Model save

README.md

@@ -1,20 +1,17 @@
 ---
 base_model: google/gemma-7b
-datasets:
-- argilla/dpo-mix-7k
 library_name: transformers
-model_name: google/gemma-7b
+model_name: gemma-7b-slic-basic-5e-5-05-v5
 tags:
 - generated_from_trainer
-- alignment-handbook
 - trl
 - orpo
 licence: license
 ---
 
-# Model Card for google/gemma-7b
+# Model Card for gemma-7b-slic-basic-5e-5-05-v5
 
-This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on the [['argilla/dpo-mix-7k']](https://huggingface.co/datasets/['argilla/dpo-mix-7k']) dataset.
+This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -30,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/silvias/huggingface/runs/qqrikeob) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/silvias/huggingface/runs/iod0j5dx) 
 
 
 This model was trained with ORPO, a method introduced in [ORPO: Monolithic Preference Optimization without Reference Model](https://huggingface.co/papers/2403.07691).

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 2.986666666666667,
     "total_flos": 0.0,
-    "train_loss": 25.709435809604706,
-    "train_runtime": 7268.7111,
+    "train_loss": 43.15912766380916,
+    "train_runtime": 7320.6498,
     "train_samples": 6750,
-    "train_samples_per_second": 2.786,
+    "train_samples_per_second": 2.766,
     "train_steps_per_second": 0.043
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 2.986666666666667,
     "total_flos": 0.0,
-    "train_loss": 25.709435809604706,
-    "train_runtime": 7268.7111,
+    "train_loss": 43.15912766380916,
+    "train_runtime": 7320.6498,
     "train_samples": 6750,
-    "train_samples_per_second": 2.786,
+    "train_samples_per_second": 2.766,
     "train_steps_per_second": 0.043
 }

trainer_state.json

@@ -10,1019 +10,1019 @@
   "log_history": [
     {
       "epoch": 0.047407407407407405,
-      "grad_norm": 1072.0,
+      "grad_norm": 1152.0,
       "learning_rate": 7.8125e-06,
-      "log_odds_chosen": -1.060240626335144,
-      "log_odds_ratio": -11.007429122924805,
-      "logps/chosen": -22.712669372558594,
-      "logps/rejected": -21.652647018432617,
-      "loss": 455.4558,
-      "nll_loss": 9.838397026062012,
+      "log_odds_chosen": -1.0631844997406006,
+      "log_odds_ratio": -11.005602836608887,
+      "logps/chosen": -22.707393646240234,
+      "logps/rejected": -21.644441604614258,
+      "loss": 467.0638,
+      "nll_loss": 9.838674545288086,
       "rewards/accuracies": 0.4781250059604645,
-      "rewards/chosen": -11.356334686279297,
-      "rewards/margins": -0.530011773109436,
-      "rewards/rejected": -10.826323509216309,
+      "rewards/chosen": -11.353696823120117,
+      "rewards/margins": -0.5314784646034241,
+      "rewards/rejected": -10.822220802307129,
       "step": 5
     },
     {
       "epoch": 0.09481481481481481,
-      "grad_norm": 592.0,
+      "grad_norm": 600.0,
       "learning_rate": 1.5625e-05,
-      "log_odds_chosen": -3.1204988956451416,
-      "log_odds_ratio": -11.043037414550781,
-      "logps/chosen": -21.994853973388672,
-      "logps/rejected": -18.874696731567383,
-      "loss": 424.5746,
-      "nll_loss": 8.752840042114258,
-      "rewards/accuracies": 0.4437499940395355,
-      "rewards/chosen": -10.997426986694336,
-      "rewards/margins": -1.5600786209106445,
-      "rewards/rejected": -9.437348365783691,
+      "log_odds_chosen": -3.1253325939178467,
+      "log_odds_ratio": -11.045626640319824,
+      "logps/chosen": -21.99468231201172,
+      "logps/rejected": -18.869693756103516,
+      "loss": 435.4043,
+      "nll_loss": 8.75256633758545,
+      "rewards/accuracies": 0.44062501192092896,
+      "rewards/chosen": -10.99734115600586,
+      "rewards/margins": -1.5624935626983643,
+      "rewards/rejected": -9.434846878051758,
       "step": 10
     },
     {
       "epoch": 0.14222222222222222,
-      "grad_norm": 454.0,
+      "grad_norm": 464.0,
       "learning_rate": 2.34375e-05,
-      "log_odds_chosen": -1.2841793298721313,
-      "log_odds_ratio": -11.318960189819336,
-      "logps/chosen": -21.992443084716797,
-      "logps/rejected": -20.706798553466797,
-      "loss": 399.2196,
-      "nll_loss": 8.00178050994873,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -10.996221542358398,
-      "rewards/margins": -0.6428214907646179,
-      "rewards/rejected": -10.353399276733398,
+      "log_odds_chosen": -1.283685564994812,
+      "log_odds_ratio": -11.318685531616211,
+      "logps/chosen": -21.985923767089844,
+      "logps/rejected": -20.700777053833008,
+      "loss": 412.295,
+      "nll_loss": 8.000757217407227,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": -10.992961883544922,
+      "rewards/margins": -0.6425749063491821,
+      "rewards/rejected": -10.350388526916504,
       "step": 15
     },
     {
       "epoch": 0.18962962962962962,
-      "grad_norm": 3008.0,
+      "grad_norm": 3360.0,
       "learning_rate": 3.125e-05,
-      "log_odds_chosen": -4.260960578918457,
-      "log_odds_ratio": -9.433350563049316,
-      "logps/chosen": -17.13646125793457,
-      "logps/rejected": -12.87755298614502,
-      "loss": 320.5827,
-      "nll_loss": 6.233663558959961,
-      "rewards/accuracies": 0.44062501192092896,
-      "rewards/chosen": -8.568230628967285,
-      "rewards/margins": -2.129453659057617,
-      "rewards/rejected": -6.43877649307251,
+      "log_odds_chosen": -4.3174591064453125,
+      "log_odds_ratio": -9.530021667480469,
+      "logps/chosen": -17.268482208251953,
+      "logps/rejected": -12.952882766723633,
+      "loss": 332.5653,
+      "nll_loss": 6.284921646118164,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -8.634241104125977,
+      "rewards/margins": -2.157799482345581,
+      "rewards/rejected": -6.476441383361816,
       "step": 20
     },
     {
       "epoch": 0.23703703703703705,
-      "grad_norm": 296.0,
+      "grad_norm": 241.0,
       "learning_rate": 3.90625e-05,
-      "log_odds_chosen": -0.10830163955688477,
-      "log_odds_ratio": -1.7876460552215576,
-      "logps/chosen": -3.861504316329956,
-      "logps/rejected": -3.7353768348693848,
-      "loss": 92.4915,
-      "nll_loss": 2.6942696571350098,
+      "log_odds_chosen": -0.13406293094158173,
+      "log_odds_ratio": -1.9662774801254272,
+      "logps/chosen": -4.281245708465576,
+      "logps/rejected": -4.128834247589111,
+      "loss": 101.8704,
+      "nll_loss": 2.7933926582336426,
       "rewards/accuracies": 0.5531250238418579,
-      "rewards/chosen": -1.930752158164978,
-      "rewards/margins": -0.06306387484073639,
-      "rewards/rejected": -1.8676884174346924,
+      "rewards/chosen": -2.140622854232788,
+      "rewards/margins": -0.07620568573474884,
+      "rewards/rejected": -2.0644171237945557,
       "step": 25
     },
     {
       "epoch": 0.28444444444444444,
-      "grad_norm": 189.0,
+      "grad_norm": 242.0,
       "learning_rate": 4.6875e-05,
-      "log_odds_chosen": 0.08914291858673096,
-      "log_odds_ratio": -0.9090879559516907,
-      "logps/chosen": -1.9305442571640015,
-      "logps/rejected": -2.0069050788879395,
-      "loss": 65.6584,
-      "nll_loss": 2.0820810794830322,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.9652721285820007,
-      "rewards/margins": 0.03818041831254959,
-      "rewards/rejected": -1.0034525394439697,
+      "log_odds_chosen": 0.10939224064350128,
+      "log_odds_ratio": -0.8504419326782227,
+      "logps/chosen": -1.855107307434082,
+      "logps/rejected": -1.9498792886734009,
+      "loss": 64.8284,
+      "nll_loss": 2.0649564266204834,
+      "rewards/accuracies": 0.546875,
+      "rewards/chosen": -0.927553653717041,
+      "rewards/margins": 0.04738599434494972,
+      "rewards/rejected": -0.9749396443367004,
       "step": 30
     },
     {
       "epoch": 0.33185185185185184,
-      "grad_norm": 247.0,
+      "grad_norm": 282.0,
       "learning_rate": 4.998613757348784e-05,
-      "log_odds_chosen": 0.3332863748073578,
-      "log_odds_ratio": -0.8136388063430786,
-      "logps/chosen": -1.8473803997039795,
-      "logps/rejected": -2.1461009979248047,
-      "loss": 58.187,
-      "nll_loss": 1.9638898372650146,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.9236901998519897,
-      "rewards/margins": 0.14936023950576782,
-      "rewards/rejected": -1.0730504989624023,
+      "log_odds_chosen": 0.3194878101348877,
+      "log_odds_ratio": -0.7833901643753052,
+      "logps/chosen": -1.7884143590927124,
+      "logps/rejected": -2.071709156036377,
+      "loss": 58.4951,
+      "nll_loss": 1.9425550699234009,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.8942071795463562,
+      "rewards/margins": 0.14164744317531586,
+      "rewards/rejected": -1.0358545780181885,
       "step": 35
     },
     {
       "epoch": 0.37925925925925924,
-      "grad_norm": 316.0,
+      "grad_norm": 214.0,
       "learning_rate": 4.990147841143462e-05,
-      "log_odds_chosen": 0.5912309885025024,
-      "log_odds_ratio": -0.6898080110549927,
-      "logps/chosen": -1.7279255390167236,
-      "logps/rejected": -2.248690128326416,
-      "loss": 53.4463,
-      "nll_loss": 1.926064133644104,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.8639627695083618,
-      "rewards/margins": 0.2603822350502014,
-      "rewards/rejected": -1.124345064163208,
+      "log_odds_chosen": 0.5328894853591919,
+      "log_odds_ratio": -0.6567676663398743,
+      "logps/chosen": -1.6281368732452393,
+      "logps/rejected": -2.09126877784729,
+      "loss": 53.2976,
+      "nll_loss": 1.8886241912841797,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": -0.8140684366226196,
+      "rewards/margins": 0.2315659075975418,
+      "rewards/rejected": -1.045634388923645,
       "step": 40
     },
     {
       "epoch": 0.4266666666666667,
-      "grad_norm": 436.0,
+      "grad_norm": 354.0,
       "learning_rate": 4.97401218720448e-05,
-      "log_odds_chosen": 1.5738911628723145,
-      "log_odds_ratio": -0.6931017637252808,
-      "logps/chosen": -1.9965721368789673,
-      "logps/rejected": -3.4913876056671143,
-      "loss": 42.7119,
-      "nll_loss": 1.9767124652862549,
-      "rewards/accuracies": 0.684374988079071,
-      "rewards/chosen": -0.9982860684394836,
-      "rewards/margins": 0.7474077939987183,
-      "rewards/rejected": -1.7456938028335571,
+      "log_odds_chosen": 1.3014285564422607,
+      "log_odds_ratio": -0.5872281789779663,
+      "logps/chosen": -1.680598497390747,
+      "logps/rejected": -2.899627208709717,
+      "loss": 46.0017,
+      "nll_loss": 1.8771553039550781,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": -0.8402992486953735,
+      "rewards/margins": 0.6095143556594849,
+      "rewards/rejected": -1.4498136043548584,
       "step": 45
     },
     {
       "epoch": 0.4740740740740741,
-      "grad_norm": 340.0,
+      "grad_norm": 226.0,
       "learning_rate": 4.9502564938797946e-05,
-      "log_odds_chosen": 0.7620365619659424,
-      "log_odds_ratio": -0.7459171414375305,
-      "logps/chosen": -1.7422816753387451,
-      "logps/rejected": -2.4515380859375,
-      "loss": 52.5226,
-      "nll_loss": 1.9401613473892212,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.8711408376693726,
-      "rewards/margins": 0.3546280860900879,
-      "rewards/rejected": -1.22576904296875,
+      "log_odds_chosen": 0.6567279696464539,
+      "log_odds_ratio": -0.6511496305465698,
+      "logps/chosen": -1.4998167753219604,
+      "logps/rejected": -2.0936269760131836,
+      "loss": 49.0928,
+      "nll_loss": 1.7653518915176392,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.7499083876609802,
+      "rewards/margins": 0.29690515995025635,
+      "rewards/rejected": -1.0468134880065918,
       "step": 50
     },
     {
       "epoch": 0.5214814814814814,
-      "grad_norm": 336.0,
+      "grad_norm": 202.0,
       "learning_rate": 4.918953929490768e-05,
-      "log_odds_chosen": 2.066755533218384,
-      "log_odds_ratio": -0.703250527381897,
-      "logps/chosen": -2.0667014122009277,
-      "logps/rejected": -4.0496721267700195,
-      "loss": 43.0197,
-      "nll_loss": 2.0524208545684814,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.0333507061004639,
-      "rewards/margins": 0.9914852380752563,
-      "rewards/rejected": -2.0248360633850098,
+      "log_odds_chosen": 1.189807653427124,
+      "log_odds_ratio": -0.617165744304657,
+      "logps/chosen": -1.565733551979065,
+      "logps/rejected": -2.669482707977295,
+      "loss": 44.8712,
+      "nll_loss": 1.786768913269043,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.7828667759895325,
+      "rewards/margins": 0.5518745183944702,
+      "rewards/rejected": -1.3347413539886475,
       "step": 55
     },
     {
       "epoch": 0.5688888888888889,
-      "grad_norm": 310.0,
+      "grad_norm": 278.0,
       "learning_rate": 4.88020090697132e-05,
-      "log_odds_chosen": 1.32530677318573,
-      "log_odds_ratio": -0.7277625799179077,
-      "logps/chosen": -1.7378164529800415,
-      "logps/rejected": -2.996918201446533,
-      "loss": 46.4442,
-      "nll_loss": 1.8602434396743774,
-      "rewards/accuracies": 0.6656249761581421,
-      "rewards/chosen": -0.8689082264900208,
-      "rewards/margins": 0.6295510530471802,
-      "rewards/rejected": -1.4984591007232666,
+      "log_odds_chosen": 1.5694297552108765,
+      "log_odds_ratio": -0.5945903062820435,
+      "logps/chosen": -1.5673797130584717,
+      "logps/rejected": -3.0414257049560547,
+      "loss": 44.6048,
+      "nll_loss": 1.8120968341827393,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.7836898565292358,
+      "rewards/margins": 0.7370229959487915,
+      "rewards/rejected": -1.5207128524780273,
       "step": 60
     },
     {
       "epoch": 0.6162962962962963,
-      "grad_norm": 418.0,
+      "grad_norm": 160.0,
       "learning_rate": 4.834116786912897e-05,
-      "log_odds_chosen": 2.172917604446411,
-      "log_odds_ratio": -0.7570880651473999,
-      "logps/chosen": -1.923484444618225,
-      "logps/rejected": -4.014767169952393,
-      "loss": 43.8154,
-      "nll_loss": 2.0151474475860596,
-      "rewards/accuracies": 0.671875,
-      "rewards/chosen": -0.9617422223091125,
-      "rewards/margins": 1.0456410646438599,
-      "rewards/rejected": -2.0073835849761963,
+      "log_odds_chosen": 1.5031869411468506,
+      "log_odds_ratio": -0.663598895072937,
+      "logps/chosen": -1.6163688898086548,
+      "logps/rejected": -3.024693250656128,
+      "loss": 46.3428,
+      "nll_loss": 1.8134605884552002,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.8081844449043274,
+      "rewards/margins": 0.7041621208190918,
+      "rewards/rejected": -1.512346625328064,
       "step": 65
     },
     {
       "epoch": 0.6637037037037037,
-      "grad_norm": 362.0,
+      "grad_norm": 298.0,
       "learning_rate": 4.7808435099299045e-05,
-      "log_odds_chosen": 3.7954392433166504,
-      "log_odds_ratio": -0.6524917483329773,
-      "logps/chosen": -2.0215845108032227,
-      "logps/rejected": -5.7059245109558105,
-      "loss": 31.4526,
-      "nll_loss": 2.0003814697265625,
+      "log_odds_chosen": 2.1792948246002197,
+      "log_odds_ratio": -0.5366494059562683,
+      "logps/chosen": -1.6390451192855835,
+      "logps/rejected": -3.6916420459747314,
+      "loss": 38.6914,
+      "nll_loss": 1.7929494380950928,
       "rewards/accuracies": 0.753125011920929,
-      "rewards/chosen": -1.0107922554016113,
-      "rewards/margins": 1.842170000076294,
-      "rewards/rejected": -2.8529622554779053,
+      "rewards/chosen": -0.8195225596427917,
+      "rewards/margins": 1.0262985229492188,
+      "rewards/rejected": -1.8458210229873657,
       "step": 70
     },
     {
       "epoch": 0.7111111111111111,
-      "grad_norm": 644.0,
+      "grad_norm": 300.0,
       "learning_rate": 4.720545159477922e-05,
-      "log_odds_chosen": 2.579057216644287,
-      "log_odds_ratio": -0.9140008687973022,
-      "logps/chosen": -2.144348382949829,
-      "logps/rejected": -4.612305641174316,
-      "loss": 40.0912,
-      "nll_loss": 1.925865888595581,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.0721741914749146,
-      "rewards/margins": 1.2339783906936646,
-      "rewards/rejected": -2.306152820587158,
+      "log_odds_chosen": 1.6375625133514404,
+      "log_odds_ratio": -0.6377304196357727,
+      "logps/chosen": -1.5810494422912598,
+      "logps/rejected": -3.109715223312378,
+      "loss": 42.8578,
+      "nll_loss": 1.7745224237442017,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.7905247211456299,
+      "rewards/margins": 0.7643328905105591,
+      "rewards/rejected": -1.554857611656189,
       "step": 75
     },
     {
       "epoch": 0.7585185185185185,
-      "grad_norm": 474.0,
+      "grad_norm": 208.0,
       "learning_rate": 4.653407456471222e-05,
-      "log_odds_chosen": 2.602337121963501,
-      "log_odds_ratio": -0.8660507202148438,
-      "logps/chosen": -2.392124891281128,
-      "logps/rejected": -4.895805358886719,
-      "loss": 45.2894,
-      "nll_loss": 2.1908349990844727,
-      "rewards/accuracies": 0.6781250238418579,
-      "rewards/chosen": -1.196062445640564,
-      "rewards/margins": 1.2518401145935059,
-      "rewards/rejected": -2.4479026794433594,
+      "log_odds_chosen": 1.206108808517456,
+      "log_odds_ratio": -0.6781158447265625,
+      "logps/chosen": -1.5513617992401123,
+      "logps/rejected": -2.6515350341796875,
+      "loss": 44.2435,
+      "nll_loss": 1.7461738586425781,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.7756808996200562,
+      "rewards/margins": 0.5500864386558533,
+      "rewards/rejected": -1.3257675170898438,
       "step": 80
     },
     {
       "epoch": 0.8059259259259259,
-      "grad_norm": 268.0,
+      "grad_norm": 282.0,
       "learning_rate": 4.579637187256222e-05,
-      "log_odds_chosen": 1.2037420272827148,
-      "log_odds_ratio": -0.6474961042404175,
-      "logps/chosen": -1.5563756227493286,
-      "logps/rejected": -2.667034864425659,
-      "loss": 40.2868,
-      "nll_loss": 1.7489856481552124,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.7781878113746643,
-      "rewards/margins": 0.5553296208381653,
-      "rewards/rejected": -1.3335174322128296,
+      "log_odds_chosen": 0.8860191106796265,
+      "log_odds_ratio": -0.5883544683456421,
+      "logps/chosen": -1.4292150735855103,
+      "logps/rejected": -2.2102513313293457,
+      "loss": 42.4359,
+      "nll_loss": 1.656638741493225,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.7146075367927551,
+      "rewards/margins": 0.3905182480812073,
+      "rewards/rejected": -1.1051256656646729,
       "step": 85
     },
     {
       "epoch": 0.8533333333333334,
-      "grad_norm": 372.0,
+      "grad_norm": 1400.0,
       "learning_rate": 4.499461566702685e-05,
-      "log_odds_chosen": 5.79607629776001,
-      "log_odds_ratio": -0.834893524646759,
-      "logps/chosen": -2.230257987976074,
-      "logps/rejected": -7.889917850494385,
-      "loss": 33.4453,
-      "nll_loss": 2.099855422973633,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.115128993988037,
-      "rewards/margins": 2.8298299312591553,
-      "rewards/rejected": -3.9449589252471924,
+      "log_odds_chosen": 2.8823413848876953,
+      "log_odds_ratio": -0.8564308285713196,
+      "logps/chosen": -2.120701789855957,
+      "logps/rejected": -4.8557209968566895,
+      "loss": 47.9855,
+      "nll_loss": 1.9995596408843994,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0603508949279785,
+      "rewards/margins": 1.3675094842910767,
+      "rewards/rejected": -2.4278604984283447,
       "step": 90
     },
     {
       "epoch": 0.9007407407407407,
-      "grad_norm": 644.0,
+      "grad_norm": 366.0,
       "learning_rate": 4.413127538374411e-05,
-      "log_odds_chosen": 2.9538180828094482,
-      "log_odds_ratio": -0.7920938730239868,
-      "logps/chosen": -1.8438174724578857,
-      "logps/rejected": -4.690665245056152,
-      "loss": 34.7808,
-      "nll_loss": 1.8597549200057983,
-      "rewards/accuracies": 0.690625011920929,
-      "rewards/chosen": -0.9219087362289429,
-      "rewards/margins": 1.4234240055084229,
-      "rewards/rejected": -2.345332622528076,
+      "log_odds_chosen": 1.6698837280273438,
+      "log_odds_ratio": -0.7055229544639587,
+      "logps/chosen": -1.6337862014770508,
+      "logps/rejected": -3.1812427043914795,
+      "loss": 41.8057,
+      "nll_loss": 1.781404733657837,
+      "rewards/accuracies": 0.684374988079071,
+      "rewards/chosen": -0.8168931007385254,
+      "rewards/margins": 0.7737283110618591,
+      "rewards/rejected": -1.5906213521957397,
       "step": 95
     },
     {
       "epoch": 0.9481481481481482,
-      "grad_norm": 296.0,
+      "grad_norm": 364.0,
       "learning_rate": 4.320901013934887e-05,
-      "log_odds_chosen": 3.595071315765381,
-      "log_odds_ratio": -0.8463969230651855,
-      "logps/chosen": -2.0342302322387695,
-      "logps/rejected": -5.500476837158203,
-      "loss": 30.0336,
-      "nll_loss": 2.020622491836548,
+      "log_odds_chosen": 1.8235197067260742,
+      "log_odds_ratio": -0.5255730748176575,
+      "logps/chosen": -1.462209939956665,
+      "logps/rejected": -3.1428921222686768,
+      "loss": 34.1059,
+      "nll_loss": 1.6777896881103516,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.0171151161193848,
-      "rewards/margins": 1.7331234216690063,
-      "rewards/rejected": -2.7502384185791016,
+      "rewards/chosen": -0.7311049699783325,
+      "rewards/margins": 0.840340793132782,
+      "rewards/rejected": -1.5714460611343384,
       "step": 100
     },
     {
       "epoch": 0.9955555555555555,
-      "grad_norm": 312.0,
+      "grad_norm": 268.0,
       "learning_rate": 4.223066054130568e-05,
-      "log_odds_chosen": 2.3394980430603027,
-      "log_odds_ratio": -0.7494302988052368,
-      "logps/chosen": -1.8197135925292969,
-      "logps/rejected": -4.058799743652344,
-      "loss": 35.016,
-      "nll_loss": 1.9035847187042236,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.9098567962646484,
-      "rewards/margins": 1.1195429563522339,
-      "rewards/rejected": -2.029399871826172,
+      "log_odds_chosen": 1.5804908275604248,
+      "log_odds_ratio": -0.580092191696167,
+      "logps/chosen": -1.4532772302627563,
+      "logps/rejected": -2.918117046356201,
+      "loss": 37.4817,
+      "nll_loss": 1.6894712448120117,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": -0.7266386151313782,
+      "rewards/margins": 0.7324199080467224,
+      "rewards/rejected": -1.4590585231781006,
       "step": 105
     },
     {
       "epoch": 1.0429629629629629,
-      "grad_norm": 1912.0,
+      "grad_norm": 286.0,
       "learning_rate": 4.1199239938743797e-05,
-      "log_odds_chosen": 4.236447334289551,
-      "log_odds_ratio": -0.4881769120693207,
-      "logps/chosen": -2.2219159603118896,
-      "logps/rejected": -6.284074306488037,
-      "loss": 22.0456,
-      "nll_loss": 2.1491634845733643,
-      "rewards/accuracies": 0.8031250238418579,
-      "rewards/chosen": -1.1109579801559448,
-      "rewards/margins": 2.031079053878784,
-      "rewards/rejected": -3.1420371532440186,
+      "log_odds_chosen": 2.7626733779907227,
+      "log_odds_ratio": -0.424811989068985,
+      "logps/chosen": -1.4315791130065918,
+      "logps/rejected": -3.964046001434326,
+      "loss": 24.2139,
+      "nll_loss": 1.5723967552185059,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.7157895565032959,
+      "rewards/margins": 1.2662335634231567,
+      "rewards/rejected": -1.982023000717163,
       "step": 110
     },
     {
       "epoch": 1.0903703703703704,
-      "grad_norm": 382.0,
+      "grad_norm": 254.0,
       "learning_rate": 4.0117925141242174e-05,
-      "log_odds_chosen": 4.837921142578125,
-      "log_odds_ratio": -0.5910585522651672,
-      "logps/chosen": -1.997674584388733,
-      "logps/rejected": -6.637750148773193,
-      "loss": 16.3477,
-      "nll_loss": 1.9437005519866943,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.9988372921943665,
-      "rewards/margins": 2.320038318634033,
-      "rewards/rejected": -3.3188750743865967,
+      "log_odds_chosen": 3.299778461456299,
+      "log_odds_ratio": -0.5258711576461792,
+      "logps/chosen": -1.3833136558532715,
+      "logps/rejected": -4.405320644378662,
+      "loss": 25.4402,
+      "nll_loss": 1.6238991022109985,
+      "rewards/accuracies": 0.784375011920929,
+      "rewards/chosen": -0.6916568279266357,
+      "rewards/margins": 1.5110036134719849,
+      "rewards/rejected": -2.202660322189331,
       "step": 115
     },
     {
       "epoch": 1.1377777777777778,
-      "grad_norm": 516.0,
+      "grad_norm": 528.0,
       "learning_rate": 3.899004663415084e-05,
-      "log_odds_chosen": 4.582813739776611,
-      "log_odds_ratio": -0.4979763627052307,
-      "logps/chosen": -1.5545316934585571,
-      "logps/rejected": -5.910370826721191,
-      "loss": 11.8075,
-      "nll_loss": 1.7471332550048828,
-      "rewards/accuracies": 0.784375011920929,
-      "rewards/chosen": -0.7772658467292786,
-      "rewards/margins": 2.177919626235962,
-      "rewards/rejected": -2.9551854133605957,
+      "log_odds_chosen": 2.667694568634033,
+      "log_odds_ratio": -0.40380731225013733,
+      "logps/chosen": -1.2623659372329712,
+      "logps/rejected": -3.6577308177948,
+      "loss": 23.1023,
+      "nll_loss": 1.5065176486968994,
+      "rewards/accuracies": 0.815625011920929,
+      "rewards/chosen": -0.6311829686164856,
+      "rewards/margins": 1.1976823806762695,
+      "rewards/rejected": -1.8288654088974,
       "step": 120
     },
     {
       "epoch": 1.1851851851851851,
-      "grad_norm": 466.0,
+      "grad_norm": 342.0,
       "learning_rate": 3.781907832058587e-05,
-      "log_odds_chosen": 5.0084943771362305,
-      "log_odds_ratio": -0.5646654367446899,
-      "logps/chosen": -1.8939094543457031,
-      "logps/rejected": -6.705332279205322,
-      "loss": 12.4279,
-      "nll_loss": 1.95928156375885,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": -0.9469547271728516,
-      "rewards/margins": 2.4057114124298096,
-      "rewards/rejected": -3.352666139602661,
+      "log_odds_chosen": 2.9293477535247803,
+      "log_odds_ratio": -0.45116695761680603,
+      "logps/chosen": -1.4864990711212158,
+      "logps/rejected": -4.153409957885742,
+      "loss": 25.0104,
+      "nll_loss": 1.6183675527572632,
+      "rewards/accuracies": 0.8031250238418579,
+      "rewards/chosen": -0.7432495355606079,
+      "rewards/margins": 1.3334555625915527,
+      "rewards/rejected": -2.076704978942871,
       "step": 125
     },
     {
       "epoch": 1.2325925925925927,
-      "grad_norm": 1016.0,
+      "grad_norm": 268.0,
       "learning_rate": 3.660862682169282e-05,
-      "log_odds_chosen": 6.262756824493408,
-      "log_odds_ratio": -0.5206202268600464,
-      "logps/chosen": -1.6548172235488892,
-      "logps/rejected": -7.668360710144043,
-      "loss": 2.6131,
-      "nll_loss": 1.758933663368225,
-      "rewards/accuracies": 0.809374988079071,
-      "rewards/chosen": -0.8274086117744446,
-      "rewards/margins": 3.006772041320801,
-      "rewards/rejected": -3.8341803550720215,
+      "log_odds_chosen": 3.8303322792053223,
+      "log_odds_ratio": -0.37618154287338257,
+      "logps/chosen": -1.4800456762313843,
+      "logps/rejected": -5.02761697769165,
+      "loss": 18.1555,
+      "nll_loss": 1.5957032442092896,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": -0.7400228381156921,
+      "rewards/margins": 1.7737855911254883,
+      "rewards/rejected": -2.513808488845825,
       "step": 130
     },
     {
       "epoch": 1.28,
-      "grad_norm": 506.0,
+      "grad_norm": 458.0,
       "learning_rate": 3.5362420368134356e-05,
-      "log_odds_chosen": 4.817102909088135,
-      "log_odds_ratio": -0.6338304281234741,
-      "logps/chosen": -1.9214130640029907,
-      "logps/rejected": -6.519143104553223,
-      "loss": 18.6154,
-      "nll_loss": 1.8606481552124023,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.9607065320014954,
-      "rewards/margins": 2.2988650798797607,
-      "rewards/rejected": -3.2595715522766113,
+      "log_odds_chosen": 2.9902822971343994,
+      "log_odds_ratio": -0.43725720047950745,
+      "logps/chosen": -1.473851203918457,
+      "logps/rejected": -4.198307991027832,
+      "loss": 22.4211,
+      "nll_loss": 1.5640933513641357,
+      "rewards/accuracies": 0.8218749761581421,
+      "rewards/chosen": -0.7369256019592285,
+      "rewards/margins": 1.362228274345398,
+      "rewards/rejected": -2.099153995513916,
       "step": 135
     },
     {
       "epoch": 1.3274074074074074,
-      "grad_norm": 956.0,
+      "grad_norm": 201.0,
       "learning_rate": 3.408429731701635e-05,
-      "log_odds_chosen": 4.097002983093262,
-      "log_odds_ratio": -0.4646216034889221,
-      "logps/chosen": -1.740180253982544,
-      "logps/rejected": -5.640007972717285,
-      "loss": 11.3345,
-      "nll_loss": 1.65927255153656,
+      "log_odds_chosen": 2.826463460922241,
+      "log_odds_ratio": -0.39234691858291626,
+      "logps/chosen": -1.3628809452056885,
+      "logps/rejected": -3.936453342437744,
+      "loss": 21.0249,
+      "nll_loss": 1.5445901155471802,
       "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.870090126991272,
-      "rewards/margins": 1.9499139785766602,
-      "rewards/rejected": -2.8200039863586426,
+      "rewards/chosen": -0.6814404726028442,
+      "rewards/margins": 1.2867863178253174,
+      "rewards/rejected": -1.968226671218872,
       "step": 140
     },
     {
       "epoch": 1.374814814814815,
-      "grad_norm": 1064.0,
+      "grad_norm": 784.0,
       "learning_rate": 3.2778194329621104e-05,
-      "log_odds_chosen": 7.051480770111084,
-      "log_odds_ratio": -0.4304323196411133,
-      "logps/chosen": -2.5247302055358887,
-      "logps/rejected": -9.359436988830566,
-      "loss": 5.1241,
-      "nll_loss": 2.1758809089660645,
-      "rewards/accuracies": 0.846875011920929,
-      "rewards/chosen": -1.2623651027679443,
-      "rewards/margins": 3.417353868484497,
-      "rewards/rejected": -4.679718494415283,
+      "log_odds_chosen": 3.861825942993164,
+      "log_odds_ratio": -0.3445121943950653,
+      "logps/chosen": -1.6509920358657837,
+      "logps/rejected": -5.243237495422363,
+      "loss": 18.864,
+      "nll_loss": 1.7057406902313232,
+      "rewards/accuracies": 0.8843749761581421,
+      "rewards/chosen": -0.8254960179328918,
+      "rewards/margins": 1.7961229085922241,
+      "rewards/rejected": -2.6216187477111816,
       "step": 145
     },
     {
       "epoch": 1.4222222222222223,
-      "grad_norm": 604.0,
+      "grad_norm": 316.0,
       "learning_rate": 3.144813424636031e-05,
-      "log_odds_chosen": 6.2552809715271,
-      "log_odds_ratio": -0.4223237931728363,
-      "logps/chosen": -1.5957365036010742,
-      "logps/rejected": -7.607705116271973,
-      "loss": 2.8133,
-      "nll_loss": 1.7030588388442993,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.7978682518005371,
-      "rewards/margins": 3.005984306335449,
-      "rewards/rejected": -3.8038525581359863,
+      "log_odds_chosen": 4.123653888702393,
+      "log_odds_ratio": -0.4155045449733734,
+      "logps/chosen": -1.6837345361709595,
+      "logps/rejected": -5.53621768951416,
+      "loss": 18.678,
+      "nll_loss": 1.6780169010162354,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.8418672680854797,
+      "rewards/margins": 1.9262415170669556,
+      "rewards/rejected": -2.76810884475708,
       "step": 150
     },
     {
       "epoch": 1.4696296296296296,
-      "grad_norm": 972.0,
+      "grad_norm": 466.0,
       "learning_rate": 3.0098213696293542e-05,
-      "log_odds_chosen": 5.868264198303223,
-      "log_odds_ratio": -0.7328829169273376,
-      "logps/chosen": -2.398165225982666,
-      "logps/rejected": -8.077600479125977,
-      "loss": 14.0782,
-      "nll_loss": 1.9348533153533936,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.199082612991333,
-      "rewards/margins": 2.839717388153076,
-      "rewards/rejected": -4.038800239562988,
+      "log_odds_chosen": 3.181971311569214,
+      "log_odds_ratio": -0.4872185289859772,
+      "logps/chosen": -1.4469890594482422,
+      "logps/rejected": -4.361490726470947,
+      "loss": 21.8554,
+      "nll_loss": 1.5998506546020508,
+      "rewards/accuracies": 0.815625011920929,
+      "rewards/chosen": -0.7234945297241211,
+      "rewards/margins": 1.457250952720642,
+      "rewards/rejected": -2.1807453632354736,
       "step": 155
     },
     {
       "epoch": 1.5170370370370372,
-      "grad_norm": 884.0,
+      "grad_norm": 412.0,
       "learning_rate": 2.8732590479375165e-05,
-      "log_odds_chosen": 7.269213676452637,
-      "log_odds_ratio": -0.5041804909706116,
-      "logps/chosen": -2.1709227561950684,
-      "logps/rejected": -9.193258285522461,
-      "loss": 1.1204,
-      "nll_loss": 2.00057315826416,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.0854613780975342,
-      "rewards/margins": 3.511167526245117,
-      "rewards/rejected": -4.5966291427612305,
+      "log_odds_chosen": 4.313605308532715,
+      "log_odds_ratio": -0.4044560492038727,
+      "logps/chosen": -1.7111812829971313,
+      "logps/rejected": -5.7456865310668945,
+      "loss": 14.9335,
+      "nll_loss": 1.6823270320892334,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": -0.8555906414985657,
+      "rewards/margins": 2.0172524452209473,
+      "rewards/rejected": -2.8728432655334473,
       "step": 160
     },
     {
       "epoch": 1.5644444444444443,
-      "grad_norm": 580.0,
+      "grad_norm": 270.0,
       "learning_rate": 2.7355470760292956e-05,
-      "log_odds_chosen": 6.614538669586182,
-      "log_odds_ratio": -0.462197482585907,
-      "logps/chosen": -1.875078797340393,
-      "logps/rejected": -8.235835075378418,
-      "loss": -2.3102,
-      "nll_loss": 1.8360283374786377,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.9375393986701965,
-      "rewards/margins": 3.1803784370422363,
-      "rewards/rejected": -4.117917537689209,
+      "log_odds_chosen": 3.449075698852539,
+      "log_odds_ratio": -0.4021359980106354,
+      "logps/chosen": -1.360383152961731,
+      "logps/rejected": -4.50086784362793,
+      "loss": 17.5439,
+      "nll_loss": 1.5542360544204712,
+      "rewards/accuracies": 0.8531249761581421,
+      "rewards/chosen": -0.6801915764808655,
+      "rewards/margins": 1.5702422857284546,
+      "rewards/rejected": -2.250433921813965,
       "step": 165
     },
     {
       "epoch": 1.6118518518518519,
-      "grad_norm": 1064.0,
+      "grad_norm": 334.0,
       "learning_rate": 2.597109611334169e-05,
-      "log_odds_chosen": 6.787781715393066,
-      "log_odds_ratio": -0.5965229272842407,
-      "logps/chosen": -2.1229805946350098,
-      "logps/rejected": -8.671282768249512,
-      "loss": -2.3052,
-      "nll_loss": 1.9179661273956299,
-      "rewards/accuracies": 0.828125,
-      "rewards/chosen": -1.0614902973175049,
-      "rewards/margins": 3.27415132522583,
-      "rewards/rejected": -4.335641384124756,
+      "log_odds_chosen": 4.0982818603515625,
+      "log_odds_ratio": -0.4611207842826843,
+      "logps/chosen": -1.5969505310058594,
+      "logps/rejected": -5.419460773468018,
+      "loss": 18.2855,
+      "nll_loss": 1.6615581512451172,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.7984752655029297,
+      "rewards/margins": 1.911254644393921,
+      "rewards/rejected": -2.709730386734009,
       "step": 170
     },
     {
       "epoch": 1.6592592592592592,
-      "grad_norm": 872.0,
+      "grad_norm": 424.0,
       "learning_rate": 2.458373045823404e-05,
-      "log_odds_chosen": 6.521327972412109,
-      "log_odds_ratio": -0.528164267539978,
-      "logps/chosen": -2.1449532508850098,
-      "logps/rejected": -8.415470123291016,
-      "loss": 1.3161,
-      "nll_loss": 2.0013487339019775,
-      "rewards/accuracies": 0.809374988079071,
-      "rewards/chosen": -1.0724766254425049,
-      "rewards/margins": 3.135258197784424,
-      "rewards/rejected": -4.207735061645508,
+      "log_odds_chosen": 3.9468979835510254,
+      "log_odds_ratio": -0.36181649565696716,
+      "logps/chosen": -1.4357500076293945,
+      "logps/rejected": -5.089698791503906,
+      "loss": 14.1876,
+      "nll_loss": 1.5684595108032227,
+      "rewards/accuracies": 0.8218749761581421,
+      "rewards/chosen": -0.7178750038146973,
+      "rewards/margins": 1.8269745111465454,
+      "rewards/rejected": -2.544849395751953,
       "step": 175
     },
     {
       "epoch": 1.7066666666666666,
-      "grad_norm": 1256.0,
+      "grad_norm": 692.0,
       "learning_rate": 2.3197646927086697e-05,
-      "log_odds_chosen": 7.416574001312256,
-      "log_odds_ratio": -0.5437217354774475,
-      "logps/chosen": -2.012195587158203,
-      "logps/rejected": -9.163311004638672,
-      "loss": -6.5787,
-      "nll_loss": 1.931572675704956,
-      "rewards/accuracies": 0.815625011920929,
-      "rewards/chosen": -1.0060977935791016,
-      "rewards/margins": 3.575557231903076,
-      "rewards/rejected": -4.581655502319336,
+      "log_odds_chosen": 3.9691085815429688,
+      "log_odds_ratio": -0.35701385140419006,
+      "logps/chosen": -1.550195336341858,
+      "logps/rejected": -5.20673942565918,
+      "loss": 14.9335,
+      "nll_loss": 1.635414719581604,
+      "rewards/accuracies": 0.8843749761581421,
+      "rewards/chosen": -0.775097668170929,
+      "rewards/margins": 1.8282718658447266,
+      "rewards/rejected": -2.60336971282959,
       "step": 180
     },
     {
       "epoch": 1.7540740740740741,
-      "grad_norm": 500.0,
+      "grad_norm": 304.0,
       "learning_rate": 2.1817114703032176e-05,
-      "log_odds_chosen": 7.6875762939453125,
-      "log_odds_ratio": -0.6660786867141724,
-      "logps/chosen": -2.3043220043182373,
-      "logps/rejected": -9.788423538208008,
-      "loss": -4.8462,
-      "nll_loss": 1.9335501194000244,
-      "rewards/accuracies": 0.784375011920929,
-      "rewards/chosen": -1.1521610021591187,
-      "rewards/margins": 3.742050886154175,
-      "rewards/rejected": -4.894211769104004,
+      "log_odds_chosen": 4.004181861877441,
+      "log_odds_ratio": -0.4124390184879303,
+      "logps/chosen": -1.5889790058135986,
+      "logps/rejected": -5.336648464202881,
+      "loss": 13.5162,
+      "nll_loss": 1.580910563468933,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": -0.7944895029067993,
+      "rewards/margins": 1.8738348484039307,
+      "rewards/rejected": -2.6683242321014404,
       "step": 185
     },
     {
       "epoch": 1.8014814814814815,
-      "grad_norm": 876.0,
+      "grad_norm": 344.0,
       "learning_rate": 2.0446385870993467e-05,
-      "log_odds_chosen": 6.940661430358887,
-      "log_odds_ratio": -0.4403323233127594,
-      "logps/chosen": -1.603877067565918,
-      "logps/rejected": -8.270362854003906,
-      "loss": -8.6702,
-      "nll_loss": 1.6499793529510498,
-      "rewards/accuracies": 0.8343750238418579,
-      "rewards/chosen": -0.801938533782959,
-      "rewards/margins": 3.333242893218994,
-      "rewards/rejected": -4.135181427001953,
+      "log_odds_chosen": 3.7639663219451904,
+      "log_odds_ratio": -0.3424893021583557,
+      "logps/chosen": -1.3625359535217285,
+      "logps/rejected": -4.8325395584106445,
+      "loss": 10.6844,
+      "nll_loss": 1.4781030416488647,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.6812679767608643,
+      "rewards/margins": 1.735001564025879,
+      "rewards/rejected": -2.4162697792053223,
       "step": 190
     },
     {
       "epoch": 1.8488888888888888,
-      "grad_norm": 1320.0,
+      "grad_norm": 632.0,
       "learning_rate": 1.9089682321121834e-05,
-      "log_odds_chosen": 9.642197608947754,
-      "log_odds_ratio": -0.6060672402381897,
-      "logps/chosen": -2.7960872650146484,
-      "logps/rejected": -12.217751502990723,
-      "loss": -4.8162,
-      "nll_loss": 2.395156145095825,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.3980436325073242,
-      "rewards/margins": 4.710832595825195,
-      "rewards/rejected": -6.108875751495361,
+      "log_odds_chosen": 4.872841835021973,
+      "log_odds_ratio": -0.3931121528148651,
+      "logps/chosen": -1.8755972385406494,
+      "logps/rejected": -6.473967552185059,
+      "loss": 14.5199,
+      "nll_loss": 1.8242985010147095,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": -0.9377986192703247,
+      "rewards/margins": 2.299185037612915,
+      "rewards/rejected": -3.2369837760925293,
       "step": 195
     },
     {
       "epoch": 1.8962962962962964,
-      "grad_norm": 1240.0,
+      "grad_norm": 438.0,
       "learning_rate": 1.775118274523545e-05,
-      "log_odds_chosen": 6.639898777008057,
-      "log_odds_ratio": -0.8482674360275269,
-      "logps/chosen": -2.435774564743042,
-      "logps/rejected": -8.842093467712402,
-      "loss": 3.4566,
-      "nll_loss": 1.9974777698516846,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.217887282371521,
-      "rewards/margins": 3.203160047531128,
-      "rewards/rejected": -4.421046733856201,
+      "log_odds_chosen": 3.7074005603790283,
+      "log_odds_ratio": -0.5268596410751343,
+      "logps/chosen": -1.7662988901138306,
+      "logps/rejected": -5.216193199157715,
+      "loss": 20.4733,
+      "nll_loss": 1.7086811065673828,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.8831494450569153,
+      "rewards/margins": 1.724947214126587,
+      "rewards/rejected": -2.6080965995788574,
       "step": 200
     },
     {
       "epoch": 1.9437037037037037,
-      "grad_norm": 900.0,
+      "grad_norm": 576.0,
       "learning_rate": 1.643500976631037e-05,
-      "log_odds_chosen": 6.316862106323242,
-      "log_odds_ratio": -0.5562707781791687,
-      "logps/chosen": -1.6930224895477295,
-      "logps/rejected": -7.736858367919922,
-      "loss": -2.5638,
-      "nll_loss": 1.7886855602264404,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.8465112447738647,
-      "rewards/margins": 3.0219178199768066,
-      "rewards/rejected": -3.868429183959961,
+      "log_odds_chosen": 3.051617383956909,
+      "log_odds_ratio": -0.38788479566574097,
+      "logps/chosen": -1.2543895244598389,
+      "logps/rejected": -4.006674289703369,
+      "loss": 16.8108,
+      "nll_loss": 1.5027841329574585,
+      "rewards/accuracies": 0.8218749761581421,
+      "rewards/chosen": -0.6271947622299194,
+      "rewards/margins": 1.3761423826217651,
+      "rewards/rejected": -2.0033371448516846,
       "step": 205
     },
     {
       "epoch": 1.991111111111111,
-      "grad_norm": 940.0,
+      "grad_norm": 386.0,
       "learning_rate": 1.514521724066537e-05,
-      "log_odds_chosen": 6.6161909103393555,
-      "log_odds_ratio": -0.8282305598258972,
-      "logps/chosen": -2.5031771659851074,
-      "logps/rejected": -8.889016151428223,
-      "loss": 3.2385,
-      "nll_loss": 1.9650413990020752,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.2515885829925537,
-      "rewards/margins": 3.1929194927215576,
-      "rewards/rejected": -4.444508075714111,
+      "log_odds_chosen": 3.231032609939575,
+      "log_odds_ratio": -0.43392324447631836,
+      "logps/chosen": -1.4704937934875488,
+      "logps/rejected": -4.4187726974487305,
+      "loss": 17.4563,
+      "nll_loss": 1.565903902053833,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.7352468967437744,
+      "rewards/margins": 1.4741394519805908,
+      "rewards/rejected": -2.2093863487243652,
       "step": 210
     },
     {
       "epoch": 2.0385185185185186,
-      "grad_norm": 652.0,
+      "grad_norm": 278.0,
       "learning_rate": 1.3885777771950348e-05,
-      "log_odds_chosen": 9.489248275756836,
-      "log_odds_ratio": -0.48209771513938904,
-      "logps/chosen": -1.8140331506729126,
-      "logps/rejected": -10.948740005493164,
-      "loss": -27.9736,
-      "nll_loss": 1.7284278869628906,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.9070165753364563,
-      "rewards/margins": 4.567353248596191,
-      "rewards/rejected": -5.474370002746582,
+      "log_odds_chosen": 5.03135871887207,
+      "log_odds_ratio": -0.3135664165019989,
+      "logps/chosen": -1.407021164894104,
+      "logps/rejected": -6.015364646911621,
+      "loss": 0.5226,
+      "nll_loss": 1.4759743213653564,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.703510582447052,
+      "rewards/margins": 2.3041718006134033,
+      "rewards/rejected": -3.0076823234558105,
       "step": 215
     },
     {
       "epoch": 2.0859259259259257,
-      "grad_norm": 1088.0,
+      "grad_norm": 378.0,
       "learning_rate": 1.2660570475395683e-05,
-      "log_odds_chosen": 9.268631935119629,
-      "log_odds_ratio": -0.5644906759262085,
-      "logps/chosen": -2.2069149017333984,
-      "logps/rejected": -11.19309139251709,
-      "loss": -25.8251,
-      "nll_loss": 1.8661258220672607,
-      "rewards/accuracies": 0.840624988079071,
-      "rewards/chosen": -1.1034574508666992,
-      "rewards/margins": 4.493088722229004,
-      "rewards/rejected": -5.596545696258545,
+      "log_odds_chosen": 5.138548374176025,
+      "log_odds_ratio": -0.25997596979141235,
+      "logps/chosen": -1.37165367603302,
+      "logps/rejected": -6.125999450683594,
+      "loss": -0.8241,
+      "nll_loss": 1.4749200344085693,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.68582683801651,
+      "rewards/margins": 2.3771729469299316,
+      "rewards/rejected": -3.062999725341797,
       "step": 220
     },
     {
       "epoch": 2.1333333333333333,
-      "grad_norm": 1536.0,
+      "grad_norm": 396.0,
       "learning_rate": 1.1473369030008974e-05,
-      "log_odds_chosen": 13.42963695526123,
-      "log_odds_ratio": -0.2936347424983978,
-      "logps/chosen": -2.7818703651428223,
-      "logps/rejected": -15.93250560760498,
-      "loss": -38.3857,
-      "nll_loss": 2.1401987075805664,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": -1.3909351825714111,
-      "rewards/margins": 6.5753173828125,
-      "rewards/rejected": -7.96625280380249,
+      "log_odds_chosen": 6.11403751373291,
+      "log_odds_ratio": -0.1846025437116623,
+      "logps/chosen": -1.402966856956482,
+      "logps/rejected": -7.111693382263184,
+      "loss": -8.5304,
+      "nll_loss": 1.5103604793548584,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -0.701483428478241,
+      "rewards/margins": 2.854363441467285,
+      "rewards/rejected": -3.555846691131592,
       "step": 225
     },
     {
       "epoch": 2.180740740740741,
-      "grad_norm": 1392.0,
+      "grad_norm": 368.0,
       "learning_rate": 1.0327830055518842e-05,
-      "log_odds_chosen": 10.711917877197266,
-      "log_odds_ratio": -0.31785595417022705,
-      "logps/chosen": -1.934106469154358,
-      "logps/rejected": -12.327714920043945,
-      "loss": -36.8357,
-      "nll_loss": 1.7895313501358032,
-      "rewards/accuracies": 0.8968750238418579,
-      "rewards/chosen": -0.967053234577179,
-      "rewards/margins": 5.196803092956543,
-      "rewards/rejected": -6.163857460021973,
+      "log_odds_chosen": 6.256018161773682,
+      "log_odds_ratio": -0.18493634462356567,
+      "logps/chosen": -1.3649917840957642,
+      "logps/rejected": -7.209232330322266,
+      "loss": -7.4943,
+      "nll_loss": 1.4895808696746826,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -0.6824958920478821,
+      "rewards/margins": 2.9221205711364746,
+      "rewards/rejected": -3.604616165161133,
       "step": 230
     },
     {
       "epoch": 2.228148148148148,
-      "grad_norm": 668.0,
+      "grad_norm": 328.0,
       "learning_rate": 9.227481849865235e-06,
-      "log_odds_chosen": 10.63416862487793,
-      "log_odds_ratio": -0.34915637969970703,
-      "logps/chosen": -1.4962049722671509,
-      "logps/rejected": -11.764998435974121,
-      "loss": -36.5253,
-      "nll_loss": 1.6011587381362915,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -0.7481024861335754,
-      "rewards/margins": 5.134397029876709,
-      "rewards/rejected": -5.8824992179870605,
+      "log_odds_chosen": 6.152120113372803,
+      "log_odds_ratio": -0.22947028279304504,
+      "logps/chosen": -1.2814245223999023,
+      "logps/rejected": -7.014758110046387,
+      "loss": -6.2174,
+      "nll_loss": 1.4657459259033203,
+      "rewards/accuracies": 0.9156249761581421,
+      "rewards/chosen": -0.6407122611999512,
+      "rewards/margins": 2.866666793823242,
+      "rewards/rejected": -3.5073790550231934,
       "step": 235
     },
     {
       "epoch": 2.2755555555555556,
-      "grad_norm": 644.0,
+      "grad_norm": 440.0,
       "learning_rate": 8.175713521924978e-06,
-      "log_odds_chosen": 12.23620891571045,
-      "log_odds_ratio": -0.1865963637828827,
-      "logps/chosen": -1.4724217653274536,
-      "logps/rejected": -13.347505569458008,
-      "loss": -51.2,
-      "nll_loss": 1.510587215423584,
-      "rewards/accuracies": 0.8968750238418579,
-      "rewards/chosen": -0.7362108826637268,
-      "rewards/margins": 5.93754243850708,
-      "rewards/rejected": -6.673752784729004,
+      "log_odds_chosen": 6.160048484802246,
+      "log_odds_ratio": -0.1592252552509308,
+      "logps/chosen": -1.235910177230835,
+      "logps/rejected": -6.972237586975098,
+      "loss": -8.9307,
+      "nll_loss": 1.388727068901062,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": -0.6179550886154175,
+      "rewards/margins": 2.8681633472442627,
+      "rewards/rejected": -3.486118793487549,
       "step": 240
     },
     {
       "epoch": 2.322962962962963,
-      "grad_norm": 1704.0,
+      "grad_norm": 412.0,
       "learning_rate": 7.1757645529443665e-06,
-      "log_odds_chosen": 15.528274536132812,
-      "log_odds_ratio": -0.33081239461898804,
-      "logps/chosen": -2.277860641479492,
-      "logps/rejected": -17.496549606323242,
-      "loss": -45.7833,
-      "nll_loss": 2.0120162963867188,
-      "rewards/accuracies": 0.9156249761581421,
-      "rewards/chosen": -1.138930320739746,
-      "rewards/margins": 7.609343528747559,
-      "rewards/rejected": -8.748274803161621,
+      "log_odds_chosen": 6.512506008148193,
+      "log_odds_ratio": -0.16299432516098022,
+      "logps/chosen": -1.2822771072387695,
+      "logps/rejected": -7.385122776031494,
+      "loss": -10.2451,
+      "nll_loss": 1.4544556140899658,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -0.6411385536193848,
+      "rewards/margins": 3.051422595977783,
+      "rewards/rejected": -3.692561388015747,
       "step": 245
     },
     {
       "epoch": 2.3703703703703702,
-      "grad_norm": 1384.0,
+      "grad_norm": 366.0,
       "learning_rate": 6.230714818829733e-06,
-      "log_odds_chosen": 14.397786140441895,
-      "log_odds_ratio": -0.28585368394851685,
-      "logps/chosen": -2.087308645248413,
-      "logps/rejected": -16.145244598388672,
-      "loss": -41.6852,
-      "nll_loss": 1.8240935802459717,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -1.0436543226242065,
-      "rewards/margins": 7.02896785736084,
-      "rewards/rejected": -8.072622299194336,
+      "log_odds_chosen": 6.763820648193359,
+      "log_odds_ratio": -0.15657763183116913,
+      "logps/chosen": -1.2465496063232422,
+      "logps/rejected": -7.550148010253906,
+      "loss": -8.9088,
+      "nll_loss": 1.4477818012237549,
+      "rewards/accuracies": 0.9281250238418579,
+      "rewards/chosen": -0.6232748031616211,
+      "rewards/margins": 3.151798963546753,
+      "rewards/rejected": -3.775074005126953,
       "step": 250
     },
     {
       "epoch": 2.417777777777778,
-      "grad_norm": 960.0,
+      "grad_norm": 348.0,
       "learning_rate": 5.343475104027743e-06,
-      "log_odds_chosen": 13.305925369262695,
-      "log_odds_ratio": -0.2490675449371338,
-      "logps/chosen": -1.9038841724395752,
-      "logps/rejected": -14.872416496276855,
-      "loss": -47.3843,
-      "nll_loss": 1.7593896389007568,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -0.9519420862197876,
-      "rewards/margins": 6.484266757965088,
-      "rewards/rejected": -7.436208248138428,
+      "log_odds_chosen": 6.812985897064209,
+      "log_odds_ratio": -0.13997027277946472,
+      "logps/chosen": -1.285111665725708,
+      "logps/rejected": -7.66811990737915,
+      "loss": -11.2052,
+      "nll_loss": 1.4328773021697998,
+      "rewards/accuracies": 0.9468749761581421,
+      "rewards/chosen": -0.642555832862854,
+      "rewards/margins": 3.1915042400360107,
+      "rewards/rejected": -3.834059953689575,
       "step": 255
     },
     {
       "epoch": 2.4651851851851854,
-      "grad_norm": 1152.0,
+      "grad_norm": 632.0,
       "learning_rate": 4.516778136213037e-06,
-      "log_odds_chosen": 13.447871208190918,
-      "log_odds_ratio": -0.271032452583313,
-      "logps/chosen": -1.6334911584854126,
-      "logps/rejected": -14.739973068237305,
-      "loss": -45.342,
-      "nll_loss": 1.6864122152328491,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -0.8167455792427063,
-      "rewards/margins": 6.553240776062012,
-      "rewards/rejected": -7.369986534118652,
+      "log_odds_chosen": 6.276130676269531,
+      "log_odds_ratio": -0.18893949687480927,
+      "logps/chosen": -1.3154428005218506,
+      "logps/rejected": -7.166337013244629,
+      "loss": -5.5648,
+      "nll_loss": 1.4617598056793213,
+      "rewards/accuracies": 0.9156249761581421,
+      "rewards/chosen": -0.6577214002609253,
+      "rewards/margins": 2.9254469871520996,
+      "rewards/rejected": -3.5831685066223145,
       "step": 260
     },
     {
       "epoch": 2.5125925925925925,
-      "grad_norm": 1080.0,
+      "grad_norm": 644.0,
       "learning_rate": 3.7531701693965554e-06,
-      "log_odds_chosen": 12.9279203414917,
-      "log_odds_ratio": -0.3936213552951813,
-      "logps/chosen": -1.8595199584960938,
-      "logps/rejected": -14.45753288269043,
-      "loss": -43.4362,
-      "nll_loss": 1.7575229406356812,
-      "rewards/accuracies": 0.8843749761581421,
-      "rewards/chosen": -0.9297599792480469,
-      "rewards/margins": 6.299006938934326,
-      "rewards/rejected": -7.228766441345215,
+      "log_odds_chosen": 6.7295122146606445,
+      "log_odds_ratio": -0.2116066962480545,
+      "logps/chosen": -1.275740623474121,
+      "logps/rejected": -7.586668968200684,
+      "loss": -9.4219,
+      "nll_loss": 1.4439576864242554,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -0.6378703117370605,
+      "rewards/margins": 3.155463933944702,
+      "rewards/rejected": -3.793334484100342,
       "step": 265
     },
     {
       "epoch": 2.56,
-      "grad_norm": 1016.0,
+      "grad_norm": 314.0,
       "learning_rate": 3.055003141378948e-06,
-      "log_odds_chosen": 13.242762565612793,
-      "log_odds_ratio": -0.2418208122253418,
-      "logps/chosen": -1.5649698972702026,
-      "logps/rejected": -14.447901725769043,
-      "loss": -49.7394,
-      "nll_loss": 1.635581612586975,
-      "rewards/accuracies": 0.903124988079071,
-      "rewards/chosen": -0.7824849486351013,
-      "rewards/margins": 6.441465854644775,
-      "rewards/rejected": -7.2239508628845215,
+      "log_odds_chosen": 6.7960004806518555,
+      "log_odds_ratio": -0.18422751128673553,
+      "logps/chosen": -1.1799513101577759,
+      "logps/rejected": -7.515390872955322,
+      "loss": -11.538,
+      "nll_loss": 1.3993252515792847,
+      "rewards/accuracies": 0.934374988079071,
+      "rewards/chosen": -0.5899756550788879,
+      "rewards/margins": 3.167720079421997,
+      "rewards/rejected": -3.757695436477661,
       "step": 270
     },
     {
       "epoch": 2.6074074074074076,
-      "grad_norm": 1080.0,
+      "grad_norm": 398.0,
       "learning_rate": 2.424427429704365e-06,
-      "log_odds_chosen": 13.131576538085938,
-      "log_odds_ratio": -0.24517115950584412,
-      "logps/chosen": -1.7483772039413452,
-      "logps/rejected": -14.538126945495605,
-      "loss": -50.0705,
-      "nll_loss": 1.7109159231185913,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": -0.8741886019706726,
-      "rewards/margins": 6.394874572753906,
-      "rewards/rejected": -7.269063472747803,
+      "log_odds_chosen": 6.602663516998291,
+      "log_odds_ratio": -0.18241380155086517,
+      "logps/chosen": -1.3822739124298096,
+      "logps/rejected": -7.560347557067871,
+      "loss": -8.7863,
+      "nll_loss": 1.474333643913269,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.6911369562149048,
+      "rewards/margins": 3.0890371799468994,
+      "rewards/rejected": -3.7801737785339355,
       "step": 275
     },
     {
       "epoch": 2.6548148148148147,
-      "grad_norm": 1320.0,
+      "grad_norm": 700.0,
       "learning_rate": 1.8633852284264508e-06,
-      "log_odds_chosen": 12.429898262023926,
-      "log_odds_ratio": -0.4006059169769287,
-      "logps/chosen": -2.0325827598571777,
-      "logps/rejected": -14.141799926757812,
-      "loss": -34.9616,
-      "nll_loss": 1.9175059795379639,
-      "rewards/accuracies": 0.8656250238418579,
-      "rewards/chosen": -1.0162913799285889,
-      "rewards/margins": 6.054608345031738,
-      "rewards/rejected": -7.070899963378906,
+      "log_odds_chosen": 6.423117637634277,
+      "log_odds_ratio": -0.18244215846061707,
+      "logps/chosen": -1.2519899606704712,
+      "logps/rejected": -7.237380027770996,
+      "loss": -7.6169,
+      "nll_loss": 1.4630454778671265,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -0.6259949803352356,
+      "rewards/margins": 2.9926953315734863,
+      "rewards/rejected": -3.618690013885498,
       "step": 280
     },
     {
       "epoch": 2.7022222222222223,
-      "grad_norm": 1280.0,
+      "grad_norm": 376.0,
       "learning_rate": 1.3736045660864034e-06,
-      "log_odds_chosen": 11.897767066955566,
-      "log_odds_ratio": -0.36061421036720276,
-      "logps/chosen": -1.7342910766601562,
-      "logps/rejected": -13.319345474243164,
-      "loss": -42.368,
-      "nll_loss": 1.7154544591903687,
-      "rewards/accuracies": 0.871874988079071,
-      "rewards/chosen": -0.8671455383300781,
-      "rewards/margins": 5.792527198791504,
-      "rewards/rejected": -6.659672737121582,
+      "log_odds_chosen": 6.383296966552734,
+      "log_odds_ratio": -0.22375822067260742,
+      "logps/chosen": -1.2126858234405518,
+      "logps/rejected": -7.17873477935791,
+      "loss": -6.2815,
+      "nll_loss": 1.423060417175293,
+      "rewards/accuracies": 0.8968750238418579,
+      "rewards/chosen": -0.6063429117202759,
+      "rewards/margins": 2.9830245971679688,
+      "rewards/rejected": -3.589367389678955,
       "step": 285
     },
     {
       "epoch": 2.74962962962963,
-      "grad_norm": 1472.0,
+      "grad_norm": 288.0,
       "learning_rate": 9.565939833279192e-07,
-      "log_odds_chosen": 12.41030216217041,
-      "log_odds_ratio": -0.48926201462745667,
-      "logps/chosen": -2.32023286819458,
-      "logps/rejected": -14.431295394897461,
-      "loss": -30.8903,
-      "nll_loss": 2.0258736610412598,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -1.16011643409729,
-      "rewards/margins": 6.0555315017700195,
-      "rewards/rejected": -7.2156476974487305,
+      "log_odds_chosen": 5.9918012619018555,
+      "log_odds_ratio": -0.24828758835792542,
+      "logps/chosen": -1.4515832662582397,
+      "logps/rejected": -7.042089939117432,
+      "loss": -2.5592,
+      "nll_loss": 1.5330219268798828,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.7257916331291199,
+      "rewards/margins": 2.7952535152435303,
+      "rewards/rejected": -3.521044969558716,
       "step": 290
     },
     {
       "epoch": 2.797037037037037,
-      "grad_norm": 920.0,
+      "grad_norm": 452.0,
       "learning_rate": 6.136378865420872e-07,
-      "log_odds_chosen": 12.211552619934082,
-      "log_odds_ratio": -0.4335884153842926,
-      "logps/chosen": -1.8138420581817627,
-      "logps/rejected": -13.713777542114258,
-      "loss": -37.5638,
-      "nll_loss": 1.7503633499145508,
-      "rewards/accuracies": 0.859375,
-      "rewards/chosen": -0.9069210290908813,
-      "rewards/margins": 5.949967384338379,
-      "rewards/rejected": -6.856888771057129,
+      "log_odds_chosen": 6.23110294342041,
+      "log_odds_ratio": -0.2531821131706238,
+      "logps/chosen": -1.3049705028533936,
+      "logps/rejected": -7.137003421783447,
+      "loss": -4.6802,
+      "nll_loss": 1.456378698348999,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.6524852514266968,
+      "rewards/margins": 2.9160165786743164,
+      "rewards/rejected": -3.5685017108917236,
       "step": 295
     },
     {
       "epoch": 2.8444444444444446,
-      "grad_norm": 1648.0,
+      "grad_norm": 354.0,
       "learning_rate": 3.45792591853214e-07,
-      "log_odds_chosen": 13.25987434387207,
-      "log_odds_ratio": -0.322501003742218,
-      "logps/chosen": -1.6583086252212524,
-      "logps/rejected": -14.573458671569824,
-      "loss": -45.0139,
-      "nll_loss": 1.6782853603363037,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -0.8291543126106262,
-      "rewards/margins": 6.45757532119751,
-      "rewards/rejected": -7.286729335784912,
+      "log_odds_chosen": 6.851447105407715,
+      "log_odds_ratio": -0.21110931038856506,
+      "logps/chosen": -1.1985723972320557,
+      "logps/rejected": -7.616302490234375,
+      "loss": -9.172,
+      "nll_loss": 1.4328322410583496,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.5992861986160278,
+      "rewards/margins": 3.20886492729187,
+      "rewards/rejected": -3.8081512451171875,
       "step": 300
     },
     {
       "epoch": 2.891851851851852,
-      "grad_norm": 1232.0,
+      "grad_norm": 468.0,
       "learning_rate": 1.538830716302092e-07,
-      "log_odds_chosen": 12.726137161254883,
-      "log_odds_ratio": -0.36502203345298767,
-      "logps/chosen": -1.7828433513641357,
-      "logps/rejected": -14.167094230651855,
-      "loss": -41.4455,
-      "nll_loss": 1.751901626586914,
-      "rewards/accuracies": 0.8843749761581421,
-      "rewards/chosen": -0.8914216756820679,
-      "rewards/margins": 6.19212532043457,
-      "rewards/rejected": -7.083547115325928,
+      "log_odds_chosen": 6.681235313415527,
+      "log_odds_ratio": -0.16963359713554382,
+      "logps/chosen": -1.2472527027130127,
+      "logps/rejected": -7.471646308898926,
+      "loss": -10.8492,
+      "nll_loss": 1.403349757194519,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -0.6236263513565063,
+      "rewards/margins": 3.112197160720825,
+      "rewards/rejected": -3.735823154449463,
       "step": 305
     },
     {
       "epoch": 2.9392592592592592,
-      "grad_norm": 1456.0,
+      "grad_norm": 384.0,
       "learning_rate": 3.8500413544415025e-08,
-      "log_odds_chosen": 13.806699752807617,
-      "log_odds_ratio": -0.15578700602054596,
-      "logps/chosen": -1.7442131042480469,
-      "logps/rejected": -15.208877563476562,
-      "loss": -47.4243,
-      "nll_loss": 1.7841014862060547,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": -0.8721065521240234,
-      "rewards/margins": 6.732332706451416,
-      "rewards/rejected": -7.604438781738281,
+      "log_odds_chosen": 6.875144958496094,
+      "log_odds_ratio": -0.12083999067544937,
+      "logps/chosen": -1.2005863189697266,
+      "logps/rejected": -7.624534606933594,
+      "loss": -11.3909,
+      "nll_loss": 1.4330816268920898,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.6002931594848633,
+      "rewards/margins": 3.2119739055633545,
+      "rewards/rejected": -3.812267303466797,
       "step": 310
     },
     {
       "epoch": 2.986666666666667,
-      "grad_norm": 1952.0,
+      "grad_norm": 354.0,
       "learning_rate": 0.0,
-      "log_odds_chosen": 12.691495895385742,
-      "log_odds_ratio": -0.2689394950866699,
-      "logps/chosen": -1.6713707447052002,
-      "logps/rejected": -13.993799209594727,
-      "loss": -43.2258,
-      "nll_loss": 1.702843427658081,
-      "rewards/accuracies": 0.8843749761581421,
-      "rewards/chosen": -0.8356853723526001,
-      "rewards/margins": 6.1612138748168945,
-      "rewards/rejected": -6.996899604797363,
+      "log_odds_chosen": 6.552358150482178,
+      "log_odds_ratio": -0.21020980179309845,
+      "logps/chosen": -1.2049369812011719,
+      "logps/rejected": -7.295576572418213,
+      "loss": -9.7319,
+      "nll_loss": 1.3733375072479248,
+      "rewards/accuracies": 0.903124988079071,
+      "rewards/chosen": -0.6024684906005859,
+      "rewards/margins": 3.0453197956085205,
+      "rewards/rejected": -3.6477882862091064,
       "step": 315
     },
     {
       "epoch": 2.986666666666667,
       "step": 315,
       "total_flos": 0.0,
-      "train_loss": 25.709435809604706,
-      "train_runtime": 7268.7111,
-      "train_samples_per_second": 2.786,
+      "train_loss": 43.15912766380916,
+      "train_runtime": 7320.6498,
+      "train_samples_per_second": 2.766,
       "train_steps_per_second": 0.043
     }
   ],