Model save

README.md

@@ -4,6 +4,7 @@ base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 model-index:
 - name: zephyr-7b-dpo-full-prometheus_consistent-reward-scale-1-rpo
@@ -17,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0340
-- Rewards/chosen: -0.0965
-- Rewards/rejected: -0.3938
+- Loss: 0.0341
+- Rewards/chosen: -0.0738
+- Rewards/rejected: -0.3717
 - Rewards/accuracies: 0.7414
-- Rewards/margins: 0.2973
-- Logps/rejected: -258.4546
-- Logps/chosen: -285.2520
-- Logits/rejected: -2.1563
-- Logits/chosen: -2.3140
+- Rewards/margins: 0.2979
+- Logps/rejected: -256.2462
+- Logps/chosen: -282.9861
+- Logits/rejected: -2.4523
+- Logits/chosen: -2.5611
 
 ## Model description
 
@@ -62,14 +63,14 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.048         | 0.1143 | 50   | 0.0428          | 0.0615         | -0.0563          | 0.7026             | 0.1178          | -224.7078      | -269.4535    | -2.4545         | -2.5520       |
-| 0.0399        | 0.2286 | 100  | 0.0385          | -0.0747        | -0.3118          | 0.75               | 0.2371          | -250.2514      | -283.0706    | -1.9893         | -2.1601       |
-| 0.0367        | 0.3429 | 150  | 0.0371          | -0.1934        | -0.4431          | 0.7672             | 0.2497          | -263.3893      | -294.9446    | -2.3057         | -2.4218       |
-| 0.0375        | 0.4571 | 200  | 0.0353          | -0.0541        | -0.3320          | 0.7672             | 0.2779          | -252.2786      | -281.0130    | -2.1436         | -2.2907       |
-| 0.0371        | 0.5714 | 250  | 0.0344          | -0.0812        | -0.3496          | 0.7629             | 0.2684          | -254.0325      | -283.7219    | -2.2615         | -2.3785       |
-| 0.0345        | 0.6857 | 300  | 0.0341          | -0.0682        | -0.3495          | 0.7457             | 0.2813          | -254.0265      | -282.4234    | -2.2130         | -2.3475       |
-| 0.0373        | 0.8    | 350  | 0.0341          | -0.0908        | -0.3849          | 0.7414             | 0.2941          | -257.5619      | -284.6819    | -2.1788         | -2.3321       |
-| 0.0367        | 0.9143 | 400  | 0.0340          | -0.0965        | -0.3938          | 0.7414             | 0.2973          | -258.4546      | -285.2520    | -2.1563         | -2.3140       |
+| 0.048         | 0.1143 | 50   | 0.0428          | 0.0621         | -0.0550          | 0.7026             | 0.1170          | -224.5715      | -269.3974    | -2.4545         | -2.5523       |
+| 0.04          | 0.2286 | 100  | 0.0385          | -0.0777        | -0.3186          | 0.75               | 0.2409          | -250.9367      | -283.3715    | -1.9565         | -2.1306       |
+| 0.0363        | 0.3429 | 150  | 0.0371          | -0.2052        | -0.4595          | 0.7543             | 0.2543          | -265.0228      | -296.1211    | -2.1955         | -2.3441       |
+| 0.0373        | 0.4571 | 200  | 0.0353          | -0.0452        | -0.3269          | 0.7716             | 0.2817          | -251.7630      | -280.1239    | -2.3848         | -2.4903       |
+| 0.0374        | 0.5714 | 250  | 0.0344          | -0.0802        | -0.3463          | 0.75               | 0.2662          | -253.7082      | -283.6198    | -2.4307         | -2.5245       |
+| 0.0346        | 0.6857 | 300  | 0.0342          | -0.0372        | -0.3195          | 0.7457             | 0.2823          | -251.0285      | -279.3270    | -2.4797         | -2.5812       |
+| 0.0375        | 0.8    | 350  | 0.0342          | -0.0783        | -0.3746          | 0.7414             | 0.2963          | -256.5389      | -283.4324    | -2.4474         | -2.5561       |
+| 0.0367        | 0.9143 | 400  | 0.0341          | -0.0738        | -0.3717          | 0.7414             | 0.2979          | -256.2462      | -282.9861    | -2.4523         | -2.5611       |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,22 @@
 {
     "epoch": 0.9988571428571429,
+    "eval_logits/chosen": -2.3148677349090576,
+    "eval_logits/rejected": -2.157057762145996,
+    "eval_logps/chosen": -285.280517578125,
+    "eval_logps/rejected": -258.34063720703125,
+    "eval_loss": 0.03401629254221916,
+    "eval_rewards/accuracies": 0.7413793206214905,
+    "eval_rewards/chosen": -0.09676530957221985,
+    "eval_rewards/margins": 0.2958797514438629,
+    "eval_rewards/rejected": -0.39264509081840515,
+    "eval_runtime": 90.641,
+    "eval_samples": 1831,
+    "eval_samples_per_second": 20.201,
+    "eval_steps_per_second": 0.32,
     "total_flos": 0.0,
-    "train_loss": 0.038760803656938146,
-    "train_runtime": 11420.4923,
+    "train_loss": 0.03879852061004333,
+    "train_runtime": 11351.3146,
     "train_samples": 55999,
-    "train_samples_per_second": 4.903,
+    "train_samples_per_second": 4.933,
     "train_steps_per_second": 0.038
 }

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 0.9988571428571429,
+    "eval_logits/chosen": -2.3148677349090576,
+    "eval_logits/rejected": -2.157057762145996,
+    "eval_logps/chosen": -285.280517578125,
+    "eval_logps/rejected": -258.34063720703125,
+    "eval_loss": 0.03401629254221916,
+    "eval_rewards/accuracies": 0.7413793206214905,
+    "eval_rewards/chosen": -0.09676530957221985,
+    "eval_rewards/margins": 0.2958797514438629,
+    "eval_rewards/rejected": -0.39264509081840515,
+    "eval_runtime": 90.641,
+    "eval_samples": 1831,
+    "eval_samples_per_second": 20.201,
+    "eval_steps_per_second": 0.32
+}

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a16996512c2f0318b313acd5cbbd2955f2de60f80643fe3362509213fc4b96e
+oid sha256:e36aaff6bebca3ccf46a29f9662e6134a9641e003c49d97a362e7df005167c6a
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a50856b4422d2977f2685ab7f0f05f160b22d5953300d21883994a161722a20
+oid sha256:c8d6e4adf391ff6afd4d06c9ae6425c1c4d3d7d377dea5ebace959c50e844972
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c01b4a827fe28570eefef7140f3d2290f7897751116084c5469bd9ec2b5d1d0f
+oid sha256:51809653eb1aabf994d17f71339dc346305d93484261fc3f2573f85d188d5891
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9988571428571429,
     "total_flos": 0.0,
-    "train_loss": 0.038760803656938146,
-    "train_runtime": 11420.4923,
+    "train_loss": 0.03879852061004333,
+    "train_runtime": 11351.3146,
     "train_samples": 55999,
-    "train_samples_per_second": 4.903,
+    "train_samples_per_second": 4.933,
     "train_steps_per_second": 0.038
 }

trainer_state.json

@@ -10,784 +10,784 @@
   "log_history": [
     {
       "epoch": 0.022857142857142857,
-      "grad_norm": 3.2870738344316774,
+      "grad_norm": 3.2784174727745485,
       "learning_rate": 1.1363636363636363e-07,
-      "logits/chosen": -2.700951099395752,
-      "logits/rejected": -2.6252384185791016,
-      "logps/chosen": -301.28485107421875,
-      "logps/rejected": -281.7684020996094,
-      "loss": 0.0556,
-      "rewards/accuracies": 0.41874998807907104,
-      "rewards/chosen": 1.3337824384507257e-05,
-      "rewards/margins": 0.00020079524256289005,
-      "rewards/rejected": -0.0001874573645181954,
+      "logits/chosen": -2.700894832611084,
+      "logits/rejected": -2.625084400177002,
+      "logps/chosen": -301.3052062988281,
+      "logps/rejected": -281.75274658203125,
+      "loss": 0.0557,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": -0.00019026221707463264,
+      "rewards/margins": -0.00015981406613718718,
+      "rewards/rejected": -3.0448136385530233e-05,
       "step": 10
     },
     {
       "epoch": 0.045714285714285714,
-      "grad_norm": 2.5214503350239097,
+      "grad_norm": 2.5255601771436815,
       "learning_rate": 2.2727272727272726e-07,
-      "logits/chosen": -2.641275405883789,
-      "logits/rejected": -2.606044292449951,
-      "logps/chosen": -278.9186706542969,
-      "logps/rejected": -254.6912384033203,
-      "loss": 0.0552,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.002525148680433631,
-      "rewards/margins": 0.0017468780279159546,
-      "rewards/rejected": 0.0007782711763866246,
+      "logits/chosen": -2.641096830368042,
+      "logits/rejected": -2.6057686805725098,
+      "logps/chosen": -278.879638671875,
+      "logps/rejected": -254.6970672607422,
+      "loss": 0.0553,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.002915444318205118,
+      "rewards/margins": 0.002195452805608511,
+      "rewards/rejected": 0.0007199913961812854,
       "step": 20
     },
     {
       "epoch": 0.06857142857142857,
-      "grad_norm": 2.7331110185448306,
+      "grad_norm": 2.762527455327235,
       "learning_rate": 3.4090909090909085e-07,
-      "logits/chosen": -2.6379804611206055,
-      "logits/rejected": -2.617119550704956,
-      "logps/chosen": -263.29449462890625,
-      "logps/rejected": -263.40509033203125,
+      "logits/chosen": -2.6387953758239746,
+      "logits/rejected": -2.6177260875701904,
+      "logps/chosen": -263.30303955078125,
+      "logps/rejected": -263.4232177734375,
       "loss": 0.0535,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.01388559676706791,
-      "rewards/margins": 0.008006090298295021,
-      "rewards/rejected": 0.00587950786575675,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.013800328597426414,
+      "rewards/margins": 0.008101832121610641,
+      "rewards/rejected": 0.005698497407138348,
       "step": 30
     },
     {
       "epoch": 0.09142857142857143,
-      "grad_norm": 2.830903095609413,
+      "grad_norm": 2.8469620893370813,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": -2.6505517959594727,
-      "logits/rejected": -2.5873801708221436,
-      "logps/chosen": -290.2440185546875,
-      "logps/rejected": -268.03863525390625,
+      "logits/chosen": -2.6508777141571045,
+      "logits/rejected": -2.5878684520721436,
+      "logps/chosen": -290.2543640136719,
+      "logps/rejected": -268.0738830566406,
       "loss": 0.0502,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.037532415241003036,
-      "rewards/margins": 0.04160536080598831,
-      "rewards/rejected": -0.004072942305356264,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.03742903843522072,
+      "rewards/margins": 0.04185490682721138,
+      "rewards/rejected": -0.004425862338393927,
       "step": 40
     },
     {
       "epoch": 0.11428571428571428,
-      "grad_norm": 6.7770519006865815,
+      "grad_norm": 7.044347610102239,
       "learning_rate": 4.997124959943201e-07,
-      "logits/chosen": -2.6873106956481934,
-      "logits/rejected": -2.6081948280334473,
-      "logps/chosen": -290.84735107421875,
-      "logps/rejected": -249.636474609375,
+      "logits/chosen": -2.68748140335083,
+      "logits/rejected": -2.6086513996124268,
+      "logps/chosen": -290.8190612792969,
+      "logps/rejected": -249.57925415039062,
       "loss": 0.048,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.05462931841611862,
-      "rewards/margins": 0.08235044032335281,
-      "rewards/rejected": -0.027721121907234192,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.05491219088435173,
+      "rewards/margins": 0.08206149935722351,
+      "rewards/rejected": -0.02714930847287178,
       "step": 50
     },
     {
       "epoch": 0.11428571428571428,
-      "eval_logits/chosen": -2.552030086517334,
-      "eval_logits/rejected": -2.4545059204101562,
-      "eval_logps/chosen": -269.4535217285156,
-      "eval_logps/rejected": -224.7078094482422,
-      "eval_loss": 0.04283829778432846,
+      "eval_logits/chosen": -2.55232834815979,
+      "eval_logits/rejected": -2.4544785022735596,
+      "eval_logps/chosen": -269.3974304199219,
+      "eval_logps/rejected": -224.57150268554688,
+      "eval_loss": 0.04283786565065384,
       "eval_rewards/accuracies": 0.7025862336158752,
-      "eval_rewards/chosen": 0.06150422990322113,
-      "eval_rewards/margins": 0.11782074719667435,
-      "eval_rewards/rejected": -0.05631651729345322,
-      "eval_runtime": 91.0985,
-      "eval_samples_per_second": 20.099,
+      "eval_rewards/chosen": 0.06206508353352547,
+      "eval_rewards/margins": 0.11701909452676773,
+      "eval_rewards/rejected": -0.054954010993242264,
+      "eval_runtime": 91.1445,
+      "eval_samples_per_second": 20.089,
       "eval_steps_per_second": 0.318,
       "step": 50
     },
     {
       "epoch": 0.13714285714285715,
-      "grad_norm": 3.063883877478527,
+      "grad_norm": 3.0003693671132106,
       "learning_rate": 4.979579212164186e-07,
-      "logits/chosen": -2.5849452018737793,
-      "logits/rejected": -2.476766347885132,
-      "logps/chosen": -280.53155517578125,
-      "logps/rejected": -259.91339111328125,
+      "logits/chosen": -2.5842273235321045,
+      "logits/rejected": -2.476365804672241,
+      "logps/chosen": -280.49664306640625,
+      "logps/rejected": -259.8401794433594,
       "loss": 0.0438,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.00025545060634613037,
-      "rewards/margins": 0.11335714906454086,
-      "rewards/rejected": -0.1136125922203064,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 9.337197843706235e-05,
+      "rewards/margins": 0.11297394335269928,
+      "rewards/rejected": -0.11288057267665863,
       "step": 60
     },
     {
       "epoch": 0.16,
-      "grad_norm": 3.212195825091894,
+      "grad_norm": 3.2128849852724057,
       "learning_rate": 4.946196886175515e-07,
-      "logits/chosen": -2.527914524078369,
-      "logits/rejected": -2.468449354171753,
-      "logps/chosen": -273.3289489746094,
-      "logps/rejected": -270.74847412109375,
-      "loss": 0.0414,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 0.024492865428328514,
-      "rewards/margins": 0.1357099711894989,
-      "rewards/rejected": -0.11121710389852524,
+      "logits/chosen": -2.5273566246032715,
+      "logits/rejected": -2.468844175338745,
+      "logps/chosen": -273.30010986328125,
+      "logps/rejected": -270.6748962402344,
+      "loss": 0.0415,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.024781424552202225,
+      "rewards/margins": 0.13526295125484467,
+      "rewards/rejected": -0.11048151552677155,
       "step": 70
     },
     {
       "epoch": 0.18285714285714286,
-      "grad_norm": 2.9871578323865613,
+      "grad_norm": 2.993790876710757,
       "learning_rate": 4.897191188239667e-07,
-      "logits/chosen": -2.435612678527832,
-      "logits/rejected": -2.3714098930358887,
-      "logps/chosen": -269.2018127441406,
-      "logps/rejected": -276.71612548828125,
+      "logits/chosen": -2.4337241649627686,
+      "logits/rejected": -2.369915008544922,
+      "logps/chosen": -269.41571044921875,
+      "logps/rejected": -276.94500732421875,
       "loss": 0.0424,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.02172762155532837,
-      "rewards/margins": 0.19234482944011688,
-      "rewards/rejected": -0.21407246589660645,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.023866862058639526,
+      "rewards/margins": 0.1924942582845688,
+      "rewards/rejected": -0.2163611203432083,
       "step": 80
     },
     {
       "epoch": 0.2057142857142857,
-      "grad_norm": 7.2824339333866,
+      "grad_norm": 7.3103992020217285,
       "learning_rate": 4.832875107981763e-07,
-      "logits/chosen": -2.3872673511505127,
-      "logits/rejected": -2.300445318222046,
-      "logps/chosen": -275.88641357421875,
-      "logps/rejected": -275.3648681640625,
-      "loss": 0.0423,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.003988614305853844,
-      "rewards/margins": 0.20215804874897003,
-      "rewards/rejected": -0.19816945493221283,
+      "logits/chosen": -2.3889706134796143,
+      "logits/rejected": -2.3026340007781982,
+      "logps/chosen": -275.86724853515625,
+      "logps/rejected": -275.403076171875,
+      "loss": 0.0422,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.004180185496807098,
+      "rewards/margins": 0.20273157954216003,
+      "rewards/rejected": -0.19855138659477234,
       "step": 90
     },
     {
       "epoch": 0.22857142857142856,
-      "grad_norm": 3.336291542294826,
+      "grad_norm": 3.5295110150949056,
       "learning_rate": 4.753659419387223e-07,
-      "logits/chosen": -2.3031492233276367,
-      "logits/rejected": -2.170569658279419,
-      "logps/chosen": -289.19390869140625,
-      "logps/rejected": -258.89019775390625,
-      "loss": 0.0399,
+      "logits/chosen": -2.3000099658966064,
+      "logits/rejected": -2.165864944458008,
+      "logps/chosen": -289.35467529296875,
+      "logps/rejected": -259.2148742675781,
+      "loss": 0.04,
       "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.028075385838747025,
-      "rewards/margins": 0.2263123095035553,
-      "rewards/rejected": -0.25438767671585083,
+      "rewards/chosen": -0.029682844877243042,
+      "rewards/margins": 0.22795169055461884,
+      "rewards/rejected": -0.25763455033302307,
       "step": 100
     },
     {
       "epoch": 0.22857142857142856,
-      "eval_logits/chosen": -2.1601011753082275,
-      "eval_logits/rejected": -1.989274263381958,
-      "eval_logps/chosen": -283.0706481933594,
-      "eval_logps/rejected": -250.25137329101562,
-      "eval_loss": 0.03850938379764557,
+      "eval_logits/chosen": -2.1305601596832275,
+      "eval_logits/rejected": -1.9564552307128906,
+      "eval_logps/chosen": -283.3714904785156,
+      "eval_logps/rejected": -250.93673706054688,
+      "eval_loss": 0.03854820877313614,
       "eval_rewards/accuracies": 0.75,
-      "eval_rewards/chosen": -0.07466702908277512,
-      "eval_rewards/margins": 0.23708537220954895,
-      "eval_rewards/rejected": -0.3117523789405823,
-      "eval_runtime": 91.3269,
-      "eval_samples_per_second": 20.049,
-      "eval_steps_per_second": 0.318,
+      "eval_rewards/chosen": -0.07767525315284729,
+      "eval_rewards/margins": 0.2409307211637497,
+      "eval_rewards/rejected": -0.31860601902008057,
+      "eval_runtime": 91.3911,
+      "eval_samples_per_second": 20.035,
+      "eval_steps_per_second": 0.317,
       "step": 100
     },
     {
       "epoch": 0.25142857142857145,
-      "grad_norm": 3.177801599242846,
+      "grad_norm": 3.082269192831021,
       "learning_rate": 4.660050057270191e-07,
-      "logits/chosen": -2.190119504928589,
-      "logits/rejected": -2.104891061782837,
-      "logps/chosen": -322.0883483886719,
-      "logps/rejected": -316.6219482421875,
-      "loss": 0.0407,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.05784715339541435,
-      "rewards/margins": 0.12927614152431488,
-      "rewards/rejected": -0.18712328374385834,
+      "logits/chosen": -2.183840036392212,
+      "logits/rejected": -2.098686456680298,
+      "logps/chosen": -322.04498291015625,
+      "logps/rejected": -316.5348205566406,
+      "loss": 0.0406,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.05741389840841293,
+      "rewards/margins": 0.1288377195596695,
+      "rewards/rejected": -0.18625161051750183,
       "step": 110
     },
     {
       "epoch": 0.2742857142857143,
-      "grad_norm": 3.162850908555599,
+      "grad_norm": 3.1619647572029415,
       "learning_rate": 4.5526448859687144e-07,
-      "logits/chosen": -2.1028637886047363,
-      "logits/rejected": -1.9295556545257568,
-      "logps/chosen": -316.31634521484375,
-      "logps/rejected": -253.72195434570312,
-      "loss": 0.0376,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.05411834642291069,
-      "rewards/margins": 0.16595926880836487,
-      "rewards/rejected": -0.22007760405540466,
+      "logits/chosen": -2.122063159942627,
+      "logits/rejected": -1.943372130393982,
+      "logps/chosen": -315.7963562011719,
+      "logps/rejected": -253.3599395751953,
+      "loss": 0.0377,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.048919014632701874,
+      "rewards/margins": 0.1675383597612381,
+      "rewards/rejected": -0.21645736694335938,
       "step": 120
     },
     {
       "epoch": 0.29714285714285715,
-      "grad_norm": 3.4487456432507195,
+      "grad_norm": 3.421686175984915,
       "learning_rate": 4.432129880904388e-07,
-      "logits/chosen": -2.1982297897338867,
-      "logits/rejected": -2.1048200130462646,
-      "logps/chosen": -314.8988037109375,
-      "logps/rejected": -276.97674560546875,
-      "loss": 0.0373,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.09742867946624756,
-      "rewards/margins": 0.16763603687286377,
-      "rewards/rejected": -0.26506468653678894,
+      "logits/chosen": -2.2705729007720947,
+      "logits/rejected": -2.1812658309936523,
+      "logps/chosen": -313.17596435546875,
+      "logps/rejected": -274.95806884765625,
+      "loss": 0.0371,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08020049333572388,
+      "rewards/margins": 0.16467759013175964,
+      "rewards/rejected": -0.24487809836864471,
       "step": 130
     },
     {
       "epoch": 0.32,
-      "grad_norm": 3.9041569099317925,
+      "grad_norm": 3.9283597913931922,
       "learning_rate": 4.299274747394055e-07,
-      "logits/chosen": -2.3467020988464355,
-      "logits/rejected": -2.3200185298919678,
-      "logps/chosen": -315.0827941894531,
-      "logps/rejected": -290.0022277832031,
+      "logits/chosen": -2.398083448410034,
+      "logits/rejected": -2.3765132427215576,
+      "logps/chosen": -314.16827392578125,
+      "logps/rejected": -288.8768310546875,
       "loss": 0.0375,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.0036486200988292694,
-      "rewards/margins": 0.18455657362937927,
-      "rewards/rejected": -0.1809079647064209,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.012793675065040588,
+      "rewards/margins": 0.18244774639606476,
+      "rewards/rejected": -0.16965410113334656,
       "step": 140
     },
     {
       "epoch": 0.34285714285714286,
-      "grad_norm": 3.1721506985515817,
+      "grad_norm": 3.018006960325513,
       "learning_rate": 4.1549280046953653e-07,
-      "logits/chosen": -2.452073097229004,
-      "logits/rejected": -2.371002674102783,
-      "logps/chosen": -296.94329833984375,
-      "logps/rejected": -307.01837158203125,
-      "loss": 0.0367,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.09957350790500641,
-      "rewards/margins": 0.2076205015182495,
-      "rewards/rejected": -0.3071940243244171,
+      "logits/chosen": -2.4656407833099365,
+      "logits/rejected": -2.367140531539917,
+      "logps/chosen": -295.70556640625,
+      "logps/rejected": -305.4766845703125,
+      "loss": 0.0363,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.08719640970230103,
+      "rewards/margins": 0.204580619931221,
+      "rewards/rejected": -0.2917770445346832,
       "step": 150
     },
     {
       "epoch": 0.34285714285714286,
-      "eval_logits/chosen": -2.4217660427093506,
-      "eval_logits/rejected": -2.3057165145874023,
-      "eval_logps/chosen": -294.94464111328125,
-      "eval_logps/rejected": -263.3892822265625,
-      "eval_loss": 0.037056975066661835,
-      "eval_rewards/accuracies": 0.767241358757019,
-      "eval_rewards/chosen": -0.19340714812278748,
-      "eval_rewards/margins": 0.24972420930862427,
-      "eval_rewards/rejected": -0.44313138723373413,
-      "eval_runtime": 91.4385,
-      "eval_samples_per_second": 20.024,
+      "eval_logits/chosen": -2.3440675735473633,
+      "eval_logits/rejected": -2.195537805557251,
+      "eval_logps/chosen": -296.12109375,
+      "eval_logps/rejected": -265.0228271484375,
+      "eval_loss": 0.03708173334598541,
+      "eval_rewards/accuracies": 0.7543103694915771,
+      "eval_rewards/chosen": -0.20517148077487946,
+      "eval_rewards/margins": 0.25429537892341614,
+      "eval_rewards/rejected": -0.4594668447971344,
+      "eval_runtime": 91.4169,
+      "eval_samples_per_second": 20.029,
       "eval_steps_per_second": 0.317,
       "step": 150
     },
     {
       "epoch": 0.3657142857142857,
-      "grad_norm": 3.2106332008237772,
+      "grad_norm": 3.319581864694512,
       "learning_rate": 4.000011566683401e-07,
-      "logits/chosen": -2.493586540222168,
-      "logits/rejected": -2.4255142211914062,
-      "logps/chosen": -316.421630859375,
-      "logps/rejected": -294.204345703125,
-      "loss": 0.0372,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.17081338167190552,
-      "rewards/margins": 0.21480269730091095,
-      "rewards/rejected": -0.38561612367630005,
+      "logits/chosen": -2.376082181930542,
+      "logits/rejected": -2.2733752727508545,
+      "logps/chosen": -320.1603088378906,
+      "logps/rejected": -300.1729431152344,
+      "loss": 0.0377,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.20820029079914093,
+      "rewards/margins": 0.23710182309150696,
+      "rewards/rejected": -0.4453020989894867,
       "step": 160
     },
     {
       "epoch": 0.38857142857142857,
-      "grad_norm": 3.305163626494215,
+      "grad_norm": 3.33265699836665,
       "learning_rate": 3.8355148537705047e-07,
-      "logits/chosen": -2.473874568939209,
-      "logits/rejected": -2.4223687648773193,
-      "logps/chosen": -309.713623046875,
-      "logps/rejected": -291.6881103515625,
-      "loss": 0.0373,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.056086935102939606,
-      "rewards/margins": 0.2034837305545807,
-      "rewards/rejected": -0.2595706582069397,
+      "logits/chosen": -2.5347208976745605,
+      "logits/rejected": -2.48330020904541,
+      "logps/chosen": -312.109619140625,
+      "logps/rejected": -294.4687805175781,
+      "loss": 0.0377,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.08004748821258545,
+      "rewards/margins": 0.2073298990726471,
+      "rewards/rejected": -0.28737738728523254,
       "step": 170
     },
     {
       "epoch": 0.4114285714285714,
-      "grad_norm": 3.211306956770945,
+      "grad_norm": 2.9907171954669534,
       "learning_rate": 3.662488473675315e-07,
-      "logits/chosen": -2.486910104751587,
-      "logits/rejected": -2.3967292308807373,
-      "logps/chosen": -334.092041015625,
-      "logps/rejected": -292.355712890625,
+      "logits/chosen": -2.622641086578369,
+      "logits/rejected": -2.556631565093994,
+      "logps/chosen": -332.7588806152344,
+      "logps/rejected": -290.630859375,
       "loss": 0.0373,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.03912293165922165,
-      "rewards/margins": 0.2849285304546356,
-      "rewards/rejected": -0.3240514397621155,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.02579095959663391,
+      "rewards/margins": 0.2810118794441223,
+      "rewards/rejected": -0.3068028390407562,
       "step": 180
     },
     {
       "epoch": 0.4342857142857143,
-      "grad_norm": 3.4199998150652564,
+      "grad_norm": 3.4010102454112205,
       "learning_rate": 3.48203751140067e-07,
-      "logits/chosen": -2.507291078567505,
-      "logits/rejected": -2.4220941066741943,
-      "logps/chosen": -285.0407409667969,
-      "logps/rejected": -261.77392578125,
-      "loss": 0.0365,
+      "logits/chosen": -2.642368793487549,
+      "logits/rejected": -2.5668444633483887,
+      "logps/chosen": -286.96087646484375,
+      "logps/rejected": -263.45306396484375,
+      "loss": 0.0366,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.08783163130283356,
-      "rewards/margins": 0.1849258691072464,
-      "rewards/rejected": -0.27275753021240234,
+      "rewards/chosen": -0.10703293979167938,
+      "rewards/margins": 0.1825161725282669,
+      "rewards/rejected": -0.2895490825176239,
       "step": 190
     },
     {
       "epoch": 0.45714285714285713,
-      "grad_norm": 2.966792213983748,
+      "grad_norm": 2.9611644003699062,
       "learning_rate": 3.2953144712759537e-07,
-      "logits/chosen": -2.508540630340576,
-      "logits/rejected": -2.376711130142212,
-      "logps/chosen": -271.0993347167969,
-      "logps/rejected": -254.6019744873047,
-      "loss": 0.0375,
+      "logits/chosen": -2.651844024658203,
+      "logits/rejected": -2.5362985134124756,
+      "logps/chosen": -272.91973876953125,
+      "logps/rejected": -257.00128173828125,
+      "loss": 0.0373,
       "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.02756611444056034,
-      "rewards/margins": 0.2296477109193802,
-      "rewards/rejected": -0.257213830947876,
+      "rewards/chosen": -0.04576968401670456,
+      "rewards/margins": 0.23543760180473328,
+      "rewards/rejected": -0.28120729327201843,
       "step": 200
     },
     {
       "epoch": 0.45714285714285713,
-      "eval_logits/chosen": -2.290728807449341,
-      "eval_logits/rejected": -2.1435999870300293,
-      "eval_logps/chosen": -281.01300048828125,
-      "eval_logps/rejected": -252.2786407470703,
-      "eval_loss": 0.03527820482850075,
-      "eval_rewards/accuracies": 0.767241358757019,
-      "eval_rewards/chosen": -0.054090581834316254,
-      "eval_rewards/margins": 0.2779344916343689,
-      "eval_rewards/rejected": -0.33202511072158813,
-      "eval_runtime": 91.8763,
-      "eval_samples_per_second": 19.929,
+      "eval_logits/chosen": -2.4903368949890137,
+      "eval_logits/rejected": -2.384784460067749,
+      "eval_logps/chosen": -280.1239318847656,
+      "eval_logps/rejected": -251.76303100585938,
+      "eval_loss": 0.035302262753248215,
+      "eval_rewards/accuracies": 0.7715517282485962,
+      "eval_rewards/chosen": -0.04520006850361824,
+      "eval_rewards/margins": 0.28166908025741577,
+      "eval_rewards/rejected": -0.3268691301345825,
+      "eval_runtime": 91.7563,
+      "eval_samples_per_second": 19.955,
       "eval_steps_per_second": 0.316,
       "step": 200
     },
     {
       "epoch": 0.48,
-      "grad_norm": 3.030911456207702,
+      "grad_norm": 2.9976100938955375,
       "learning_rate": 3.103511916141658e-07,
-      "logits/chosen": -2.2013955116271973,
-      "logits/rejected": -2.0953996181488037,
-      "logps/chosen": -276.37432861328125,
-      "logps/rejected": -280.0777893066406,
+      "logits/chosen": -2.4521584510803223,
+      "logits/rejected": -2.38016939163208,
+      "logps/chosen": -272.8417053222656,
+      "logps/rejected": -276.30755615234375,
       "loss": 0.0362,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.12920817732810974,
-      "rewards/margins": 0.2187701165676117,
-      "rewards/rejected": -0.3479783535003662,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.09388158470392227,
+      "rewards/margins": 0.21639461815357208,
+      "rewards/rejected": -0.31027621030807495,
       "step": 210
     },
     {
       "epoch": 0.5028571428571429,
-      "grad_norm": 3.521691949347102,
+      "grad_norm": 3.3697037289842515,
       "learning_rate": 2.9078548506882117e-07,
-      "logits/chosen": -2.1590688228607178,
-      "logits/rejected": -2.0675222873687744,
-      "logps/chosen": -301.6694030761719,
-      "logps/rejected": -281.8822326660156,
-      "loss": 0.0387,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.16603410243988037,
-      "rewards/margins": 0.1909538209438324,
-      "rewards/rejected": -0.35698795318603516,
+      "logits/chosen": -2.486403226852417,
+      "logits/rejected": -2.43949556350708,
+      "logps/chosen": -299.24713134765625,
+      "logps/rejected": -279.8057556152344,
+      "loss": 0.0389,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.14181171357631683,
+      "rewards/margins": 0.19441165030002594,
+      "rewards/rejected": -0.3362233638763428,
       "step": 220
     },
     {
       "epoch": 0.5257142857142857,
-      "grad_norm": 3.271109178578225,
+      "grad_norm": 3.1432532704884797,
       "learning_rate": 2.709592897595191e-07,
-      "logits/chosen": -2.195312023162842,
-      "logits/rejected": -2.095759391784668,
-      "logps/chosen": -293.96038818359375,
-      "logps/rejected": -271.3498229980469,
-      "loss": 0.0366,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.1290125697851181,
-      "rewards/margins": 0.19161902368068695,
-      "rewards/rejected": -0.32063156366348267,
+      "logits/chosen": -2.507004499435425,
+      "logits/rejected": -2.437407970428467,
+      "logps/chosen": -295.362060546875,
+      "logps/rejected": -273.03448486328125,
+      "loss": 0.0364,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.1430293470621109,
+      "rewards/margins": 0.19444888830184937,
+      "rewards/rejected": -0.3374782204627991,
       "step": 230
     },
     {
       "epoch": 0.5485714285714286,
-      "grad_norm": 3.788405528579881,
+      "grad_norm": 3.601368929942566,
       "learning_rate": 2.509992316440332e-07,
-      "logits/chosen": -2.3450160026550293,
-      "logits/rejected": -2.233682870864868,
-      "logps/chosen": -311.3532409667969,
-      "logps/rejected": -312.9554138183594,
+      "logits/chosen": -2.5563902854919434,
+      "logits/rejected": -2.469142436981201,
+      "logps/chosen": -310.55108642578125,
+      "logps/rejected": -312.4164733886719,
       "loss": 0.0359,
       "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.12643404304981232,
-      "rewards/margins": 0.2889019250869751,
-      "rewards/rejected": -0.4153359532356262,
+      "rewards/chosen": -0.1184125691652298,
+      "rewards/margins": 0.2915334701538086,
+      "rewards/rejected": -0.4099460542201996,
       "step": 240
     },
     {
       "epoch": 0.5714285714285714,
-      "grad_norm": 2.975794895949038,
+      "grad_norm": 2.959775188259923,
       "learning_rate": 2.3103279163519918e-07,
-      "logits/chosen": -2.361924648284912,
-      "logits/rejected": -2.305227041244507,
-      "logps/chosen": -292.22430419921875,
-      "logps/rejected": -304.44610595703125,
-      "loss": 0.0371,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.11598072946071625,
-      "rewards/margins": 0.22647562623023987,
-      "rewards/rejected": -0.3424563705921173,
+      "logits/chosen": -2.5154330730438232,
+      "logits/rejected": -2.477905750274658,
+      "logps/chosen": -291.4244689941406,
+      "logps/rejected": -302.8068542480469,
+      "loss": 0.0374,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1079820767045021,
+      "rewards/margins": 0.21808166801929474,
+      "rewards/rejected": -0.32606372237205505,
       "step": 250
     },
     {
       "epoch": 0.5714285714285714,
-      "eval_logits/chosen": -2.378539800643921,
-      "eval_logits/rejected": -2.2614917755126953,
-      "eval_logps/chosen": -283.72186279296875,
-      "eval_logps/rejected": -254.03253173828125,
-      "eval_loss": 0.0344076044857502,
-      "eval_rewards/accuracies": 0.7629310488700867,
-      "eval_rewards/chosen": -0.08117903023958206,
-      "eval_rewards/margins": 0.2683848738670349,
-      "eval_rewards/rejected": -0.3495638966560364,
-      "eval_runtime": 90.3763,
-      "eval_samples_per_second": 20.26,
-      "eval_steps_per_second": 0.321,
+      "eval_logits/chosen": -2.5244851112365723,
+      "eval_logits/rejected": -2.4306986331939697,
+      "eval_logps/chosen": -283.6198425292969,
+      "eval_logps/rejected": -253.70823669433594,
+      "eval_loss": 0.034400373697280884,
+      "eval_rewards/accuracies": 0.75,
+      "eval_rewards/chosen": -0.08015903830528259,
+      "eval_rewards/margins": 0.26616212725639343,
+      "eval_rewards/rejected": -0.346321165561676,
+      "eval_runtime": 91.0071,
+      "eval_samples_per_second": 20.119,
+      "eval_steps_per_second": 0.319,
       "step": 250
     },
     {
       "epoch": 0.5942857142857143,
-      "grad_norm": 3.4496793515012127,
+      "grad_norm": 3.439430942929606,
       "learning_rate": 2.1118749140573358e-07,
-      "logits/chosen": -2.378929615020752,
-      "logits/rejected": -2.367784023284912,
-      "logps/chosen": -292.9862365722656,
-      "logps/rejected": -303.8517150878906,
-      "loss": 0.0378,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.1319950819015503,
-      "rewards/margins": 0.17309217154979706,
-      "rewards/rejected": -0.30508726835250854,
+      "logits/chosen": -2.5266551971435547,
+      "logits/rejected": -2.515558958053589,
+      "logps/chosen": -293.00262451171875,
+      "logps/rejected": -303.7609558105469,
+      "loss": 0.038,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.13215936720371246,
+      "rewards/margins": 0.17202024161815643,
+      "rewards/rejected": -0.3041796088218689,
       "step": 260
     },
     {
       "epoch": 0.6171428571428571,
-      "grad_norm": 3.309695226000979,
+      "grad_norm": 3.3473957187400707,
       "learning_rate": 1.9159007893272703e-07,
-      "logits/chosen": -2.4140243530273438,
-      "logits/rejected": -2.2664575576782227,
-      "logps/chosen": -284.2572326660156,
-      "logps/rejected": -267.2565612792969,
-      "loss": 0.0344,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.11731357872486115,
-      "rewards/margins": 0.2501775622367859,
-      "rewards/rejected": -0.3674911856651306,
+      "logits/chosen": -2.563744068145752,
+      "logits/rejected": -2.4393467903137207,
+      "logps/chosen": -282.9759216308594,
+      "logps/rejected": -266.39312744140625,
+      "loss": 0.0343,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.10450073331594467,
+      "rewards/margins": 0.2543557584285736,
+      "rewards/rejected": -0.3588564991950989,
       "step": 270
     },
     {
       "epoch": 0.64,
-      "grad_norm": 3.6035910497695895,
+      "grad_norm": 3.487492966730279,
       "learning_rate": 1.7236571898357766e-07,
-      "logits/chosen": -2.3951687812805176,
-      "logits/rejected": -2.2973971366882324,
-      "logps/chosen": -286.1737365722656,
-      "logps/rejected": -301.73150634765625,
-      "loss": 0.0359,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.13135617971420288,
-      "rewards/margins": 0.25489118695259094,
-      "rewards/rejected": -0.38624733686447144,
+      "logits/chosen": -2.576411724090576,
+      "logits/rejected": -2.5091185569763184,
+      "logps/chosen": -281.6929931640625,
+      "logps/rejected": -297.8419189453125,
+      "loss": 0.036,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.08654884248971939,
+      "rewards/margins": 0.2608022391796112,
+      "rewards/rejected": -0.34735107421875,
       "step": 280
     },
     {
       "epoch": 0.6628571428571428,
-      "grad_norm": 4.1650025414210665,
+      "grad_norm": 3.951421197104747,
       "learning_rate": 1.5363719371356882e-07,
-      "logits/chosen": -2.3873696327209473,
-      "logits/rejected": -2.2862212657928467,
-      "logps/chosen": -316.511962890625,
-      "logps/rejected": -304.41741943359375,
+      "logits/chosen": -2.622135877609253,
+      "logits/rejected": -2.544238328933716,
+      "logps/chosen": -311.6895751953125,
+      "logps/rejected": -299.3029479980469,
       "loss": 0.0349,
       "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.15606102347373962,
-      "rewards/margins": 0.2326282560825348,
-      "rewards/rejected": -0.3886892795562744,
+      "rewards/chosen": -0.10783736407756805,
+      "rewards/margins": 0.2297072857618332,
+      "rewards/rejected": -0.33754467964172363,
       "step": 290
     },
     {
       "epoch": 0.6857142857142857,
-      "grad_norm": 2.987221364071687,
+      "grad_norm": 3.035652974254887,
       "learning_rate": 1.3552411848071565e-07,
-      "logits/chosen": -2.387683391571045,
-      "logits/rejected": -2.2113590240478516,
-      "logps/chosen": -317.22491455078125,
-      "logps/rejected": -293.2046813964844,
-      "loss": 0.0345,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.1531573385000229,
-      "rewards/margins": 0.25246819853782654,
-      "rewards/rejected": -0.40562552213668823,
+      "logits/chosen": -2.646151304244995,
+      "logits/rejected": -2.5031981468200684,
+      "logps/chosen": -313.28375244140625,
+      "logps/rejected": -289.15740966796875,
+      "loss": 0.0346,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.11374591290950775,
+      "rewards/margins": 0.25140708684921265,
+      "rewards/rejected": -0.3651530146598816,
       "step": 300
     },
     {
       "epoch": 0.6857142857142857,
-      "eval_logits/chosen": -2.3475093841552734,
-      "eval_logits/rejected": -2.2130320072174072,
-      "eval_logps/chosen": -282.4234313964844,
-      "eval_logps/rejected": -254.02647399902344,
-      "eval_loss": 0.03413059934973717,
+      "eval_logits/chosen": -2.5812017917633057,
+      "eval_logits/rejected": -2.479748487472534,
+      "eval_logps/chosen": -279.3269958496094,
+      "eval_logps/rejected": -251.0284881591797,
+      "eval_loss": 0.03423487767577171,
       "eval_rewards/accuracies": 0.7456896305084229,
-      "eval_rewards/chosen": -0.06819500029087067,
-      "eval_rewards/margins": 0.28130805492401123,
-      "eval_rewards/rejected": -0.3495030105113983,
-      "eval_runtime": 90.5949,
-      "eval_samples_per_second": 20.211,
-      "eval_steps_per_second": 0.32,
+      "eval_rewards/chosen": -0.03723056614398956,
+      "eval_rewards/margins": 0.2822931110858917,
+      "eval_rewards/rejected": -0.3195236325263977,
+      "eval_runtime": 90.8627,
+      "eval_samples_per_second": 20.151,
+      "eval_steps_per_second": 0.319,
       "step": 300
     },
     {
       "epoch": 0.7085714285714285,
-      "grad_norm": 2.83762479667315,
+      "grad_norm": 2.850744846118877,
       "learning_rate": 1.1814217788631473e-07,
-      "logits/chosen": -2.3416621685028076,
-      "logits/rejected": -2.2815122604370117,
-      "logps/chosen": -267.0755310058594,
-      "logps/rejected": -267.9870300292969,
-      "loss": 0.0367,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.10771362483501434,
-      "rewards/margins": 0.20020349323749542,
-      "rewards/rejected": -0.30791711807250977,
+      "logits/chosen": -2.6023502349853516,
+      "logits/rejected": -2.559641122817993,
+      "logps/chosen": -263.7620849609375,
+      "logps/rejected": -264.8592224121094,
+      "loss": 0.0369,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.07457949221134186,
+      "rewards/margins": 0.20205926895141602,
+      "rewards/rejected": -0.27663877606391907,
       "step": 310
     },
     {
       "epoch": 0.7314285714285714,
-      "grad_norm": 3.3425045367761492,
+      "grad_norm": 3.4445308568499575,
       "learning_rate": 1.0160238692045331e-07,
-      "logits/chosen": -2.3024659156799316,
-      "logits/rejected": -2.2111096382141113,
-      "logps/chosen": -257.63531494140625,
-      "logps/rejected": -272.8826904296875,
+      "logits/chosen": -2.5764858722686768,
+      "logits/rejected": -2.4977431297302246,
+      "logps/chosen": -255.32528686523438,
+      "logps/rejected": -270.0458984375,
       "loss": 0.0364,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.13352924585342407,
-      "rewards/margins": 0.18541577458381653,
-      "rewards/rejected": -0.3189450204372406,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.11042918264865875,
+      "rewards/margins": 0.18014779686927795,
+      "rewards/rejected": -0.2905769944190979,
       "step": 320
     },
     {
       "epoch": 0.7542857142857143,
-      "grad_norm": 2.769882287509293,
+      "grad_norm": 2.7891304584846908,
       "learning_rate": 8.601038193139438e-08,
-      "logits/chosen": -2.4251229763031006,
-      "logits/rejected": -2.29219388961792,
-      "logps/chosen": -307.23333740234375,
-      "logps/rejected": -280.0276794433594,
+      "logits/chosen": -2.669182300567627,
+      "logits/rejected": -2.559919595718384,
+      "logps/chosen": -304.69049072265625,
+      "logps/rejected": -277.16827392578125,
       "loss": 0.0348,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.08015141636133194,
-      "rewards/margins": 0.24224629998207092,
-      "rewards/rejected": -0.3223976790904999,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.05472307652235031,
+      "rewards/margins": 0.23908019065856934,
+      "rewards/rejected": -0.29380327463150024,
       "step": 330
     },
     {
       "epoch": 0.7771428571428571,
-      "grad_norm": 3.0959592695193763,
+      "grad_norm": 3.0804434627972372,
       "learning_rate": 7.146574594727572e-08,
-      "logits/chosen": -2.3518755435943604,
-      "logits/rejected": -2.3008193969726562,
-      "logps/chosen": -277.1438293457031,
-      "logps/rejected": -279.57427978515625,
-      "loss": 0.0353,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.11192858219146729,
-      "rewards/margins": 0.2635765075683594,
-      "rewards/rejected": -0.3755050301551819,
+      "logits/chosen": -2.5831470489501953,
+      "logits/rejected": -2.54848051071167,
+      "logps/chosen": -275.44757080078125,
+      "logps/rejected": -277.88311767578125,
+      "loss": 0.0352,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.09496594220399857,
+      "rewards/margins": 0.2636275291442871,
+      "rewards/rejected": -0.3585934638977051,
       "step": 340
     },
     {
       "epoch": 0.8,
-      "grad_norm": 3.411772564497867,
+      "grad_norm": 3.3935330484921606,
       "learning_rate": 5.8061372659157306e-08,
-      "logits/chosen": -2.4039082527160645,
-      "logits/rejected": -2.2675061225891113,
-      "logps/chosen": -300.5389099121094,
-      "logps/rejected": -285.20794677734375,
-      "loss": 0.0373,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.10831709951162338,
-      "rewards/margins": 0.20078572630882263,
-      "rewards/rejected": -0.3091028332710266,
+      "logits/chosen": -2.6102333068847656,
+      "logits/rejected": -2.5170912742614746,
+      "logps/chosen": -299.60601806640625,
+      "logps/rejected": -284.296630859375,
+      "loss": 0.0375,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.09898806363344193,
+      "rewards/margins": 0.20100173354148865,
+      "rewards/rejected": -0.29998978972435,
       "step": 350
     },
     {
       "epoch": 0.8,
-      "eval_logits/chosen": -2.3320775032043457,
-      "eval_logits/rejected": -2.178847312927246,
-      "eval_logps/chosen": -284.68194580078125,
-      "eval_logps/rejected": -257.5619201660156,
-      "eval_loss": 0.03409451246261597,
+      "eval_logits/chosen": -2.5560529232025146,
+      "eval_logits/rejected": -2.447376012802124,
+      "eval_logps/chosen": -283.43243408203125,
+      "eval_logps/rejected": -256.5389099121094,
+      "eval_loss": 0.03421488776803017,
       "eval_rewards/accuracies": 0.7413793206214905,
-      "eval_rewards/chosen": -0.09078007191419601,
-      "eval_rewards/margins": 0.2940778136253357,
-      "eval_rewards/rejected": -0.38485783338546753,
-      "eval_runtime": 90.2939,
-      "eval_samples_per_second": 20.278,
-      "eval_steps_per_second": 0.321,
+      "eval_rewards/chosen": -0.07828506827354431,
+      "eval_rewards/margins": 0.29634276032447815,
+      "eval_rewards/rejected": -0.37462782859802246,
+      "eval_runtime": 90.9316,
+      "eval_samples_per_second": 20.136,
+      "eval_steps_per_second": 0.319,
       "step": 350
     },
     {
       "epoch": 0.8228571428571428,
-      "grad_norm": 2.905033636374422,
+      "grad_norm": 2.9199488201620456,
       "learning_rate": 4.5882873127531614e-08,
-      "logits/chosen": -2.4123928546905518,
-      "logits/rejected": -2.2830798625946045,
-      "logps/chosen": -295.31842041015625,
-      "logps/rejected": -286.4782409667969,
+      "logits/chosen": -2.6238226890563965,
+      "logits/rejected": -2.5243430137634277,
+      "logps/chosen": -294.04046630859375,
+      "logps/rejected": -285.36041259765625,
       "loss": 0.0342,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.09913380444049835,
-      "rewards/margins": 0.25478315353393555,
-      "rewards/rejected": -0.3539169430732727,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.08635450899600983,
+      "rewards/margins": 0.2563842833042145,
+      "rewards/rejected": -0.3427388072013855,
       "step": 360
     },
     {
       "epoch": 0.8457142857142858,
-      "grad_norm": 3.0252642177231617,
+      "grad_norm": 3.0306421043965925,
       "learning_rate": 3.500802900154412e-08,
-      "logits/chosen": -2.3916738033294678,
-      "logits/rejected": -2.244600296020508,
-      "logps/chosen": -278.41217041015625,
-      "logps/rejected": -273.110595703125,
+      "logits/chosen": -2.61911940574646,
+      "logits/rejected": -2.5165693759918213,
+      "logps/chosen": -277.0403137207031,
+      "logps/rejected": -271.5109558105469,
       "loss": 0.034,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.07350718975067139,
-      "rewards/margins": 0.273443341255188,
-      "rewards/rejected": -0.346950501203537,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.05978885293006897,
+      "rewards/margins": 0.2711651027202606,
+      "rewards/rejected": -0.330953985452652,
       "step": 370
     },
     {
       "epoch": 0.8685714285714285,
-      "grad_norm": 3.1477774029837864,
+      "grad_norm": 3.142763842399202,
       "learning_rate": 2.550629574310309e-08,
-      "logits/chosen": -2.4462573528289795,
-      "logits/rejected": -2.309199094772339,
-      "logps/chosen": -329.63348388671875,
-      "logps/rejected": -289.4853515625,
+      "logits/chosen": -2.68263578414917,
+      "logits/rejected": -2.584908962249756,
+      "logps/chosen": -327.3045959472656,
+      "logps/rejected": -287.1323547363281,
       "loss": 0.0352,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.11577162891626358,
-      "rewards/margins": 0.20508745312690735,
-      "rewards/rejected": -0.32085904479026794,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.09248252213001251,
+      "rewards/margins": 0.20484676957130432,
+      "rewards/rejected": -0.29732924699783325,
       "step": 380
     },
     {
       "epoch": 0.8914285714285715,
-      "grad_norm": 3.5873354511870312,
+      "grad_norm": 3.6385746368012355,
       "learning_rate": 1.7438359028687983e-08,
-      "logits/chosen": -2.403637170791626,
-      "logits/rejected": -2.321964740753174,
-      "logps/chosen": -318.7025146484375,
-      "logps/rejected": -324.0418395996094,
-      "loss": 0.0358,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.07329531759023666,
-      "rewards/margins": 0.21939659118652344,
-      "rewards/rejected": -0.2926918864250183,
+      "logits/chosen": -2.6608521938323975,
+      "logits/rejected": -2.607428789138794,
+      "logps/chosen": -316.05438232421875,
+      "logps/rejected": -321.3371887207031,
+      "loss": 0.0359,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.04681408405303955,
+      "rewards/margins": 0.2188311070203781,
+      "rewards/rejected": -0.26564517617225647,
       "step": 390
     },
     {
       "epoch": 0.9142857142857143,
-      "grad_norm": 5.030788992854401,
+      "grad_norm": 5.02358291913839,
       "learning_rate": 1.0855747162029361e-08,
-      "logits/chosen": -2.3374335765838623,
-      "logits/rejected": -2.3087658882141113,
-      "logps/chosen": -292.0116271972656,
-      "logps/rejected": -298.6130065917969,
+      "logits/chosen": -2.590088367462158,
+      "logits/rejected": -2.570838451385498,
+      "logps/chosen": -289.6832580566406,
+      "logps/rejected": -296.08538818359375,
       "loss": 0.0367,
       "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.13278374075889587,
-      "rewards/margins": 0.18760989606380463,
-      "rewards/rejected": -0.3203936815261841,
+      "rewards/chosen": -0.10950014740228653,
+      "rewards/margins": 0.18561770021915436,
+      "rewards/rejected": -0.2951178550720215,
       "step": 400
     },
     {
       "epoch": 0.9142857142857143,
-      "eval_logits/chosen": -2.3139522075653076,
-      "eval_logits/rejected": -2.1562979221343994,
-      "eval_logps/chosen": -285.25201416015625,
-      "eval_logps/rejected": -258.45458984375,
-      "eval_loss": 0.034024760127067566,
+      "eval_logits/chosen": -2.561089277267456,
+      "eval_logits/rejected": -2.4522511959075928,
+      "eval_logps/chosen": -282.986083984375,
+      "eval_logps/rejected": -256.24615478515625,
+      "eval_loss": 0.03414054214954376,
       "eval_rewards/accuracies": 0.7413793206214905,
-      "eval_rewards/chosen": -0.09648066759109497,
-      "eval_rewards/margins": 0.2973036468029022,
-      "eval_rewards/rejected": -0.3937843143939972,
-      "eval_runtime": 90.5725,
-      "eval_samples_per_second": 20.216,
-      "eval_steps_per_second": 0.32,
+      "eval_rewards/chosen": -0.07382136583328247,
+      "eval_rewards/margins": 0.2978789508342743,
+      "eval_rewards/rejected": -0.3717002868652344,
+      "eval_runtime": 90.9735,
+      "eval_samples_per_second": 20.127,
+      "eval_steps_per_second": 0.319,
       "step": 400
     },
     {
       "epoch": 0.9371428571428572,
-      "grad_norm": 3.2590392659418335,
+      "grad_norm": 3.2423722180060786,
       "learning_rate": 5.8005019731033615e-09,
-      "logits/chosen": -2.3900809288024902,
-      "logits/rejected": -2.28753924369812,
-      "logps/chosen": -301.38275146484375,
-      "logps/rejected": -294.66497802734375,
+      "logits/chosen": -2.6259548664093018,
+      "logits/rejected": -2.5545239448547363,
+      "logps/chosen": -299.16107177734375,
+      "logps/rejected": -292.75714111328125,
       "loss": 0.0352,
       "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.13202829658985138,
-      "rewards/margins": 0.21732142567634583,
-      "rewards/rejected": -0.349349707365036,
+      "rewards/chosen": -0.10981108993291855,
+      "rewards/margins": 0.2204599827528,
+      "rewards/rejected": -0.3302710950374603,
       "step": 410
     },
     {
       "epoch": 0.96,
-      "grad_norm": 3.0742600085581127,
+      "grad_norm": 3.0950678320925222,
       "learning_rate": 2.3049103053431886e-09,
-      "logits/chosen": -2.326852798461914,
-      "logits/rejected": -2.221895694732666,
-      "logps/chosen": -288.41644287109375,
-      "logps/rejected": -271.6980895996094,
+      "logits/chosen": -2.5695414543151855,
+      "logits/rejected": -2.504147529602051,
+      "logps/chosen": -286.32177734375,
+      "logps/rejected": -269.1705627441406,
       "loss": 0.0345,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.056683529168367386,
-      "rewards/margins": 0.32042860984802246,
-      "rewards/rejected": -0.37711212038993835,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.03573674336075783,
+      "rewards/margins": 0.31610041856765747,
+      "rewards/rejected": -0.351837158203125,
       "step": 420
     },
     {
       "epoch": 0.9828571428571429,
-      "grad_norm": 3.069144533981935,
+      "grad_norm": 2.978033708452065,
       "learning_rate": 3.9129780600541397e-10,
-      "logits/chosen": -2.312039613723755,
-      "logits/rejected": -2.213510751724243,
-      "logps/chosen": -293.47271728515625,
-      "logps/rejected": -299.9048156738281,
+      "logits/chosen": -2.5734434127807617,
+      "logits/rejected": -2.4971184730529785,
+      "logps/chosen": -290.81646728515625,
+      "logps/rejected": -297.24932861328125,
       "loss": 0.0344,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.11240877956151962,
-      "rewards/margins": 0.24994345009326935,
-      "rewards/rejected": -0.3623522222042084,
+      "rewards/chosen": -0.08584621548652649,
+      "rewards/margins": 0.24995127320289612,
+      "rewards/rejected": -0.33579742908477783,
       "step": 430
     },
     {
       "epoch": 0.9988571428571429,
       "step": 437,
       "total_flos": 0.0,
-      "train_loss": 0.038760803656938146,
-      "train_runtime": 11420.4923,
-      "train_samples_per_second": 4.903,
+      "train_loss": 0.03879852061004333,
+      "train_runtime": 11351.3146,
+      "train_samples_per_second": 4.933,
       "train_steps_per_second": 0.038
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d2f1aff6e6758906dd6c2076fc4cec217e811316cd11064de2c28addf6c5700
+oid sha256:d3c6dc0bb8685ef4f4a6dddea7302f644bfb846df5915cda57dbfd6c378e690f
 size 7672