Training in progress, epoch 1, checkpoint

last-checkpoint/global_step313/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e525aaf68ee1ef799d102f8ac29c0f92de91d69236641af2f3d2ee7dee81745a
+size 13476836524

last-checkpoint/global_step313/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67a425e323e6de53a947fc63bccd7ad559d74fe78103898bd601affcb1a358d9
+size 13476836524

last-checkpoint/global_step313/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42439561d72bb6f266a7c556d9a2be61ac7e1057d388d45d57dc36a81f5a203c
+size 13476836524

last-checkpoint/global_step313/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dea8b0ab0216933e18b8abda975de86d7b7edd46cb63df5971773fbf72f1b4d5
+size 13476836524

last-checkpoint/global_step313/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d8314cf4f5600084d8a4b1876eeffb6fc6b6f02b835d1591892fd690287d0d3
+size 150693

last-checkpoint/global_step313/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20269c5f664b468c45f1de4f3c4e0c1c4b60b82d70c7fe71aac546c322e493c7
+size 150693

last-checkpoint/global_step313/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6aaf6725b72e7ff7a5246ff292b36215359ab5df553980b4af50abe70487e6dd
+size 150693

last-checkpoint/global_step313/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29c4b2089384f4a5b72e6485203303e84687da7a5128aa8af2207045f0b953f9
+size 150693

last-checkpoint/latest

@@ -1 +1 @@
-global_step189
+global_step313

last-checkpoint/model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e39b6a24339b6efce07b3fe0bd3de7f0d3326e4a4e46f1f6f867690e202b94ef
+oid sha256:5231b2f328f2643a2d9789a86b8e10370a2324a4150b1bd106badfe55d7b1866
 size 4938985352

last-checkpoint/model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c276c3b5a01ce2bab646fd194f8e61095229f92f8757ed79ce40a46582eaa07
+oid sha256:c529b2cb15dcc95f696d572500eb93c3a14c5d99f45f3a3034a1b0ec90fb38fb
 size 4947390880

last-checkpoint/model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1b5c13a284586496a8da5a37ca35b977da8d2d2f2ccbf96a7b0186fcaeab960
+oid sha256:8dafef7fc1c913b3bf18c99e485c87aa9e89b5bcbe34eb6a46f886f4b21552eb
 size 3590488816

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab7cf0d34d60dfda516cf9661904550e2e294e723edd07c25c738f05e8ba92d1
+oid sha256:f1ce770862b76ceecb3453a5d0b9ef65311eab0f6a60f52997726a89d718de08
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7b183324e8227a51a9556d86b2ad893a8c4c52205ed4a737356c6611dac7353
+oid sha256:2bcae6573442578b752be3e988d4ccb38056b45c31b5c02eb579ec6cebcfa62e
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac69e994090f4818cb1fa6f6cefa363178552c3c731c6507ff195bcb07fd5bef
+oid sha256:398ff45072fee2975e88b9e078f915103d75d94bc08753303cf855b915973623
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f68e61b63402f8afb1f69c960f7944965655dac11e3ccf29919c282f23931f86
+oid sha256:7f4b210bc832d4c648e3bee8c21dca26e5a8b365d6ec90c638062005a052e57b
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c988df4a306242fbc45c908424c576078313f1b5bb472bee13c3b6371b94095e
+oid sha256:6b9aad3a748e15719e20120f0a760f22686338e176ee3f1c011b6dab2f3ab19b
 size 1064

last-checkpoint/trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.0,
+  "epoch": 1.0,
   "eval_steps": 500,
-  "global_step": 189,
+  "global_step": 313,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.015873015873015872,
-      "grad_norm": 8.48122634477978,
-      "learning_rate": 2.6315789473684208e-08,
-      "logits/chosen": -1.015625,
-      "logits/rejected": -1.390625,
-      "logps/chosen": -45.5,
-      "logps/rejected": -80.5,
+      "epoch": 0.003194888178913738,
+      "grad_norm": 26.70895145343874,
+      "learning_rate": 5.3191489361702125e-09,
+      "logits/chosen": -1.4921875,
+      "logits/rejected": -1.4921875,
+      "logps/chosen": -247.0,
+      "logps/rejected": -252.0,
       "loss": 0.6914,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,326 +24,489 @@
       "step": 1
     },
     {
-      "epoch": 0.15873015873015872,
-      "grad_norm": 7.666414237190791,
-      "learning_rate": 2.631578947368421e-07,
-      "logits/chosen": -1.1953125,
-      "logits/rejected": -1.21875,
-      "logps/chosen": -55.0,
-      "logps/rejected": -57.0,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.0833333358168602,
-      "rewards/chosen": -0.00017452239990234375,
-      "rewards/margins": -0.00244140625,
-      "rewards/rejected": 0.00225830078125,
+      "epoch": 0.03194888178913738,
+      "grad_norm": 27.524419104861234,
+      "learning_rate": 5.3191489361702123e-08,
+      "logits/chosen": -1.5390625,
+      "logits/rejected": -1.59375,
+      "logps/chosen": -252.0,
+      "logps/rejected": -290.0,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.1805555522441864,
+      "rewards/chosen": 0.004180908203125,
+      "rewards/margins": 0.0069580078125,
+      "rewards/rejected": -0.0027618408203125,
       "step": 10
     },
     {
-      "epoch": 0.31746031746031744,
-      "grad_norm": 6.673773390918632,
-      "learning_rate": 4.970588235294118e-07,
-      "logits/chosen": -1.09375,
-      "logits/rejected": -1.21875,
-      "logps/chosen": -53.75,
-      "logps/rejected": -56.75,
-      "loss": 0.69,
-      "rewards/accuracies": 0.20000000298023224,
-      "rewards/chosen": -0.01470947265625,
-      "rewards/margins": 0.0024871826171875,
-      "rewards/rejected": -0.0172119140625,
+      "epoch": 0.06389776357827476,
+      "grad_norm": 25.84743681377418,
+      "learning_rate": 1.0638297872340425e-07,
+      "logits/chosen": -1.5546875,
+      "logits/rejected": -1.6015625,
+      "logps/chosen": -244.0,
+      "logps/rejected": -284.0,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.16249999403953552,
+      "rewards/chosen": -0.00750732421875,
+      "rewards/margins": -0.00750732421875,
+      "rewards/rejected": -6.109476089477539e-06,
       "step": 20
     },
     {
-      "epoch": 0.47619047619047616,
-      "grad_norm": 7.379593048881886,
-      "learning_rate": 4.676470588235294e-07,
-      "logits/chosen": -1.1796875,
-      "logits/rejected": -1.1015625,
-      "logps/chosen": -60.0,
-      "logps/rejected": -57.0,
-      "loss": 0.6804,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -0.06689453125,
-      "rewards/margins": 0.030029296875,
-      "rewards/rejected": -0.09716796875,
+      "epoch": 0.09584664536741214,
+      "grad_norm": 26.132304612422136,
+      "learning_rate": 1.5957446808510638e-07,
+      "logits/chosen": -1.5625,
+      "logits/rejected": -1.5625,
+      "logps/chosen": -245.0,
+      "logps/rejected": -282.0,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.038818359375,
+      "rewards/margins": 0.0244140625,
+      "rewards/rejected": 0.014404296875,
       "step": 30
     },
     {
-      "epoch": 0.6349206349206349,
-      "grad_norm": 9.322532970919411,
-      "learning_rate": 4.38235294117647e-07,
-      "logits/chosen": -1.3359375,
-      "logits/rejected": -1.265625,
-      "logps/chosen": -63.25,
-      "logps/rejected": -56.5,
-      "loss": 0.661,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.0185546875,
-      "rewards/margins": 0.06494140625,
-      "rewards/rejected": -0.08349609375,
+      "epoch": 0.12779552715654952,
+      "grad_norm": 24.407986537974015,
+      "learning_rate": 2.127659574468085e-07,
+      "logits/chosen": -1.515625,
+      "logits/rejected": -1.5859375,
+      "logps/chosen": -237.0,
+      "logps/rejected": -308.0,
+      "loss": 0.6435,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.1591796875,
+      "rewards/margins": 0.1240234375,
+      "rewards/rejected": 0.03515625,
       "step": 40
     },
     {
-      "epoch": 0.7936507936507936,
-      "grad_norm": 7.879751058232231,
-      "learning_rate": 4.0882352941176465e-07,
-      "logits/chosen": -1.171875,
-      "logits/rejected": -1.21875,
-      "logps/chosen": -63.5,
-      "logps/rejected": -55.5,
-      "loss": 0.6611,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": 0.0830078125,
-      "rewards/margins": 0.1220703125,
-      "rewards/rejected": -0.0390625,
+      "epoch": 0.1597444089456869,
+      "grad_norm": 21.9426563067355,
+      "learning_rate": 2.659574468085106e-07,
+      "logits/chosen": -1.5625,
+      "logits/rejected": -1.640625,
+      "logps/chosen": -239.0,
+      "logps/rejected": -286.0,
+      "loss": 0.6137,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.310546875,
+      "rewards/margins": 0.2275390625,
+      "rewards/rejected": 0.08251953125,
       "step": 50
     },
     {
-      "epoch": 0.9523809523809523,
-      "grad_norm": 7.002381076132743,
-      "learning_rate": 3.7941176470588235e-07,
-      "logits/chosen": -1.25,
-      "logits/rejected": -1.2265625,
-      "logps/chosen": -62.5,
-      "logps/rejected": -56.25,
-      "loss": 0.645,
-      "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": 0.1005859375,
-      "rewards/margins": 0.12451171875,
-      "rewards/rejected": -0.024169921875,
+      "epoch": 0.19169329073482427,
+      "grad_norm": 20.96670132168061,
+      "learning_rate": 3.1914893617021275e-07,
+      "logits/chosen": -1.5859375,
+      "logits/rejected": -1.6171875,
+      "logps/chosen": -242.0,
+      "logps/rejected": -292.0,
+      "loss": 0.5732,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.5,
+      "rewards/margins": 0.376953125,
+      "rewards/rejected": 0.12109375,
       "step": 60
     },
     {
-      "epoch": 1.0,
-      "eval_logits/chosen": -1.2578125,
-      "eval_logits/rejected": -1.171875,
-      "eval_logps/chosen": -65.0,
-      "eval_logps/rejected": -57.25,
-      "eval_loss": 0.6340625286102295,
-      "eval_rewards/accuracies": 0.3928571343421936,
-      "eval_rewards/chosen": 0.02734375,
-      "eval_rewards/margins": 0.1787109375,
-      "eval_rewards/rejected": -0.1513671875,
-      "eval_runtime": 12.2628,
-      "eval_samples_per_second": 16.309,
-      "eval_steps_per_second": 0.571,
-      "step": 63
-    },
-    {
-      "epoch": 1.1111111111111112,
-      "grad_norm": 7.403427316810219,
-      "learning_rate": 3.5e-07,
-      "logits/chosen": -1.203125,
-      "logits/rejected": -1.2109375,
-      "logps/chosen": -58.75,
-      "logps/rejected": -57.75,
-      "loss": 0.6193,
-      "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": 0.044677734375,
-      "rewards/margins": 0.1630859375,
-      "rewards/rejected": -0.119140625,
+      "epoch": 0.22364217252396165,
+      "grad_norm": 20.027837530897173,
+      "learning_rate": 3.7234042553191484e-07,
+      "logits/chosen": -1.515625,
+      "logits/rejected": -1.625,
+      "logps/chosen": -235.0,
+      "logps/rejected": -296.0,
+      "loss": 0.5562,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.6953125,
+      "rewards/margins": 0.5390625,
+      "rewards/rejected": 0.1552734375,
       "step": 70
     },
     {
-      "epoch": 1.2698412698412698,
-      "grad_norm": 7.698447091140595,
-      "learning_rate": 3.205882352941177e-07,
-      "logits/chosen": -1.1015625,
-      "logits/rejected": -1.1640625,
-      "logps/chosen": -56.5,
-      "logps/rejected": -57.75,
-      "loss": 0.5965,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.035400390625,
-      "rewards/margins": 0.232421875,
-      "rewards/rejected": -0.26953125,
+      "epoch": 0.25559105431309903,
+      "grad_norm": 20.104841517252364,
+      "learning_rate": 4.25531914893617e-07,
+      "logits/chosen": -1.5859375,
+      "logits/rejected": -1.6328125,
+      "logps/chosen": -245.0,
+      "logps/rejected": -274.0,
+      "loss": 0.5644,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.6875,
+      "rewards/margins": 0.40234375,
+      "rewards/rejected": 0.28515625,
       "step": 80
     },
     {
-      "epoch": 1.4285714285714286,
-      "grad_norm": 5.644436155568419,
-      "learning_rate": 2.911764705882353e-07,
-      "logits/chosen": -1.25,
-      "logits/rejected": -1.234375,
-      "logps/chosen": -57.25,
-      "logps/rejected": -58.75,
-      "loss": 0.5885,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": 0.0103759765625,
-      "rewards/margins": 0.29296875,
-      "rewards/rejected": -0.283203125,
+      "epoch": 0.28753993610223644,
+      "grad_norm": 18.79087036985414,
+      "learning_rate": 4.787234042553192e-07,
+      "logits/chosen": -1.515625,
+      "logits/rejected": -1.6015625,
+      "logps/chosen": -236.0,
+      "logps/rejected": -284.0,
+      "loss": 0.5118,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.87109375,
+      "rewards/margins": 0.5859375,
+      "rewards/rejected": 0.287109375,
       "step": 90
     },
     {
-      "epoch": 1.5873015873015874,
-      "grad_norm": 7.133267052353533,
-      "learning_rate": 2.6176470588235295e-07,
-      "logits/chosen": -1.15625,
-      "logits/rejected": -1.296875,
-      "logps/chosen": -60.5,
-      "logps/rejected": -61.5,
-      "loss": 0.5796,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.025146484375,
-      "rewards/margins": 0.3671875,
-      "rewards/rejected": -0.392578125,
+      "epoch": 0.3194888178913738,
+      "grad_norm": 22.066743357299675,
+      "learning_rate": 4.964497041420119e-07,
+      "logits/chosen": -1.578125,
+      "logits/rejected": -1.6328125,
+      "logps/chosen": -231.0,
+      "logps/rejected": -288.0,
+      "loss": 0.5409,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 1.0078125,
+      "rewards/margins": 0.890625,
+      "rewards/rejected": 0.12060546875,
       "step": 100
     },
     {
-      "epoch": 1.746031746031746,
-      "grad_norm": 3.8242948995128176,
-      "learning_rate": 2.323529411764706e-07,
-      "logits/chosen": -1.0234375,
-      "logits/rejected": -1.171875,
-      "logps/chosen": -61.75,
-      "logps/rejected": -62.5,
-      "loss": 0.5951,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.0732421875,
-      "rewards/margins": 0.318359375,
-      "rewards/rejected": -0.390625,
+      "epoch": 0.3514376996805112,
+      "grad_norm": 19.3322147725973,
+      "learning_rate": 4.905325443786982e-07,
+      "logits/chosen": -1.546875,
+      "logits/rejected": -1.6328125,
+      "logps/chosen": -248.0,
+      "logps/rejected": -280.0,
+      "loss": 0.5656,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.69921875,
+      "rewards/margins": 0.458984375,
+      "rewards/rejected": 0.2392578125,
       "step": 110
     },
     {
-      "epoch": 1.9047619047619047,
-      "grad_norm": 5.60510875244782,
-      "learning_rate": 2.0294117647058823e-07,
-      "logits/chosen": -1.1328125,
-      "logits/rejected": -1.2265625,
-      "logps/chosen": -61.5,
-      "logps/rejected": -67.0,
-      "loss": 0.5691,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.12451171875,
-      "rewards/margins": 0.35546875,
-      "rewards/rejected": -0.478515625,
+      "epoch": 0.38338658146964855,
+      "grad_norm": 18.958252526138903,
+      "learning_rate": 4.846153846153846e-07,
+      "logits/chosen": -1.625,
+      "logits/rejected": -1.703125,
+      "logps/chosen": -240.0,
+      "logps/rejected": -292.0,
+      "loss": 0.5077,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.8671875,
+      "rewards/margins": 0.75390625,
+      "rewards/rejected": 0.11376953125,
       "step": 120
     },
     {
-      "epoch": 2.0,
-      "eval_logits/chosen": -1.21875,
-      "eval_logits/rejected": -1.109375,
-      "eval_logps/chosen": -66.5,
-      "eval_logps/rejected": -60.0,
-      "eval_loss": 0.6115624904632568,
-      "eval_rewards/accuracies": 0.3928571343421936,
-      "eval_rewards/chosen": -0.11279296875,
-      "eval_rewards/margins": 0.30078125,
-      "eval_rewards/rejected": -0.4140625,
-      "eval_runtime": 14.9499,
-      "eval_samples_per_second": 13.378,
-      "eval_steps_per_second": 0.468,
-      "step": 126
-    },
-    {
-      "epoch": 2.0634920634920633,
-      "grad_norm": 4.8582765968746875,
-      "learning_rate": 1.7352941176470587e-07,
-      "logits/chosen": -1.265625,
-      "logits/rejected": -1.1875,
-      "logps/chosen": -59.5,
-      "logps/rejected": -65.0,
-      "loss": 0.5577,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.002899169921875,
-      "rewards/margins": 0.408203125,
-      "rewards/rejected": -0.41015625,
+      "epoch": 0.41533546325878595,
+      "grad_norm": 24.459723181791798,
+      "learning_rate": 4.78698224852071e-07,
+      "logits/chosen": -1.5625,
+      "logits/rejected": -1.625,
+      "logps/chosen": -235.0,
+      "logps/rejected": -294.0,
+      "loss": 0.4762,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.86328125,
+      "rewards/margins": 0.81640625,
+      "rewards/rejected": 0.04638671875,
       "step": 130
     },
     {
-      "epoch": 2.2222222222222223,
-      "grad_norm": 3.9951840250758166,
-      "learning_rate": 1.441176470588235e-07,
-      "logits/chosen": -1.171875,
-      "logits/rejected": -1.15625,
-      "logps/chosen": -57.75,
-      "logps/rejected": -58.0,
-      "loss": 0.5505,
-      "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": -0.052978515625,
-      "rewards/margins": 0.33203125,
-      "rewards/rejected": -0.38671875,
+      "epoch": 0.4472843450479233,
+      "grad_norm": 24.8748418291271,
+      "learning_rate": 4.727810650887574e-07,
+      "logits/chosen": -1.578125,
+      "logits/rejected": -1.6484375,
+      "logps/chosen": -242.0,
+      "logps/rejected": -282.0,
+      "loss": 0.5377,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.7890625,
+      "rewards/margins": 0.640625,
+      "rewards/rejected": 0.1484375,
       "step": 140
     },
     {
-      "epoch": 2.380952380952381,
-      "grad_norm": 6.94549887559355,
-      "learning_rate": 1.1470588235294116e-07,
-      "logits/chosen": -1.21875,
-      "logits/rejected": -1.109375,
-      "logps/chosen": -66.0,
-      "logps/rejected": -57.75,
-      "loss": 0.5602,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.048095703125,
-      "rewards/margins": 0.3984375,
-      "rewards/rejected": -0.4453125,
+      "epoch": 0.4792332268370607,
+      "grad_norm": 21.92108677224862,
+      "learning_rate": 4.668639053254438e-07,
+      "logits/chosen": -1.578125,
+      "logits/rejected": -1.671875,
+      "logps/chosen": -236.0,
+      "logps/rejected": -302.0,
+      "loss": 0.5036,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.82421875,
+      "rewards/margins": 0.79296875,
+      "rewards/rejected": 0.029541015625,
       "step": 150
     },
     {
-      "epoch": 2.5396825396825395,
-      "grad_norm": 4.543691810100308,
-      "learning_rate": 8.529411764705883e-08,
-      "logits/chosen": -1.1484375,
-      "logits/rejected": -1.3046875,
-      "logps/chosen": -61.25,
-      "logps/rejected": -65.0,
-      "loss": 0.5427,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.11962890625,
-      "rewards/margins": 0.53515625,
-      "rewards/rejected": -0.65234375,
+      "epoch": 0.5111821086261981,
+      "grad_norm": 23.160240290542404,
+      "learning_rate": 4.6094674556213014e-07,
+      "logits/chosen": -1.6328125,
+      "logits/rejected": -1.6640625,
+      "logps/chosen": -236.0,
+      "logps/rejected": -290.0,
+      "loss": 0.5215,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.84765625,
+      "rewards/margins": 0.984375,
+      "rewards/rejected": -0.1337890625,
       "step": 160
     },
     {
-      "epoch": 2.6984126984126986,
-      "grad_norm": 8.364914220576363,
-      "learning_rate": 5.5882352941176474e-08,
-      "logits/chosen": -1.171875,
-      "logits/rejected": -1.15625,
-      "logps/chosen": -57.0,
-      "logps/rejected": -66.0,
-      "loss": 0.5554,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.1552734375,
-      "rewards/margins": 0.412109375,
-      "rewards/rejected": -0.56640625,
+      "epoch": 0.5431309904153354,
+      "grad_norm": 15.682473129649416,
+      "learning_rate": 4.5502958579881655e-07,
+      "logits/chosen": -1.6171875,
+      "logits/rejected": -1.6875,
+      "logps/chosen": -230.0,
+      "logps/rejected": -292.0,
+      "loss": 0.4507,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.9296875,
+      "rewards/margins": 0.9609375,
+      "rewards/rejected": -0.02880859375,
       "step": 170
     },
     {
-      "epoch": 2.857142857142857,
-      "grad_norm": 5.985657271989901,
-      "learning_rate": 2.6470588235294116e-08,
-      "logits/chosen": -0.96875,
-      "logits/rejected": -1.125,
-      "logps/chosen": -61.25,
-      "logps/rejected": -62.0,
-      "loss": 0.5432,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.047607421875,
-      "rewards/margins": 0.48828125,
-      "rewards/rejected": -0.53515625,
+      "epoch": 0.5750798722044729,
+      "grad_norm": 18.270260505484856,
+      "learning_rate": 4.491124260355029e-07,
+      "logits/chosen": -1.5859375,
+      "logits/rejected": -1.609375,
+      "logps/chosen": -229.0,
+      "logps/rejected": -298.0,
+      "loss": 0.4635,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 1.0078125,
+      "rewards/margins": 1.0234375,
+      "rewards/rejected": -0.01531982421875,
       "step": 180
     },
     {
-      "epoch": 3.0,
-      "eval_logits/chosen": -1.1953125,
-      "eval_logits/rejected": -1.0703125,
-      "eval_logps/chosen": -67.5,
-      "eval_logps/rejected": -61.25,
-      "eval_loss": 0.6068750023841858,
-      "eval_rewards/accuracies": 0.375,
-      "eval_rewards/chosen": -0.2021484375,
-      "eval_rewards/margins": 0.34375,
-      "eval_rewards/rejected": -0.546875,
-      "eval_runtime": 14.9623,
-      "eval_samples_per_second": 13.367,
-      "eval_steps_per_second": 0.468,
-      "step": 189
+      "epoch": 0.6070287539936102,
+      "grad_norm": 20.98631167244082,
+      "learning_rate": 4.4319526627218936e-07,
+      "logits/chosen": -1.6171875,
+      "logits/rejected": -1.65625,
+      "logps/chosen": -230.0,
+      "logps/rejected": -294.0,
+      "loss": 0.4752,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.9296875,
+      "rewards/margins": 0.953125,
+      "rewards/rejected": -0.021484375,
+      "step": 190
+    },
+    {
+      "epoch": 0.6389776357827476,
+      "grad_norm": 22.61128808819513,
+      "learning_rate": 4.372781065088757e-07,
+      "logits/chosen": -1.59375,
+      "logits/rejected": -1.6953125,
+      "logps/chosen": -233.0,
+      "logps/rejected": -294.0,
+      "loss": 0.4676,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.90234375,
+      "rewards/margins": 1.0,
+      "rewards/rejected": -0.1005859375,
+      "step": 200
+    },
+    {
+      "epoch": 0.670926517571885,
+      "grad_norm": 20.472996932111148,
+      "learning_rate": 4.313609467455621e-07,
+      "logits/chosen": -1.5625,
+      "logits/rejected": -1.65625,
+      "logps/chosen": -236.0,
+      "logps/rejected": -292.0,
+      "loss": 0.4827,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.890625,
+      "rewards/margins": 0.98828125,
+      "rewards/rejected": -0.1015625,
+      "step": 210
+    },
+    {
+      "epoch": 0.7028753993610224,
+      "grad_norm": 25.186112119569938,
+      "learning_rate": 4.2544378698224847e-07,
+      "logits/chosen": -1.5390625,
+      "logits/rejected": -1.5859375,
+      "logps/chosen": -238.0,
+      "logps/rejected": -286.0,
+      "loss": 0.5209,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.76171875,
+      "rewards/margins": 0.92578125,
+      "rewards/rejected": -0.1640625,
+      "step": 220
+    },
+    {
+      "epoch": 0.7348242811501597,
+      "grad_norm": 15.532080715300815,
+      "learning_rate": 4.195266272189349e-07,
+      "logits/chosen": -1.578125,
+      "logits/rejected": -1.578125,
+      "logps/chosen": -236.0,
+      "logps/rejected": -290.0,
+      "loss": 0.4544,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.7578125,
+      "rewards/margins": 0.8515625,
+      "rewards/rejected": -0.095703125,
+      "step": 230
+    },
+    {
+      "epoch": 0.7667731629392971,
+      "grad_norm": 21.562659615610453,
+      "learning_rate": 4.1360946745562133e-07,
+      "logits/chosen": -1.625,
+      "logits/rejected": -1.640625,
+      "logps/chosen": -238.0,
+      "logps/rejected": -292.0,
+      "loss": 0.4312,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.82421875,
+      "rewards/margins": 0.96484375,
+      "rewards/rejected": -0.1416015625,
+      "step": 240
+    },
+    {
+      "epoch": 0.7987220447284346,
+      "grad_norm": 19.697962308480218,
+      "learning_rate": 4.076923076923077e-07,
+      "logits/chosen": -1.578125,
+      "logits/rejected": -1.671875,
+      "logps/chosen": -241.0,
+      "logps/rejected": -286.0,
+      "loss": 0.4683,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.7421875,
+      "rewards/margins": 0.97265625,
+      "rewards/rejected": -0.23046875,
+      "step": 250
+    },
+    {
+      "epoch": 0.8306709265175719,
+      "grad_norm": 19.922270403408632,
+      "learning_rate": 4.017751479289941e-07,
+      "logits/chosen": -1.5859375,
+      "logits/rejected": -1.6875,
+      "logps/chosen": -235.0,
+      "logps/rejected": -298.0,
+      "loss": 0.4589,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.84765625,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -0.33203125,
+      "step": 260
+    },
+    {
+      "epoch": 0.8626198083067093,
+      "grad_norm": 23.03342264376932,
+      "learning_rate": 3.9585798816568044e-07,
+      "logits/chosen": -1.5703125,
+      "logits/rejected": -1.6484375,
+      "logps/chosen": -246.0,
+      "logps/rejected": -292.0,
+      "loss": 0.4974,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.609375,
+      "rewards/margins": 0.8125,
+      "rewards/rejected": -0.203125,
+      "step": 270
+    },
+    {
+      "epoch": 0.8945686900958466,
+      "grad_norm": 18.556977289134693,
+      "learning_rate": 3.8994082840236685e-07,
+      "logits/chosen": -1.625,
+      "logits/rejected": -1.6640625,
+      "logps/chosen": -248.0,
+      "logps/rejected": -280.0,
+      "loss": 0.4835,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.66796875,
+      "rewards/margins": 0.859375,
+      "rewards/rejected": -0.1904296875,
+      "step": 280
+    },
+    {
+      "epoch": 0.9265175718849841,
+      "grad_norm": 20.159977656950176,
+      "learning_rate": 3.840236686390532e-07,
+      "logits/chosen": -1.578125,
+      "logits/rejected": -1.6484375,
+      "logps/chosen": -227.0,
+      "logps/rejected": -298.0,
+      "loss": 0.4328,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.79296875,
+      "rewards/margins": 1.046875,
+      "rewards/rejected": -0.251953125,
+      "step": 290
+    },
+    {
+      "epoch": 0.9584664536741214,
+      "grad_norm": 18.20653927593905,
+      "learning_rate": 3.7810650887573966e-07,
+      "logits/chosen": -1.515625,
+      "logits/rejected": -1.59375,
+      "logps/chosen": -239.0,
+      "logps/rejected": -296.0,
+      "loss": 0.4848,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.68359375,
+      "rewards/margins": 1.09375,
+      "rewards/rejected": -0.412109375,
+      "step": 300
+    },
+    {
+      "epoch": 0.9904153354632588,
+      "grad_norm": 22.32128048324979,
+      "learning_rate": 3.72189349112426e-07,
+      "logits/chosen": -1.578125,
+      "logits/rejected": -1.59375,
+      "logps/chosen": -241.0,
+      "logps/rejected": -290.0,
+      "loss": 0.5057,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.60546875,
+      "rewards/margins": 0.86328125,
+      "rewards/rejected": -0.255859375,
+      "step": 310
+    },
+    {
+      "epoch": 1.0,
+      "eval_logits/chosen": -1.609375,
+      "eval_logits/rejected": -1.6171875,
+      "eval_logps/chosen": -237.0,
+      "eval_logps/rejected": -296.0,
+      "eval_loss": 0.43648436665534973,
+      "eval_rewards/accuracies": 0.7857142686843872,
+      "eval_rewards/chosen": 0.58203125,
+      "eval_rewards/margins": 0.9375,
+      "eval_rewards/rejected": -0.35546875,
+      "eval_runtime": 12.3846,
+      "eval_samples_per_second": 16.149,
+      "eval_steps_per_second": 0.565,
+      "step": 313
     }
   ],
   "logging_steps": 10,
-  "max_steps": 189,
+  "max_steps": 939,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
   "save_steps": 500,
@@ -354,7 +517,7 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff37ba364cdb9d523375e7721085f5bcee45f2bf0257fa418e0fa5bd10cd876d
-size 7800
+oid sha256:8aef5b63aa3b5f0736db5e2a3a04b28bb4e7f00aab0514f05bff1b95ce95c231
+size 7864