Training in progress, step 84, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4cc8e98653996ccbe8e6e7055cadbd604a38886cb78c8c9dabe55c550b2d8bd2
+oid sha256:8390484e99a6ca25506ccaf92cb645bb0bf1ab6983a116f36fae158cd06bbfc2
 size 478211024

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2538df05ce2e3d95ac456f624673838f53b318273ce47ca1b7c8a886d4261f6d
+oid sha256:928608a38f32520a33ef7af17d06b62dc27cd037a2bf71cb5c4720dcb06e835b
 size 243337876

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e3484a8fae8bef26e8aceca1a4986225c025687e241c60c48ff1e088b574d9c
+oid sha256:a889f36d0b3b0e6dbbd898a2660bec9b7db6408bce16221a02808a207d0a2e39
 size 15984

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87b0a5e2553ff606038df9001dd88a817d7fc5872b495490bae91e01fad1ec5c
+oid sha256:5dd921a952ea9c7967e8db6ce5ee54cda5b37007fbc220247a9a4d513537e8b4
 size 15984

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54bad5f88df8ac9b5c20646448922516fa39d7287816baa41b79627fe21c46aa
+oid sha256:462ac9d41f9ae16f3d7503a703b674e354b5ce1c9d7b0ab6fc3347bf4f3f0629
 size 15984

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96a326ecc1330f08d92e39047259a8624f4099061baa191d3c258d0e544825d6
+oid sha256:92252b6027cdc118ad9dfaf1397e24f9e5791e684dfb65e9d19bca24bf212bb0
 size 15984

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f646d116a4b94d580a5f63a0a9a98584c51569731fd11c856d15da201f219f9
+oid sha256:91153713420b5f9d63d1f5a6eaee63174c1b2d5735a06a7bbdf0aaa08ff45016
 size 15984

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:330623ee88337c01c8bd095f67d83bf6ad0846e58dd146fa5dde80c5db29f6f9
+oid sha256:c7affa3a740eb7dfccbe1f59366cf137ac9d6884cf98b06d914099fee6c3e958
 size 15984

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b61ef68b78f474a58968515752e2de9b80e4d7b7fc05d6b1707f02a47d657621
+oid sha256:c9db0008f3524b7fd4944ab5a33b7a25383d04061f998c04fc1c61e76c29b40c
 size 15984

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6bcd1e4f18fb0864f5598bac2c33d66314f526665ba40514ba24df7c912075fd
+oid sha256:6901d2a29b14e7a40d89ae1c8eea7922a2dac90b3d788847d854c8acda7fea8f
 size 15984

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ea1834b19ac5c7c4cdbdb524938202c9fe17deb719aea1234ea8bf8c83879be
+oid sha256:7f033506de37408b1d728cf1b729bd4839fc41c8f2e858dbc7623579a6647247
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 9.761539459228516,
   "best_model_checkpoint": "miner_id_24/checkpoint-50",
-  "epoch": 1.8198198198198199,
+  "epoch": 3.081081081081081,
   "eval_steps": 50,
-  "global_step": 50,
+  "global_step": 84,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -373,6 +373,244 @@
       "eval_samples_per_second": 131.211,
       "eval_steps_per_second": 4.221,
       "step": 50
+    },
+    {
+      "epoch": 1.855855855855856,
+      "grad_norm": 1.0086657164815892e+18,
+      "learning_rate": 4.528945529245924e-05,
+      "loss": 8.9588,
+      "step": 51
+    },
+    {
+      "epoch": 1.8918918918918919,
+      "grad_norm": 1.4040368349708288e+17,
+      "learning_rate": 4.301797978523293e-05,
+      "loss": 9.0851,
+      "step": 52
+    },
+    {
+      "epoch": 1.9279279279279278,
+      "grad_norm": Infinity,
+      "learning_rate": 4.076719565451069e-05,
+      "loss": 10.925,
+      "step": 53
+    },
+    {
+      "epoch": 1.9639639639639639,
+      "grad_norm": Infinity,
+      "learning_rate": 3.854115896542801e-05,
+      "loss": 8.9936,
+      "step": 54
+    },
+    {
+      "epoch": 2.018018018018018,
+      "grad_norm": Infinity,
+      "learning_rate": 3.6343881186557866e-05,
+      "loss": 17.2423,
+      "step": 55
+    },
+    {
+      "epoch": 2.054054054054054,
+      "grad_norm": Infinity,
+      "learning_rate": 3.417932196097319e-05,
+      "loss": 9.8011,
+      "step": 56
+    },
+    {
+      "epoch": 2.09009009009009,
+      "grad_norm": Infinity,
+      "learning_rate": 3.205138197070241e-05,
+      "loss": 9.3979,
+      "step": 57
+    },
+    {
+      "epoch": 2.126126126126126,
+      "grad_norm": Infinity,
+      "learning_rate": 2.99638959074368e-05,
+      "loss": 9.1279,
+      "step": 58
+    },
+    {
+      "epoch": 2.1621621621621623,
+      "grad_norm": 1.0414861110829122e+19,
+      "learning_rate": 2.7920625562156932e-05,
+      "loss": 8.6924,
+      "step": 59
+    },
+    {
+      "epoch": 2.1981981981981984,
+      "grad_norm": 2.4940312447156224e+16,
+      "learning_rate": 2.5925253046130883e-05,
+      "loss": 8.528,
+      "step": 60
+    },
+    {
+      "epoch": 2.234234234234234,
+      "grad_norm": Infinity,
+      "learning_rate": 2.3981374155501056e-05,
+      "loss": 8.5312,
+      "step": 61
+    },
+    {
+      "epoch": 2.27027027027027,
+      "grad_norm": Infinity,
+      "learning_rate": 2.2092491891416384e-05,
+      "loss": 12.068,
+      "step": 62
+    },
+    {
+      "epoch": 2.3063063063063063,
+      "grad_norm": Infinity,
+      "learning_rate": 2.026201014738749e-05,
+      "loss": 9.5087,
+      "step": 63
+    },
+    {
+      "epoch": 2.3423423423423424,
+      "grad_norm": 8.642452215144907e+18,
+      "learning_rate": 1.849322757524057e-05,
+      "loss": 9.183,
+      "step": 64
+    },
+    {
+      "epoch": 2.3783783783783785,
+      "grad_norm": 7.275224349412426e+18,
+      "learning_rate": 1.6789331640723738e-05,
+      "loss": 8.6685,
+      "step": 65
+    },
+    {
+      "epoch": 2.4144144144144146,
+      "grad_norm": 6.894451927841505e+18,
+      "learning_rate": 1.515339287947842e-05,
+      "loss": 8.5355,
+      "step": 66
+    },
+    {
+      "epoch": 2.4504504504504503,
+      "grad_norm": 563174499680256.0,
+      "learning_rate": 1.3588359363726518e-05,
+      "loss": 4.2306,
+      "step": 67
+    },
+    {
+      "epoch": 2.4864864864864864,
+      "grad_norm": Infinity,
+      "learning_rate": 1.2097051389645058e-05,
+      "loss": 16.6928,
+      "step": 68
+    },
+    {
+      "epoch": 2.5225225225225225,
+      "grad_norm": Infinity,
+      "learning_rate": 1.0682156395001747e-05,
+      "loss": 9.3695,
+      "step": 69
+    },
+    {
+      "epoch": 2.5585585585585586,
+      "grad_norm": 9.359216148710687e+17,
+      "learning_rate": 9.346224116210607e-06,
+      "loss": 8.805,
+      "step": 70
+    },
+    {
+      "epoch": 2.5945945945945947,
+      "grad_norm": 4.12155610163839e+17,
+      "learning_rate": 8.091661993534535e-06,
+      "loss": 8.7406,
+      "step": 71
+    },
+    {
+      "epoch": 2.6306306306306304,
+      "grad_norm": 4.016165506934374e+16,
+      "learning_rate": 6.920730832715246e-06,
+      "loss": 8.5016,
+      "step": 72
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 1.49182067510136e+18,
+      "learning_rate": 5.835540730848524e-06,
+      "loss": 4.957,
+      "step": 73
+    },
+    {
+      "epoch": 2.7027027027027026,
+      "grad_norm": Infinity,
+      "learning_rate": 4.838047273846599e-06,
+      "loss": 15.8309,
+      "step": 74
+    },
+    {
+      "epoch": 2.7387387387387387,
+      "grad_norm": Infinity,
+      "learning_rate": 3.930048012340241e-06,
+      "loss": 9.5723,
+      "step": 75
+    },
+    {
+      "epoch": 2.774774774774775,
+      "grad_norm": Infinity,
+      "learning_rate": 3.113179222370911e-06,
+      "loss": 9.0965,
+      "step": 76
+    },
+    {
+      "epoch": 2.810810810810811,
+      "grad_norm": 3.073233680802513e+18,
+      "learning_rate": 2.388912956710829e-06,
+      "loss": 8.6315,
+      "step": 77
+    },
+    {
+      "epoch": 2.846846846846847,
+      "grad_norm": 6.825145861652087e+18,
+      "learning_rate": 1.7585543921243115e-06,
+      "loss": 8.6798,
+      "step": 78
+    },
+    {
+      "epoch": 2.8828828828828827,
+      "grad_norm": 7784822034399232.0,
+      "learning_rate": 1.2232394773511053e-06,
+      "loss": 8.6864,
+      "step": 79
+    },
+    {
+      "epoch": 2.918918918918919,
+      "grad_norm": Infinity,
+      "learning_rate": 7.839328860498892e-07,
+      "loss": 10.5033,
+      "step": 80
+    },
+    {
+      "epoch": 2.954954954954955,
+      "grad_norm": Infinity,
+      "learning_rate": 4.414262783912126e-07,
+      "loss": 8.1782,
+      "step": 81
+    },
+    {
+      "epoch": 3.009009009009009,
+      "grad_norm": Infinity,
+      "learning_rate": 1.9633687443230459e-07,
+      "loss": 15.877,
+      "step": 82
+    },
+    {
+      "epoch": 3.045045045045045,
+      "grad_norm": Infinity,
+      "learning_rate": 4.9106341844862845e-08,
+      "loss": 9.9209,
+      "step": 83
+    },
+    {
+      "epoch": 3.081081081081081,
+      "grad_norm": Infinity,
+      "learning_rate": 0.0,
+      "loss": 9.3869,
+      "step": 84
     }
   ],
   "logging_steps": 1,
@@ -396,12 +634,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 3.081994717227909e+17,
+  "total_flos": 5.181981315112632e+17,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null