Training in progress, step 950, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dca569a5bee748c34c51ac0d5b0f16a410ab4a71a296a7c172247ce0d6801519
+oid sha256:80df5e3a8d607d66820e6fd586076f2617165a54c1dd83c001587d2b85b8c1ef
 size 29034840

last-checkpoint/global_step950/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e233d0aa5b09d0dc9346bf7bb973b9b39be72451e5bbc62feeec79eb51fbca3
+size 43429616

last-checkpoint/global_step950/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a18d7c135c18df72fb2bb6dc1f9d7d84cc2d4d9509446abf4e7f9b8eda81120
+size 43429616

last-checkpoint/global_step950/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44849a34cc26d6fe94df3a9755d666d0035040fbc9e9e5d34f8ef9a19737370e
+size 43429616

last-checkpoint/global_step950/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9329ec1841ccea84942072d3c9fef8201a303a99281369f732ec7fac5af3c024
+size 43429616

last-checkpoint/global_step950/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de8d2452415f67f6cb314134176f636d47ded9477de6e168c2925a95f1e50399
+size 637299

last-checkpoint/global_step950/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59a0c15d11bab9ab36ba888fa09e8a77ddc15c19181e193c722b9f734d07accd
+size 637171

last-checkpoint/global_step950/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8010ba1747feb31117a6b69f86638d1fc0741ad088c9024682efeba498962221
+size 637171

last-checkpoint/global_step950/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39c9dd240d6e20c6b0f5d0f32f43209c894bb5b7adbab13d992cf542fcdcdb1c
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step900
+global_step950

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8f22ced19e790cc864cefe3b7c711d9ae631c44f95d42fb4829688cc3de0153
+oid sha256:7f2439da621f14c22b4f733e91bfc9de6b506d28d7b8d6f3eaca2e0b4f24c078
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e0407513eba77d34cbf3adf0e59a58bd80716f4f00f414854253637e82be43d
+oid sha256:c9e3fb386557f376b8946af5b8c91f9418f374dddb2ad9da4868b1ef16778c32
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6060636c023258ce9b965e244b8a58b4c99d5784dde4405b39737550ef50cd4f
+oid sha256:dc7774d06045635bece9e960378fdc6913bf7bbbc903444cc570d1ca6ac25645
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c24ccdfdcde39cb2265c82c50c36ffdfcc670f757aba4bcf4bb0fdc6d1373c4c
+oid sha256:d98c54a80a914fecf43d06ea81432499f46e70664f1d04651bf339163e30fa9e
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:728e48e0d034d64ded7c746a218c0b746489e6573b6bc352ba39a730bda96bbd
+oid sha256:fe7b1ea5625ff613528c3f7f1e76a7521cccec3038e0a450f6a24711c8e1b799
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.7039459347724915,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-850",
-  "epoch": 0.4635591037857327,
+  "best_metric": 0.6830747723579407,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-950",
+  "epoch": 0.4893123873293845,
   "eval_steps": 50,
-  "global_step": 900,
+  "global_step": 950,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1609,11 +1609,100 @@
       "eval_steps_per_second": 0.762,
       "num_input_tokens_seen": 10526712,
       "step": 900
+    },
+    {
+      "epoch": 0.46613443214009787,
+      "grad_norm": 7.649426233722995,
+      "learning_rate": 9.112590869053359e-05,
+      "loss": 0.6794,
+      "num_input_tokens_seen": 10585232,
+      "step": 905
+    },
+    {
+      "epoch": 0.46870976049446306,
+      "grad_norm": 6.066459270532772,
+      "learning_rate": 9.100438409496444e-05,
+      "loss": 0.6817,
+      "num_input_tokens_seen": 10643728,
+      "step": 910
+    },
+    {
+      "epoch": 0.47128508884882825,
+      "grad_norm": 7.144597127673979,
+      "learning_rate": 9.088211524207497e-05,
+      "loss": 0.6503,
+      "num_input_tokens_seen": 10702240,
+      "step": 915
+    },
+    {
+      "epoch": 0.4738604172031934,
+      "grad_norm": 9.676112884447143,
+      "learning_rate": 9.075910435112766e-05,
+      "loss": 0.6903,
+      "num_input_tokens_seen": 10760656,
+      "step": 920
+    },
+    {
+      "epoch": 0.4764357455575586,
+      "grad_norm": 12.206584747037537,
+      "learning_rate": 9.063535365485341e-05,
+      "loss": 0.6611,
+      "num_input_tokens_seen": 10819128,
+      "step": 925
+    },
+    {
+      "epoch": 0.47901107391192377,
+      "grad_norm": 8.724970113237934,
+      "learning_rate": 9.051086539941108e-05,
+      "loss": 0.6361,
+      "num_input_tokens_seen": 10877600,
+      "step": 930
+    },
+    {
+      "epoch": 0.48158640226628896,
+      "grad_norm": 26.26773221921971,
+      "learning_rate": 9.038564184434676e-05,
+      "loss": 0.7006,
+      "num_input_tokens_seen": 10936088,
+      "step": 935
+    },
+    {
+      "epoch": 0.48416173062065415,
+      "grad_norm": 6.223867131390233,
+      "learning_rate": 9.025968526255275e-05,
+      "loss": 0.7012,
+      "num_input_tokens_seen": 10994560,
+      "step": 940
+    },
+    {
+      "epoch": 0.4867370589750193,
+      "grad_norm": 6.1072541360418295,
+      "learning_rate": 9.013299794022622e-05,
+      "loss": 0.6968,
+      "num_input_tokens_seen": 11053016,
+      "step": 945
+    },
+    {
+      "epoch": 0.4893123873293845,
+      "grad_norm": 8.372921238626587,
+      "learning_rate": 9.00055821768278e-05,
+      "loss": 0.6825,
+      "num_input_tokens_seen": 11111520,
+      "step": 950
+    },
+    {
+      "epoch": 0.4893123873293845,
+      "eval_loss": 0.6830747723579407,
+      "eval_runtime": 19.7357,
+      "eval_samples_per_second": 3.04,
+      "eval_steps_per_second": 0.76,
+      "num_input_tokens_seen": 11111520,
+      "step": 950
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 10526712,
+  "num_input_tokens_seen": 11111520,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -1628,7 +1717,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 694543977283584.0,
+  "total_flos": 733134262632448.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null