Training in progress, step 750, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:086f56d8253ad2de065083ad9ed3e401f206ca6169266148720dc9056e97e11d
+oid sha256:9c65b6b24daabfe64ef0f6c6a0cc775ba02b62fb7a4c18bdac4a973c02aef24f
 size 29034840

last-checkpoint/global_step750/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:beb47aeb14a8461d8746421a15ccdc6228c1c958b75efa59b9309c850bc3df92
+size 43429616

last-checkpoint/global_step750/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6636dd3c88527c8676a1c724e2fb7a0720bed30ea3ffbe86356ebc9fea193980
+size 43429616

last-checkpoint/global_step750/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfdedc3d830cac142e8e593bff3ac4c5b883c6c4aa3b08d2c0b666c1b9f94cc1
+size 43429616

last-checkpoint/global_step750/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d48b4016ca4042b4b89b2e2a2f9d5265a3ac8b47e10f4d0fe183c7ef59fa4c94
+size 43429616

last-checkpoint/global_step750/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b7ceefcf5f9aaf423f675a59b70c7f49dd63bf2969dd0f241c706afca330069
+size 637299

last-checkpoint/global_step750/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88f0e9d3d0064652385de4668d7d1493464ea47cc90272985cc5f82378fe9407
+size 637171

last-checkpoint/global_step750/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e2747d5b9a430ea06aa5e313062964f76ced72bafedfb86736e283a3440555b
+size 637171

last-checkpoint/global_step750/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3df739d3279f7e7c62c77653c661bc9074781cffe48c2820c6dcb897ac09d46b
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step700
+global_step750

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e7d74de51245105e1fbf57a6707ef3538b353952485508f6e2f8f74dc5d479d4
+oid sha256:66b4ef73f3603a1b91082ee108fa8299ebe45fb3cdeec7d0bdca1982af5bf07d
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0617c9eb6cf7df57b2e0bb53cfe17c05f0910de56fe5b14427fe39ab54a44782
+oid sha256:882eccb2a272cf97cd645050bd077c971e48e78584f717a1b1cc9b5f1c9326dc
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed68a365057022897d9645ee60902a77102f43215dcdf2ddd5d3842b6a8446d8
+oid sha256:1626bac54f5806a604b7efdd749c5b65d63bbb40fc55c3744aae6130aa24f3de
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63ebaa0c302cadbdfcd9f8ee2289e35ecf9c9fc8c9968fc0c05f100dac20c6b9
+oid sha256:40cf83ff997228172cf0b991f9d5209728ccf2f0a75841db5e31e647779a1ad2
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c2f570b3fae67de7e649855487e96bf548cf918d7ba3d40d1baf265735ed69a0
+oid sha256:063f87ae0f4d3da8752b393e2d03482641fa84f93994bbbeba6ee3fa87c622b1
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.7318872809410095,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-700",
-  "epoch": 0.3605459696111254,
+  "epoch": 0.38629925315477726,
   "eval_steps": 50,
-  "global_step": 700,
+  "global_step": 750,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1253,11 +1253,100 @@
       "eval_steps_per_second": 0.762,
       "num_input_tokens_seen": 8187320,
       "step": 700
+    },
+    {
+      "epoch": 0.3631212979654906,
+      "grad_norm": 14.492396688166755,
+      "learning_rate": 9.535282851249103e-05,
+      "loss": 0.765,
+      "num_input_tokens_seen": 8245776,
+      "step": 705
+    },
+    {
+      "epoch": 0.3656966263198558,
+      "grad_norm": 16.054951836007135,
+      "learning_rate": 9.526273465583646e-05,
+      "loss": 0.7287,
+      "num_input_tokens_seen": 8304280,
+      "step": 710
+    },
+    {
+      "epoch": 0.36827195467422097,
+      "grad_norm": 21.4994447839661,
+      "learning_rate": 9.517181924991716e-05,
+      "loss": 0.758,
+      "num_input_tokens_seen": 8362728,
+      "step": 715
+    },
+    {
+      "epoch": 0.37084728302858616,
+      "grad_norm": 10.676012702912917,
+      "learning_rate": 9.508008394490926e-05,
+      "loss": 0.795,
+      "num_input_tokens_seen": 8421224,
+      "step": 720
+    },
+    {
+      "epoch": 0.37342261138295135,
+      "grad_norm": 9.802559177691224,
+      "learning_rate": 9.498753040587066e-05,
+      "loss": 0.6901,
+      "num_input_tokens_seen": 8479720,
+      "step": 725
+    },
+    {
+      "epoch": 0.3759979397373165,
+      "grad_norm": 12.345047855457121,
+      "learning_rate": 9.48941603127107e-05,
+      "loss": 0.7618,
+      "num_input_tokens_seen": 8538192,
+      "step": 730
+    },
+    {
+      "epoch": 0.3785732680916817,
+      "grad_norm": 9.115843124142248,
+      "learning_rate": 9.479997536015977e-05,
+      "loss": 0.7481,
+      "num_input_tokens_seen": 8596664,
+      "step": 735
+    },
+    {
+      "epoch": 0.3811485964460469,
+      "grad_norm": 13.754407712653018,
+      "learning_rate": 9.47049772577385e-05,
+      "loss": 0.746,
+      "num_input_tokens_seen": 8655128,
+      "step": 740
+    },
+    {
+      "epoch": 0.38372392480041206,
+      "grad_norm": 11.612003816357428,
+      "learning_rate": 9.460916772972672e-05,
+      "loss": 0.812,
+      "num_input_tokens_seen": 8713624,
+      "step": 745
+    },
+    {
+      "epoch": 0.38629925315477726,
+      "grad_norm": 27.606819697848053,
+      "learning_rate": 9.451254851513222e-05,
+      "loss": 0.6751,
+      "num_input_tokens_seen": 8772104,
+      "step": 750
+    },
+    {
+      "epoch": 0.38629925315477726,
+      "eval_loss": 0.732211709022522,
+      "eval_runtime": 19.8828,
+      "eval_samples_per_second": 3.018,
+      "eval_steps_per_second": 0.754,
+      "num_input_tokens_seen": 8772104,
+      "step": 750
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 8187320,
+  "num_input_tokens_seen": 8772104,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -1272,7 +1361,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 540175003287552.0,
+  "total_flos": 578762919116800.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null