Training in progress, step 3000, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc8b033038b93e3de0bc922cc8f2b022fe9db637f004a766d4052071c9c2e30e
+oid sha256:ae6fb1f14ee1009f01ea7663b113fbf73bbec357852271439aa98fd9b151857f
 size 29034840

last-checkpoint/global_step2999/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b05b8f19e80af96dfd06b95e3286d82ffabbba9745fd88555cc350799c328a45
+size 43429616

last-checkpoint/global_step2999/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a16297865ca8e35bf16a93dfa65524ec8f4d46339b8afb4d196b9fb70444b0cd
+size 43429616

last-checkpoint/global_step2999/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20fc49baf52aaa80eda4198b8bba3d3d78bd08ec7892c3ea85854e49e6fbf793
+size 43429616

last-checkpoint/global_step2999/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56a03c805b6c083d2785d4c62aaa381d2d9fbd336ff2ac213906bc8bfe45a61c
+size 43429616

last-checkpoint/global_step2999/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bcc61ff9a283b0a1d9eb08deb6c03dcf9be742e81cbf1b8a9aeca5a7ece2f77
+size 637299

last-checkpoint/global_step2999/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3aed9bbcf0429217297595ec063bb407ce0ca0daf0eb96f5f2297d372f98e1e
+size 637171

last-checkpoint/global_step2999/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79ae875e6693af32ea495a37aa8e950f37781721acd4a701339f4bc3225c49f6
+size 637171

last-checkpoint/global_step2999/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7805dfe9ba12d6c3eb9816c02b895c880e764b65c43bf7210459893ae338b0e7
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2949
+global_step2999

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8ac6af6ac2b73603409d1721537224310f2ce061bc8c1c1c6f959231ed2e31e
+oid sha256:6f30f7f3c3620ccd30020e1ea4b81d1a56ee511b742c91370577d55399c14412
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:693cfdc542e1c8c319f7052d02602310660cb04e6571aa78525e03834c8b9930
+oid sha256:440cc1f8e138e1a90606722bd350460b4460991a2f3671f46f880f5743522dca
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ca85a038ef549f3dd6ca18c0dadc240e57927cad9adbcbd9ff224f3da3ae003
+oid sha256:d78d5df466f5b34e85649b90825d4b168464f6d49c668313415473184409b799
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c2a98b011c36f42f0d59506d21a897e487e2865babe045af4934af8b2748950
+oid sha256:6d8be6df9e66c8bba407df289d2aa6b4422668727a1e2419037de032213393a4
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3669b260cb8d49d29e38ac63ff4021b4682b4274090168fce7ad038be94b6302
+oid sha256:dc3e447bbd6be172b5767d058649279aa3d0985585618f7a30101af7a3f18799
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.18780523538589478,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2650",
-  "epoch": 1.5191861962400206,
+  "epoch": 1.5449394797836724,
   "eval_steps": 50,
-  "global_step": 2950,
+  "global_step": 3000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5258,11 +5258,100 @@
       "eval_steps_per_second": 0.777,
       "num_input_tokens_seen": 34497216,
       "step": 2950
+    },
+    {
+      "epoch": 1.5217615245943859,
+      "grad_norm": 7.283233671161431,
+      "learning_rate": 1.4803172119621661e-05,
+      "loss": 0.2811,
+      "num_input_tokens_seen": 34555704,
+      "step": 2955
+    },
+    {
+      "epoch": 1.524336852948751,
+      "grad_norm": 5.1545528457087295,
+      "learning_rate": 1.4652193074033188e-05,
+      "loss": 0.2056,
+      "num_input_tokens_seen": 34614168,
+      "step": 2960
+    },
+    {
+      "epoch": 1.5269121813031161,
+      "grad_norm": 5.345680235559109,
+      "learning_rate": 1.4501855615061376e-05,
+      "loss": 0.2097,
+      "num_input_tokens_seen": 34672624,
+      "step": 2965
+    },
+    {
+      "epoch": 1.5294875096574814,
+      "grad_norm": 5.662685942725516,
+      "learning_rate": 1.4352162471432872e-05,
+      "loss": 0.1913,
+      "num_input_tokens_seen": 34731056,
+      "step": 2970
+    },
+    {
+      "epoch": 1.5320628380118464,
+      "grad_norm": 7.857317097258681,
+      "learning_rate": 1.420311636017953e-05,
+      "loss": 0.1996,
+      "num_input_tokens_seen": 34789552,
+      "step": 2975
+    },
+    {
+      "epoch": 1.5346381663662116,
+      "grad_norm": 5.32739810275719,
+      "learning_rate": 1.4054719986589148e-05,
+      "loss": 0.2102,
+      "num_input_tokens_seen": 34848064,
+      "step": 2980
+    },
+    {
+      "epoch": 1.537213494720577,
+      "grad_norm": 18.39581577669835,
+      "learning_rate": 1.39069760441563e-05,
+      "loss": 0.232,
+      "num_input_tokens_seen": 34906592,
+      "step": 2985
+    },
+    {
+      "epoch": 1.5397888230749421,
+      "grad_norm": 8.502936446963005,
+      "learning_rate": 1.3759887214533557e-05,
+      "loss": 0.2109,
+      "num_input_tokens_seen": 34965064,
+      "step": 2990
+    },
+    {
+      "epoch": 1.5423641514293074,
+      "grad_norm": 5.97229161259624,
+      "learning_rate": 1.3613456167482685e-05,
+      "loss": 0.1989,
+      "num_input_tokens_seen": 35023552,
+      "step": 2995
+    },
+    {
+      "epoch": 1.5449394797836724,
+      "grad_norm": 17.874594504088744,
+      "learning_rate": 1.3467685560826315e-05,
+      "loss": 0.2027,
+      "num_input_tokens_seen": 35082056,
+      "step": 3000
+    },
+    {
+      "epoch": 1.5449394797836724,
+      "eval_loss": 0.2371213436126709,
+      "eval_runtime": 19.9605,
+      "eval_samples_per_second": 3.006,
+      "eval_steps_per_second": 0.751,
+      "num_input_tokens_seen": 35082056,
+      "step": 3000
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 34497216,
+  "num_input_tokens_seen": 35082056,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5277,7 +5366,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2276284685090816.0,
+  "total_flos": 2314876033433600.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null