Training in progress, step 2750, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:493ef098b4e142b34c8f8ffa03b228d10404d9550092e5415ecc2ddce2804e56
+oid sha256:dcf51604c7fbe68fb5bec81cac3b4632fb619bd371e32820e3fd872616836861
 size 29034840

last-checkpoint/global_step2749/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16f749fa7242f50a15167a3cfc8f3f0ca21db7ef95db11825b34fe8477b9894f
+size 43429616

last-checkpoint/global_step2749/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e282acf71b51832288b6023e31e80ec7d4a33addbf3307769341a67a260a5a8b
+size 43429616

last-checkpoint/global_step2749/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05689186bf64e352efea034b0526442cf4788fb5ca574778846dbd5212a52ac4
+size 43429616

last-checkpoint/global_step2749/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88284573bab80830396943150f719948ed82a68f579a0cb3e5bb27493aaf54b8
+size 43429616

last-checkpoint/global_step2749/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1051febab85db07da6f2d5583551a973689717beb73a989075986b4a249b7e3d
+size 637299

last-checkpoint/global_step2749/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07b952e7c1470e1848899b6793f447e4d319957d8b2072e541dcb23e86436686
+size 637171

last-checkpoint/global_step2749/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:162e2a9507b0126ea6d58b1dff2e6435a0794913e45421499748680c5934a764
+size 637171

last-checkpoint/global_step2749/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db1b6b12ab5689d184e76c5f3f7284d49c453eb7a4a7f2e7bb88f4bc020c628a
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2699
+global_step2749

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d8d3c7739f9787ea797b86ff1b3a51f9e68197835ba3178915a8a77558f67fc
+oid sha256:49db5a9fd0c84d580c671e52905ebeffc155b36537e76ff966d2e82906708999
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a22a57799bc43e59db67d9a787ed73040020c5f35990602033f4dab1318787d7
+oid sha256:c8ca224562d8d97aaa131b3516288bb99f68d7dcf62170494326662bda0bb206
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29a624b936b77a04d6bfb6940acdd65a710bf39452e419e7ddb5c40fb2261072
+oid sha256:56e86a11c89dba78d60e1b2a1855a651b90a5a22ef131ce65d26af83668c154e
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a79306817d4440cd621149537e8cf216b60f847fc6f9531a6147426aa02bb07
+oid sha256:3dd63019c923e9692431619aced46b91aaf3fd22e1c22ec0a64347f2fe635a0e
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:55fc50c620128befbe42c0ee9c82ae426d9d0e14e63b4929582516d207937ad9
+oid sha256:2d5206e8eee1fd5c926f6f8750c100fc15237f73c48e6ae482a3b1e4fadc9091
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.18780523538589478,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2650",
-  "epoch": 1.3904197785217616,
+  "epoch": 1.4161730620654134,
   "eval_steps": 50,
-  "global_step": 2700,
+  "global_step": 2750,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4813,11 +4813,100 @@
       "eval_steps_per_second": 0.774,
       "num_input_tokens_seen": 31573240,
       "step": 2700
+    },
+    {
+      "epoch": 1.3929951068761266,
+      "grad_norm": 23.383619982845065,
+      "learning_rate": 2.3106579887919894e-05,
+      "loss": 0.2339,
+      "num_input_tokens_seen": 31631688,
+      "step": 2705
+    },
+    {
+      "epoch": 1.3955704352304918,
+      "grad_norm": 4.588108718468969,
+      "learning_rate": 2.2927244003753694e-05,
+      "loss": 0.2197,
+      "num_input_tokens_seen": 31690200,
+      "step": 2710
+    },
+    {
+      "epoch": 1.398145763584857,
+      "grad_norm": 3.4659557649616284,
+      "learning_rate": 2.2748399508429834e-05,
+      "loss": 0.1999,
+      "num_input_tokens_seen": 31748664,
+      "step": 2715
+    },
+    {
+      "epoch": 1.4007210919392223,
+      "grad_norm": 5.297859489588456,
+      "learning_rate": 2.2570049648096902e-05,
+      "loss": 0.2757,
+      "num_input_tokens_seen": 31807176,
+      "step": 2720
+    },
+    {
+      "epoch": 1.4032964202935874,
+      "grad_norm": 8.718394323594142,
+      "learning_rate": 2.239219765992557e-05,
+      "loss": 0.2128,
+      "num_input_tokens_seen": 31865696,
+      "step": 2725
+    },
+    {
+      "epoch": 1.4058717486479526,
+      "grad_norm": 11.91241028531398,
+      "learning_rate": 2.221484677204978e-05,
+      "loss": 0.2657,
+      "num_input_tokens_seen": 31924200,
+      "step": 2730
+    },
+    {
+      "epoch": 1.4084470770023179,
+      "grad_norm": 6.407371884386196,
+      "learning_rate": 2.203800020350819e-05,
+      "loss": 0.2349,
+      "num_input_tokens_seen": 31982712,
+      "step": 2735
+    },
+    {
+      "epoch": 1.4110224053566829,
+      "grad_norm": 4.998470631619502,
+      "learning_rate": 2.1861661164185652e-05,
+      "loss": 0.2188,
+      "num_input_tokens_seen": 32041176,
+      "step": 2740
+    },
+    {
+      "epoch": 1.4135977337110481,
+      "grad_norm": 6.503122447364567,
+      "learning_rate": 2.1685832854755115e-05,
+      "loss": 0.2531,
+      "num_input_tokens_seen": 32099656,
+      "step": 2745
+    },
+    {
+      "epoch": 1.4161730620654134,
+      "grad_norm": 3.8963890551505123,
+      "learning_rate": 2.1510518466619334e-05,
+      "loss": 0.1416,
+      "num_input_tokens_seen": 32158144,
+      "step": 2750
+    },
+    {
+      "epoch": 1.4161730620654134,
+      "eval_loss": 0.24715903401374817,
+      "eval_runtime": 19.5042,
+      "eval_samples_per_second": 3.076,
+      "eval_steps_per_second": 0.769,
+      "num_input_tokens_seen": 32158144,
+      "step": 2750
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 31573240,
+  "num_input_tokens_seen": 32158144,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4832,7 +4921,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2083346708430848.0,
+  "total_flos": 2121940811644928.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null