Training in progress, step 2700, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:764955504d6e3b1e90ca51e30bd2f4eb4f8626760b453dba421653e4148c6798
+oid sha256:493ef098b4e142b34c8f8ffa03b228d10404d9550092e5415ecc2ddce2804e56
 size 29034840

last-checkpoint/global_step2699/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6479c80783220f5b315b8cdfb27c69f066a48968353bae9fdd0c6ad51f103be3
+size 43429616

last-checkpoint/global_step2699/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b762b6ca4b4997a5ea83080a7f39306dd23f33bc2b94445246b8ce782127192
+size 43429616

last-checkpoint/global_step2699/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7b8abf2472099a388a11d1952eee5504aba8ae96731236f488d7f52753ae7e3
+size 43429616

last-checkpoint/global_step2699/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd79ab9b573afc088ff9dec4313acf1c44f97bb89ba0e4fb8695dd337c692876
+size 43429616

last-checkpoint/global_step2699/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b0db3b78beff3e56858d2436c1d0a2e545d33d040af80b20f8200dd2b4f2a56
+size 637299

last-checkpoint/global_step2699/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de5fe7df8657de59dc79c17929c6c467fcfb8dde65fda88f1345df754f88c846
+size 637171

last-checkpoint/global_step2699/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08279f88d37f7f8bbe8a892a3e3b3bc04d94c80affdedddfd9a3aef1850c302a
+size 637171

last-checkpoint/global_step2699/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a071bcb76fa2506fab196add6a363d182ee704d850d67a025e43d7d63627b2c
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2649
+global_step2699

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e599331812a34463d102d64a4034a0b702a893f362f752003aa577fe71dcc1d
+oid sha256:5d8d3c7739f9787ea797b86ff1b3a51f9e68197835ba3178915a8a77558f67fc
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ed431e5e71393a0174ad2fd492755f8c1142596f1af3bfe7827c1f8f815dd80
+oid sha256:a22a57799bc43e59db67d9a787ed73040020c5f35990602033f4dab1318787d7
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2e70789f26a9f56b6b779e87cb1a405615af81562a256e5afe579f40972e827
+oid sha256:29a624b936b77a04d6bfb6940acdd65a710bf39452e419e7ddb5c40fb2261072
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c8c18bc74d5211e761da269c814d7da0687633993838ec22e81ac939a14e91b
+oid sha256:3a79306817d4440cd621149537e8cf216b60f847fc6f9531a6147426aa02bb07
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b2d21491c81c396d8a264b58575a84e3c59e8d20650a3ba41b1f862eb888fcb7
+oid sha256:55fc50c620128befbe42c0ee9c82ae426d9d0e14e63b4929582516d207937ad9
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.18780523538589478,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2650",
-  "epoch": 1.3646664949781098,
+  "epoch": 1.3904197785217616,
   "eval_steps": 50,
-  "global_step": 2650,
+  "global_step": 2700,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4724,11 +4724,100 @@
       "eval_steps_per_second": 0.773,
       "num_input_tokens_seen": 30988344,
       "step": 2650
+    },
+    {
+      "epoch": 1.367241823332475,
+      "grad_norm": 7.2025231601123085,
+      "learning_rate": 2.4926244662411734e-05,
+      "loss": 0.2109,
+      "num_input_tokens_seen": 31046848,
+      "step": 2655
+    },
+    {
+      "epoch": 1.36981715168684,
+      "grad_norm": 4.944078503740031,
+      "learning_rate": 2.474217536081342e-05,
+      "loss": 0.2544,
+      "num_input_tokens_seen": 31105352,
+      "step": 2660
+    },
+    {
+      "epoch": 1.3723924800412053,
+      "grad_norm": 20.15441745302564,
+      "learning_rate": 2.4558564505825088e-05,
+      "loss": 0.2616,
+      "num_input_tokens_seen": 31163848,
+      "step": 2665
+    },
+    {
+      "epoch": 1.3749678083955703,
+      "grad_norm": 9.97997619530482,
+      "learning_rate": 2.4375415430107977e-05,
+      "loss": 0.1722,
+      "num_input_tokens_seen": 31222360,
+      "step": 2670
+    },
+    {
+      "epoch": 1.3775431367499356,
+      "grad_norm": 2.1763725749744465,
+      "learning_rate": 2.4192731457941805e-05,
+      "loss": 0.1966,
+      "num_input_tokens_seen": 31280840,
+      "step": 2675
+    },
+    {
+      "epoch": 1.3801184651043008,
+      "grad_norm": 10.60754914489294,
+      "learning_rate": 2.4010515905164243e-05,
+      "loss": 0.2578,
+      "num_input_tokens_seen": 31339264,
+      "step": 2680
+    },
+    {
+      "epoch": 1.382693793458666,
+      "grad_norm": 15.190148047284161,
+      "learning_rate": 2.3828772079110907e-05,
+      "loss": 0.3076,
+      "num_input_tokens_seen": 31397744,
+      "step": 2685
+    },
+    {
+      "epoch": 1.385269121813031,
+      "grad_norm": 20.889251542966374,
+      "learning_rate": 2.3647503278555233e-05,
+      "loss": 0.1911,
+      "num_input_tokens_seen": 31456240,
+      "step": 2690
+    },
+    {
+      "epoch": 1.3878444501673963,
+      "grad_norm": 4.797824757647722,
+      "learning_rate": 2.3466712793648638e-05,
+      "loss": 0.2204,
+      "num_input_tokens_seen": 31514744,
+      "step": 2695
+    },
+    {
+      "epoch": 1.3904197785217616,
+      "grad_norm": 16.36975645045513,
+      "learning_rate": 2.3286403905860733e-05,
+      "loss": 0.2269,
+      "num_input_tokens_seen": 31573240,
+      "step": 2700
+    },
+    {
+      "epoch": 1.3904197785217616,
+      "eval_loss": 0.24001093208789825,
+      "eval_runtime": 19.3768,
+      "eval_samples_per_second": 3.096,
+      "eval_steps_per_second": 0.774,
+      "num_input_tokens_seen": 31573240,
+      "step": 2700
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 30988344,
+  "num_input_tokens_seen": 31573240,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4743,7 +4832,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2044752555933696.0,
+  "total_flos": 2083346708430848.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null