Training in progress, step 2600, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4309cbe22308fad27e8b5e7c85714a164b3e5d38d0b75c9561151c809911db4b
+oid sha256:822e6dcdbe992039c470c7d5345a56c4a6d718bf3296adb52b926cfb46d4ab8e
 size 29034840

last-checkpoint/global_step2599/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e095771b1dd0a77832675a32c1e0adba637ce893192708948322e4ae0e108d85
+size 43429616

last-checkpoint/global_step2599/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d8bfec6d94c896829153a3df913f4031650e5f5e6cd918e31315a75f8de3c9c
+size 43429616

last-checkpoint/global_step2599/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:011fb90eb0ac52df7b6b124720eba263bde86534222e1c70c19d554967ee6a75
+size 43429616

last-checkpoint/global_step2599/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8af9b47439108751362a728f0e1f6ff8284c1aac5c0f85286e109f3864ad8508
+size 43429616

last-checkpoint/global_step2599/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a33d0c44522250f6271beb76c30d602be4c8a12e9374721e9ac3f7ed9aa4cdf0
+size 637299

last-checkpoint/global_step2599/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1f057ab5447205b9eb14de207e7b2fd1b440d8f383f812bb4efbe5197c5bf59
+size 637171

last-checkpoint/global_step2599/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bffd784ee7b57957293b50b7f079fdfd39c1d272fb838f2f34e66e8e8a1176d
+size 637171

last-checkpoint/global_step2599/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2fc176fcd1b7abc3279e54d33aef1bf0c3f4582cf78c44322368da5ceeeb2a3
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2549
+global_step2599

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91caf226cd72269cf1248d5f2f1a1d0c92100844814086cd6878cfb06293dc0c
+oid sha256:7bf1e520c0ffd0912cdaf36292baf0834dd187b6292436dd343acbd7d39f37df
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a2264023ead4b957c8737f57b7f130753adf9857726b38e1a7c000a4c721672
+oid sha256:480a3937172137fc1b4a3886bc929d7dd5dd0507aebd6b20448e5d03ffe9f33f
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3af91d2c27fa6123938cc5723642d3bc4b5f5e9e58a6f0d15fdf45fd12cb685e
+oid sha256:9cc7d6afeed03ecf805019b6cd001604bbcd5e4ef0cbc38e4dc6f5587e48dbf2
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:57930146d6b40a72f7f121792ad7f144f1863c40a705666c20e64c1e1c54a94a
+oid sha256:48c0ace59112a18d7f177314d1a9eb2b6b894f2028df1dd368b8c67c8732e18d
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5184a5762c3654cd8e62f1aa51b8146cd229ac960662a48fb2daefc27ab2811c
+oid sha256:2794fc624dc0bb5af51b88db4a8a47eb798dc318755b3b6d0e646a1f265656ea
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.22014015913009644,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2250",
-  "epoch": 1.3131599278908062,
+  "best_metric": 0.20262545347213745,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2600",
+  "epoch": 1.338913211434458,
   "eval_steps": 50,
-  "global_step": 2550,
+  "global_step": 2600,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4546,11 +4546,100 @@
       "eval_steps_per_second": 0.77,
       "num_input_tokens_seen": 29818856,
       "step": 2550
+    },
+    {
+      "epoch": 1.3157352562451712,
+      "grad_norm": 5.864909751734038,
+      "learning_rate": 2.8698700556002178e-05,
+      "loss": 0.2231,
+      "num_input_tokens_seen": 29877288,
+      "step": 2555
+    },
+    {
+      "epoch": 1.3183105845995364,
+      "grad_norm": 11.12053960539336,
+      "learning_rate": 2.8506174548135695e-05,
+      "loss": 0.3007,
+      "num_input_tokens_seen": 29935776,
+      "step": 2560
+    },
+    {
+      "epoch": 1.3208859129539017,
+      "grad_norm": 5.495356583672569,
+      "learning_rate": 2.8314038667746732e-05,
+      "loss": 0.1895,
+      "num_input_tokens_seen": 29994232,
+      "step": 2565
+    },
+    {
+      "epoch": 1.3234612413082667,
+      "grad_norm": 11.482142819244615,
+      "learning_rate": 2.81222964022315e-05,
+      "loss": 0.2728,
+      "num_input_tokens_seen": 30052760,
+      "step": 2570
+    },
+    {
+      "epoch": 1.326036569662632,
+      "grad_norm": 8.217608228490523,
+      "learning_rate": 2.793095123184193e-05,
+      "loss": 0.2827,
+      "num_input_tokens_seen": 30111200,
+      "step": 2575
+    },
+    {
+      "epoch": 1.3286118980169972,
+      "grad_norm": 5.535041581031614,
+      "learning_rate": 2.77400066296223e-05,
+      "loss": 0.2191,
+      "num_input_tokens_seen": 30169680,
+      "step": 2580
+    },
+    {
+      "epoch": 1.3311872263713624,
+      "grad_norm": 6.639368336077178,
+      "learning_rate": 2.7549466061346386e-05,
+      "loss": 0.2317,
+      "num_input_tokens_seen": 30228176,
+      "step": 2585
+    },
+    {
+      "epoch": 1.3337625547257275,
+      "grad_norm": 5.625387657466934,
+      "learning_rate": 2.7359332985454443e-05,
+      "loss": 0.2293,
+      "num_input_tokens_seen": 30286656,
+      "step": 2590
+    },
+    {
+      "epoch": 1.3363378830800927,
+      "grad_norm": 3.631245501478199,
+      "learning_rate": 2.716961085299049e-05,
+      "loss": 0.1903,
+      "num_input_tokens_seen": 30345160,
+      "step": 2595
+    },
+    {
+      "epoch": 1.338913211434458,
+      "grad_norm": 11.391263066967507,
+      "learning_rate": 2.698030310753958e-05,
+      "loss": 0.224,
+      "num_input_tokens_seen": 30403640,
+      "step": 2600
+    },
+    {
+      "epoch": 1.338913211434458,
+      "eval_loss": 0.20262545347213745,
+      "eval_runtime": 19.6125,
+      "eval_samples_per_second": 3.059,
+      "eval_steps_per_second": 0.765,
+      "num_input_tokens_seen": 30403640,
+      "step": 2600
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 29818856,
+  "num_input_tokens_seen": 30403640,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4565,7 +4654,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1967582528405504.0,
+  "total_flos": 2006172016574464.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null