Training in progress, step 2250, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f759f407b1897a80281dc7418f8b9856395bc3d571214488014c389b60bc84f7
+oid sha256:9efa5a507b1e2d13741d6dd8d920f108bb4ba17026ae4bb85a703fca7207dad8
 size 29034840

last-checkpoint/global_step2249/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ec8c956cd7f0b3ee292ff3d33951cc4172158cf1321eb2d9043b6245f345ef9
+size 43429616

last-checkpoint/global_step2249/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe3921fc532639d534ae9f17a894fc5accb247b315ee6e244460bf7a661410d8
+size 43429616

last-checkpoint/global_step2249/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8631ca14af8fe60d40f7a92203f3d54f42a50a84f3e0ee211dbc497ea10abc08
+size 43429616

last-checkpoint/global_step2249/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:843b10f1922017cf256aa50636cec5e12a2e31975cf67777e2d523c7b8298232
+size 43429616

last-checkpoint/global_step2249/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b734279c942fb522083927c9cf15334b8f92f36cf5cd7b8a2c9d3272281f45e
+size 637299

last-checkpoint/global_step2249/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3364a47ef048489172fae575c6478c201fa1b93f3504029cbf44dceb15c1e7f7
+size 637171

last-checkpoint/global_step2249/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e495c9d8b9b9cb7f4ae7247fc2c82061262ab9dc2a40bf356c51abbbe0b61a21
+size 637171

last-checkpoint/global_step2249/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:114dfd732dee90b2a833cfa17190e3281255e1bc8b16e35df789bcc362c3817f
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2199
+global_step2249

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3187a61ccc2722c440dc24ae4a6eefe6b9e5daccf9e92473bbb4483c7751ea77
+oid sha256:1d25cbcbbaa0866ea9c7365cb49b84e805db119693e615f5a1898a6ebfe997e8
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0f2a0df922fb3337cf2562745ebe8d5adf433ca45cb4e3da33a21b48183c000
+oid sha256:a883389afac12125c2c6bf62631b7de0220fdb0020d24cd0c6e8f8858dd3b362
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f4d84b5276f687f44c9af60b1e41cd7b93a6d1659e36831a7bc021b5635d663b
+oid sha256:577d49de6d60035e159d9ebb1e6eabef79a55787b14ecea93a6a93c242661779
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d69159433c88b97106cf21b92eb5a3f66f0c826aa268d82a47b3faed1ac86cd
+oid sha256:b5f2dd1c21e06806a9ce39eeab45734dfb8a62b829f91a86d1f65f13102d6242
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ff8fbfa0ee7724c753a28dc8879941724df5ef1edec9dbdcc7e9583ed5bd0a6
+oid sha256:e9cbcebe45b9faaca29ead6f9f59deeeb362fed9bbabf34da80537e86d967e85
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.2472737729549408,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1950",
-  "epoch": 1.1328869430852433,
+  "best_metric": 0.22014015913009644,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2250",
+  "epoch": 1.158640226628895,
   "eval_steps": 50,
-  "global_step": 2200,
+  "global_step": 2250,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3923,11 +3923,100 @@
       "eval_steps_per_second": 0.782,
       "num_input_tokens_seen": 25725560,
       "step": 2200
+    },
+    {
+      "epoch": 1.1354622714396085,
+      "grad_norm": 18.05516134286847,
+      "learning_rate": 4.2930264789752974e-05,
+      "loss": 0.3121,
+      "num_input_tokens_seen": 25784048,
+      "step": 2205
+    },
+    {
+      "epoch": 1.1380375997939738,
+      "grad_norm": 5.276257572884321,
+      "learning_rate": 4.271945150160815e-05,
+      "loss": 0.2448,
+      "num_input_tokens_seen": 25842512,
+      "step": 2210
+    },
+    {
+      "epoch": 1.140612928148339,
+      "grad_norm": 9.772532033416248,
+      "learning_rate": 4.250877036034552e-05,
+      "loss": 0.3141,
+      "num_input_tokens_seen": 25900992,
+      "step": 2215
+    },
+    {
+      "epoch": 1.143188256502704,
+      "grad_norm": 4.984185148660987,
+      "learning_rate": 4.2298225189970424e-05,
+      "loss": 0.2589,
+      "num_input_tokens_seen": 25959448,
+      "step": 2220
+    },
+    {
+      "epoch": 1.1457635848570693,
+      "grad_norm": 5.610565971454716,
+      "learning_rate": 4.2087819812020155e-05,
+      "loss": 0.3071,
+      "num_input_tokens_seen": 26017936,
+      "step": 2225
+    },
+    {
+      "epoch": 1.1483389132114346,
+      "grad_norm": 6.203972820387514,
+      "learning_rate": 4.1877558045494735e-05,
+      "loss": 0.1913,
+      "num_input_tokens_seen": 26076440,
+      "step": 2230
+    },
+    {
+      "epoch": 1.1509142415657996,
+      "grad_norm": 6.8902811910693185,
+      "learning_rate": 4.166744370678748e-05,
+      "loss": 0.1998,
+      "num_input_tokens_seen": 26134896,
+      "step": 2235
+    },
+    {
+      "epoch": 1.1534895699201648,
+      "grad_norm": 13.51914586072303,
+      "learning_rate": 4.1457480609615866e-05,
+      "loss": 0.2382,
+      "num_input_tokens_seen": 26193368,
+      "step": 2240
+    },
+    {
+      "epoch": 1.15606489827453,
+      "grad_norm": 10.238408014223214,
+      "learning_rate": 4.124767256495217e-05,
+      "loss": 0.231,
+      "num_input_tokens_seen": 26251848,
+      "step": 2245
+    },
+    {
+      "epoch": 1.158640226628895,
+      "grad_norm": 3.473707001713829,
+      "learning_rate": 4.1038023380954415e-05,
+      "loss": 0.273,
+      "num_input_tokens_seen": 26310336,
+      "step": 2250
+    },
+    {
+      "epoch": 1.158640226628895,
+      "eval_loss": 0.22014015913009644,
+      "eval_runtime": 20.315,
+      "eval_samples_per_second": 2.953,
+      "eval_steps_per_second": 0.738,
+      "num_input_tokens_seen": 26310336,
+      "step": 2250
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 25725560,
+  "num_input_tokens_seen": 26310336,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -3942,7 +4031,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1697482620796928.0,
+  "total_flos": 1736069012258816.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null