Training in progress, step 1900, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:715ff69f50f84d39187c1f0fc35de081fab6e8a1a5b66268497bd57f97e40762
+oid sha256:920101966caf9f63708128eacbc2e0f1c2ccd11ad0e5ce041ad3a77fc5419432
 size 29034840

last-checkpoint/global_step1900/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10004449ffc6fbe7243c0d014ae1bf0c7a32216c00dcb88aaf05da089e831e9e
+size 43429616

last-checkpoint/global_step1900/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c671b50f276fe4e6b55eb594721794eb78e97dfdcb1d79983a027142ec1ed050
+size 43429616

last-checkpoint/global_step1900/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bfe3408bf5e3534dc83c88bd98779cbd9b2aea263d697b0bad3c51905a8109d
+size 43429616

last-checkpoint/global_step1900/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae526fb8f169175aee500162ad359c6134710c3f5d82c9ba5253afe601763db8
+size 43429616

last-checkpoint/global_step1900/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b73c80d353e86bc0831c17b078c24040fdd0c4e8ab08514a49f32c8d30bdd05f
+size 637299

last-checkpoint/global_step1900/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1bf77037608fc752c877fb66d69006a641580266e173e61625e593bf57be3c5
+size 637171

last-checkpoint/global_step1900/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:231be1114d61d22226ebe1d59cbf8739f7815ea661c2bb2f951d7ed5587a3666
+size 637171

last-checkpoint/global_step1900/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d051a7de4c2283ae0bbc87046a8a57e26442a6c23b4a66eb9f4fe4aeb78f7dd
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1850
+global_step1900

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7cc13c69d2b97530a3e18634e2f473678ea6880a6b34244c9c86a457f70137e1
+oid sha256:585b8316bac42fce8431ad71be5542adc79fdd5ff1839401f8374f6ab8226086
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fcfd90610e35a8f40098e1413d3e3a1658276c859045d3450f227cab64c9081a
+oid sha256:cc4c7cc343f6d985da07523b86ffd7b02d0b8ae40ff925936c48aff8a7385f39
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:663ae184d05dd950d27e1f419e0306ca3b141b9d7f075a80804cada62f64a363
+oid sha256:b364386e5b3618cb74f99527a8e31fa3c7d8d93018d2d0cf0b819ae4c3c1794b
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec4f0a72927b3837a0e73cb5612acf4318c503f3ba2108b3925f750332a49b60
+oid sha256:ed71fe0f2fd47c414d9fd305b847e65b2f95946543d7b867318fdc05f761e6bc
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:657e1e43dbcdd82c1b5b25483acdceb69febc0e5b31b87d49bda24ccdcfd4221
+oid sha256:ee7d3b553d79cd5d86dc19d51dc31e4f97cde4fa9e0ae7f8da298e252f169e81
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.2935050129890442,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1850",
-  "epoch": 0.9528714911151172,
+  "epoch": 0.978624774658769,
   "eval_steps": 50,
-  "global_step": 1850,
+  "global_step": 1900,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3300,11 +3300,100 @@
       "eval_steps_per_second": 0.77,
       "num_input_tokens_seen": 21637848,
       "step": 1850
+    },
+    {
+      "epoch": 0.9554468194694824,
+      "grad_norm": 11.793922029031792,
+      "learning_rate": 5.778595395022226e-05,
+      "loss": 0.4205,
+      "num_input_tokens_seen": 21696328,
+      "step": 1855
+    },
+    {
+      "epoch": 0.9580221478238475,
+      "grad_norm": 7.242120499330675,
+      "learning_rate": 5.757546423837314e-05,
+      "loss": 0.3075,
+      "num_input_tokens_seen": 21754816,
+      "step": 1860
+    },
+    {
+      "epoch": 0.9605974761782127,
+      "grad_norm": 15.811484589504676,
+      "learning_rate": 5.736483702672155e-05,
+      "loss": 0.2604,
+      "num_input_tokens_seen": 21813296,
+      "step": 1865
+    },
+    {
+      "epoch": 0.9631728045325779,
+      "grad_norm": 13.712784450465344,
+      "learning_rate": 5.7154076138293914e-05,
+      "loss": 0.342,
+      "num_input_tokens_seen": 21871800,
+      "step": 1870
+    },
+    {
+      "epoch": 0.9657481328869431,
+      "grad_norm": 7.941289555505942,
+      "learning_rate": 5.694318539854297e-05,
+      "loss": 0.411,
+      "num_input_tokens_seen": 21930272,
+      "step": 1875
+    },
+    {
+      "epoch": 0.9683234612413083,
+      "grad_norm": 13.422362176081661,
+      "learning_rate": 5.673216863527836e-05,
+      "loss": 0.2606,
+      "num_input_tokens_seen": 21988736,
+      "step": 1880
+    },
+    {
+      "epoch": 0.9708987895956734,
+      "grad_norm": 3.5902749610502247,
+      "learning_rate": 5.652102967859715e-05,
+      "loss": 0.2647,
+      "num_input_tokens_seen": 22047200,
+      "step": 1885
+    },
+    {
+      "epoch": 0.9734741179500386,
+      "grad_norm": 10.785857611570844,
+      "learning_rate": 5.6309772360814295e-05,
+      "loss": 0.2961,
+      "num_input_tokens_seen": 22105664,
+      "step": 1890
+    },
+    {
+      "epoch": 0.9760494463044038,
+      "grad_norm": 12.758145270494738,
+      "learning_rate": 5.6098400516393065e-05,
+      "loss": 0.2776,
+      "num_input_tokens_seen": 22164184,
+      "step": 1895
+    },
+    {
+      "epoch": 0.978624774658769,
+      "grad_norm": 9.907554223635353,
+      "learning_rate": 5.5886917981875485e-05,
+      "loss": 0.3781,
+      "num_input_tokens_seen": 22222632,
+      "step": 1900
+    },
+    {
+      "epoch": 0.978624774658769,
+      "eval_loss": 0.297338604927063,
+      "eval_runtime": 19.4745,
+      "eval_samples_per_second": 3.081,
+      "eval_steps_per_second": 0.77,
+      "num_input_tokens_seen": 22222632,
+      "step": 1900
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 21637848,
+  "num_input_tokens_seen": 22222632,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -3319,7 +3408,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1427712275382272.0,
+  "total_flos": 1466299246968832.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null