Training in progress, step 1050, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eec35e19707d55e085c42f2fc7f96384949ae1c2b6909dea425372d9568c8a7a
+oid sha256:9ec2597cb66cc8bf7c12cf0511653b2a0fba5900300120225835d32ef88d42ca
 size 29034840

last-checkpoint/global_step1050/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de2e167999736a3e47f82ccd89dd02140ccd2fb9e1416493e0d58739b4df6c86
+size 43429616

last-checkpoint/global_step1050/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65ffb0f2b0651eaf6c98bc5de8e998c605ae188e16936b9958297b9809eb66a7
+size 43429616

last-checkpoint/global_step1050/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94cf56191a583e6143daf3d907055192e9d16a3a366fe400ec28191e559cadb2
+size 43429616

last-checkpoint/global_step1050/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ccfaaa1c7bf29b7d738cd613b7a8c1cc7a2aad384ef1de5c5ce601c568d6969
+size 43429616

last-checkpoint/global_step1050/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5740907eeaf461a960ab6da055d43d89f41f6417899be566427552f76b967abe
+size 637299

last-checkpoint/global_step1050/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28dcc960a14c72ef47fb9722c4b98cf1679b58db19513a1020156883879227c7
+size 637171

last-checkpoint/global_step1050/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:055280373e3009fe56853d9b9ef4c853cc09e485fa66a9bd20cba222d17c0bf5
+size 637171

last-checkpoint/global_step1050/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:179e95a89c200a847bd6de0c47ca8b3d721ccf69a889371f83c8cb19695431d6
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1000
+global_step1050

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d73dfcc09cf3d6f08149535e03920234febc15f7e9a166987f3bc01ee871abf
+oid sha256:67db742b8aa1744a8224bf2a1f79d89caff63b15f78a455d92bb666df82183ea
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4396a64b6da4868d060d1e3c7c9ccb12c39d63bd0f7b146d2512400aff4c769c
+oid sha256:c0f9acb7e6f8bbfb305c3601c71eb6189af24942fab5f99046412c03bb10c3eb
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95877efc8fb5eb302819ee7effca4222569cdcfdebb9fa5d9846e68ed9e833fe
+oid sha256:713783338342f7486f6f186abd03c5963a0d22368f403efb2bf903ed083d2b64
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9fa4f23377f00fdde731da68a8690098617a1fdd912e03cdaa8bde87c493179
+oid sha256:83aaec0b5b7d8a2da4577075066cf434ce6e9feb9327edbea6677a2e51d76466
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:22246d81041447d5284c93bc1d01007e3c5bfb83233c72fb1f7878544bb0fc38
+oid sha256:15418e5c97d5f142900d492dc3d977b1b3706e9b18920657077086449e4a88dc
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.6830747723579407,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-950",
-  "epoch": 0.5150656708730363,
+  "best_metric": 0.6824291348457336,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1050",
+  "epoch": 0.5408189544166881,
   "eval_steps": 50,
-  "global_step": 1000,
+  "global_step": 1050,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1787,11 +1787,100 @@
       "eval_steps_per_second": 0.762,
       "num_input_tokens_seen": 11696200,
       "step": 1000
+    },
+    {
+      "epoch": 0.5176409992274015,
+      "grad_norm": 9.38238507668501,
+      "learning_rate": 8.855660246460895e-05,
+      "loss": 0.6959,
+      "num_input_tokens_seen": 11754720,
+      "step": 1005
+    },
+    {
+      "epoch": 0.5202163275817667,
+      "grad_norm": 8.602013574457626,
+      "learning_rate": 8.842062953648023e-05,
+      "loss": 0.6918,
+      "num_input_tokens_seen": 11813216,
+      "step": 1010
+    },
+    {
+      "epoch": 0.5227916559361319,
+      "grad_norm": 6.752768596735988,
+      "learning_rate": 8.828395924792497e-05,
+      "loss": 0.7269,
+      "num_input_tokens_seen": 11871712,
+      "step": 1015
+    },
+    {
+      "epoch": 0.525366984290497,
+      "grad_norm": 3.9336242665009187,
+      "learning_rate": 8.814659407960141e-05,
+      "loss": 0.7026,
+      "num_input_tokens_seen": 11930200,
+      "step": 1020
+    },
+    {
+      "epoch": 0.5279423126448622,
+      "grad_norm": 5.739197643968765,
+      "learning_rate": 8.800853652478028e-05,
+      "loss": 0.6467,
+      "num_input_tokens_seen": 11988704,
+      "step": 1025
+    },
+    {
+      "epoch": 0.5305176409992274,
+      "grad_norm": 4.583970239100745,
+      "learning_rate": 8.786978908929966e-05,
+      "loss": 0.6155,
+      "num_input_tokens_seen": 12047176,
+      "step": 1030
+    },
+    {
+      "epoch": 0.5330929693535926,
+      "grad_norm": 7.891614334520996,
+      "learning_rate": 8.773035429151937e-05,
+      "loss": 0.7849,
+      "num_input_tokens_seen": 12105680,
+      "step": 1035
+    },
+    {
+      "epoch": 0.5356682977079578,
+      "grad_norm": 10.308514200015722,
+      "learning_rate": 8.759023466227538e-05,
+      "loss": 0.6341,
+      "num_input_tokens_seen": 12164208,
+      "step": 1040
+    },
+    {
+      "epoch": 0.5382436260623229,
+      "grad_norm": 5.0434906291132995,
+      "learning_rate": 8.744943274483376e-05,
+      "loss": 0.7189,
+      "num_input_tokens_seen": 12222672,
+      "step": 1045
+    },
+    {
+      "epoch": 0.5408189544166881,
+      "grad_norm": 4.3092898145567125,
+      "learning_rate": 8.730795109484461e-05,
+      "loss": 0.6914,
+      "num_input_tokens_seen": 12281072,
+      "step": 1050
+    },
+    {
+      "epoch": 0.5408189544166881,
+      "eval_loss": 0.6824291348457336,
+      "eval_runtime": 19.7949,
+      "eval_samples_per_second": 3.031,
+      "eval_steps_per_second": 0.758,
+      "num_input_tokens_seen": 12281072,
+      "step": 1050
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 11696200,
+  "num_input_tokens_seen": 12281072,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -1806,7 +1895,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 771713061093376.0,
+  "total_flos": 810306416410624.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null