Training in progress, step 1100, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ec2597cb66cc8bf7c12cf0511653b2a0fba5900300120225835d32ef88d42ca
+oid sha256:a5dad64d30fcdb47a2e15bc593335933f84181ea038dde168b8ebbd434b58442
 size 29034840

last-checkpoint/global_step1100/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ccba41fa9c69863760c3b5bcd9d61f13b46c515b62b014a6d61c06c032f951a
+size 43429616

last-checkpoint/global_step1100/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70b88ddf43bccc562bf81975ebb9b42ae702870f931e36cd1c4319b9c01be85c
+size 43429616

last-checkpoint/global_step1100/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da9bb536698e2adeee2fa246c6e2d30c74e6a30bf9cd88dc1d3e4d4bf5d604e4
+size 43429616

last-checkpoint/global_step1100/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:507419dffbde31d35bfb3ced0ea01ddd508cf02375f4e6869caa54a2586997da
+size 43429616

last-checkpoint/global_step1100/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0594a06a960a2b82b79ae3375fcc5157d04c096adbc8e37daafab8145646afd
+size 637299

last-checkpoint/global_step1100/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:602bae77c5e406ad93d955e22f82404c93c6d54927e02491854e9d1b05064fd2
+size 637171

last-checkpoint/global_step1100/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62e29c7a76a09448e9c9a0f2bf6ede9bafd0bf3776d255f4d3e4a617869514ec
+size 637171

last-checkpoint/global_step1100/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f47a3e9d4caf314582b7b91a420e64941ca9a069f6f32e6e6bbdd05a980a717e
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1050
+global_step1100

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67db742b8aa1744a8224bf2a1f79d89caff63b15f78a455d92bb666df82183ea
+oid sha256:9d9fea52fb92cc51e76feeb2b139ce35723c0cb651da383e4f7eec2606ed6c2a
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0f9acb7e6f8bbfb305c3601c71eb6189af24942fab5f99046412c03bb10c3eb
+oid sha256:1b5820ebfcc2e1cfe1ad2619a05ea9a484ff21635e13e386bf14abd302f2c0f7
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:713783338342f7486f6f186abd03c5963a0d22368f403efb2bf903ed083d2b64
+oid sha256:eb8d96a68e732fca41980516622a50990bbd3ee989e72076a35c8608d9b4d136
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83aaec0b5b7d8a2da4577075066cf434ce6e9feb9327edbea6677a2e51d76466
+oid sha256:ce6f1db6d09f4d89a9b2bd8dc8eeb99f1fada2ec04376e23b5a7a13004994005
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15418e5c97d5f142900d492dc3d977b1b3706e9b18920657077086449e4a88dc
+oid sha256:9c5d08a4bbe1ee22ad6f9fd0a36b9c3ef20361a28d31a9dd65a011a3a2857a08
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.6824291348457336,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1050",
-  "epoch": 0.5408189544166881,
+  "best_metric": 0.6431913375854492,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1100",
+  "epoch": 0.56657223796034,
   "eval_steps": 50,
-  "global_step": 1050,
+  "global_step": 1100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1876,11 +1876,100 @@
       "eval_steps_per_second": 0.758,
       "num_input_tokens_seen": 12281072,
       "step": 1050
+    },
+    {
+      "epoch": 0.5433942827710533,
+      "grad_norm": 6.118699079738975,
+      "learning_rate": 8.716579228029562e-05,
+      "loss": 0.6567,
+      "num_input_tokens_seen": 12339544,
+      "step": 1055
+    },
+    {
+      "epoch": 0.5459696111254185,
+      "grad_norm": 10.911613071983485,
+      "learning_rate": 8.702295888146548e-05,
+      "loss": 0.6155,
+      "num_input_tokens_seen": 12397992,
+      "step": 1060
+    },
+    {
+      "epoch": 0.5485449394797837,
+      "grad_norm": 7.643873763070769,
+      "learning_rate": 8.687945349087703e-05,
+      "loss": 0.6029,
+      "num_input_tokens_seen": 12456480,
+      "step": 1065
+    },
+    {
+      "epoch": 0.5511202678341488,
+      "grad_norm": 8.136001935771205,
+      "learning_rate": 8.673527871325022e-05,
+      "loss": 0.6129,
+      "num_input_tokens_seen": 12515000,
+      "step": 1070
+    },
+    {
+      "epoch": 0.553695596188514,
+      "grad_norm": 11.244237678841547,
+      "learning_rate": 8.659043716545485e-05,
+      "loss": 0.6825,
+      "num_input_tokens_seen": 12573504,
+      "step": 1075
+    },
+    {
+      "epoch": 0.5562709245428792,
+      "grad_norm": 7.506618014617338,
+      "learning_rate": 8.644493147646302e-05,
+      "loss": 0.6828,
+      "num_input_tokens_seen": 12632008,
+      "step": 1080
+    },
+    {
+      "epoch": 0.5588462528972444,
+      "grad_norm": 6.1353619139783415,
+      "learning_rate": 8.629876428730145e-05,
+      "loss": 0.6286,
+      "num_input_tokens_seen": 12690520,
+      "step": 1085
+    },
+    {
+      "epoch": 0.5614215812516096,
+      "grad_norm": 4.639695384779573,
+      "learning_rate": 8.615193825100355e-05,
+      "loss": 0.6361,
+      "num_input_tokens_seen": 12749032,
+      "step": 1090
+    },
+    {
+      "epoch": 0.5639969096059748,
+      "grad_norm": 8.458071149567038,
+      "learning_rate": 8.600445603256123e-05,
+      "loss": 0.6087,
+      "num_input_tokens_seen": 12807504,
+      "step": 1095
+    },
+    {
+      "epoch": 0.56657223796034,
+      "grad_norm": 7.622756865192901,
+      "learning_rate": 8.585632030887658e-05,
+      "loss": 0.5825,
+      "num_input_tokens_seen": 12865992,
+      "step": 1100
+    },
+    {
+      "epoch": 0.56657223796034,
+      "eval_loss": 0.6431913375854492,
+      "eval_runtime": 19.2744,
+      "eval_samples_per_second": 3.113,
+      "eval_steps_per_second": 0.778,
+      "num_input_tokens_seen": 12865992,
+      "step": 1100
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 12281072,
+  "num_input_tokens_seen": 12865992,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -1895,7 +1984,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 810306416410624.0,
+  "total_flos": 848901994971136.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null