Training in progress, step 1600, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81c47427b42705c35bffe681bec49fac21a6b19f781031ccb1d03b2dd6ac2efa
+oid sha256:36f722668f1a0618c0a9e43f223d905e82d0c1068f32211684745e00d5aaa272
 size 29034840

last-checkpoint/global_step1600/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e091a8c7fceb415905f7821beab43c7ed45ca60df330f7eb6bca5d2495e0afd
+size 43429616

last-checkpoint/global_step1600/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc8c286a1f817f5fa92ee5287137ba1da1dd812b56fe458a4b50e828a9ef57ce
+size 43429616

last-checkpoint/global_step1600/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e860255e84d5ed5472a1e731852a76784b39bcc513d580bce574a8f9843c030d
+size 43429616

last-checkpoint/global_step1600/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71678fd629b82fabf0f9b710e39e9496a4c007151a404a0928b1d62ea948f6b2
+size 43429616

last-checkpoint/global_step1600/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d4afb4aaf2fe09b00f009636090ac8140a92a92061b0beb56860dbb1563a322
+size 637299

last-checkpoint/global_step1600/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99752500f7623dfbbb0b804e1c22e1cbd188635e5967f6ff430fde200bbb865f
+size 637171

last-checkpoint/global_step1600/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf3c8ffb4a388a7c685af82e29e62c38dac41b19f00d659db4227bfe73f04b08
+size 637171

last-checkpoint/global_step1600/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bab2075b924174a0c53aaeb6926f645d6f84f031f1c7a5a84a1e25330a267f11
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1550
+global_step1600

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f1e48a120d69830576f7b582aa6cc46f0ca41d30015a7a674eaec3dcdfc0f09
+oid sha256:9279ed4b01716237e789d2631c1f29bc5d43c5633c014d4401de21b672c1b355
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4dbabb9273d3983e52a4a981b5f60f8c2e19da375765d05bb9f2caad284b9652
+oid sha256:ca1990d68e57c70df5c56d395dd3f3befbe07b380521f4144677c20f6fe2a3eb
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:554ac925bb9c9ea292b7a41caac1cf75285511cf8aa440f37090891ee457a178
+oid sha256:e0790066885525e1b9a9390a40ae27abd57abb47f031abface27890732f9e684
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5be5e00123fc0a321e41599b50e07be02f4c165504c601192e5c73f5f5437c30
+oid sha256:1325a2034fe48ebad4f00ac8a2b32ab5c4c43c2497712169a8e3b1112363d916
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:009e196049daa80a75a37312338f9f37a038a260e8bdb3c8e7bae80b1332e3b1
+oid sha256:05a385f582b86d8b587b56eaada5930afef800d2a4f1e7413c113f427a6fcef2
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.3963810205459595,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1500",
-  "epoch": 0.7983517898532063,
+  "epoch": 0.8241050733968581,
   "eval_steps": 50,
-  "global_step": 1550,
+  "global_step": 1600,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2766,11 +2766,100 @@
       "eval_steps_per_second": 0.772,
       "num_input_tokens_seen": 18129304,
       "step": 1550
+    },
+    {
+      "epoch": 0.8009271182075715,
+      "grad_norm": 6.105522334359041,
+      "learning_rate": 7.002109533435066e-05,
+      "loss": 0.4305,
+      "num_input_tokens_seen": 18187728,
+      "step": 1555
+    },
+    {
+      "epoch": 0.8035024465619367,
+      "grad_norm": 7.852746488368037,
+      "learning_rate": 6.982571905530669e-05,
+      "loss": 0.4529,
+      "num_input_tokens_seen": 18246192,
+      "step": 1560
+    },
+    {
+      "epoch": 0.8060777749163018,
+      "grad_norm": 5.8884808032636275,
+      "learning_rate": 6.962998292604517e-05,
+      "loss": 0.4569,
+      "num_input_tokens_seen": 18304632,
+      "step": 1565
+    },
+    {
+      "epoch": 0.808653103270667,
+      "grad_norm": 8.738504630489588,
+      "learning_rate": 6.943389049930931e-05,
+      "loss": 0.3936,
+      "num_input_tokens_seen": 18363136,
+      "step": 1570
+    },
+    {
+      "epoch": 0.8112284316250322,
+      "grad_norm": 8.410645723765466,
+      "learning_rate": 6.923744533430937e-05,
+      "loss": 0.4083,
+      "num_input_tokens_seen": 18421592,
+      "step": 1575
+    },
+    {
+      "epoch": 0.8138037599793974,
+      "grad_norm": 7.654296364994705,
+      "learning_rate": 6.904065099665803e-05,
+      "loss": 0.4564,
+      "num_input_tokens_seen": 18480104,
+      "step": 1580
+    },
+    {
+      "epoch": 0.8163790883337626,
+      "grad_norm": 17.3461741974751,
+      "learning_rate": 6.884351105830568e-05,
+      "loss": 0.3928,
+      "num_input_tokens_seen": 18538600,
+      "step": 1585
+    },
+    {
+      "epoch": 0.8189544166881277,
+      "grad_norm": 16.63308936698189,
+      "learning_rate": 6.864602909747563e-05,
+      "loss": 0.4278,
+      "num_input_tokens_seen": 18597104,
+      "step": 1590
+    },
+    {
+      "epoch": 0.8215297450424929,
+      "grad_norm": 10.83446543348463,
+      "learning_rate": 6.84482086985991e-05,
+      "loss": 0.3956,
+      "num_input_tokens_seen": 18655584,
+      "step": 1595
+    },
+    {
+      "epoch": 0.8241050733968581,
+      "grad_norm": 7.98539427128297,
+      "learning_rate": 6.825005345225019e-05,
+      "loss": 0.3834,
+      "num_input_tokens_seen": 18714072,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8241050733968581,
+      "eval_loss": 0.44774264097213745,
+      "eval_runtime": 19.316,
+      "eval_samples_per_second": 3.106,
+      "eval_steps_per_second": 0.777,
+      "num_input_tokens_seen": 18714072,
+      "step": 1600
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 18129304,
+  "num_input_tokens_seen": 18714072,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2785,7 +2874,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1196197017812992.0,
+  "total_flos": 1234783772868608.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null