Training in progress, step 1550, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c9a15524ca69ba3d1a171c59eaf574e64f0642e13435155688ec32a8d7af040
+oid sha256:81c47427b42705c35bffe681bec49fac21a6b19f781031ccb1d03b2dd6ac2efa
 size 29034840

last-checkpoint/global_step1550/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf681e5efd3e47f682c063da3d38d4db386b4d08c53fea2bfbb83e783e63635a
+size 43429616

last-checkpoint/global_step1550/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b67aec5c32eeb07d9ea90510bd992498a7b889457bdf24a2347c7ada1ac571ec
+size 43429616

last-checkpoint/global_step1550/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2b96525d4693047b0ce756922976df4a7eceb5cee4198eae42b69e16babccaa
+size 43429616

last-checkpoint/global_step1550/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8797a0fb24bb222a68365b1418a17c430c8945d7791d9c4904b6cd53ccba1b57
+size 43429616

last-checkpoint/global_step1550/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a43dd54cbbc70405ab57738adb3ec4581722d997b526b922cea67d12d7a3154
+size 637299

last-checkpoint/global_step1550/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2160593af1ce08c3f12b9b90e1c3cbef9f94fa69d7b34c91af296952a4c32634
+size 637171

last-checkpoint/global_step1550/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a26b6fdb593b50a1684d8681f4281e3d55be3afd1e2403c04c5de137e4315716
+size 637171

last-checkpoint/global_step1550/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4062e6de2aea9cf48856d37c0a6068948c993876856dc4d6c5e69e86ba09a28e
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1500
+global_step1550

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd3566049ad1f65d2f434d990deb65584d2b2dcb1aac8e89c68ea37dc533eab7
+oid sha256:7f1e48a120d69830576f7b582aa6cc46f0ca41d30015a7a674eaec3dcdfc0f09
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f96a47dc4176412bc893ccb49c004c8fa1cc8c306d67689d87ed20944233c62
+oid sha256:4dbabb9273d3983e52a4a981b5f60f8c2e19da375765d05bb9f2caad284b9652
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ebd9fa3db1079ccf750b71f4eeedbf1f04422fc748026a4b866afe133f9fbfd1
+oid sha256:554ac925bb9c9ea292b7a41caac1cf75285511cf8aa440f37090891ee457a178
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:383e9e252cc8292eef0120c964bdc8033972e800c085c97c42af97379e6b4b5c
+oid sha256:5be5e00123fc0a321e41599b50e07be02f4c165504c601192e5c73f5f5437c30
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a102b72949813698fac5ec632a1bf25ec2ce711ec639920c48d52212dc276b1
+oid sha256:009e196049daa80a75a37312338f9f37a038a260e8bdb3c8e7bae80b1332e3b1
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.3963810205459595,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1500",
-  "epoch": 0.7725985063095545,
+  "epoch": 0.7983517898532063,
   "eval_steps": 50,
-  "global_step": 1500,
+  "global_step": 1550,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2677,11 +2677,100 @@
       "eval_steps_per_second": 0.779,
       "num_input_tokens_seen": 17544440,
       "step": 1500
+    },
+    {
+      "epoch": 0.7751738346639196,
+      "grad_norm": 10.358996803929989,
+      "learning_rate": 7.195428951199204e-05,
+      "loss": 0.3397,
+      "num_input_tokens_seen": 17602952,
+      "step": 1505
+    },
+    {
+      "epoch": 0.7777491630182848,
+      "grad_norm": 11.954413440576635,
+      "learning_rate": 7.176270566729904e-05,
+      "loss": 0.5039,
+      "num_input_tokens_seen": 17661440,
+      "step": 1510
+    },
+    {
+      "epoch": 0.78032449137265,
+      "grad_norm": 15.247523980815545,
+      "learning_rate": 7.157072681477069e-05,
+      "loss": 0.426,
+      "num_input_tokens_seen": 17719928,
+      "step": 1515
+    },
+    {
+      "epoch": 0.7828998197270152,
+      "grad_norm": 11.560314722738736,
+      "learning_rate": 7.137835643895305e-05,
+      "loss": 0.3587,
+      "num_input_tokens_seen": 17778424,
+      "step": 1520
+    },
+    {
+      "epoch": 0.7854751480813804,
+      "grad_norm": 9.214361802688169,
+      "learning_rate": 7.118559803149865e-05,
+      "loss": 0.4849,
+      "num_input_tokens_seen": 17836936,
+      "step": 1525
+    },
+    {
+      "epoch": 0.7880504764357456,
+      "grad_norm": 9.825924936343675,
+      "learning_rate": 7.099245509110299e-05,
+      "loss": 0.3795,
+      "num_input_tokens_seen": 17895392,
+      "step": 1530
+    },
+    {
+      "epoch": 0.7906258047901107,
+      "grad_norm": 28.893219975863104,
+      "learning_rate": 7.079893112344118e-05,
+      "loss": 0.4206,
+      "num_input_tokens_seen": 17953872,
+      "step": 1535
+    },
+    {
+      "epoch": 0.7932011331444759,
+      "grad_norm": 9.96463686082691,
+      "learning_rate": 7.060502964110418e-05,
+      "loss": 0.3567,
+      "num_input_tokens_seen": 18012320,
+      "step": 1540
+    },
+    {
+      "epoch": 0.7957764614988411,
+      "grad_norm": 16.21585017609129,
+      "learning_rate": 7.041075416353513e-05,
+      "loss": 0.4172,
+      "num_input_tokens_seen": 18070792,
+      "step": 1545
+    },
+    {
+      "epoch": 0.7983517898532063,
+      "grad_norm": 7.729155059823011,
+      "learning_rate": 7.02161082169654e-05,
+      "loss": 0.4666,
+      "num_input_tokens_seen": 18129304,
+      "step": 1550
+    },
+    {
+      "epoch": 0.7983517898532063,
+      "eval_loss": 0.42660555243492126,
+      "eval_runtime": 19.4313,
+      "eval_samples_per_second": 3.088,
+      "eval_steps_per_second": 0.772,
+      "num_input_tokens_seen": 18129304,
+      "step": 1550
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 17544440,
+  "num_input_tokens_seen": 18129304,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2696,7 +2785,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1157604242882560.0,
+  "total_flos": 1196197017812992.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null