Training in progress, step 3100, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e06b707bfa3ec5abcc1d86c5faaa69df3813668bd2420483f88e9d7e67c4d0c6
+oid sha256:7239a3f5625e1a3761226609117c1f747a15f2afb3ec951ab6136b481594307b
 size 29034840

last-checkpoint/global_step3099/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5848a54367bfaea0f562a92f949ce89c940f9a4825026a2e85aff5fcaa1886b0
+size 43429616

last-checkpoint/global_step3099/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b108a72650a52bf88206f95453fc52054dd8aa304df8cddd6fbef009c31f18ef
+size 43429616

last-checkpoint/global_step3099/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fca00a84e9f4c8c7e31b17bac6b2376db807b60acf7d050eeaaed464e6ec0a53
+size 43429616

last-checkpoint/global_step3099/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45bf750487d15fecdccd69502889cae8daf31861b0529c38b22efaada644d9ce
+size 43429616

last-checkpoint/global_step3099/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96fba03550c3418465ff6dda65b74bfb4da6fb5ab4bf90bd5b14c45c919bd26b
+size 637299

last-checkpoint/global_step3099/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fa2624f16f6d92ee668571339fbe4c0d54e94330926d0ce8e1c6a2cd661669e
+size 637171

last-checkpoint/global_step3099/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1309e735812e3063d45cbfcaa0b7eb0aae54c5f4d5b38ae996fda77a5249bb81
+size 637171

last-checkpoint/global_step3099/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44a5979aa592927362fe11f24be0664036c310ebafc765d001d928553c447cd0
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step3049
+global_step3099

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:381f090b12cbb3fcce976bd2e72d07a7786154c2848cc881d75715648e7c4dc2
+oid sha256:f819cf2c7fe8719d427eaa6ec9775983252517a512a5eb46040621049bb676c6
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4321dc1beb6c245d4a16a8fb2bed2ce2a40d89e5a9c611c4572f63a09523846c
+oid sha256:2a78180f5e29cd98437c35591ab402d42bd98c8a12a813c23dfdfe59b5a850b5
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b79020117e301cd96518c9d1e3eb43a609bc85799f14bac63d9c572fc04cb89
+oid sha256:5f43520b5104ee75a59749a66f5367557a7f191243b389309b6acad668af81d2
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5429623c7323b4a820ea3d76194bc456c2affeeb1f3af8978aec5aee11b2d1ef
+oid sha256:2c502c4157180ff49243cd3eeb13c063be673f3bf094a73b62e8315ca6f65442
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:345d0f997c1fbe695e06c254895683e58eecedd54abdbc10558bebd9f3e1183b
+oid sha256:c9fee80c953c89df2c10ece901ddfacf40bd3457db41f37010a06f05c3916293
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.18780523538589478,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2650",
-  "epoch": 1.5706927633273242,
+  "epoch": 1.596446046870976,
   "eval_steps": 50,
-  "global_step": 3050,
+  "global_step": 3100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5436,11 +5436,100 @@
       "eval_steps_per_second": 0.745,
       "num_input_tokens_seen": 35666976,
       "step": 3050
+    },
+    {
+      "epoch": 1.5732680916816895,
+      "grad_norm": 2.403638244773669,
+      "learning_rate": 1.1908545995993253e-05,
+      "loss": 0.1686,
+      "num_input_tokens_seen": 35725456,
+      "step": 3055
+    },
+    {
+      "epoch": 1.5758434200360547,
+      "grad_norm": 14.854500855372818,
+      "learning_rate": 1.1770903573173398e-05,
+      "loss": 0.2366,
+      "num_input_tokens_seen": 35783912,
+      "step": 3060
+    },
+    {
+      "epoch": 1.5784187483904197,
+      "grad_norm": 4.505603869422207,
+      "learning_rate": 1.1633955034324507e-05,
+      "loss": 0.2029,
+      "num_input_tokens_seen": 35842376,
+      "step": 3065
+    },
+    {
+      "epoch": 1.580994076744785,
+      "grad_norm": 6.561062574184342,
+      "learning_rate": 1.1497702865155274e-05,
+      "loss": 0.2116,
+      "num_input_tokens_seen": 35900824,
+      "step": 3070
+    },
+    {
+      "epoch": 1.58356940509915,
+      "grad_norm": 3.075946215121841,
+      "learning_rate": 1.1362149538734745e-05,
+      "loss": 0.1676,
+      "num_input_tokens_seen": 35959264,
+      "step": 3075
+    },
+    {
+      "epoch": 1.5861447334535153,
+      "grad_norm": 3.932942100960138,
+      "learning_rate": 1.1227297515447543e-05,
+      "loss": 0.1811,
+      "num_input_tokens_seen": 36017760,
+      "step": 3080
+    },
+    {
+      "epoch": 1.5887200618078805,
+      "grad_norm": 2.914398659228269,
+      "learning_rate": 1.1093149242949152e-05,
+      "loss": 0.1298,
+      "num_input_tokens_seen": 36076280,
+      "step": 3085
+    },
+    {
+      "epoch": 1.5912953901622457,
+      "grad_norm": 7.36077869474613,
+      "learning_rate": 1.0959707156121506e-05,
+      "loss": 0.1979,
+      "num_input_tokens_seen": 36134784,
+      "step": 3090
+    },
+    {
+      "epoch": 1.593870718516611,
+      "grad_norm": 7.180782753385591,
+      "learning_rate": 1.0826973677028767e-05,
+      "loss": 0.2539,
+      "num_input_tokens_seen": 36193288,
+      "step": 3095
+    },
+    {
+      "epoch": 1.596446046870976,
+      "grad_norm": 5.80837569754525,
+      "learning_rate": 1.0694951214873427e-05,
+      "loss": 0.2139,
+      "num_input_tokens_seen": 36251744,
+      "step": 3100
+    },
+    {
+      "epoch": 1.596446046870976,
+      "eval_loss": 0.2746882140636444,
+      "eval_runtime": 20.8164,
+      "eval_samples_per_second": 2.882,
+      "eval_steps_per_second": 0.721,
+      "num_input_tokens_seen": 36251744,
+      "step": 3100
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 35666976,
+  "num_input_tokens_seen": 36251744,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5455,7 +5544,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2353472555974656.0,
+  "total_flos": 2392060569321472.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null