Training in progress, step 2450, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9d5ec3c39553916ddeed05ebc73c5ecc3358eb275ee803834c3cbefd37b2102
+oid sha256:e815dfd4544e66028785a2440a38262f9d7e3b347ea1c255fd921b0ddf689205
 size 29034840

last-checkpoint/global_step2449/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df913b606dc17e04f911185b2e90ac4c6d2f835c77acde131fac03606ba59b57
+size 43429616

last-checkpoint/global_step2449/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40bf06dc89bc81ae43dd663a01a967bb52ec7e73cc89537eae97f9ecc404c291
+size 43429616

last-checkpoint/global_step2449/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea547cde4ba847bc195d4961853f5087a4f24ce42d0ed6863652de7516898a41
+size 43429616

last-checkpoint/global_step2449/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:854eb496733ec15bd9c4450b945123107d61526ca981135a7ae727e338ff167e
+size 43429616

last-checkpoint/global_step2449/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a82c5df390d3257a5352379097918600d041610d4e5ba1e974b3e229bdd67a8c
+size 637299

last-checkpoint/global_step2449/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97da9743c1e36ab23675aa22a5b8411e2e0ed805854ceca8f076e43f31695414
+size 637171

last-checkpoint/global_step2449/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a135547822e5461284280560c9a7414587b25a0dc806306e40ba6b78474ea00e
+size 637171

last-checkpoint/global_step2449/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a161b670106cc9758bc9709b5057592d4ed400d53d5d1dc95f84e4f1e869d6ad
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2399
+global_step2449

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b8aff7a1897a7eaf48c78ea1f8115c061edfa2b6fa42280e2c1c58fe66b1f8a
+oid sha256:a97c73c15a2a5b2de7dc426a700b2053aee43809425431c513cc5e3aab6c2107
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d73a7524f07999ef35d5d9b107dcc1678eae2ada841644e1bd00ec0734368c2
+oid sha256:1296b339c1b16ab7e14352a269004d20ede428aef748283fb0a6650d62f58129
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:149a2ed30e88bf94d622f8d7693f382286a49ac536a3f63efc50cab63f6b9f39
+oid sha256:000b1637f5e73170f2337500a6a083df3a43d967d642b6c3a68f60deb6c3b960
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:18f12c1b5aae2b7d4bb968649839fc7ff1ce6131508baad4b633693b04cee910
+oid sha256:b8bcb6e7802f6d888bc099642911087298cfb1adf7053a2d43a67192a53404ef
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6242510adeb4d59877ccb2f625a69ecf628b2bfccb966d6143251f5bc2806c24
+oid sha256:ca831f953944cb39f4c82163568f885af5876e2e989b1167b35966dbf1db8e86
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.22014015913009644,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2250",
-  "epoch": 1.2359000772598505,
+  "epoch": 1.2616533608035025,
   "eval_steps": 50,
-  "global_step": 2400,
+  "global_step": 2450,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4279,11 +4279,100 @@
       "eval_steps_per_second": 0.755,
       "num_input_tokens_seen": 28064552,
       "step": 2400
+    },
+    {
+      "epoch": 1.2384754056142158,
+      "grad_norm": 16.379510205091783,
+      "learning_rate": 3.463831801557577e-05,
+      "loss": 0.1933,
+      "num_input_tokens_seen": 28123016,
+      "step": 2405
+    },
+    {
+      "epoch": 1.241050733968581,
+      "grad_norm": 3.6631986381781556,
+      "learning_rate": 3.443574263935062e-05,
+      "loss": 0.2014,
+      "num_input_tokens_seen": 28181464,
+      "step": 2410
+    },
+    {
+      "epoch": 1.2436260623229463,
+      "grad_norm": 5.654529592966364,
+      "learning_rate": 3.42334497649294e-05,
+      "loss": 0.2383,
+      "num_input_tokens_seen": 28239920,
+      "step": 2415
+    },
+    {
+      "epoch": 1.2462013906773113,
+      "grad_norm": 14.005033039787135,
+      "learning_rate": 3.403144306406466e-05,
+      "loss": 0.2074,
+      "num_input_tokens_seen": 28298432,
+      "step": 2420
+    },
+    {
+      "epoch": 1.2487767190316765,
+      "grad_norm": 7.470199127973674,
+      "learning_rate": 3.382972620331475e-05,
+      "loss": 0.2202,
+      "num_input_tokens_seen": 28356904,
+      "step": 2425
+    },
+    {
+      "epoch": 1.2513520473860418,
+      "grad_norm": 3.9239650586287484,
+      "learning_rate": 3.362830284397716e-05,
+      "loss": 0.1756,
+      "num_input_tokens_seen": 28415384,
+      "step": 2430
+    },
+    {
+      "epoch": 1.2539273757404068,
+      "grad_norm": 32.67443269914418,
+      "learning_rate": 3.342717664202223e-05,
+      "loss": 0.2564,
+      "num_input_tokens_seen": 28473848,
+      "step": 2435
+    },
+    {
+      "epoch": 1.256502704094772,
+      "grad_norm": 8.79661830656602,
+      "learning_rate": 3.322635124802658e-05,
+      "loss": 0.242,
+      "num_input_tokens_seen": 28532312,
+      "step": 2440
+    },
+    {
+      "epoch": 1.2590780324491373,
+      "grad_norm": 9.41003020617472,
+      "learning_rate": 3.3025830307107035e-05,
+      "loss": 0.3455,
+      "num_input_tokens_seen": 28590784,
+      "step": 2445
+    },
+    {
+      "epoch": 1.2616533608035025,
+      "grad_norm": 23.884422578805488,
+      "learning_rate": 3.2825617458854376e-05,
+      "loss": 0.3076,
+      "num_input_tokens_seen": 28649256,
+      "step": 2450
+    },
+    {
+      "epoch": 1.2616533608035025,
+      "eval_loss": 0.2371988147497177,
+      "eval_runtime": 19.6997,
+      "eval_samples_per_second": 3.046,
+      "eval_steps_per_second": 0.761,
+      "num_input_tokens_seen": 28649256,
+      "step": 2450
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 28064552,
+  "num_input_tokens_seen": 28649256,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4298,7 +4387,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1851822579122176.0,
+  "total_flos": 1890404691345408.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null