Training in progress, step 2550, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9966048df26861231f31ce0ddfe797df85f054e16dbf2091f97f17eda09a0921
+oid sha256:4309cbe22308fad27e8b5e7c85714a164b3e5d38d0b75c9561151c809911db4b
 size 29034840

last-checkpoint/global_step2549/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:116250004a1929d0c076717857b44333856b786e32f05ae9fbc0307de4a8b032
+size 43429616

last-checkpoint/global_step2549/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c91c4e6a7ea34a84805bc2387f121f7034be970ba6d4cd8ddfe0f6ecc08f4b6
+size 43429616

last-checkpoint/global_step2549/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad76f92356447842720adbce3c984e60959184fe2dcd8166a73004defe8aafbd
+size 43429616

last-checkpoint/global_step2549/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f82f3269a7588c654b27ac5eba958161192da8dfa1af90b9d5c411ab43e7c18c
+size 43429616

last-checkpoint/global_step2549/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:149016bb2c0c5b8c8a4e0a7bc0cee8f6b5193b3ab8371a46b6db6818f5879d1b
+size 637299

last-checkpoint/global_step2549/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d7c40326f16894d1ff25c39d8dba54d5a1be41123dce9090e3e658e3905944b
+size 637171

last-checkpoint/global_step2549/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e27bc879b7b5595e9c7b5a57dec365bece22370f9b4e617a4d6945910a5409d9
+size 637171

last-checkpoint/global_step2549/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:963088ee6518684b564cfdf26a0ea840b35584a38d3167e45f610f6198373461
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2499
+global_step2549

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0362dfd92e8da01e4a0deedcbd1c493b8162d5d1d84d5a4c1cd210c556f2cf9b
+oid sha256:91caf226cd72269cf1248d5f2f1a1d0c92100844814086cd6878cfb06293dc0c
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e65c5adee1a22c5343e38495a6905880496fb22d5e3ec5b16b87aadb731969d2
+oid sha256:3a2264023ead4b957c8737f57b7f130753adf9857726b38e1a7c000a4c721672
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8d8858483b0c6944d55621cc2633469e3e0d04c48b6671eee92d4abab2352c2
+oid sha256:3af91d2c27fa6123938cc5723642d3bc4b5f5e9e58a6f0d15fdf45fd12cb685e
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca42ef4f7a2f8c2285c4cf6cef585dcc0b132b21e8bb33d96d53b6db837f5e54
+oid sha256:57930146d6b40a72f7f121792ad7f144f1863c40a705666c20e64c1e1c54a94a
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6bfc8f82217a5679ff2e12a0352f5107734a71e0cf25a7278e2a7645ea398cd
+oid sha256:5184a5762c3654cd8e62f1aa51b8146cd229ac960662a48fb2daefc27ab2811c
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.22014015913009644,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2250",
-  "epoch": 1.2874066443471541,
+  "epoch": 1.3131599278908062,
   "eval_steps": 50,
-  "global_step": 2500,
+  "global_step": 2550,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4457,11 +4457,100 @@
       "eval_steps_per_second": 0.768,
       "num_input_tokens_seen": 29233968,
       "step": 2500
+    },
+    {
+      "epoch": 1.2899819727015194,
+      "grad_norm": 3.8096764598668864,
+      "learning_rate": 3.064464543763057e-05,
+      "loss": 0.1617,
+      "num_input_tokens_seen": 29292464,
+      "step": 2505
+    },
+    {
+      "epoch": 1.2925573010558846,
+      "grad_norm": 8.299858124153992,
+      "learning_rate": 3.0448411488123862e-05,
+      "loss": 0.2068,
+      "num_input_tokens_seen": 29350952,
+      "step": 2510
+    },
+    {
+      "epoch": 1.2951326294102499,
+      "grad_norm": 10.959075314673317,
+      "learning_rate": 3.0252532413179825e-05,
+      "loss": 0.2188,
+      "num_input_tokens_seen": 29409432,
+      "step": 2515
+    },
+    {
+      "epoch": 1.2977079577646151,
+      "grad_norm": 7.014945794108391,
+      "learning_rate": 3.0057011768136223e-05,
+      "loss": 0.1948,
+      "num_input_tokens_seen": 29467880,
+      "step": 2520
+    },
+    {
+      "epoch": 1.3002832861189801,
+      "grad_norm": 5.3992920228602665,
+      "learning_rate": 2.9861853101825094e-05,
+      "loss": 0.2366,
+      "num_input_tokens_seen": 29526384,
+      "step": 2525
+    },
+    {
+      "epoch": 1.3028586144733454,
+      "grad_norm": 7.369149060264004,
+      "learning_rate": 2.966705995650827e-05,
+      "loss": 0.2601,
+      "num_input_tokens_seen": 29584880,
+      "step": 2530
+    },
+    {
+      "epoch": 1.3054339428277104,
+      "grad_norm": 8.85591028977132,
+      "learning_rate": 2.9472635867813185e-05,
+      "loss": 0.1913,
+      "num_input_tokens_seen": 29643344,
+      "step": 2535
+    },
+    {
+      "epoch": 1.3080092711820757,
+      "grad_norm": 6.292489461251922,
+      "learning_rate": 2.927858436466861e-05,
+      "loss": 0.2,
+      "num_input_tokens_seen": 29701864,
+      "step": 2540
+    },
+    {
+      "epoch": 1.310584599536441,
+      "grad_norm": 10.28646529424139,
+      "learning_rate": 2.9084908969240664e-05,
+      "loss": 0.2503,
+      "num_input_tokens_seen": 29760376,
+      "step": 2545
+    },
+    {
+      "epoch": 1.3131599278908062,
+      "grad_norm": 6.3979585721541135,
+      "learning_rate": 2.8891613196868816e-05,
+      "loss": 0.2192,
+      "num_input_tokens_seen": 29818856,
+      "step": 2550
+    },
+    {
+      "epoch": 1.3131599278908062,
+      "eval_loss": 0.24317465722560883,
+      "eval_runtime": 19.477,
+      "eval_samples_per_second": 3.081,
+      "eval_steps_per_second": 0.77,
+      "num_input_tokens_seen": 29818856,
+      "step": 2550
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 29233968,
+  "num_input_tokens_seen": 29818856,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4476,7 +4565,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1928988327673856.0,
+  "total_flos": 1967582528405504.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null