Training in progress, step 500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7156b16f127d3e2296e6ada4ebf9cbb8c48381f16d088d0a874a522c40e99a16
+oid sha256:55382d84797424bf555d465074d1d58ecfa91cc23250d3509fdb9cd78d1cbeff
 size 29034840

last-checkpoint/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc106c6fc6d9c584cf59e5adf55bdf127c07781019ffa591f00cefaa2aa092a8
+size 43429616

last-checkpoint/global_step500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1206ffbd9ab0ca5b2df0bd5e6d788b9cbd993d61d56e60e0d2b4e38ea1bafd79
+size 43429616

last-checkpoint/global_step500/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95b13646a15e0a5bb9ffcadaa56a12480f686f005cc3f4f5b35ece1084e33102
+size 43429616

last-checkpoint/global_step500/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc8c9ef603b49122a9ba6b122660d8ee061f9e18ccb68b40d01da50ed3e11949
+size 43429616

last-checkpoint/global_step500/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:982b6b8112b1a68d2e38cae94a225f18bd9ea392e1654dbc7cc130776bff950b
+size 637299

last-checkpoint/global_step500/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:854e2832a5dae385b495dad0465c766c553cf722b470925f5286a7230114f652
+size 637171

last-checkpoint/global_step500/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da17084af120106739c69dcfe393381f051b6adbcd329aae7c5a39f5b06e3345
+size 637171

last-checkpoint/global_step500/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7066b428556bc0b3f73235845964ecb2a136bcb07c44570c346b66a0f2f5992
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step450
+global_step500

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7dbc6521b0b64cb12d818506108fcf257a4089ca8a9b1e453776ed3e032e7176
+oid sha256:bdd1f02cb20d3f4f7e0dd26fea62af57e5e71316163f926a28ed6cf89a9f3777
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b13e3da1b0679cab1bab94f893e385a9a224d3335b5a6f62602f33c2be88d03
+oid sha256:fc6d54ba2aa85e2f895439a1b787ec947b848a1c34ea5a3a28821572bf2b9fec
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6a24f0e0f117b5a8236e0d12594c0c358f41ef00068d4460002e95ad1cc3cb1c
+oid sha256:8b6927d26551cddd8e35b34b43e79bd58f8b6027b6a481bb6a563a3652addeb4
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e46e4eab6c4a25d84ad36ddf1357401788adeeb6388c03cefa35a63b52ee7610
+oid sha256:f8295b1be8e66b4b30cb905dc48cfc717c027e427937b8142d00ae9de8106c6a
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:581bd51b94cb26100e335b86a6da2b6d11272f4e7dfeb8188f963bb8c6dcbb27
+oid sha256:8779534db128a38380f8ecdd5b697bdb59a398f35fe4ad4de1318e871fcc0e6f
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.8932263255119324,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-300",
-  "epoch": 0.23177955189286634,
+  "best_metric": 0.8268976211547852,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-500",
+  "epoch": 0.25753283543651817,
   "eval_steps": 50,
-  "global_step": 450,
+  "global_step": 500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -808,11 +808,100 @@
       "eval_steps_per_second": 0.718,
       "num_input_tokens_seen": 5263304,
       "step": 450
+    },
+    {
+      "epoch": 0.23435488024723153,
+      "grad_norm": 0.7549819464827967,
+      "learning_rate": 9.877802252293474e-05,
+      "loss": 0.8891,
+      "num_input_tokens_seen": 5321760,
+      "step": 455
+    },
+    {
+      "epoch": 0.2369302086015967,
+      "grad_norm": 0.9960909370043465,
+      "learning_rate": 9.873077323567488e-05,
+      "loss": 0.9026,
+      "num_input_tokens_seen": 5380224,
+      "step": 460
+    },
+    {
+      "epoch": 0.23950553695596188,
+      "grad_norm": 1.5888386521989892,
+      "learning_rate": 9.868263945190312e-05,
+      "loss": 0.8707,
+      "num_input_tokens_seen": 5438704,
+      "step": 465
+    },
+    {
+      "epoch": 0.24208086531032708,
+      "grad_norm": 3.0542478842411587,
+      "learning_rate": 9.863362204528024e-05,
+      "loss": 0.9051,
+      "num_input_tokens_seen": 5497208,
+      "step": 470
+    },
+    {
+      "epoch": 0.24465619366469224,
+      "grad_norm": 1.2908325061552137,
+      "learning_rate": 9.858372190550533e-05,
+      "loss": 0.8711,
+      "num_input_tokens_seen": 5555704,
+      "step": 475
+    },
+    {
+      "epoch": 0.24723152201905743,
+      "grad_norm": 3.1989324866235744,
+      "learning_rate": 9.853293993829969e-05,
+      "loss": 0.885,
+      "num_input_tokens_seen": 5614160,
+      "step": 480
+    },
+    {
+      "epoch": 0.24980685037342262,
+      "grad_norm": 3.591366302378185,
+      "learning_rate": 9.848127706539039e-05,
+      "loss": 0.8615,
+      "num_input_tokens_seen": 5672640,
+      "step": 485
+    },
+    {
+      "epoch": 0.2523821787277878,
+      "grad_norm": 2.053833335696007,
+      "learning_rate": 9.842873422449354e-05,
+      "loss": 0.9057,
+      "num_input_tokens_seen": 5731072,
+      "step": 490
+    },
+    {
+      "epoch": 0.254957507082153,
+      "grad_norm": 1.4501486574941083,
+      "learning_rate": 9.837531236929726e-05,
+      "loss": 0.8818,
+      "num_input_tokens_seen": 5789544,
+      "step": 495
+    },
+    {
+      "epoch": 0.25753283543651817,
+      "grad_norm": 2.1068404021122866,
+      "learning_rate": 9.832101246944439e-05,
+      "loss": 0.8576,
+      "num_input_tokens_seen": 5848048,
+      "step": 500
+    },
+    {
+      "epoch": 0.25753283543651817,
+      "eval_loss": 0.8268976211547852,
+      "eval_runtime": 19.6346,
+      "eval_samples_per_second": 3.056,
+      "eval_steps_per_second": 0.764,
+      "num_input_tokens_seen": 5848048,
+      "step": 500
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 5263304,
+  "num_input_tokens_seen": 5848048,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -827,7 +916,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 347233811955712.0,
+  "total_flos": 385817567035392.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null