Training in progress, step 600, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:82e427cc7e624394004d64620be182b50bd10e2a67a34cf622de25a3f138b7e0
+oid sha256:168cc62ce0f9bc823e9f05cfca486c4f8b12cfdb3adf1b70687137fa417f7b65
 size 29034840

last-checkpoint/global_step600/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f9fe0148996918ae64e3daf55de125922b1aac01d65237cb6d580396eb5edb5
+size 43429616

last-checkpoint/global_step600/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:981a666d180eb07860966c360bb3546e8d109fe8e734a3cc93556dd84b087068
+size 43429616

last-checkpoint/global_step600/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:206025f90c8c7ddfc483b10a17cdbd7607d47453ab345d0d2adba2482f60148c
+size 43429616

last-checkpoint/global_step600/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36b63312a138941da8cb52882eb69e743ebac4a3d9a0f0e5c118e50224999076
+size 43429616

last-checkpoint/global_step600/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11e8027e9407df0ac39e2b3b0f9b391ceffcc0365bf7b524d551f5dbe3c76e79
+size 637299

last-checkpoint/global_step600/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c20dbc40aa3a9738159e35e517f2b9b468c6b1bee8cb810efadc972e0821f0fc
+size 637171

last-checkpoint/global_step600/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:585439023d752dde2525d7bdccd458f3fb79989825110ba63e0cce828093c299
+size 637171

last-checkpoint/global_step600/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e197c45213ad104a71f307e0f73afc54979e8843c65bd3e2a9fc618c43b98f06
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step550
+global_step600

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae78313eb528c8d3695eebaf4de3539bd0a0bc6ee18c66af1ee183442f1758a0
+oid sha256:a81e3916b1392c4c49afb171dee5415c15f5a5a5af8749b28195fcfa0596699c
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b38031f60d9e88601d369ef46bcdcf2b5b03f2cb4ba93853bcb2328df7ebb7c
+oid sha256:9a781038dd714b87b8adb1aac8dbc8217ceb607428a992133954ad522365236e
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f58092375c93d237cd0e3149aecfbf83e2acdae46279e07a32920d01cb507e64
+oid sha256:9446c3db15f382a5546f13622787fc99392a5e0bc8a9ca2da1838de7ab621a37
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83cd4bbff9962da7ec6787fcea8d65df7096917f9a5902e249ba7aee8887fe5f
+oid sha256:1f11e7a6b3faa884fc23044e3772ff9dd72c257f02e121665061e2a03d518bd9
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e969c9b0ecef9c1209a6397ff63db034af1cc51341323dc3dc14016347fe871
+oid sha256:c97a1e2f2542883d462e18c679fb75515cd51cbf96416fbbbdc7ed7d003e43a9
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.7909801602363586,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-550",
-  "epoch": 0.28328611898017,
+  "best_metric": 0.787663459777832,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-600",
+  "epoch": 0.3090394025238218,
   "eval_steps": 50,
-  "global_step": 550,
+  "global_step": 600,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -986,11 +986,100 @@
       "eval_steps_per_second": 0.755,
       "num_input_tokens_seen": 6432936,
       "step": 550
+    },
+    {
+      "epoch": 0.28586144733453517,
+      "grad_norm": 3.529163852540611,
+      "learning_rate": 9.7666055857936e-05,
+      "loss": 0.8264,
+      "num_input_tokens_seen": 6491400,
+      "step": 555
+    },
+    {
+      "epoch": 0.28843677568890036,
+      "grad_norm": 4.044590312854015,
+      "learning_rate": 9.760130083696595e-05,
+      "loss": 0.8456,
+      "num_input_tokens_seen": 6549872,
+      "step": 560
+    },
+    {
+      "epoch": 0.2910121040432655,
+      "grad_norm": 4.650808013267891,
+      "learning_rate": 9.75356818201724e-05,
+      "loss": 0.8032,
+      "num_input_tokens_seen": 6608296,
+      "step": 565
+    },
+    {
+      "epoch": 0.2935874323976307,
+      "grad_norm": 6.577223054225459,
+      "learning_rate": 9.746919999858492e-05,
+      "loss": 0.8081,
+      "num_input_tokens_seen": 6666768,
+      "step": 570
+    },
+    {
+      "epoch": 0.2961627607519959,
+      "grad_norm": 7.1732684079932545,
+      "learning_rate": 9.740185657889357e-05,
+      "loss": 0.8398,
+      "num_input_tokens_seen": 6725248,
+      "step": 575
+    },
+    {
+      "epoch": 0.29873808910636107,
+      "grad_norm": 21.451661035438484,
+      "learning_rate": 9.733365278342696e-05,
+      "loss": 0.8908,
+      "num_input_tokens_seen": 6783680,
+      "step": 580
+    },
+    {
+      "epoch": 0.30131341746072626,
+      "grad_norm": 4.031699151478832,
+      "learning_rate": 9.726458985013017e-05,
+      "loss": 0.8248,
+      "num_input_tokens_seen": 6842144,
+      "step": 585
+    },
+    {
+      "epoch": 0.3038887458150914,
+      "grad_norm": 3.45579530759462,
+      "learning_rate": 9.719466903254215e-05,
+      "loss": 0.829,
+      "num_input_tokens_seen": 6900656,
+      "step": 590
+    },
+    {
+      "epoch": 0.3064640741694566,
+      "grad_norm": 4.518719062630672,
+      "learning_rate": 9.712389159977307e-05,
+      "loss": 0.8269,
+      "num_input_tokens_seen": 6959128,
+      "step": 595
+    },
+    {
+      "epoch": 0.3090394025238218,
+      "grad_norm": 12.728221405806083,
+      "learning_rate": 9.705225883648121e-05,
+      "loss": 0.7997,
+      "num_input_tokens_seen": 7017576,
+      "step": 600
+    },
+    {
+      "epoch": 0.3090394025238218,
+      "eval_loss": 0.787663459777832,
+      "eval_runtime": 19.6121,
+      "eval_samples_per_second": 3.059,
+      "eval_steps_per_second": 0.765,
+      "num_input_tokens_seen": 7017576,
+      "step": 600
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 6432936,
+  "num_input_tokens_seen": 7017576,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -1005,7 +1094,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 424412082339840.0,
+  "total_flos": 462990180876288.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null