Training in progress, step 2500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e815dfd4544e66028785a2440a38262f9d7e3b347ea1c255fd921b0ddf689205
+oid sha256:9966048df26861231f31ce0ddfe797df85f054e16dbf2091f97f17eda09a0921
 size 29034840

last-checkpoint/global_step2499/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4b433ab564c44660c5768a12dcb6f566e0fb6e921d6519e63dfb5f600dd5ba8
+size 43429616

last-checkpoint/global_step2499/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d1ed86de19f51135c95073461ad5c3b27d9f7970bef2aa4bc784b3290ffc013
+size 43429616

last-checkpoint/global_step2499/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f82875e94321d319c1175271ac5e6a226fce428c7076c7b48a75d6b5af6c04e4
+size 43429616

last-checkpoint/global_step2499/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:301bcb66e17799f3ce082f82e7c18707562d9a3990239469cb0c702779e2af6b
+size 43429616

last-checkpoint/global_step2499/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b68057968ae8a6712b220c63c51737d021b970e8bc7eae3b478483df645f816b
+size 637299

last-checkpoint/global_step2499/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e556ac7579ecbafd661a7651fefa61a7c19c100f36f7c9b8b31e0715c28cccc
+size 637171

last-checkpoint/global_step2499/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe1aa2ebfc9ea794e8a4e19de5c025e81597c8300f18fc96bb87266383e6cca9
+size 637171

last-checkpoint/global_step2499/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dab7388ad26aa2d426b25a026e976559019c9b2ae8f851138b711373b8827c01
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2449
+global_step2499

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a97c73c15a2a5b2de7dc426a700b2053aee43809425431c513cc5e3aab6c2107
+oid sha256:0362dfd92e8da01e4a0deedcbd1c493b8162d5d1d84d5a4c1cd210c556f2cf9b
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1296b339c1b16ab7e14352a269004d20ede428aef748283fb0a6650d62f58129
+oid sha256:e65c5adee1a22c5343e38495a6905880496fb22d5e3ec5b16b87aadb731969d2
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:000b1637f5e73170f2337500a6a083df3a43d967d642b6c3a68f60deb6c3b960
+oid sha256:f8d8858483b0c6944d55621cc2633469e3e0d04c48b6671eee92d4abab2352c2
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8bcb6e7802f6d888bc099642911087298cfb1adf7053a2d43a67192a53404ef
+oid sha256:ca42ef4f7a2f8c2285c4cf6cef585dcc0b132b21e8bb33d96d53b6db837f5e54
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca831f953944cb39f4c82163568f885af5876e2e989b1167b35966dbf1db8e86
+oid sha256:d6bfc8f82217a5679ff2e12a0352f5107734a71e0cf25a7278e2a7645ea398cd
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.22014015913009644,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2250",
-  "epoch": 1.2616533608035025,
+  "epoch": 1.2874066443471541,
   "eval_steps": 50,
-  "global_step": 2450,
+  "global_step": 2500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4368,11 +4368,100 @@
       "eval_steps_per_second": 0.761,
       "num_input_tokens_seen": 28649256,
       "step": 2450
+    },
+    {
+      "epoch": 1.2642286891578676,
+      "grad_norm": 13.48575395754194,
+      "learning_rate": 3.2625716337267295e-05,
+      "loss": 0.2041,
+      "num_input_tokens_seen": 28707736,
+      "step": 2455
+    },
+    {
+      "epoch": 1.2668040175122328,
+      "grad_norm": 3.1641318472119986,
+      "learning_rate": 3.242613057068641e-05,
+      "loss": 0.3022,
+      "num_input_tokens_seen": 28766232,
+      "step": 2460
+    },
+    {
+      "epoch": 1.269379345866598,
+      "grad_norm": 17.472204317681474,
+      "learning_rate": 3.222686378172847e-05,
+      "loss": 0.2756,
+      "num_input_tokens_seen": 28824688,
+      "step": 2465
+    },
+    {
+      "epoch": 1.271954674220963,
+      "grad_norm": 4.051662934956441,
+      "learning_rate": 3.2027919587220516e-05,
+      "loss": 0.2395,
+      "num_input_tokens_seen": 28883176,
+      "step": 2470
+    },
+    {
+      "epoch": 1.2745300025753283,
+      "grad_norm": 5.058807389542336,
+      "learning_rate": 3.1829301598134355e-05,
+      "loss": 0.2204,
+      "num_input_tokens_seen": 28941592,
+      "step": 2475
+    },
+    {
+      "epoch": 1.2771053309296936,
+      "grad_norm": 4.820344297526228,
+      "learning_rate": 3.1631013419520855e-05,
+      "loss": 0.2477,
+      "num_input_tokens_seen": 29000064,
+      "step": 2480
+    },
+    {
+      "epoch": 1.2796806592840588,
+      "grad_norm": 6.098120285465299,
+      "learning_rate": 3.143305865044467e-05,
+      "loss": 0.2645,
+      "num_input_tokens_seen": 29058496,
+      "step": 2485
+    },
+    {
+      "epoch": 1.2822559876384239,
+      "grad_norm": 10.021136804794267,
+      "learning_rate": 3.123544088391881e-05,
+      "loss": 0.2184,
+      "num_input_tokens_seen": 29116992,
+      "step": 2490
+    },
+    {
+      "epoch": 1.284831315992789,
+      "grad_norm": 32.07716009409277,
+      "learning_rate": 3.10381637068395e-05,
+      "loss": 0.1857,
+      "num_input_tokens_seen": 29175496,
+      "step": 2495
+    },
+    {
+      "epoch": 1.2874066443471541,
+      "grad_norm": 10.697788786097991,
+      "learning_rate": 3.084123069992096e-05,
+      "loss": 0.257,
+      "num_input_tokens_seen": 29233968,
+      "step": 2500
+    },
+    {
+      "epoch": 1.2874066443471541,
+      "eval_loss": 0.2488754242658615,
+      "eval_runtime": 19.5416,
+      "eval_samples_per_second": 3.07,
+      "eval_steps_per_second": 0.768,
+      "num_input_tokens_seen": 29233968,
+      "step": 2500
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 28649256,
+  "num_input_tokens_seen": 29233968,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4387,7 +4476,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1890404691345408.0,
+  "total_flos": 1928988327673856.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null