Training in progress, step 2150, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a111e8798c83c643eeba02e2a149e66e3437bb97bb843d76a35308962c3f1d78
+oid sha256:2034eebd508e7ac8ecb0a317db22f0ed34690e812dc78be4e22f97f074447d0f
 size 29034840

last-checkpoint/global_step2149/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76a834493c770996dd31bdf289d8d416720d9a0c1163a4a28ce363bfb9f56125
+size 43429616

last-checkpoint/global_step2149/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66ae8328db5cbe4350cadbe9b03e5f94da9266623aaad8962b9c3013a29fa6c9
+size 43429616

last-checkpoint/global_step2149/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67ffae581e43860255f53fc49344f4c9abc8aee6837d3d4073ea36f38bd660de
+size 43429616

last-checkpoint/global_step2149/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb9994cac84231dddba4ebe94b420877e83b181dd1edf79cf1d2a5be45661faa
+size 43429616

last-checkpoint/global_step2149/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f000a8026012dca201de2d8edd792d3f11c18221cb0b0ca2bd6c4102d3442db7
+size 637299

last-checkpoint/global_step2149/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93290cbef16db999968f1763b76a46df6e094b05b64c8ac94f7e82559e0e8bcf
+size 637171

last-checkpoint/global_step2149/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf708fc6c44da8bc3d14d7da12c3598394376bcf4601fa8e8035ef28bbd430db
+size 637171

last-checkpoint/global_step2149/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea2fbbc7294531a75063dddb972957a050968b045541575f6b4e3e81281fc0fb
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2099
+global_step2149

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b105708e2c99c8661b46698b8ccc5799ac83c1f0fc6a30c2d41c9fbfb349d480
+oid sha256:1c2f72d01585273766959f0cc9805fab753b53f20e581399855a293176ace988
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc44cd2015d8c8fc2f109f07c797876873a52f478c57b0350b8a2cf5dcb17f25
+oid sha256:3fd1ecda2bb159be37a2a23800e098324f5b0334e7189df47c343ca6cb7605a2
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9843dec201b5a542ebd69abfc596f99ad5a000cf81dab52c6a2c52a5b9224ea7
+oid sha256:cf71c84ea2995fbc545b918d03f7f94c92293ca2e33343f177e6fd04531b7b19
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2296f0efda653dd4c7e861f5a867baa09d6d8bb50e57bc69af930268b40de9ef
+oid sha256:72c53116f0f4c80841c24cd681d5fbd5a5992b259583a4cfb493f8f3e4544d82
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e06f479f50d6b90756aab0e5f1d2b8944be9014d46f9719fade9db1f0995f9b
+oid sha256:2a5fa7220e971f22064d8e3cc972d3be539d9ac37e650399fd40813d74479992
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.2472737729549408,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1950",
-  "epoch": 1.0813803759979397,
+  "epoch": 1.1071336595415915,
   "eval_steps": 50,
-  "global_step": 2100,
+  "global_step": 2150,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3745,11 +3745,100 @@
       "eval_steps_per_second": 0.774,
       "num_input_tokens_seen": 24555904,
       "step": 2100
+    },
+    {
+      "epoch": 1.083955704352305,
+      "grad_norm": 10.785937986790138,
+      "learning_rate": 4.716837353150695e-05,
+      "loss": 0.2942,
+      "num_input_tokens_seen": 24614392,
+      "step": 2105
+    },
+    {
+      "epoch": 1.0865310327066702,
+      "grad_norm": 7.201431051550072,
+      "learning_rate": 4.695572353154438e-05,
+      "loss": 0.2357,
+      "num_input_tokens_seen": 24672848,
+      "step": 2110
+    },
+    {
+      "epoch": 1.0891063610610352,
+      "grad_norm": 10.470411807057468,
+      "learning_rate": 4.674312878725985e-05,
+      "loss": 0.2476,
+      "num_input_tokens_seen": 24731344,
+      "step": 2115
+    },
+    {
+      "epoch": 1.0916816894154004,
+      "grad_norm": 10.029698195020993,
+      "learning_rate": 4.653059315739188e-05,
+      "loss": 0.3875,
+      "num_input_tokens_seen": 24789832,
+      "step": 2120
+    },
+    {
+      "epoch": 1.0942570177697657,
+      "grad_norm": 5.623217865601237,
+      "learning_rate": 4.631812049960595e-05,
+      "loss": 0.3313,
+      "num_input_tokens_seen": 24848264,
+      "step": 2125
+    },
+    {
+      "epoch": 1.0968323461241307,
+      "grad_norm": 4.705669392169022,
+      "learning_rate": 4.61057146704246e-05,
+      "loss": 0.2385,
+      "num_input_tokens_seen": 24906744,
+      "step": 2130
+    },
+    {
+      "epoch": 1.099407674478496,
+      "grad_norm": 9.334998248379781,
+      "learning_rate": 4.589337952515736e-05,
+      "loss": 0.2478,
+      "num_input_tokens_seen": 24965256,
+      "step": 2135
+    },
+    {
+      "epoch": 1.1019830028328612,
+      "grad_norm": 8.45475259780102,
+      "learning_rate": 4.5681118917830835e-05,
+      "loss": 0.214,
+      "num_input_tokens_seen": 25023768,
+      "step": 2140
+    },
+    {
+      "epoch": 1.1045583311872265,
+      "grad_norm": 23.234169875193494,
+      "learning_rate": 4.546893670111866e-05,
+      "loss": 0.2806,
+      "num_input_tokens_seen": 25082248,
+      "step": 2145
+    },
+    {
+      "epoch": 1.1071336595415915,
+      "grad_norm": 7.852580632471935,
+      "learning_rate": 4.525683672627168e-05,
+      "loss": 0.2536,
+      "num_input_tokens_seen": 25140752,
+      "step": 2150
+    },
+    {
+      "epoch": 1.1071336595415915,
+      "eval_loss": 0.30806976556777954,
+      "eval_runtime": 19.3086,
+      "eval_samples_per_second": 3.107,
+      "eval_steps_per_second": 0.777,
+      "num_input_tokens_seen": 25140752,
+      "step": 2150
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 24555904,
+  "num_input_tokens_seen": 25140752,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -3764,7 +3853,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1620301350436864.0,
+  "total_flos": 1658893279166464.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null