Training in progress, step 2300, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9efa5a507b1e2d13741d6dd8d920f108bb4ba17026ae4bb85a703fca7207dad8
+oid sha256:928aa21b0c49c49462d0aedc8af44db37ae597ff96ee656e5763e930bfac1a19
 size 29034840

last-checkpoint/global_step2299/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8b88893fe1583c5ce33e461bf4ce59a1626ce53ff44ba50c21267848f362c07
+size 43429616

last-checkpoint/global_step2299/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21b29af43f75c1390f228bf54b7e627ffe91ec6cf44730afbc0ecaeafec7cc06
+size 43429616

last-checkpoint/global_step2299/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bdc5d2fa6d94db687b3268482eb07108d2c5fe4aace0924ed3981fc45fe3ce7
+size 43429616

last-checkpoint/global_step2299/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9eee0fa02aab662f6dd71836e00a139cf13412379a8590bd664cfe160c8e8a1d
+size 43429616

last-checkpoint/global_step2299/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3db9a66f062c06d4ae22ce00381d04aaf88febb31a1deb876ef8f5f7e037b0b
+size 637299

last-checkpoint/global_step2299/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d4d0f45bd74f8f11098175a2c697bf0d440274790a3e45d2430d91452939d2d
+size 637171

last-checkpoint/global_step2299/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e17a1e9ce61b5f6d7436055412d41d8bb09874b16c91328ac0303c620f7e7754
+size 637171

last-checkpoint/global_step2299/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a51f66021906f63ad256a9e41a3333820be76fc6433e1d490fb47829b67200bf
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2249
+global_step2299

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d25cbcbbaa0866ea9c7365cb49b84e805db119693e615f5a1898a6ebfe997e8
+oid sha256:f12bf3da75454e5aae4644f2a1d46fdf90f68e680dbf5bdaa86861f825d32d80
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a883389afac12125c2c6bf62631b7de0220fdb0020d24cd0c6e8f8858dd3b362
+oid sha256:2faef1b8798e7516fd96ee7b3363866a8f97ca2d0ec5a8dd27bbfe70b0c6a733
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:577d49de6d60035e159d9ebb1e6eabef79a55787b14ecea93a6a93c242661779
+oid sha256:7f35f6d27fbd414dd4285d91816b37fb6b97ff10fbba4c074d56ad2a7f723033
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b5f2dd1c21e06806a9ce39eeab45734dfb8a62b829f91a86d1f65f13102d6242
+oid sha256:ea5c64dfc7e3b2729065483dfef8e4bb0af0d9bae32df888d258ee3c2859d676
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9cbcebe45b9faaca29ead6f9f59deeeb362fed9bbabf34da80537e86d967e85
+oid sha256:9ba68b7498f6ef6c27be2e178ae1a286d6e6aacbd9a34a6f944fd623ea7cf386
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.22014015913009644,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2250",
-  "epoch": 1.158640226628895,
+  "epoch": 1.184393510172547,
   "eval_steps": 50,
-  "global_step": 2250,
+  "global_step": 2300,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4012,11 +4012,100 @@
       "eval_steps_per_second": 0.738,
       "num_input_tokens_seen": 26310336,
       "step": 2250
+    },
+    {
+      "epoch": 1.1612155549832603,
+      "grad_norm": 8.283787630987762,
+      "learning_rate": 4.0828536862897125e-05,
+      "loss": 0.2986,
+      "num_input_tokens_seen": 26368840,
+      "step": 2255
+    },
+    {
+      "epoch": 1.1637908833376256,
+      "grad_norm": 20.215982883136522,
+      "learning_rate": 4.061921681310241e-05,
+      "loss": 0.263,
+      "num_input_tokens_seen": 26427280,
+      "step": 2260
+    },
+    {
+      "epoch": 1.1663662116919906,
+      "grad_norm": 6.802291856358878,
+      "learning_rate": 4.04100670308708e-05,
+      "loss": 0.2282,
+      "num_input_tokens_seen": 26485760,
+      "step": 2265
+    },
+    {
+      "epoch": 1.1689415400463559,
+      "grad_norm": 8.985818465549714,
+      "learning_rate": 4.0201091312412394e-05,
+      "loss": 0.2311,
+      "num_input_tokens_seen": 26544224,
+      "step": 2270
+    },
+    {
+      "epoch": 1.1715168684007211,
+      "grad_norm": 8.383045025215052,
+      "learning_rate": 3.999229345077789e-05,
+      "loss": 0.2939,
+      "num_input_tokens_seen": 26602712,
+      "step": 2275
+    },
+    {
+      "epoch": 1.1740921967550864,
+      "grad_norm": 6.558546292884369,
+      "learning_rate": 3.978367723578981e-05,
+      "loss": 0.2165,
+      "num_input_tokens_seen": 26661160,
+      "step": 2280
+    },
+    {
+      "epoch": 1.1766675251094514,
+      "grad_norm": 7.108641756919162,
+      "learning_rate": 3.957524645397359e-05,
+      "loss": 0.1823,
+      "num_input_tokens_seen": 26719648,
+      "step": 2285
+    },
+    {
+      "epoch": 1.1792428534638166,
+      "grad_norm": 11.468609977960636,
+      "learning_rate": 3.936700488848899e-05,
+      "loss": 0.2827,
+      "num_input_tokens_seen": 26778112,
+      "step": 2290
+    },
+    {
+      "epoch": 1.1818181818181819,
+      "grad_norm": 11.548102910785317,
+      "learning_rate": 3.915895631906135e-05,
+      "loss": 0.2897,
+      "num_input_tokens_seen": 26836592,
+      "step": 2295
+    },
+    {
+      "epoch": 1.184393510172547,
+      "grad_norm": 4.826501140271565,
+      "learning_rate": 3.895110452191301e-05,
+      "loss": 0.2903,
+      "num_input_tokens_seen": 26895096,
+      "step": 2300
+    },
+    {
+      "epoch": 1.184393510172547,
+      "eval_loss": 0.2940039336681366,
+      "eval_runtime": 20.1001,
+      "eval_samples_per_second": 2.985,
+      "eval_steps_per_second": 0.746,
+      "num_input_tokens_seen": 26895096,
+      "step": 2300
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 26310336,
+  "num_input_tokens_seen": 26895096,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4031,7 +4120,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1736069012258816.0,
+  "total_flos": 1774654872354816.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null