Training in progress, step 1350, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f02e158d7a0ecd4f15ebd388f3f9321f3c48f48b6c240184415bff2106b4225
+oid sha256:6aef5102eef70c31f0044c654484ffa32b1a0562551889340924debb001cc356
 size 29034840

last-checkpoint/global_step1350/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91bca888e4c33e24509ed239e1cace5cfcc74b657e45601aa17195a818a995d9
+size 43429616

last-checkpoint/global_step1350/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4ab6b925af54924e652426e5130c56f13868f1543ea6c23dcedb092eada1390
+size 43429616

last-checkpoint/global_step1350/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4bff5c2841762d4c5f5f2f84c319b2519954e346f9b70ea4c14e9a1f980b3e7
+size 43429616

last-checkpoint/global_step1350/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e3d62e714ef5a158d1249d0f9b65bb0f82f150761b83dc2cf2e0aa76b322d75
+size 43429616

last-checkpoint/global_step1350/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf6a04ed575d4bbe95b19e28c451beac968995fe7f81b545d3f82185be8aab33
+size 637299

last-checkpoint/global_step1350/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6797229513b11978627019edbf1729dfa52f86c1151aac9777d9693a2be4536b
+size 637171

last-checkpoint/global_step1350/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3dff6f8ada3bacd5905b06ee331028a57bd1b141fb55f651aca5cbe974a3a95
+size 637171

last-checkpoint/global_step1350/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0cfc18b2fc2a54e8761219cee72f1cb08510e09bfc522456e8dbdebcd2dde79
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1300
+global_step1350

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c364a094b8b4b8d6b015687012206e88b2233dd7d6a4f6f395d7aef77752ea67
+oid sha256:9747fe881253e52a47314f48068ef9649032bec4cb284b1b4becbb8787f37faa
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2bd2e297a13ac051fc5f3cce9c34767e51a5cb4574835aa8bd1309d8cdc48053
+oid sha256:3ac02a5554a5ef9e3473dcd2926626ae41f4777354859c7d2bf0a0c1188c0583
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:028047dd1753d92e11bf971ee14a5c981a9a3ea6631f228e38475027eb5ae430
+oid sha256:9405d230cc78dac3f3b2ab887674631c15f66fedab0042ab7bc1bd83b8575344
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:000eb4cb4096582f15856f380d5f2a9a00eaecdbb95f2289a7a81a0a624fdf72
+oid sha256:b15237547030ac62d49d70a5465b2e29515e6334f62416eb16c0c6d073f7c6bf
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d597028ddc0c214d0751afd211631bdacdc9a86d2364a66b2294abb4eba3117
+oid sha256:4c2345b3e418be58bce3abfab9e2138cf24cde9ab37ff4c2ee4ae37ffd9029bf
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.49471279978752136,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1300",
-  "epoch": 0.6695853721349472,
+  "epoch": 0.695338655678599,
   "eval_steps": 50,
-  "global_step": 1300,
+  "global_step": 1350,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2321,11 +2321,100 @@
       "eval_steps_per_second": 0.778,
       "num_input_tokens_seen": 15205064,
       "step": 1300
+    },
+    {
+      "epoch": 0.6721607004893124,
+      "grad_norm": 10.254593495214214,
+      "learning_rate": 7.925467901247996e-05,
+      "loss": 0.4589,
+      "num_input_tokens_seen": 15263560,
+      "step": 1305
+    },
+    {
+      "epoch": 0.6747360288436776,
+      "grad_norm": 10.097755490465826,
+      "learning_rate": 7.908166355425475e-05,
+      "loss": 0.457,
+      "num_input_tokens_seen": 15322016,
+      "step": 1310
+    },
+    {
+      "epoch": 0.6773113571980427,
+      "grad_norm": 10.727169120741898,
+      "learning_rate": 7.890812024415555e-05,
+      "loss": 0.51,
+      "num_input_tokens_seen": 15380504,
+      "step": 1315
+    },
+    {
+      "epoch": 0.6798866855524079,
+      "grad_norm": 11.12869176509039,
+      "learning_rate": 7.873405223211087e-05,
+      "loss": 0.4994,
+      "num_input_tokens_seen": 15438944,
+      "step": 1320
+    },
+    {
+      "epoch": 0.6824620139067731,
+      "grad_norm": 8.499688286725998,
+      "learning_rate": 7.855946267757295e-05,
+      "loss": 0.4501,
+      "num_input_tokens_seen": 15497384,
+      "step": 1325
+    },
+    {
+      "epoch": 0.6850373422611383,
+      "grad_norm": 10.352066646699223,
+      "learning_rate": 7.838435474946034e-05,
+      "loss": 0.4807,
+      "num_input_tokens_seen": 15555856,
+      "step": 1330
+    },
+    {
+      "epoch": 0.6876126706155035,
+      "grad_norm": 14.714795422962215,
+      "learning_rate": 7.820873162610044e-05,
+      "loss": 0.5112,
+      "num_input_tokens_seen": 15614368,
+      "step": 1335
+    },
+    {
+      "epoch": 0.6901879989698687,
+      "grad_norm": 8.466874504995866,
+      "learning_rate": 7.803259649517178e-05,
+      "loss": 0.4825,
+      "num_input_tokens_seen": 15672864,
+      "step": 1340
+    },
+    {
+      "epoch": 0.6927633273242338,
+      "grad_norm": 7.62934190428385,
+      "learning_rate": 7.78559525536462e-05,
+      "loss": 0.5147,
+      "num_input_tokens_seen": 15731376,
+      "step": 1345
+    },
+    {
+      "epoch": 0.695338655678599,
+      "grad_norm": 9.019045929732858,
+      "learning_rate": 7.767880300773074e-05,
+      "loss": 0.4702,
+      "num_input_tokens_seen": 15789848,
+      "step": 1350
+    },
+    {
+      "epoch": 0.695338655678599,
+      "eval_loss": 0.5247787237167358,
+      "eval_runtime": 19.436,
+      "eval_samples_per_second": 3.087,
+      "eval_steps_per_second": 0.772,
+      "num_input_tokens_seen": 15789848,
+      "step": 1350
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 15205064,
+  "num_input_tokens_seen": 15789848,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2340,7 +2429,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1003241463611392.0,
+  "total_flos": 1041830008324096.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null