Training in progress, step 400, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:613cdc2baa395a9cb64ef1794cbc658fcd70f510e6627131b053d1c268735db7
+oid sha256:d1ad4a76028a7f83ee44275d3e3a7c646bb4cd4ceb75431117f2d8b0f219b774
 size 29034840

last-checkpoint/global_step400/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acf90ffece8a7f06b8cffc8e83d916754e6dd1c79e6cc5a7f76683610f413ae0
+size 43429616

last-checkpoint/global_step400/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5587702d9ee2de9a94b9b3799aa31bf78e76373bee6164a560bab2343aa2321
+size 43429616

last-checkpoint/global_step400/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e87f899c238a56abcdcf140d5320ff1671db1f6aa6e7fa46d7af9798fbfabac
+size 43429616

last-checkpoint/global_step400/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1764bcf91937cec4e4ec99fd961319ee759e1950cb286d2e7b485f4dcf0015af
+size 43429616

last-checkpoint/global_step400/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5156b71863558eb940883c2a53abc6ea0ecf35171e1c6ef6fd48a17669031953
+size 637299

last-checkpoint/global_step400/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c22d7604f5e9a963cc0beedf938177c8d8489af65df205ebac4c98a1b8ec451
+size 637171

last-checkpoint/global_step400/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3741599f9c6a30d8f169f2bdf2dbef5196058896ed7f387c615c11e68675c1bf
+size 637171

last-checkpoint/global_step400/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:945fe57eb537c5ffd3d2e0d80ed5dfc750e49073d3d12e111bfb6c6bd93acce2
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step350
+global_step400

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee97cd82dba4d425fdd8dfdb88d4a43d0d4b1979b5c81ab4a24914fb00d4f332
+oid sha256:543ef05f530d40ee20b8d626b07a69b86597aca643e48897571062f973efe84f
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91dad95440fb85dc4a31745642117165c1a72173b2e389679ea8c0b2b6fcd7e2
+oid sha256:7a23f732e43838ce0398d2636885ac16badbb9bcbc04d1406069ba3027bc5ae0
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:98698326b023c2af02c94f18726ce52c7f7a6fe290734dd7edbe99bc807fcfa0
+oid sha256:e10cce960e7068b051c05e35ed6160656be9091c63f13796ac2ed7e9c84e5a72
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:708e7c6b5bf8a327e688779ebc08830ce249928bcb1ff5c82b1b1d0bf6d2660b
+oid sha256:b6f6049e212b1df5cefc5d834afcd8cc052c73f1457449e9fe8a38d514f54078
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a6b1313b92e71cbfe6fd28b5f67fce5b969bc1141062c32ede2fbf27949b277
+oid sha256:7779b5dd7047c5846ea43ed002061c3588a7e29e0d9f0ca42655eab3f8410e99
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.8932263255119324,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-300",
-  "epoch": 0.1802729848055627,
+  "epoch": 0.20602626834921453,
   "eval_steps": 50,
-  "global_step": 350,
+  "global_step": 400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -630,11 +630,100 @@
       "eval_steps_per_second": 0.747,
       "num_input_tokens_seen": 4093688,
       "step": 350
+    },
+    {
+      "epoch": 0.1828483131599279,
+      "grad_norm": 0.5786050277605497,
+      "learning_rate": 9.953606121739074e-05,
+      "loss": 0.8795,
+      "num_input_tokens_seen": 4152160,
+      "step": 355
+    },
+    {
+      "epoch": 0.18542364151429308,
+      "grad_norm": 1.3309062985901938,
+      "learning_rate": 9.950666049947653e-05,
+      "loss": 0.9143,
+      "num_input_tokens_seen": 4210648,
+      "step": 360
+    },
+    {
+      "epoch": 0.18799896986865824,
+      "grad_norm": 0.5010297124723248,
+      "learning_rate": 9.947636120217155e-05,
+      "loss": 0.9164,
+      "num_input_tokens_seen": 4269136,
+      "step": 365
+    },
+    {
+      "epoch": 0.19057429822302344,
+      "grad_norm": 0.7250841632803818,
+      "learning_rate": 9.944516387542852e-05,
+      "loss": 0.9061,
+      "num_input_tokens_seen": 4327664,
+      "step": 370
+    },
+    {
+      "epoch": 0.19314962657738863,
+      "grad_norm": 0.4506280653909736,
+      "learning_rate": 9.941306908550005e-05,
+      "loss": 0.8873,
+      "num_input_tokens_seen": 4386120,
+      "step": 375
+    },
+    {
+      "epoch": 0.1957249549317538,
+      "grad_norm": 0.6467175538087946,
+      "learning_rate": 9.938007741492828e-05,
+      "loss": 0.9038,
+      "num_input_tokens_seen": 4444560,
+      "step": 380
+    },
+    {
+      "epoch": 0.19830028328611898,
+      "grad_norm": 0.6742654767461002,
+      "learning_rate": 9.934618946253437e-05,
+      "loss": 0.9116,
+      "num_input_tokens_seen": 4503016,
+      "step": 385
+    },
+    {
+      "epoch": 0.20087561164048418,
+      "grad_norm": 0.418516338281364,
+      "learning_rate": 9.931140584340761e-05,
+      "loss": 0.9023,
+      "num_input_tokens_seen": 4561496,
+      "step": 390
+    },
+    {
+      "epoch": 0.20345093999484934,
+      "grad_norm": 0.5738919793445436,
+      "learning_rate": 9.92757271888942e-05,
+      "loss": 0.8901,
+      "num_input_tokens_seen": 4619944,
+      "step": 395
+    },
+    {
+      "epoch": 0.20602626834921453,
+      "grad_norm": 0.3600152815973316,
+      "learning_rate": 9.923915414658587e-05,
+      "loss": 0.9033,
+      "num_input_tokens_seen": 4678384,
+      "step": 400
+    },
+    {
+      "epoch": 0.20602626834921453,
+      "eval_loss": 0.906301736831665,
+      "eval_runtime": 19.8079,
+      "eval_samples_per_second": 3.029,
+      "eval_steps_per_second": 0.757,
+      "num_input_tokens_seen": 4678384,
+      "step": 400
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 4093688,
+  "num_input_tokens_seen": 4678384,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -649,7 +738,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 270058588995584.0,
+  "total_flos": 308638862540800.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null