Training in progress, step 2800, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dcf51604c7fbe68fb5bec81cac3b4632fb619bd371e32820e3fd872616836861
+oid sha256:2f209712b49d72a0a61d59f7b8f5e51be948aa2ecb02420dc257c2b1ba5b38a8
 size 29034840

last-checkpoint/global_step2799/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2046c56a4678d6e319a2f603d0b1e9398b496663d16ed2bf5a036ebfea22add
+size 43429616

last-checkpoint/global_step2799/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:370ccbaecefb9ce961e671a4165e47d7a69af72ff6915ddf34336361fff7160c
+size 43429616

last-checkpoint/global_step2799/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5c5eac2a9b2ebfbebab9b21261d5722ad12e898f434fa74e8730c56b7cb34e8
+size 43429616

last-checkpoint/global_step2799/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15279a29771bcf9661f5dd1bbdfe635434bf014fc851029796f713fc908492ca
+size 43429616

last-checkpoint/global_step2799/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb57e085d33592674e15e103c61ee81ee7e4de7c83d7b3166bbe37cbc7a3f507
+size 637299

last-checkpoint/global_step2799/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d5f662afab50ff642f02642e2e708fff5b1b0e4c174cd936f632115944c55d2
+size 637171

last-checkpoint/global_step2799/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f802cd49f46b34e8f236eac45af3046f74c89f9b8b521ac23f5c2e931c0055ab
+size 637171

last-checkpoint/global_step2799/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86ecedc8766b6b51fc2ce655bc577f9ed891c4aca1eb78f64ebdbdb3bc691604
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2749
+global_step2799

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49db5a9fd0c84d580c671e52905ebeffc155b36537e76ff966d2e82906708999
+oid sha256:ff8dba2341c0517760edfde50521977f02a5bd982ffd3bc03de6109439c4f478
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8ca224562d8d97aaa131b3516288bb99f68d7dcf62170494326662bda0bb206
+oid sha256:d2bf831df9fbade9ac2a8db79798bc2a7b1afb85a78a6e463ec7a7db4acc0f8e
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56e86a11c89dba78d60e1b2a1855a651b90a5a22ef131ce65d26af83668c154e
+oid sha256:8602ff0a0fa366d46b61c0ef2b23ce468387898cf2bc1027e5450de73ddf647f
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3dd63019c923e9692431619aced46b91aaf3fd22e1c22ec0a64347f2fe635a0e
+oid sha256:4bb51d675cf23603b1b765cd645f53d6b66ddb104d56d48674e9c798e086f696
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d5206e8eee1fd5c926f6f8750c100fc15237f73c48e6ae482a3b1e4fadc9091
+oid sha256:b2248ae5cd08b24968d749ba4f37513ee0a4cfb5059768755ed9ef6b0b1fd810
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.18780523538589478,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2650",
-  "epoch": 1.4161730620654134,
+  "epoch": 1.4419263456090652,
   "eval_steps": 50,
-  "global_step": 2750,
+  "global_step": 2800,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4902,11 +4902,100 @@
       "eval_steps_per_second": 0.769,
       "num_input_tokens_seen": 32158144,
       "step": 2750
+    },
+    {
+      "epoch": 1.4187483904197786,
+      "grad_norm": 8.62691802455825,
+      "learning_rate": 2.133572118185313e-05,
+      "loss": 0.2459,
+      "num_input_tokens_seen": 32216656,
+      "step": 2755
+    },
+    {
+      "epoch": 1.4213237187741437,
+      "grad_norm": 11.518284151131217,
+      "learning_rate": 2.1161444173145468e-05,
+      "loss": 0.2656,
+      "num_input_tokens_seen": 32275168,
+      "step": 2760
+    },
+    {
+      "epoch": 1.423899047128509,
+      "grad_norm": 5.717001204241147,
+      "learning_rate": 2.098769060374206e-05,
+      "loss": 0.199,
+      "num_input_tokens_seen": 32333664,
+      "step": 2765
+    },
+    {
+      "epoch": 1.4264743754828741,
+      "grad_norm": 5.060469427631098,
+      "learning_rate": 2.0814463627387744e-05,
+      "loss": 0.2552,
+      "num_input_tokens_seen": 32392088,
+      "step": 2770
+    },
+    {
+      "epoch": 1.4290497038372392,
+      "grad_norm": 10.068742247308663,
+      "learning_rate": 2.0641766388269345e-05,
+      "loss": 0.2123,
+      "num_input_tokens_seen": 32450584,
+      "step": 2775
+    },
+    {
+      "epoch": 1.4316250321916044,
+      "grad_norm": 8.04329511861408,
+      "learning_rate": 2.046960202095866e-05,
+      "loss": 0.1827,
+      "num_input_tokens_seen": 32509096,
+      "step": 2780
+    },
+    {
+      "epoch": 1.4342003605459697,
+      "grad_norm": 5.111975388475009,
+      "learning_rate": 2.0297973650355435e-05,
+      "loss": 0.3038,
+      "num_input_tokens_seen": 32567560,
+      "step": 2785
+    },
+    {
+      "epoch": 1.436775688900335,
+      "grad_norm": 2.2165480876124666,
+      "learning_rate": 2.012688439163075e-05,
+      "loss": 0.2078,
+      "num_input_tokens_seen": 32626056,
+      "step": 2790
+    },
+    {
+      "epoch": 1.4393510172547,
+      "grad_norm": 5.208837069631258,
+      "learning_rate": 1.9956337350170446e-05,
+      "loss": 0.23,
+      "num_input_tokens_seen": 32684520,
+      "step": 2795
+    },
+    {
+      "epoch": 1.4419263456090652,
+      "grad_norm": 14.804471075670579,
+      "learning_rate": 1.978633562151875e-05,
+      "loss": 0.2162,
+      "num_input_tokens_seen": 32743032,
+      "step": 2800
+    },
+    {
+      "epoch": 1.4419263456090652,
+      "eval_loss": 0.2771253287792206,
+      "eval_runtime": 19.4256,
+      "eval_samples_per_second": 3.089,
+      "eval_steps_per_second": 0.772,
+      "num_input_tokens_seen": 32743032,
+      "step": 2800
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 32158144,
+  "num_input_tokens_seen": 32743032,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4921,7 +5010,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2121940811644928.0,
+  "total_flos": 2160533754609664.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null