Training in progress, step 2850, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f209712b49d72a0a61d59f7b8f5e51be948aa2ecb02420dc257c2b1ba5b38a8
+oid sha256:73c7f2377d2df91cf17de2733b3a5bbe82d66442455824ef9aa239771fcb5c26
 size 29034840

last-checkpoint/global_step2849/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45ab5885128ab15acfeda63788268d09370a0f0245104f50cb23ab0e1debf5d1
+size 43429616

last-checkpoint/global_step2849/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cc72c29b8a71621fc9556a48734394c203d607b44ea67f32726da75dcf76d54
+size 43429616

last-checkpoint/global_step2849/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a1c10bdfdd75e7e8018958e4aff8bde9e76b0bde1ebb0be5da4e566b9eccbb9
+size 43429616

last-checkpoint/global_step2849/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a183a41b7c8207214291440122475dc391e27b810782729e2ee71e10f4e0d37
+size 43429616

last-checkpoint/global_step2849/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7180ca4205568132765e6d2ead963d258ddb856ab3eae9d1bfcb22efb6872c96
+size 637299

last-checkpoint/global_step2849/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae55ddd6b4e14375cd0c4d3f967ab0daccde3cb162e3e12251839e83e0195956
+size 637171

last-checkpoint/global_step2849/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ee9490ec9c9853920e4c9155fb808160a417e30fd2bff714d8b01355280bd6f
+size 637171

last-checkpoint/global_step2849/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e82babdf84d259edd42d8f571b3a5cbfeeb5c920141b498673efd54dfbd4c85d
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2799
+global_step2849

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff8dba2341c0517760edfde50521977f02a5bd982ffd3bc03de6109439c4f478
+oid sha256:ce92cea831a04716b4b472f1dad1cc986b2021dee9aac057217f5d455b27ec42
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d2bf831df9fbade9ac2a8db79798bc2a7b1afb85a78a6e463ec7a7db4acc0f8e
+oid sha256:3cddb73bbdf0f6f6a2c3182d70f7ad5d587353b164c08dd4f383b940d6b61e4e
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8602ff0a0fa366d46b61c0ef2b23ce468387898cf2bc1027e5450de73ddf647f
+oid sha256:b24b508e466beb446d37377d2a04757d3bc2b4230de3ac56b25a65d7753a74c1
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4bb51d675cf23603b1b765cd645f53d6b66ddb104d56d48674e9c798e086f696
+oid sha256:b4c6a18a7de8b25b21673ba2ff7efbaaae00ec8c453c7975b467c1df87b87022
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b2248ae5cd08b24968d749ba4f37513ee0a4cfb5059768755ed9ef6b0b1fd810
+oid sha256:06c7a8724990bbfa42e474ca2bea837a85b83fc6dd9afb66285c6f4108456bdd
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.18780523538589478,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2650",
-  "epoch": 1.4419263456090652,
+  "epoch": 1.467679629152717,
   "eval_steps": 50,
-  "global_step": 2800,
+  "global_step": 2850,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4991,11 +4991,100 @@
       "eval_steps_per_second": 0.772,
       "num_input_tokens_seen": 32743032,
       "step": 2800
+    },
+    {
+      "epoch": 1.4445016739634302,
+      "grad_norm": 5.47081802170478,
+      "learning_rate": 1.9616882291322043e-05,
+      "loss": 0.2128,
+      "num_input_tokens_seen": 32801504,
+      "step": 2805
+    },
+    {
+      "epoch": 1.4470770023177955,
+      "grad_norm": 8.153473173608221,
+      "learning_rate": 1.9447980435272982e-05,
+      "loss": 0.226,
+      "num_input_tokens_seen": 32860032,
+      "step": 2810
+    },
+    {
+      "epoch": 1.4496523306721607,
+      "grad_norm": 3.2012899866356275,
+      "learning_rate": 1.9279633119054524e-05,
+      "loss": 0.1945,
+      "num_input_tokens_seen": 32918472,
+      "step": 2815
+    },
+    {
+      "epoch": 1.452227659026526,
+      "grad_norm": 3.275737115662877,
+      "learning_rate": 1.9111843398284412e-05,
+      "loss": 0.1593,
+      "num_input_tokens_seen": 32976944,
+      "step": 2820
+    },
+    {
+      "epoch": 1.4548029873808912,
+      "grad_norm": 5.6817295869650755,
+      "learning_rate": 1.8944614318459604e-05,
+      "loss": 0.2154,
+      "num_input_tokens_seen": 33035424,
+      "step": 2825
+    },
+    {
+      "epoch": 1.4573783157352562,
+      "grad_norm": 4.248487908153702,
+      "learning_rate": 1.8777948914901066e-05,
+      "loss": 0.2266,
+      "num_input_tokens_seen": 33093880,
+      "step": 2830
+    },
+    {
+      "epoch": 1.4599536440896215,
+      "grad_norm": 4.65833753095402,
+      "learning_rate": 1.8611850212698678e-05,
+      "loss": 0.1948,
+      "num_input_tokens_seen": 33152328,
+      "step": 2835
+    },
+    {
+      "epoch": 1.4625289724439865,
+      "grad_norm": 6.9576190671694205,
+      "learning_rate": 1.84463212266563e-05,
+      "loss": 0.2159,
+      "num_input_tokens_seen": 33210816,
+      "step": 2840
+    },
+    {
+      "epoch": 1.4651043007983517,
+      "grad_norm": 28.44341760505862,
+      "learning_rate": 1.8281364961237013e-05,
+      "loss": 0.2326,
+      "num_input_tokens_seen": 33269320,
+      "step": 2845
+    },
+    {
+      "epoch": 1.467679629152717,
+      "grad_norm": 12.408239979973144,
+      "learning_rate": 1.8116984410508696e-05,
+      "loss": 0.1912,
+      "num_input_tokens_seen": 33327720,
+      "step": 2850
+    },
+    {
+      "epoch": 1.467679629152717,
+      "eval_loss": 0.2647402584552765,
+      "eval_runtime": 19.6149,
+      "eval_samples_per_second": 3.059,
+      "eval_steps_per_second": 0.765,
+      "num_input_tokens_seen": 33327720,
+      "step": 2850
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 32743032,
+  "num_input_tokens_seen": 33327720,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5010,7 +5099,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2160533754609664.0,
+  "total_flos": 2199114391224320.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null