Training in progress, step 2900, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:73c7f2377d2df91cf17de2733b3a5bbe82d66442455824ef9aa239771fcb5c26
+oid sha256:6efdabb869140223f58ba89adb95c0f30e574545e43fb41d7bac2b2f9150b3f4
 size 29034840

last-checkpoint/global_step2899/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe584b25009f277bbd1ff8bd68498986caeb876dde90db7fb191b1828a4b09d4
+size 43429616

last-checkpoint/global_step2899/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90b37aad3ae90c9555a3255e733ac7286f6ed53445b5a691924da68415107bc1
+size 43429616

last-checkpoint/global_step2899/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05f1397e484268e9c78c25d048eb927a0958f13b099e7e38cb6464cdf8260894
+size 43429616

last-checkpoint/global_step2899/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6a4704ace1d93a396e94eb31be253766aae3bc1786ff5cc58e88bbab5d5aae8
+size 43429616

last-checkpoint/global_step2899/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b07ba6cef623fc319725e939f863369b1349e53b309afcc6f961d7d3ec8bab6a
+size 637299

last-checkpoint/global_step2899/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:561aced656a1a87db26d9d074566496b417f0e1a340fe32041d82d5bb86ca954
+size 637171

last-checkpoint/global_step2899/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b47d8d65772e4ad33bcd2aac19666fd9bd00c11fffef415c34820da3d7491dfb
+size 637171

last-checkpoint/global_step2899/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff1814429e9f6159d228ad9418a3daad31d09f23fd7299fd0145ec2f5640a696
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2849
+global_step2899

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce92cea831a04716b4b472f1dad1cc986b2021dee9aac057217f5d455b27ec42
+oid sha256:bfc5d4e344535f1dd0ff5275633ec3d55bb6249e432442365ff24445d82ec35c
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3cddb73bbdf0f6f6a2c3182d70f7ad5d587353b164c08dd4f383b940d6b61e4e
+oid sha256:8a898928042c09dc123c1025557279997043b7f607bc91ee2ff2d4b4d2b9f1ba
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b24b508e466beb446d37377d2a04757d3bc2b4230de3ac56b25a65d7753a74c1
+oid sha256:6923d07d979aa78d66765208f598662fd5092b5227cd87920feedfb923fa375f
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4c6a18a7de8b25b21673ba2ff7efbaaae00ec8c453c7975b467c1df87b87022
+oid sha256:e9ba35b9b3c512fbe857d909557329ba47dbefe5f521014123c05901c32edb6d
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:06c7a8724990bbfa42e474ca2bea837a85b83fc6dd9afb66285c6f4108456bdd
+oid sha256:2ab87135f8169af50b16a6f8b1198988096b14bc0e5836b645aff14a930bb3f5
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.18780523538589478,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-2650",
-  "epoch": 1.467679629152717,
+  "epoch": 1.4934329126963688,
   "eval_steps": 50,
-  "global_step": 2850,
+  "global_step": 2900,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5080,11 +5080,100 @@
       "eval_steps_per_second": 0.765,
       "num_input_tokens_seen": 33327720,
       "step": 2850
+    },
+    {
+      "epoch": 1.4702549575070822,
+      "grad_norm": 8.379574283067335,
+      "learning_rate": 1.795318255808956e-05,
+      "loss": 0.2166,
+      "num_input_tokens_seen": 33386160,
+      "step": 2855
+    },
+    {
+      "epoch": 1.4728302858614473,
+      "grad_norm": 9.376274649781424,
+      "learning_rate": 1.7789962377094088e-05,
+      "loss": 0.1843,
+      "num_input_tokens_seen": 33444648,
+      "step": 2860
+    },
+    {
+      "epoch": 1.4754056142158125,
+      "grad_norm": 2.5653090157172724,
+      "learning_rate": 1.762732683007902e-05,
+      "loss": 0.1988,
+      "num_input_tokens_seen": 33503144,
+      "step": 2865
+    },
+    {
+      "epoch": 1.4779809425701778,
+      "grad_norm": 12.205543974310553,
+      "learning_rate": 1.746527886898962e-05,
+      "loss": 0.1715,
+      "num_input_tokens_seen": 33561600,
+      "step": 2870
+    },
+    {
+      "epoch": 1.4805562709245428,
+      "grad_norm": 2.016900668078144,
+      "learning_rate": 1.7303821435105998e-05,
+      "loss": 0.1577,
+      "num_input_tokens_seen": 33620056,
+      "step": 2875
+    },
+    {
+      "epoch": 1.483131599278908,
+      "grad_norm": 7.53734329898392,
+      "learning_rate": 1.714295745898989e-05,
+      "loss": 0.2204,
+      "num_input_tokens_seen": 33678512,
+      "step": 2880
+    },
+    {
+      "epoch": 1.4857069276332733,
+      "grad_norm": 25.928242388695963,
+      "learning_rate": 1.6982689860431283e-05,
+      "loss": 0.2283,
+      "num_input_tokens_seen": 33736960,
+      "step": 2885
+    },
+    {
+      "epoch": 1.4882822559876385,
+      "grad_norm": 21.45493152674021,
+      "learning_rate": 1.682302154839558e-05,
+      "loss": 0.2632,
+      "num_input_tokens_seen": 33795416,
+      "step": 2890
+    },
+    {
+      "epoch": 1.4908575843420036,
+      "grad_norm": 8.396238950090229,
+      "learning_rate": 1.6663955420970667e-05,
+      "loss": 0.1844,
+      "num_input_tokens_seen": 33853936,
+      "step": 2895
+    },
+    {
+      "epoch": 1.4934329126963688,
+      "grad_norm": 4.710042357794161,
+      "learning_rate": 1.650549436531442e-05,
+      "loss": 0.2015,
+      "num_input_tokens_seen": 33912440,
+      "step": 2900
+    },
+    {
+      "epoch": 1.4934329126963688,
+      "eval_loss": 0.23923428356647491,
+      "eval_runtime": 19.3819,
+      "eval_samples_per_second": 3.096,
+      "eval_steps_per_second": 0.774,
+      "num_input_tokens_seen": 33912440,
+      "step": 2900
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 33327720,
+  "num_input_tokens_seen": 33912440,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5099,7 +5188,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2199114391224320.0,
+  "total_flos": 2237698608201728.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null