Training in progress, step 1750, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6956268c1ac0ce3f63b81eb8b75c0c4b5ea990b8cf8dfa819dbcd8de1270e39d
+oid sha256:adf72d38ec00ef740c0dd6070b4c5e7e4025640731572f71c655ba6e0033d445
 size 29034840

last-checkpoint/global_step1750/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e193ff7d129d9dbf6944bc24b4bb5e73fed2bd2059133820931e732fda273a46
+size 43429616

last-checkpoint/global_step1750/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5f60ad1dacc982d980bf31f45b19fb5140e46df15f5ff6549b16f20788d2491
+size 43429616

last-checkpoint/global_step1750/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec7266955b43732e772c94076abc46d7296dfbc857b23dcbbd3f7a538576c0b8
+size 43429616

last-checkpoint/global_step1750/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71e1e05402c94f950750d8c500f91432b71381b7c277a132094b0f4b7332b275
+size 43429616

last-checkpoint/global_step1750/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32f8c5b1da88eb2c3bd967f43f6931692786200c22aa97a202ebf66082557677
+size 637299

last-checkpoint/global_step1750/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:037fcd6af5c00ed77baab88df849e8698d197844c05cc017806d8e74ae2f1b95
+size 637171

last-checkpoint/global_step1750/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99970103805cf54854383f6ea3849fdf6038c8a68a2a992a04bd9ebaf6f4251d
+size 637171

last-checkpoint/global_step1750/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd3d1f71573606393b3cb5f1deddf23fe9aa63fa2404d1a9570e793efcfa8328
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1700
+global_step1750

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e7c17922709137dd2f358be87cc431f7959a56821e2d051582e81679e2d388e
+oid sha256:222e5f04f66dfcca4efb2d648f5a480d56c2a07755d7a1bae8232d01d4b479ce
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eaf0ec3cc431efdc1cb595b27e74fc020021d53f5c11850d7490a79bf42b71d6
+oid sha256:150a9cfa07bd33135b1b8b22033907e44137689de662dda0a482f3af84c5a1ff
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:148fd0a663c121acf74d815964bc0d08d07065e8503b03adfd967fdaf63a6abc
+oid sha256:c0c0d7619e94c90efa0c89a1f208db53b48c726519761710da6fbc31a80651d8
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56589f6c34527656fe0dc752874cfc1460efbb5c0c3f874c4cd09d6415dd16c1
+oid sha256:aa66a4b2f749b1ec7e8868668dc670ff3f6d8df765a5249122f980ae5aae6a54
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac6991effd213fdc73293940aef8bf4a082cc635f49d6fda49012c0f681c0564
+oid sha256:8bd9a4264d505b81d4c17e3b911ead7aa7b48174cf389f137f6abc777b68bcd1
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.34382957220077515,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1700",
-  "epoch": 0.8756116404841617,
+  "best_metric": 0.3325226604938507,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1750",
+  "epoch": 0.9013649240278135,
   "eval_steps": 50,
-  "global_step": 1700,
+  "global_step": 1750,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3033,11 +3033,100 @@
       "eval_steps_per_second": 0.773,
       "num_input_tokens_seen": 19883504,
       "step": 1700
+    },
+    {
+      "epoch": 0.8781869688385269,
+      "grad_norm": 18.837461373103466,
+      "learning_rate": 6.401786421227686e-05,
+      "loss": 0.3453,
+      "num_input_tokens_seen": 19942000,
+      "step": 1705
+    },
+    {
+      "epoch": 0.8807622971928921,
+      "grad_norm": 10.315591123527769,
+      "learning_rate": 6.381326233990644e-05,
+      "loss": 0.3278,
+      "num_input_tokens_seen": 20000440,
+      "step": 1710
+    },
+    {
+      "epoch": 0.8833376255472573,
+      "grad_norm": 14.473376988028903,
+      "learning_rate": 6.360840974747666e-05,
+      "loss": 0.3695,
+      "num_input_tokens_seen": 20058920,
+      "step": 1715
+    },
+    {
+      "epoch": 0.8859129539016225,
+      "grad_norm": 13.569903569955356,
+      "learning_rate": 6.340331015320073e-05,
+      "loss": 0.4362,
+      "num_input_tokens_seen": 20117416,
+      "step": 1720
+    },
+    {
+      "epoch": 0.8884882822559876,
+      "grad_norm": 8.610173422350266,
+      "learning_rate": 6.319796727977509e-05,
+      "loss": 0.3178,
+      "num_input_tokens_seen": 20175936,
+      "step": 1725
+    },
+    {
+      "epoch": 0.8910636106103528,
+      "grad_norm": 13.406485921668688,
+      "learning_rate": 6.299238485431183e-05,
+      "loss": 0.3679,
+      "num_input_tokens_seen": 20234368,
+      "step": 1730
+    },
+    {
+      "epoch": 0.893638938964718,
+      "grad_norm": 10.15746393869747,
+      "learning_rate": 6.27865666082711e-05,
+      "loss": 0.3529,
+      "num_input_tokens_seen": 20292856,
+      "step": 1735
+    },
+    {
+      "epoch": 0.8962142673190832,
+      "grad_norm": 6.0390850533631,
+      "learning_rate": 6.258051627739337e-05,
+      "loss": 0.281,
+      "num_input_tokens_seen": 20351272,
+      "step": 1740
+    },
+    {
+      "epoch": 0.8987895956734484,
+      "grad_norm": 11.746443511207032,
+      "learning_rate": 6.237423760163156e-05,
+      "loss": 0.3148,
+      "num_input_tokens_seen": 20409744,
+      "step": 1745
+    },
+    {
+      "epoch": 0.9013649240278135,
+      "grad_norm": 23.680917449871366,
+      "learning_rate": 6.216773432508325e-05,
+      "loss": 0.3233,
+      "num_input_tokens_seen": 20468200,
+      "step": 1750
+    },
+    {
+      "epoch": 0.9013649240278135,
+      "eval_loss": 0.3325226604938507,
+      "eval_runtime": 19.3673,
+      "eval_samples_per_second": 3.098,
+      "eval_steps_per_second": 0.775,
+      "num_input_tokens_seen": 20468200,
+      "step": 1750
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 19883504,
+  "num_input_tokens_seen": 20468200,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -3052,7 +3141,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1311949109854208.0,
+  "total_flos": 1350532214292480.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null