Training in progress, step 1450, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1cfac0e8b9dcb659e1e1aecd6c6484a664c1bc40a9c86d4a35709450060ad76
+oid sha256:140829d96675f8a17bdb0fa18b56b080defbfc9f26441176908846cb9afcc2fb
 size 29034840

last-checkpoint/global_step1450/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5b61b026899fd0585bd426d26032e3c9083b6a89ea4493964360fc99676263a
+size 43429616

last-checkpoint/global_step1450/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73dacdf7217df5dfd37fcdc5dd4b401f2df2b413654832de71d26b4696586cfd
+size 43429616

last-checkpoint/global_step1450/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e39c9918463cb14ac43fc2d7c6b82ab9b56446d6919894153bf82eb0539edc4a
+size 43429616

last-checkpoint/global_step1450/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a557d4c9a85e33df7e72f07f4e4cc4f4f4e9f05c1991fb944018d1142dcdcd0
+size 43429616

last-checkpoint/global_step1450/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07bcf228cd3445b5f81d82193f9981b94412753caac46837372bccbefabc8afb
+size 637299

last-checkpoint/global_step1450/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:451a08a86c16b5c280887423b2e3f77cbc4ff52bdf0818c12b05618d2438332c
+size 637171

last-checkpoint/global_step1450/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d37d114e8d4c71e3b602ca91992bcc10700074bf3f6d72c53989b941258aac05
+size 637171

last-checkpoint/global_step1450/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0fa53cd0688eb5fe1f2eba519558bf9b4822cd883935db7d14ffea34356251
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1400
+global_step1450

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe66a68e61de2221b30fd9749bc68b45a1474bb2cc95901bca9557ac87909355
+oid sha256:dd4f3298d54e9509917373bcd359e11c92a8e0aa77b2cc0825602efd186ad77e
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0cd4f3162e46c3bb0f1fc4d3c52c7c33e60f56764458e0c8a73c3810b0a25f8c
+oid sha256:5fdab02a791039ff46df83a272c972ab0f366fcea91338790dc016dbbbf6de80
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:185cc99aaa81b1b49b3ddc74aa6f97aa3036330983a7b69d52bd191057f9a5d5
+oid sha256:e2808762a5f80587f05c1e2e8de26b5aef1010331cea366649319d593dbb0e66
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e37403c30cb4309e54e5defdde1906486716fc859274035d44aaac5d48a97ba
+oid sha256:c052112cbe3f325fd4543a02558091b80c9c8abcd8db8822be9a8306eab6f2b7
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23a4dfbc7cb0c7bdf1a7b9ae411289d72515b198f3487d165a8d6a0736490572
+oid sha256:c787e9070735d1c645006b3e548e9bd4e98919282115042a730b3d795d99d4b1
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.47318556904792786,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1400",
-  "epoch": 0.7210919392222508,
+  "best_metric": 0.43503889441490173,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1450",
+  "epoch": 0.7468452227659027,
   "eval_steps": 50,
-  "global_step": 1400,
+  "global_step": 1450,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2499,11 +2499,100 @@
       "eval_steps_per_second": 0.779,
       "num_input_tokens_seen": 16374784,
       "step": 1400
+    },
+    {
+      "epoch": 0.723667267576616,
+      "grad_norm": 9.384799103772558,
+      "learning_rate": 7.569771415706126e-05,
+      "loss": 0.444,
+      "num_input_tokens_seen": 16433288,
+      "step": 1405
+    },
+    {
+      "epoch": 0.7262425959309812,
+      "grad_norm": 16.840151224890615,
+      "learning_rate": 7.551475072432237e-05,
+      "loss": 0.4971,
+      "num_input_tokens_seen": 16491784,
+      "step": 1410
+    },
+    {
+      "epoch": 0.7288179242853464,
+      "grad_norm": 10.190076716766358,
+      "learning_rate": 7.533132418159119e-05,
+      "loss": 0.4284,
+      "num_input_tokens_seen": 16550272,
+      "step": 1415
+    },
+    {
+      "epoch": 0.7313932526397116,
+      "grad_norm": 7.663975666748285,
+      "learning_rate": 7.514743785818361e-05,
+      "loss": 0.414,
+      "num_input_tokens_seen": 16608704,
+      "step": 1420
+    },
+    {
+      "epoch": 0.7339685809940767,
+      "grad_norm": 7.403751704217463,
+      "learning_rate": 7.496309509176082e-05,
+      "loss": 0.3715,
+      "num_input_tokens_seen": 16667200,
+      "step": 1425
+    },
+    {
+      "epoch": 0.7365439093484419,
+      "grad_norm": 15.05075115467088,
+      "learning_rate": 7.477829922826883e-05,
+      "loss": 0.4723,
+      "num_input_tokens_seen": 16725704,
+      "step": 1430
+    },
+    {
+      "epoch": 0.7391192377028071,
+      "grad_norm": 8.817475778915243,
+      "learning_rate": 7.459305362187756e-05,
+      "loss": 0.4357,
+      "num_input_tokens_seen": 16784192,
+      "step": 1435
+    },
+    {
+      "epoch": 0.7416945660571723,
+      "grad_norm": 14.070137984729548,
+      "learning_rate": 7.440736163492016e-05,
+      "loss": 0.4402,
+      "num_input_tokens_seen": 16842704,
+      "step": 1440
+    },
+    {
+      "epoch": 0.7442698944115375,
+      "grad_norm": 14.494333760708628,
+      "learning_rate": 7.422122663783187e-05,
+      "loss": 0.4457,
+      "num_input_tokens_seen": 16901176,
+      "step": 1445
+    },
+    {
+      "epoch": 0.7468452227659027,
+      "grad_norm": 8.298786058991697,
+      "learning_rate": 7.403465200908883e-05,
+      "loss": 0.4353,
+      "num_input_tokens_seen": 16959632,
+      "step": 1450
+    },
+    {
+      "epoch": 0.7468452227659027,
+      "eval_loss": 0.43503889441490173,
+      "eval_runtime": 19.5652,
+      "eval_samples_per_second": 3.067,
+      "eval_steps_per_second": 0.767,
+      "num_input_tokens_seen": 16959632,
+      "step": 1450
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 16374784,
+  "num_input_tokens_seen": 16959632,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2518,7 +2607,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1080424859697152.0,
+  "total_flos": 1119014586417152.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null