Training in progress, step 1700, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a98d193ab7c674712e3c18a30e49e6fde0bcc59efee6c60450baf3a08afe8f78
+oid sha256:6956268c1ac0ce3f63b81eb8b75c0c4b5ea990b8cf8dfa819dbcd8de1270e39d
 size 29034840

last-checkpoint/global_step1700/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f91b41ee41f6d8ffe96d90a30b1833d8800fc4473d410257defc40606fba2b0
+size 43429616

last-checkpoint/global_step1700/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0c602c2e1fa159485ae5b6e5e6656b25b47460423fcc950708304a8521a4cb5
+size 43429616

last-checkpoint/global_step1700/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37e589d92c862ed47bfaea6f877bfa19325cd0e8679ac559e5a6c26ed12e9ec7
+size 43429616

last-checkpoint/global_step1700/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5024d6ee19d4d162bb27db47bbfd93299df19b31c6bd68a223a6360630f1e7ff
+size 43429616

last-checkpoint/global_step1700/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06ba06283edb02eda2732a8265d8207d60eb1fef7f9e3ae5c1ea8d90c17b0715
+size 637299

last-checkpoint/global_step1700/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da84df26158916db26311a33b5233d31790e0ff29bbd78fcce9a0b80073a2f46
+size 637171

last-checkpoint/global_step1700/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0650a802205c7a65d67b752891c3c2aa99e1d36f1b08f1727af28cb5064e46fe
+size 637171

last-checkpoint/global_step1700/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db65b2203742cda52b00e5cd5927c8817de12627c99794dbca6286f744f71a1d
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1650
+global_step1700

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a90384755f5b036b42b1a465b39dbf24a925a02c04294f9d684bc1de7f4db1e5
+oid sha256:9e7c17922709137dd2f358be87cc431f7959a56821e2d051582e81679e2d388e
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7621b41e55056218f97d5b32ae116de3304a677b9f27b6a62170d83a2bbff176
+oid sha256:eaf0ec3cc431efdc1cb595b27e74fc020021d53f5c11850d7490a79bf42b71d6
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:997e9debadfd125b5c8b66ee6dd79ced3d40d353ff9250475f3814fd950012a6
+oid sha256:148fd0a663c121acf74d815964bc0d08d07065e8503b03adfd967fdaf63a6abc
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f0256032419959580948d742425f66782bc8eb029126a091669a42c6ee0eba4
+oid sha256:56589f6c34527656fe0dc752874cfc1460efbb5c0c3f874c4cd09d6415dd16c1
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:73abf8d1cae561939afcea6d9d77e3583686a8bac69cce204a4cb222cd869363
+oid sha256:ac6991effd213fdc73293940aef8bf4a082cc635f49d6fda49012c0f681c0564
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.35129043459892273,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1650",
-  "epoch": 0.8498583569405099,
+  "best_metric": 0.34382957220077515,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1700",
+  "epoch": 0.8756116404841617,
   "eval_steps": 50,
-  "global_step": 1650,
+  "global_step": 1700,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2944,11 +2944,100 @@
       "eval_steps_per_second": 0.773,
       "num_input_tokens_seen": 19298848,
       "step": 1650
+    },
+    {
+      "epoch": 0.8524336852948751,
+      "grad_norm": 6.358459669534092,
+      "learning_rate": 6.604927866620031e-05,
+      "loss": 0.3414,
+      "num_input_tokens_seen": 19357304,
+      "step": 1655
+    },
+    {
+      "epoch": 0.8550090136492403,
+      "grad_norm": 9.39994214990231,
+      "learning_rate": 6.584738746168762e-05,
+      "loss": 0.4728,
+      "num_input_tokens_seen": 19415800,
+      "step": 1660
+    },
+    {
+      "epoch": 0.8575843420036054,
+      "grad_norm": 14.50676995208697,
+      "learning_rate": 6.564520861636808e-05,
+      "loss": 0.4264,
+      "num_input_tokens_seen": 19474256,
+      "step": 1665
+    },
+    {
+      "epoch": 0.8601596703579707,
+      "grad_norm": 4.791356534368335,
+      "learning_rate": 6.544274579992453e-05,
+      "loss": 0.4038,
+      "num_input_tokens_seen": 19532736,
+      "step": 1670
+    },
+    {
+      "epoch": 0.8627349987123358,
+      "grad_norm": 6.4285821594117225,
+      "learning_rate": 6.524000268719411e-05,
+      "loss": 0.3241,
+      "num_input_tokens_seen": 19591184,
+      "step": 1675
+    },
+    {
+      "epoch": 0.865310327066701,
+      "grad_norm": 8.268794944409354,
+      "learning_rate": 6.503698295810154e-05,
+      "loss": 0.3248,
+      "num_input_tokens_seen": 19649656,
+      "step": 1680
+    },
+    {
+      "epoch": 0.8678856554210662,
+      "grad_norm": 12.365745303187085,
+      "learning_rate": 6.483369029759229e-05,
+      "loss": 0.4295,
+      "num_input_tokens_seen": 19708144,
+      "step": 1685
+    },
+    {
+      "epoch": 0.8704609837754314,
+      "grad_norm": 6.3136571538884505,
+      "learning_rate": 6.463012839556569e-05,
+      "loss": 0.3785,
+      "num_input_tokens_seen": 19766592,
+      "step": 1690
+    },
+    {
+      "epoch": 0.8730363121297966,
+      "grad_norm": 6.1513513385401195,
+      "learning_rate": 6.442630094680805e-05,
+      "loss": 0.3376,
+      "num_input_tokens_seen": 19825064,
+      "step": 1695
+    },
+    {
+      "epoch": 0.8756116404841617,
+      "grad_norm": 9.59752445262446,
+      "learning_rate": 6.42222116509255e-05,
+      "loss": 0.3752,
+      "num_input_tokens_seen": 19883504,
+      "step": 1700
+    },
+    {
+      "epoch": 0.8756116404841617,
+      "eval_loss": 0.34382957220077515,
+      "eval_runtime": 19.3987,
+      "eval_samples_per_second": 3.093,
+      "eval_steps_per_second": 0.773,
+      "num_input_tokens_seen": 19883504,
+      "step": 1700
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 19298848,
+  "num_input_tokens_seen": 19883504,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2963,7 +3052,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1273370793476096.0,
+  "total_flos": 1311949109854208.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null