Training in progress, step 200, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d6cd128d7c617876ad46ba2cf72bcffc9405810c45769e1284c0504c35ae59f
+oid sha256:cfb788f0d6ef0dec08ed5f5eb5575155c2531033fdd3222d6ffa1678d4305eab
 size 29034840

last-checkpoint/global_step200/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33398e00a4f2862e8adc6ce36c52babdb6e88238ef6cf9be57c08ca6ce426ad9
+size 43429616

last-checkpoint/global_step200/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:975a0bed84695b3a40282da9ca40ccba2cc3989b43f0bae7eb56e3c79a64c89a
+size 43429616

last-checkpoint/global_step200/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96823a0e06f59312795c8c2116d9fb89cd2e36411387b48ed029d300d01dfb6b
+size 43429616

last-checkpoint/global_step200/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da04baff4c25198a65f9d675bb21eb1bae33371cd95286a9e1e9186418c390e3
+size 43429616

last-checkpoint/global_step200/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9c8aed679b2d5c060599974f1ab45390fb83c42d37e4ed63003977054cda112
+size 637299

last-checkpoint/global_step200/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7ceb1154d5065a4632b1731df2acb5e51080a1f0388344212eaec5524452314
+size 637171

last-checkpoint/global_step200/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30465b90efa0eb73fc80dc870a336c0cc691a34eaf088821a381a9f71ee7d652
+size 637171

last-checkpoint/global_step200/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab524daca08828de12f8863c12ac5a32a74f179700a4a2850b77f7debcf459b3
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step150
+global_step200

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70cc56408014c410353d4dd58ae9b03f4be043f5f800324f66fd8e20e99b840e
+oid sha256:5c5e18f922d0af74d820247ae97bee506ab412554a58345ddf2558abc94ee3e3
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49d1438e98cc9c53a6852464635ce62e9788e61eb3646b73e33813f487c4b6ae
+oid sha256:2a2dcca6d9741f46592359768ea2212b9321da6408d1fd7d3a80b017bf37f434
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4388add9cec90932f8ff0100d27a0574d98e1bad52ff89d44e31967d2b4fbfde
+oid sha256:69420ece2c255923c5cbb3c6c9c4a6b9cb38fb57e5d3033c8b7d436a1faf6f13
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a705d6dfaae4f2c1b4b2be6b25a6eb521ffae6fcba21cc1531e97b60037ed079
+oid sha256:66f278b40a1e23b88a657c4e5d03afa8dbbbe14dfeb16f6b4beedaece6cdd0b9
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a3a5e990aae1d037304be1158121f10142a3f8a9085eb5cce549d9f326711508
+oid sha256:a350ac726ec3a2e00c35455a1b9a7e1e8a4f7899bd79cd1f9094f9c7f1fa8509
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.8934853076934814,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-150",
-  "epoch": 0.07725985063095545,
+  "epoch": 0.10301313417460727,
   "eval_steps": 50,
-  "global_step": 150,
+  "global_step": 200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -274,11 +274,100 @@
       "eval_steps_per_second": 0.755,
       "num_input_tokens_seen": 1754512,
       "step": 150
+    },
+    {
+      "epoch": 0.07983517898532062,
+      "grad_norm": 0.5167982536583405,
+      "learning_rate": 7.948717948717948e-05,
+      "loss": 0.8992,
+      "num_input_tokens_seen": 1812976,
+      "step": 155
+    },
+    {
+      "epoch": 0.08241050733968582,
+      "grad_norm": 0.4971816797735092,
+      "learning_rate": 8.205128205128205e-05,
+      "loss": 0.8965,
+      "num_input_tokens_seen": 1871464,
+      "step": 160
+    },
+    {
+      "epoch": 0.08498583569405099,
+      "grad_norm": 0.6561749633642688,
+      "learning_rate": 8.461538461538461e-05,
+      "loss": 0.9094,
+      "num_input_tokens_seen": 1929928,
+      "step": 165
+    },
+    {
+      "epoch": 0.08756116404841617,
+      "grad_norm": 0.5010857314708574,
+      "learning_rate": 8.717948717948718e-05,
+      "loss": 0.903,
+      "num_input_tokens_seen": 1988432,
+      "step": 170
+    },
+    {
+      "epoch": 0.09013649240278135,
+      "grad_norm": 0.48794512034251364,
+      "learning_rate": 8.974358974358975e-05,
+      "loss": 0.902,
+      "num_input_tokens_seen": 2046920,
+      "step": 175
+    },
+    {
+      "epoch": 0.09271182075714654,
+      "grad_norm": 0.4040014684262414,
+      "learning_rate": 9.230769230769232e-05,
+      "loss": 0.9006,
+      "num_input_tokens_seen": 2105392,
+      "step": 180
+    },
+    {
+      "epoch": 0.09528714911151172,
+      "grad_norm": 0.5312840597942438,
+      "learning_rate": 9.487179487179487e-05,
+      "loss": 0.9042,
+      "num_input_tokens_seen": 2163872,
+      "step": 185
+    },
+    {
+      "epoch": 0.0978624774658769,
+      "grad_norm": 0.3535119366494406,
+      "learning_rate": 9.743589743589744e-05,
+      "loss": 0.9096,
+      "num_input_tokens_seen": 2222352,
+      "step": 190
+    },
+    {
+      "epoch": 0.10043780582024209,
+      "grad_norm": 0.30590378285024006,
+      "learning_rate": 0.0001,
+      "loss": 0.9037,
+      "num_input_tokens_seen": 2280800,
+      "step": 195
+    },
+    {
+      "epoch": 0.10301313417460727,
+      "grad_norm": 0.3055264226667786,
+      "learning_rate": 9.999954623308172e-05,
+      "loss": 0.904,
+      "num_input_tokens_seen": 2339304,
+      "step": 200
+    },
+    {
+      "epoch": 0.10301313417460727,
+      "eval_loss": 0.8980139493942261,
+      "eval_runtime": 19.316,
+      "eval_samples_per_second": 3.106,
+      "eval_steps_per_second": 0.777,
+      "num_input_tokens_seen": 2339304,
+      "step": 200
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 1754512,
+  "num_input_tokens_seen": 2339304,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -293,7 +382,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 115707171307520.0,
+  "total_flos": 154295037853696.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null