Training in progress, step 1650, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:36f722668f1a0618c0a9e43f223d905e82d0c1068f32211684745e00d5aaa272
+oid sha256:a98d193ab7c674712e3c18a30e49e6fde0bcc59efee6c60450baf3a08afe8f78
 size 29034840

last-checkpoint/global_step1650/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c92b99d09b208a5e54f54701424a9351685533109a2b3f0141fdfff156b409a9
+size 43429616

last-checkpoint/global_step1650/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b524a5a5ce1f117b997580292cb026d46cd9b79da9f9b1c06f9530a5b8aa996c
+size 43429616

last-checkpoint/global_step1650/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5489eb43fcb6a44359d04a1bd2ae923acbf692327d488e0afc70ab73f91dbdd8
+size 43429616

last-checkpoint/global_step1650/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b247918492721ac2a81189d9733f60f35c3feef1bda359a86a48fd8fb11aa4c
+size 43429616

last-checkpoint/global_step1650/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4c79466c87dc7a7bb2278fbdb456e69e6c6b0a01149fe0952fd362a1881b0c3
+size 637299

last-checkpoint/global_step1650/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4009b72c3e7da18b4a07fc0d3d8de5ac6d846834d93ff9946ac1044932cdd63
+size 637171

last-checkpoint/global_step1650/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a7cde2a2fb17884fd88468a6046779089bf8060a77027c24f561b10b27cca30
+size 637171

last-checkpoint/global_step1650/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9f660827bea0980ccb09f606308a98aeb8817e9ee50af4a17e22e579fd1e42d
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1600
+global_step1650

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9279ed4b01716237e789d2631c1f29bc5d43c5633c014d4401de21b672c1b355
+oid sha256:a90384755f5b036b42b1a465b39dbf24a925a02c04294f9d684bc1de7f4db1e5
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca1990d68e57c70df5c56d395dd3f3befbe07b380521f4144677c20f6fe2a3eb
+oid sha256:7621b41e55056218f97d5b32ae116de3304a677b9f27b6a62170d83a2bbff176
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0790066885525e1b9a9390a40ae27abd57abb47f031abface27890732f9e684
+oid sha256:997e9debadfd125b5c8b66ee6dd79ced3d40d353ff9250475f3814fd950012a6
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1325a2034fe48ebad4f00ac8a2b32ab5c4c43c2497712169a8e3b1112363d916
+oid sha256:0f0256032419959580948d742425f66782bc8eb029126a091669a42c6ee0eba4
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05a385f582b86d8b587b56eaada5930afef800d2a4f1e7413c113f427a6fcef2
+oid sha256:73abf8d1cae561939afcea6d9d77e3583686a8bac69cce204a4cb222cd869363
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.3963810205459595,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1500",
-  "epoch": 0.8241050733968581,
+  "best_metric": 0.35129043459892273,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4/lora/sft/checkpoint-1650",
+  "epoch": 0.8498583569405099,
   "eval_steps": 50,
-  "global_step": 1600,
+  "global_step": 1650,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2855,11 +2855,100 @@
       "eval_steps_per_second": 0.777,
       "num_input_tokens_seen": 18714072,
       "step": 1600
+    },
+    {
+      "epoch": 0.8266804017512233,
+      "grad_norm": 6.7599286769908815,
+      "learning_rate": 6.805156695508075e-05,
+      "loss": 0.4136,
+      "num_input_tokens_seen": 18772552,
+      "step": 1605
+    },
+    {
+      "epoch": 0.8292557301055885,
+      "grad_norm": 5.6959660380818,
+      "learning_rate": 6.7852752809755e-05,
+      "loss": 0.3523,
+      "num_input_tokens_seen": 18830992,
+      "step": 1610
+    },
+    {
+      "epoch": 0.8318310584599536,
+      "grad_norm": 5.054175814859553,
+      "learning_rate": 6.765361462488424e-05,
+      "loss": 0.2983,
+      "num_input_tokens_seen": 18889480,
+      "step": 1615
+    },
+    {
+      "epoch": 0.8344063868143188,
+      "grad_norm": 8.9830923866359,
+      "learning_rate": 6.745415601496127e-05,
+      "loss": 0.3987,
+      "num_input_tokens_seen": 18947936,
+      "step": 1620
+    },
+    {
+      "epoch": 0.836981715168684,
+      "grad_norm": 9.543713321488678,
+      "learning_rate": 6.725438060029485e-05,
+      "loss": 0.5012,
+      "num_input_tokens_seen": 19006432,
+      "step": 1625
+    },
+    {
+      "epoch": 0.8395570435230492,
+      "grad_norm": 9.329917566254677,
+      "learning_rate": 6.705429200694396e-05,
+      "loss": 0.36,
+      "num_input_tokens_seen": 19064920,
+      "step": 1630
+    },
+    {
+      "epoch": 0.8421323718774144,
+      "grad_norm": 4.257164474078224,
+      "learning_rate": 6.685389386665197e-05,
+      "loss": 0.3816,
+      "num_input_tokens_seen": 19123376,
+      "step": 1635
+    },
+    {
+      "epoch": 0.8447077002317795,
+      "grad_norm": 5.420005475317537,
+      "learning_rate": 6.665318981678072e-05,
+      "loss": 0.3503,
+      "num_input_tokens_seen": 19181864,
+      "step": 1640
+    },
+    {
+      "epoch": 0.8472830285861447,
+      "grad_norm": 6.544911058180563,
+      "learning_rate": 6.645218350024456e-05,
+      "loss": 0.3644,
+      "num_input_tokens_seen": 19240352,
+      "step": 1645
+    },
+    {
+      "epoch": 0.8498583569405099,
+      "grad_norm": 9.593843549483235,
+      "learning_rate": 6.625087856544416e-05,
+      "loss": 0.475,
+      "num_input_tokens_seen": 19298848,
+      "step": 1650
+    },
+    {
+      "epoch": 0.8498583569405099,
+      "eval_loss": 0.35129043459892273,
+      "eval_runtime": 19.4001,
+      "eval_samples_per_second": 3.093,
+      "eval_steps_per_second": 0.773,
+      "num_input_tokens_seen": 19298848,
+      "step": 1650
     }
   ],
   "logging_steps": 5,
   "max_steps": 3882,
-  "num_input_tokens_seen": 18714072,
+  "num_input_tokens_seen": 19298848,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2874,7 +2963,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1234783772868608.0,
+  "total_flos": 1273370793476096.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null