Model save

README.md

@@ -0,0 +1,69 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- ndcg
+- generated_from_trainer
+model-index:
+- name: mistral-7b-base-lipo-listsize_8-beta_0.05-batchsize_128
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhaoyang1/huggingface/runs/qrzgbz3l)
+# mistral-7b-base-lipo-listsize_8-beta_0.05-batchsize_128
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0021
+- Logps: -330.3465
+- Logits: -1.7109
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 1
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 16
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 128
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Logps     | Logits  |
+|:-------------:|:------:|:----:|:---------------:|:---------:|:-------:|
+| 0.0024        | 0.5343 | 250  | 0.0021          | -330.3465 | -1.7109 |
+
+
+### Framework versions
+
+- Transformers 4.42.0
+- Pytorch 2.3.0+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9981298423724285,
+    "total_flos": 0.0,
+    "train_loss": 0.0024537650077569853,
+    "train_runtime": 43696.9945,
+    "train_samples": 59880,
+    "train_samples_per_second": 1.37,
+    "train_steps_per_second": 0.011
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.42.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7808319aa61d29bf39c802d368cc3f93ab19d22ada69333bd8445ebe00542940
+size 4943162240

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a7635f3c0800cf3dbd3e12af09426333044a167f5c49d0776c73b20e04bea90
+size 4999819232

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44b9f19b827843139b772730347ea28b89bb6a84132912ef9452ffeee17068af
+size 4540516256

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9981298423724285,
+    "total_flos": 0.0,
+    "train_loss": 0.0024537650077569853,
+    "train_runtime": 43696.9945,
+    "train_samples": 59880,
+    "train_samples_per_second": 1.37,
+    "train_steps_per_second": 0.011
+}

trainer_state.json

@@ -0,0 +1,898 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9981298423724285,
+  "eval_steps": 250,
+  "global_step": 467,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0021373230029388193,
+      "grad_norm": 0.9382448260470807,
+      "learning_rate": 1.0638297872340425e-08,
+      "logits": -2.7276527881622314,
+      "logps": -123.19757843017578,
+      "loss": 0.0132,
+      "step": 1
+    },
+    {
+      "epoch": 0.010686615014694095,
+      "grad_norm": 0.6077971078164804,
+      "learning_rate": 5.3191489361702123e-08,
+      "logits": -2.8715453147888184,
+      "logps": -234.58905029296875,
+      "loss": 0.0122,
+      "step": 5
+    },
+    {
+      "epoch": 0.02137323002938819,
+      "grad_norm": 0.5751992418185391,
+      "learning_rate": 1.0638297872340425e-07,
+      "logits": -2.848034381866455,
+      "logps": -247.9265899658203,
+      "loss": 0.0084,
+      "step": 10
+    },
+    {
+      "epoch": 0.03205984504408229,
+      "grad_norm": 0.7990885869956119,
+      "learning_rate": 1.5957446808510638e-07,
+      "logits": -2.789750099182129,
+      "logps": -228.93344116210938,
+      "loss": 0.0079,
+      "step": 15
+    },
+    {
+      "epoch": 0.04274646005877638,
+      "grad_norm": 0.4707531974619967,
+      "learning_rate": 1.9148936170212765e-07,
+      "logits": -2.790773868560791,
+      "logps": -203.69082641601562,
+      "loss": 0.0067,
+      "step": 20
+    },
+    {
+      "epoch": 0.053433075073470476,
+      "grad_norm": 0.4013429152996433,
+      "learning_rate": 2.4468085106382976e-07,
+      "logits": -2.9664595127105713,
+      "logps": -289.47760009765625,
+      "loss": 0.0053,
+      "step": 25
+    },
+    {
+      "epoch": 0.06411969008816458,
+      "grad_norm": 0.2606608287569671,
+      "learning_rate": 2.872340425531915e-07,
+      "logits": -2.9390180110931396,
+      "logps": -280.2230224609375,
+      "loss": 0.0048,
+      "step": 30
+    },
+    {
+      "epoch": 0.07480630510285867,
+      "grad_norm": 0.17101584180223114,
+      "learning_rate": 3.404255319148936e-07,
+      "logits": -2.9212167263031006,
+      "logps": -236.70346069335938,
+      "loss": 0.0038,
+      "step": 35
+    },
+    {
+      "epoch": 0.08549292011755276,
+      "grad_norm": 0.2209413248930559,
+      "learning_rate": 3.9361702127659574e-07,
+      "logits": -2.9314119815826416,
+      "logps": -237.0644073486328,
+      "loss": 0.0033,
+      "step": 40
+    },
+    {
+      "epoch": 0.09617953513224686,
+      "grad_norm": 0.19081193545603162,
+      "learning_rate": 4.4680851063829783e-07,
+      "logits": -2.8745718002319336,
+      "logps": -262.94085693359375,
+      "loss": 0.0031,
+      "step": 45
+    },
+    {
+      "epoch": 0.10686615014694095,
+      "grad_norm": 0.29000989304909724,
+      "learning_rate": 5e-07,
+      "logits": -2.735830545425415,
+      "logps": -279.531494140625,
+      "loss": 0.0031,
+      "step": 50
+    },
+    {
+      "epoch": 0.11755276516163506,
+      "grad_norm": 0.1736830507456117,
+      "learning_rate": 4.998251761970996e-07,
+      "logits": -2.9761738777160645,
+      "logps": -290.0188903808594,
+      "loss": 0.0028,
+      "step": 55
+    },
+    {
+      "epoch": 0.12823938017632916,
+      "grad_norm": 0.19619045059609302,
+      "learning_rate": 4.993009492952949e-07,
+      "logits": -2.8654844760894775,
+      "logps": -253.6826629638672,
+      "loss": 0.0029,
+      "step": 60
+    },
+    {
+      "epoch": 0.13892599519102325,
+      "grad_norm": 0.15658392797413836,
+      "learning_rate": 4.984280524733107e-07,
+      "logits": -2.9113190174102783,
+      "logps": -287.5055847167969,
+      "loss": 0.0024,
+      "step": 65
+    },
+    {
+      "epoch": 0.14961261020571734,
+      "grad_norm": 0.280826775873623,
+      "learning_rate": 4.972077065562821e-07,
+      "logits": -2.7052161693573,
+      "logps": -270.00244140625,
+      "loss": 0.0031,
+      "step": 70
+    },
+    {
+      "epoch": 0.16029922522041143,
+      "grad_norm": 0.19625817049433308,
+      "learning_rate": 4.956416183083221e-07,
+      "logits": -2.6010804176330566,
+      "logps": -264.93060302734375,
+      "loss": 0.0026,
+      "step": 75
+    },
+    {
+      "epoch": 0.17098584023510552,
+      "grad_norm": 0.20134121833820165,
+      "learning_rate": 4.937319780454559e-07,
+      "logits": -2.8801825046539307,
+      "logps": -306.81439208984375,
+      "loss": 0.0036,
+      "step": 80
+    },
+    {
+      "epoch": 0.18167245524979964,
+      "grad_norm": 0.20678619269403417,
+      "learning_rate": 4.91481456572267e-07,
+      "logits": -2.620666027069092,
+      "logps": -306.35552978515625,
+      "loss": 0.0024,
+      "step": 85
+    },
+    {
+      "epoch": 0.19235907026449373,
+      "grad_norm": 0.24234453306339507,
+      "learning_rate": 4.894377041712326e-07,
+      "logits": -2.769148826599121,
+      "logps": -240.70516967773438,
+      "loss": 0.0028,
+      "step": 90
+    },
+    {
+      "epoch": 0.20304568527918782,
+      "grad_norm": 0.1774806126244229,
+      "learning_rate": 4.86581845949791e-07,
+      "logits": -2.5769433975219727,
+      "logps": -304.4381408691406,
+      "loss": 0.0025,
+      "step": 95
+    },
+    {
+      "epoch": 0.2137323002938819,
+      "grad_norm": 0.12713835218927472,
+      "learning_rate": 4.833951066243004e-07,
+      "logits": -2.529529094696045,
+      "logps": -258.7133483886719,
+      "loss": 0.0034,
+      "step": 100
+    },
+    {
+      "epoch": 0.224418915308576,
+      "grad_norm": 0.17305634898925182,
+      "learning_rate": 4.798819431378626e-07,
+      "logits": -2.5534040927886963,
+      "logps": -299.6002502441406,
+      "loss": 0.0023,
+      "step": 105
+    },
+    {
+      "epoch": 0.2351055303232701,
+      "grad_norm": 0.17241734067358394,
+      "learning_rate": 4.7604726896728496e-07,
+      "logits": -2.5559840202331543,
+      "logps": -313.1445007324219,
+      "loss": 0.0022,
+      "step": 110
+    },
+    {
+      "epoch": 0.2457921453379642,
+      "grad_norm": 0.37047445770869264,
+      "learning_rate": 4.718964472511385e-07,
+      "logits": -2.405649185180664,
+      "logps": -295.0511779785156,
+      "loss": 0.0032,
+      "step": 115
+    },
+    {
+      "epoch": 0.2564787603526583,
+      "grad_norm": 0.351138414882941,
+      "learning_rate": 4.6743528328892384e-07,
+      "logits": -2.518690586090088,
+      "logps": -265.8407287597656,
+      "loss": 0.0021,
+      "step": 120
+    },
+    {
+      "epoch": 0.2671653753673524,
+      "grad_norm": 0.15375436361571293,
+      "learning_rate": 4.626700164218349e-07,
+      "logits": -2.5620675086975098,
+      "logps": -289.02581787109375,
+      "loss": 0.0024,
+      "step": 125
+    },
+    {
+      "epoch": 0.2778519903820465,
+      "grad_norm": 0.17841955950576954,
+      "learning_rate": 4.576073113064759e-07,
+      "logits": -2.5668206214904785,
+      "logps": -285.832763671875,
+      "loss": 0.0021,
+      "step": 130
+    },
+    {
+      "epoch": 0.2885386053967406,
+      "grad_norm": 0.25549759297732705,
+      "learning_rate": 4.5225424859373684e-07,
+      "logits": -2.520317792892456,
+      "logps": -280.8033447265625,
+      "loss": 0.0022,
+      "step": 135
+    },
+    {
+      "epoch": 0.2992252204114347,
+      "grad_norm": 0.17168885542346832,
+      "learning_rate": 4.4661831502586244e-07,
+      "logits": -2.5304956436157227,
+      "logps": -315.8266296386719,
+      "loss": 0.0019,
+      "step": 140
+    },
+    {
+      "epoch": 0.30991183542612877,
+      "grad_norm": 0.2617360849968264,
+      "learning_rate": 4.407073929655666e-07,
+      "logits": -2.33339786529541,
+      "logps": -294.7607116699219,
+      "loss": 0.0023,
+      "step": 145
+    },
+    {
+      "epoch": 0.32059845044082286,
+      "grad_norm": 0.24053970622842247,
+      "learning_rate": 4.345297493718352e-07,
+      "logits": -2.4368555545806885,
+      "logps": -288.93389892578125,
+      "loss": 0.0034,
+      "step": 150
+    },
+    {
+      "epoch": 0.33128506545551695,
+      "grad_norm": 0.17732642380059183,
+      "learning_rate": 4.280940242378362e-07,
+      "logits": -2.405362844467163,
+      "logps": -320.8726501464844,
+      "loss": 0.0024,
+      "step": 155
+    },
+    {
+      "epoch": 0.34197168047021104,
+      "grad_norm": 0.1355464253010723,
+      "learning_rate": 4.2140921850710855e-07,
+      "logits": -2.2018160820007324,
+      "logps": -305.10626220703125,
+      "loss": 0.0019,
+      "step": 160
+    },
+    {
+      "epoch": 0.3526582954849052,
+      "grad_norm": 0.16961017757765015,
+      "learning_rate": 4.1448468148492814e-07,
+      "logits": -2.360142230987549,
+      "logps": -313.4556884765625,
+      "loss": 0.0019,
+      "step": 165
+    },
+    {
+      "epoch": 0.36334491049959927,
+      "grad_norm": 0.31250910225151746,
+      "learning_rate": 4.0733009776245937e-07,
+      "logits": -2.2883636951446533,
+      "logps": -278.54327392578125,
+      "loss": 0.0019,
+      "step": 170
+    },
+    {
+      "epoch": 0.37403152551429336,
+      "grad_norm": 0.16247846899403953,
+      "learning_rate": 3.9995547367197843e-07,
+      "logits": -2.2985727787017822,
+      "logps": -287.256591796875,
+      "loss": 0.0031,
+      "step": 175
+    },
+    {
+      "epoch": 0.38471814052898745,
+      "grad_norm": 0.18553209072310442,
+      "learning_rate": 3.92371123292113e-07,
+      "logits": -2.257999897003174,
+      "logps": -307.00048828125,
+      "loss": 0.0022,
+      "step": 180
+    },
+    {
+      "epoch": 0.39540475554368154,
+      "grad_norm": 0.11718026937432835,
+      "learning_rate": 3.8458765402267056e-07,
+      "logits": -2.169142246246338,
+      "logps": -253.388671875,
+      "loss": 0.0019,
+      "step": 185
+    },
+    {
+      "epoch": 0.40609137055837563,
+      "grad_norm": 0.12113177807346075,
+      "learning_rate": 3.766159517492307e-07,
+      "logits": -2.4464828968048096,
+      "logps": -285.96881103515625,
+      "loss": 0.0017,
+      "step": 190
+    },
+    {
+      "epoch": 0.4167779855730697,
+      "grad_norm": 0.30240590885572066,
+      "learning_rate": 3.6846716561824967e-07,
+      "logits": -2.2569479942321777,
+      "logps": -346.3056945800781,
+      "loss": 0.0017,
+      "step": 195
+    },
+    {
+      "epoch": 0.4274646005877638,
+      "grad_norm": 0.17258433198404743,
+      "learning_rate": 3.601526924439709e-07,
+      "logits": -2.108452320098877,
+      "logps": -281.43035888671875,
+      "loss": 0.0023,
+      "step": 200
+    },
+    {
+      "epoch": 0.4381512156024579,
+      "grad_norm": 0.17727230407582378,
+      "learning_rate": 3.5338962624163016e-07,
+      "logits": -1.8794915676116943,
+      "logps": -298.4540100097656,
+      "loss": 0.0021,
+      "step": 205
+    },
+    {
+      "epoch": 0.448837830617152,
+      "grad_norm": 0.20837993856290382,
+      "learning_rate": 3.448063656632321e-07,
+      "logits": -1.7576067447662354,
+      "logps": -303.8121643066406,
+      "loss": 0.0025,
+      "step": 210
+    },
+    {
+      "epoch": 0.45952444563184613,
+      "grad_norm": 0.2386532784792278,
+      "learning_rate": 3.360905098097587e-07,
+      "logits": -1.7971121072769165,
+      "logps": -280.39599609375,
+      "loss": 0.0016,
+      "step": 215
+    },
+    {
+      "epoch": 0.4702110606465402,
+      "grad_norm": 0.25342590908847396,
+      "learning_rate": 3.272542485937368e-07,
+      "logits": -1.7364866733551025,
+      "logps": -245.9429168701172,
+      "loss": 0.0031,
+      "step": 220
+    },
+    {
+      "epoch": 0.4808976756612343,
+      "grad_norm": 0.28055644744363367,
+      "learning_rate": 3.1830994032548e-07,
+      "logits": -1.8645261526107788,
+      "logps": -326.64093017578125,
+      "loss": 0.0017,
+      "step": 225
+    },
+    {
+      "epoch": 0.4915842906759284,
+      "grad_norm": 0.23825267279541876,
+      "learning_rate": 3.0927009442887437e-07,
+      "logits": -1.740753173828125,
+      "logps": -346.83685302734375,
+      "loss": 0.0026,
+      "step": 230
+    },
+    {
+      "epoch": 0.5022709056906225,
+      "grad_norm": 0.17759949053402713,
+      "learning_rate": 3.001473539458182e-07,
+      "logits": -1.6421802043914795,
+      "logps": -284.00006103515625,
+      "loss": 0.0024,
+      "step": 235
+    },
+    {
+      "epoch": 0.5129575207053166,
+      "grad_norm": 0.17993669671987442,
+      "learning_rate": 2.909544778537844e-07,
+      "logits": -1.6951431035995483,
+      "logps": -319.45477294921875,
+      "loss": 0.0033,
+      "step": 240
+    },
+    {
+      "epoch": 0.5236441357200107,
+      "grad_norm": 0.1819192210456823,
+      "learning_rate": 2.817043232212371e-07,
+      "logits": -1.9256826639175415,
+      "logps": -294.8505859375,
+      "loss": 0.0021,
+      "step": 245
+    },
+    {
+      "epoch": 0.5343307507347048,
+      "grad_norm": 0.13679569396445074,
+      "learning_rate": 2.7240982722585837e-07,
+      "logits": -2.037923574447632,
+      "logps": -286.97491455078125,
+      "loss": 0.0024,
+      "step": 250
+    },
+    {
+      "epoch": 0.5343307507347048,
+      "eval_logits": -1.7109427452087402,
+      "eval_logps": -330.3465270996094,
+      "eval_loss": 0.002132988767698407,
+      "eval_runtime": 2920.2979,
+      "eval_samples_per_second": 0.674,
+      "eval_steps_per_second": 0.042,
+      "step": 250
+    },
+    {
+      "epoch": 0.5450173657493989,
+      "grad_norm": 0.3067475776023036,
+      "learning_rate": 2.63083989060736e-07,
+      "logits": -1.3564677238464355,
+      "logps": -311.0221252441406,
+      "loss": 0.0017,
+      "step": 255
+    },
+    {
+      "epoch": 0.555703980764093,
+      "grad_norm": 0.23225688810898162,
+      "learning_rate": 2.537398517538159e-07,
+      "logits": -1.5437383651733398,
+      "logps": -308.88275146484375,
+      "loss": 0.0017,
+      "step": 260
+    },
+    {
+      "epoch": 0.566390595778787,
+      "grad_norm": 0.16415456855040866,
+      "learning_rate": 2.4439048392604877e-07,
+      "logits": -1.9235122203826904,
+      "logps": -291.63970947265625,
+      "loss": 0.0016,
+      "step": 265
+    },
+    {
+      "epoch": 0.5770772107934812,
+      "grad_norm": 0.2626871431753474,
+      "learning_rate": 2.3504896151374144e-07,
+      "logits": -1.6461937427520752,
+      "logps": -291.2130126953125,
+      "loss": 0.0017,
+      "step": 270
+    },
+    {
+      "epoch": 0.5877638258081752,
+      "grad_norm": 0.2142095147404927,
+      "learning_rate": 2.2572834948067795e-07,
+      "logits": -2.053422451019287,
+      "logps": -299.7975158691406,
+      "loss": 0.0015,
+      "step": 275
+    },
+    {
+      "epoch": 0.5984504408228694,
+      "grad_norm": 0.1729835472792332,
+      "learning_rate": 2.164416835455862e-07,
+      "logits": -1.5931049585342407,
+      "logps": -330.79833984375,
+      "loss": 0.0017,
+      "step": 280
+    },
+    {
+      "epoch": 0.6091370558375635,
+      "grad_norm": 0.13229138105992105,
+      "learning_rate": 2.072019519505062e-07,
+      "logits": -1.6202490329742432,
+      "logps": -274.6128234863281,
+      "loss": 0.0014,
+      "step": 285
+    },
+    {
+      "epoch": 0.6198236708522575,
+      "grad_norm": 0.13384864632448829,
+      "learning_rate": 1.980220772955602e-07,
+      "logits": -1.6888185739517212,
+      "logps": -331.9822998046875,
+      "loss": 0.0017,
+      "step": 290
+    },
+    {
+      "epoch": 0.6305102858669517,
+      "grad_norm": 0.2570231775542848,
+      "learning_rate": 1.8891489846552644e-07,
+      "logits": -1.2151944637298584,
+      "logps": -277.94366455078125,
+      "loss": 0.0015,
+      "step": 295
+    },
+    {
+      "epoch": 0.6411969008816457,
+      "grad_norm": 0.32823420976163764,
+      "learning_rate": 1.7989315267349933e-07,
+      "logits": -1.591247797012329,
+      "logps": -297.7398681640625,
+      "loss": 0.0018,
+      "step": 300
+    },
+    {
+      "epoch": 0.6518835158963399,
+      "grad_norm": 0.25010102936467765,
+      "learning_rate": 1.7096945764674398e-07,
+      "logits": -1.675546646118164,
+      "logps": -316.112060546875,
+      "loss": 0.0016,
+      "step": 305
+    },
+    {
+      "epoch": 0.6625701309110339,
+      "grad_norm": 0.4034349897330769,
+      "learning_rate": 1.6390949019024118e-07,
+      "logits": -1.6305315494537354,
+      "logps": -296.0116271972656,
+      "loss": 0.0018,
+      "step": 310
+    },
+    {
+      "epoch": 0.673256745925728,
+      "grad_norm": 0.12684350054052765,
+      "learning_rate": 1.5519363433676791e-07,
+      "logits": -1.9057996273040771,
+      "logps": -345.0762634277344,
+      "loss": 0.0015,
+      "step": 315
+    },
+    {
+      "epoch": 0.6839433609404221,
+      "grad_norm": 0.15556386654085777,
+      "learning_rate": 1.4661037375836987e-07,
+      "logits": -1.8693593740463257,
+      "logps": -288.85662841796875,
+      "loss": 0.0015,
+      "step": 320
+    },
+    {
+      "epoch": 0.6946299759551162,
+      "grad_norm": 0.09912065480120762,
+      "learning_rate": 1.381717129210918e-07,
+      "logits": -2.061394453048706,
+      "logps": -339.9905700683594,
+      "loss": 0.0021,
+      "step": 325
+    },
+    {
+      "epoch": 0.7053165909698104,
+      "grad_norm": 0.19349172782619234,
+      "learning_rate": 1.2988945405516565e-07,
+      "logits": -1.9794120788574219,
+      "logps": -321.97857666015625,
+      "loss": 0.0015,
+      "step": 330
+    },
+    {
+      "epoch": 0.7160032059845044,
+      "grad_norm": 0.1156190072354086,
+      "learning_rate": 1.2177518064852348e-07,
+      "logits": -1.5535465478897095,
+      "logps": -356.76190185546875,
+      "loss": 0.0012,
+      "step": 335
+    },
+    {
+      "epoch": 0.7266898209991985,
+      "grad_norm": 0.14209863022427785,
+      "learning_rate": 1.1384024124624322e-07,
+      "logits": -1.6569770574569702,
+      "logps": -293.7080993652344,
+      "loss": 0.0015,
+      "step": 340
+    },
+    {
+      "epoch": 0.7373764360138926,
+      "grad_norm": 0.19451581998026854,
+      "learning_rate": 1.0609573357858165e-07,
+      "logits": -1.84344482421875,
+      "logps": -326.22711181640625,
+      "loss": 0.0015,
+      "step": 345
+    },
+    {
+      "epoch": 0.7480630510285867,
+      "grad_norm": 0.13229136201466776,
+      "learning_rate": 9.855248903979505e-08,
+      "logits": -1.7190221548080444,
+      "logps": -290.7681579589844,
+      "loss": 0.0013,
+      "step": 350
+    },
+    {
+      "epoch": 0.7587496660432808,
+      "grad_norm": 0.17506198326200426,
+      "learning_rate": 9.12210575394553e-08,
+      "logits": -1.9327806234359741,
+      "logps": -286.25860595703125,
+      "loss": 0.0023,
+      "step": 355
+    },
+    {
+      "epoch": 0.7694362810579749,
+      "grad_norm": 0.2082357023255924,
+      "learning_rate": 8.411169274744723e-08,
+      "logits": -1.7046730518341064,
+      "logps": -314.0340270996094,
+      "loss": 0.0015,
+      "step": 360
+    },
+    {
+      "epoch": 0.7801228960726689,
+      "grad_norm": 0.19562312404071172,
+      "learning_rate": 7.723433775328384e-08,
+      "logits": -1.8511289358139038,
+      "logps": -292.6779479980469,
+      "loss": 0.0016,
+      "step": 365
+    },
+    {
+      "epoch": 0.7908095110873631,
+      "grad_norm": 0.12344514414713371,
+      "learning_rate": 7.059861115979701e-08,
+      "logits": -1.7055408954620361,
+      "logps": -299.0326843261719,
+      "loss": 0.0019,
+      "step": 370
+    },
+    {
+      "epoch": 0.8014961261020572,
+      "grad_norm": 1.3213611658871371,
+      "learning_rate": 6.42137936306514e-08,
+      "logits": -1.5638744831085205,
+      "logps": -307.2875671386719,
+      "loss": 0.0022,
+      "step": 375
+    },
+    {
+      "epoch": 0.8121827411167513,
+      "grad_norm": 0.1250994703925717,
+      "learning_rate": 5.808881491049722e-08,
+      "logits": -1.5469547510147095,
+      "logps": -311.3077697753906,
+      "loss": 0.0013,
+      "step": 380
+    },
+    {
+      "epoch": 0.8228693561314454,
+      "grad_norm": 0.1745123714331581,
+      "learning_rate": 5.223224133591475e-08,
+      "logits": -1.286905288696289,
+      "logps": -325.25128173828125,
+      "loss": 0.0024,
+      "step": 385
+    },
+    {
+      "epoch": 0.8335559711461394,
+      "grad_norm": 0.19638092392041206,
+      "learning_rate": 4.6652263854618016e-08,
+      "logits": -1.4627269506454468,
+      "logps": -309.8140869140625,
+      "loss": 0.0016,
+      "step": 390
+    },
+    {
+      "epoch": 0.8442425861608336,
+      "grad_norm": 0.11362891391073889,
+      "learning_rate": 4.1356686569674335e-08,
+      "logits": -1.6027376651763916,
+      "logps": -287.57379150390625,
+      "loss": 0.0024,
+      "step": 395
+    },
+    {
+      "epoch": 0.8549292011755276,
+      "grad_norm": 0.23114987885798075,
+      "learning_rate": 3.635291582475963e-08,
+      "logits": -1.7361466884613037,
+      "logps": -318.19342041015625,
+      "loss": 0.0015,
+      "step": 400
+    },
+    {
+      "epoch": 0.8656158161902218,
+      "grad_norm": 0.12775948188820038,
+      "learning_rate": 3.1647949845717585e-08,
+      "logits": -1.696567177772522,
+      "logps": -331.56842041015625,
+      "loss": 0.0015,
+      "step": 405
+    },
+    {
+      "epoch": 0.8763024312049158,
+      "grad_norm": 0.08933009596567083,
+      "learning_rate": 2.724836895290805e-08,
+      "logits": -1.4884788990020752,
+      "logps": -306.1937561035156,
+      "loss": 0.0015,
+      "step": 410
+    },
+    {
+      "epoch": 0.88698904621961,
+      "grad_norm": 0.1285330434833646,
+      "learning_rate": 2.3160326358033778e-08,
+      "logits": -1.578226089477539,
+      "logps": -289.6263122558594,
+      "loss": 0.0014,
+      "step": 415
+    },
+    {
+      "epoch": 0.897675661234304,
+      "grad_norm": 0.29555100432245013,
+      "learning_rate": 1.938953955831771e-08,
+      "logits": -1.9324296712875366,
+      "logps": -302.9831848144531,
+      "loss": 0.0018,
+      "step": 420
+    },
+    {
+      "epoch": 0.9083622762489981,
+      "grad_norm": 0.17141977440091252,
+      "learning_rate": 1.5941282340065697e-08,
+      "logits": -1.6776586771011353,
+      "logps": -301.5423889160156,
+      "loss": 0.0018,
+      "step": 425
+    },
+    {
+      "epoch": 0.9190488912636923,
+      "grad_norm": 0.1346355295397476,
+      "learning_rate": 1.2820377402800064e-08,
+      "logits": -1.7960350513458252,
+      "logps": -332.9203796386719,
+      "loss": 0.0013,
+      "step": 430
+    },
+    {
+      "epoch": 0.9297355062783863,
+      "grad_norm": 0.10793693371388081,
+      "learning_rate": 1.0031189614277763e-08,
+      "logits": -1.9094654321670532,
+      "logps": -301.3127136230469,
+      "loss": 0.0016,
+      "step": 435
+    },
+    {
+      "epoch": 0.9404221212930804,
+      "grad_norm": 0.1268480914372309,
+      "learning_rate": 7.577619905828281e-09,
+      "logits": -1.781385064125061,
+      "logps": -280.0783386230469,
+      "loss": 0.0021,
+      "step": 440
+    },
+    {
+      "epoch": 0.9511087363077745,
+      "grad_norm": 0.3706774830055159,
+      "learning_rate": 5.463099816548577e-09,
+      "logits": -1.7710893154144287,
+      "logps": -344.9855651855469,
+      "loss": 0.0021,
+      "step": 445
+    },
+    {
+      "epoch": 0.9617953513224686,
+      "grad_norm": 0.5771827809657561,
+      "learning_rate": 3.6905866939851983e-09,
+      "logits": -1.7609302997589111,
+      "logps": -303.08831787109375,
+      "loss": 0.0018,
+      "step": 450
+    },
+    {
+      "epoch": 0.9724819663371627,
+      "grad_norm": 0.1383764641180946,
+      "learning_rate": 2.2625595580163247e-09,
+      "logits": -1.5160179138183594,
+      "logps": -303.7861633300781,
+      "loss": 0.0014,
+      "step": 455
+    },
+    {
+      "epoch": 0.9831685813518568,
+      "grad_norm": 0.11550175444430529,
+      "learning_rate": 1.1810156337183908e-09,
+      "logits": -1.5184260606765747,
+      "logps": -268.4586181640625,
+      "loss": 0.0014,
+      "step": 460
+    },
+    {
+      "epoch": 0.9938551963665508,
+      "grad_norm": 0.24801238262954162,
+      "learning_rate": 4.4746755806621126e-10,
+      "logits": -1.7884981632232666,
+      "logps": -305.1263732910156,
+      "loss": 0.0022,
+      "step": 465
+    },
+    {
+      "epoch": 0.9981298423724285,
+      "step": 467,
+      "total_flos": 0.0,
+      "train_loss": 0.0024537650077569853,
+      "train_runtime": 43696.9945,
+      "train_samples_per_second": 1.37,
+      "train_steps_per_second": 0.011
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 467,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 250,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}