Model save

README.md

@@ -0,0 +1,72 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- generated_from_trainer
+model-index:
+- name: zephyr-spin-phi3-data
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-spin-phi3-data
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0000
+- Rewards/real: 0.3501
+- Rewards/generated: -23.8099
+- Rewards/accuracies: 1.0
+- Rewards/margins: 24.1600
+- Logps/generated: -916.0912
+- Logps/real: -130.5047
+- Logits/generated: -2.8901
+- Logits/real: -2.6092
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/real | Rewards/generated | Rewards/accuracies | Rewards/margins | Logps/generated | Logps/real | Logits/generated | Logits/real |
+|:-------------:|:-----:|:----:|:---------------:|:------------:|:-----------------:|:------------------:|:---------------:|:---------------:|:----------:|:----------------:|:-----------:|
+| 0.0           | 0.32  | 500  | 0.0001          | 0.3931       | -14.9119          | 1.0                | 15.3050         | -827.1113       | -130.0747  | -2.7616          | -2.7055     |
+| 0.0           | 0.64  | 1000 | 0.0000          | 0.4203       | -16.5266          | 1.0                | 16.9469         | -843.2585       | -129.8029  | -2.8325          | -2.7464     |
+| 0.0           | 0.96  | 1500 | 0.0000          | 0.3501       | -23.8099          | 1.0                | 24.1600         | -916.0912       | -130.5047  | -2.8901          | -2.6092     |
+
+
+### Framework versions
+
+- Transformers 4.37.0
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.009621814649877188,
+    "train_runtime": 14787.3165,
+    "train_samples": 50000,
+    "train_samples_per_second": 3.381,
+    "train_steps_per_second": 0.106
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.37.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e124988a82bd9aeac3905129b54ac6413131f35f489af3491f39d2771f1f1709
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87fcb459a7638cd05f80e05b0715f2e11e1959156c303265fca07d3128e64650
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3c2d29fe8bc880ec39aa519881a4328fa33478212ac88ec5c48a4fd192a54d7
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Jul21_15-16-37_gilbreth-j001.rcac.purdue.edu/events.out.tfevents.1721589594.gilbreth-j001.rcac.purdue.edu.235220.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9ec812da382e7b630602ec7d974ae400703cb9f5617012765b43deb9d311dc3
-size 101946
+oid sha256:aa8a085f85db684c97c96f4bf579603cc7bbeb1c3c26667807f694bf34802532
+size 106086

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.009621814649877188,
+    "train_runtime": 14787.3165,
+    "train_samples": 50000,
+    "train_samples_per_second": 3.381,
+    "train_steps_per_second": 0.106
+}

trainer_state.json

@@ -0,0 +1,2276 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1563,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.1847133757961784e-09,
+      "logits/generated": -2.8295888900756836,
+      "logits/real": -2.849569320678711,
+      "logps/generated": -609.6478881835938,
+      "logps/real": -115.27798461914062,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/generated": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/real": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.184713375796178e-08,
+      "logits/generated": -2.6501715183258057,
+      "logits/real": -2.821021318435669,
+      "logps/generated": -688.5430297851562,
+      "logps/real": -131.44122314453125,
+      "loss": 0.5833,
+      "rewards/accuracies": 0.6388888955116272,
+      "rewards/generated": -0.34704259037971497,
+      "rewards/margins": 0.3495745360851288,
+      "rewards/real": 0.0025319471023976803,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.369426751592356e-08,
+      "logits/generated": -2.714224338531494,
+      "logits/real": -2.780827522277832,
+      "logps/generated": -677.5689086914062,
+      "logps/real": -129.50588989257812,
+      "loss": 0.3907,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/generated": -1.4221687316894531,
+      "rewards/margins": 1.4617396593093872,
+      "rewards/real": 0.039571087807416916,
+      "step": 20
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.554140127388536e-08,
+      "logits/generated": -2.7694613933563232,
+      "logits/real": -2.860546588897705,
+      "logps/generated": -698.780517578125,
+      "logps/real": -139.16226196289062,
+      "loss": 0.2306,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -2.204209089279175,
+      "rewards/margins": 2.362700939178467,
+      "rewards/real": 0.15849189460277557,
+      "step": 30
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.2738853503184713e-07,
+      "logits/generated": -2.678589344024658,
+      "logits/real": -2.7937369346618652,
+      "logps/generated": -756.0526123046875,
+      "logps/real": -132.27268981933594,
+      "loss": 0.1129,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -4.11299467086792,
+      "rewards/margins": 4.405646800994873,
+      "rewards/real": 0.29265230894088745,
+      "step": 40
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.592356687898089e-07,
+      "logits/generated": -2.719285726547241,
+      "logits/real": -2.8095831871032715,
+      "logps/generated": -749.7476806640625,
+      "logps/real": -123.66046142578125,
+      "loss": 0.0573,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -4.803214073181152,
+      "rewards/margins": 5.2087225914001465,
+      "rewards/real": 0.4055088460445404,
+      "step": 50
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9108280254777072e-07,
+      "logits/generated": -2.685657024383545,
+      "logits/real": -2.817525625228882,
+      "logps/generated": -743.1283569335938,
+      "logps/real": -132.15084838867188,
+      "loss": 0.0325,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -4.882467746734619,
+      "rewards/margins": 5.401439189910889,
+      "rewards/real": 0.5189720392227173,
+      "step": 60
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.2292993630573247e-07,
+      "logits/generated": -2.6736221313476562,
+      "logits/real": -2.782536029815674,
+      "logps/generated": -779.1280517578125,
+      "logps/real": -136.8399200439453,
+      "loss": 0.0175,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -5.401805400848389,
+      "rewards/margins": 5.999195098876953,
+      "rewards/real": 0.597389817237854,
+      "step": 70
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.5477707006369425e-07,
+      "logits/generated": -2.716283082962036,
+      "logits/real": -2.7836098670959473,
+      "logps/generated": -763.261962890625,
+      "logps/real": -121.11332702636719,
+      "loss": 0.0154,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -5.937032222747803,
+      "rewards/margins": 6.565484046936035,
+      "rewards/real": 0.6284510493278503,
+      "step": 80
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.86624203821656e-07,
+      "logits/generated": -2.752387285232544,
+      "logits/real": -2.8120365142822266,
+      "logps/generated": -760.432373046875,
+      "logps/real": -132.00917053222656,
+      "loss": 0.0078,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -6.373486518859863,
+      "rewards/margins": 7.068659782409668,
+      "rewards/real": 0.6951735615730286,
+      "step": 90
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.184713375796178e-07,
+      "logits/generated": -2.702080488204956,
+      "logits/real": -2.792630910873413,
+      "logps/generated": -744.688232421875,
+      "logps/real": -120.1148910522461,
+      "loss": 0.0086,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -5.977693557739258,
+      "rewards/margins": 6.694817543029785,
+      "rewards/real": 0.7171245217323303,
+      "step": 100
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.5031847133757957e-07,
+      "logits/generated": -2.6754021644592285,
+      "logits/real": -2.786886215209961,
+      "logps/generated": -726.2047119140625,
+      "logps/real": -138.05221557617188,
+      "loss": 0.006,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -6.919286251068115,
+      "rewards/margins": 7.560235500335693,
+      "rewards/real": 0.6409494280815125,
+      "step": 110
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.8216560509554143e-07,
+      "logits/generated": -2.7799072265625,
+      "logits/real": -2.834855556488037,
+      "logps/generated": -731.4354248046875,
+      "logps/real": -123.99066162109375,
+      "loss": 0.0041,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -6.255575656890869,
+      "rewards/margins": 7.035998344421387,
+      "rewards/real": 0.7804235219955444,
+      "step": 120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.140127388535032e-07,
+      "logits/generated": -2.695237398147583,
+      "logits/real": -2.808246612548828,
+      "logps/generated": -781.197265625,
+      "logps/real": -134.5619354248047,
+      "loss": 0.0041,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -7.42046594619751,
+      "rewards/margins": 8.036266326904297,
+      "rewards/real": 0.6158010363578796,
+      "step": 130
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.4585987261146494e-07,
+      "logits/generated": -2.690974712371826,
+      "logits/real": -2.7843871116638184,
+      "logps/generated": -769.6416625976562,
+      "logps/real": -118.09663391113281,
+      "loss": 0.0029,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -8.030922889709473,
+      "rewards/margins": 8.821279525756836,
+      "rewards/real": 0.7903567552566528,
+      "step": 140
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.777070063694267e-07,
+      "logits/generated": -2.6999502182006836,
+      "logits/real": -2.845780372619629,
+      "logps/generated": -756.4619750976562,
+      "logps/real": -128.97555541992188,
+      "loss": 0.0027,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -7.640128135681152,
+      "rewards/margins": 8.349076271057129,
+      "rewards/real": 0.7089481353759766,
+      "step": 150
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.989331436699858e-07,
+      "logits/generated": -2.770327568054199,
+      "logits/real": -2.7911763191223145,
+      "logps/generated": -772.9364624023438,
+      "logps/real": -125.63675689697266,
+      "loss": 0.0019,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -8.297707557678223,
+      "rewards/margins": 9.060877799987793,
+      "rewards/real": 0.7631710767745972,
+      "step": 160
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.953769559032717e-07,
+      "logits/generated": -2.700230121612549,
+      "logits/real": -2.770098924636841,
+      "logps/generated": -789.2719116210938,
+      "logps/real": -118.48409271240234,
+      "loss": 0.0015,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -8.936752319335938,
+      "rewards/margins": 9.651647567749023,
+      "rewards/real": 0.7148973345756531,
+      "step": 170
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.918207681365576e-07,
+      "logits/generated": -2.715512752532959,
+      "logits/real": -2.8020758628845215,
+      "logps/generated": -785.4468383789062,
+      "logps/real": -141.07501220703125,
+      "loss": 0.0016,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -9.002473831176758,
+      "rewards/margins": 9.73188591003418,
+      "rewards/real": 0.7294121980667114,
+      "step": 180
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.882645803698435e-07,
+      "logits/generated": -2.7685036659240723,
+      "logits/real": -2.789585590362549,
+      "logps/generated": -773.3211669921875,
+      "logps/real": -132.91026306152344,
+      "loss": 0.0009,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -9.269811630249023,
+      "rewards/margins": 10.108678817749023,
+      "rewards/real": 0.8388668298721313,
+      "step": 190
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.847083926031294e-07,
+      "logits/generated": -2.7428407669067383,
+      "logits/real": -2.7933878898620605,
+      "logps/generated": -822.3059692382812,
+      "logps/real": -119.96354675292969,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -9.959307670593262,
+      "rewards/margins": 10.764963150024414,
+      "rewards/real": 0.8056550025939941,
+      "step": 200
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.811522048364154e-07,
+      "logits/generated": -2.6588082313537598,
+      "logits/real": -2.753288984298706,
+      "logps/generated": -809.4946899414062,
+      "logps/real": -112.04827880859375,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -9.868528366088867,
+      "rewards/margins": 10.617731094360352,
+      "rewards/real": 0.7492026090621948,
+      "step": 210
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.775960170697012e-07,
+      "logits/generated": -2.6897635459899902,
+      "logits/real": -2.770383596420288,
+      "logps/generated": -840.3517456054688,
+      "logps/real": -111.0347900390625,
+      "loss": 0.0004,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -10.592704772949219,
+      "rewards/margins": 11.492315292358398,
+      "rewards/real": 0.8996096849441528,
+      "step": 220
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.7403982930298717e-07,
+      "logits/generated": -2.7826247215270996,
+      "logits/real": -2.786170482635498,
+      "logps/generated": -804.2281494140625,
+      "logps/real": -119.8484115600586,
+      "loss": 0.0011,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -9.724154472351074,
+      "rewards/margins": 10.552526473999023,
+      "rewards/real": 0.8283706903457642,
+      "step": 230
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.7048364153627306e-07,
+      "logits/generated": -2.7883107662200928,
+      "logits/real": -2.7733795642852783,
+      "logps/generated": -787.4295654296875,
+      "logps/real": -148.79747009277344,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -9.756797790527344,
+      "rewards/margins": 10.487835884094238,
+      "rewards/real": 0.7310384511947632,
+      "step": 240
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.66927453769559e-07,
+      "logits/generated": -2.7394678592681885,
+      "logits/real": -2.790409803390503,
+      "logps/generated": -819.8644409179688,
+      "logps/real": -116.63028717041016,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -10.328279495239258,
+      "rewards/margins": 11.112763404846191,
+      "rewards/real": 0.7844842672348022,
+      "step": 250
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.633712660028449e-07,
+      "logits/generated": -2.727818012237549,
+      "logits/real": -2.7909488677978516,
+      "logps/generated": -797.3067016601562,
+      "logps/real": -117.8537826538086,
+      "loss": 0.0004,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -10.560891151428223,
+      "rewards/margins": 11.291653633117676,
+      "rewards/real": 0.7307616472244263,
+      "step": 260
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.5981507823613085e-07,
+      "logits/generated": -2.767531633377075,
+      "logits/real": -2.7978005409240723,
+      "logps/generated": -810.1624755859375,
+      "logps/real": -121.3199691772461,
+      "loss": 0.0004,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -10.349993705749512,
+      "rewards/margins": 11.145318984985352,
+      "rewards/real": 0.7953254580497742,
+      "step": 270
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.562588904694168e-07,
+      "logits/generated": -2.778958559036255,
+      "logits/real": -2.767667293548584,
+      "logps/generated": -821.5338134765625,
+      "logps/real": -130.97152709960938,
+      "loss": 0.0005,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -11.092814445495605,
+      "rewards/margins": 11.828493118286133,
+      "rewards/real": 0.7356794476509094,
+      "step": 280
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.5270270270270264e-07,
+      "logits/generated": -2.732811450958252,
+      "logits/real": -2.771510601043701,
+      "logps/generated": -757.4833374023438,
+      "logps/real": -126.4487075805664,
+      "loss": 0.0006,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -10.417075157165527,
+      "rewards/margins": 11.195660591125488,
+      "rewards/real": 0.7785850167274475,
+      "step": 290
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.491465149359886e-07,
+      "logits/generated": -2.713351249694824,
+      "logits/real": -2.756260395050049,
+      "logps/generated": -806.474609375,
+      "logps/real": -118.60646057128906,
+      "loss": 0.0003,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -11.823250770568848,
+      "rewards/margins": 12.591789245605469,
+      "rewards/real": 0.7685383558273315,
+      "step": 300
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.4559032716927454e-07,
+      "logits/generated": -2.8016388416290283,
+      "logits/real": -2.796672821044922,
+      "logps/generated": -763.6119995117188,
+      "logps/real": -142.29685974121094,
+      "loss": 0.0003,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -10.680870056152344,
+      "rewards/margins": 11.322160720825195,
+      "rewards/real": 0.6412909030914307,
+      "step": 310
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.420341394025605e-07,
+      "logits/generated": -2.7270781993865967,
+      "logits/real": -2.7888123989105225,
+      "logps/generated": -799.5844116210938,
+      "logps/real": -130.8887481689453,
+      "loss": 0.0002,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -11.834752082824707,
+      "rewards/margins": 12.512211799621582,
+      "rewards/real": 0.6774585247039795,
+      "step": 320
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.384779516358463e-07,
+      "logits/generated": -2.79298734664917,
+      "logits/real": -2.784541606903076,
+      "logps/generated": -788.7904052734375,
+      "logps/real": -134.79293823242188,
+      "loss": 0.0003,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -11.506429672241211,
+      "rewards/margins": 12.11829948425293,
+      "rewards/real": 0.6118704080581665,
+      "step": 330
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.3492176386913227e-07,
+      "logits/generated": -2.8363544940948486,
+      "logits/real": -2.8054802417755127,
+      "logps/generated": -768.1728515625,
+      "logps/real": -139.9955291748047,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -11.797532081604004,
+      "rewards/margins": 12.470538139343262,
+      "rewards/real": 0.6730067133903503,
+      "step": 340
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.313655761024182e-07,
+      "logits/generated": -2.7400827407836914,
+      "logits/real": -2.7338576316833496,
+      "logps/generated": -811.0081787109375,
+      "logps/real": -131.45535278320312,
+      "loss": 0.0002,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -12.710081100463867,
+      "rewards/margins": 13.333477973937988,
+      "rewards/real": 0.6233970522880554,
+      "step": 350
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.278093883357041e-07,
+      "logits/generated": -2.776153087615967,
+      "logits/real": -2.750797748565674,
+      "logps/generated": -819.32666015625,
+      "logps/real": -126.93217468261719,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -12.181897163391113,
+      "rewards/margins": 12.906854629516602,
+      "rewards/real": 0.7249582409858704,
+      "step": 360
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.2425320056899e-07,
+      "logits/generated": -2.7904210090637207,
+      "logits/real": -2.7961840629577637,
+      "logps/generated": -789.5787963867188,
+      "logps/real": -147.4117431640625,
+      "loss": 0.0003,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -11.834062576293945,
+      "rewards/margins": 12.519264221191406,
+      "rewards/real": 0.6852015256881714,
+      "step": 370
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.2069701280227595e-07,
+      "logits/generated": -2.750471830368042,
+      "logits/real": -2.772777795791626,
+      "logps/generated": -819.3651123046875,
+      "logps/real": -135.7245330810547,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -12.894061088562012,
+      "rewards/margins": 13.609522819519043,
+      "rewards/real": 0.7154618501663208,
+      "step": 380
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.1714082503556185e-07,
+      "logits/generated": -2.7443814277648926,
+      "logits/real": -2.745856285095215,
+      "logps/generated": -827.7913208007812,
+      "logps/real": -126.5484390258789,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -13.79316234588623,
+      "rewards/margins": 14.394973754882812,
+      "rewards/real": 0.601812481880188,
+      "step": 390
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.135846372688478e-07,
+      "logits/generated": -2.812390089035034,
+      "logits/real": -2.73819637298584,
+      "logps/generated": -808.9031982421875,
+      "logps/real": -131.18746948242188,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -12.755112648010254,
+      "rewards/margins": 13.416218757629395,
+      "rewards/real": 0.6611047983169556,
+      "step": 400
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.100284495021337e-07,
+      "logits/generated": -2.820923089981079,
+      "logits/real": -2.7527689933776855,
+      "logps/generated": -823.16796875,
+      "logps/real": -127.68003845214844,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -12.786894798278809,
+      "rewards/margins": 13.430368423461914,
+      "rewards/real": 0.6434718370437622,
+      "step": 410
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.064722617354196e-07,
+      "logits/generated": -2.678879737854004,
+      "logits/real": -2.7136893272399902,
+      "logps/generated": -872.8970947265625,
+      "logps/real": -131.42593383789062,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -13.947868347167969,
+      "rewards/margins": 14.58533000946045,
+      "rewards/real": 0.637461245059967,
+      "step": 420
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.0291607396870553e-07,
+      "logits/generated": -2.7224462032318115,
+      "logits/real": -2.7085747718811035,
+      "logps/generated": -834.8814697265625,
+      "logps/real": -122.5090560913086,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -13.601274490356445,
+      "rewards/margins": 14.206278800964355,
+      "rewards/real": 0.6050056219100952,
+      "step": 430
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 3.993598862019915e-07,
+      "logits/generated": -2.783932685852051,
+      "logits/real": -2.7148799896240234,
+      "logps/generated": -828.7579956054688,
+      "logps/real": -125.92060852050781,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -13.426411628723145,
+      "rewards/margins": 14.101513862609863,
+      "rewards/real": 0.6751025915145874,
+      "step": 440
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 3.9580369843527737e-07,
+      "logits/generated": -2.774967670440674,
+      "logits/real": -2.701488971710205,
+      "logps/generated": -873.3244018554688,
+      "logps/real": -123.95247650146484,
+      "loss": 0.0002,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -14.389918327331543,
+      "rewards/margins": 14.915544509887695,
+      "rewards/real": 0.525626540184021,
+      "step": 450
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 3.9224751066856327e-07,
+      "logits/generated": -2.735586643218994,
+      "logits/real": -2.7377548217773438,
+      "logps/generated": -805.3878173828125,
+      "logps/real": -155.18560791015625,
+      "loss": 0.0002,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -14.252492904663086,
+      "rewards/margins": 14.655688285827637,
+      "rewards/real": 0.4031934142112732,
+      "step": 460
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 3.886913229018492e-07,
+      "logits/generated": -2.7526440620422363,
+      "logits/real": -2.7074286937713623,
+      "logps/generated": -821.6927490234375,
+      "logps/real": -128.5563507080078,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -14.784818649291992,
+      "rewards/margins": 15.3468599319458,
+      "rewards/real": 0.562040388584137,
+      "step": 470
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 3.851351351351351e-07,
+      "logits/generated": -2.7571702003479004,
+      "logits/real": -2.737308979034424,
+      "logps/generated": -852.9948120117188,
+      "logps/real": -132.78759765625,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -15.543283462524414,
+      "rewards/margins": 16.045238494873047,
+      "rewards/real": 0.5019546747207642,
+      "step": 480
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 3.8157894736842105e-07,
+      "logits/generated": -2.706204891204834,
+      "logits/real": -2.6772992610931396,
+      "logps/generated": -849.1552734375,
+      "logps/real": -120.03173828125,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -15.504430770874023,
+      "rewards/margins": 15.993593215942383,
+      "rewards/real": 0.4891592860221863,
+      "step": 490
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 3.7802275960170695e-07,
+      "logits/generated": -2.7938857078552246,
+      "logits/real": -2.740180253982544,
+      "logps/generated": -782.1716918945312,
+      "logps/real": -129.24673461914062,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -14.547981262207031,
+      "rewards/margins": 15.090237617492676,
+      "rewards/real": 0.5422547459602356,
+      "step": 500
+    },
+    {
+      "epoch": 0.32,
+      "eval_logits/generated": -2.761601686477661,
+      "eval_logits/real": -2.705458402633667,
+      "eval_logps/generated": -827.1112670898438,
+      "eval_logps/real": -130.07472229003906,
+      "eval_loss": 5.503268039319664e-05,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/generated": -14.911882400512695,
+      "eval_rewards/margins": 15.304994583129883,
+      "eval_rewards/real": 0.3931117355823517,
+      "eval_runtime": 66.014,
+      "eval_samples_per_second": 7.574,
+      "eval_steps_per_second": 0.242,
+      "step": 500
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 3.7446657183499284e-07,
+      "logits/generated": -2.8136253356933594,
+      "logits/real": -2.669490337371826,
+      "logps/generated": -792.15380859375,
+      "logps/real": -127.56230163574219,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -14.737091064453125,
+      "rewards/margins": 15.237916946411133,
+      "rewards/real": 0.5008259415626526,
+      "step": 510
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 3.709103840682788e-07,
+      "logits/generated": -2.787266731262207,
+      "logits/real": -2.670997142791748,
+      "logps/generated": -824.7960205078125,
+      "logps/real": -124.60465240478516,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -15.136065483093262,
+      "rewards/margins": 15.58125114440918,
+      "rewards/real": 0.4451850950717926,
+      "step": 520
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 3.6735419630156474e-07,
+      "logits/generated": -2.704446315765381,
+      "logits/real": -2.6110129356384277,
+      "logps/generated": -878.9093017578125,
+      "logps/real": -118.95606994628906,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -16.33749008178711,
+      "rewards/margins": 16.747507095336914,
+      "rewards/real": 0.41001471877098083,
+      "step": 530
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 3.637980085348506e-07,
+      "logits/generated": -2.794490098953247,
+      "logits/real": -2.6710212230682373,
+      "logps/generated": -834.6387939453125,
+      "logps/real": -130.42050170898438,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -15.052263259887695,
+      "rewards/margins": 15.429656982421875,
+      "rewards/real": 0.37739241123199463,
+      "step": 540
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 3.602418207681365e-07,
+      "logits/generated": -2.7407026290893555,
+      "logits/real": -2.6537132263183594,
+      "logps/generated": -880.5545043945312,
+      "logps/real": -135.30288696289062,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -16.217912673950195,
+      "rewards/margins": 16.690380096435547,
+      "rewards/real": 0.47246813774108887,
+      "step": 550
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.5668563300142247e-07,
+      "logits/generated": -2.81174898147583,
+      "logits/real": -2.6751866340637207,
+      "logps/generated": -854.27490234375,
+      "logps/real": -126.11138916015625,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -16.003740310668945,
+      "rewards/margins": 16.378582000732422,
+      "rewards/real": 0.3748398423194885,
+      "step": 560
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.5312944523470837e-07,
+      "logits/generated": -2.741673469543457,
+      "logits/real": -2.5907273292541504,
+      "logps/generated": -888.6085815429688,
+      "logps/real": -128.69569396972656,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -17.06721305847168,
+      "rewards/margins": 17.444976806640625,
+      "rewards/real": 0.37776434421539307,
+      "step": 570
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.495732574679943e-07,
+      "logits/generated": -2.7112770080566406,
+      "logits/real": -2.647355794906616,
+      "logps/generated": -842.6921997070312,
+      "logps/real": -126.5383529663086,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -16.2426815032959,
+      "rewards/margins": 16.684232711791992,
+      "rewards/real": 0.4415510296821594,
+      "step": 580
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.460170697012802e-07,
+      "logits/generated": -2.7611031532287598,
+      "logits/real": -2.587040901184082,
+      "logps/generated": -856.2335815429688,
+      "logps/real": -131.83444213867188,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -16.540231704711914,
+      "rewards/margins": 16.8332576751709,
+      "rewards/real": 0.29302695393562317,
+      "step": 590
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.424608819345661e-07,
+      "logits/generated": -2.749112844467163,
+      "logits/real": -2.5918571949005127,
+      "logps/generated": -806.6871948242188,
+      "logps/real": -124.9672622680664,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -15.760574340820312,
+      "rewards/margins": 16.10599136352539,
+      "rewards/real": 0.3454182744026184,
+      "step": 600
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.3890469416785205e-07,
+      "logits/generated": -2.7701778411865234,
+      "logits/real": -2.6194324493408203,
+      "logps/generated": -830.5565185546875,
+      "logps/real": -129.43264770507812,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -16.803186416625977,
+      "rewards/margins": 17.231233596801758,
+      "rewards/real": 0.42804789543151855,
+      "step": 610
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.35348506401138e-07,
+      "logits/generated": -2.814532995223999,
+      "logits/real": -2.5871658325195312,
+      "logps/generated": -834.9091796875,
+      "logps/real": -127.82197570800781,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -17.46357536315918,
+      "rewards/margins": 17.733022689819336,
+      "rewards/real": 0.2694476246833801,
+      "step": 620
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.3179231863442384e-07,
+      "logits/generated": -2.748018741607666,
+      "logits/real": -2.5878210067749023,
+      "logps/generated": -869.302734375,
+      "logps/real": -141.6997833251953,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.122358322143555,
+      "rewards/margins": 18.34942626953125,
+      "rewards/real": 0.22706761956214905,
+      "step": 630
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.282361308677098e-07,
+      "logits/generated": -2.792604684829712,
+      "logits/real": -2.5345077514648438,
+      "logps/generated": -875.5255737304688,
+      "logps/real": -133.2880096435547,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.701038360595703,
+      "rewards/margins": 19.063264846801758,
+      "rewards/real": 0.3622281849384308,
+      "step": 640
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.2467994310099573e-07,
+      "logits/generated": -2.775300979614258,
+      "logits/real": -2.560939311981201,
+      "logps/generated": -871.24853515625,
+      "logps/real": -142.5642852783203,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.207752227783203,
+      "rewards/margins": 18.60503387451172,
+      "rewards/real": 0.3972865343093872,
+      "step": 650
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.211237553342817e-07,
+      "logits/generated": -2.835697650909424,
+      "logits/real": -2.570935010910034,
+      "logps/generated": -844.2117309570312,
+      "logps/real": -129.60166931152344,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -17.626232147216797,
+      "rewards/margins": 17.888334274291992,
+      "rewards/real": 0.26210257411003113,
+      "step": 660
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.175675675675675e-07,
+      "logits/generated": -2.7552971839904785,
+      "logits/real": -2.5506412982940674,
+      "logps/generated": -894.5694580078125,
+      "logps/real": -119.1685791015625,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.028125762939453,
+      "rewards/margins": 19.389427185058594,
+      "rewards/real": 0.36130291223526,
+      "step": 670
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.1401137980085347e-07,
+      "logits/generated": -2.7464940547943115,
+      "logits/real": -2.5819218158721924,
+      "logps/generated": -898.6015625,
+      "logps/real": -131.2238311767578,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.07114601135254,
+      "rewards/margins": 19.419193267822266,
+      "rewards/real": 0.34804823994636536,
+      "step": 680
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.104551920341394e-07,
+      "logits/generated": -2.7470998764038086,
+      "logits/real": -2.5765349864959717,
+      "logps/generated": -885.7230224609375,
+      "logps/real": -134.91915893554688,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.37563705444336,
+      "rewards/margins": 18.524120330810547,
+      "rewards/real": 0.14848431944847107,
+      "step": 690
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.068990042674253e-07,
+      "logits/generated": -2.734856128692627,
+      "logits/real": -2.512298107147217,
+      "logps/generated": -853.0060424804688,
+      "logps/real": -120.59394836425781,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.26091194152832,
+      "rewards/margins": 18.471248626708984,
+      "rewards/real": 0.21033525466918945,
+      "step": 700
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.033428165007112e-07,
+      "logits/generated": -2.7379255294799805,
+      "logits/real": -2.524719715118408,
+      "logps/generated": -912.4431762695312,
+      "logps/real": -123.26702880859375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.675243377685547,
+      "rewards/margins": 19.047819137573242,
+      "rewards/real": 0.37257617712020874,
+      "step": 710
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 2.9978662873399715e-07,
+      "logits/generated": -2.7684216499328613,
+      "logits/real": -2.531463146209717,
+      "logps/generated": -881.7340698242188,
+      "logps/real": -134.31008911132812,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.38132667541504,
+      "rewards/margins": 19.59175682067871,
+      "rewards/real": 0.21043212711811066,
+      "step": 720
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 2.9623044096728305e-07,
+      "logits/generated": -2.7709155082702637,
+      "logits/real": -2.562648057937622,
+      "logps/generated": -887.0978393554688,
+      "logps/real": -145.66043090820312,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.221771240234375,
+      "rewards/margins": 19.359745025634766,
+      "rewards/real": 0.13797567784786224,
+      "step": 730
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 2.92674253200569e-07,
+      "logits/generated": -2.8770792484283447,
+      "logits/real": -2.5876846313476562,
+      "logps/generated": -835.0736083984375,
+      "logps/real": -131.42913818359375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.772159576416016,
+      "rewards/margins": 19.107730865478516,
+      "rewards/real": 0.33557194471359253,
+      "step": 740
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 2.8911806543385494e-07,
+      "logits/generated": -2.734930992126465,
+      "logits/real": -2.5578300952911377,
+      "logps/generated": -874.3038330078125,
+      "logps/real": -145.36695861816406,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.031780242919922,
+      "rewards/margins": 20.371191024780273,
+      "rewards/real": 0.33940908312797546,
+      "step": 750
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.855618776671408e-07,
+      "logits/generated": -2.6914491653442383,
+      "logits/real": -2.576624631881714,
+      "logps/generated": -893.9830322265625,
+      "logps/real": -153.35513305664062,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.32299041748047,
+      "rewards/margins": 19.533132553100586,
+      "rewards/real": 0.21014323830604553,
+      "step": 760
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.8200568990042673e-07,
+      "logits/generated": -2.7508046627044678,
+      "logits/real": -2.489609956741333,
+      "logps/generated": -886.3199462890625,
+      "logps/real": -132.56236267089844,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.033309936523438,
+      "rewards/margins": 20.369976043701172,
+      "rewards/real": 0.3366653025150299,
+      "step": 770
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.784495021337127e-07,
+      "logits/generated": -2.7089645862579346,
+      "logits/real": -2.4881272315979004,
+      "logps/generated": -903.6559448242188,
+      "logps/real": -122.16401672363281,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.558847427368164,
+      "rewards/margins": 19.884899139404297,
+      "rewards/real": 0.32605427503585815,
+      "step": 780
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.7489331436699857e-07,
+      "logits/generated": -2.7490928173065186,
+      "logits/real": -2.536649703979492,
+      "logps/generated": -893.4107666015625,
+      "logps/real": -141.30215454101562,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.618242263793945,
+      "rewards/margins": 19.87813949584961,
+      "rewards/real": 0.25990021228790283,
+      "step": 790
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.7133712660028446e-07,
+      "logits/generated": -2.754713535308838,
+      "logits/real": -2.4825220108032227,
+      "logps/generated": -877.2716064453125,
+      "logps/real": -122.8569107055664,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.619098663330078,
+      "rewards/margins": 19.970170974731445,
+      "rewards/real": 0.35107091069221497,
+      "step": 800
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.677809388335704e-07,
+      "logits/generated": -2.7757420539855957,
+      "logits/real": -2.5053367614746094,
+      "logps/generated": -907.7996826171875,
+      "logps/real": -124.29368591308594,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.24595069885254,
+      "rewards/margins": 20.557300567626953,
+      "rewards/real": 0.3113483488559723,
+      "step": 810
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.642247510668563e-07,
+      "logits/generated": -2.7591538429260254,
+      "logits/real": -2.488976001739502,
+      "logps/generated": -878.0640869140625,
+      "logps/real": -129.9168701171875,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.200183868408203,
+      "rewards/margins": 20.459766387939453,
+      "rewards/real": 0.25958216190338135,
+      "step": 820
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6066856330014225e-07,
+      "logits/generated": -2.717893600463867,
+      "logits/real": -2.475963830947876,
+      "logps/generated": -886.1345825195312,
+      "logps/real": -136.38031005859375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -21.064132690429688,
+      "rewards/margins": 21.22158432006836,
+      "rewards/real": 0.1574556827545166,
+      "step": 830
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5711237553342815e-07,
+      "logits/generated": -2.7576115131378174,
+      "logits/real": -2.520620107650757,
+      "logps/generated": -919.7344970703125,
+      "logps/real": -146.73341369628906,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -21.55472755432129,
+      "rewards/margins": 21.70217514038086,
+      "rewards/real": 0.14745107293128967,
+      "step": 840
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5355618776671404e-07,
+      "logits/generated": -2.758734941482544,
+      "logits/real": -2.4575321674346924,
+      "logps/generated": -924.5079956054688,
+      "logps/real": -137.92759704589844,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.882726669311523,
+      "rewards/margins": 21.117658615112305,
+      "rewards/real": 0.23493008315563202,
+      "step": 850
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5e-07,
+      "logits/generated": -2.76533579826355,
+      "logits/real": -2.473336696624756,
+      "logps/generated": -872.0582885742188,
+      "logps/real": -128.65528869628906,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.465499877929688,
+      "rewards/margins": 20.760725021362305,
+      "rewards/real": 0.29522615671157837,
+      "step": 860
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4644381223328594e-07,
+      "logits/generated": -2.8043251037597656,
+      "logits/real": -2.4596962928771973,
+      "logps/generated": -831.9053955078125,
+      "logps/real": -136.89483642578125,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.220478057861328,
+      "rewards/margins": 20.537092208862305,
+      "rewards/real": 0.31661272048950195,
+      "step": 870
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4288762446657183e-07,
+      "logits/generated": -2.7661118507385254,
+      "logits/real": -2.463319778442383,
+      "logps/generated": -896.66796875,
+      "logps/real": -140.58807373046875,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -21.672061920166016,
+      "rewards/margins": 21.854246139526367,
+      "rewards/real": 0.18218322098255157,
+      "step": 880
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.393314366998578e-07,
+      "logits/generated": -2.8111281394958496,
+      "logits/real": -2.45881986618042,
+      "logps/generated": -883.6256713867188,
+      "logps/real": -133.40512084960938,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -21.710323333740234,
+      "rewards/margins": 22.020999908447266,
+      "rewards/real": 0.3106769621372223,
+      "step": 890
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.3577524893314365e-07,
+      "logits/generated": -2.7077133655548096,
+      "logits/real": -2.4282584190368652,
+      "logps/generated": -929.71923828125,
+      "logps/real": -125.48017883300781,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -21.90422821044922,
+      "rewards/margins": 22.404098510742188,
+      "rewards/real": 0.49986690282821655,
+      "step": 900
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.322190611664296e-07,
+      "logits/generated": -2.8797926902770996,
+      "logits/real": -2.449512004852295,
+      "logps/generated": -898.6605224609375,
+      "logps/real": -145.88131713867188,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.765823364257812,
+      "rewards/margins": 21.118236541748047,
+      "rewards/real": 0.35241395235061646,
+      "step": 910
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.2866287339971549e-07,
+      "logits/generated": -2.7840793132781982,
+      "logits/real": -2.467308282852173,
+      "logps/generated": -892.7574462890625,
+      "logps/real": -133.6017303466797,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -21.309818267822266,
+      "rewards/margins": 21.834148406982422,
+      "rewards/real": 0.524328351020813,
+      "step": 920
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.251066856330014e-07,
+      "logits/generated": -2.7835748195648193,
+      "logits/real": -2.430983781814575,
+      "logps/generated": -926.97900390625,
+      "logps/real": -121.66536712646484,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -22.335927963256836,
+      "rewards/margins": 22.66562271118164,
+      "rewards/real": 0.32969528436660767,
+      "step": 930
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.2155049786628733e-07,
+      "logits/generated": -2.7988877296447754,
+      "logits/real": -2.470797061920166,
+      "logps/generated": -906.01904296875,
+      "logps/real": -139.35302734375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -21.41358757019043,
+      "rewards/margins": 21.88725471496582,
+      "rewards/real": 0.4736654758453369,
+      "step": 940
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.1799431009957325e-07,
+      "logits/generated": -2.7693393230438232,
+      "logits/real": -2.449216842651367,
+      "logps/generated": -971.3763427734375,
+      "logps/real": -120.10380554199219,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -22.712810516357422,
+      "rewards/margins": 23.097557067871094,
+      "rewards/real": 0.38474756479263306,
+      "step": 950
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.1443812233285914e-07,
+      "logits/generated": -2.8496899604797363,
+      "logits/real": -2.6208953857421875,
+      "logps/generated": -853.5813598632812,
+      "logps/real": -140.28988647460938,
+      "loss": 0.0002,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.387523651123047,
+      "rewards/margins": 18.704341888427734,
+      "rewards/real": 0.31681886315345764,
+      "step": 960
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.108819345661451e-07,
+      "logits/generated": -2.81449556350708,
+      "logits/real": -2.7593271732330322,
+      "logps/generated": -824.1788330078125,
+      "logps/real": -129.34524536132812,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -15.003524780273438,
+      "rewards/margins": 15.52760124206543,
+      "rewards/real": 0.5240752100944519,
+      "step": 970
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 2.0732574679943098e-07,
+      "logits/generated": -2.8338940143585205,
+      "logits/real": -2.6659107208251953,
+      "logps/generated": -828.3829956054688,
+      "logps/real": -113.12556457519531,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -15.830032348632812,
+      "rewards/margins": 16.479970932006836,
+      "rewards/real": 0.6499394178390503,
+      "step": 980
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 2.0376955903271693e-07,
+      "logits/generated": -2.8800129890441895,
+      "logits/real": -2.7851107120513916,
+      "logps/generated": -825.91015625,
+      "logps/real": -145.60104370117188,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -15.22101879119873,
+      "rewards/margins": 15.671483993530273,
+      "rewards/real": 0.45046553015708923,
+      "step": 990
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 2.0021337126600283e-07,
+      "logits/generated": -2.8783576488494873,
+      "logits/real": -2.719095468521118,
+      "logps/generated": -829.0347900390625,
+      "logps/real": -117.2816162109375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -16.659259796142578,
+      "rewards/margins": 17.303306579589844,
+      "rewards/real": 0.6440474390983582,
+      "step": 1000
+    },
+    {
+      "epoch": 0.64,
+      "eval_logits/generated": -2.8324971199035645,
+      "eval_logits/real": -2.7463560104370117,
+      "eval_logps/generated": -843.258544921875,
+      "eval_logps/real": -129.80291748046875,
+      "eval_loss": 2.3505108401877806e-05,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/generated": -16.52660369873047,
+      "eval_rewards/margins": 16.946895599365234,
+      "eval_rewards/real": 0.42029163241386414,
+      "eval_runtime": 65.6308,
+      "eval_samples_per_second": 7.618,
+      "eval_steps_per_second": 0.244,
+      "step": 1000
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.9665718349928875e-07,
+      "logits/generated": -2.7597239017486572,
+      "logits/real": -2.70881986618042,
+      "logps/generated": -878.791015625,
+      "logps/real": -121.81756591796875,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -17.580312728881836,
+      "rewards/margins": 18.086267471313477,
+      "rewards/real": 0.5059542655944824,
+      "step": 1010
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.931009957325747e-07,
+      "logits/generated": -2.8248562812805176,
+      "logits/real": -2.6810784339904785,
+      "logps/generated": -852.2976684570312,
+      "logps/real": -110.82057189941406,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -17.118757247924805,
+      "rewards/margins": 17.73995590209961,
+      "rewards/real": 0.6211975812911987,
+      "step": 1020
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.895448079658606e-07,
+      "logits/generated": -2.8053200244903564,
+      "logits/real": -2.70365309715271,
+      "logps/generated": -900.5974731445312,
+      "logps/real": -135.4402313232422,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -17.847537994384766,
+      "rewards/margins": 18.353679656982422,
+      "rewards/real": 0.5061434507369995,
+      "step": 1030
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.859886201991465e-07,
+      "logits/generated": -2.898444414138794,
+      "logits/real": -2.7453322410583496,
+      "logps/generated": -820.4622802734375,
+      "logps/real": -140.00946044921875,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -17.400270462036133,
+      "rewards/margins": 17.893169403076172,
+      "rewards/real": 0.4929002821445465,
+      "step": 1040
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.8243243243243243e-07,
+      "logits/generated": -2.839688777923584,
+      "logits/real": -2.6923739910125732,
+      "logps/generated": -838.73291015625,
+      "logps/real": -132.70223999023438,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.139698028564453,
+      "rewards/margins": 18.552087783813477,
+      "rewards/real": 0.41239088773727417,
+      "step": 1050
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.7887624466571835e-07,
+      "logits/generated": -2.833216667175293,
+      "logits/real": -2.7041759490966797,
+      "logps/generated": -861.8936767578125,
+      "logps/real": -126.6530990600586,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.731300354003906,
+      "rewards/margins": 19.197338104248047,
+      "rewards/real": 0.4660395085811615,
+      "step": 1060
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.7532005689900424e-07,
+      "logits/generated": -2.8046717643737793,
+      "logits/real": -2.69667387008667,
+      "logps/generated": -875.9267578125,
+      "logps/real": -128.2639617919922,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.01252555847168,
+      "rewards/margins": 18.45262908935547,
+      "rewards/real": 0.44010037183761597,
+      "step": 1070
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.717638691322902e-07,
+      "logits/generated": -2.83022141456604,
+      "logits/real": -2.692930221557617,
+      "logps/generated": -854.4952392578125,
+      "logps/real": -121.74955749511719,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -17.961904525756836,
+      "rewards/margins": 18.5115909576416,
+      "rewards/real": 0.5496853590011597,
+      "step": 1080
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.6820768136557609e-07,
+      "logits/generated": -2.8350539207458496,
+      "logits/real": -2.6970601081848145,
+      "logps/generated": -863.2819213867188,
+      "logps/real": -123.15059661865234,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.864200592041016,
+      "rewards/margins": 19.36715316772461,
+      "rewards/real": 0.5029550194740295,
+      "step": 1090
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.64651493598862e-07,
+      "logits/generated": -2.8632559776306152,
+      "logits/real": -2.677931308746338,
+      "logps/generated": -879.8753662109375,
+      "logps/real": -141.77952575683594,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.380382537841797,
+      "rewards/margins": 18.56157684326172,
+      "rewards/real": 0.18119129538536072,
+      "step": 1100
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.6109530583214793e-07,
+      "logits/generated": -2.803745746612549,
+      "logits/real": -2.6646134853363037,
+      "logps/generated": -901.3361206054688,
+      "logps/real": -126.1727523803711,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.827922821044922,
+      "rewards/margins": 20.339210510253906,
+      "rewards/real": 0.5112860202789307,
+      "step": 1110
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.5753911806543385e-07,
+      "logits/generated": -2.8503870964050293,
+      "logits/real": -2.6645989418029785,
+      "logps/generated": -888.7047119140625,
+      "logps/real": -129.1335906982422,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.367877960205078,
+      "rewards/margins": 19.824716567993164,
+      "rewards/real": 0.4568362236022949,
+      "step": 1120
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.5398293029871974e-07,
+      "logits/generated": -2.7480721473693848,
+      "logits/real": -2.6507885456085205,
+      "logps/generated": -934.6629028320312,
+      "logps/real": -135.6553497314453,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.353116989135742,
+      "rewards/margins": 20.787641525268555,
+      "rewards/real": 0.4345230162143707,
+      "step": 1130
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.504267425320057e-07,
+      "logits/generated": -2.797375202178955,
+      "logits/real": -2.6887059211730957,
+      "logps/generated": -881.1536254882812,
+      "logps/real": -125.61927795410156,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.214933395385742,
+      "rewards/margins": 19.779462814331055,
+      "rewards/real": 0.5645291209220886,
+      "step": 1140
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 1.4687055476529158e-07,
+      "logits/generated": -2.870006799697876,
+      "logits/real": -2.654900550842285,
+      "logps/generated": -834.4885864257812,
+      "logps/real": -136.41250610351562,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -18.738508224487305,
+      "rewards/margins": 19.11133575439453,
+      "rewards/real": 0.37282687425613403,
+      "step": 1150
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 1.4331436699857753e-07,
+      "logits/generated": -2.8572959899902344,
+      "logits/real": -2.7063486576080322,
+      "logps/generated": -870.64404296875,
+      "logps/real": -134.53292846679688,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.98287582397461,
+      "rewards/margins": 20.453279495239258,
+      "rewards/real": 0.4704047739505768,
+      "step": 1160
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.3975817923186345e-07,
+      "logits/generated": -2.871398448944092,
+      "logits/real": -2.707024335861206,
+      "logps/generated": -850.9390869140625,
+      "logps/real": -148.42337036132812,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.16036605834961,
+      "rewards/margins": 20.592147827148438,
+      "rewards/real": 0.43178051710128784,
+      "step": 1170
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.3620199146514935e-07,
+      "logits/generated": -2.807039976119995,
+      "logits/real": -2.723776340484619,
+      "logps/generated": -929.8095703125,
+      "logps/real": -115.76216125488281,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.871318817138672,
+      "rewards/margins": 21.316526412963867,
+      "rewards/real": 0.44520822167396545,
+      "step": 1180
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 1.326458036984353e-07,
+      "logits/generated": -2.8286869525909424,
+      "logits/real": -2.6350340843200684,
+      "logps/generated": -862.5518798828125,
+      "logps/real": -126.08731842041016,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.82559585571289,
+      "rewards/margins": 20.303485870361328,
+      "rewards/real": 0.4778921604156494,
+      "step": 1190
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.290896159317212e-07,
+      "logits/generated": -2.8613972663879395,
+      "logits/real": -2.638763904571533,
+      "logps/generated": -921.1492919921875,
+      "logps/real": -123.17964172363281,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.54897689819336,
+      "rewards/margins": 20.937849044799805,
+      "rewards/real": 0.3888731598854065,
+      "step": 1200
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.255334281650071e-07,
+      "logits/generated": -2.876451253890991,
+      "logits/real": -2.6815617084503174,
+      "logps/generated": -831.7360229492188,
+      "logps/real": -139.11557006835938,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -19.613943099975586,
+      "rewards/margins": 19.776029586791992,
+      "rewards/real": 0.16208769381046295,
+      "step": 1210
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 1.2197724039829303e-07,
+      "logits/generated": -2.8330130577087402,
+      "logits/real": -2.566429615020752,
+      "logps/generated": -911.4781494140625,
+      "logps/real": -122.40309143066406,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -22.14625358581543,
+      "rewards/margins": 22.701326370239258,
+      "rewards/real": 0.5550734400749207,
+      "step": 1220
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 1.1842105263157894e-07,
+      "logits/generated": -2.808868885040283,
+      "logits/real": -2.6208791732788086,
+      "logps/generated": -873.4892578125,
+      "logps/real": -114.96858215332031,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.708131790161133,
+      "rewards/margins": 21.098825454711914,
+      "rewards/real": 0.3906935155391693,
+      "step": 1230
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 1.1486486486486487e-07,
+      "logits/generated": -2.8322298526763916,
+      "logits/real": -2.6367688179016113,
+      "logps/generated": -940.2215576171875,
+      "logps/real": -132.8704071044922,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -22.91935920715332,
+      "rewards/margins": 23.39228630065918,
+      "rewards/real": 0.4729260802268982,
+      "step": 1240
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.1130867709815078e-07,
+      "logits/generated": -2.8651223182678223,
+      "logits/real": -2.6476199626922607,
+      "logps/generated": -869.115234375,
+      "logps/real": -129.05712890625,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.786222457885742,
+      "rewards/margins": 21.287538528442383,
+      "rewards/real": 0.5013141632080078,
+      "step": 1250
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.077524893314367e-07,
+      "logits/generated": -2.7978148460388184,
+      "logits/real": -2.5825142860412598,
+      "logps/generated": -897.38330078125,
+      "logps/real": -118.57264709472656,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -21.1629638671875,
+      "rewards/margins": 21.7834415435791,
+      "rewards/real": 0.6204766631126404,
+      "step": 1260
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.0419630156472262e-07,
+      "logits/generated": -2.866764783859253,
+      "logits/real": -2.685533046722412,
+      "logps/generated": -919.8984375,
+      "logps/real": -146.31027221679688,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -21.15250587463379,
+      "rewards/margins": 21.48689079284668,
+      "rewards/real": 0.3343891501426697,
+      "step": 1270
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.0064011379800854e-07,
+      "logits/generated": -2.8685061931610107,
+      "logits/real": -2.649932384490967,
+      "logps/generated": -902.7081909179688,
+      "logps/real": -141.6209716796875,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -22.167659759521484,
+      "rewards/margins": 22.551496505737305,
+      "rewards/real": 0.38383588194847107,
+      "step": 1280
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.708392603129445e-08,
+      "logits/generated": -2.8944077491760254,
+      "logits/real": -2.6382641792297363,
+      "logps/generated": -916.2838745117188,
+      "logps/real": -130.5482177734375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -21.300434112548828,
+      "rewards/margins": 21.502178192138672,
+      "rewards/real": 0.20173999667167664,
+      "step": 1290
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.352773826458037e-08,
+      "logits/generated": -2.8912646770477295,
+      "logits/real": -2.6097311973571777,
+      "logps/generated": -894.2330322265625,
+      "logps/real": -132.25819396972656,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -21.95718765258789,
+      "rewards/margins": 22.406177520751953,
+      "rewards/real": 0.44899100065231323,
+      "step": 1300
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.997155049786629e-08,
+      "logits/generated": -2.8901562690734863,
+      "logits/real": -2.551877021789551,
+      "logps/generated": -932.6094970703125,
+      "logps/real": -119.0418930053711,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -22.05977439880371,
+      "rewards/margins": 22.581966400146484,
+      "rewards/real": 0.5221914052963257,
+      "step": 1310
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.64153627311522e-08,
+      "logits/generated": -2.8296782970428467,
+      "logits/real": -2.5993740558624268,
+      "logps/generated": -896.4122314453125,
+      "logps/real": -128.8374481201172,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -21.528562545776367,
+      "rewards/margins": 22.051563262939453,
+      "rewards/real": 0.5230005979537964,
+      "step": 1320
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 8.285917496443812e-08,
+      "logits/generated": -2.860327959060669,
+      "logits/real": -2.5901761054992676,
+      "logps/generated": -898.0784912109375,
+      "logps/real": -132.93478393554688,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -22.043405532836914,
+      "rewards/margins": 22.58662223815918,
+      "rewards/real": 0.543217658996582,
+      "step": 1330
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.930298719772404e-08,
+      "logits/generated": -2.8273463249206543,
+      "logits/real": -2.5543017387390137,
+      "logps/generated": -961.3826904296875,
+      "logps/real": -140.56031799316406,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -23.900615692138672,
+      "rewards/margins": 24.335386276245117,
+      "rewards/real": 0.43477168679237366,
+      "step": 1340
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.574679943100994e-08,
+      "logits/generated": -2.8639349937438965,
+      "logits/real": -2.6260411739349365,
+      "logps/generated": -885.5602416992188,
+      "logps/real": -112.6806869506836,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -20.910247802734375,
+      "rewards/margins": 21.491928100585938,
+      "rewards/real": 0.5816811919212341,
+      "step": 1350
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 7.219061166429587e-08,
+      "logits/generated": -2.8940956592559814,
+      "logits/real": -2.5862889289855957,
+      "logps/generated": -917.0275268554688,
+      "logps/real": -125.59222412109375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -23.038272857666016,
+      "rewards/margins": 23.486886978149414,
+      "rewards/real": 0.44861316680908203,
+      "step": 1360
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.863442389758179e-08,
+      "logits/generated": -2.8932290077209473,
+      "logits/real": -2.5974230766296387,
+      "logps/generated": -886.7428588867188,
+      "logps/real": -143.0746307373047,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -22.268463134765625,
+      "rewards/margins": 22.662071228027344,
+      "rewards/real": 0.3936085104942322,
+      "step": 1370
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.507823613086771e-08,
+      "logits/generated": -2.922111749649048,
+      "logits/real": -2.594691276550293,
+      "logps/generated": -906.36767578125,
+      "logps/real": -140.33889770507812,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -22.82131004333496,
+      "rewards/margins": 23.025859832763672,
+      "rewards/real": 0.20455090701580048,
+      "step": 1380
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.152204836415363e-08,
+      "logits/generated": -2.914600133895874,
+      "logits/real": -2.6084065437316895,
+      "logps/generated": -941.8387451171875,
+      "logps/real": -133.2527313232422,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -23.14767837524414,
+      "rewards/margins": 23.540363311767578,
+      "rewards/real": 0.39268168807029724,
+      "step": 1390
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.796586059743954e-08,
+      "logits/generated": -2.8385584354400635,
+      "logits/real": -2.6445257663726807,
+      "logps/generated": -907.6339721679688,
+      "logps/real": -132.3414764404297,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -24.239797592163086,
+      "rewards/margins": 24.71152114868164,
+      "rewards/real": 0.47172126173973083,
+      "step": 1400
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.4409672830725456e-08,
+      "logits/generated": -2.8101682662963867,
+      "logits/real": -2.6199960708618164,
+      "logps/generated": -945.5979614257812,
+      "logps/real": -132.10206604003906,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -24.88088607788086,
+      "rewards/margins": 25.367450714111328,
+      "rewards/real": 0.4865630567073822,
+      "step": 1410
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 5.0853485064011376e-08,
+      "logits/generated": -2.8484458923339844,
+      "logits/real": -2.563117504119873,
+      "logps/generated": -928.4225463867188,
+      "logps/real": -119.01268005371094,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -23.193946838378906,
+      "rewards/margins": 23.61981201171875,
+      "rewards/real": 0.4258663058280945,
+      "step": 1420
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.72972972972973e-08,
+      "logits/generated": -2.8349921703338623,
+      "logits/real": -2.5548834800720215,
+      "logps/generated": -979.7244873046875,
+      "logps/real": -130.8174285888672,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -25.183971405029297,
+      "rewards/margins": 25.632709503173828,
+      "rewards/real": 0.44873887300491333,
+      "step": 1430
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.374110953058322e-08,
+      "logits/generated": -2.837606906890869,
+      "logits/real": -2.537325620651245,
+      "logps/generated": -992.2394409179688,
+      "logps/real": -128.49644470214844,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -25.086801528930664,
+      "rewards/margins": 25.410724639892578,
+      "rewards/real": 0.32392334938049316,
+      "step": 1440
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 4.018492176386913e-08,
+      "logits/generated": -2.91603422164917,
+      "logits/real": -2.5341243743896484,
+      "logps/generated": -898.00146484375,
+      "logps/real": -129.14276123046875,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -22.930097579956055,
+      "rewards/margins": 23.440217971801758,
+      "rewards/real": 0.5101193189620972,
+      "step": 1450
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6628733997155046e-08,
+      "logits/generated": -2.8616137504577637,
+      "logits/real": -2.5504488945007324,
+      "logps/generated": -912.74560546875,
+      "logps/real": -108.04595947265625,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -23.54279899597168,
+      "rewards/margins": 23.969791412353516,
+      "rewards/real": 0.4269927442073822,
+      "step": 1460
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.3072546230440967e-08,
+      "logits/generated": -2.85672926902771,
+      "logits/real": -2.5588877201080322,
+      "logps/generated": -940.3358154296875,
+      "logps/real": -125.9631118774414,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -24.594558715820312,
+      "rewards/margins": 24.99027442932129,
+      "rewards/real": 0.39571598172187805,
+      "step": 1470
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.9516358463726884e-08,
+      "logits/generated": -2.89031720161438,
+      "logits/real": -2.5603203773498535,
+      "logps/generated": -972.2039184570312,
+      "logps/real": -137.25588989257812,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -24.38454818725586,
+      "rewards/margins": 24.839740753173828,
+      "rewards/real": 0.45519551634788513,
+      "step": 1480
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.59601706970128e-08,
+      "logits/generated": -2.890516996383667,
+      "logits/real": -2.5641016960144043,
+      "logps/generated": -936.7041015625,
+      "logps/real": -140.62559509277344,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -25.00693702697754,
+      "rewards/margins": 25.536272048950195,
+      "rewards/real": 0.5293352603912354,
+      "step": 1490
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.240398293029872e-08,
+      "logits/generated": -2.8512871265411377,
+      "logits/real": -2.5838348865509033,
+      "logps/generated": -910.3527221679688,
+      "logps/real": -126.60355377197266,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -23.73545265197754,
+      "rewards/margins": 24.106916427612305,
+      "rewards/real": 0.3714631199836731,
+      "step": 1500
+    },
+    {
+      "epoch": 0.96,
+      "eval_logits/generated": -2.8901188373565674,
+      "eval_logits/real": -2.609180450439453,
+      "eval_logps/generated": -916.0912475585938,
+      "eval_logps/real": -130.5047149658203,
+      "eval_loss": 2.595016326267796e-07,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/generated": -23.809871673583984,
+      "eval_rewards/margins": 24.159982681274414,
+      "eval_rewards/real": 0.35011160373687744,
+      "eval_runtime": 65.5536,
+      "eval_samples_per_second": 7.627,
+      "eval_steps_per_second": 0.244,
+      "step": 1500
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.8847795163584636e-08,
+      "logits/generated": -2.900836229324341,
+      "logits/real": -2.5513949394226074,
+      "logps/generated": -931.4461059570312,
+      "logps/real": -129.80133056640625,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -24.57596206665039,
+      "rewards/margins": 24.972017288208008,
+      "rewards/real": 0.3960537910461426,
+      "step": 1510
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5291607396870554e-08,
+      "logits/generated": -2.862175941467285,
+      "logits/real": -2.5267205238342285,
+      "logps/generated": -934.8350830078125,
+      "logps/real": -126.36529541015625,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -24.785350799560547,
+      "rewards/margins": 25.167552947998047,
+      "rewards/real": 0.38220247626304626,
+      "step": 1520
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1735419630156473e-08,
+      "logits/generated": -2.877037525177002,
+      "logits/real": -2.5817883014678955,
+      "logps/generated": -916.3255004882812,
+      "logps/real": -128.97787475585938,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -24.590730667114258,
+      "rewards/margins": 24.93622589111328,
+      "rewards/real": 0.3454935848712921,
+      "step": 1530
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.179231863442388e-09,
+      "logits/generated": -2.9152793884277344,
+      "logits/real": -2.5522732734680176,
+      "logps/generated": -968.6594848632812,
+      "logps/real": -132.7301025390625,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -24.529855728149414,
+      "rewards/margins": 25.114501953125,
+      "rewards/real": 0.5846462249755859,
+      "step": 1540
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.623044096728307e-09,
+      "logits/generated": -2.8607215881347656,
+      "logits/real": -2.5737948417663574,
+      "logps/generated": -897.6732177734375,
+      "logps/real": -137.73817443847656,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -23.409955978393555,
+      "rewards/margins": 23.72184181213379,
+      "rewards/real": 0.31188473105430603,
+      "step": 1550
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0668563300142248e-09,
+      "logits/generated": -2.8556289672851562,
+      "logits/real": -2.54154896736145,
+      "logps/generated": -959.1483154296875,
+      "logps/real": -127.1308822631836,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -25.255569458007812,
+      "rewards/margins": 25.72182273864746,
+      "rewards/real": 0.46625250577926636,
+      "step": 1560
+    },
+    {
+      "epoch": 1.0,
+      "step": 1563,
+      "total_flos": 0.0,
+      "train_loss": 0.009621814649877188,
+      "train_runtime": 14787.3165,
+      "train_samples_per_second": 3.381,
+      "train_steps_per_second": 0.106
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1563,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}