c-alfano commited on
Commit
4a5e5f6
·
verified ·
1 Parent(s): 1accc35

Model save

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ license: gemma
4
+ base_model: google/gemma-7b
5
+ tags:
6
+ - trl
7
+ - orpo
8
+ - generated_from_trainer
9
+ model-index:
10
+ - name: gemma-7b-borpo-low-quality-v3
11
+ results: []
12
+ ---
13
+
14
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
15
+ should probably proofread and complete it, then remove this comment. -->
16
+
17
+ # gemma-7b-borpo-low-quality-v3
18
+
19
+ This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on an unknown dataset.
20
+ It achieves the following results on the evaluation set:
21
+ - Loss: 2.1095
22
+ - Rewards/chosen: -0.6954
23
+ - Rewards/rejected: -0.8346
24
+ - Rewards/accuracies: 0.5571
25
+ - Rewards/margins: 0.1392
26
+ - Logps/rejected: -1.6692
27
+ - Logps/chosen: -1.3909
28
+ - Logits/rejected: 262.5518
29
+ - Logits/chosen: 319.3429
30
+ - Nll Loss: 1.7836
31
+ - Log Odds Ratio: -0.6395
32
+ - Log Odds Chosen: 0.4455
33
+
34
+ ## Model description
35
+
36
+ More information needed
37
+
38
+ ## Intended uses & limitations
39
+
40
+ More information needed
41
+
42
+ ## Training and evaluation data
43
+
44
+ More information needed
45
+
46
+ ## Training procedure
47
+
48
+ ### Training hyperparameters
49
+
50
+ The following hyperparameters were used during training:
51
+ - learning_rate: 1e-05
52
+ - train_batch_size: 2
53
+ - eval_batch_size: 1
54
+ - seed: 42
55
+ - distributed_type: multi-GPU
56
+ - num_devices: 8
57
+ - gradient_accumulation_steps: 2
58
+ - total_train_batch_size: 32
59
+ - total_eval_batch_size: 8
60
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
61
+ - lr_scheduler_type: inverse_sqrt
62
+ - lr_scheduler_warmup_steps: 100
63
+ - num_epochs: 3
64
+
65
+ ### Training results
66
+
67
+ | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Nll Loss | Log Odds Ratio | Log Odds Chosen |
68
+ |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------:|:--------------:|:---------------:|
69
+ | 1.9721 | 1.0 | 168 | 1.9526 | -0.6072 | -0.7027 | 0.5571 | 0.0955 | -1.4054 | -1.2144 | 282.1215 | 336.2867 | 1.6515 | -0.6573 | 0.2649 |
70
+ | 1.3299 | 2.0 | 336 | 1.9015 | -0.5986 | -0.6805 | 0.5 | 0.0820 | -1.3611 | -1.1972 | 293.2820 | 345.2333 | 1.5933 | -0.6792 | 0.2173 |
71
+ | 0.6266 | 3.0 | 504 | 2.1095 | -0.6954 | -0.8346 | 0.5571 | 0.1392 | -1.6692 | -1.3909 | 262.5518 | 319.3429 | 1.7836 | -0.6395 | 0.4455 |
72
+
73
+
74
+ ### Framework versions
75
+
76
+ - Transformers 4.44.2
77
+ - Pytorch 2.4.1+cu121
78
+ - Datasets 3.0.0
79
+ - Tokenizers 0.19.1
all_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 3.0,
3
+ "total_flos": 0.0,
4
+ "train_loss": 1.593297282854716,
5
+ "train_runtime": 15168.4679,
6
+ "train_samples": 5364,
7
+ "train_samples_per_second": 1.061,
8
+ "train_steps_per_second": 0.033
9
+ }
config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "google/gemma-7b",
3
+ "architectures": [
4
+ "GemmaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 106,
9
+ "eos_token_id": 107,
10
+ "head_dim": 256,
11
+ "hidden_act": "gelu",
12
+ "hidden_activation": "gelu_pytorch_tanh",
13
+ "hidden_size": 3072,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 24576,
16
+ "max_position_embeddings": 8192,
17
+ "model_type": "gemma",
18
+ "num_attention_heads": 16,
19
+ "num_hidden_layers": 28,
20
+ "num_key_value_heads": 16,
21
+ "pad_token_id": 107,
22
+ "rms_norm_eps": 1e-06,
23
+ "rope_scaling": null,
24
+ "rope_theta": 10000.0,
25
+ "torch_dtype": "bfloat16",
26
+ "transformers_version": "4.44.2",
27
+ "use_cache": false,
28
+ "vocab_size": 256000
29
+ }
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 106,
4
+ "eos_token_id": 107,
5
+ "pad_token_id": 107,
6
+ "transformers_version": "4.44.2"
7
+ }
model-00001-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:22a60be7bed9aac03b59f76ab9bb81d3c6f9ce4e49bc291ca973795722bcd739
3
+ size 4995496656
model-00002-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:227e60bee0a89f31ba221fdaf134cba8dddbfa784d0e5c3968dbd005a7c20942
3
+ size 4982953168
model-00003-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:096927ddfe54789478d6a01c24372b8f03913d5740dbfc1a3c6c882a716f6eb6
3
+ size 4982953200
model-00004-of-00004.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fe469255a13131924a87430d208e6cc491f54302952a48bf17c8c399c589716b
3
+ size 2113988336
model.safetensors.index.json ADDED
@@ -0,0 +1,261 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 17075361792
4
+ },
5
+ "weight_map": {
6
+ "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
7
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
8
+ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
9
+ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
10
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
11
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
12
+ "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
13
+ "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
14
+ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
15
+ "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
16
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
17
+ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
18
+ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
19
+ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
20
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
21
+ "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
22
+ "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
23
+ "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
24
+ "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
25
+ "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
26
+ "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
27
+ "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
28
+ "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
29
+ "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
30
+ "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
31
+ "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
32
+ "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
33
+ "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
34
+ "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
35
+ "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
36
+ "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
37
+ "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
38
+ "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
39
+ "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
40
+ "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
41
+ "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
42
+ "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
43
+ "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
44
+ "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
45
+ "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
46
+ "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
47
+ "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
48
+ "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
49
+ "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
50
+ "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
51
+ "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
52
+ "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
53
+ "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
54
+ "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
55
+ "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
56
+ "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
57
+ "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
58
+ "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
59
+ "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
60
+ "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
61
+ "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
62
+ "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
63
+ "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
64
+ "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
65
+ "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
66
+ "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
67
+ "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
68
+ "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
69
+ "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
70
+ "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors",
71
+ "model.layers.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
72
+ "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
73
+ "model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
74
+ "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
75
+ "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
76
+ "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
77
+ "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
78
+ "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
79
+ "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
80
+ "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
81
+ "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
82
+ "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
83
+ "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
84
+ "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
85
+ "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
86
+ "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
87
+ "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
88
+ "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
89
+ "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
90
+ "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
91
+ "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
92
+ "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
93
+ "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
94
+ "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
95
+ "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
96
+ "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
97
+ "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
98
+ "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
99
+ "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
100
+ "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
101
+ "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
102
+ "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
103
+ "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
104
+ "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
105
+ "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
106
+ "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
107
+ "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
108
+ "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
109
+ "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
110
+ "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
111
+ "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
112
+ "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
113
+ "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
114
+ "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
115
+ "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
116
+ "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
117
+ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
118
+ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
119
+ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
120
+ "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
121
+ "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
122
+ "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
123
+ "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
124
+ "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
125
+ "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
126
+ "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
127
+ "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
128
+ "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
129
+ "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
130
+ "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
131
+ "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
132
+ "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
133
+ "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
134
+ "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
135
+ "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
136
+ "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
137
+ "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
138
+ "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
139
+ "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
140
+ "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
141
+ "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
142
+ "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
143
+ "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
144
+ "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
145
+ "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
146
+ "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
147
+ "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
148
+ "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
149
+ "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
150
+ "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
151
+ "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
152
+ "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
153
+ "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
154
+ "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
155
+ "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
156
+ "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
157
+ "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
158
+ "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
159
+ "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
160
+ "model.layers.24.input_layernorm.weight": "model-00004-of-00004.safetensors",
161
+ "model.layers.24.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
162
+ "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
163
+ "model.layers.24.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
164
+ "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
165
+ "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
166
+ "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
167
+ "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
168
+ "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
169
+ "model.layers.25.input_layernorm.weight": "model-00004-of-00004.safetensors",
170
+ "model.layers.25.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
171
+ "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
172
+ "model.layers.25.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
173
+ "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
174
+ "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
175
+ "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
176
+ "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
177
+ "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
178
+ "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
179
+ "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
180
+ "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
181
+ "model.layers.26.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
182
+ "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
183
+ "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
184
+ "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
185
+ "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
186
+ "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
187
+ "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
188
+ "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
189
+ "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
190
+ "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
191
+ "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
192
+ "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
193
+ "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
194
+ "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
195
+ "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
196
+ "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
197
+ "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
198
+ "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
199
+ "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
200
+ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
201
+ "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
202
+ "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
203
+ "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
204
+ "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
205
+ "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
206
+ "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
207
+ "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
208
+ "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
209
+ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
210
+ "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
211
+ "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
212
+ "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
213
+ "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
214
+ "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
215
+ "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
216
+ "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
217
+ "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
218
+ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
219
+ "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
220
+ "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
221
+ "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
222
+ "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
223
+ "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
224
+ "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
225
+ "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
226
+ "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
227
+ "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
228
+ "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
229
+ "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
230
+ "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
231
+ "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
232
+ "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
233
+ "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
234
+ "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
235
+ "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
236
+ "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
237
+ "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
238
+ "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
239
+ "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
240
+ "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
241
+ "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
242
+ "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
243
+ "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
244
+ "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
245
+ "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
246
+ "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
247
+ "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
248
+ "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
249
+ "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
250
+ "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
251
+ "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
252
+ "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
253
+ "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
254
+ "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
255
+ "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
256
+ "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
257
+ "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
258
+ "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
259
+ "model.norm.weight": "model-00004-of-00004.safetensors"
260
+ }
261
+ }
runs/Sep22_12-11-01_zizgpu06.cpu.stats.ox.ac.uk/events.out.tfevents.1727003560.zizgpu06.cpu.stats.ox.ac.uk.3873994.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dcdd79df0494ff06f048c56a83e32cdabed25ae9013fce84535c6b12ddb12217
3
+ size 94140
special_tokens_map.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ {
4
+ "content": "<|im_start|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false
9
+ },
10
+ {
11
+ "content": "<|im_end|>",
12
+ "lstrip": false,
13
+ "normalized": false,
14
+ "rstrip": false,
15
+ "single_word": false
16
+ }
17
+ ],
18
+ "bos_token": "<|im_start|>",
19
+ "eos_token": "<|im_end|>",
20
+ "pad_token": "<|im_end|>",
21
+ "unk_token": {
22
+ "content": "<unk>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false
27
+ }
28
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:322a5f52ab5cab196761ab397a022d6fa3a2e1418585e532bb6efb2fedd2ae94
3
+ size 17477501
tokenizer_config.json ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<pad>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<eos>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "<bos>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "3": {
30
+ "content": "<unk>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "106": {
38
+ "content": "<|im_start|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "107": {
46
+ "content": "<|im_end|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ }
53
+ },
54
+ "additional_special_tokens": [
55
+ "<|im_start|>",
56
+ "<|im_end|>"
57
+ ],
58
+ "bos_token": "<|im_start|>",
59
+ "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
60
+ "clean_up_tokenization_spaces": false,
61
+ "eos_token": "<|im_end|>",
62
+ "legacy": null,
63
+ "model_max_length": 2048,
64
+ "pad_token": "<|im_end|>",
65
+ "sp_model_kwargs": {},
66
+ "spaces_between_special_tokens": false,
67
+ "tokenizer_class": "GemmaTokenizer",
68
+ "unk_token": "<unk>",
69
+ "use_default_system_prompt": false
70
+ }
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 3.0,
3
+ "total_flos": 0.0,
4
+ "train_loss": 1.593297282854716,
5
+ "train_runtime": 15168.4679,
6
+ "train_samples": 5364,
7
+ "train_samples_per_second": 1.061,
8
+ "train_steps_per_second": 0.033
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,1899 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 3.0,
5
+ "eval_steps": 500,
6
+ "global_step": 504,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.02976190476190476,
13
+ "grad_norm": 1643.52392578125,
14
+ "learning_rate": 5.000000000000001e-07,
15
+ "log_odds_chosen": -0.34639421105384827,
16
+ "log_odds_ratio": -1.0579421520233154,
17
+ "logits/chosen": 125.67509460449219,
18
+ "logits/rejected": 180.79092407226562,
19
+ "logps/chosen": -15.30119514465332,
20
+ "logps/rejected": -14.954809188842773,
21
+ "loss": 15.2275,
22
+ "nll_loss": 14.854708671569824,
23
+ "rewards/accuracies": 0.25,
24
+ "rewards/chosen": -7.65059757232666,
25
+ "rewards/margins": -0.1731930673122406,
26
+ "rewards/rejected": -7.477404594421387,
27
+ "step": 5
28
+ },
29
+ {
30
+ "epoch": 0.05952380952380952,
31
+ "grad_norm": 859.1522827148438,
32
+ "learning_rate": 1.0000000000000002e-06,
33
+ "log_odds_chosen": -0.04612647369503975,
34
+ "log_odds_ratio": -0.8707455396652222,
35
+ "logits/chosen": 226.11355590820312,
36
+ "logits/rejected": 262.4227600097656,
37
+ "logps/chosen": -10.240728378295898,
38
+ "logps/rejected": -10.194613456726074,
39
+ "loss": 10.6297,
40
+ "nll_loss": 10.136636734008789,
41
+ "rewards/accuracies": 0.4000000059604645,
42
+ "rewards/chosen": -5.120364189147949,
43
+ "rewards/margins": -0.023057078942656517,
44
+ "rewards/rejected": -5.097306728363037,
45
+ "step": 10
46
+ },
47
+ {
48
+ "epoch": 0.08928571428571429,
49
+ "grad_norm": 192.85568237304688,
50
+ "learning_rate": 1.5e-06,
51
+ "log_odds_chosen": -0.1222303956747055,
52
+ "log_odds_ratio": -0.8104267120361328,
53
+ "logits/chosen": 293.97882080078125,
54
+ "logits/rejected": 282.1849670410156,
55
+ "logps/chosen": -5.908555507659912,
56
+ "logps/rejected": -5.787174224853516,
57
+ "loss": 6.2671,
58
+ "nll_loss": 5.8525919914245605,
59
+ "rewards/accuracies": 0.4000000059604645,
60
+ "rewards/chosen": -2.954277753829956,
61
+ "rewards/margins": -0.06069115549325943,
62
+ "rewards/rejected": -2.893587112426758,
63
+ "step": 15
64
+ },
65
+ {
66
+ "epoch": 0.11904761904761904,
67
+ "grad_norm": 181.04627990722656,
68
+ "learning_rate": 2.0000000000000003e-06,
69
+ "log_odds_chosen": 0.1651725322008133,
70
+ "log_odds_ratio": -0.8192933797836304,
71
+ "logits/chosen": 280.179931640625,
72
+ "logits/rejected": 263.22296142578125,
73
+ "logps/chosen": -3.191753625869751,
74
+ "logps/rejected": -3.352200984954834,
75
+ "loss": 3.8066,
76
+ "nll_loss": 3.3215103149414062,
77
+ "rewards/accuracies": 0.6000000238418579,
78
+ "rewards/chosen": -1.5958768129348755,
79
+ "rewards/margins": 0.08022388815879822,
80
+ "rewards/rejected": -1.676100492477417,
81
+ "step": 20
82
+ },
83
+ {
84
+ "epoch": 0.1488095238095238,
85
+ "grad_norm": 124.88711547851562,
86
+ "learning_rate": 2.5e-06,
87
+ "log_odds_chosen": 0.21610824763774872,
88
+ "log_odds_ratio": -0.6467095613479614,
89
+ "logits/chosen": 337.0777587890625,
90
+ "logits/rejected": 361.3624267578125,
91
+ "logps/chosen": -2.110491991043091,
92
+ "logps/rejected": -2.2703230381011963,
93
+ "loss": 2.9167,
94
+ "nll_loss": 2.676313638687134,
95
+ "rewards/accuracies": 0.550000011920929,
96
+ "rewards/chosen": -1.0552459955215454,
97
+ "rewards/margins": 0.07991557568311691,
98
+ "rewards/rejected": -1.1351615190505981,
99
+ "step": 25
100
+ },
101
+ {
102
+ "epoch": 0.17857142857142858,
103
+ "grad_norm": 120.593994140625,
104
+ "learning_rate": 3e-06,
105
+ "log_odds_chosen": 0.2517230808734894,
106
+ "log_odds_ratio": -0.639795184135437,
107
+ "logits/chosen": 322.35894775390625,
108
+ "logits/rejected": 409.290771484375,
109
+ "logps/chosen": -2.053133964538574,
110
+ "logps/rejected": -2.262042999267578,
111
+ "loss": 2.4846,
112
+ "nll_loss": 2.1683788299560547,
113
+ "rewards/accuracies": 0.6499999761581421,
114
+ "rewards/chosen": -1.026566982269287,
115
+ "rewards/margins": 0.10445437580347061,
116
+ "rewards/rejected": -1.131021499633789,
117
+ "step": 30
118
+ },
119
+ {
120
+ "epoch": 0.20833333333333334,
121
+ "grad_norm": 45.12126541137695,
122
+ "learning_rate": 3.5e-06,
123
+ "log_odds_chosen": 0.5470780730247498,
124
+ "log_odds_ratio": -0.5346588492393494,
125
+ "logits/chosen": 374.33734130859375,
126
+ "logits/rejected": 398.3028259277344,
127
+ "logps/chosen": -1.4850542545318604,
128
+ "logps/rejected": -1.9500999450683594,
129
+ "loss": 2.3067,
130
+ "nll_loss": 1.7573463916778564,
131
+ "rewards/accuracies": 0.699999988079071,
132
+ "rewards/chosen": -0.7425271272659302,
133
+ "rewards/margins": 0.23252280056476593,
134
+ "rewards/rejected": -0.9750499725341797,
135
+ "step": 35
136
+ },
137
+ {
138
+ "epoch": 0.23809523809523808,
139
+ "grad_norm": 34.55175018310547,
140
+ "learning_rate": 4.000000000000001e-06,
141
+ "log_odds_chosen": 0.48626986145973206,
142
+ "log_odds_ratio": -0.5875475406646729,
143
+ "logits/chosen": 343.8048400878906,
144
+ "logits/rejected": 384.3130187988281,
145
+ "logps/chosen": -1.6461362838745117,
146
+ "logps/rejected": -2.0677056312561035,
147
+ "loss": 2.2439,
148
+ "nll_loss": 1.988368272781372,
149
+ "rewards/accuracies": 0.6000000238418579,
150
+ "rewards/chosen": -0.8230681419372559,
151
+ "rewards/margins": 0.2107846736907959,
152
+ "rewards/rejected": -1.0338528156280518,
153
+ "step": 40
154
+ },
155
+ {
156
+ "epoch": 0.26785714285714285,
157
+ "grad_norm": 45.629512786865234,
158
+ "learning_rate": 4.5e-06,
159
+ "log_odds_chosen": -0.03384453058242798,
160
+ "log_odds_ratio": -0.7663249969482422,
161
+ "logits/chosen": 391.72265625,
162
+ "logits/rejected": 376.98858642578125,
163
+ "logps/chosen": -1.8141120672225952,
164
+ "logps/rejected": -1.77732253074646,
165
+ "loss": 2.1803,
166
+ "nll_loss": 1.894079566001892,
167
+ "rewards/accuracies": 0.44999998807907104,
168
+ "rewards/chosen": -0.9070560336112976,
169
+ "rewards/margins": -0.018394792452454567,
170
+ "rewards/rejected": -0.88866126537323,
171
+ "step": 45
172
+ },
173
+ {
174
+ "epoch": 0.2976190476190476,
175
+ "grad_norm": 47.596744537353516,
176
+ "learning_rate": 5e-06,
177
+ "log_odds_chosen": 0.6274509429931641,
178
+ "log_odds_ratio": -0.5308854579925537,
179
+ "logits/chosen": 436.2086486816406,
180
+ "logits/rejected": 442.98974609375,
181
+ "logps/chosen": -1.5697168111801147,
182
+ "logps/rejected": -2.105163097381592,
183
+ "loss": 2.1885,
184
+ "nll_loss": 2.0384533405303955,
185
+ "rewards/accuracies": 0.800000011920929,
186
+ "rewards/chosen": -0.7848584055900574,
187
+ "rewards/margins": 0.26772308349609375,
188
+ "rewards/rejected": -1.052581548690796,
189
+ "step": 50
190
+ },
191
+ {
192
+ "epoch": 0.3273809523809524,
193
+ "grad_norm": 35.20968246459961,
194
+ "learning_rate": 5.500000000000001e-06,
195
+ "log_odds_chosen": 0.3415950834751129,
196
+ "log_odds_ratio": -0.6474028825759888,
197
+ "logits/chosen": 370.7908020019531,
198
+ "logits/rejected": 369.24395751953125,
199
+ "logps/chosen": -1.539645791053772,
200
+ "logps/rejected": -1.8215446472167969,
201
+ "loss": 2.2186,
202
+ "nll_loss": 1.9263942241668701,
203
+ "rewards/accuracies": 0.6000000238418579,
204
+ "rewards/chosen": -0.769822895526886,
205
+ "rewards/margins": 0.14094945788383484,
206
+ "rewards/rejected": -0.9107723236083984,
207
+ "step": 55
208
+ },
209
+ {
210
+ "epoch": 0.35714285714285715,
211
+ "grad_norm": 56.7140998840332,
212
+ "learning_rate": 6e-06,
213
+ "log_odds_chosen": 0.23950794339179993,
214
+ "log_odds_ratio": -0.634242057800293,
215
+ "logits/chosen": 393.73089599609375,
216
+ "logits/rejected": 364.10833740234375,
217
+ "logps/chosen": -1.4558594226837158,
218
+ "logps/rejected": -1.6270105838775635,
219
+ "loss": 2.0389,
220
+ "nll_loss": 2.0331270694732666,
221
+ "rewards/accuracies": 0.5,
222
+ "rewards/chosen": -0.7279297113418579,
223
+ "rewards/margins": 0.08557556569576263,
224
+ "rewards/rejected": -0.8135052919387817,
225
+ "step": 60
226
+ },
227
+ {
228
+ "epoch": 0.3869047619047619,
229
+ "grad_norm": 35.615726470947266,
230
+ "learning_rate": 6.5000000000000004e-06,
231
+ "log_odds_chosen": 0.3183743357658386,
232
+ "log_odds_ratio": -0.6254990696907043,
233
+ "logits/chosen": 390.70111083984375,
234
+ "logits/rejected": 394.5395812988281,
235
+ "logps/chosen": -1.4384119510650635,
236
+ "logps/rejected": -1.6543052196502686,
237
+ "loss": 1.9284,
238
+ "nll_loss": 1.6583305597305298,
239
+ "rewards/accuracies": 0.699999988079071,
240
+ "rewards/chosen": -0.7192059755325317,
241
+ "rewards/margins": 0.10794667154550552,
242
+ "rewards/rejected": -0.8271526098251343,
243
+ "step": 65
244
+ },
245
+ {
246
+ "epoch": 0.4166666666666667,
247
+ "grad_norm": 39.04438018798828,
248
+ "learning_rate": 7e-06,
249
+ "log_odds_chosen": 0.40005454421043396,
250
+ "log_odds_ratio": -0.5705705881118774,
251
+ "logits/chosen": 397.2193603515625,
252
+ "logits/rejected": 392.36126708984375,
253
+ "logps/chosen": -1.761604905128479,
254
+ "logps/rejected": -2.098273515701294,
255
+ "loss": 2.0169,
256
+ "nll_loss": 1.894805908203125,
257
+ "rewards/accuracies": 0.75,
258
+ "rewards/chosen": -0.8808024525642395,
259
+ "rewards/margins": 0.16833437979221344,
260
+ "rewards/rejected": -1.049136757850647,
261
+ "step": 70
262
+ },
263
+ {
264
+ "epoch": 0.44642857142857145,
265
+ "grad_norm": 81.70245361328125,
266
+ "learning_rate": 7.500000000000001e-06,
267
+ "log_odds_chosen": 0.6616519689559937,
268
+ "log_odds_ratio": -0.5056720972061157,
269
+ "logits/chosen": 399.26202392578125,
270
+ "logits/rejected": 387.3080139160156,
271
+ "logps/chosen": -1.2724144458770752,
272
+ "logps/rejected": -1.7848689556121826,
273
+ "loss": 1.9258,
274
+ "nll_loss": 1.534188985824585,
275
+ "rewards/accuracies": 0.800000011920929,
276
+ "rewards/chosen": -0.6362072229385376,
277
+ "rewards/margins": 0.2562272548675537,
278
+ "rewards/rejected": -0.8924344778060913,
279
+ "step": 75
280
+ },
281
+ {
282
+ "epoch": 0.47619047619047616,
283
+ "grad_norm": 50.36211395263672,
284
+ "learning_rate": 8.000000000000001e-06,
285
+ "log_odds_chosen": 0.6271190643310547,
286
+ "log_odds_ratio": -0.5201026797294617,
287
+ "logits/chosen": 406.4871520996094,
288
+ "logits/rejected": 419.87884521484375,
289
+ "logps/chosen": -1.585078239440918,
290
+ "logps/rejected": -2.038538694381714,
291
+ "loss": 2.0136,
292
+ "nll_loss": 1.7463384866714478,
293
+ "rewards/accuracies": 0.75,
294
+ "rewards/chosen": -0.792539119720459,
295
+ "rewards/margins": 0.22673015296459198,
296
+ "rewards/rejected": -1.019269347190857,
297
+ "step": 80
298
+ },
299
+ {
300
+ "epoch": 0.5059523809523809,
301
+ "grad_norm": 63.27934265136719,
302
+ "learning_rate": 8.5e-06,
303
+ "log_odds_chosen": -0.1369583010673523,
304
+ "log_odds_ratio": -0.9432764053344727,
305
+ "logits/chosen": 416.71893310546875,
306
+ "logits/rejected": 396.90789794921875,
307
+ "logps/chosen": -2.0069127082824707,
308
+ "logps/rejected": -1.7957664728164673,
309
+ "loss": 2.0904,
310
+ "nll_loss": 2.148869514465332,
311
+ "rewards/accuracies": 0.6499999761581421,
312
+ "rewards/chosen": -1.0034563541412354,
313
+ "rewards/margins": -0.10557299852371216,
314
+ "rewards/rejected": -0.8978832364082336,
315
+ "step": 85
316
+ },
317
+ {
318
+ "epoch": 0.5357142857142857,
319
+ "grad_norm": 43.4862060546875,
320
+ "learning_rate": 9e-06,
321
+ "log_odds_chosen": 1.0741676092147827,
322
+ "log_odds_ratio": -0.4483945965766907,
323
+ "logits/chosen": 430.83160400390625,
324
+ "logits/rejected": 437.00006103515625,
325
+ "logps/chosen": -1.3649173974990845,
326
+ "logps/rejected": -2.2547924518585205,
327
+ "loss": 1.8406,
328
+ "nll_loss": 1.6345653533935547,
329
+ "rewards/accuracies": 0.75,
330
+ "rewards/chosen": -0.6824586987495422,
331
+ "rewards/margins": 0.444937527179718,
332
+ "rewards/rejected": -1.1273962259292603,
333
+ "step": 90
334
+ },
335
+ {
336
+ "epoch": 0.5654761904761905,
337
+ "grad_norm": 158.6744384765625,
338
+ "learning_rate": 9.5e-06,
339
+ "log_odds_chosen": 0.3580256700515747,
340
+ "log_odds_ratio": -0.5983911752700806,
341
+ "logits/chosen": 365.0293273925781,
342
+ "logits/rejected": 395.13739013671875,
343
+ "logps/chosen": -1.2539499998092651,
344
+ "logps/rejected": -1.4669833183288574,
345
+ "loss": 1.9122,
346
+ "nll_loss": 1.5921684503555298,
347
+ "rewards/accuracies": 0.6499999761581421,
348
+ "rewards/chosen": -0.6269749999046326,
349
+ "rewards/margins": 0.10651664435863495,
350
+ "rewards/rejected": -0.7334916591644287,
351
+ "step": 95
352
+ },
353
+ {
354
+ "epoch": 0.5952380952380952,
355
+ "grad_norm": 60.67824172973633,
356
+ "learning_rate": 1e-05,
357
+ "log_odds_chosen": 0.5569584965705872,
358
+ "log_odds_ratio": -0.5973536372184753,
359
+ "logits/chosen": 427.83270263671875,
360
+ "logits/rejected": 444.78289794921875,
361
+ "logps/chosen": -1.552487850189209,
362
+ "logps/rejected": -2.052288770675659,
363
+ "loss": 1.9997,
364
+ "nll_loss": 1.694284439086914,
365
+ "rewards/accuracies": 0.75,
366
+ "rewards/chosen": -0.7762439250946045,
367
+ "rewards/margins": 0.2499004304409027,
368
+ "rewards/rejected": -1.0261443853378296,
369
+ "step": 100
370
+ },
371
+ {
372
+ "epoch": 0.625,
373
+ "grad_norm": 66.58687591552734,
374
+ "learning_rate": 9.759000729485331e-06,
375
+ "log_odds_chosen": 0.6742503643035889,
376
+ "log_odds_ratio": -0.5492368340492249,
377
+ "logits/chosen": 421.02435302734375,
378
+ "logits/rejected": 404.02593994140625,
379
+ "logps/chosen": -1.4385154247283936,
380
+ "logps/rejected": -1.9352591037750244,
381
+ "loss": 1.9337,
382
+ "nll_loss": 1.8405609130859375,
383
+ "rewards/accuracies": 0.800000011920929,
384
+ "rewards/chosen": -0.7192577123641968,
385
+ "rewards/margins": 0.24837179481983185,
386
+ "rewards/rejected": -0.9676295518875122,
387
+ "step": 105
388
+ },
389
+ {
390
+ "epoch": 0.6547619047619048,
391
+ "grad_norm": 378.0513916015625,
392
+ "learning_rate": 9.534625892455923e-06,
393
+ "log_odds_chosen": 0.7585327625274658,
394
+ "log_odds_ratio": -0.467681884765625,
395
+ "logits/chosen": 386.55181884765625,
396
+ "logits/rejected": 365.63861083984375,
397
+ "logps/chosen": -1.359151840209961,
398
+ "logps/rejected": -1.9564090967178345,
399
+ "loss": 2.0541,
400
+ "nll_loss": 1.9982373714447021,
401
+ "rewards/accuracies": 0.800000011920929,
402
+ "rewards/chosen": -0.6795759201049805,
403
+ "rewards/margins": 0.29862865805625916,
404
+ "rewards/rejected": -0.9782045483589172,
405
+ "step": 110
406
+ },
407
+ {
408
+ "epoch": 0.6845238095238095,
409
+ "grad_norm": 68.57817077636719,
410
+ "learning_rate": 9.325048082403139e-06,
411
+ "log_odds_chosen": 0.9356454610824585,
412
+ "log_odds_ratio": -0.41365212202072144,
413
+ "logits/chosen": 406.5351867675781,
414
+ "logits/rejected": 452.47161865234375,
415
+ "logps/chosen": -1.2247552871704102,
416
+ "logps/rejected": -1.8875564336776733,
417
+ "loss": 2.0096,
418
+ "nll_loss": 1.6129186153411865,
419
+ "rewards/accuracies": 0.8999999761581421,
420
+ "rewards/chosen": -0.6123776435852051,
421
+ "rewards/margins": 0.33140069246292114,
422
+ "rewards/rejected": -0.9437782168388367,
423
+ "step": 115
424
+ },
425
+ {
426
+ "epoch": 0.7142857142857143,
427
+ "grad_norm": 70.14920806884766,
428
+ "learning_rate": 9.12870929175277e-06,
429
+ "log_odds_chosen": 0.6218617558479309,
430
+ "log_odds_ratio": -0.5581934452056885,
431
+ "logits/chosen": 400.44232177734375,
432
+ "logits/rejected": 403.1569519042969,
433
+ "logps/chosen": -1.2707955837249756,
434
+ "logps/rejected": -1.6316139698028564,
435
+ "loss": 2.0261,
436
+ "nll_loss": 1.7148948907852173,
437
+ "rewards/accuracies": 0.6499999761581421,
438
+ "rewards/chosen": -0.6353977918624878,
439
+ "rewards/margins": 0.18040914833545685,
440
+ "rewards/rejected": -0.8158069849014282,
441
+ "step": 120
442
+ },
443
+ {
444
+ "epoch": 0.7440476190476191,
445
+ "grad_norm": 28.353559494018555,
446
+ "learning_rate": 8.94427190999916e-06,
447
+ "log_odds_chosen": 0.06747283786535263,
448
+ "log_odds_ratio": -0.7747208476066589,
449
+ "logits/chosen": 394.8547668457031,
450
+ "logits/rejected": 402.99114990234375,
451
+ "logps/chosen": -1.4064565896987915,
452
+ "logps/rejected": -1.443865180015564,
453
+ "loss": 1.9533,
454
+ "nll_loss": 1.529036283493042,
455
+ "rewards/accuracies": 0.5,
456
+ "rewards/chosen": -0.7032282948493958,
457
+ "rewards/margins": 0.018704283982515335,
458
+ "rewards/rejected": -0.721932590007782,
459
+ "step": 125
460
+ },
461
+ {
462
+ "epoch": 0.7738095238095238,
463
+ "grad_norm": 50.000038146972656,
464
+ "learning_rate": 8.770580193070294e-06,
465
+ "log_odds_chosen": 0.487846702337265,
466
+ "log_odds_ratio": -0.62076336145401,
467
+ "logits/chosen": 419.7767639160156,
468
+ "logits/rejected": 387.2704772949219,
469
+ "logps/chosen": -1.2864845991134644,
470
+ "logps/rejected": -1.5687696933746338,
471
+ "loss": 1.9848,
472
+ "nll_loss": 1.5705018043518066,
473
+ "rewards/accuracies": 0.5,
474
+ "rewards/chosen": -0.6432422995567322,
475
+ "rewards/margins": 0.14114244282245636,
476
+ "rewards/rejected": -0.7843848466873169,
477
+ "step": 130
478
+ },
479
+ {
480
+ "epoch": 0.8035714285714286,
481
+ "grad_norm": 32.915321350097656,
482
+ "learning_rate": 8.606629658238705e-06,
483
+ "log_odds_chosen": 0.5996901988983154,
484
+ "log_odds_ratio": -0.5842069387435913,
485
+ "logits/chosen": 392.77679443359375,
486
+ "logits/rejected": 423.60272216796875,
487
+ "logps/chosen": -1.3194632530212402,
488
+ "logps/rejected": -1.8708839416503906,
489
+ "loss": 1.9914,
490
+ "nll_loss": 1.6256046295166016,
491
+ "rewards/accuracies": 0.699999988079071,
492
+ "rewards/chosen": -0.6597316265106201,
493
+ "rewards/margins": 0.2757102847099304,
494
+ "rewards/rejected": -0.9354419708251953,
495
+ "step": 135
496
+ },
497
+ {
498
+ "epoch": 0.8333333333333334,
499
+ "grad_norm": 60.51993179321289,
500
+ "learning_rate": 8.451542547285167e-06,
501
+ "log_odds_chosen": 0.14726313948631287,
502
+ "log_odds_ratio": -0.6693505048751831,
503
+ "logits/chosen": 411.6986389160156,
504
+ "logits/rejected": 375.326416015625,
505
+ "logps/chosen": -1.321712851524353,
506
+ "logps/rejected": -1.4512436389923096,
507
+ "loss": 1.9797,
508
+ "nll_loss": 1.6686627864837646,
509
+ "rewards/accuracies": 0.6499999761581421,
510
+ "rewards/chosen": -0.6608564257621765,
511
+ "rewards/margins": 0.0647653192281723,
512
+ "rewards/rejected": -0.7256218194961548,
513
+ "step": 140
514
+ },
515
+ {
516
+ "epoch": 0.8630952380952381,
517
+ "grad_norm": 31.022499084472656,
518
+ "learning_rate": 8.304547985373997e-06,
519
+ "log_odds_chosen": 0.1282106339931488,
520
+ "log_odds_ratio": -0.6646836996078491,
521
+ "logits/chosen": 381.526611328125,
522
+ "logits/rejected": 395.548095703125,
523
+ "logps/chosen": -1.3501091003417969,
524
+ "logps/rejected": -1.4500309228897095,
525
+ "loss": 1.9768,
526
+ "nll_loss": 1.590570330619812,
527
+ "rewards/accuracies": 0.699999988079071,
528
+ "rewards/chosen": -0.6750545501708984,
529
+ "rewards/margins": 0.0499609112739563,
530
+ "rewards/rejected": -0.7250154614448547,
531
+ "step": 145
532
+ },
533
+ {
534
+ "epoch": 0.8928571428571429,
535
+ "grad_norm": 48.54278564453125,
536
+ "learning_rate": 8.164965809277262e-06,
537
+ "log_odds_chosen": 1.1463629007339478,
538
+ "log_odds_ratio": -0.46680259704589844,
539
+ "logits/chosen": 403.89031982421875,
540
+ "logits/rejected": 409.86151123046875,
541
+ "logps/chosen": -1.3419028520584106,
542
+ "logps/rejected": -2.324528217315674,
543
+ "loss": 1.901,
544
+ "nll_loss": 1.6430673599243164,
545
+ "rewards/accuracies": 0.800000011920929,
546
+ "rewards/chosen": -0.6709514260292053,
547
+ "rewards/margins": 0.49131274223327637,
548
+ "rewards/rejected": -1.162264108657837,
549
+ "step": 150
550
+ },
551
+ {
552
+ "epoch": 0.9226190476190477,
553
+ "grad_norm": 44.35458755493164,
554
+ "learning_rate": 8.03219328902499e-06,
555
+ "log_odds_chosen": 0.22458314895629883,
556
+ "log_odds_ratio": -0.6567160487174988,
557
+ "logits/chosen": 399.331298828125,
558
+ "logits/rejected": 395.67767333984375,
559
+ "logps/chosen": -1.445773959159851,
560
+ "logps/rejected": -1.61488938331604,
561
+ "loss": 1.8732,
562
+ "nll_loss": 1.569331407546997,
563
+ "rewards/accuracies": 0.6499999761581421,
564
+ "rewards/chosen": -0.7228869795799255,
565
+ "rewards/margins": 0.08455771207809448,
566
+ "rewards/rejected": -0.80744469165802,
567
+ "step": 155
568
+ },
569
+ {
570
+ "epoch": 0.9523809523809523,
571
+ "grad_norm": 44.52214431762695,
572
+ "learning_rate": 7.905694150420949e-06,
573
+ "log_odds_chosen": 0.3395001292228699,
574
+ "log_odds_ratio": -0.6508662104606628,
575
+ "logits/chosen": 404.9471435546875,
576
+ "logits/rejected": 443.93865966796875,
577
+ "logps/chosen": -1.2228091955184937,
578
+ "logps/rejected": -1.4965049028396606,
579
+ "loss": 1.8769,
580
+ "nll_loss": 1.4934804439544678,
581
+ "rewards/accuracies": 0.44999998807907104,
582
+ "rewards/chosen": -0.6114045977592468,
583
+ "rewards/margins": 0.13684777915477753,
584
+ "rewards/rejected": -0.7482524514198303,
585
+ "step": 160
586
+ },
587
+ {
588
+ "epoch": 0.9821428571428571,
589
+ "grad_norm": 51.537784576416016,
590
+ "learning_rate": 7.78498944161523e-06,
591
+ "log_odds_chosen": 0.2777930200099945,
592
+ "log_odds_ratio": -0.6248766779899597,
593
+ "logits/chosen": 440.94366455078125,
594
+ "logits/rejected": 444.90240478515625,
595
+ "logps/chosen": -1.3909051418304443,
596
+ "logps/rejected": -1.5702444314956665,
597
+ "loss": 1.9721,
598
+ "nll_loss": 1.7042747735977173,
599
+ "rewards/accuracies": 0.6499999761581421,
600
+ "rewards/chosen": -0.6954525709152222,
601
+ "rewards/margins": 0.08966972678899765,
602
+ "rewards/rejected": -0.7851222157478333,
603
+ "step": 165
604
+ },
605
+ {
606
+ "epoch": 1.0,
607
+ "eval_log_odds_chosen": 0.26492369174957275,
608
+ "eval_log_odds_ratio": -0.6573201417922974,
609
+ "eval_logits/chosen": 336.2867126464844,
610
+ "eval_logits/rejected": 282.1214904785156,
611
+ "eval_logps/chosen": -1.2143747806549072,
612
+ "eval_logps/rejected": -1.4054285287857056,
613
+ "eval_loss": 1.9526195526123047,
614
+ "eval_nll_loss": 1.6514703035354614,
615
+ "eval_rewards/accuracies": 0.5571428537368774,
616
+ "eval_rewards/chosen": -0.6071873903274536,
617
+ "eval_rewards/margins": 0.09552692621946335,
618
+ "eval_rewards/rejected": -0.7027142643928528,
619
+ "eval_runtime": 201.1918,
620
+ "eval_samples_per_second": 2.749,
621
+ "eval_steps_per_second": 0.348,
622
+ "step": 168
623
+ },
624
+ {
625
+ "epoch": 1.0119047619047619,
626
+ "grad_norm": 31.192705154418945,
627
+ "learning_rate": 7.669649888473705e-06,
628
+ "log_odds_chosen": 0.9334543347358704,
629
+ "log_odds_ratio": -0.48719945549964905,
630
+ "logits/chosen": 377.3502502441406,
631
+ "logits/rejected": 415.46356201171875,
632
+ "logps/chosen": -1.0376964807510376,
633
+ "logps/rejected": -1.7006464004516602,
634
+ "loss": 1.7058,
635
+ "nll_loss": 1.2853295803070068,
636
+ "rewards/accuracies": 0.699999988079071,
637
+ "rewards/chosen": -0.5188482403755188,
638
+ "rewards/margins": 0.3314751386642456,
639
+ "rewards/rejected": -0.8503232002258301,
640
+ "step": 170
641
+ },
642
+ {
643
+ "epoch": 1.0416666666666667,
644
+ "grad_norm": 23.733516693115234,
645
+ "learning_rate": 7.559289460184545e-06,
646
+ "log_odds_chosen": 1.4257056713104248,
647
+ "log_odds_ratio": -0.36321204900741577,
648
+ "logits/chosen": 406.3605651855469,
649
+ "logits/rejected": 437.1375427246094,
650
+ "logps/chosen": -0.8629204034805298,
651
+ "logps/rejected": -1.8850151300430298,
652
+ "loss": 1.2977,
653
+ "nll_loss": 1.387385368347168,
654
+ "rewards/accuracies": 0.8500000238418579,
655
+ "rewards/chosen": -0.4314602017402649,
656
+ "rewards/margins": 0.5110472440719604,
657
+ "rewards/rejected": -0.9425075650215149,
658
+ "step": 175
659
+ },
660
+ {
661
+ "epoch": 1.0714285714285714,
662
+ "grad_norm": 24.075864791870117,
663
+ "learning_rate": 7.4535599249993e-06,
664
+ "log_odds_chosen": 1.4416855573654175,
665
+ "log_odds_ratio": -0.30401644110679626,
666
+ "logits/chosen": 357.9011535644531,
667
+ "logits/rejected": 352.5140380859375,
668
+ "logps/chosen": -0.899684727191925,
669
+ "logps/rejected": -1.9276078939437866,
670
+ "loss": 1.3288,
671
+ "nll_loss": 1.3482264280319214,
672
+ "rewards/accuracies": 0.8500000238418579,
673
+ "rewards/chosen": -0.4498423635959625,
674
+ "rewards/margins": 0.513961672782898,
675
+ "rewards/rejected": -0.9638039469718933,
676
+ "step": 180
677
+ },
678
+ {
679
+ "epoch": 1.1011904761904763,
680
+ "grad_norm": 39.60210418701172,
681
+ "learning_rate": 7.352146220938079e-06,
682
+ "log_odds_chosen": 2.3201422691345215,
683
+ "log_odds_ratio": -0.20258066058158875,
684
+ "logits/chosen": 449.66485595703125,
685
+ "logits/rejected": 380.39923095703125,
686
+ "logps/chosen": -0.8746849894523621,
687
+ "logps/rejected": -2.625277042388916,
688
+ "loss": 1.2651,
689
+ "nll_loss": 1.2770421504974365,
690
+ "rewards/accuracies": 0.949999988079071,
691
+ "rewards/chosen": -0.43734249472618103,
692
+ "rewards/margins": 0.8752959370613098,
693
+ "rewards/rejected": -1.312638521194458,
694
+ "step": 185
695
+ },
696
+ {
697
+ "epoch": 1.130952380952381,
698
+ "grad_norm": 19.941560745239258,
699
+ "learning_rate": 7.254762501100117e-06,
700
+ "log_odds_chosen": 1.2973986864089966,
701
+ "log_odds_ratio": -0.36347365379333496,
702
+ "logits/chosen": 403.78106689453125,
703
+ "logits/rejected": 423.5147399902344,
704
+ "logps/chosen": -0.9356532096862793,
705
+ "logps/rejected": -1.7905277013778687,
706
+ "loss": 1.3584,
707
+ "nll_loss": 1.2576459646224976,
708
+ "rewards/accuracies": 0.800000011920929,
709
+ "rewards/chosen": -0.46782660484313965,
710
+ "rewards/margins": 0.4274372160434723,
711
+ "rewards/rejected": -0.8952638506889343,
712
+ "step": 190
713
+ },
714
+ {
715
+ "epoch": 1.1607142857142858,
716
+ "grad_norm": 22.545778274536133,
717
+ "learning_rate": 7.1611487403943295e-06,
718
+ "log_odds_chosen": 1.494046688079834,
719
+ "log_odds_ratio": -0.28673815727233887,
720
+ "logits/chosen": 383.939697265625,
721
+ "logits/rejected": 414.34100341796875,
722
+ "logps/chosen": -1.043168067932129,
723
+ "logps/rejected": -2.1456902027130127,
724
+ "loss": 1.3101,
725
+ "nll_loss": 1.2422401905059814,
726
+ "rewards/accuracies": 0.949999988079071,
727
+ "rewards/chosen": -0.5215840339660645,
728
+ "rewards/margins": 0.5512610673904419,
729
+ "rewards/rejected": -1.0728451013565063,
730
+ "step": 195
731
+ },
732
+ {
733
+ "epoch": 1.1904761904761905,
734
+ "grad_norm": 29.15802574157715,
735
+ "learning_rate": 7.0710678118654756e-06,
736
+ "log_odds_chosen": 2.355477809906006,
737
+ "log_odds_ratio": -0.23161384463310242,
738
+ "logits/chosen": 443.63848876953125,
739
+ "logits/rejected": 404.1456298828125,
740
+ "logps/chosen": -0.6128617525100708,
741
+ "logps/rejected": -2.293651580810547,
742
+ "loss": 1.2885,
743
+ "nll_loss": 1.0000107288360596,
744
+ "rewards/accuracies": 0.8999999761581421,
745
+ "rewards/chosen": -0.3064308762550354,
746
+ "rewards/margins": 0.8403949737548828,
747
+ "rewards/rejected": -1.1468257904052734,
748
+ "step": 200
749
+ },
750
+ {
751
+ "epoch": 1.2202380952380953,
752
+ "grad_norm": 25.433277130126953,
753
+ "learning_rate": 6.984302957695783e-06,
754
+ "log_odds_chosen": 2.206387758255005,
755
+ "log_odds_ratio": -0.20685645937919617,
756
+ "logits/chosen": 345.31134033203125,
757
+ "logits/rejected": 396.1018371582031,
758
+ "logps/chosen": -0.8278995752334595,
759
+ "logps/rejected": -2.5073094367980957,
760
+ "loss": 1.2908,
761
+ "nll_loss": 1.2252819538116455,
762
+ "rewards/accuracies": 1.0,
763
+ "rewards/chosen": -0.41394978761672974,
764
+ "rewards/margins": 0.8397049903869629,
765
+ "rewards/rejected": -1.2536547183990479,
766
+ "step": 205
767
+ },
768
+ {
769
+ "epoch": 1.25,
770
+ "grad_norm": 20.637460708618164,
771
+ "learning_rate": 6.900655593423542e-06,
772
+ "log_odds_chosen": 1.841051697731018,
773
+ "log_odds_ratio": -0.25199171900749207,
774
+ "logits/chosen": 375.1767272949219,
775
+ "logits/rejected": 373.2859191894531,
776
+ "logps/chosen": -0.8490890264511108,
777
+ "logps/rejected": -2.084282159805298,
778
+ "loss": 1.1767,
779
+ "nll_loss": 1.2052141427993774,
780
+ "rewards/accuracies": 1.0,
781
+ "rewards/chosen": -0.4245445132255554,
782
+ "rewards/margins": 0.617596447467804,
783
+ "rewards/rejected": -1.042141079902649,
784
+ "step": 210
785
+ },
786
+ {
787
+ "epoch": 1.2797619047619047,
788
+ "grad_norm": 44.84423065185547,
789
+ "learning_rate": 6.819943394704736e-06,
790
+ "log_odds_chosen": 2.0608296394348145,
791
+ "log_odds_ratio": -0.26352304220199585,
792
+ "logits/chosen": 417.9048767089844,
793
+ "logits/rejected": 423.32275390625,
794
+ "logps/chosen": -0.8630622029304504,
795
+ "logps/rejected": -2.449791669845581,
796
+ "loss": 1.2804,
797
+ "nll_loss": 1.2531594038009644,
798
+ "rewards/accuracies": 0.949999988079071,
799
+ "rewards/chosen": -0.4315311014652252,
800
+ "rewards/margins": 0.7933648824691772,
801
+ "rewards/rejected": -1.2248958349227905,
802
+ "step": 215
803
+ },
804
+ {
805
+ "epoch": 1.3095238095238095,
806
+ "grad_norm": 27.052371978759766,
807
+ "learning_rate": 6.741998624632421e-06,
808
+ "log_odds_chosen": 1.9642305374145508,
809
+ "log_odds_ratio": -0.2985631823539734,
810
+ "logits/chosen": 365.51226806640625,
811
+ "logits/rejected": 383.2918395996094,
812
+ "logps/chosen": -0.7986623048782349,
813
+ "logps/rejected": -2.1426806449890137,
814
+ "loss": 1.2905,
815
+ "nll_loss": 1.1290483474731445,
816
+ "rewards/accuracies": 0.8999999761581421,
817
+ "rewards/chosen": -0.39933115243911743,
818
+ "rewards/margins": 0.6720091700553894,
819
+ "rewards/rejected": -1.0713403224945068,
820
+ "step": 220
821
+ },
822
+ {
823
+ "epoch": 1.3392857142857144,
824
+ "grad_norm": 33.49964141845703,
825
+ "learning_rate": 6.666666666666667e-06,
826
+ "log_odds_chosen": 1.2250487804412842,
827
+ "log_odds_ratio": -0.34452176094055176,
828
+ "logits/chosen": 399.16046142578125,
829
+ "logits/rejected": 376.3241882324219,
830
+ "logps/chosen": -1.1176211833953857,
831
+ "logps/rejected": -2.0609307289123535,
832
+ "loss": 1.2819,
833
+ "nll_loss": 1.3445219993591309,
834
+ "rewards/accuracies": 0.8500000238418579,
835
+ "rewards/chosen": -0.5588105916976929,
836
+ "rewards/margins": 0.47165459394454956,
837
+ "rewards/rejected": -1.0304653644561768,
838
+ "step": 225
839
+ },
840
+ {
841
+ "epoch": 1.369047619047619,
842
+ "grad_norm": 20.881301879882812,
843
+ "learning_rate": 6.593804733957872e-06,
844
+ "log_odds_chosen": 2.1563384532928467,
845
+ "log_odds_ratio": -0.24819080531597137,
846
+ "logits/chosen": 367.4872131347656,
847
+ "logits/rejected": 376.99822998046875,
848
+ "logps/chosen": -0.9552156329154968,
849
+ "logps/rejected": -2.5535261631011963,
850
+ "loss": 1.2569,
851
+ "nll_loss": 0.9765374064445496,
852
+ "rewards/accuracies": 0.8999999761581421,
853
+ "rewards/chosen": -0.4776078164577484,
854
+ "rewards/margins": 0.7991552948951721,
855
+ "rewards/rejected": -1.2767630815505981,
856
+ "step": 230
857
+ },
858
+ {
859
+ "epoch": 1.3988095238095237,
860
+ "grad_norm": 20.453393936157227,
861
+ "learning_rate": 6.523280730534423e-06,
862
+ "log_odds_chosen": 2.148529529571533,
863
+ "log_odds_ratio": -0.2209801971912384,
864
+ "logits/chosen": 396.610595703125,
865
+ "logits/rejected": 402.3414306640625,
866
+ "logps/chosen": -0.6876020431518555,
867
+ "logps/rejected": -2.1799914836883545,
868
+ "loss": 1.2833,
869
+ "nll_loss": 1.0352197885513306,
870
+ "rewards/accuracies": 0.949999988079071,
871
+ "rewards/chosen": -0.34380102157592773,
872
+ "rewards/margins": 0.7461946606636047,
873
+ "rewards/rejected": -1.0899957418441772,
874
+ "step": 235
875
+ },
876
+ {
877
+ "epoch": 1.4285714285714286,
878
+ "grad_norm": 23.43726348876953,
879
+ "learning_rate": 6.4549722436790284e-06,
880
+ "log_odds_chosen": 1.4042203426361084,
881
+ "log_odds_ratio": -0.30760836601257324,
882
+ "logits/chosen": 426.6971740722656,
883
+ "logits/rejected": 434.68292236328125,
884
+ "logps/chosen": -0.9825772047042847,
885
+ "logps/rejected": -2.0324692726135254,
886
+ "loss": 1.258,
887
+ "nll_loss": 1.1937551498413086,
888
+ "rewards/accuracies": 0.949999988079071,
889
+ "rewards/chosen": -0.49128860235214233,
890
+ "rewards/margins": 0.5249461531639099,
891
+ "rewards/rejected": -1.0162346363067627,
892
+ "step": 240
893
+ },
894
+ {
895
+ "epoch": 1.4583333333333333,
896
+ "grad_norm": 20.512117385864258,
897
+ "learning_rate": 6.3887656499994e-06,
898
+ "log_odds_chosen": 1.7208242416381836,
899
+ "log_odds_ratio": -0.2485727071762085,
900
+ "logits/chosen": 406.27337646484375,
901
+ "logits/rejected": 381.36297607421875,
902
+ "logps/chosen": -0.7839127779006958,
903
+ "logps/rejected": -2.025153398513794,
904
+ "loss": 1.3187,
905
+ "nll_loss": 1.2436693906784058,
906
+ "rewards/accuracies": 0.949999988079071,
907
+ "rewards/chosen": -0.3919563889503479,
908
+ "rewards/margins": 0.6206203699111938,
909
+ "rewards/rejected": -1.012576699256897,
910
+ "step": 245
911
+ },
912
+ {
913
+ "epoch": 1.4880952380952381,
914
+ "grad_norm": 31.212268829345703,
915
+ "learning_rate": 6.324555320336759e-06,
916
+ "log_odds_chosen": 2.171504497528076,
917
+ "log_odds_ratio": -0.2626289427280426,
918
+ "logits/chosen": 385.3293151855469,
919
+ "logits/rejected": 403.4792785644531,
920
+ "logps/chosen": -0.912436842918396,
921
+ "logps/rejected": -2.581498146057129,
922
+ "loss": 1.2409,
923
+ "nll_loss": 0.9795435070991516,
924
+ "rewards/accuracies": 0.8999999761581421,
925
+ "rewards/chosen": -0.456218421459198,
926
+ "rewards/margins": 0.8345306515693665,
927
+ "rewards/rejected": -1.2907490730285645,
928
+ "step": 250
929
+ },
930
+ {
931
+ "epoch": 1.5178571428571428,
932
+ "grad_norm": 15.0794095993042,
933
+ "learning_rate": 6.262242910851496e-06,
934
+ "log_odds_chosen": 1.983006238937378,
935
+ "log_odds_ratio": -0.19495443999767303,
936
+ "logits/chosen": 411.68487548828125,
937
+ "logits/rejected": 416.602783203125,
938
+ "logps/chosen": -0.9126818776130676,
939
+ "logps/rejected": -2.4313483238220215,
940
+ "loss": 1.2696,
941
+ "nll_loss": 1.102388620376587,
942
+ "rewards/accuracies": 1.0,
943
+ "rewards/chosen": -0.4563409388065338,
944
+ "rewards/margins": 0.7593332529067993,
945
+ "rewards/rejected": -1.2156741619110107,
946
+ "step": 255
947
+ },
948
+ {
949
+ "epoch": 1.5476190476190477,
950
+ "grad_norm": 27.49896812438965,
951
+ "learning_rate": 6.2017367294604225e-06,
952
+ "log_odds_chosen": 1.9481725692749023,
953
+ "log_odds_ratio": -0.2753888964653015,
954
+ "logits/chosen": 386.07427978515625,
955
+ "logits/rejected": 435.198974609375,
956
+ "logps/chosen": -0.995970606803894,
957
+ "logps/rejected": -2.5512473583221436,
958
+ "loss": 1.2264,
959
+ "nll_loss": 1.1618086099624634,
960
+ "rewards/accuracies": 0.8500000238418579,
961
+ "rewards/chosen": -0.497985303401947,
962
+ "rewards/margins": 0.7776384353637695,
963
+ "rewards/rejected": -1.2756236791610718,
964
+ "step": 260
965
+ },
966
+ {
967
+ "epoch": 1.5773809523809523,
968
+ "grad_norm": 23.22437286376953,
969
+ "learning_rate": 6.142951168339513e-06,
970
+ "log_odds_chosen": 1.3944361209869385,
971
+ "log_odds_ratio": -0.4270511567592621,
972
+ "logits/chosen": 435.14678955078125,
973
+ "logits/rejected": 427.51226806640625,
974
+ "logps/chosen": -1.104048252105713,
975
+ "logps/rejected": -2.160891056060791,
976
+ "loss": 1.4157,
977
+ "nll_loss": 1.3079249858856201,
978
+ "rewards/accuracies": 0.75,
979
+ "rewards/chosen": -0.5520241260528564,
980
+ "rewards/margins": 0.5284214019775391,
981
+ "rewards/rejected": -1.0804455280303955,
982
+ "step": 265
983
+ },
984
+ {
985
+ "epoch": 1.6071428571428572,
986
+ "grad_norm": 16.965179443359375,
987
+ "learning_rate": 6.0858061945018455e-06,
988
+ "log_odds_chosen": 1.3110793828964233,
989
+ "log_odds_ratio": -0.33642929792404175,
990
+ "logits/chosen": 370.2662658691406,
991
+ "logits/rejected": 376.83587646484375,
992
+ "logps/chosen": -1.1280148029327393,
993
+ "logps/rejected": -2.0509979724884033,
994
+ "loss": 1.3064,
995
+ "nll_loss": 1.3204824924468994,
996
+ "rewards/accuracies": 0.8999999761581421,
997
+ "rewards/chosen": -0.5640074014663696,
998
+ "rewards/margins": 0.46149152517318726,
999
+ "rewards/rejected": -1.0254989862442017,
1000
+ "step": 270
1001
+ },
1002
+ {
1003
+ "epoch": 1.6369047619047619,
1004
+ "grad_norm": 16.231523513793945,
1005
+ "learning_rate": 6.030226891555273e-06,
1006
+ "log_odds_chosen": 1.2828176021575928,
1007
+ "log_odds_ratio": -0.31571871042251587,
1008
+ "logits/chosen": 336.78143310546875,
1009
+ "logits/rejected": 352.0708312988281,
1010
+ "logps/chosen": -0.8390641212463379,
1011
+ "logps/rejected": -1.6753209829330444,
1012
+ "loss": 1.2993,
1013
+ "nll_loss": 1.0436512231826782,
1014
+ "rewards/accuracies": 1.0,
1015
+ "rewards/chosen": -0.41953206062316895,
1016
+ "rewards/margins": 0.4181283414363861,
1017
+ "rewards/rejected": -0.8376604914665222,
1018
+ "step": 275
1019
+ },
1020
+ {
1021
+ "epoch": 1.6666666666666665,
1022
+ "grad_norm": 19.623210906982422,
1023
+ "learning_rate": 5.976143046671968e-06,
1024
+ "log_odds_chosen": 1.4294779300689697,
1025
+ "log_odds_ratio": -0.3383074104785919,
1026
+ "logits/chosen": 406.8611145019531,
1027
+ "logits/rejected": 369.0583190917969,
1028
+ "logps/chosen": -0.7519195675849915,
1029
+ "logps/rejected": -1.7197290658950806,
1030
+ "loss": 1.3075,
1031
+ "nll_loss": 1.0520720481872559,
1032
+ "rewards/accuracies": 0.8999999761581421,
1033
+ "rewards/chosen": -0.3759597837924957,
1034
+ "rewards/margins": 0.4839046597480774,
1035
+ "rewards/rejected": -0.8598645329475403,
1036
+ "step": 280
1037
+ },
1038
+ {
1039
+ "epoch": 1.6964285714285714,
1040
+ "grad_norm": 27.15911293029785,
1041
+ "learning_rate": 5.923488777590924e-06,
1042
+ "log_odds_chosen": 1.6469166278839111,
1043
+ "log_odds_ratio": -0.31998661160469055,
1044
+ "logits/chosen": 351.8115234375,
1045
+ "logits/rejected": 378.2767333984375,
1046
+ "logps/chosen": -0.8496532440185547,
1047
+ "logps/rejected": -2.034947156906128,
1048
+ "loss": 1.2238,
1049
+ "nll_loss": 1.0820589065551758,
1050
+ "rewards/accuracies": 0.949999988079071,
1051
+ "rewards/chosen": -0.42482662200927734,
1052
+ "rewards/margins": 0.5926468968391418,
1053
+ "rewards/rejected": -1.017473578453064,
1054
+ "step": 285
1055
+ },
1056
+ {
1057
+ "epoch": 1.7261904761904763,
1058
+ "grad_norm": 20.430383682250977,
1059
+ "learning_rate": 5.8722021951470355e-06,
1060
+ "log_odds_chosen": 2.2868614196777344,
1061
+ "log_odds_ratio": -0.23290471732616425,
1062
+ "logits/chosen": 376.9328918457031,
1063
+ "logits/rejected": 444.3408203125,
1064
+ "logps/chosen": -0.8962064981460571,
1065
+ "logps/rejected": -2.7214858531951904,
1066
+ "loss": 1.2798,
1067
+ "nll_loss": 1.1513203382492065,
1068
+ "rewards/accuracies": 0.8999999761581421,
1069
+ "rewards/chosen": -0.44810324907302856,
1070
+ "rewards/margins": 0.9126396179199219,
1071
+ "rewards/rejected": -1.3607429265975952,
1072
+ "step": 290
1073
+ },
1074
+ {
1075
+ "epoch": 1.755952380952381,
1076
+ "grad_norm": 23.614900588989258,
1077
+ "learning_rate": 5.822225097395821e-06,
1078
+ "log_odds_chosen": 2.100496768951416,
1079
+ "log_odds_ratio": -0.2871156930923462,
1080
+ "logits/chosen": 391.25726318359375,
1081
+ "logits/rejected": 399.76214599609375,
1082
+ "logps/chosen": -1.0318877696990967,
1083
+ "logps/rejected": -2.5897607803344727,
1084
+ "loss": 1.318,
1085
+ "nll_loss": 1.339220404624939,
1086
+ "rewards/accuracies": 0.8500000238418579,
1087
+ "rewards/chosen": -0.5159438848495483,
1088
+ "rewards/margins": 0.778936505317688,
1089
+ "rewards/rejected": -1.2948803901672363,
1090
+ "step": 295
1091
+ },
1092
+ {
1093
+ "epoch": 1.7857142857142856,
1094
+ "grad_norm": 24.594526290893555,
1095
+ "learning_rate": 5.773502691896259e-06,
1096
+ "log_odds_chosen": 1.4617587327957153,
1097
+ "log_odds_ratio": -0.268736869096756,
1098
+ "logits/chosen": 392.9894714355469,
1099
+ "logits/rejected": 392.593994140625,
1100
+ "logps/chosen": -0.716581404209137,
1101
+ "logps/rejected": -1.6443252563476562,
1102
+ "loss": 1.2342,
1103
+ "nll_loss": 1.0071418285369873,
1104
+ "rewards/accuracies": 0.8999999761581421,
1105
+ "rewards/chosen": -0.3582907021045685,
1106
+ "rewards/margins": 0.46387186646461487,
1107
+ "rewards/rejected": -0.8221626281738281,
1108
+ "step": 300
1109
+ },
1110
+ {
1111
+ "epoch": 1.8154761904761905,
1112
+ "grad_norm": 18.39592742919922,
1113
+ "learning_rate": 5.725983343138682e-06,
1114
+ "log_odds_chosen": 1.3188327550888062,
1115
+ "log_odds_ratio": -0.39420002698898315,
1116
+ "logits/chosen": 443.49505615234375,
1117
+ "logits/rejected": 423.3663635253906,
1118
+ "logps/chosen": -1.0477676391601562,
1119
+ "logps/rejected": -2.009697914123535,
1120
+ "loss": 1.2633,
1121
+ "nll_loss": 1.2299957275390625,
1122
+ "rewards/accuracies": 0.8500000238418579,
1123
+ "rewards/chosen": -0.5238838195800781,
1124
+ "rewards/margins": 0.4809652864933014,
1125
+ "rewards/rejected": -1.0048489570617676,
1126
+ "step": 305
1127
+ },
1128
+ {
1129
+ "epoch": 1.8452380952380953,
1130
+ "grad_norm": 16.925289154052734,
1131
+ "learning_rate": 5.679618342470648e-06,
1132
+ "log_odds_chosen": 1.8564685583114624,
1133
+ "log_odds_ratio": -0.2928754985332489,
1134
+ "logits/chosen": 358.899169921875,
1135
+ "logits/rejected": 364.58160400390625,
1136
+ "logps/chosen": -0.94548100233078,
1137
+ "logps/rejected": -2.3730039596557617,
1138
+ "loss": 1.2961,
1139
+ "nll_loss": 1.2056444883346558,
1140
+ "rewards/accuracies": 1.0,
1141
+ "rewards/chosen": -0.47274050116539,
1142
+ "rewards/margins": 0.7137616276741028,
1143
+ "rewards/rejected": -1.1865019798278809,
1144
+ "step": 310
1145
+ },
1146
+ {
1147
+ "epoch": 1.875,
1148
+ "grad_norm": 27.0323543548584,
1149
+ "learning_rate": 5.63436169819011e-06,
1150
+ "log_odds_chosen": 1.889154076576233,
1151
+ "log_odds_ratio": -0.3951663076877594,
1152
+ "logits/chosen": 385.2926025390625,
1153
+ "logits/rejected": 391.605712890625,
1154
+ "logps/chosen": -1.166025161743164,
1155
+ "logps/rejected": -2.7089405059814453,
1156
+ "loss": 1.3311,
1157
+ "nll_loss": 1.3940035104751587,
1158
+ "rewards/accuracies": 0.800000011920929,
1159
+ "rewards/chosen": -0.583012580871582,
1160
+ "rewards/margins": 0.7714576125144958,
1161
+ "rewards/rejected": -1.3544702529907227,
1162
+ "step": 315
1163
+ },
1164
+ {
1165
+ "epoch": 1.9047619047619047,
1166
+ "grad_norm": 18.699766159057617,
1167
+ "learning_rate": 5.590169943749475e-06,
1168
+ "log_odds_chosen": 1.6885192394256592,
1169
+ "log_odds_ratio": -0.2244713306427002,
1170
+ "logits/chosen": 402.7138366699219,
1171
+ "logits/rejected": 435.62646484375,
1172
+ "logps/chosen": -0.8159521222114563,
1173
+ "logps/rejected": -2.033271551132202,
1174
+ "loss": 1.2117,
1175
+ "nll_loss": 0.9898951649665833,
1176
+ "rewards/accuracies": 1.0,
1177
+ "rewards/chosen": -0.40797606110572815,
1178
+ "rewards/margins": 0.6086598634719849,
1179
+ "rewards/rejected": -1.016635775566101,
1180
+ "step": 320
1181
+ },
1182
+ {
1183
+ "epoch": 1.9345238095238095,
1184
+ "grad_norm": 29.9803409576416,
1185
+ "learning_rate": 5.547001962252292e-06,
1186
+ "log_odds_chosen": 2.429278612136841,
1187
+ "log_odds_ratio": -0.19841960072517395,
1188
+ "logits/chosen": 411.4713439941406,
1189
+ "logits/rejected": 433.3556213378906,
1190
+ "logps/chosen": -0.8200858235359192,
1191
+ "logps/rejected": -2.58778977394104,
1192
+ "loss": 1.1848,
1193
+ "nll_loss": 1.0703952312469482,
1194
+ "rewards/accuracies": 0.949999988079071,
1195
+ "rewards/chosen": -0.4100429117679596,
1196
+ "rewards/margins": 0.8838518857955933,
1197
+ "rewards/rejected": -1.29389488697052,
1198
+ "step": 325
1199
+ },
1200
+ {
1201
+ "epoch": 1.9642857142857144,
1202
+ "grad_norm": 15.92293643951416,
1203
+ "learning_rate": 5.504818825631804e-06,
1204
+ "log_odds_chosen": 1.846983551979065,
1205
+ "log_odds_ratio": -0.29716789722442627,
1206
+ "logits/chosen": 407.4903869628906,
1207
+ "logits/rejected": 413.0613708496094,
1208
+ "logps/chosen": -0.9037710428237915,
1209
+ "logps/rejected": -2.2958996295928955,
1210
+ "loss": 1.245,
1211
+ "nll_loss": 0.9782114028930664,
1212
+ "rewards/accuracies": 0.949999988079071,
1213
+ "rewards/chosen": -0.45188552141189575,
1214
+ "rewards/margins": 0.6960643529891968,
1215
+ "rewards/rejected": -1.1479498147964478,
1216
+ "step": 330
1217
+ },
1218
+ {
1219
+ "epoch": 1.994047619047619,
1220
+ "grad_norm": 15.895071029663086,
1221
+ "learning_rate": 5.4635836470815305e-06,
1222
+ "log_odds_chosen": 1.2502014636993408,
1223
+ "log_odds_ratio": -0.3804013133049011,
1224
+ "logits/chosen": 390.8824157714844,
1225
+ "logits/rejected": 404.8577575683594,
1226
+ "logps/chosen": -1.2015631198883057,
1227
+ "logps/rejected": -2.143710136413574,
1228
+ "loss": 1.3299,
1229
+ "nll_loss": 1.439396619796753,
1230
+ "rewards/accuracies": 0.800000011920929,
1231
+ "rewards/chosen": -0.6007815599441528,
1232
+ "rewards/margins": 0.47107353806495667,
1233
+ "rewards/rejected": -1.071855068206787,
1234
+ "step": 335
1235
+ },
1236
+ {
1237
+ "epoch": 2.0,
1238
+ "eval_log_odds_chosen": 0.2173391729593277,
1239
+ "eval_log_odds_ratio": -0.6792100071907043,
1240
+ "eval_logits/chosen": 345.2332763671875,
1241
+ "eval_logits/rejected": 293.2820129394531,
1242
+ "eval_logps/chosen": -1.1971725225448608,
1243
+ "eval_logps/rejected": -1.3610831499099731,
1244
+ "eval_loss": 1.9015214443206787,
1245
+ "eval_nll_loss": 1.5933443307876587,
1246
+ "eval_rewards/accuracies": 0.5,
1247
+ "eval_rewards/chosen": -0.5985862612724304,
1248
+ "eval_rewards/margins": 0.08195527642965317,
1249
+ "eval_rewards/rejected": -0.6805415749549866,
1250
+ "eval_runtime": 201.3424,
1251
+ "eval_samples_per_second": 2.747,
1252
+ "eval_steps_per_second": 0.348,
1253
+ "step": 336
1254
+ },
1255
+ {
1256
+ "epoch": 2.0238095238095237,
1257
+ "grad_norm": 18.064809799194336,
1258
+ "learning_rate": 5.423261445466404e-06,
1259
+ "log_odds_chosen": 3.1630072593688965,
1260
+ "log_odds_ratio": -0.14342114329338074,
1261
+ "logits/chosen": 407.103759765625,
1262
+ "logits/rejected": 396.9122009277344,
1263
+ "logps/chosen": -0.5497515201568604,
1264
+ "logps/rejected": -2.797853946685791,
1265
+ "loss": 0.7506,
1266
+ "nll_loss": 0.7164371609687805,
1267
+ "rewards/accuracies": 0.949999988079071,
1268
+ "rewards/chosen": -0.2748757600784302,
1269
+ "rewards/margins": 1.1240513324737549,
1270
+ "rewards/rejected": -1.3989269733428955,
1271
+ "step": 340
1272
+ },
1273
+ {
1274
+ "epoch": 2.0535714285714284,
1275
+ "grad_norm": 12.874629974365234,
1276
+ "learning_rate": 5.383819020581656e-06,
1277
+ "log_odds_chosen": 4.300416946411133,
1278
+ "log_odds_ratio": -0.10298861563205719,
1279
+ "logits/chosen": 406.04144287109375,
1280
+ "logits/rejected": 365.400390625,
1281
+ "logps/chosen": -0.5357804894447327,
1282
+ "logps/rejected": -3.574709415435791,
1283
+ "loss": 0.6675,
1284
+ "nll_loss": 0.8001909255981445,
1285
+ "rewards/accuracies": 0.8999999761581421,
1286
+ "rewards/chosen": -0.26789024472236633,
1287
+ "rewards/margins": 1.5194646120071411,
1288
+ "rewards/rejected": -1.7873547077178955,
1289
+ "step": 345
1290
+ },
1291
+ {
1292
+ "epoch": 2.0833333333333335,
1293
+ "grad_norm": 13.677016258239746,
1294
+ "learning_rate": 5.345224838248489e-06,
1295
+ "log_odds_chosen": 4.409786224365234,
1296
+ "log_odds_ratio": -0.03061295673251152,
1297
+ "logits/chosen": 372.30560302734375,
1298
+ "logits/rejected": 401.3008728027344,
1299
+ "logps/chosen": -0.46024495363235474,
1300
+ "logps/rejected": -3.737445116043091,
1301
+ "loss": 0.5993,
1302
+ "nll_loss": 0.6028808355331421,
1303
+ "rewards/accuracies": 1.0,
1304
+ "rewards/chosen": -0.23012247681617737,
1305
+ "rewards/margins": 1.6386003494262695,
1306
+ "rewards/rejected": -1.8687225580215454,
1307
+ "step": 350
1308
+ },
1309
+ {
1310
+ "epoch": 2.113095238095238,
1311
+ "grad_norm": 10.955292701721191,
1312
+ "learning_rate": 5.307448924342753e-06,
1313
+ "log_odds_chosen": 4.361363410949707,
1314
+ "log_odds_ratio": -0.04152694344520569,
1315
+ "logits/chosen": 373.2965393066406,
1316
+ "logits/rejected": 396.0318603515625,
1317
+ "logps/chosen": -0.4621841311454773,
1318
+ "logps/rejected": -3.721182346343994,
1319
+ "loss": 0.5932,
1320
+ "nll_loss": 0.5964406132698059,
1321
+ "rewards/accuracies": 1.0,
1322
+ "rewards/chosen": -0.23109206557273865,
1323
+ "rewards/margins": 1.6294991970062256,
1324
+ "rewards/rejected": -1.860591173171997,
1325
+ "step": 355
1326
+ },
1327
+ {
1328
+ "epoch": 2.142857142857143,
1329
+ "grad_norm": 28.92525863647461,
1330
+ "learning_rate": 5.270462766947299e-06,
1331
+ "log_odds_chosen": 4.147698402404785,
1332
+ "log_odds_ratio": -0.032929155975580215,
1333
+ "logits/chosen": 380.5809631347656,
1334
+ "logits/rejected": 416.3023986816406,
1335
+ "logps/chosen": -0.381146639585495,
1336
+ "logps/rejected": -3.1521759033203125,
1337
+ "loss": 0.612,
1338
+ "nll_loss": 0.6254645586013794,
1339
+ "rewards/accuracies": 1.0,
1340
+ "rewards/chosen": -0.1905733197927475,
1341
+ "rewards/margins": 1.3855146169662476,
1342
+ "rewards/rejected": -1.5760879516601562,
1343
+ "step": 360
1344
+ },
1345
+ {
1346
+ "epoch": 2.1726190476190474,
1347
+ "grad_norm": 12.443119049072266,
1348
+ "learning_rate": 5.234239225902137e-06,
1349
+ "log_odds_chosen": 4.118858337402344,
1350
+ "log_odds_ratio": -0.03749427944421768,
1351
+ "logits/chosen": 345.683349609375,
1352
+ "logits/rejected": 312.6221923828125,
1353
+ "logps/chosen": -0.32834386825561523,
1354
+ "logps/rejected": -3.117326498031616,
1355
+ "loss": 0.5807,
1356
+ "nll_loss": 0.531657338142395,
1357
+ "rewards/accuracies": 1.0,
1358
+ "rewards/chosen": -0.16417193412780762,
1359
+ "rewards/margins": 1.3944913148880005,
1360
+ "rewards/rejected": -1.558663249015808,
1361
+ "step": 365
1362
+ },
1363
+ {
1364
+ "epoch": 2.2023809523809526,
1365
+ "grad_norm": 10.90052318572998,
1366
+ "learning_rate": 5.198752449100364e-06,
1367
+ "log_odds_chosen": 3.4755806922912598,
1368
+ "log_odds_ratio": -0.04619182273745537,
1369
+ "logits/chosen": 332.7835693359375,
1370
+ "logits/rejected": 340.1016845703125,
1371
+ "logps/chosen": -0.4380973279476166,
1372
+ "logps/rejected": -2.8499865531921387,
1373
+ "loss": 0.6158,
1374
+ "nll_loss": 0.6762995719909668,
1375
+ "rewards/accuracies": 1.0,
1376
+ "rewards/chosen": -0.2190486639738083,
1377
+ "rewards/margins": 1.2059446573257446,
1378
+ "rewards/rejected": -1.4249932765960693,
1379
+ "step": 370
1380
+ },
1381
+ {
1382
+ "epoch": 2.232142857142857,
1383
+ "grad_norm": 14.661003112792969,
1384
+ "learning_rate": 5.163977794943223e-06,
1385
+ "log_odds_chosen": 3.5086026191711426,
1386
+ "log_odds_ratio": -0.11584819853305817,
1387
+ "logits/chosen": 396.9617614746094,
1388
+ "logits/rejected": 412.2447814941406,
1389
+ "logps/chosen": -0.4914408326148987,
1390
+ "logps/rejected": -2.892894744873047,
1391
+ "loss": 0.6302,
1392
+ "nll_loss": 0.5993391275405884,
1393
+ "rewards/accuracies": 0.8999999761581421,
1394
+ "rewards/chosen": -0.24572041630744934,
1395
+ "rewards/margins": 1.2007267475128174,
1396
+ "rewards/rejected": -1.4464473724365234,
1397
+ "step": 375
1398
+ },
1399
+ {
1400
+ "epoch": 2.261904761904762,
1401
+ "grad_norm": 13.687707901000977,
1402
+ "learning_rate": 5.129891760425772e-06,
1403
+ "log_odds_chosen": 3.4452381134033203,
1404
+ "log_odds_ratio": -0.04614276438951492,
1405
+ "logits/chosen": 386.617919921875,
1406
+ "logits/rejected": 348.6525573730469,
1407
+ "logps/chosen": -0.43144264817237854,
1408
+ "logps/rejected": -2.679448366165161,
1409
+ "loss": 0.5916,
1410
+ "nll_loss": 0.61055588722229,
1411
+ "rewards/accuracies": 1.0,
1412
+ "rewards/chosen": -0.21572132408618927,
1413
+ "rewards/margins": 1.1240026950836182,
1414
+ "rewards/rejected": -1.3397241830825806,
1415
+ "step": 380
1416
+ },
1417
+ {
1418
+ "epoch": 2.2916666666666665,
1419
+ "grad_norm": 12.487543106079102,
1420
+ "learning_rate": 5.096471914376255e-06,
1421
+ "log_odds_chosen": 4.143012046813965,
1422
+ "log_odds_ratio": -0.055028241127729416,
1423
+ "logits/chosen": 369.7757873535156,
1424
+ "logits/rejected": 385.3197326660156,
1425
+ "logps/chosen": -0.42009004950523376,
1426
+ "logps/rejected": -3.1088523864746094,
1427
+ "loss": 0.5661,
1428
+ "nll_loss": 0.5417618155479431,
1429
+ "rewards/accuracies": 0.949999988079071,
1430
+ "rewards/chosen": -0.21004502475261688,
1431
+ "rewards/margins": 1.3443810939788818,
1432
+ "rewards/rejected": -1.5544261932373047,
1433
+ "step": 385
1434
+ },
1435
+ {
1436
+ "epoch": 2.3214285714285716,
1437
+ "grad_norm": 14.711711883544922,
1438
+ "learning_rate": 5.0636968354183334e-06,
1439
+ "log_odds_chosen": 3.86775541305542,
1440
+ "log_odds_ratio": -0.051612865179777145,
1441
+ "logits/chosen": 386.55340576171875,
1442
+ "logits/rejected": 420.259033203125,
1443
+ "logps/chosen": -0.40563899278640747,
1444
+ "logps/rejected": -3.13773775100708,
1445
+ "loss": 0.6336,
1446
+ "nll_loss": 0.5235159993171692,
1447
+ "rewards/accuracies": 1.0,
1448
+ "rewards/chosen": -0.20281949639320374,
1449
+ "rewards/margins": 1.3660494089126587,
1450
+ "rewards/rejected": -1.56886887550354,
1451
+ "step": 390
1452
+ },
1453
+ {
1454
+ "epoch": 2.3511904761904763,
1455
+ "grad_norm": 14.341813087463379,
1456
+ "learning_rate": 5.031546054266276e-06,
1457
+ "log_odds_chosen": 4.3614888191223145,
1458
+ "log_odds_ratio": -0.02659059502184391,
1459
+ "logits/chosen": 381.09478759765625,
1460
+ "logits/rejected": 371.54730224609375,
1461
+ "logps/chosen": -0.3334965705871582,
1462
+ "logps/rejected": -3.2248425483703613,
1463
+ "loss": 0.6006,
1464
+ "nll_loss": 0.5441581010818481,
1465
+ "rewards/accuracies": 1.0,
1466
+ "rewards/chosen": -0.1667482852935791,
1467
+ "rewards/margins": 1.4456731081008911,
1468
+ "rewards/rejected": -1.6124212741851807,
1469
+ "step": 395
1470
+ },
1471
+ {
1472
+ "epoch": 2.380952380952381,
1473
+ "grad_norm": 10.141727447509766,
1474
+ "learning_rate": 5e-06,
1475
+ "log_odds_chosen": 5.030593395233154,
1476
+ "log_odds_ratio": -0.060914844274520874,
1477
+ "logits/chosen": 360.45758056640625,
1478
+ "logits/rejected": 407.6477966308594,
1479
+ "logps/chosen": -0.41616684198379517,
1480
+ "logps/rejected": -4.148199558258057,
1481
+ "loss": 0.5815,
1482
+ "nll_loss": 0.6026356220245361,
1483
+ "rewards/accuracies": 0.949999988079071,
1484
+ "rewards/chosen": -0.20808342099189758,
1485
+ "rewards/margins": 1.8660163879394531,
1486
+ "rewards/rejected": -2.0740997791290283,
1487
+ "step": 400
1488
+ },
1489
+ {
1490
+ "epoch": 2.4107142857142856,
1491
+ "grad_norm": 12.312828063964844,
1492
+ "learning_rate": 4.969039949999534e-06,
1493
+ "log_odds_chosen": 3.7259132862091064,
1494
+ "log_odds_ratio": -0.05210161954164505,
1495
+ "logits/chosen": 441.5244140625,
1496
+ "logits/rejected": 379.39215087890625,
1497
+ "logps/chosen": -0.31666994094848633,
1498
+ "logps/rejected": -2.7463951110839844,
1499
+ "loss": 0.6506,
1500
+ "nll_loss": 0.5460222959518433,
1501
+ "rewards/accuracies": 1.0,
1502
+ "rewards/chosen": -0.15833497047424316,
1503
+ "rewards/margins": 1.214862585067749,
1504
+ "rewards/rejected": -1.3731975555419922,
1505
+ "step": 405
1506
+ },
1507
+ {
1508
+ "epoch": 2.4404761904761907,
1509
+ "grad_norm": 10.084498405456543,
1510
+ "learning_rate": 4.938647983247949e-06,
1511
+ "log_odds_chosen": 4.13731575012207,
1512
+ "log_odds_ratio": -0.07053720951080322,
1513
+ "logits/chosen": 375.71875,
1514
+ "logits/rejected": 404.51898193359375,
1515
+ "logps/chosen": -0.4222637116909027,
1516
+ "logps/rejected": -3.3887131214141846,
1517
+ "loss": 0.5941,
1518
+ "nll_loss": 0.5796958804130554,
1519
+ "rewards/accuracies": 0.949999988079071,
1520
+ "rewards/chosen": -0.21113185584545135,
1521
+ "rewards/margins": 1.483224630355835,
1522
+ "rewards/rejected": -1.6943565607070923,
1523
+ "step": 410
1524
+ },
1525
+ {
1526
+ "epoch": 2.4702380952380953,
1527
+ "grad_norm": 10.309317588806152,
1528
+ "learning_rate": 4.9088069367381605e-06,
1529
+ "log_odds_chosen": 4.0256781578063965,
1530
+ "log_odds_ratio": -0.02946905419230461,
1531
+ "logits/chosen": 410.3990783691406,
1532
+ "logits/rejected": 388.10638427734375,
1533
+ "logps/chosen": -0.34235674142837524,
1534
+ "logps/rejected": -2.9506092071533203,
1535
+ "loss": 0.6024,
1536
+ "nll_loss": 0.5277774333953857,
1537
+ "rewards/accuracies": 1.0,
1538
+ "rewards/chosen": -0.17117837071418762,
1539
+ "rewards/margins": 1.3041261434555054,
1540
+ "rewards/rejected": -1.4753046035766602,
1541
+ "step": 415
1542
+ },
1543
+ {
1544
+ "epoch": 2.5,
1545
+ "grad_norm": 17.37739372253418,
1546
+ "learning_rate": 4.8795003647426654e-06,
1547
+ "log_odds_chosen": 4.524823188781738,
1548
+ "log_odds_ratio": -0.054420001804828644,
1549
+ "logits/chosen": 391.82403564453125,
1550
+ "logits/rejected": 331.8238220214844,
1551
+ "logps/chosen": -0.296555757522583,
1552
+ "logps/rejected": -3.383528470993042,
1553
+ "loss": 0.5781,
1554
+ "nll_loss": 0.6599777340888977,
1555
+ "rewards/accuracies": 0.949999988079071,
1556
+ "rewards/chosen": -0.1482778787612915,
1557
+ "rewards/margins": 1.5434863567352295,
1558
+ "rewards/rejected": -1.691764235496521,
1559
+ "step": 420
1560
+ },
1561
+ {
1562
+ "epoch": 2.5297619047619047,
1563
+ "grad_norm": 14.520256042480469,
1564
+ "learning_rate": 4.850712500726659e-06,
1565
+ "log_odds_chosen": 4.5146284103393555,
1566
+ "log_odds_ratio": -0.020355457440018654,
1567
+ "logits/chosen": 364.44293212890625,
1568
+ "logits/rejected": 381.6509094238281,
1569
+ "logps/chosen": -0.4376618266105652,
1570
+ "logps/rejected": -3.6549148559570312,
1571
+ "loss": 0.6064,
1572
+ "nll_loss": 0.7541596293449402,
1573
+ "rewards/accuracies": 1.0,
1574
+ "rewards/chosen": -0.2188309133052826,
1575
+ "rewards/margins": 1.6086266040802002,
1576
+ "rewards/rejected": -1.8274574279785156,
1577
+ "step": 425
1578
+ },
1579
+ {
1580
+ "epoch": 2.5595238095238093,
1581
+ "grad_norm": 18.008617401123047,
1582
+ "learning_rate": 4.822428221704122e-06,
1583
+ "log_odds_chosen": 4.2900166511535645,
1584
+ "log_odds_ratio": -0.05251041799783707,
1585
+ "logits/chosen": 418.76812744140625,
1586
+ "logits/rejected": 422.1298828125,
1587
+ "logps/chosen": -0.3535473942756653,
1588
+ "logps/rejected": -3.3458237648010254,
1589
+ "loss": 0.5961,
1590
+ "nll_loss": 0.49905306100845337,
1591
+ "rewards/accuracies": 1.0,
1592
+ "rewards/chosen": -0.17677369713783264,
1593
+ "rewards/margins": 1.496138334274292,
1594
+ "rewards/rejected": -1.6729118824005127,
1595
+ "step": 430
1596
+ },
1597
+ {
1598
+ "epoch": 2.5892857142857144,
1599
+ "grad_norm": 14.501811027526855,
1600
+ "learning_rate": 4.794633014853843e-06,
1601
+ "log_odds_chosen": 3.7202048301696777,
1602
+ "log_odds_ratio": -0.08040798455476761,
1603
+ "logits/chosen": 378.04962158203125,
1604
+ "logits/rejected": 374.47149658203125,
1605
+ "logps/chosen": -0.4141341745853424,
1606
+ "logps/rejected": -3.029280424118042,
1607
+ "loss": 0.6443,
1608
+ "nll_loss": 0.5974027514457703,
1609
+ "rewards/accuracies": 1.0,
1610
+ "rewards/chosen": -0.2070670872926712,
1611
+ "rewards/margins": 1.3075730800628662,
1612
+ "rewards/rejected": -1.514640212059021,
1613
+ "step": 435
1614
+ },
1615
+ {
1616
+ "epoch": 2.619047619047619,
1617
+ "grad_norm": 13.21033000946045,
1618
+ "learning_rate": 4.767312946227961e-06,
1619
+ "log_odds_chosen": 4.036019325256348,
1620
+ "log_odds_ratio": -0.0349675677716732,
1621
+ "logits/chosen": 374.5437927246094,
1622
+ "logits/rejected": 379.2315368652344,
1623
+ "logps/chosen": -0.357850581407547,
1624
+ "logps/rejected": -3.0918896198272705,
1625
+ "loss": 0.6481,
1626
+ "nll_loss": 0.5479155778884888,
1627
+ "rewards/accuracies": 1.0,
1628
+ "rewards/chosen": -0.1789252907037735,
1629
+ "rewards/margins": 1.3670194149017334,
1630
+ "rewards/rejected": -1.5459448099136353,
1631
+ "step": 440
1632
+ },
1633
+ {
1634
+ "epoch": 2.6488095238095237,
1635
+ "grad_norm": 10.330572128295898,
1636
+ "learning_rate": 4.740454631399773e-06,
1637
+ "log_odds_chosen": 4.903168678283691,
1638
+ "log_odds_ratio": -0.02322390116751194,
1639
+ "logits/chosen": 356.780029296875,
1640
+ "logits/rejected": 385.78631591796875,
1641
+ "logps/chosen": -0.3364141881465912,
1642
+ "logps/rejected": -3.9139695167541504,
1643
+ "loss": 0.5982,
1644
+ "nll_loss": 0.5599361658096313,
1645
+ "rewards/accuracies": 1.0,
1646
+ "rewards/chosen": -0.1682070940732956,
1647
+ "rewards/margins": 1.7887779474258423,
1648
+ "rewards/rejected": -1.9569847583770752,
1649
+ "step": 445
1650
+ },
1651
+ {
1652
+ "epoch": 2.678571428571429,
1653
+ "grad_norm": 11.032569885253906,
1654
+ "learning_rate": 4.714045207910318e-06,
1655
+ "log_odds_chosen": 4.497644901275635,
1656
+ "log_odds_ratio": -0.030209308490157127,
1657
+ "logits/chosen": 394.76336669921875,
1658
+ "logits/rejected": 376.0134582519531,
1659
+ "logps/chosen": -0.41811317205429077,
1660
+ "logps/rejected": -3.552691698074341,
1661
+ "loss": 0.6268,
1662
+ "nll_loss": 0.5461404919624329,
1663
+ "rewards/accuracies": 1.0,
1664
+ "rewards/chosen": -0.20905658602714539,
1665
+ "rewards/margins": 1.567289113998413,
1666
+ "rewards/rejected": -1.7763458490371704,
1667
+ "step": 450
1668
+ },
1669
+ {
1670
+ "epoch": 2.7083333333333335,
1671
+ "grad_norm": 17.51761245727539,
1672
+ "learning_rate": 4.688072309384955e-06,
1673
+ "log_odds_chosen": 4.075136661529541,
1674
+ "log_odds_ratio": -0.025673285126686096,
1675
+ "logits/chosen": 384.58154296875,
1676
+ "logits/rejected": 360.6435546875,
1677
+ "logps/chosen": -0.29184406995773315,
1678
+ "logps/rejected": -2.7901484966278076,
1679
+ "loss": 0.6721,
1680
+ "nll_loss": 0.5232545733451843,
1681
+ "rewards/accuracies": 1.0,
1682
+ "rewards/chosen": -0.14592203497886658,
1683
+ "rewards/margins": 1.249152421951294,
1684
+ "rewards/rejected": -1.3950742483139038,
1685
+ "step": 455
1686
+ },
1687
+ {
1688
+ "epoch": 2.738095238095238,
1689
+ "grad_norm": 13.356114387512207,
1690
+ "learning_rate": 4.662524041201569e-06,
1691
+ "log_odds_chosen": 3.7381584644317627,
1692
+ "log_odds_ratio": -0.041400760412216187,
1693
+ "logits/chosen": 424.15606689453125,
1694
+ "logits/rejected": 436.59417724609375,
1695
+ "logps/chosen": -0.3177054226398468,
1696
+ "logps/rejected": -2.7719643115997314,
1697
+ "loss": 0.6171,
1698
+ "nll_loss": 0.5559448599815369,
1699
+ "rewards/accuracies": 1.0,
1700
+ "rewards/chosen": -0.1588527113199234,
1701
+ "rewards/margins": 1.2271292209625244,
1702
+ "rewards/rejected": -1.3859821557998657,
1703
+ "step": 460
1704
+ },
1705
+ {
1706
+ "epoch": 2.767857142857143,
1707
+ "grad_norm": 16.436243057250977,
1708
+ "learning_rate": 4.6373889576016826e-06,
1709
+ "log_odds_chosen": 3.9720139503479004,
1710
+ "log_odds_ratio": -0.04150586202740669,
1711
+ "logits/chosen": 419.3877868652344,
1712
+ "logits/rejected": 399.4032287597656,
1713
+ "logps/chosen": -0.3887273669242859,
1714
+ "logps/rejected": -3.1958889961242676,
1715
+ "loss": 0.6517,
1716
+ "nll_loss": 0.7023509740829468,
1717
+ "rewards/accuracies": 1.0,
1718
+ "rewards/chosen": -0.19436368346214294,
1719
+ "rewards/margins": 1.403580665588379,
1720
+ "rewards/rejected": -1.5979444980621338,
1721
+ "step": 465
1722
+ },
1723
+ {
1724
+ "epoch": 2.7976190476190474,
1725
+ "grad_norm": 13.92647647857666,
1726
+ "learning_rate": 4.6126560401444256e-06,
1727
+ "log_odds_chosen": 3.6036312580108643,
1728
+ "log_odds_ratio": -0.0841737613081932,
1729
+ "logits/chosen": 376.7361145019531,
1730
+ "logits/rejected": 346.24578857421875,
1731
+ "logps/chosen": -0.38905754685401917,
1732
+ "logps/rejected": -2.8610401153564453,
1733
+ "loss": 0.5864,
1734
+ "nll_loss": 0.5663691163063049,
1735
+ "rewards/accuracies": 0.949999988079071,
1736
+ "rewards/chosen": -0.19452877342700958,
1737
+ "rewards/margins": 1.235991358757019,
1738
+ "rewards/rejected": -1.4305200576782227,
1739
+ "step": 470
1740
+ },
1741
+ {
1742
+ "epoch": 2.8273809523809526,
1743
+ "grad_norm": 22.44719123840332,
1744
+ "learning_rate": 4.588314677411235e-06,
1745
+ "log_odds_chosen": 4.282876014709473,
1746
+ "log_odds_ratio": -0.02305762842297554,
1747
+ "logits/chosen": 351.80535888671875,
1748
+ "logits/rejected": 435.5419006347656,
1749
+ "logps/chosen": -0.376276433467865,
1750
+ "logps/rejected": -3.4008407592773438,
1751
+ "loss": 0.6159,
1752
+ "nll_loss": 0.5037115216255188,
1753
+ "rewards/accuracies": 1.0,
1754
+ "rewards/chosen": -0.1881382167339325,
1755
+ "rewards/margins": 1.512282133102417,
1756
+ "rewards/rejected": -1.7004203796386719,
1757
+ "step": 475
1758
+ },
1759
+ {
1760
+ "epoch": 2.857142857142857,
1761
+ "grad_norm": 11.814839363098145,
1762
+ "learning_rate": 4.564354645876385e-06,
1763
+ "log_odds_chosen": 5.438357353210449,
1764
+ "log_odds_ratio": -0.03407539427280426,
1765
+ "logits/chosen": 393.2117614746094,
1766
+ "logits/rejected": 400.9731140136719,
1767
+ "logps/chosen": -0.38579824566841125,
1768
+ "logps/rejected": -4.569952964782715,
1769
+ "loss": 0.5976,
1770
+ "nll_loss": 0.6172084212303162,
1771
+ "rewards/accuracies": 1.0,
1772
+ "rewards/chosen": -0.19289912283420563,
1773
+ "rewards/margins": 2.0920770168304443,
1774
+ "rewards/rejected": -2.2849764823913574,
1775
+ "step": 480
1776
+ },
1777
+ {
1778
+ "epoch": 2.886904761904762,
1779
+ "grad_norm": 12.605895042419434,
1780
+ "learning_rate": 4.540766091864998e-06,
1781
+ "log_odds_chosen": 3.7765631675720215,
1782
+ "log_odds_ratio": -0.03285397216677666,
1783
+ "logits/chosen": 376.1605529785156,
1784
+ "logits/rejected": 400.92620849609375,
1785
+ "logps/chosen": -0.4142071604728699,
1786
+ "logps/rejected": -3.060633659362793,
1787
+ "loss": 0.5919,
1788
+ "nll_loss": 0.5258246660232544,
1789
+ "rewards/accuracies": 1.0,
1790
+ "rewards/chosen": -0.20710358023643494,
1791
+ "rewards/margins": 1.3232133388519287,
1792
+ "rewards/rejected": -1.5303168296813965,
1793
+ "step": 485
1794
+ },
1795
+ {
1796
+ "epoch": 2.9166666666666665,
1797
+ "grad_norm": 10.655911445617676,
1798
+ "learning_rate": 4.517539514526257e-06,
1799
+ "log_odds_chosen": 3.3025474548339844,
1800
+ "log_odds_ratio": -0.07940138876438141,
1801
+ "logits/chosen": 416.90509033203125,
1802
+ "logits/rejected": 319.48687744140625,
1803
+ "logps/chosen": -0.6181944012641907,
1804
+ "logps/rejected": -2.8924221992492676,
1805
+ "loss": 0.6671,
1806
+ "nll_loss": 0.5718962550163269,
1807
+ "rewards/accuracies": 0.949999988079071,
1808
+ "rewards/chosen": -0.30909720063209534,
1809
+ "rewards/margins": 1.1371139287948608,
1810
+ "rewards/rejected": -1.4462110996246338,
1811
+ "step": 490
1812
+ },
1813
+ {
1814
+ "epoch": 2.946428571428571,
1815
+ "grad_norm": 11.349733352661133,
1816
+ "learning_rate": 4.4946657497549474e-06,
1817
+ "log_odds_chosen": 4.556464672088623,
1818
+ "log_odds_ratio": -0.033807143568992615,
1819
+ "logits/chosen": 363.4445495605469,
1820
+ "logits/rejected": 406.9335021972656,
1821
+ "logps/chosen": -0.3999672532081604,
1822
+ "logps/rejected": -3.7019195556640625,
1823
+ "loss": 0.6043,
1824
+ "nll_loss": 0.5209922790527344,
1825
+ "rewards/accuracies": 1.0,
1826
+ "rewards/chosen": -0.1999836266040802,
1827
+ "rewards/margins": 1.6509761810302734,
1828
+ "rewards/rejected": -1.8509597778320312,
1829
+ "step": 495
1830
+ },
1831
+ {
1832
+ "epoch": 2.9761904761904763,
1833
+ "grad_norm": 10.738035202026367,
1834
+ "learning_rate": 4.47213595499958e-06,
1835
+ "log_odds_chosen": 4.959429740905762,
1836
+ "log_odds_ratio": -0.03518088907003403,
1837
+ "logits/chosen": 417.14154052734375,
1838
+ "logits/rejected": 379.5878601074219,
1839
+ "logps/chosen": -0.2811127007007599,
1840
+ "logps/rejected": -3.7553272247314453,
1841
+ "loss": 0.6266,
1842
+ "nll_loss": 0.42872363328933716,
1843
+ "rewards/accuracies": 1.0,
1844
+ "rewards/chosen": -0.14055635035037994,
1845
+ "rewards/margins": 1.7371070384979248,
1846
+ "rewards/rejected": -1.8776636123657227,
1847
+ "step": 500
1848
+ },
1849
+ {
1850
+ "epoch": 3.0,
1851
+ "eval_log_odds_chosen": 0.44554343819618225,
1852
+ "eval_log_odds_ratio": -0.6395100355148315,
1853
+ "eval_logits/chosen": 319.3428649902344,
1854
+ "eval_logits/rejected": 262.5517883300781,
1855
+ "eval_logps/chosen": -1.3908888101577759,
1856
+ "eval_logps/rejected": -1.6692452430725098,
1857
+ "eval_loss": 2.1094672679901123,
1858
+ "eval_nll_loss": 1.7835990190505981,
1859
+ "eval_rewards/accuracies": 0.5571428537368774,
1860
+ "eval_rewards/chosen": -0.6954444050788879,
1861
+ "eval_rewards/margins": 0.13917820155620575,
1862
+ "eval_rewards/rejected": -0.8346226215362549,
1863
+ "eval_runtime": 201.4063,
1864
+ "eval_samples_per_second": 2.746,
1865
+ "eval_steps_per_second": 0.348,
1866
+ "step": 504
1867
+ },
1868
+ {
1869
+ "epoch": 3.0,
1870
+ "step": 504,
1871
+ "total_flos": 0.0,
1872
+ "train_loss": 1.593297282854716,
1873
+ "train_runtime": 15168.4679,
1874
+ "train_samples_per_second": 1.061,
1875
+ "train_steps_per_second": 0.033
1876
+ }
1877
+ ],
1878
+ "logging_steps": 5,
1879
+ "max_steps": 504,
1880
+ "num_input_tokens_seen": 0,
1881
+ "num_train_epochs": 3,
1882
+ "save_steps": 500,
1883
+ "stateful_callbacks": {
1884
+ "TrainerControl": {
1885
+ "args": {
1886
+ "should_epoch_stop": false,
1887
+ "should_evaluate": false,
1888
+ "should_log": false,
1889
+ "should_save": false,
1890
+ "should_training_stop": false
1891
+ },
1892
+ "attributes": {}
1893
+ }
1894
+ },
1895
+ "total_flos": 0.0,
1896
+ "train_batch_size": 2,
1897
+ "trial_name": null,
1898
+ "trial_params": null
1899
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:486e0036e86edb72aded2adfe67a129b169da9b403bc7d940f225a2265436ad7
3
+ size 6776