sfulay commited on
Commit
a15b904
·
verified ·
1 Parent(s): 5d5ba27

Model save

Browse files
README.md CHANGED
@@ -4,6 +4,7 @@ base_model: alignment-handbook/zephyr-7b-sft-full
4
  tags:
5
  - trl
6
  - dpo
 
7
  - generated_from_trainer
8
  model-index:
9
  - name: zephyr-7b-dpo-full-prometheus_consistent-low-curriculum
@@ -17,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
17
 
18
  This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
19
  It achieves the following results on the evaluation set:
20
- - Loss: 0.5216
21
- - Rewards/chosen: -0.8203
22
- - Rewards/rejected: -1.7663
23
- - Rewards/accuracies: 0.75
24
- - Rewards/margins: 0.9460
25
- - Logps/rejected: -395.7078
26
- - Logps/chosen: -357.6330
27
- - Logits/rejected: 0.0656
28
- - Logits/chosen: -1.2359
29
 
30
  ## Model description
31
 
@@ -62,14 +63,14 @@ The following hyperparameters were used during training:
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
- | 0.662 | 0.1143 | 50 | 0.6466 | -0.0212 | -0.1670 | 0.6940 | 0.1458 | -235.7726 | -277.7203 | -2.4687 | -2.5688 |
66
- | 0.5829 | 0.2286 | 100 | 0.5717 | -0.4507 | -1.0582 | 0.75 | 0.6075 | -324.8917 | -320.6731 | -0.8703 | -1.2813 |
67
- | 0.5449 | 0.3429 | 150 | 0.5481 | -0.4787 | -1.2088 | 0.7586 | 0.7301 | -339.9565 | -323.4756 | -0.5998 | -1.4586 |
68
- | 0.5265 | 0.4571 | 200 | 0.5408 | -0.5815 | -1.4843 | 0.7629 | 0.9029 | -367.5099 | -333.7512 | -0.3317 | -1.4393 |
69
- | 0.5634 | 0.5714 | 250 | 0.5355 | -1.0776 | -1.9985 | 0.7672 | 0.9209 | -418.9265 | -383.3671 | 0.4274 | -0.9575 |
70
- | 0.521 | 0.6857 | 300 | 0.5250 | -0.8302 | -1.8422 | 0.7672 | 1.0119 | -403.2920 | -358.6284 | 0.4320 | -1.0318 |
71
- | 0.5252 | 0.8 | 350 | 0.5232 | -0.9076 | -1.8790 | 0.7457 | 0.9714 | -406.9760 | -366.3648 | 0.3793 | -1.0407 |
72
- | 0.5357 | 0.9143 | 400 | 0.5216 | -0.8203 | -1.7663 | 0.75 | 0.9460 | -395.7078 | -357.6330 | 0.0656 | -1.2359 |
73
 
74
 
75
  ### Framework versions
 
4
  tags:
5
  - trl
6
  - dpo
7
+ - alignment-handbook
8
  - generated_from_trainer
9
  model-index:
10
  - name: zephyr-7b-dpo-full-prometheus_consistent-low-curriculum
 
18
 
19
  This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
20
  It achieves the following results on the evaluation set:
21
+ - Loss: 0.5227
22
+ - Rewards/chosen: -0.6988
23
+ - Rewards/rejected: -1.6296
24
+ - Rewards/accuracies: 0.7629
25
+ - Rewards/margins: 0.9308
26
+ - Logps/rejected: -382.0352
27
+ - Logps/chosen: -345.4830
28
+ - Logits/rejected: 0.7649
29
+ - Logits/chosen: -0.3664
30
 
31
  ## Model description
32
 
 
63
 
64
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
65
  |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
66
+ | 0.6619 | 0.1143 | 50 | 0.6465 | -0.0188 | -0.1651 | 0.6940 | 0.1464 | -235.5894 | -277.4793 | -2.4695 | -2.5696 |
67
+ | 0.5843 | 0.2286 | 100 | 0.5720 | -0.4443 | -1.0204 | 0.7457 | 0.5761 | -321.1169 | -320.0323 | -1.1914 | -1.3850 |
68
+ | 0.5509 | 0.3429 | 150 | 0.5467 | -0.4786 | -1.2647 | 0.7328 | 0.7861 | -345.5482 | -323.4684 | -0.6635 | -1.3552 |
69
+ | 0.5275 | 0.4571 | 200 | 0.5396 | -0.5220 | -1.3807 | 0.7716 | 0.8587 | -357.1487 | -327.8021 | -0.0895 | -1.0377 |
70
+ | 0.5665 | 0.5714 | 250 | 0.5405 | -1.0842 | -1.9608 | 0.7629 | 0.8766 | -415.1577 | -384.0230 | 1.5254 | 0.2878 |
71
+ | 0.5202 | 0.6857 | 300 | 0.5275 | -0.7424 | -1.7347 | 0.7716 | 0.9923 | -392.5496 | -349.8434 | 0.9934 | -0.2192 |
72
+ | 0.5261 | 0.8 | 350 | 0.5234 | -0.7613 | -1.7196 | 0.7586 | 0.9584 | -391.0397 | -351.7310 | 0.9008 | -0.2697 |
73
+ | 0.5343 | 0.9143 | 400 | 0.5227 | -0.6988 | -1.6296 | 0.7629 | 0.9308 | -382.0352 | -345.4830 | 0.7649 | -0.3664 |
74
 
75
 
76
  ### Framework versions
all_results.json CHANGED
@@ -1,9 +1,22 @@
1
  {
2
  "epoch": 0.9988571428571429,
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  "total_flos": 0.0,
4
- "train_loss": 0.5593291070968662,
5
- "train_runtime": 11268.1264,
6
  "train_samples": 55999,
7
- "train_samples_per_second": 4.97,
8
  "train_steps_per_second": 0.039
9
  }
 
1
  {
2
  "epoch": 0.9988571428571429,
3
+ "eval_logits/chosen": -1.2403382062911987,
4
+ "eval_logits/rejected": 0.062164660543203354,
5
+ "eval_logps/chosen": -357.4463806152344,
6
+ "eval_logps/rejected": -395.7604064941406,
7
+ "eval_loss": 0.5215353965759277,
8
+ "eval_rewards/accuracies": 0.7456896305084229,
9
+ "eval_rewards/chosen": -0.8184244632720947,
10
+ "eval_rewards/margins": 0.948418378829956,
11
+ "eval_rewards/rejected": -1.7668428421020508,
12
+ "eval_runtime": 92.4063,
13
+ "eval_samples": 1831,
14
+ "eval_samples_per_second": 19.815,
15
+ "eval_steps_per_second": 0.314,
16
  "total_flos": 0.0,
17
+ "train_loss": 0.5600041279258117,
18
+ "train_runtime": 11271.3049,
19
  "train_samples": 55999,
20
+ "train_samples_per_second": 4.968,
21
  "train_steps_per_second": 0.039
22
  }
eval_results.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.9988571428571429,
3
+ "eval_logits/chosen": -1.2403382062911987,
4
+ "eval_logits/rejected": 0.062164660543203354,
5
+ "eval_logps/chosen": -357.4463806152344,
6
+ "eval_logps/rejected": -395.7604064941406,
7
+ "eval_loss": 0.5215353965759277,
8
+ "eval_rewards/accuracies": 0.7456896305084229,
9
+ "eval_rewards/chosen": -0.8184244632720947,
10
+ "eval_rewards/margins": 0.948418378829956,
11
+ "eval_rewards/rejected": -1.7668428421020508,
12
+ "eval_runtime": 92.4063,
13
+ "eval_samples": 1831,
14
+ "eval_samples_per_second": 19.815,
15
+ "eval_steps_per_second": 0.314
16
+ }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6e92c22e590ba358ee27bef464a1762f96a789fa9abded669bd438565b9cae89
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8e623605fa3bbafa67cb410816687a0721c2410e0270a2013e1995f114ec7ae6
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:525d54af197c91282b153fe09952efe865cb827e9fbc4ddcacb934e2f6b24655
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:245a992275365f6ee90a40eb8598df9cf1fae14f09a85445924cdb0bda5ce59f
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:073618ac24da9ff46472e8fadc10d471253af6e69b5d0ab002b7641e1af25055
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7d92997f328d5b8a521cb3504f1e718907ff6f7f8f7edea04e3c80ea37cc84d
3
  size 4540516344
train_results.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "epoch": 0.9988571428571429,
3
  "total_flos": 0.0,
4
- "train_loss": 0.5593291070968662,
5
- "train_runtime": 11268.1264,
6
  "train_samples": 55999,
7
- "train_samples_per_second": 4.97,
8
  "train_steps_per_second": 0.039
9
  }
 
1
  {
2
  "epoch": 0.9988571428571429,
3
  "total_flos": 0.0,
4
+ "train_loss": 0.5600041279258117,
5
+ "train_runtime": 11271.3049,
6
  "train_samples": 55999,
7
+ "train_samples_per_second": 4.968,
8
  "train_steps_per_second": 0.039
9
  }
trainer_state.json CHANGED
@@ -10,784 +10,784 @@
10
  "log_history": [
11
  {
12
  "epoch": 0.022857142857142857,
13
- "grad_norm": 8.107446577217848,
14
  "learning_rate": 1.1363636363636363e-07,
15
- "logits/chosen": -2.6695966720581055,
16
- "logits/rejected": -2.6076207160949707,
17
- "logps/chosen": -296.6441650390625,
18
- "logps/rejected": -254.7543487548828,
19
- "loss": 0.6932,
20
- "rewards/accuracies": 0.4312500059604645,
21
- "rewards/chosen": 9.977961599361151e-05,
22
- "rewards/margins": 9.826607856666669e-05,
23
- "rewards/rejected": 1.5136087085920735e-06,
24
  "step": 10
25
  },
26
  {
27
  "epoch": 0.045714285714285714,
28
- "grad_norm": 9.775215212899207,
29
  "learning_rate": 2.2727272727272726e-07,
30
- "logits/chosen": -2.689204692840576,
31
- "logits/rejected": -2.623645305633545,
32
- "logps/chosen": -294.1236267089844,
33
- "logps/rejected": -254.0284881591797,
34
  "loss": 0.6922,
35
- "rewards/accuracies": 0.612500011920929,
36
- "rewards/chosen": 0.0009323403355665505,
37
- "rewards/margins": 0.0030489463824778795,
38
- "rewards/rejected": -0.002116605406627059,
39
  "step": 20
40
  },
41
  {
42
  "epoch": 0.06857142857142857,
43
- "grad_norm": 8.467128246471026,
44
  "learning_rate": 3.4090909090909085e-07,
45
- "logits/chosen": -2.6579556465148926,
46
- "logits/rejected": -2.56276273727417,
47
- "logps/chosen": -285.685791015625,
48
- "logps/rejected": -247.09814453125,
49
- "loss": 0.688,
50
- "rewards/accuracies": 0.643750011920929,
51
- "rewards/chosen": 0.01188767608255148,
52
- "rewards/margins": 0.01455541979521513,
53
- "rewards/rejected": -0.0026677455753087997,
54
  "step": 30
55
  },
56
  {
57
  "epoch": 0.09142857142857143,
58
- "grad_norm": 10.382674354799562,
59
  "learning_rate": 4.545454545454545e-07,
60
- "logits/chosen": -2.624577760696411,
61
- "logits/rejected": -2.5955650806427,
62
- "logps/chosen": -269.8204345703125,
63
- "logps/rejected": -273.1560974121094,
64
- "loss": 0.6792,
65
- "rewards/accuracies": 0.668749988079071,
66
- "rewards/chosen": 0.024544764310121536,
67
- "rewards/margins": 0.020519858226180077,
68
- "rewards/rejected": 0.004024903755635023,
69
  "step": 40
70
  },
71
  {
72
  "epoch": 0.11428571428571428,
73
- "grad_norm": 8.79099848469502,
74
  "learning_rate": 4.997124959943201e-07,
75
- "logits/chosen": -2.613304615020752,
76
- "logits/rejected": -2.575565814971924,
77
- "logps/chosen": -305.5827941894531,
78
- "logps/rejected": -271.3218994140625,
79
- "loss": 0.662,
80
  "rewards/accuracies": 0.7124999761581421,
81
- "rewards/chosen": -0.010970579460263252,
82
- "rewards/margins": 0.06770028173923492,
83
- "rewards/rejected": -0.07867084443569183,
84
  "step": 50
85
  },
86
  {
87
  "epoch": 0.11428571428571428,
88
- "eval_logits/chosen": -2.5687849521636963,
89
- "eval_logits/rejected": -2.4686827659606934,
90
- "eval_logps/chosen": -277.7203369140625,
91
- "eval_logps/rejected": -235.77255249023438,
92
- "eval_loss": 0.646622896194458,
93
  "eval_rewards/accuracies": 0.693965494632721,
94
- "eval_rewards/chosen": -0.021163778379559517,
95
- "eval_rewards/margins": 0.14580054581165314,
96
- "eval_rewards/rejected": -0.1669643372297287,
97
- "eval_runtime": 91.1434,
98
- "eval_samples_per_second": 20.089,
99
- "eval_steps_per_second": 0.318,
100
  "step": 50
101
  },
102
  {
103
  "epoch": 0.13714285714285715,
104
- "grad_norm": 11.503278845369955,
105
  "learning_rate": 4.979579212164186e-07,
106
- "logits/chosen": -2.599658489227295,
107
- "logits/rejected": -2.5006279945373535,
108
- "logps/chosen": -299.0731201171875,
109
- "logps/rejected": -302.80633544921875,
110
- "loss": 0.6306,
111
  "rewards/accuracies": 0.71875,
112
- "rewards/chosen": -0.05183562636375427,
113
- "rewards/margins": 0.1643407940864563,
114
- "rewards/rejected": -0.21617642045021057,
115
  "step": 60
116
  },
117
  {
118
  "epoch": 0.16,
119
- "grad_norm": 15.91025838952606,
120
  "learning_rate": 4.946196886175515e-07,
121
- "logits/chosen": -2.632972002029419,
122
- "logits/rejected": -2.559365749359131,
123
- "logps/chosen": -280.2527160644531,
124
- "logps/rejected": -274.1287536621094,
125
- "loss": 0.6184,
126
  "rewards/accuracies": 0.6625000238418579,
127
- "rewards/chosen": -0.16350679099559784,
128
- "rewards/margins": 0.2035621702671051,
129
- "rewards/rejected": -0.36706894636154175,
130
  "step": 70
131
  },
132
  {
133
  "epoch": 0.18285714285714286,
134
- "grad_norm": 21.12456805447821,
135
  "learning_rate": 4.897191188239667e-07,
136
- "logits/chosen": -2.6217806339263916,
137
- "logits/rejected": -2.561105251312256,
138
- "logps/chosen": -323.9505615234375,
139
- "logps/rejected": -302.5912170410156,
140
- "loss": 0.6108,
141
- "rewards/accuracies": 0.762499988079071,
142
- "rewards/chosen": -0.1250532567501068,
143
- "rewards/margins": 0.27992114424705505,
144
- "rewards/rejected": -0.40497440099716187,
145
  "step": 80
146
  },
147
  {
148
  "epoch": 0.2057142857142857,
149
- "grad_norm": 17.008248515885608,
150
  "learning_rate": 4.832875107981763e-07,
151
- "logits/chosen": -2.3575804233551025,
152
- "logits/rejected": -2.253711462020874,
153
- "logps/chosen": -308.32257080078125,
154
- "logps/rejected": -306.1024169921875,
155
- "loss": 0.6101,
156
- "rewards/accuracies": 0.699999988079071,
157
- "rewards/chosen": -0.32767707109451294,
158
- "rewards/margins": 0.27739250659942627,
159
- "rewards/rejected": -0.6050695180892944,
160
  "step": 90
161
  },
162
  {
163
  "epoch": 0.22857142857142856,
164
- "grad_norm": 17.46746330094676,
165
  "learning_rate": 4.753659419387223e-07,
166
- "logits/chosen": -1.1981137990951538,
167
- "logits/rejected": -1.0265915393829346,
168
- "logps/chosen": -315.5753173828125,
169
- "logps/rejected": -326.49658203125,
170
- "loss": 0.5829,
171
- "rewards/accuracies": 0.65625,
172
- "rewards/chosen": -0.6649550795555115,
173
- "rewards/margins": 0.34251198172569275,
174
- "rewards/rejected": -1.0074671506881714,
175
  "step": 100
176
  },
177
  {
178
  "epoch": 0.22857142857142856,
179
- "eval_logits/chosen": -1.281318187713623,
180
- "eval_logits/rejected": -0.8703082799911499,
181
- "eval_logps/chosen": -320.673095703125,
182
- "eval_logps/rejected": -324.8917236328125,
183
- "eval_loss": 0.5717055797576904,
184
- "eval_rewards/accuracies": 0.75,
185
- "eval_rewards/chosen": -0.4506916403770447,
186
- "eval_rewards/margins": 0.6074643731117249,
187
- "eval_rewards/rejected": -1.0581560134887695,
188
- "eval_runtime": 90.9709,
189
- "eval_samples_per_second": 20.127,
190
- "eval_steps_per_second": 0.319,
191
  "step": 100
192
  },
193
  {
194
  "epoch": 0.25142857142857145,
195
- "grad_norm": 19.89210167952385,
196
  "learning_rate": 4.660050057270191e-07,
197
- "logits/chosen": -1.352695345878601,
198
- "logits/rejected": -1.1257222890853882,
199
- "logps/chosen": -337.44989013671875,
200
- "logps/rejected": -390.30242919921875,
201
- "loss": 0.562,
202
- "rewards/accuracies": 0.6937500238418579,
203
- "rewards/chosen": -0.5830885767936707,
204
- "rewards/margins": 0.4551981985569,
205
- "rewards/rejected": -1.0382866859436035,
206
  "step": 110
207
  },
208
  {
209
  "epoch": 0.2742857142857143,
210
- "grad_norm": 14.443049697702785,
211
  "learning_rate": 4.5526448859687144e-07,
212
- "logits/chosen": -1.0166410207748413,
213
- "logits/rejected": -0.8261572122573853,
214
- "logps/chosen": -349.50250244140625,
215
- "logps/rejected": -375.82318115234375,
216
- "loss": 0.5911,
217
- "rewards/accuracies": 0.675000011920929,
218
- "rewards/chosen": -0.8073323965072632,
219
- "rewards/margins": 0.39745792746543884,
220
- "rewards/rejected": -1.2047903537750244,
221
  "step": 120
222
  },
223
  {
224
  "epoch": 0.29714285714285715,
225
- "grad_norm": 19.783756043123564,
226
  "learning_rate": 4.432129880904388e-07,
227
- "logits/chosen": -1.3571927547454834,
228
- "logits/rejected": -0.9383346438407898,
229
- "logps/chosen": -344.55084228515625,
230
- "logps/rejected": -390.46051025390625,
231
- "loss": 0.55,
232
- "rewards/accuracies": 0.78125,
233
- "rewards/chosen": -0.592494785785675,
234
- "rewards/margins": 0.5841384530067444,
235
- "rewards/rejected": -1.1766332387924194,
236
  "step": 130
237
  },
238
  {
239
  "epoch": 0.32,
240
- "grad_norm": 26.124919398425305,
241
  "learning_rate": 4.299274747394055e-07,
242
- "logits/chosen": -0.37819042801856995,
243
- "logits/rejected": 0.003760010004043579,
244
- "logps/chosen": -378.787353515625,
245
- "logps/rejected": -400.87371826171875,
246
- "loss": 0.5655,
247
- "rewards/accuracies": 0.675000011920929,
248
- "rewards/chosen": -0.8012701869010925,
249
- "rewards/margins": 0.4693904519081116,
250
- "rewards/rejected": -1.270660638809204,
251
  "step": 140
252
  },
253
  {
254
  "epoch": 0.34285714285714286,
255
- "grad_norm": 19.20697683094284,
256
  "learning_rate": 4.1549280046953653e-07,
257
- "logits/chosen": -1.5754578113555908,
258
- "logits/rejected": -0.9163841009140015,
259
- "logps/chosen": -307.23358154296875,
260
- "logps/rejected": -347.45172119140625,
261
- "loss": 0.5449,
262
- "rewards/accuracies": 0.7437499761581421,
263
- "rewards/chosen": -0.5083099603652954,
264
- "rewards/margins": 0.6555605530738831,
265
- "rewards/rejected": -1.1638705730438232,
266
  "step": 150
267
  },
268
  {
269
  "epoch": 0.34285714285714286,
270
- "eval_logits/chosen": -1.4585620164871216,
271
- "eval_logits/rejected": -0.599774956703186,
272
- "eval_logps/chosen": -323.4755859375,
273
- "eval_logps/rejected": -339.95648193359375,
274
- "eval_loss": 0.5480508804321289,
275
- "eval_rewards/accuracies": 0.7586206793785095,
276
- "eval_rewards/chosen": -0.47871649265289307,
277
- "eval_rewards/margins": 0.7300868630409241,
278
- "eval_rewards/rejected": -1.2088035345077515,
279
- "eval_runtime": 91.5755,
280
- "eval_samples_per_second": 19.994,
281
- "eval_steps_per_second": 0.317,
282
  "step": 150
283
  },
284
  {
285
  "epoch": 0.3657142857142857,
286
- "grad_norm": 23.89400472742195,
287
  "learning_rate": 4.000011566683401e-07,
288
- "logits/chosen": -1.0969589948654175,
289
- "logits/rejected": -0.4010998606681824,
290
- "logps/chosen": -336.60321044921875,
291
- "logps/rejected": -360.08514404296875,
292
- "loss": 0.5504,
293
- "rewards/accuracies": 0.6875,
294
- "rewards/chosen": -0.7269619107246399,
295
- "rewards/margins": 0.5191264152526855,
296
- "rewards/rejected": -1.2460882663726807,
297
  "step": 160
298
  },
299
  {
300
  "epoch": 0.38857142857142857,
301
- "grad_norm": 25.854828716677858,
302
  "learning_rate": 3.8355148537705047e-07,
303
- "logits/chosen": -0.5157957673072815,
304
- "logits/rejected": 0.3562891483306885,
305
- "logps/chosen": -358.5205993652344,
306
- "logps/rejected": -408.76837158203125,
307
- "loss": 0.5316,
308
- "rewards/accuracies": 0.731249988079071,
309
- "rewards/chosen": -0.7713872194290161,
310
- "rewards/margins": 0.7046753764152527,
311
- "rewards/rejected": -1.476062536239624,
312
  "step": 170
313
  },
314
  {
315
  "epoch": 0.4114285714285714,
316
- "grad_norm": 24.398761455182004,
317
  "learning_rate": 3.662488473675315e-07,
318
- "logits/chosen": -0.9975932240486145,
319
- "logits/rejected": -0.4696226716041565,
320
- "logps/chosen": -342.71319580078125,
321
- "logps/rejected": -386.1841735839844,
322
- "loss": 0.5437,
323
- "rewards/accuracies": 0.6812499761581421,
324
- "rewards/chosen": -0.7768923044204712,
325
- "rewards/margins": 0.5349052548408508,
326
- "rewards/rejected": -1.3117974996566772,
327
  "step": 180
328
  },
329
  {
330
  "epoch": 0.4342857142857143,
331
- "grad_norm": 24.140520074610162,
332
  "learning_rate": 3.48203751140067e-07,
333
- "logits/chosen": -0.8493292927742004,
334
- "logits/rejected": -0.2633267939090729,
335
- "logps/chosen": -340.5964050292969,
336
- "logps/rejected": -400.925048828125,
337
- "loss": 0.5274,
338
- "rewards/accuracies": 0.6875,
339
- "rewards/chosen": -0.7818416357040405,
340
- "rewards/margins": 0.6584270596504211,
341
- "rewards/rejected": -1.4402687549591064,
342
  "step": 190
343
  },
344
  {
345
  "epoch": 0.45714285714285713,
346
- "grad_norm": 22.1583738183432,
347
  "learning_rate": 3.2953144712759537e-07,
348
- "logits/chosen": -0.6653417348861694,
349
- "logits/rejected": 0.03831692412495613,
350
- "logps/chosen": -358.6188049316406,
351
- "logps/rejected": -412.5543518066406,
352
- "loss": 0.5265,
353
- "rewards/accuracies": 0.706250011920929,
354
- "rewards/chosen": -0.9225257039070129,
355
- "rewards/margins": 0.6431353688240051,
356
- "rewards/rejected": -1.565661072731018,
357
  "step": 200
358
  },
359
  {
360
  "epoch": 0.45714285714285713,
361
- "eval_logits/chosen": -1.4393001794815063,
362
- "eval_logits/rejected": -0.33169230818748474,
363
- "eval_logps/chosen": -333.7511901855469,
364
- "eval_logps/rejected": -367.50994873046875,
365
- "eval_loss": 0.5407785773277283,
366
- "eval_rewards/accuracies": 0.7629310488700867,
367
- "eval_rewards/chosen": -0.5814722776412964,
368
- "eval_rewards/margins": 0.902865469455719,
369
- "eval_rewards/rejected": -1.4843376874923706,
370
- "eval_runtime": 90.4868,
371
- "eval_samples_per_second": 20.235,
372
- "eval_steps_per_second": 0.32,
373
  "step": 200
374
  },
375
  {
376
  "epoch": 0.48,
377
- "grad_norm": 23.568312031862824,
378
  "learning_rate": 3.103511916141658e-07,
379
- "logits/chosen": -1.483643651008606,
380
- "logits/rejected": -0.6250999569892883,
381
- "logps/chosen": -355.1040954589844,
382
- "logps/rejected": -402.67877197265625,
383
- "loss": 0.5204,
384
- "rewards/accuracies": 0.71875,
385
- "rewards/chosen": -0.7539708614349365,
386
- "rewards/margins": 0.7451109886169434,
387
- "rewards/rejected": -1.4990818500518799,
388
  "step": 210
389
  },
390
  {
391
  "epoch": 0.5028571428571429,
392
- "grad_norm": 18.51942784655271,
393
  "learning_rate": 2.9078548506882117e-07,
394
- "logits/chosen": -1.191624402999878,
395
- "logits/rejected": -0.27054792642593384,
396
- "logps/chosen": -393.39544677734375,
397
- "logps/rejected": -429.14691162109375,
398
- "loss": 0.5337,
399
- "rewards/accuracies": 0.7437499761581421,
400
- "rewards/chosen": -0.9885727763175964,
401
- "rewards/margins": 0.7009212374687195,
402
- "rewards/rejected": -1.6894941329956055,
403
  "step": 220
404
  },
405
  {
406
  "epoch": 0.5257142857142857,
407
- "grad_norm": 21.886955438328442,
408
  "learning_rate": 2.709592897595191e-07,
409
- "logits/chosen": -1.5971931219100952,
410
- "logits/rejected": -1.0103317499160767,
411
- "logps/chosen": -361.65167236328125,
412
- "logps/rejected": -410.632568359375,
413
- "loss": 0.5249,
414
- "rewards/accuracies": 0.762499988079071,
415
- "rewards/chosen": -0.76312655210495,
416
- "rewards/margins": 0.6484739780426025,
417
- "rewards/rejected": -1.4116004705429077,
418
  "step": 230
419
  },
420
  {
421
  "epoch": 0.5485714285714286,
422
- "grad_norm": 26.481170298366607,
423
  "learning_rate": 2.509992316440332e-07,
424
- "logits/chosen": -1.4730970859527588,
425
- "logits/rejected": -0.5076150894165039,
426
- "logps/chosen": -383.16644287109375,
427
- "logps/rejected": -402.82269287109375,
428
- "loss": 0.5023,
429
- "rewards/accuracies": 0.78125,
430
- "rewards/chosen": -0.7350863218307495,
431
- "rewards/margins": 0.7647216320037842,
432
- "rewards/rejected": -1.4998080730438232,
433
  "step": 240
434
  },
435
  {
436
  "epoch": 0.5714285714285714,
437
- "grad_norm": 25.028338430854234,
438
  "learning_rate": 2.3103279163519918e-07,
439
- "logits/chosen": -0.8250116109848022,
440
- "logits/rejected": 0.11155863106250763,
441
- "logps/chosen": -420.82861328125,
442
- "logps/rejected": -422.923095703125,
443
- "loss": 0.5634,
444
- "rewards/accuracies": 0.6812499761581421,
445
- "rewards/chosen": -1.3357810974121094,
446
- "rewards/margins": 0.5469622015953064,
447
- "rewards/rejected": -1.8827431201934814,
448
  "step": 250
449
  },
450
  {
451
  "epoch": 0.5714285714285714,
452
- "eval_logits/chosen": -0.9574553370475769,
453
- "eval_logits/rejected": 0.42744484543800354,
454
- "eval_logps/chosen": -383.36712646484375,
455
- "eval_logps/rejected": -418.92645263671875,
456
- "eval_loss": 0.53549724817276,
457
- "eval_rewards/accuracies": 0.767241358757019,
458
- "eval_rewards/chosen": -1.077631950378418,
459
- "eval_rewards/margins": 0.9208711385726929,
460
- "eval_rewards/rejected": -1.9985028505325317,
461
- "eval_runtime": 91.7472,
462
- "eval_samples_per_second": 19.957,
463
- "eval_steps_per_second": 0.316,
464
  "step": 250
465
  },
466
  {
467
  "epoch": 0.5942857142857143,
468
- "grad_norm": 27.976258157727464,
469
  "learning_rate": 2.1118749140573358e-07,
470
- "logits/chosen": -1.1772454977035522,
471
- "logits/rejected": -0.11464130878448486,
472
- "logps/chosen": -414.1883239746094,
473
- "logps/rejected": -459.7842712402344,
474
- "loss": 0.5177,
475
- "rewards/accuracies": 0.793749988079071,
476
- "rewards/chosen": -0.9650837182998657,
477
- "rewards/margins": 0.8455003499984741,
478
- "rewards/rejected": -1.8105840682983398,
479
  "step": 260
480
  },
481
  {
482
  "epoch": 0.6171428571428571,
483
- "grad_norm": 21.800567423584056,
484
  "learning_rate": 1.9159007893272703e-07,
485
- "logits/chosen": -0.9937992095947266,
486
- "logits/rejected": -0.10799358040094376,
487
- "logps/chosen": -359.657470703125,
488
- "logps/rejected": -412.29693603515625,
489
- "loss": 0.5307,
490
- "rewards/accuracies": 0.675000011920929,
491
- "rewards/chosen": -0.9359390139579773,
492
- "rewards/margins": 0.6931002140045166,
493
- "rewards/rejected": -1.6290391683578491,
494
  "step": 270
495
  },
496
  {
497
  "epoch": 0.64,
498
- "grad_norm": 24.431728608368438,
499
  "learning_rate": 1.7236571898357766e-07,
500
- "logits/chosen": -0.9994579553604126,
501
- "logits/rejected": 0.06377071142196655,
502
- "logps/chosen": -370.099365234375,
503
- "logps/rejected": -412.1414489746094,
504
- "loss": 0.516,
505
- "rewards/accuracies": 0.762499988079071,
506
- "rewards/chosen": -0.8733994364738464,
507
- "rewards/margins": 0.8108373880386353,
508
- "rewards/rejected": -1.6842365264892578,
509
  "step": 280
510
  },
511
  {
512
  "epoch": 0.6628571428571428,
513
- "grad_norm": 21.614177487404678,
514
  "learning_rate": 1.5363719371356882e-07,
515
- "logits/chosen": -0.8323847055435181,
516
- "logits/rejected": -0.23326265811920166,
517
- "logps/chosen": -359.58868408203125,
518
- "logps/rejected": -413.9085998535156,
519
- "loss": 0.534,
520
- "rewards/accuracies": 0.7124999761581421,
521
- "rewards/chosen": -0.9346914291381836,
522
- "rewards/margins": 0.6113588213920593,
523
- "rewards/rejected": -1.5460503101348877,
524
  "step": 290
525
  },
526
  {
527
  "epoch": 0.6857142857142857,
528
- "grad_norm": 25.95153399510747,
529
  "learning_rate": 1.3552411848071565e-07,
530
- "logits/chosen": -1.046464204788208,
531
- "logits/rejected": 0.03388619422912598,
532
- "logps/chosen": -337.25665283203125,
533
- "logps/rejected": -431.7433166503906,
534
- "loss": 0.521,
535
- "rewards/accuracies": 0.75,
536
- "rewards/chosen": -0.8008958101272583,
537
- "rewards/margins": 0.9349392652511597,
538
- "rewards/rejected": -1.735835075378418,
539
  "step": 300
540
  },
541
  {
542
  "epoch": 0.6857142857142857,
543
- "eval_logits/chosen": -1.0317851305007935,
544
- "eval_logits/rejected": 0.4320092499256134,
545
- "eval_logps/chosen": -358.62835693359375,
546
- "eval_logps/rejected": -403.2919921875,
547
- "eval_loss": 0.524986743927002,
548
- "eval_rewards/accuracies": 0.767241358757019,
549
- "eval_rewards/chosen": -0.8302440643310547,
550
- "eval_rewards/margins": 1.0119147300720215,
551
- "eval_rewards/rejected": -1.842158555984497,
552
- "eval_runtime": 92.5899,
553
- "eval_samples_per_second": 19.775,
554
- "eval_steps_per_second": 0.313,
555
  "step": 300
556
  },
557
  {
558
  "epoch": 0.7085714285714285,
559
- "grad_norm": 23.25880052276584,
560
  "learning_rate": 1.1814217788631473e-07,
561
- "logits/chosen": -0.9328817129135132,
562
- "logits/rejected": -0.3298904299736023,
563
- "logps/chosen": -365.3583068847656,
564
- "logps/rejected": -441.9790954589844,
565
- "loss": 0.5312,
566
  "rewards/accuracies": 0.675000011920929,
567
- "rewards/chosen": -0.9027471542358398,
568
- "rewards/margins": 0.7032585740089417,
569
- "rewards/rejected": -1.6060056686401367,
570
  "step": 310
571
  },
572
  {
573
  "epoch": 0.7314285714285714,
574
- "grad_norm": 24.482378169212087,
575
  "learning_rate": 1.0160238692045331e-07,
576
- "logits/chosen": -1.1010204553604126,
577
- "logits/rejected": -0.25174209475517273,
578
- "logps/chosen": -366.65863037109375,
579
- "logps/rejected": -422.2201232910156,
580
- "loss": 0.5348,
581
- "rewards/accuracies": 0.7250000238418579,
582
- "rewards/chosen": -0.9146193265914917,
583
- "rewards/margins": 0.7960137724876404,
584
- "rewards/rejected": -1.7106330394744873,
585
  "step": 320
586
  },
587
  {
588
  "epoch": 0.7542857142857143,
589
- "grad_norm": 22.578433848449897,
590
  "learning_rate": 8.601038193139438e-08,
591
- "logits/chosen": -1.17572820186615,
592
- "logits/rejected": -0.389732301235199,
593
- "logps/chosen": -381.45098876953125,
594
- "logps/rejected": -406.979248046875,
595
- "loss": 0.5228,
596
- "rewards/accuracies": 0.731249988079071,
597
- "rewards/chosen": -0.999869167804718,
598
- "rewards/margins": 0.6249268651008606,
599
- "rewards/rejected": -1.6247961521148682,
600
  "step": 330
601
  },
602
  {
603
  "epoch": 0.7771428571428571,
604
- "grad_norm": 25.423274681043992,
605
  "learning_rate": 7.146574594727572e-08,
606
- "logits/chosen": -1.3077609539031982,
607
- "logits/rejected": -0.20081695914268494,
608
- "logps/chosen": -363.9648742675781,
609
- "logps/rejected": -409.0540466308594,
610
- "loss": 0.5155,
611
- "rewards/accuracies": 0.7250000238418579,
612
- "rewards/chosen": -0.8527425527572632,
613
- "rewards/margins": 0.8432666659355164,
614
- "rewards/rejected": -1.6960090398788452,
615
  "step": 340
616
  },
617
  {
618
  "epoch": 0.8,
619
- "grad_norm": 23.93668405920085,
620
  "learning_rate": 5.8061372659157306e-08,
621
- "logits/chosen": -0.9727778434753418,
622
- "logits/rejected": 0.11374668776988983,
623
- "logps/chosen": -378.41851806640625,
624
- "logps/rejected": -403.8486328125,
625
- "loss": 0.5252,
626
- "rewards/accuracies": 0.675000011920929,
627
- "rewards/chosen": -0.9569555521011353,
628
- "rewards/margins": 0.6800653338432312,
629
- "rewards/rejected": -1.6370208263397217,
630
  "step": 350
631
  },
632
  {
633
  "epoch": 0.8,
634
- "eval_logits/chosen": -1.0407053232192993,
635
- "eval_logits/rejected": 0.37925437092781067,
636
- "eval_logps/chosen": -366.3647766113281,
637
- "eval_logps/rejected": -406.97601318359375,
638
- "eval_loss": 0.5231528878211975,
639
- "eval_rewards/accuracies": 0.7456896305084229,
640
- "eval_rewards/chosen": -0.9076080322265625,
641
- "eval_rewards/margins": 0.9713909029960632,
642
- "eval_rewards/rejected": -1.87899911403656,
643
- "eval_runtime": 90.6244,
644
- "eval_samples_per_second": 20.204,
645
- "eval_steps_per_second": 0.32,
646
  "step": 350
647
  },
648
  {
649
  "epoch": 0.8228571428571428,
650
- "grad_norm": 26.827956646239386,
651
  "learning_rate": 4.5882873127531614e-08,
652
- "logits/chosen": -0.8992304801940918,
653
- "logits/rejected": -0.09379217773675919,
654
- "logps/chosen": -403.6158142089844,
655
- "logps/rejected": -442.5780334472656,
656
- "loss": 0.5084,
657
- "rewards/accuracies": 0.6937500238418579,
658
- "rewards/chosen": -0.9882488250732422,
659
- "rewards/margins": 0.6592365503311157,
660
- "rewards/rejected": -1.647485375404358,
661
  "step": 360
662
  },
663
  {
664
  "epoch": 0.8457142857142858,
665
- "grad_norm": 29.24997495992518,
666
  "learning_rate": 3.500802900154412e-08,
667
- "logits/chosen": -0.8916434049606323,
668
- "logits/rejected": 0.1490824669599533,
669
- "logps/chosen": -329.18121337890625,
670
- "logps/rejected": -396.29486083984375,
671
- "loss": 0.5272,
672
- "rewards/accuracies": 0.7124999761581421,
673
- "rewards/chosen": -0.8961926698684692,
674
- "rewards/margins": 0.7349058985710144,
675
- "rewards/rejected": -1.6310985088348389,
676
  "step": 370
677
  },
678
  {
679
  "epoch": 0.8685714285714285,
680
- "grad_norm": 26.372757513524967,
681
  "learning_rate": 2.550629574310309e-08,
682
- "logits/chosen": -0.9773521423339844,
683
- "logits/rejected": -0.07595231384038925,
684
- "logps/chosen": -364.446533203125,
685
- "logps/rejected": -418.685302734375,
686
- "loss": 0.5242,
687
- "rewards/accuracies": 0.699999988079071,
688
- "rewards/chosen": -0.9272258877754211,
689
- "rewards/margins": 0.684902548789978,
690
- "rewards/rejected": -1.612128496170044,
691
  "step": 380
692
  },
693
  {
694
  "epoch": 0.8914285714285715,
695
- "grad_norm": 24.006363343018684,
696
  "learning_rate": 1.7438359028687983e-08,
697
- "logits/chosen": -0.9853193163871765,
698
- "logits/rejected": -0.48293977975845337,
699
- "logps/chosen": -355.98504638671875,
700
- "logps/rejected": -426.0884704589844,
701
- "loss": 0.5198,
702
- "rewards/accuracies": 0.668749988079071,
703
- "rewards/chosen": -0.9684079885482788,
704
- "rewards/margins": 0.5912572741508484,
705
- "rewards/rejected": -1.5596654415130615,
706
  "step": 390
707
  },
708
  {
709
  "epoch": 0.9142857142857143,
710
- "grad_norm": 26.577783975986762,
711
  "learning_rate": 1.0855747162029361e-08,
712
- "logits/chosen": -1.1757861375808716,
713
- "logits/rejected": -0.06207038834691048,
714
- "logps/chosen": -375.5000305175781,
715
- "logps/rejected": -405.329833984375,
716
- "loss": 0.5357,
717
- "rewards/accuracies": 0.7124999761581421,
718
- "rewards/chosen": -0.8609127998352051,
719
- "rewards/margins": 0.7547314763069153,
720
- "rewards/rejected": -1.6156442165374756,
721
  "step": 400
722
  },
723
  {
724
  "epoch": 0.9142857142857143,
725
- "eval_logits/chosen": -1.235926866531372,
726
- "eval_logits/rejected": 0.06562583893537521,
727
- "eval_logps/chosen": -357.6330261230469,
728
- "eval_logps/rejected": -395.7077941894531,
729
- "eval_loss": 0.5215969681739807,
730
- "eval_rewards/accuracies": 0.75,
731
- "eval_rewards/chosen": -0.8202905058860779,
732
- "eval_rewards/margins": 0.9460263848304749,
733
- "eval_rewards/rejected": -1.7663167715072632,
734
- "eval_runtime": 90.7178,
735
- "eval_samples_per_second": 20.183,
736
- "eval_steps_per_second": 0.32,
737
  "step": 400
738
  },
739
  {
740
  "epoch": 0.9371428571428572,
741
- "grad_norm": 25.1148424255179,
742
  "learning_rate": 5.8005019731033615e-09,
743
- "logits/chosen": -1.2540085315704346,
744
- "logits/rejected": -0.18423935770988464,
745
- "logps/chosen": -349.39703369140625,
746
- "logps/rejected": -408.9049987792969,
747
- "loss": 0.5166,
748
- "rewards/accuracies": 0.800000011920929,
749
- "rewards/chosen": -0.7661619782447815,
750
- "rewards/margins": 0.9154095649719238,
751
- "rewards/rejected": -1.681571364402771,
752
  "step": 410
753
  },
754
  {
755
  "epoch": 0.96,
756
- "grad_norm": 22.71468084905044,
757
  "learning_rate": 2.3049103053431886e-09,
758
- "logits/chosen": -1.1918666362762451,
759
- "logits/rejected": -0.38211789727211,
760
- "logps/chosen": -398.10797119140625,
761
- "logps/rejected": -393.56427001953125,
762
- "loss": 0.5365,
763
- "rewards/accuracies": 0.6499999761581421,
764
- "rewards/chosen": -0.9075633883476257,
765
- "rewards/margins": 0.6042085289955139,
766
- "rewards/rejected": -1.5117719173431396,
767
  "step": 420
768
  },
769
  {
770
  "epoch": 0.9828571428571429,
771
- "grad_norm": 23.51283465587864,
772
  "learning_rate": 3.9129780600541397e-10,
773
- "logits/chosen": -1.0148117542266846,
774
- "logits/rejected": -0.13587214052677155,
775
- "logps/chosen": -376.6526184082031,
776
- "logps/rejected": -404.432373046875,
777
- "loss": 0.5327,
778
- "rewards/accuracies": 0.731249988079071,
779
- "rewards/chosen": -0.9118492007255554,
780
- "rewards/margins": 0.7239080667495728,
781
- "rewards/rejected": -1.6357572078704834,
782
  "step": 430
783
  },
784
  {
785
  "epoch": 0.9988571428571429,
786
  "step": 437,
787
  "total_flos": 0.0,
788
- "train_loss": 0.5593291070968662,
789
- "train_runtime": 11268.1264,
790
- "train_samples_per_second": 4.97,
791
  "train_steps_per_second": 0.039
792
  }
793
  ],
 
10
  "log_history": [
11
  {
12
  "epoch": 0.022857142857142857,
13
+ "grad_norm": 8.11412987933583,
14
  "learning_rate": 1.1363636363636363e-07,
15
+ "logits/chosen": -2.66959810256958,
16
+ "logits/rejected": -2.6077542304992676,
17
+ "logps/chosen": -296.6876220703125,
18
+ "logps/rejected": -254.7753448486328,
19
+ "loss": 0.6933,
20
+ "rewards/accuracies": 0.3812499940395355,
21
+ "rewards/chosen": -0.0003348872414790094,
22
+ "rewards/margins": -0.00012643556692637503,
23
+ "rewards/rejected": -0.0002084516454488039,
24
  "step": 10
25
  },
26
  {
27
  "epoch": 0.045714285714285714,
28
+ "grad_norm": 9.934680850734814,
29
  "learning_rate": 2.2727272727272726e-07,
30
+ "logits/chosen": -2.6891586780548096,
31
+ "logits/rejected": -2.6237130165100098,
32
+ "logps/chosen": -294.1405334472656,
33
+ "logps/rejected": -254.05810546875,
34
  "loss": 0.6922,
35
+ "rewards/accuracies": 0.65625,
36
+ "rewards/chosen": 0.0007635392248630524,
37
+ "rewards/margins": 0.0031762172002345324,
38
+ "rewards/rejected": -0.0024126782082021236,
39
  "step": 20
40
  },
41
  {
42
  "epoch": 0.06857142857142857,
43
+ "grad_norm": 8.46839233994518,
44
  "learning_rate": 3.4090909090909085e-07,
45
+ "logits/chosen": -2.657853364944458,
46
+ "logits/rejected": -2.562720775604248,
47
+ "logps/chosen": -285.6708984375,
48
+ "logps/rejected": -247.06838989257812,
49
+ "loss": 0.6879,
50
+ "rewards/accuracies": 0.6625000238418579,
51
+ "rewards/chosen": 0.012036588974297047,
52
+ "rewards/margins": 0.01440697442740202,
53
+ "rewards/rejected": -0.002370386151596904,
54
  "step": 30
55
  },
56
  {
57
  "epoch": 0.09142857142857143,
58
+ "grad_norm": 10.38860192358711,
59
  "learning_rate": 4.545454545454545e-07,
60
+ "logits/chosen": -2.6249475479125977,
61
+ "logits/rejected": -2.5956408977508545,
62
+ "logps/chosen": -269.8529357910156,
63
+ "logps/rejected": -273.1573791503906,
64
+ "loss": 0.6793,
65
+ "rewards/accuracies": 0.65625,
66
+ "rewards/chosen": 0.02421986497938633,
67
+ "rewards/margins": 0.020207645371556282,
68
+ "rewards/rejected": 0.004012218676507473,
69
  "step": 40
70
  },
71
  {
72
  "epoch": 0.11428571428571428,
73
+ "grad_norm": 8.812178369158405,
74
  "learning_rate": 4.997124959943201e-07,
75
+ "logits/chosen": -2.6135976314544678,
76
+ "logits/rejected": -2.5756285190582275,
77
+ "logps/chosen": -305.51312255859375,
78
+ "logps/rejected": -271.22247314453125,
79
+ "loss": 0.6619,
80
  "rewards/accuracies": 0.7124999761581421,
81
+ "rewards/chosen": -0.010274273343384266,
82
+ "rewards/margins": 0.067402184009552,
83
+ "rewards/rejected": -0.0776764526963234,
84
  "step": 50
85
  },
86
  {
87
  "epoch": 0.11428571428571428,
88
+ "eval_logits/chosen": -2.569648265838623,
89
+ "eval_logits/rejected": -2.4695067405700684,
90
+ "eval_logps/chosen": -277.47930908203125,
91
+ "eval_logps/rejected": -235.5894012451172,
92
+ "eval_loss": 0.6465452313423157,
93
  "eval_rewards/accuracies": 0.693965494632721,
94
+ "eval_rewards/chosen": -0.018753662705421448,
95
+ "eval_rewards/margins": 0.1463788002729416,
96
+ "eval_rewards/rejected": -0.16513246297836304,
97
+ "eval_runtime": 90.334,
98
+ "eval_samples_per_second": 20.269,
99
+ "eval_steps_per_second": 0.321,
100
  "step": 50
101
  },
102
  {
103
  "epoch": 0.13714285714285715,
104
+ "grad_norm": 11.6151396248973,
105
  "learning_rate": 4.979579212164186e-07,
106
+ "logits/chosen": -2.6002144813537598,
107
+ "logits/rejected": -2.501227855682373,
108
+ "logps/chosen": -298.59063720703125,
109
+ "logps/rejected": -302.35577392578125,
110
+ "loss": 0.6307,
111
  "rewards/accuracies": 0.71875,
112
+ "rewards/chosen": -0.047010406851768494,
113
+ "rewards/margins": 0.16466036438941956,
114
+ "rewards/rejected": -0.21167078614234924,
115
  "step": 60
116
  },
117
  {
118
  "epoch": 0.16,
119
+ "grad_norm": 14.811620243521006,
120
  "learning_rate": 4.946196886175515e-07,
121
+ "logits/chosen": -2.6356024742126465,
122
+ "logits/rejected": -2.5619795322418213,
123
+ "logps/chosen": -280.7294921875,
124
+ "logps/rejected": -274.32647705078125,
125
+ "loss": 0.6194,
126
  "rewards/accuracies": 0.6625000238418579,
127
+ "rewards/chosen": -0.16827444732189178,
128
+ "rewards/margins": 0.20077195763587952,
129
+ "rewards/rejected": -0.3690463900566101,
130
  "step": 70
131
  },
132
  {
133
  "epoch": 0.18285714285714286,
134
+ "grad_norm": 16.993875634534675,
135
  "learning_rate": 4.897191188239667e-07,
136
+ "logits/chosen": -2.642766237258911,
137
+ "logits/rejected": -2.582462787628174,
138
+ "logps/chosen": -321.19744873046875,
139
+ "logps/rejected": -298.94171142578125,
140
+ "loss": 0.6087,
141
+ "rewards/accuracies": 0.7437499761581421,
142
+ "rewards/chosen": -0.0975220575928688,
143
+ "rewards/margins": 0.27095723152160645,
144
+ "rewards/rejected": -0.36847931146621704,
145
  "step": 80
146
  },
147
  {
148
  "epoch": 0.2057142857142857,
149
+ "grad_norm": 17.671501985453162,
150
  "learning_rate": 4.832875107981763e-07,
151
+ "logits/chosen": -2.4848644733428955,
152
+ "logits/rejected": -2.404571533203125,
153
+ "logps/chosen": -310.0569152832031,
154
+ "logps/rejected": -313.3294982910156,
155
+ "loss": 0.6012,
156
+ "rewards/accuracies": 0.7124999761581421,
157
+ "rewards/chosen": -0.345020592212677,
158
+ "rewards/margins": 0.33231958746910095,
159
+ "rewards/rejected": -0.6773402690887451,
160
  "step": 90
161
  },
162
  {
163
  "epoch": 0.22857142857142856,
164
+ "grad_norm": 17.978608029535753,
165
  "learning_rate": 4.753659419387223e-07,
166
+ "logits/chosen": -1.396976351737976,
167
+ "logits/rejected": -1.3131816387176514,
168
+ "logps/chosen": -309.3507385253906,
169
+ "logps/rejected": -321.52056884765625,
170
+ "loss": 0.5843,
171
+ "rewards/accuracies": 0.643750011920929,
172
+ "rewards/chosen": -0.6027094125747681,
173
+ "rewards/margins": 0.35499778389930725,
174
+ "rewards/rejected": -0.9577071070671082,
175
  "step": 100
176
  },
177
  {
178
  "epoch": 0.22857142857142856,
179
+ "eval_logits/chosen": -1.3850308656692505,
180
+ "eval_logits/rejected": -1.1914278268814087,
181
+ "eval_logps/chosen": -320.03228759765625,
182
+ "eval_logps/rejected": -321.1168518066406,
183
+ "eval_loss": 0.5719701647758484,
184
+ "eval_rewards/accuracies": 0.7456896305084229,
185
+ "eval_rewards/chosen": -0.44428348541259766,
186
+ "eval_rewards/margins": 0.5761240124702454,
187
+ "eval_rewards/rejected": -1.0204075574874878,
188
+ "eval_runtime": 89.9125,
189
+ "eval_samples_per_second": 20.364,
190
+ "eval_steps_per_second": 0.323,
191
  "step": 100
192
  },
193
  {
194
  "epoch": 0.25142857142857145,
195
+ "grad_norm": 30.473603379477254,
196
  "learning_rate": 4.660050057270191e-07,
197
+ "logits/chosen": -1.5161502361297607,
198
+ "logits/rejected": -1.4007251262664795,
199
+ "logps/chosen": -326.7287292480469,
200
+ "logps/rejected": -381.71234130859375,
201
+ "loss": 0.5578,
202
+ "rewards/accuracies": 0.699999988079071,
203
+ "rewards/chosen": -0.47587689757347107,
204
+ "rewards/margins": 0.4765087962150574,
205
+ "rewards/rejected": -0.9523857235908508,
206
  "step": 110
207
  },
208
  {
209
  "epoch": 0.2742857142857143,
210
+ "grad_norm": 16.07677825536776,
211
  "learning_rate": 4.5526448859687144e-07,
212
+ "logits/chosen": -1.329465627670288,
213
+ "logits/rejected": -1.2029626369476318,
214
+ "logps/chosen": -313.04150390625,
215
+ "logps/rejected": -344.30377197265625,
216
+ "loss": 0.59,
217
+ "rewards/accuracies": 0.6625000238418579,
218
+ "rewards/chosen": -0.4427226483821869,
219
+ "rewards/margins": 0.44687420129776,
220
+ "rewards/rejected": -0.8895969390869141,
221
  "step": 120
222
  },
223
  {
224
  "epoch": 0.29714285714285715,
225
+ "grad_norm": 19.25626895436831,
226
  "learning_rate": 4.432129880904388e-07,
227
+ "logits/chosen": -1.0987221002578735,
228
+ "logits/rejected": -0.7802125215530396,
229
+ "logps/chosen": -344.36712646484375,
230
+ "logps/rejected": -391.39154052734375,
231
+ "loss": 0.5477,
232
+ "rewards/accuracies": 0.7749999761581421,
233
+ "rewards/chosen": -0.5906578302383423,
234
+ "rewards/margins": 0.5952860116958618,
235
+ "rewards/rejected": -1.1859437227249146,
236
  "step": 130
237
  },
238
  {
239
  "epoch": 0.32,
240
+ "grad_norm": 32.057510195911,
241
  "learning_rate": 4.299274747394055e-07,
242
+ "logits/chosen": -0.759337306022644,
243
+ "logits/rejected": -0.5684966444969177,
244
+ "logps/chosen": -352.746826171875,
245
+ "logps/rejected": -371.2802734375,
246
+ "loss": 0.5676,
247
+ "rewards/accuracies": 0.6499999761581421,
248
+ "rewards/chosen": -0.540864884853363,
249
+ "rewards/margins": 0.43386125564575195,
250
+ "rewards/rejected": -0.9747260808944702,
251
  "step": 140
252
  },
253
  {
254
  "epoch": 0.34285714285714286,
255
+ "grad_norm": 31.167990046129457,
256
  "learning_rate": 4.1549280046953653e-07,
257
+ "logits/chosen": -1.4192949533462524,
258
+ "logits/rejected": -0.8910300135612488,
259
+ "logps/chosen": -302.2892150878906,
260
+ "logps/rejected": -339.9471435546875,
261
+ "loss": 0.5509,
262
+ "rewards/accuracies": 0.731249988079071,
263
+ "rewards/chosen": -0.45886653661727905,
264
+ "rewards/margins": 0.62995845079422,
265
+ "rewards/rejected": -1.0888248682022095,
266
  "step": 150
267
  },
268
  {
269
  "epoch": 0.34285714285714286,
270
+ "eval_logits/chosen": -1.355178952217102,
271
+ "eval_logits/rejected": -0.6634992361068726,
272
+ "eval_logps/chosen": -323.4683532714844,
273
+ "eval_logps/rejected": -345.5482482910156,
274
+ "eval_loss": 0.5466835498809814,
275
+ "eval_rewards/accuracies": 0.732758641242981,
276
+ "eval_rewards/chosen": -0.4786438047885895,
277
+ "eval_rewards/margins": 0.7860775589942932,
278
+ "eval_rewards/rejected": -1.264721393585205,
279
+ "eval_runtime": 90.391,
280
+ "eval_samples_per_second": 20.256,
281
+ "eval_steps_per_second": 0.321,
282
  "step": 150
283
  },
284
  {
285
  "epoch": 0.3657142857142857,
286
+ "grad_norm": 24.250974026024966,
287
  "learning_rate": 4.000011566683401e-07,
288
+ "logits/chosen": -1.3669617176055908,
289
+ "logits/rejected": -0.9052613377571106,
290
+ "logps/chosen": -333.5279846191406,
291
+ "logps/rejected": -360.4810791015625,
292
+ "loss": 0.55,
293
+ "rewards/accuracies": 0.7124999761581421,
294
+ "rewards/chosen": -0.6962090730667114,
295
+ "rewards/margins": 0.5538384318351746,
296
+ "rewards/rejected": -1.2500474452972412,
297
  "step": 160
298
  },
299
  {
300
  "epoch": 0.38857142857142857,
301
+ "grad_norm": 24.704591344446357,
302
  "learning_rate": 3.8355148537705047e-07,
303
+ "logits/chosen": -1.3166093826293945,
304
+ "logits/rejected": -0.7781628966331482,
305
+ "logps/chosen": -343.19940185546875,
306
+ "logps/rejected": -390.14239501953125,
307
+ "loss": 0.5266,
308
+ "rewards/accuracies": 0.7124999761581421,
309
+ "rewards/chosen": -0.6181753873825073,
310
+ "rewards/margins": 0.671627402305603,
311
+ "rewards/rejected": -1.2898027896881104,
312
  "step": 170
313
  },
314
  {
315
  "epoch": 0.4114285714285714,
316
+ "grad_norm": 22.05889500026718,
317
  "learning_rate": 3.662488473675315e-07,
318
+ "logits/chosen": -1.254248023033142,
319
+ "logits/rejected": -0.8255653381347656,
320
+ "logps/chosen": -341.5926818847656,
321
+ "logps/rejected": -382.6631774902344,
322
+ "loss": 0.542,
323
+ "rewards/accuracies": 0.6625000238418579,
324
+ "rewards/chosen": -0.7656871676445007,
325
+ "rewards/margins": 0.5109010338783264,
326
+ "rewards/rejected": -1.2765882015228271,
327
  "step": 180
328
  },
329
  {
330
  "epoch": 0.4342857142857143,
331
+ "grad_norm": 23.909926187935003,
332
  "learning_rate": 3.48203751140067e-07,
333
+ "logits/chosen": -1.0761396884918213,
334
+ "logits/rejected": -0.6647660136222839,
335
+ "logps/chosen": -325.56683349609375,
336
+ "logps/rejected": -379.4165344238281,
337
+ "loss": 0.5356,
338
+ "rewards/accuracies": 0.6937500238418579,
339
+ "rewards/chosen": -0.6315457820892334,
340
+ "rewards/margins": 0.5936378240585327,
341
+ "rewards/rejected": -1.2251836061477661,
342
  "step": 190
343
  },
344
  {
345
  "epoch": 0.45714285714285713,
346
+ "grad_norm": 20.73701530382175,
347
  "learning_rate": 3.2953144712759537e-07,
348
+ "logits/chosen": -0.5213090181350708,
349
+ "logits/rejected": 0.013022899627685547,
350
+ "logps/chosen": -359.0880432128906,
351
+ "logps/rejected": -410.6299743652344,
352
+ "loss": 0.5275,
353
+ "rewards/accuracies": 0.7124999761581421,
354
+ "rewards/chosen": -0.927217960357666,
355
+ "rewards/margins": 0.6191995739936829,
356
+ "rewards/rejected": -1.5464175939559937,
357
  "step": 200
358
  },
359
  {
360
  "epoch": 0.45714285714285713,
361
+ "eval_logits/chosen": -1.0376836061477661,
362
+ "eval_logits/rejected": -0.08949049562215805,
363
+ "eval_logps/chosen": -327.8020935058594,
364
+ "eval_logps/rejected": -357.148681640625,
365
+ "eval_loss": 0.5396182537078857,
366
+ "eval_rewards/accuracies": 0.7715517282485962,
367
+ "eval_rewards/chosen": -0.5219810605049133,
368
+ "eval_rewards/margins": 0.8587445020675659,
369
+ "eval_rewards/rejected": -1.380725622177124,
370
+ "eval_runtime": 90.1866,
371
+ "eval_samples_per_second": 20.302,
372
+ "eval_steps_per_second": 0.322,
373
  "step": 200
374
  },
375
  {
376
  "epoch": 0.48,
377
+ "grad_norm": 23.237204035063847,
378
  "learning_rate": 3.103511916141658e-07,
379
+ "logits/chosen": -0.9574594497680664,
380
+ "logits/rejected": -0.16778725385665894,
381
+ "logps/chosen": -348.38385009765625,
382
+ "logps/rejected": -397.10321044921875,
383
+ "loss": 0.5219,
384
+ "rewards/accuracies": 0.7250000238418579,
385
+ "rewards/chosen": -0.686768651008606,
386
+ "rewards/margins": 0.7565571665763855,
387
+ "rewards/rejected": -1.4433258771896362,
388
  "step": 210
389
  },
390
  {
391
  "epoch": 0.5028571428571429,
392
+ "grad_norm": 19.18259747527528,
393
  "learning_rate": 2.9078548506882117e-07,
394
+ "logits/chosen": -0.37646159529685974,
395
+ "logits/rejected": 0.516203761100769,
396
+ "logps/chosen": -383.25811767578125,
397
+ "logps/rejected": -413.0987243652344,
398
+ "loss": 0.5326,
399
+ "rewards/accuracies": 0.7250000238418579,
400
+ "rewards/chosen": -0.8871996998786926,
401
+ "rewards/margins": 0.6418129205703735,
402
+ "rewards/rejected": -1.529012680053711,
403
  "step": 220
404
  },
405
  {
406
  "epoch": 0.5257142857142857,
407
+ "grad_norm": 21.792119076799803,
408
  "learning_rate": 2.709592897595191e-07,
409
+ "logits/chosen": -0.94196617603302,
410
+ "logits/rejected": -0.2367326021194458,
411
+ "logps/chosen": -343.68157958984375,
412
+ "logps/rejected": -391.78729248046875,
413
+ "loss": 0.5313,
414
+ "rewards/accuracies": 0.768750011920929,
415
+ "rewards/chosen": -0.583425760269165,
416
+ "rewards/margins": 0.6397222280502319,
417
+ "rewards/rejected": -1.2231481075286865,
418
  "step": 230
419
  },
420
  {
421
  "epoch": 0.5485714285714286,
422
+ "grad_norm": 23.250665470567046,
423
  "learning_rate": 2.509992316440332e-07,
424
+ "logits/chosen": -0.7590802907943726,
425
+ "logits/rejected": 0.23232534527778625,
426
+ "logps/chosen": -375.7669982910156,
427
+ "logps/rejected": -393.1439208984375,
428
+ "loss": 0.5083,
429
+ "rewards/accuracies": 0.75,
430
+ "rewards/chosen": -0.661091685295105,
431
+ "rewards/margins": 0.7419286370277405,
432
+ "rewards/rejected": -1.4030205011367798,
433
  "step": 240
434
  },
435
  {
436
  "epoch": 0.5714285714285714,
437
+ "grad_norm": 23.283347220975337,
438
  "learning_rate": 2.3103279163519918e-07,
439
+ "logits/chosen": 0.3902924656867981,
440
+ "logits/rejected": 1.2702767848968506,
441
+ "logps/chosen": -430.42510986328125,
442
+ "logps/rejected": -429.90240478515625,
443
+ "loss": 0.5665,
444
+ "rewards/accuracies": 0.668749988079071,
445
+ "rewards/chosen": -1.4317457675933838,
446
+ "rewards/margins": 0.5207899808883667,
447
+ "rewards/rejected": -1.95253586769104,
448
  "step": 250
449
  },
450
  {
451
  "epoch": 0.5714285714285714,
452
+ "eval_logits/chosen": 0.2878158688545227,
453
+ "eval_logits/rejected": 1.5253632068634033,
454
+ "eval_logps/chosen": -384.02301025390625,
455
+ "eval_logps/rejected": -415.1577453613281,
456
+ "eval_loss": 0.5404527187347412,
457
+ "eval_rewards/accuracies": 0.7629310488700867,
458
+ "eval_rewards/chosen": -1.0841907262802124,
459
+ "eval_rewards/margins": 0.8766254782676697,
460
+ "eval_rewards/rejected": -1.9608159065246582,
461
+ "eval_runtime": 91.5614,
462
+ "eval_samples_per_second": 19.998,
463
+ "eval_steps_per_second": 0.317,
464
  "step": 250
465
  },
466
  {
467
  "epoch": 0.5942857142857143,
468
+ "grad_norm": 28.334390897274133,
469
  "learning_rate": 2.1118749140573358e-07,
470
+ "logits/chosen": -0.2290700227022171,
471
+ "logits/rejected": 0.8288987278938293,
472
+ "logps/chosen": -408.00201416015625,
473
+ "logps/rejected": -452.023681640625,
474
+ "loss": 0.5197,
475
+ "rewards/accuracies": 0.7875000238418579,
476
+ "rewards/chosen": -0.9032201766967773,
477
+ "rewards/margins": 0.8297585248947144,
478
+ "rewards/rejected": -1.7329788208007812,
479
  "step": 260
480
  },
481
  {
482
  "epoch": 0.6171428571428571,
483
+ "grad_norm": 23.854460344353054,
484
  "learning_rate": 1.9159007893272703e-07,
485
+ "logits/chosen": -0.17421701550483704,
486
+ "logits/rejected": 0.6511275172233582,
487
+ "logps/chosen": -342.1952209472656,
488
+ "logps/rejected": -396.54083251953125,
489
+ "loss": 0.5294,
490
+ "rewards/accuracies": 0.706250011920929,
491
+ "rewards/chosen": -0.7613161206245422,
492
+ "rewards/margins": 0.7101620435714722,
493
+ "rewards/rejected": -1.4714782238006592,
494
  "step": 270
495
  },
496
  {
497
  "epoch": 0.64,
498
+ "grad_norm": 27.414501100794606,
499
  "learning_rate": 1.7236571898357766e-07,
500
+ "logits/chosen": -0.08564956486225128,
501
+ "logits/rejected": 0.9427372813224792,
502
+ "logps/chosen": -367.49407958984375,
503
+ "logps/rejected": -409.3499755859375,
504
+ "loss": 0.518,
505
+ "rewards/accuracies": 0.7562500238418579,
506
+ "rewards/chosen": -0.8473467826843262,
507
+ "rewards/margins": 0.8089747428894043,
508
+ "rewards/rejected": -1.6563212871551514,
509
  "step": 280
510
  },
511
  {
512
  "epoch": 0.6628571428571428,
513
+ "grad_norm": 23.166387688948994,
514
  "learning_rate": 1.5363719371356882e-07,
515
+ "logits/chosen": 0.06339935958385468,
516
+ "logits/rejected": 0.5719184875488281,
517
+ "logps/chosen": -360.3900451660156,
518
+ "logps/rejected": -409.3319091796875,
519
+ "loss": 0.5387,
520
+ "rewards/accuracies": 0.71875,
521
+ "rewards/chosen": -0.9427051544189453,
522
+ "rewards/margins": 0.5575781464576721,
523
+ "rewards/rejected": -1.500283122062683,
524
  "step": 290
525
  },
526
  {
527
  "epoch": 0.6857142857142857,
528
+ "grad_norm": 28.449941474840944,
529
  "learning_rate": 1.3552411848071565e-07,
530
+ "logits/chosen": -0.3682107627391815,
531
+ "logits/rejected": 0.5333132743835449,
532
+ "logps/chosen": -333.92449951171875,
533
+ "logps/rejected": -425.3599548339844,
534
+ "loss": 0.5202,
535
+ "rewards/accuracies": 0.7749999761581421,
536
+ "rewards/chosen": -0.7675741314888,
537
+ "rewards/margins": 0.9044278264045715,
538
+ "rewards/rejected": -1.6720020771026611,
539
  "step": 300
540
  },
541
  {
542
  "epoch": 0.6857142857142857,
543
+ "eval_logits/chosen": -0.21915225684642792,
544
+ "eval_logits/rejected": 0.9934114813804626,
545
+ "eval_logps/chosen": -349.8433837890625,
546
+ "eval_logps/rejected": -392.549560546875,
547
+ "eval_loss": 0.5274777412414551,
548
+ "eval_rewards/accuracies": 0.7715517282485962,
549
+ "eval_rewards/chosen": -0.7423940300941467,
550
+ "eval_rewards/margins": 0.9923400282859802,
551
+ "eval_rewards/rejected": -1.734734058380127,
552
+ "eval_runtime": 90.6157,
553
+ "eval_samples_per_second": 20.206,
554
+ "eval_steps_per_second": 0.32,
555
  "step": 300
556
  },
557
  {
558
  "epoch": 0.7085714285714285,
559
+ "grad_norm": 21.530490280501045,
560
  "learning_rate": 1.1814217788631473e-07,
561
+ "logits/chosen": -0.17677690088748932,
562
+ "logits/rejected": 0.3225722908973694,
563
+ "logps/chosen": -356.5892639160156,
564
+ "logps/rejected": -431.662109375,
565
+ "loss": 0.5328,
566
  "rewards/accuracies": 0.675000011920929,
567
+ "rewards/chosen": -0.8150566816329956,
568
+ "rewards/margins": 0.6877792477607727,
569
+ "rewards/rejected": -1.502835988998413,
570
  "step": 310
571
  },
572
  {
573
  "epoch": 0.7314285714285714,
574
+ "grad_norm": 26.395907969219987,
575
  "learning_rate": 1.0160238692045331e-07,
576
+ "logits/chosen": -0.2509198486804962,
577
+ "logits/rejected": 0.4886396527290344,
578
+ "logps/chosen": -359.4185485839844,
579
+ "logps/rejected": -413.3671875,
580
+ "loss": 0.535,
581
+ "rewards/accuracies": 0.71875,
582
+ "rewards/chosen": -0.8422183990478516,
583
+ "rewards/margins": 0.7798849940299988,
584
+ "rewards/rejected": -1.6221033334732056,
585
  "step": 320
586
  },
587
  {
588
  "epoch": 0.7542857142857143,
589
+ "grad_norm": 25.409253983308545,
590
  "learning_rate": 8.601038193139438e-08,
591
+ "logits/chosen": -0.13014790415763855,
592
+ "logits/rejected": 0.5298113822937012,
593
+ "logps/chosen": -378.82012939453125,
594
+ "logps/rejected": -402.17156982421875,
595
+ "loss": 0.5302,
596
+ "rewards/accuracies": 0.6937500238418579,
597
+ "rewards/chosen": -0.973560631275177,
598
+ "rewards/margins": 0.6031589508056641,
599
+ "rewards/rejected": -1.5767196416854858,
600
  "step": 330
601
  },
602
  {
603
  "epoch": 0.7771428571428571,
604
+ "grad_norm": 26.91341277336219,
605
  "learning_rate": 7.146574594727572e-08,
606
+ "logits/chosen": -0.4229533076286316,
607
+ "logits/rejected": 0.5619645714759827,
608
+ "logps/chosen": -363.5431213378906,
609
+ "logps/rejected": -404.77935791015625,
610
+ "loss": 0.5192,
611
+ "rewards/accuracies": 0.71875,
612
+ "rewards/chosen": -0.8485255241394043,
613
+ "rewards/margins": 0.8047366142272949,
614
+ "rewards/rejected": -1.6532618999481201,
615
  "step": 340
616
  },
617
  {
618
  "epoch": 0.8,
619
+ "grad_norm": 24.565353732521466,
620
  "learning_rate": 5.8061372659157306e-08,
621
+ "logits/chosen": -0.17489977180957794,
622
+ "logits/rejected": 0.7508963346481323,
623
+ "logps/chosen": -371.7098693847656,
624
+ "logps/rejected": -395.7353210449219,
625
+ "loss": 0.5261,
626
+ "rewards/accuracies": 0.6625000238418579,
627
+ "rewards/chosen": -0.8898698091506958,
628
+ "rewards/margins": 0.6660181879997253,
629
+ "rewards/rejected": -1.5558879375457764,
630
  "step": 350
631
  },
632
  {
633
  "epoch": 0.8,
634
+ "eval_logits/chosen": -0.2696850597858429,
635
+ "eval_logits/rejected": 0.9007923007011414,
636
+ "eval_logps/chosen": -351.73101806640625,
637
+ "eval_logps/rejected": -391.03973388671875,
638
+ "eval_loss": 0.5234382748603821,
639
+ "eval_rewards/accuracies": 0.7586206793785095,
640
+ "eval_rewards/chosen": -0.76127028465271,
641
+ "eval_rewards/margins": 0.9583660364151001,
642
+ "eval_rewards/rejected": -1.71963632106781,
643
+ "eval_runtime": 90.1922,
644
+ "eval_samples_per_second": 20.301,
645
+ "eval_steps_per_second": 0.322,
646
  "step": 350
647
  },
648
  {
649
  "epoch": 0.8228571428571428,
650
+ "grad_norm": 26.528490406026805,
651
  "learning_rate": 4.5882873127531614e-08,
652
+ "logits/chosen": -0.18714679777622223,
653
+ "logits/rejected": 0.5642833113670349,
654
+ "logps/chosen": -389.9281921386719,
655
+ "logps/rejected": -425.6780700683594,
656
+ "loss": 0.5053,
657
+ "rewards/accuracies": 0.71875,
658
+ "rewards/chosen": -0.8513727188110352,
659
+ "rewards/margins": 0.6271126866340637,
660
+ "rewards/rejected": -1.478485345840454,
661
  "step": 360
662
  },
663
  {
664
  "epoch": 0.8457142857142858,
665
+ "grad_norm": 26.160208453826513,
666
  "learning_rate": 3.500802900154412e-08,
667
+ "logits/chosen": -0.11366554349660873,
668
+ "logits/rejected": 0.7036272287368774,
669
+ "logps/chosen": -315.89349365234375,
670
+ "logps/rejected": -380.2809143066406,
671
+ "loss": 0.5279,
672
+ "rewards/accuracies": 0.6937500238418579,
673
+ "rewards/chosen": -0.7633152008056641,
674
+ "rewards/margins": 0.7076437473297119,
675
+ "rewards/rejected": -1.470958948135376,
676
  "step": 370
677
  },
678
  {
679
  "epoch": 0.8685714285714285,
680
+ "grad_norm": 22.845593504615394,
681
  "learning_rate": 2.550629574310309e-08,
682
+ "logits/chosen": -0.2266564667224884,
683
+ "logits/rejected": 0.5173102021217346,
684
+ "logps/chosen": -349.7990417480469,
685
+ "logps/rejected": -402.1055603027344,
686
+ "loss": 0.5236,
687
+ "rewards/accuracies": 0.6937500238418579,
688
+ "rewards/chosen": -0.7807506918907166,
689
+ "rewards/margins": 0.665580689907074,
690
+ "rewards/rejected": -1.4463313817977905,
691
  "step": 380
692
  },
693
  {
694
  "epoch": 0.8914285714285715,
695
+ "grad_norm": 22.30415183791987,
696
  "learning_rate": 1.7438359028687983e-08,
697
+ "logits/chosen": -0.18654844164848328,
698
+ "logits/rejected": 0.17555546760559082,
699
+ "logps/chosen": -343.80609130859375,
700
+ "logps/rejected": -413.258544921875,
701
+ "loss": 0.5215,
702
+ "rewards/accuracies": 0.6875,
703
+ "rewards/chosen": -0.8466179966926575,
704
+ "rewards/margins": 0.5847481489181519,
705
+ "rewards/rejected": -1.431365966796875,
706
  "step": 390
707
  },
708
  {
709
  "epoch": 0.9142857142857143,
710
+ "grad_norm": 25.36293321655948,
711
  "learning_rate": 1.0855747162029361e-08,
712
+ "logits/chosen": -0.39323678612709045,
713
+ "logits/rejected": 0.6321589946746826,
714
+ "logps/chosen": -364.44781494140625,
715
+ "logps/rejected": -393.2674865722656,
716
+ "loss": 0.5343,
717
+ "rewards/accuracies": 0.731249988079071,
718
+ "rewards/chosen": -0.7503901720046997,
719
+ "rewards/margins": 0.7446303963661194,
720
+ "rewards/rejected": -1.4950206279754639,
721
  "step": 400
722
  },
723
  {
724
  "epoch": 0.9142857142857143,
725
+ "eval_logits/chosen": -0.3663737177848816,
726
+ "eval_logits/rejected": 0.7649080157279968,
727
+ "eval_logps/chosen": -345.4830017089844,
728
+ "eval_logps/rejected": -382.0352478027344,
729
+ "eval_loss": 0.5226701498031616,
730
+ "eval_rewards/accuracies": 0.7629310488700867,
731
+ "eval_rewards/chosen": -0.6987906098365784,
732
+ "eval_rewards/margins": 0.9308006763458252,
733
+ "eval_rewards/rejected": -1.6295913457870483,
734
+ "eval_runtime": 91.5738,
735
+ "eval_samples_per_second": 19.995,
736
+ "eval_steps_per_second": 0.317,
737
  "step": 400
738
  },
739
  {
740
  "epoch": 0.9371428571428572,
741
+ "grad_norm": 21.80106996482806,
742
  "learning_rate": 5.8005019731033615e-09,
743
+ "logits/chosen": -0.4477500021457672,
744
+ "logits/rejected": 0.4871234893798828,
745
+ "logps/chosen": -340.3522033691406,
746
+ "logps/rejected": -401.26629638671875,
747
+ "loss": 0.5153,
748
+ "rewards/accuracies": 0.7875000238418579,
749
+ "rewards/chosen": -0.6757135391235352,
750
+ "rewards/margins": 0.9294708371162415,
751
+ "rewards/rejected": -1.6051843166351318,
752
  "step": 410
753
  },
754
  {
755
  "epoch": 0.96,
756
+ "grad_norm": 23.103516202896476,
757
  "learning_rate": 2.3049103053431886e-09,
758
+ "logits/chosen": -0.3995040953159332,
759
+ "logits/rejected": 0.3316659927368164,
760
+ "logps/chosen": -386.6695861816406,
761
+ "logps/rejected": -383.6183776855469,
762
+ "loss": 0.5348,
763
+ "rewards/accuracies": 0.6812499761581421,
764
+ "rewards/chosen": -0.7931796312332153,
765
+ "rewards/margins": 0.6191332936286926,
766
+ "rewards/rejected": -1.4123131036758423,
767
  "step": 420
768
  },
769
  {
770
  "epoch": 0.9828571428571429,
771
+ "grad_norm": 22.91261742161674,
772
  "learning_rate": 3.9129780600541397e-10,
773
+ "logits/chosen": -0.2376430332660675,
774
+ "logits/rejected": 0.6147540807723999,
775
+ "logps/chosen": -368.62451171875,
776
+ "logps/rejected": -394.5757751464844,
777
+ "loss": 0.5369,
778
+ "rewards/accuracies": 0.7124999761581421,
779
+ "rewards/chosen": -0.8315681219100952,
780
+ "rewards/margins": 0.7056232690811157,
781
+ "rewards/rejected": -1.53719162940979,
782
  "step": 430
783
  },
784
  {
785
  "epoch": 0.9988571428571429,
786
  "step": 437,
787
  "total_flos": 0.0,
788
+ "train_loss": 0.5600041279258117,
789
+ "train_runtime": 11271.3049,
790
+ "train_samples_per_second": 4.968,
791
  "train_steps_per_second": 0.039
792
  }
793
  ],
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:065513940c426cbc6cd81ec6ac6c3f4a03305dfea92609285d389c3ad04a41d9
3
  size 7608
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6bf17a6b11cca2cbecdb0160512e7fcc2ca49ba133f6a4311e37f4633e4bf58
3
  size 7608