Model save
Browse files- README.md +18 -17
- all_results.json +16 -3
- eval_results.json +16 -0
- model-00001-of-00003.safetensors +1 -1
- model-00002-of-00003.safetensors +1 -1
- model-00003-of-00003.safetensors +1 -1
- train_results.json +3 -3
- trainer_state.json +523 -523
- training_args.bin +1 -1
README.md
CHANGED
@@ -4,6 +4,7 @@ base_model: alignment-handbook/zephyr-7b-sft-full
|
|
4 |
tags:
|
5 |
- trl
|
6 |
- dpo
|
|
|
7 |
- generated_from_trainer
|
8 |
model-index:
|
9 |
- name: zephyr-7b-dpo-full-prometheus_consistent-low-curriculum
|
@@ -17,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
|
|
17 |
|
18 |
This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
|
19 |
It achieves the following results on the evaluation set:
|
20 |
-
- Loss: 0.
|
21 |
-
- Rewards/chosen: -0.
|
22 |
-
- Rewards/rejected: -1.
|
23 |
-
- Rewards/accuracies: 0.
|
24 |
-
- Rewards/margins: 0.
|
25 |
-
- Logps/rejected: -
|
26 |
-
- Logps/chosen: -
|
27 |
-
- Logits/rejected: 0.
|
28 |
-
- Logits/chosen: -
|
29 |
|
30 |
## Model description
|
31 |
|
@@ -62,14 +63,14 @@ The following hyperparameters were used during training:
|
|
62 |
|
63 |
| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
|
64 |
|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
|
65 |
-
| 0.
|
66 |
-
| 0.
|
67 |
-
| 0.
|
68 |
-
| 0.
|
69 |
-
| 0.
|
70 |
-
| 0.
|
71 |
-
| 0.
|
72 |
-
| 0.
|
73 |
|
74 |
|
75 |
### Framework versions
|
|
|
4 |
tags:
|
5 |
- trl
|
6 |
- dpo
|
7 |
+
- alignment-handbook
|
8 |
- generated_from_trainer
|
9 |
model-index:
|
10 |
- name: zephyr-7b-dpo-full-prometheus_consistent-low-curriculum
|
|
|
18 |
|
19 |
This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
|
20 |
It achieves the following results on the evaluation set:
|
21 |
+
- Loss: 0.5227
|
22 |
+
- Rewards/chosen: -0.6988
|
23 |
+
- Rewards/rejected: -1.6296
|
24 |
+
- Rewards/accuracies: 0.7629
|
25 |
+
- Rewards/margins: 0.9308
|
26 |
+
- Logps/rejected: -382.0352
|
27 |
+
- Logps/chosen: -345.4830
|
28 |
+
- Logits/rejected: 0.7649
|
29 |
+
- Logits/chosen: -0.3664
|
30 |
|
31 |
## Model description
|
32 |
|
|
|
63 |
|
64 |
| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
|
65 |
|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
|
66 |
+
| 0.6619 | 0.1143 | 50 | 0.6465 | -0.0188 | -0.1651 | 0.6940 | 0.1464 | -235.5894 | -277.4793 | -2.4695 | -2.5696 |
|
67 |
+
| 0.5843 | 0.2286 | 100 | 0.5720 | -0.4443 | -1.0204 | 0.7457 | 0.5761 | -321.1169 | -320.0323 | -1.1914 | -1.3850 |
|
68 |
+
| 0.5509 | 0.3429 | 150 | 0.5467 | -0.4786 | -1.2647 | 0.7328 | 0.7861 | -345.5482 | -323.4684 | -0.6635 | -1.3552 |
|
69 |
+
| 0.5275 | 0.4571 | 200 | 0.5396 | -0.5220 | -1.3807 | 0.7716 | 0.8587 | -357.1487 | -327.8021 | -0.0895 | -1.0377 |
|
70 |
+
| 0.5665 | 0.5714 | 250 | 0.5405 | -1.0842 | -1.9608 | 0.7629 | 0.8766 | -415.1577 | -384.0230 | 1.5254 | 0.2878 |
|
71 |
+
| 0.5202 | 0.6857 | 300 | 0.5275 | -0.7424 | -1.7347 | 0.7716 | 0.9923 | -392.5496 | -349.8434 | 0.9934 | -0.2192 |
|
72 |
+
| 0.5261 | 0.8 | 350 | 0.5234 | -0.7613 | -1.7196 | 0.7586 | 0.9584 | -391.0397 | -351.7310 | 0.9008 | -0.2697 |
|
73 |
+
| 0.5343 | 0.9143 | 400 | 0.5227 | -0.6988 | -1.6296 | 0.7629 | 0.9308 | -382.0352 | -345.4830 | 0.7649 | -0.3664 |
|
74 |
|
75 |
|
76 |
### Framework versions
|
all_results.json
CHANGED
@@ -1,9 +1,22 @@
|
|
1 |
{
|
2 |
"epoch": 0.9988571428571429,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
3 |
"total_flos": 0.0,
|
4 |
-
"train_loss": 0.
|
5 |
-
"train_runtime":
|
6 |
"train_samples": 55999,
|
7 |
-
"train_samples_per_second": 4.
|
8 |
"train_steps_per_second": 0.039
|
9 |
}
|
|
|
1 |
{
|
2 |
"epoch": 0.9988571428571429,
|
3 |
+
"eval_logits/chosen": -1.2403382062911987,
|
4 |
+
"eval_logits/rejected": 0.062164660543203354,
|
5 |
+
"eval_logps/chosen": -357.4463806152344,
|
6 |
+
"eval_logps/rejected": -395.7604064941406,
|
7 |
+
"eval_loss": 0.5215353965759277,
|
8 |
+
"eval_rewards/accuracies": 0.7456896305084229,
|
9 |
+
"eval_rewards/chosen": -0.8184244632720947,
|
10 |
+
"eval_rewards/margins": 0.948418378829956,
|
11 |
+
"eval_rewards/rejected": -1.7668428421020508,
|
12 |
+
"eval_runtime": 92.4063,
|
13 |
+
"eval_samples": 1831,
|
14 |
+
"eval_samples_per_second": 19.815,
|
15 |
+
"eval_steps_per_second": 0.314,
|
16 |
"total_flos": 0.0,
|
17 |
+
"train_loss": 0.5600041279258117,
|
18 |
+
"train_runtime": 11271.3049,
|
19 |
"train_samples": 55999,
|
20 |
+
"train_samples_per_second": 4.968,
|
21 |
"train_steps_per_second": 0.039
|
22 |
}
|
eval_results.json
ADDED
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"epoch": 0.9988571428571429,
|
3 |
+
"eval_logits/chosen": -1.2403382062911987,
|
4 |
+
"eval_logits/rejected": 0.062164660543203354,
|
5 |
+
"eval_logps/chosen": -357.4463806152344,
|
6 |
+
"eval_logps/rejected": -395.7604064941406,
|
7 |
+
"eval_loss": 0.5215353965759277,
|
8 |
+
"eval_rewards/accuracies": 0.7456896305084229,
|
9 |
+
"eval_rewards/chosen": -0.8184244632720947,
|
10 |
+
"eval_rewards/margins": 0.948418378829956,
|
11 |
+
"eval_rewards/rejected": -1.7668428421020508,
|
12 |
+
"eval_runtime": 92.4063,
|
13 |
+
"eval_samples": 1831,
|
14 |
+
"eval_samples_per_second": 19.815,
|
15 |
+
"eval_steps_per_second": 0.314
|
16 |
+
}
|
model-00001-of-00003.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 4943162336
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:8e623605fa3bbafa67cb410816687a0721c2410e0270a2013e1995f114ec7ae6
|
3 |
size 4943162336
|
model-00002-of-00003.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 4999819336
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:245a992275365f6ee90a40eb8598df9cf1fae14f09a85445924cdb0bda5ce59f
|
3 |
size 4999819336
|
model-00003-of-00003.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 4540516344
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:e7d92997f328d5b8a521cb3504f1e718907ff6f7f8f7edea04e3c80ea37cc84d
|
3 |
size 4540516344
|
train_results.json
CHANGED
@@ -1,9 +1,9 @@
|
|
1 |
{
|
2 |
"epoch": 0.9988571428571429,
|
3 |
"total_flos": 0.0,
|
4 |
-
"train_loss": 0.
|
5 |
-
"train_runtime":
|
6 |
"train_samples": 55999,
|
7 |
-
"train_samples_per_second": 4.
|
8 |
"train_steps_per_second": 0.039
|
9 |
}
|
|
|
1 |
{
|
2 |
"epoch": 0.9988571428571429,
|
3 |
"total_flos": 0.0,
|
4 |
+
"train_loss": 0.5600041279258117,
|
5 |
+
"train_runtime": 11271.3049,
|
6 |
"train_samples": 55999,
|
7 |
+
"train_samples_per_second": 4.968,
|
8 |
"train_steps_per_second": 0.039
|
9 |
}
|
trainer_state.json
CHANGED
@@ -10,784 +10,784 @@
|
|
10 |
"log_history": [
|
11 |
{
|
12 |
"epoch": 0.022857142857142857,
|
13 |
-
"grad_norm": 8.
|
14 |
"learning_rate": 1.1363636363636363e-07,
|
15 |
-
"logits/chosen": -2.
|
16 |
-
"logits/rejected": -2.
|
17 |
-
"logps/chosen": -296.
|
18 |
-
"logps/rejected": -254.
|
19 |
-
"loss": 0.
|
20 |
-
"rewards/accuracies": 0.
|
21 |
-
"rewards/chosen":
|
22 |
-
"rewards/margins":
|
23 |
-
"rewards/rejected":
|
24 |
"step": 10
|
25 |
},
|
26 |
{
|
27 |
"epoch": 0.045714285714285714,
|
28 |
-
"grad_norm": 9.
|
29 |
"learning_rate": 2.2727272727272726e-07,
|
30 |
-
"logits/chosen": -2.
|
31 |
-
"logits/rejected": -2.
|
32 |
-
"logps/chosen": -294.
|
33 |
-
"logps/rejected": -254.
|
34 |
"loss": 0.6922,
|
35 |
-
"rewards/accuracies": 0.
|
36 |
-
"rewards/chosen": 0.
|
37 |
-
"rewards/margins": 0.
|
38 |
-
"rewards/rejected": -0.
|
39 |
"step": 20
|
40 |
},
|
41 |
{
|
42 |
"epoch": 0.06857142857142857,
|
43 |
-
"grad_norm": 8.
|
44 |
"learning_rate": 3.4090909090909085e-07,
|
45 |
-
"logits/chosen": -2.
|
46 |
-
"logits/rejected": -2.
|
47 |
-
"logps/chosen": -285.
|
48 |
-
"logps/rejected": -247.
|
49 |
-
"loss": 0.
|
50 |
-
"rewards/accuracies": 0.
|
51 |
-
"rewards/chosen": 0.
|
52 |
-
"rewards/margins": 0.
|
53 |
-
"rewards/rejected": -0.
|
54 |
"step": 30
|
55 |
},
|
56 |
{
|
57 |
"epoch": 0.09142857142857143,
|
58 |
-
"grad_norm": 10.
|
59 |
"learning_rate": 4.545454545454545e-07,
|
60 |
-
"logits/chosen": -2.
|
61 |
-
"logits/rejected": -2.
|
62 |
-
"logps/chosen": -269.
|
63 |
-
"logps/rejected": -273.
|
64 |
-
"loss": 0.
|
65 |
-
"rewards/accuracies": 0.
|
66 |
-
"rewards/chosen": 0.
|
67 |
-
"rewards/margins": 0.
|
68 |
-
"rewards/rejected": 0.
|
69 |
"step": 40
|
70 |
},
|
71 |
{
|
72 |
"epoch": 0.11428571428571428,
|
73 |
-
"grad_norm": 8.
|
74 |
"learning_rate": 4.997124959943201e-07,
|
75 |
-
"logits/chosen": -2.
|
76 |
-
"logits/rejected": -2.
|
77 |
-
"logps/chosen": -305.
|
78 |
-
"logps/rejected": -271.
|
79 |
-
"loss": 0.
|
80 |
"rewards/accuracies": 0.7124999761581421,
|
81 |
-
"rewards/chosen": -0.
|
82 |
-
"rewards/margins": 0.
|
83 |
-
"rewards/rejected": -0.
|
84 |
"step": 50
|
85 |
},
|
86 |
{
|
87 |
"epoch": 0.11428571428571428,
|
88 |
-
"eval_logits/chosen": -2.
|
89 |
-
"eval_logits/rejected": -2.
|
90 |
-
"eval_logps/chosen": -277.
|
91 |
-
"eval_logps/rejected": -235.
|
92 |
-
"eval_loss": 0.
|
93 |
"eval_rewards/accuracies": 0.693965494632721,
|
94 |
-
"eval_rewards/chosen": -0.
|
95 |
-
"eval_rewards/margins": 0.
|
96 |
-
"eval_rewards/rejected": -0.
|
97 |
-
"eval_runtime":
|
98 |
-
"eval_samples_per_second": 20.
|
99 |
-
"eval_steps_per_second": 0.
|
100 |
"step": 50
|
101 |
},
|
102 |
{
|
103 |
"epoch": 0.13714285714285715,
|
104 |
-
"grad_norm": 11.
|
105 |
"learning_rate": 4.979579212164186e-07,
|
106 |
-
"logits/chosen": -2.
|
107 |
-
"logits/rejected": -2.
|
108 |
-
"logps/chosen": -
|
109 |
-
"logps/rejected": -302.
|
110 |
-
"loss": 0.
|
111 |
"rewards/accuracies": 0.71875,
|
112 |
-
"rewards/chosen": -0.
|
113 |
-
"rewards/margins": 0.
|
114 |
-
"rewards/rejected": -0.
|
115 |
"step": 60
|
116 |
},
|
117 |
{
|
118 |
"epoch": 0.16,
|
119 |
-
"grad_norm":
|
120 |
"learning_rate": 4.946196886175515e-07,
|
121 |
-
"logits/chosen": -2.
|
122 |
-
"logits/rejected": -2.
|
123 |
-
"logps/chosen": -280.
|
124 |
-
"logps/rejected": -274.
|
125 |
-
"loss": 0.
|
126 |
"rewards/accuracies": 0.6625000238418579,
|
127 |
-
"rewards/chosen": -0.
|
128 |
-
"rewards/margins": 0.
|
129 |
-
"rewards/rejected": -0.
|
130 |
"step": 70
|
131 |
},
|
132 |
{
|
133 |
"epoch": 0.18285714285714286,
|
134 |
-
"grad_norm":
|
135 |
"learning_rate": 4.897191188239667e-07,
|
136 |
-
"logits/chosen": -2.
|
137 |
-
"logits/rejected": -2.
|
138 |
-
"logps/chosen": -
|
139 |
-
"logps/rejected": -
|
140 |
-
"loss": 0.
|
141 |
-
"rewards/accuracies": 0.
|
142 |
-
"rewards/chosen": -0.
|
143 |
-
"rewards/margins": 0.
|
144 |
-
"rewards/rejected": -0.
|
145 |
"step": 80
|
146 |
},
|
147 |
{
|
148 |
"epoch": 0.2057142857142857,
|
149 |
-
"grad_norm": 17.
|
150 |
"learning_rate": 4.832875107981763e-07,
|
151 |
-
"logits/chosen": -2.
|
152 |
-
"logits/rejected": -2.
|
153 |
-
"logps/chosen": -
|
154 |
-
"logps/rejected": -
|
155 |
-
"loss": 0.
|
156 |
-
"rewards/accuracies": 0.
|
157 |
-
"rewards/chosen": -0.
|
158 |
-
"rewards/margins": 0.
|
159 |
-
"rewards/rejected": -0.
|
160 |
"step": 90
|
161 |
},
|
162 |
{
|
163 |
"epoch": 0.22857142857142856,
|
164 |
-
"grad_norm": 17.
|
165 |
"learning_rate": 4.753659419387223e-07,
|
166 |
-
"logits/chosen": -1.
|
167 |
-
"logits/rejected": -1.
|
168 |
-
"logps/chosen": -
|
169 |
-
"logps/rejected": -
|
170 |
-
"loss": 0.
|
171 |
-
"rewards/accuracies": 0.
|
172 |
-
"rewards/chosen": -0.
|
173 |
-
"rewards/margins": 0.
|
174 |
-
"rewards/rejected": -
|
175 |
"step": 100
|
176 |
},
|
177 |
{
|
178 |
"epoch": 0.22857142857142856,
|
179 |
-
"eval_logits/chosen": -1.
|
180 |
-
"eval_logits/rejected": -
|
181 |
-
"eval_logps/chosen": -320.
|
182 |
-
"eval_logps/rejected": -
|
183 |
-
"eval_loss": 0.
|
184 |
-
"eval_rewards/accuracies": 0.
|
185 |
-
"eval_rewards/chosen": -0.
|
186 |
-
"eval_rewards/margins": 0.
|
187 |
-
"eval_rewards/rejected": -1.
|
188 |
-
"eval_runtime":
|
189 |
-
"eval_samples_per_second": 20.
|
190 |
-
"eval_steps_per_second": 0.
|
191 |
"step": 100
|
192 |
},
|
193 |
{
|
194 |
"epoch": 0.25142857142857145,
|
195 |
-
"grad_norm":
|
196 |
"learning_rate": 4.660050057270191e-07,
|
197 |
-
"logits/chosen": -1.
|
198 |
-
"logits/rejected": -1.
|
199 |
-
"logps/chosen": -
|
200 |
-
"logps/rejected": -
|
201 |
-
"loss": 0.
|
202 |
-
"rewards/accuracies": 0.
|
203 |
-
"rewards/chosen": -0.
|
204 |
-
"rewards/margins": 0.
|
205 |
-
"rewards/rejected": -
|
206 |
"step": 110
|
207 |
},
|
208 |
{
|
209 |
"epoch": 0.2742857142857143,
|
210 |
-
"grad_norm":
|
211 |
"learning_rate": 4.5526448859687144e-07,
|
212 |
-
"logits/chosen": -1.
|
213 |
-
"logits/rejected": -
|
214 |
-
"logps/chosen": -
|
215 |
-
"logps/rejected": -
|
216 |
-
"loss": 0.
|
217 |
-
"rewards/accuracies": 0.
|
218 |
-
"rewards/chosen": -0.
|
219 |
-
"rewards/margins": 0.
|
220 |
-
"rewards/rejected": -
|
221 |
"step": 120
|
222 |
},
|
223 |
{
|
224 |
"epoch": 0.29714285714285715,
|
225 |
-
"grad_norm": 19.
|
226 |
"learning_rate": 4.432129880904388e-07,
|
227 |
-
"logits/chosen": -1.
|
228 |
-
"logits/rejected": -0.
|
229 |
-
"logps/chosen": -344.
|
230 |
-
"logps/rejected": -
|
231 |
-
"loss": 0.
|
232 |
-
"rewards/accuracies": 0.
|
233 |
-
"rewards/chosen": -0.
|
234 |
-
"rewards/margins": 0.
|
235 |
-
"rewards/rejected": -1.
|
236 |
"step": 130
|
237 |
},
|
238 |
{
|
239 |
"epoch": 0.32,
|
240 |
-
"grad_norm":
|
241 |
"learning_rate": 4.299274747394055e-07,
|
242 |
-
"logits/chosen": -0.
|
243 |
-
"logits/rejected": 0.
|
244 |
-
"logps/chosen": -
|
245 |
-
"logps/rejected": -
|
246 |
-
"loss": 0.
|
247 |
-
"rewards/accuracies": 0.
|
248 |
-
"rewards/chosen": -0.
|
249 |
-
"rewards/margins": 0.
|
250 |
-
"rewards/rejected": -
|
251 |
"step": 140
|
252 |
},
|
253 |
{
|
254 |
"epoch": 0.34285714285714286,
|
255 |
-
"grad_norm":
|
256 |
"learning_rate": 4.1549280046953653e-07,
|
257 |
-
"logits/chosen": -1.
|
258 |
-
"logits/rejected": -0.
|
259 |
-
"logps/chosen": -
|
260 |
-
"logps/rejected": -
|
261 |
-
"loss": 0.
|
262 |
-
"rewards/accuracies": 0.
|
263 |
-
"rewards/chosen": -0.
|
264 |
-
"rewards/margins": 0.
|
265 |
-
"rewards/rejected": -1.
|
266 |
"step": 150
|
267 |
},
|
268 |
{
|
269 |
"epoch": 0.34285714285714286,
|
270 |
-
"eval_logits/chosen": -1.
|
271 |
-
"eval_logits/rejected": -0.
|
272 |
-
"eval_logps/chosen": -323.
|
273 |
-
"eval_logps/rejected": -
|
274 |
-
"eval_loss": 0.
|
275 |
-
"eval_rewards/accuracies": 0.
|
276 |
-
"eval_rewards/chosen": -0.
|
277 |
-
"eval_rewards/margins": 0.
|
278 |
-
"eval_rewards/rejected": -1.
|
279 |
-
"eval_runtime":
|
280 |
-
"eval_samples_per_second":
|
281 |
-
"eval_steps_per_second": 0.
|
282 |
"step": 150
|
283 |
},
|
284 |
{
|
285 |
"epoch": 0.3657142857142857,
|
286 |
-
"grad_norm":
|
287 |
"learning_rate": 4.000011566683401e-07,
|
288 |
-
"logits/chosen": -1.
|
289 |
-
"logits/rejected": -0.
|
290 |
-
"logps/chosen": -
|
291 |
-
"logps/rejected": -360.
|
292 |
-
"loss": 0.
|
293 |
-
"rewards/accuracies": 0.
|
294 |
-
"rewards/chosen": -0.
|
295 |
-
"rewards/margins": 0.
|
296 |
-
"rewards/rejected": -1.
|
297 |
"step": 160
|
298 |
},
|
299 |
{
|
300 |
"epoch": 0.38857142857142857,
|
301 |
-
"grad_norm":
|
302 |
"learning_rate": 3.8355148537705047e-07,
|
303 |
-
"logits/chosen": -
|
304 |
-
"logits/rejected": 0.
|
305 |
-
"logps/chosen": -
|
306 |
-
"logps/rejected": -
|
307 |
-
"loss": 0.
|
308 |
-
"rewards/accuracies": 0.
|
309 |
-
"rewards/chosen": -0.
|
310 |
-
"rewards/margins": 0.
|
311 |
-
"rewards/rejected": -1.
|
312 |
"step": 170
|
313 |
},
|
314 |
{
|
315 |
"epoch": 0.4114285714285714,
|
316 |
-
"grad_norm":
|
317 |
"learning_rate": 3.662488473675315e-07,
|
318 |
-
"logits/chosen": -
|
319 |
-
"logits/rejected": -0.
|
320 |
-
"logps/chosen": -
|
321 |
-
"logps/rejected": -
|
322 |
-
"loss": 0.
|
323 |
-
"rewards/accuracies": 0.
|
324 |
-
"rewards/chosen": -0.
|
325 |
-
"rewards/margins": 0.
|
326 |
-
"rewards/rejected": -1.
|
327 |
"step": 180
|
328 |
},
|
329 |
{
|
330 |
"epoch": 0.4342857142857143,
|
331 |
-
"grad_norm":
|
332 |
"learning_rate": 3.48203751140067e-07,
|
333 |
-
"logits/chosen": -
|
334 |
-
"logits/rejected": -0.
|
335 |
-
"logps/chosen": -
|
336 |
-
"logps/rejected": -
|
337 |
-
"loss": 0.
|
338 |
-
"rewards/accuracies": 0.
|
339 |
-
"rewards/chosen": -0.
|
340 |
-
"rewards/margins": 0.
|
341 |
-
"rewards/rejected": -1.
|
342 |
"step": 190
|
343 |
},
|
344 |
{
|
345 |
"epoch": 0.45714285714285713,
|
346 |
-
"grad_norm":
|
347 |
"learning_rate": 3.2953144712759537e-07,
|
348 |
-
"logits/chosen": -0.
|
349 |
-
"logits/rejected": 0.
|
350 |
-
"logps/chosen": -
|
351 |
-
"logps/rejected": -
|
352 |
-
"loss": 0.
|
353 |
-
"rewards/accuracies": 0.
|
354 |
-
"rewards/chosen": -0.
|
355 |
-
"rewards/margins": 0.
|
356 |
-
"rewards/rejected": -1.
|
357 |
"step": 200
|
358 |
},
|
359 |
{
|
360 |
"epoch": 0.45714285714285713,
|
361 |
-
"eval_logits/chosen": -1.
|
362 |
-
"eval_logits/rejected": -0.
|
363 |
-
"eval_logps/chosen": -
|
364 |
-
"eval_logps/rejected": -
|
365 |
-
"eval_loss": 0.
|
366 |
-
"eval_rewards/accuracies": 0.
|
367 |
-
"eval_rewards/chosen": -0.
|
368 |
-
"eval_rewards/margins": 0.
|
369 |
-
"eval_rewards/rejected": -1.
|
370 |
-
"eval_runtime": 90.
|
371 |
-
"eval_samples_per_second": 20.
|
372 |
-
"eval_steps_per_second": 0.
|
373 |
"step": 200
|
374 |
},
|
375 |
{
|
376 |
"epoch": 0.48,
|
377 |
-
"grad_norm": 23.
|
378 |
"learning_rate": 3.103511916141658e-07,
|
379 |
-
"logits/chosen": -
|
380 |
-
"logits/rejected": -0.
|
381 |
-
"logps/chosen": -
|
382 |
-
"logps/rejected": -
|
383 |
-
"loss": 0.
|
384 |
-
"rewards/accuracies": 0.
|
385 |
-
"rewards/chosen": -0.
|
386 |
-
"rewards/margins": 0.
|
387 |
-
"rewards/rejected": -1.
|
388 |
"step": 210
|
389 |
},
|
390 |
{
|
391 |
"epoch": 0.5028571428571429,
|
392 |
-
"grad_norm":
|
393 |
"learning_rate": 2.9078548506882117e-07,
|
394 |
-
"logits/chosen": -
|
395 |
-
"logits/rejected":
|
396 |
-
"logps/chosen": -
|
397 |
-
"logps/rejected": -
|
398 |
-
"loss": 0.
|
399 |
-
"rewards/accuracies": 0.
|
400 |
-
"rewards/chosen": -0.
|
401 |
-
"rewards/margins": 0.
|
402 |
-
"rewards/rejected": -1.
|
403 |
"step": 220
|
404 |
},
|
405 |
{
|
406 |
"epoch": 0.5257142857142857,
|
407 |
-
"grad_norm": 21.
|
408 |
"learning_rate": 2.709592897595191e-07,
|
409 |
-
"logits/chosen": -
|
410 |
-
"logits/rejected": -
|
411 |
-
"logps/chosen": -
|
412 |
-
"logps/rejected": -
|
413 |
-
"loss": 0.
|
414 |
-
"rewards/accuracies": 0.
|
415 |
-
"rewards/chosen": -0.
|
416 |
-
"rewards/margins": 0.
|
417 |
-
"rewards/rejected": -1.
|
418 |
"step": 230
|
419 |
},
|
420 |
{
|
421 |
"epoch": 0.5485714285714286,
|
422 |
-
"grad_norm":
|
423 |
"learning_rate": 2.509992316440332e-07,
|
424 |
-
"logits/chosen": -
|
425 |
-
"logits/rejected":
|
426 |
-
"logps/chosen": -
|
427 |
-
"logps/rejected": -
|
428 |
-
"loss": 0.
|
429 |
-
"rewards/accuracies": 0.
|
430 |
-
"rewards/chosen": -0.
|
431 |
-
"rewards/margins": 0.
|
432 |
-
"rewards/rejected": -1.
|
433 |
"step": 240
|
434 |
},
|
435 |
{
|
436 |
"epoch": 0.5714285714285714,
|
437 |
-
"grad_norm":
|
438 |
"learning_rate": 2.3103279163519918e-07,
|
439 |
-
"logits/chosen":
|
440 |
-
"logits/rejected":
|
441 |
-
"logps/chosen": -
|
442 |
-
"logps/rejected": -
|
443 |
-
"loss": 0.
|
444 |
-
"rewards/accuracies": 0.
|
445 |
-
"rewards/chosen": -1.
|
446 |
-
"rewards/margins": 0.
|
447 |
-
"rewards/rejected": -1.
|
448 |
"step": 250
|
449 |
},
|
450 |
{
|
451 |
"epoch": 0.5714285714285714,
|
452 |
-
"eval_logits/chosen":
|
453 |
-
"eval_logits/rejected":
|
454 |
-
"eval_logps/chosen": -
|
455 |
-
"eval_logps/rejected": -
|
456 |
-
"eval_loss": 0.
|
457 |
-
"eval_rewards/accuracies": 0.
|
458 |
-
"eval_rewards/chosen": -1.
|
459 |
-
"eval_rewards/margins": 0.
|
460 |
-
"eval_rewards/rejected": -1.
|
461 |
-
"eval_runtime": 91.
|
462 |
-
"eval_samples_per_second": 19.
|
463 |
-
"eval_steps_per_second": 0.
|
464 |
"step": 250
|
465 |
},
|
466 |
{
|
467 |
"epoch": 0.5942857142857143,
|
468 |
-
"grad_norm":
|
469 |
"learning_rate": 2.1118749140573358e-07,
|
470 |
-
"logits/chosen": -
|
471 |
-
"logits/rejected":
|
472 |
-
"logps/chosen": -
|
473 |
-
"logps/rejected": -
|
474 |
-
"loss": 0.
|
475 |
-
"rewards/accuracies": 0.
|
476 |
-
"rewards/chosen": -0.
|
477 |
-
"rewards/margins": 0.
|
478 |
-
"rewards/rejected": -1.
|
479 |
"step": 260
|
480 |
},
|
481 |
{
|
482 |
"epoch": 0.6171428571428571,
|
483 |
-
"grad_norm":
|
484 |
"learning_rate": 1.9159007893272703e-07,
|
485 |
-
"logits/chosen": -0.
|
486 |
-
"logits/rejected":
|
487 |
-
"logps/chosen": -
|
488 |
-
"logps/rejected": -
|
489 |
-
"loss": 0.
|
490 |
-
"rewards/accuracies": 0.
|
491 |
-
"rewards/chosen": -0.
|
492 |
-
"rewards/margins": 0.
|
493 |
-
"rewards/rejected": -1.
|
494 |
"step": 270
|
495 |
},
|
496 |
{
|
497 |
"epoch": 0.64,
|
498 |
-
"grad_norm":
|
499 |
"learning_rate": 1.7236571898357766e-07,
|
500 |
-
"logits/chosen": -0.
|
501 |
-
"logits/rejected": 0.
|
502 |
-
"logps/chosen": -
|
503 |
-
"logps/rejected": -
|
504 |
-
"loss": 0.
|
505 |
-
"rewards/accuracies": 0.
|
506 |
-
"rewards/chosen": -0.
|
507 |
-
"rewards/margins": 0.
|
508 |
-
"rewards/rejected": -1.
|
509 |
"step": 280
|
510 |
},
|
511 |
{
|
512 |
"epoch": 0.6628571428571428,
|
513 |
-
"grad_norm":
|
514 |
"learning_rate": 1.5363719371356882e-07,
|
515 |
-
"logits/chosen":
|
516 |
-
"logits/rejected":
|
517 |
-
"logps/chosen": -
|
518 |
-
"logps/rejected": -
|
519 |
-
"loss": 0.
|
520 |
-
"rewards/accuracies": 0.
|
521 |
-
"rewards/chosen": -0.
|
522 |
-
"rewards/margins": 0.
|
523 |
-
"rewards/rejected": -1.
|
524 |
"step": 290
|
525 |
},
|
526 |
{
|
527 |
"epoch": 0.6857142857142857,
|
528 |
-
"grad_norm":
|
529 |
"learning_rate": 1.3552411848071565e-07,
|
530 |
-
"logits/chosen": -
|
531 |
-
"logits/rejected": 0.
|
532 |
-
"logps/chosen": -
|
533 |
-
"logps/rejected": -
|
534 |
-
"loss": 0.
|
535 |
-
"rewards/accuracies": 0.
|
536 |
-
"rewards/chosen": -0.
|
537 |
-
"rewards/margins": 0.
|
538 |
-
"rewards/rejected": -1.
|
539 |
"step": 300
|
540 |
},
|
541 |
{
|
542 |
"epoch": 0.6857142857142857,
|
543 |
-
"eval_logits/chosen": -
|
544 |
-
"eval_logits/rejected": 0.
|
545 |
-
"eval_logps/chosen": -
|
546 |
-
"eval_logps/rejected": -
|
547 |
-
"eval_loss": 0.
|
548 |
-
"eval_rewards/accuracies": 0.
|
549 |
-
"eval_rewards/chosen": -0.
|
550 |
-
"eval_rewards/margins":
|
551 |
-
"eval_rewards/rejected": -1.
|
552 |
-
"eval_runtime":
|
553 |
-
"eval_samples_per_second":
|
554 |
-
"eval_steps_per_second": 0.
|
555 |
"step": 300
|
556 |
},
|
557 |
{
|
558 |
"epoch": 0.7085714285714285,
|
559 |
-
"grad_norm":
|
560 |
"learning_rate": 1.1814217788631473e-07,
|
561 |
-
"logits/chosen": -0.
|
562 |
-
"logits/rejected":
|
563 |
-
"logps/chosen": -
|
564 |
-
"logps/rejected": -
|
565 |
-
"loss": 0.
|
566 |
"rewards/accuracies": 0.675000011920929,
|
567 |
-
"rewards/chosen": -0.
|
568 |
-
"rewards/margins": 0.
|
569 |
-
"rewards/rejected": -1.
|
570 |
"step": 310
|
571 |
},
|
572 |
{
|
573 |
"epoch": 0.7314285714285714,
|
574 |
-
"grad_norm":
|
575 |
"learning_rate": 1.0160238692045331e-07,
|
576 |
-
"logits/chosen": -
|
577 |
-
"logits/rejected":
|
578 |
-
"logps/chosen": -
|
579 |
-
"logps/rejected": -
|
580 |
-
"loss": 0.
|
581 |
-
"rewards/accuracies": 0.
|
582 |
-
"rewards/chosen": -0.
|
583 |
-
"rewards/margins": 0.
|
584 |
-
"rewards/rejected": -1.
|
585 |
"step": 320
|
586 |
},
|
587 |
{
|
588 |
"epoch": 0.7542857142857143,
|
589 |
-
"grad_norm":
|
590 |
"learning_rate": 8.601038193139438e-08,
|
591 |
-
"logits/chosen": -
|
592 |
-
"logits/rejected":
|
593 |
-
"logps/chosen": -
|
594 |
-
"logps/rejected": -
|
595 |
-
"loss": 0.
|
596 |
-
"rewards/accuracies": 0.
|
597 |
-
"rewards/chosen": -0.
|
598 |
-
"rewards/margins": 0.
|
599 |
-
"rewards/rejected": -1.
|
600 |
"step": 330
|
601 |
},
|
602 |
{
|
603 |
"epoch": 0.7771428571428571,
|
604 |
-
"grad_norm":
|
605 |
"learning_rate": 7.146574594727572e-08,
|
606 |
-
"logits/chosen": -
|
607 |
-
"logits/rejected":
|
608 |
-
"logps/chosen": -363.
|
609 |
-
"logps/rejected": -
|
610 |
-
"loss": 0.
|
611 |
-
"rewards/accuracies": 0.
|
612 |
-
"rewards/chosen": -0.
|
613 |
-
"rewards/margins": 0.
|
614 |
-
"rewards/rejected": -1.
|
615 |
"step": 340
|
616 |
},
|
617 |
{
|
618 |
"epoch": 0.8,
|
619 |
-
"grad_norm":
|
620 |
"learning_rate": 5.8061372659157306e-08,
|
621 |
-
"logits/chosen": -0.
|
622 |
-
"logits/rejected": 0.
|
623 |
-
"logps/chosen": -
|
624 |
-
"logps/rejected": -
|
625 |
-
"loss": 0.
|
626 |
-
"rewards/accuracies": 0.
|
627 |
-
"rewards/chosen": -0.
|
628 |
-
"rewards/margins": 0.
|
629 |
-
"rewards/rejected": -1.
|
630 |
"step": 350
|
631 |
},
|
632 |
{
|
633 |
"epoch": 0.8,
|
634 |
-
"eval_logits/chosen": -
|
635 |
-
"eval_logits/rejected": 0.
|
636 |
-
"eval_logps/chosen": -
|
637 |
-
"eval_logps/rejected": -
|
638 |
-
"eval_loss": 0.
|
639 |
-
"eval_rewards/accuracies": 0.
|
640 |
-
"eval_rewards/chosen": -0.
|
641 |
-
"eval_rewards/margins": 0.
|
642 |
-
"eval_rewards/rejected": -1.
|
643 |
-
"eval_runtime": 90.
|
644 |
-
"eval_samples_per_second": 20.
|
645 |
-
"eval_steps_per_second": 0.
|
646 |
"step": 350
|
647 |
},
|
648 |
{
|
649 |
"epoch": 0.8228571428571428,
|
650 |
-
"grad_norm": 26.
|
651 |
"learning_rate": 4.5882873127531614e-08,
|
652 |
-
"logits/chosen": -0.
|
653 |
-
"logits/rejected":
|
654 |
-
"logps/chosen": -
|
655 |
-
"logps/rejected": -
|
656 |
-
"loss": 0.
|
657 |
-
"rewards/accuracies": 0.
|
658 |
-
"rewards/chosen": -0.
|
659 |
-
"rewards/margins": 0.
|
660 |
-
"rewards/rejected": -1.
|
661 |
"step": 360
|
662 |
},
|
663 |
{
|
664 |
"epoch": 0.8457142857142858,
|
665 |
-
"grad_norm":
|
666 |
"learning_rate": 3.500802900154412e-08,
|
667 |
-
"logits/chosen": -0.
|
668 |
-
"logits/rejected": 0.
|
669 |
-
"logps/chosen": -
|
670 |
-
"logps/rejected": -
|
671 |
-
"loss": 0.
|
672 |
-
"rewards/accuracies": 0.
|
673 |
-
"rewards/chosen": -0.
|
674 |
-
"rewards/margins": 0.
|
675 |
-
"rewards/rejected": -1.
|
676 |
"step": 370
|
677 |
},
|
678 |
{
|
679 |
"epoch": 0.8685714285714285,
|
680 |
-
"grad_norm":
|
681 |
"learning_rate": 2.550629574310309e-08,
|
682 |
-
"logits/chosen": -0.
|
683 |
-
"logits/rejected":
|
684 |
-
"logps/chosen": -
|
685 |
-
"logps/rejected": -
|
686 |
-
"loss": 0.
|
687 |
-
"rewards/accuracies": 0.
|
688 |
-
"rewards/chosen": -0.
|
689 |
-
"rewards/margins": 0.
|
690 |
-
"rewards/rejected": -1.
|
691 |
"step": 380
|
692 |
},
|
693 |
{
|
694 |
"epoch": 0.8914285714285715,
|
695 |
-
"grad_norm":
|
696 |
"learning_rate": 1.7438359028687983e-08,
|
697 |
-
"logits/chosen": -0.
|
698 |
-
"logits/rejected":
|
699 |
-
"logps/chosen": -
|
700 |
-
"logps/rejected": -
|
701 |
-
"loss": 0.
|
702 |
-
"rewards/accuracies": 0.
|
703 |
-
"rewards/chosen": -0.
|
704 |
-
"rewards/margins": 0.
|
705 |
-
"rewards/rejected": -1.
|
706 |
"step": 390
|
707 |
},
|
708 |
{
|
709 |
"epoch": 0.9142857142857143,
|
710 |
-
"grad_norm":
|
711 |
"learning_rate": 1.0855747162029361e-08,
|
712 |
-
"logits/chosen": -
|
713 |
-
"logits/rejected":
|
714 |
-
"logps/chosen": -
|
715 |
-
"logps/rejected": -
|
716 |
-
"loss": 0.
|
717 |
-
"rewards/accuracies": 0.
|
718 |
-
"rewards/chosen": -0.
|
719 |
-
"rewards/margins": 0.
|
720 |
-
"rewards/rejected": -1.
|
721 |
"step": 400
|
722 |
},
|
723 |
{
|
724 |
"epoch": 0.9142857142857143,
|
725 |
-
"eval_logits/chosen": -
|
726 |
-
"eval_logits/rejected": 0.
|
727 |
-
"eval_logps/chosen": -
|
728 |
-
"eval_logps/rejected": -
|
729 |
-
"eval_loss": 0.
|
730 |
-
"eval_rewards/accuracies": 0.
|
731 |
-
"eval_rewards/chosen": -0.
|
732 |
-
"eval_rewards/margins": 0.
|
733 |
-
"eval_rewards/rejected": -1.
|
734 |
-
"eval_runtime":
|
735 |
-
"eval_samples_per_second":
|
736 |
-
"eval_steps_per_second": 0.
|
737 |
"step": 400
|
738 |
},
|
739 |
{
|
740 |
"epoch": 0.9371428571428572,
|
741 |
-
"grad_norm":
|
742 |
"learning_rate": 5.8005019731033615e-09,
|
743 |
-
"logits/chosen": -
|
744 |
-
"logits/rejected":
|
745 |
-
"logps/chosen": -
|
746 |
-
"logps/rejected": -
|
747 |
-
"loss": 0.
|
748 |
-
"rewards/accuracies": 0.
|
749 |
-
"rewards/chosen": -0.
|
750 |
-
"rewards/margins": 0.
|
751 |
-
"rewards/rejected": -1.
|
752 |
"step": 410
|
753 |
},
|
754 |
{
|
755 |
"epoch": 0.96,
|
756 |
-
"grad_norm":
|
757 |
"learning_rate": 2.3049103053431886e-09,
|
758 |
-
"logits/chosen": -
|
759 |
-
"logits/rejected":
|
760 |
-
"logps/chosen": -
|
761 |
-
"logps/rejected": -
|
762 |
-
"loss": 0.
|
763 |
-
"rewards/accuracies": 0.
|
764 |
-
"rewards/chosen": -0.
|
765 |
-
"rewards/margins": 0.
|
766 |
-
"rewards/rejected": -1.
|
767 |
"step": 420
|
768 |
},
|
769 |
{
|
770 |
"epoch": 0.9828571428571429,
|
771 |
-
"grad_norm":
|
772 |
"learning_rate": 3.9129780600541397e-10,
|
773 |
-
"logits/chosen": -
|
774 |
-
"logits/rejected":
|
775 |
-
"logps/chosen": -
|
776 |
-
"logps/rejected": -
|
777 |
-
"loss": 0.
|
778 |
-
"rewards/accuracies": 0.
|
779 |
-
"rewards/chosen": -0.
|
780 |
-
"rewards/margins": 0.
|
781 |
-
"rewards/rejected": -1.
|
782 |
"step": 430
|
783 |
},
|
784 |
{
|
785 |
"epoch": 0.9988571428571429,
|
786 |
"step": 437,
|
787 |
"total_flos": 0.0,
|
788 |
-
"train_loss": 0.
|
789 |
-
"train_runtime":
|
790 |
-
"train_samples_per_second": 4.
|
791 |
"train_steps_per_second": 0.039
|
792 |
}
|
793 |
],
|
|
|
10 |
"log_history": [
|
11 |
{
|
12 |
"epoch": 0.022857142857142857,
|
13 |
+
"grad_norm": 8.11412987933583,
|
14 |
"learning_rate": 1.1363636363636363e-07,
|
15 |
+
"logits/chosen": -2.66959810256958,
|
16 |
+
"logits/rejected": -2.6077542304992676,
|
17 |
+
"logps/chosen": -296.6876220703125,
|
18 |
+
"logps/rejected": -254.7753448486328,
|
19 |
+
"loss": 0.6933,
|
20 |
+
"rewards/accuracies": 0.3812499940395355,
|
21 |
+
"rewards/chosen": -0.0003348872414790094,
|
22 |
+
"rewards/margins": -0.00012643556692637503,
|
23 |
+
"rewards/rejected": -0.0002084516454488039,
|
24 |
"step": 10
|
25 |
},
|
26 |
{
|
27 |
"epoch": 0.045714285714285714,
|
28 |
+
"grad_norm": 9.934680850734814,
|
29 |
"learning_rate": 2.2727272727272726e-07,
|
30 |
+
"logits/chosen": -2.6891586780548096,
|
31 |
+
"logits/rejected": -2.6237130165100098,
|
32 |
+
"logps/chosen": -294.1405334472656,
|
33 |
+
"logps/rejected": -254.05810546875,
|
34 |
"loss": 0.6922,
|
35 |
+
"rewards/accuracies": 0.65625,
|
36 |
+
"rewards/chosen": 0.0007635392248630524,
|
37 |
+
"rewards/margins": 0.0031762172002345324,
|
38 |
+
"rewards/rejected": -0.0024126782082021236,
|
39 |
"step": 20
|
40 |
},
|
41 |
{
|
42 |
"epoch": 0.06857142857142857,
|
43 |
+
"grad_norm": 8.46839233994518,
|
44 |
"learning_rate": 3.4090909090909085e-07,
|
45 |
+
"logits/chosen": -2.657853364944458,
|
46 |
+
"logits/rejected": -2.562720775604248,
|
47 |
+
"logps/chosen": -285.6708984375,
|
48 |
+
"logps/rejected": -247.06838989257812,
|
49 |
+
"loss": 0.6879,
|
50 |
+
"rewards/accuracies": 0.6625000238418579,
|
51 |
+
"rewards/chosen": 0.012036588974297047,
|
52 |
+
"rewards/margins": 0.01440697442740202,
|
53 |
+
"rewards/rejected": -0.002370386151596904,
|
54 |
"step": 30
|
55 |
},
|
56 |
{
|
57 |
"epoch": 0.09142857142857143,
|
58 |
+
"grad_norm": 10.38860192358711,
|
59 |
"learning_rate": 4.545454545454545e-07,
|
60 |
+
"logits/chosen": -2.6249475479125977,
|
61 |
+
"logits/rejected": -2.5956408977508545,
|
62 |
+
"logps/chosen": -269.8529357910156,
|
63 |
+
"logps/rejected": -273.1573791503906,
|
64 |
+
"loss": 0.6793,
|
65 |
+
"rewards/accuracies": 0.65625,
|
66 |
+
"rewards/chosen": 0.02421986497938633,
|
67 |
+
"rewards/margins": 0.020207645371556282,
|
68 |
+
"rewards/rejected": 0.004012218676507473,
|
69 |
"step": 40
|
70 |
},
|
71 |
{
|
72 |
"epoch": 0.11428571428571428,
|
73 |
+
"grad_norm": 8.812178369158405,
|
74 |
"learning_rate": 4.997124959943201e-07,
|
75 |
+
"logits/chosen": -2.6135976314544678,
|
76 |
+
"logits/rejected": -2.5756285190582275,
|
77 |
+
"logps/chosen": -305.51312255859375,
|
78 |
+
"logps/rejected": -271.22247314453125,
|
79 |
+
"loss": 0.6619,
|
80 |
"rewards/accuracies": 0.7124999761581421,
|
81 |
+
"rewards/chosen": -0.010274273343384266,
|
82 |
+
"rewards/margins": 0.067402184009552,
|
83 |
+
"rewards/rejected": -0.0776764526963234,
|
84 |
"step": 50
|
85 |
},
|
86 |
{
|
87 |
"epoch": 0.11428571428571428,
|
88 |
+
"eval_logits/chosen": -2.569648265838623,
|
89 |
+
"eval_logits/rejected": -2.4695067405700684,
|
90 |
+
"eval_logps/chosen": -277.47930908203125,
|
91 |
+
"eval_logps/rejected": -235.5894012451172,
|
92 |
+
"eval_loss": 0.6465452313423157,
|
93 |
"eval_rewards/accuracies": 0.693965494632721,
|
94 |
+
"eval_rewards/chosen": -0.018753662705421448,
|
95 |
+
"eval_rewards/margins": 0.1463788002729416,
|
96 |
+
"eval_rewards/rejected": -0.16513246297836304,
|
97 |
+
"eval_runtime": 90.334,
|
98 |
+
"eval_samples_per_second": 20.269,
|
99 |
+
"eval_steps_per_second": 0.321,
|
100 |
"step": 50
|
101 |
},
|
102 |
{
|
103 |
"epoch": 0.13714285714285715,
|
104 |
+
"grad_norm": 11.6151396248973,
|
105 |
"learning_rate": 4.979579212164186e-07,
|
106 |
+
"logits/chosen": -2.6002144813537598,
|
107 |
+
"logits/rejected": -2.501227855682373,
|
108 |
+
"logps/chosen": -298.59063720703125,
|
109 |
+
"logps/rejected": -302.35577392578125,
|
110 |
+
"loss": 0.6307,
|
111 |
"rewards/accuracies": 0.71875,
|
112 |
+
"rewards/chosen": -0.047010406851768494,
|
113 |
+
"rewards/margins": 0.16466036438941956,
|
114 |
+
"rewards/rejected": -0.21167078614234924,
|
115 |
"step": 60
|
116 |
},
|
117 |
{
|
118 |
"epoch": 0.16,
|
119 |
+
"grad_norm": 14.811620243521006,
|
120 |
"learning_rate": 4.946196886175515e-07,
|
121 |
+
"logits/chosen": -2.6356024742126465,
|
122 |
+
"logits/rejected": -2.5619795322418213,
|
123 |
+
"logps/chosen": -280.7294921875,
|
124 |
+
"logps/rejected": -274.32647705078125,
|
125 |
+
"loss": 0.6194,
|
126 |
"rewards/accuracies": 0.6625000238418579,
|
127 |
+
"rewards/chosen": -0.16827444732189178,
|
128 |
+
"rewards/margins": 0.20077195763587952,
|
129 |
+
"rewards/rejected": -0.3690463900566101,
|
130 |
"step": 70
|
131 |
},
|
132 |
{
|
133 |
"epoch": 0.18285714285714286,
|
134 |
+
"grad_norm": 16.993875634534675,
|
135 |
"learning_rate": 4.897191188239667e-07,
|
136 |
+
"logits/chosen": -2.642766237258911,
|
137 |
+
"logits/rejected": -2.582462787628174,
|
138 |
+
"logps/chosen": -321.19744873046875,
|
139 |
+
"logps/rejected": -298.94171142578125,
|
140 |
+
"loss": 0.6087,
|
141 |
+
"rewards/accuracies": 0.7437499761581421,
|
142 |
+
"rewards/chosen": -0.0975220575928688,
|
143 |
+
"rewards/margins": 0.27095723152160645,
|
144 |
+
"rewards/rejected": -0.36847931146621704,
|
145 |
"step": 80
|
146 |
},
|
147 |
{
|
148 |
"epoch": 0.2057142857142857,
|
149 |
+
"grad_norm": 17.671501985453162,
|
150 |
"learning_rate": 4.832875107981763e-07,
|
151 |
+
"logits/chosen": -2.4848644733428955,
|
152 |
+
"logits/rejected": -2.404571533203125,
|
153 |
+
"logps/chosen": -310.0569152832031,
|
154 |
+
"logps/rejected": -313.3294982910156,
|
155 |
+
"loss": 0.6012,
|
156 |
+
"rewards/accuracies": 0.7124999761581421,
|
157 |
+
"rewards/chosen": -0.345020592212677,
|
158 |
+
"rewards/margins": 0.33231958746910095,
|
159 |
+
"rewards/rejected": -0.6773402690887451,
|
160 |
"step": 90
|
161 |
},
|
162 |
{
|
163 |
"epoch": 0.22857142857142856,
|
164 |
+
"grad_norm": 17.978608029535753,
|
165 |
"learning_rate": 4.753659419387223e-07,
|
166 |
+
"logits/chosen": -1.396976351737976,
|
167 |
+
"logits/rejected": -1.3131816387176514,
|
168 |
+
"logps/chosen": -309.3507385253906,
|
169 |
+
"logps/rejected": -321.52056884765625,
|
170 |
+
"loss": 0.5843,
|
171 |
+
"rewards/accuracies": 0.643750011920929,
|
172 |
+
"rewards/chosen": -0.6027094125747681,
|
173 |
+
"rewards/margins": 0.35499778389930725,
|
174 |
+
"rewards/rejected": -0.9577071070671082,
|
175 |
"step": 100
|
176 |
},
|
177 |
{
|
178 |
"epoch": 0.22857142857142856,
|
179 |
+
"eval_logits/chosen": -1.3850308656692505,
|
180 |
+
"eval_logits/rejected": -1.1914278268814087,
|
181 |
+
"eval_logps/chosen": -320.03228759765625,
|
182 |
+
"eval_logps/rejected": -321.1168518066406,
|
183 |
+
"eval_loss": 0.5719701647758484,
|
184 |
+
"eval_rewards/accuracies": 0.7456896305084229,
|
185 |
+
"eval_rewards/chosen": -0.44428348541259766,
|
186 |
+
"eval_rewards/margins": 0.5761240124702454,
|
187 |
+
"eval_rewards/rejected": -1.0204075574874878,
|
188 |
+
"eval_runtime": 89.9125,
|
189 |
+
"eval_samples_per_second": 20.364,
|
190 |
+
"eval_steps_per_second": 0.323,
|
191 |
"step": 100
|
192 |
},
|
193 |
{
|
194 |
"epoch": 0.25142857142857145,
|
195 |
+
"grad_norm": 30.473603379477254,
|
196 |
"learning_rate": 4.660050057270191e-07,
|
197 |
+
"logits/chosen": -1.5161502361297607,
|
198 |
+
"logits/rejected": -1.4007251262664795,
|
199 |
+
"logps/chosen": -326.7287292480469,
|
200 |
+
"logps/rejected": -381.71234130859375,
|
201 |
+
"loss": 0.5578,
|
202 |
+
"rewards/accuracies": 0.699999988079071,
|
203 |
+
"rewards/chosen": -0.47587689757347107,
|
204 |
+
"rewards/margins": 0.4765087962150574,
|
205 |
+
"rewards/rejected": -0.9523857235908508,
|
206 |
"step": 110
|
207 |
},
|
208 |
{
|
209 |
"epoch": 0.2742857142857143,
|
210 |
+
"grad_norm": 16.07677825536776,
|
211 |
"learning_rate": 4.5526448859687144e-07,
|
212 |
+
"logits/chosen": -1.329465627670288,
|
213 |
+
"logits/rejected": -1.2029626369476318,
|
214 |
+
"logps/chosen": -313.04150390625,
|
215 |
+
"logps/rejected": -344.30377197265625,
|
216 |
+
"loss": 0.59,
|
217 |
+
"rewards/accuracies": 0.6625000238418579,
|
218 |
+
"rewards/chosen": -0.4427226483821869,
|
219 |
+
"rewards/margins": 0.44687420129776,
|
220 |
+
"rewards/rejected": -0.8895969390869141,
|
221 |
"step": 120
|
222 |
},
|
223 |
{
|
224 |
"epoch": 0.29714285714285715,
|
225 |
+
"grad_norm": 19.25626895436831,
|
226 |
"learning_rate": 4.432129880904388e-07,
|
227 |
+
"logits/chosen": -1.0987221002578735,
|
228 |
+
"logits/rejected": -0.7802125215530396,
|
229 |
+
"logps/chosen": -344.36712646484375,
|
230 |
+
"logps/rejected": -391.39154052734375,
|
231 |
+
"loss": 0.5477,
|
232 |
+
"rewards/accuracies": 0.7749999761581421,
|
233 |
+
"rewards/chosen": -0.5906578302383423,
|
234 |
+
"rewards/margins": 0.5952860116958618,
|
235 |
+
"rewards/rejected": -1.1859437227249146,
|
236 |
"step": 130
|
237 |
},
|
238 |
{
|
239 |
"epoch": 0.32,
|
240 |
+
"grad_norm": 32.057510195911,
|
241 |
"learning_rate": 4.299274747394055e-07,
|
242 |
+
"logits/chosen": -0.759337306022644,
|
243 |
+
"logits/rejected": -0.5684966444969177,
|
244 |
+
"logps/chosen": -352.746826171875,
|
245 |
+
"logps/rejected": -371.2802734375,
|
246 |
+
"loss": 0.5676,
|
247 |
+
"rewards/accuracies": 0.6499999761581421,
|
248 |
+
"rewards/chosen": -0.540864884853363,
|
249 |
+
"rewards/margins": 0.43386125564575195,
|
250 |
+
"rewards/rejected": -0.9747260808944702,
|
251 |
"step": 140
|
252 |
},
|
253 |
{
|
254 |
"epoch": 0.34285714285714286,
|
255 |
+
"grad_norm": 31.167990046129457,
|
256 |
"learning_rate": 4.1549280046953653e-07,
|
257 |
+
"logits/chosen": -1.4192949533462524,
|
258 |
+
"logits/rejected": -0.8910300135612488,
|
259 |
+
"logps/chosen": -302.2892150878906,
|
260 |
+
"logps/rejected": -339.9471435546875,
|
261 |
+
"loss": 0.5509,
|
262 |
+
"rewards/accuracies": 0.731249988079071,
|
263 |
+
"rewards/chosen": -0.45886653661727905,
|
264 |
+
"rewards/margins": 0.62995845079422,
|
265 |
+
"rewards/rejected": -1.0888248682022095,
|
266 |
"step": 150
|
267 |
},
|
268 |
{
|
269 |
"epoch": 0.34285714285714286,
|
270 |
+
"eval_logits/chosen": -1.355178952217102,
|
271 |
+
"eval_logits/rejected": -0.6634992361068726,
|
272 |
+
"eval_logps/chosen": -323.4683532714844,
|
273 |
+
"eval_logps/rejected": -345.5482482910156,
|
274 |
+
"eval_loss": 0.5466835498809814,
|
275 |
+
"eval_rewards/accuracies": 0.732758641242981,
|
276 |
+
"eval_rewards/chosen": -0.4786438047885895,
|
277 |
+
"eval_rewards/margins": 0.7860775589942932,
|
278 |
+
"eval_rewards/rejected": -1.264721393585205,
|
279 |
+
"eval_runtime": 90.391,
|
280 |
+
"eval_samples_per_second": 20.256,
|
281 |
+
"eval_steps_per_second": 0.321,
|
282 |
"step": 150
|
283 |
},
|
284 |
{
|
285 |
"epoch": 0.3657142857142857,
|
286 |
+
"grad_norm": 24.250974026024966,
|
287 |
"learning_rate": 4.000011566683401e-07,
|
288 |
+
"logits/chosen": -1.3669617176055908,
|
289 |
+
"logits/rejected": -0.9052613377571106,
|
290 |
+
"logps/chosen": -333.5279846191406,
|
291 |
+
"logps/rejected": -360.4810791015625,
|
292 |
+
"loss": 0.55,
|
293 |
+
"rewards/accuracies": 0.7124999761581421,
|
294 |
+
"rewards/chosen": -0.6962090730667114,
|
295 |
+
"rewards/margins": 0.5538384318351746,
|
296 |
+
"rewards/rejected": -1.2500474452972412,
|
297 |
"step": 160
|
298 |
},
|
299 |
{
|
300 |
"epoch": 0.38857142857142857,
|
301 |
+
"grad_norm": 24.704591344446357,
|
302 |
"learning_rate": 3.8355148537705047e-07,
|
303 |
+
"logits/chosen": -1.3166093826293945,
|
304 |
+
"logits/rejected": -0.7781628966331482,
|
305 |
+
"logps/chosen": -343.19940185546875,
|
306 |
+
"logps/rejected": -390.14239501953125,
|
307 |
+
"loss": 0.5266,
|
308 |
+
"rewards/accuracies": 0.7124999761581421,
|
309 |
+
"rewards/chosen": -0.6181753873825073,
|
310 |
+
"rewards/margins": 0.671627402305603,
|
311 |
+
"rewards/rejected": -1.2898027896881104,
|
312 |
"step": 170
|
313 |
},
|
314 |
{
|
315 |
"epoch": 0.4114285714285714,
|
316 |
+
"grad_norm": 22.05889500026718,
|
317 |
"learning_rate": 3.662488473675315e-07,
|
318 |
+
"logits/chosen": -1.254248023033142,
|
319 |
+
"logits/rejected": -0.8255653381347656,
|
320 |
+
"logps/chosen": -341.5926818847656,
|
321 |
+
"logps/rejected": -382.6631774902344,
|
322 |
+
"loss": 0.542,
|
323 |
+
"rewards/accuracies": 0.6625000238418579,
|
324 |
+
"rewards/chosen": -0.7656871676445007,
|
325 |
+
"rewards/margins": 0.5109010338783264,
|
326 |
+
"rewards/rejected": -1.2765882015228271,
|
327 |
"step": 180
|
328 |
},
|
329 |
{
|
330 |
"epoch": 0.4342857142857143,
|
331 |
+
"grad_norm": 23.909926187935003,
|
332 |
"learning_rate": 3.48203751140067e-07,
|
333 |
+
"logits/chosen": -1.0761396884918213,
|
334 |
+
"logits/rejected": -0.6647660136222839,
|
335 |
+
"logps/chosen": -325.56683349609375,
|
336 |
+
"logps/rejected": -379.4165344238281,
|
337 |
+
"loss": 0.5356,
|
338 |
+
"rewards/accuracies": 0.6937500238418579,
|
339 |
+
"rewards/chosen": -0.6315457820892334,
|
340 |
+
"rewards/margins": 0.5936378240585327,
|
341 |
+
"rewards/rejected": -1.2251836061477661,
|
342 |
"step": 190
|
343 |
},
|
344 |
{
|
345 |
"epoch": 0.45714285714285713,
|
346 |
+
"grad_norm": 20.73701530382175,
|
347 |
"learning_rate": 3.2953144712759537e-07,
|
348 |
+
"logits/chosen": -0.5213090181350708,
|
349 |
+
"logits/rejected": 0.013022899627685547,
|
350 |
+
"logps/chosen": -359.0880432128906,
|
351 |
+
"logps/rejected": -410.6299743652344,
|
352 |
+
"loss": 0.5275,
|
353 |
+
"rewards/accuracies": 0.7124999761581421,
|
354 |
+
"rewards/chosen": -0.927217960357666,
|
355 |
+
"rewards/margins": 0.6191995739936829,
|
356 |
+
"rewards/rejected": -1.5464175939559937,
|
357 |
"step": 200
|
358 |
},
|
359 |
{
|
360 |
"epoch": 0.45714285714285713,
|
361 |
+
"eval_logits/chosen": -1.0376836061477661,
|
362 |
+
"eval_logits/rejected": -0.08949049562215805,
|
363 |
+
"eval_logps/chosen": -327.8020935058594,
|
364 |
+
"eval_logps/rejected": -357.148681640625,
|
365 |
+
"eval_loss": 0.5396182537078857,
|
366 |
+
"eval_rewards/accuracies": 0.7715517282485962,
|
367 |
+
"eval_rewards/chosen": -0.5219810605049133,
|
368 |
+
"eval_rewards/margins": 0.8587445020675659,
|
369 |
+
"eval_rewards/rejected": -1.380725622177124,
|
370 |
+
"eval_runtime": 90.1866,
|
371 |
+
"eval_samples_per_second": 20.302,
|
372 |
+
"eval_steps_per_second": 0.322,
|
373 |
"step": 200
|
374 |
},
|
375 |
{
|
376 |
"epoch": 0.48,
|
377 |
+
"grad_norm": 23.237204035063847,
|
378 |
"learning_rate": 3.103511916141658e-07,
|
379 |
+
"logits/chosen": -0.9574594497680664,
|
380 |
+
"logits/rejected": -0.16778725385665894,
|
381 |
+
"logps/chosen": -348.38385009765625,
|
382 |
+
"logps/rejected": -397.10321044921875,
|
383 |
+
"loss": 0.5219,
|
384 |
+
"rewards/accuracies": 0.7250000238418579,
|
385 |
+
"rewards/chosen": -0.686768651008606,
|
386 |
+
"rewards/margins": 0.7565571665763855,
|
387 |
+
"rewards/rejected": -1.4433258771896362,
|
388 |
"step": 210
|
389 |
},
|
390 |
{
|
391 |
"epoch": 0.5028571428571429,
|
392 |
+
"grad_norm": 19.18259747527528,
|
393 |
"learning_rate": 2.9078548506882117e-07,
|
394 |
+
"logits/chosen": -0.37646159529685974,
|
395 |
+
"logits/rejected": 0.516203761100769,
|
396 |
+
"logps/chosen": -383.25811767578125,
|
397 |
+
"logps/rejected": -413.0987243652344,
|
398 |
+
"loss": 0.5326,
|
399 |
+
"rewards/accuracies": 0.7250000238418579,
|
400 |
+
"rewards/chosen": -0.8871996998786926,
|
401 |
+
"rewards/margins": 0.6418129205703735,
|
402 |
+
"rewards/rejected": -1.529012680053711,
|
403 |
"step": 220
|
404 |
},
|
405 |
{
|
406 |
"epoch": 0.5257142857142857,
|
407 |
+
"grad_norm": 21.792119076799803,
|
408 |
"learning_rate": 2.709592897595191e-07,
|
409 |
+
"logits/chosen": -0.94196617603302,
|
410 |
+
"logits/rejected": -0.2367326021194458,
|
411 |
+
"logps/chosen": -343.68157958984375,
|
412 |
+
"logps/rejected": -391.78729248046875,
|
413 |
+
"loss": 0.5313,
|
414 |
+
"rewards/accuracies": 0.768750011920929,
|
415 |
+
"rewards/chosen": -0.583425760269165,
|
416 |
+
"rewards/margins": 0.6397222280502319,
|
417 |
+
"rewards/rejected": -1.2231481075286865,
|
418 |
"step": 230
|
419 |
},
|
420 |
{
|
421 |
"epoch": 0.5485714285714286,
|
422 |
+
"grad_norm": 23.250665470567046,
|
423 |
"learning_rate": 2.509992316440332e-07,
|
424 |
+
"logits/chosen": -0.7590802907943726,
|
425 |
+
"logits/rejected": 0.23232534527778625,
|
426 |
+
"logps/chosen": -375.7669982910156,
|
427 |
+
"logps/rejected": -393.1439208984375,
|
428 |
+
"loss": 0.5083,
|
429 |
+
"rewards/accuracies": 0.75,
|
430 |
+
"rewards/chosen": -0.661091685295105,
|
431 |
+
"rewards/margins": 0.7419286370277405,
|
432 |
+
"rewards/rejected": -1.4030205011367798,
|
433 |
"step": 240
|
434 |
},
|
435 |
{
|
436 |
"epoch": 0.5714285714285714,
|
437 |
+
"grad_norm": 23.283347220975337,
|
438 |
"learning_rate": 2.3103279163519918e-07,
|
439 |
+
"logits/chosen": 0.3902924656867981,
|
440 |
+
"logits/rejected": 1.2702767848968506,
|
441 |
+
"logps/chosen": -430.42510986328125,
|
442 |
+
"logps/rejected": -429.90240478515625,
|
443 |
+
"loss": 0.5665,
|
444 |
+
"rewards/accuracies": 0.668749988079071,
|
445 |
+
"rewards/chosen": -1.4317457675933838,
|
446 |
+
"rewards/margins": 0.5207899808883667,
|
447 |
+
"rewards/rejected": -1.95253586769104,
|
448 |
"step": 250
|
449 |
},
|
450 |
{
|
451 |
"epoch": 0.5714285714285714,
|
452 |
+
"eval_logits/chosen": 0.2878158688545227,
|
453 |
+
"eval_logits/rejected": 1.5253632068634033,
|
454 |
+
"eval_logps/chosen": -384.02301025390625,
|
455 |
+
"eval_logps/rejected": -415.1577453613281,
|
456 |
+
"eval_loss": 0.5404527187347412,
|
457 |
+
"eval_rewards/accuracies": 0.7629310488700867,
|
458 |
+
"eval_rewards/chosen": -1.0841907262802124,
|
459 |
+
"eval_rewards/margins": 0.8766254782676697,
|
460 |
+
"eval_rewards/rejected": -1.9608159065246582,
|
461 |
+
"eval_runtime": 91.5614,
|
462 |
+
"eval_samples_per_second": 19.998,
|
463 |
+
"eval_steps_per_second": 0.317,
|
464 |
"step": 250
|
465 |
},
|
466 |
{
|
467 |
"epoch": 0.5942857142857143,
|
468 |
+
"grad_norm": 28.334390897274133,
|
469 |
"learning_rate": 2.1118749140573358e-07,
|
470 |
+
"logits/chosen": -0.2290700227022171,
|
471 |
+
"logits/rejected": 0.8288987278938293,
|
472 |
+
"logps/chosen": -408.00201416015625,
|
473 |
+
"logps/rejected": -452.023681640625,
|
474 |
+
"loss": 0.5197,
|
475 |
+
"rewards/accuracies": 0.7875000238418579,
|
476 |
+
"rewards/chosen": -0.9032201766967773,
|
477 |
+
"rewards/margins": 0.8297585248947144,
|
478 |
+
"rewards/rejected": -1.7329788208007812,
|
479 |
"step": 260
|
480 |
},
|
481 |
{
|
482 |
"epoch": 0.6171428571428571,
|
483 |
+
"grad_norm": 23.854460344353054,
|
484 |
"learning_rate": 1.9159007893272703e-07,
|
485 |
+
"logits/chosen": -0.17421701550483704,
|
486 |
+
"logits/rejected": 0.6511275172233582,
|
487 |
+
"logps/chosen": -342.1952209472656,
|
488 |
+
"logps/rejected": -396.54083251953125,
|
489 |
+
"loss": 0.5294,
|
490 |
+
"rewards/accuracies": 0.706250011920929,
|
491 |
+
"rewards/chosen": -0.7613161206245422,
|
492 |
+
"rewards/margins": 0.7101620435714722,
|
493 |
+
"rewards/rejected": -1.4714782238006592,
|
494 |
"step": 270
|
495 |
},
|
496 |
{
|
497 |
"epoch": 0.64,
|
498 |
+
"grad_norm": 27.414501100794606,
|
499 |
"learning_rate": 1.7236571898357766e-07,
|
500 |
+
"logits/chosen": -0.08564956486225128,
|
501 |
+
"logits/rejected": 0.9427372813224792,
|
502 |
+
"logps/chosen": -367.49407958984375,
|
503 |
+
"logps/rejected": -409.3499755859375,
|
504 |
+
"loss": 0.518,
|
505 |
+
"rewards/accuracies": 0.7562500238418579,
|
506 |
+
"rewards/chosen": -0.8473467826843262,
|
507 |
+
"rewards/margins": 0.8089747428894043,
|
508 |
+
"rewards/rejected": -1.6563212871551514,
|
509 |
"step": 280
|
510 |
},
|
511 |
{
|
512 |
"epoch": 0.6628571428571428,
|
513 |
+
"grad_norm": 23.166387688948994,
|
514 |
"learning_rate": 1.5363719371356882e-07,
|
515 |
+
"logits/chosen": 0.06339935958385468,
|
516 |
+
"logits/rejected": 0.5719184875488281,
|
517 |
+
"logps/chosen": -360.3900451660156,
|
518 |
+
"logps/rejected": -409.3319091796875,
|
519 |
+
"loss": 0.5387,
|
520 |
+
"rewards/accuracies": 0.71875,
|
521 |
+
"rewards/chosen": -0.9427051544189453,
|
522 |
+
"rewards/margins": 0.5575781464576721,
|
523 |
+
"rewards/rejected": -1.500283122062683,
|
524 |
"step": 290
|
525 |
},
|
526 |
{
|
527 |
"epoch": 0.6857142857142857,
|
528 |
+
"grad_norm": 28.449941474840944,
|
529 |
"learning_rate": 1.3552411848071565e-07,
|
530 |
+
"logits/chosen": -0.3682107627391815,
|
531 |
+
"logits/rejected": 0.5333132743835449,
|
532 |
+
"logps/chosen": -333.92449951171875,
|
533 |
+
"logps/rejected": -425.3599548339844,
|
534 |
+
"loss": 0.5202,
|
535 |
+
"rewards/accuracies": 0.7749999761581421,
|
536 |
+
"rewards/chosen": -0.7675741314888,
|
537 |
+
"rewards/margins": 0.9044278264045715,
|
538 |
+
"rewards/rejected": -1.6720020771026611,
|
539 |
"step": 300
|
540 |
},
|
541 |
{
|
542 |
"epoch": 0.6857142857142857,
|
543 |
+
"eval_logits/chosen": -0.21915225684642792,
|
544 |
+
"eval_logits/rejected": 0.9934114813804626,
|
545 |
+
"eval_logps/chosen": -349.8433837890625,
|
546 |
+
"eval_logps/rejected": -392.549560546875,
|
547 |
+
"eval_loss": 0.5274777412414551,
|
548 |
+
"eval_rewards/accuracies": 0.7715517282485962,
|
549 |
+
"eval_rewards/chosen": -0.7423940300941467,
|
550 |
+
"eval_rewards/margins": 0.9923400282859802,
|
551 |
+
"eval_rewards/rejected": -1.734734058380127,
|
552 |
+
"eval_runtime": 90.6157,
|
553 |
+
"eval_samples_per_second": 20.206,
|
554 |
+
"eval_steps_per_second": 0.32,
|
555 |
"step": 300
|
556 |
},
|
557 |
{
|
558 |
"epoch": 0.7085714285714285,
|
559 |
+
"grad_norm": 21.530490280501045,
|
560 |
"learning_rate": 1.1814217788631473e-07,
|
561 |
+
"logits/chosen": -0.17677690088748932,
|
562 |
+
"logits/rejected": 0.3225722908973694,
|
563 |
+
"logps/chosen": -356.5892639160156,
|
564 |
+
"logps/rejected": -431.662109375,
|
565 |
+
"loss": 0.5328,
|
566 |
"rewards/accuracies": 0.675000011920929,
|
567 |
+
"rewards/chosen": -0.8150566816329956,
|
568 |
+
"rewards/margins": 0.6877792477607727,
|
569 |
+
"rewards/rejected": -1.502835988998413,
|
570 |
"step": 310
|
571 |
},
|
572 |
{
|
573 |
"epoch": 0.7314285714285714,
|
574 |
+
"grad_norm": 26.395907969219987,
|
575 |
"learning_rate": 1.0160238692045331e-07,
|
576 |
+
"logits/chosen": -0.2509198486804962,
|
577 |
+
"logits/rejected": 0.4886396527290344,
|
578 |
+
"logps/chosen": -359.4185485839844,
|
579 |
+
"logps/rejected": -413.3671875,
|
580 |
+
"loss": 0.535,
|
581 |
+
"rewards/accuracies": 0.71875,
|
582 |
+
"rewards/chosen": -0.8422183990478516,
|
583 |
+
"rewards/margins": 0.7798849940299988,
|
584 |
+
"rewards/rejected": -1.6221033334732056,
|
585 |
"step": 320
|
586 |
},
|
587 |
{
|
588 |
"epoch": 0.7542857142857143,
|
589 |
+
"grad_norm": 25.409253983308545,
|
590 |
"learning_rate": 8.601038193139438e-08,
|
591 |
+
"logits/chosen": -0.13014790415763855,
|
592 |
+
"logits/rejected": 0.5298113822937012,
|
593 |
+
"logps/chosen": -378.82012939453125,
|
594 |
+
"logps/rejected": -402.17156982421875,
|
595 |
+
"loss": 0.5302,
|
596 |
+
"rewards/accuracies": 0.6937500238418579,
|
597 |
+
"rewards/chosen": -0.973560631275177,
|
598 |
+
"rewards/margins": 0.6031589508056641,
|
599 |
+
"rewards/rejected": -1.5767196416854858,
|
600 |
"step": 330
|
601 |
},
|
602 |
{
|
603 |
"epoch": 0.7771428571428571,
|
604 |
+
"grad_norm": 26.91341277336219,
|
605 |
"learning_rate": 7.146574594727572e-08,
|
606 |
+
"logits/chosen": -0.4229533076286316,
|
607 |
+
"logits/rejected": 0.5619645714759827,
|
608 |
+
"logps/chosen": -363.5431213378906,
|
609 |
+
"logps/rejected": -404.77935791015625,
|
610 |
+
"loss": 0.5192,
|
611 |
+
"rewards/accuracies": 0.71875,
|
612 |
+
"rewards/chosen": -0.8485255241394043,
|
613 |
+
"rewards/margins": 0.8047366142272949,
|
614 |
+
"rewards/rejected": -1.6532618999481201,
|
615 |
"step": 340
|
616 |
},
|
617 |
{
|
618 |
"epoch": 0.8,
|
619 |
+
"grad_norm": 24.565353732521466,
|
620 |
"learning_rate": 5.8061372659157306e-08,
|
621 |
+
"logits/chosen": -0.17489977180957794,
|
622 |
+
"logits/rejected": 0.7508963346481323,
|
623 |
+
"logps/chosen": -371.7098693847656,
|
624 |
+
"logps/rejected": -395.7353210449219,
|
625 |
+
"loss": 0.5261,
|
626 |
+
"rewards/accuracies": 0.6625000238418579,
|
627 |
+
"rewards/chosen": -0.8898698091506958,
|
628 |
+
"rewards/margins": 0.6660181879997253,
|
629 |
+
"rewards/rejected": -1.5558879375457764,
|
630 |
"step": 350
|
631 |
},
|
632 |
{
|
633 |
"epoch": 0.8,
|
634 |
+
"eval_logits/chosen": -0.2696850597858429,
|
635 |
+
"eval_logits/rejected": 0.9007923007011414,
|
636 |
+
"eval_logps/chosen": -351.73101806640625,
|
637 |
+
"eval_logps/rejected": -391.03973388671875,
|
638 |
+
"eval_loss": 0.5234382748603821,
|
639 |
+
"eval_rewards/accuracies": 0.7586206793785095,
|
640 |
+
"eval_rewards/chosen": -0.76127028465271,
|
641 |
+
"eval_rewards/margins": 0.9583660364151001,
|
642 |
+
"eval_rewards/rejected": -1.71963632106781,
|
643 |
+
"eval_runtime": 90.1922,
|
644 |
+
"eval_samples_per_second": 20.301,
|
645 |
+
"eval_steps_per_second": 0.322,
|
646 |
"step": 350
|
647 |
},
|
648 |
{
|
649 |
"epoch": 0.8228571428571428,
|
650 |
+
"grad_norm": 26.528490406026805,
|
651 |
"learning_rate": 4.5882873127531614e-08,
|
652 |
+
"logits/chosen": -0.18714679777622223,
|
653 |
+
"logits/rejected": 0.5642833113670349,
|
654 |
+
"logps/chosen": -389.9281921386719,
|
655 |
+
"logps/rejected": -425.6780700683594,
|
656 |
+
"loss": 0.5053,
|
657 |
+
"rewards/accuracies": 0.71875,
|
658 |
+
"rewards/chosen": -0.8513727188110352,
|
659 |
+
"rewards/margins": 0.6271126866340637,
|
660 |
+
"rewards/rejected": -1.478485345840454,
|
661 |
"step": 360
|
662 |
},
|
663 |
{
|
664 |
"epoch": 0.8457142857142858,
|
665 |
+
"grad_norm": 26.160208453826513,
|
666 |
"learning_rate": 3.500802900154412e-08,
|
667 |
+
"logits/chosen": -0.11366554349660873,
|
668 |
+
"logits/rejected": 0.7036272287368774,
|
669 |
+
"logps/chosen": -315.89349365234375,
|
670 |
+
"logps/rejected": -380.2809143066406,
|
671 |
+
"loss": 0.5279,
|
672 |
+
"rewards/accuracies": 0.6937500238418579,
|
673 |
+
"rewards/chosen": -0.7633152008056641,
|
674 |
+
"rewards/margins": 0.7076437473297119,
|
675 |
+
"rewards/rejected": -1.470958948135376,
|
676 |
"step": 370
|
677 |
},
|
678 |
{
|
679 |
"epoch": 0.8685714285714285,
|
680 |
+
"grad_norm": 22.845593504615394,
|
681 |
"learning_rate": 2.550629574310309e-08,
|
682 |
+
"logits/chosen": -0.2266564667224884,
|
683 |
+
"logits/rejected": 0.5173102021217346,
|
684 |
+
"logps/chosen": -349.7990417480469,
|
685 |
+
"logps/rejected": -402.1055603027344,
|
686 |
+
"loss": 0.5236,
|
687 |
+
"rewards/accuracies": 0.6937500238418579,
|
688 |
+
"rewards/chosen": -0.7807506918907166,
|
689 |
+
"rewards/margins": 0.665580689907074,
|
690 |
+
"rewards/rejected": -1.4463313817977905,
|
691 |
"step": 380
|
692 |
},
|
693 |
{
|
694 |
"epoch": 0.8914285714285715,
|
695 |
+
"grad_norm": 22.30415183791987,
|
696 |
"learning_rate": 1.7438359028687983e-08,
|
697 |
+
"logits/chosen": -0.18654844164848328,
|
698 |
+
"logits/rejected": 0.17555546760559082,
|
699 |
+
"logps/chosen": -343.80609130859375,
|
700 |
+
"logps/rejected": -413.258544921875,
|
701 |
+
"loss": 0.5215,
|
702 |
+
"rewards/accuracies": 0.6875,
|
703 |
+
"rewards/chosen": -0.8466179966926575,
|
704 |
+
"rewards/margins": 0.5847481489181519,
|
705 |
+
"rewards/rejected": -1.431365966796875,
|
706 |
"step": 390
|
707 |
},
|
708 |
{
|
709 |
"epoch": 0.9142857142857143,
|
710 |
+
"grad_norm": 25.36293321655948,
|
711 |
"learning_rate": 1.0855747162029361e-08,
|
712 |
+
"logits/chosen": -0.39323678612709045,
|
713 |
+
"logits/rejected": 0.6321589946746826,
|
714 |
+
"logps/chosen": -364.44781494140625,
|
715 |
+
"logps/rejected": -393.2674865722656,
|
716 |
+
"loss": 0.5343,
|
717 |
+
"rewards/accuracies": 0.731249988079071,
|
718 |
+
"rewards/chosen": -0.7503901720046997,
|
719 |
+
"rewards/margins": 0.7446303963661194,
|
720 |
+
"rewards/rejected": -1.4950206279754639,
|
721 |
"step": 400
|
722 |
},
|
723 |
{
|
724 |
"epoch": 0.9142857142857143,
|
725 |
+
"eval_logits/chosen": -0.3663737177848816,
|
726 |
+
"eval_logits/rejected": 0.7649080157279968,
|
727 |
+
"eval_logps/chosen": -345.4830017089844,
|
728 |
+
"eval_logps/rejected": -382.0352478027344,
|
729 |
+
"eval_loss": 0.5226701498031616,
|
730 |
+
"eval_rewards/accuracies": 0.7629310488700867,
|
731 |
+
"eval_rewards/chosen": -0.6987906098365784,
|
732 |
+
"eval_rewards/margins": 0.9308006763458252,
|
733 |
+
"eval_rewards/rejected": -1.6295913457870483,
|
734 |
+
"eval_runtime": 91.5738,
|
735 |
+
"eval_samples_per_second": 19.995,
|
736 |
+
"eval_steps_per_second": 0.317,
|
737 |
"step": 400
|
738 |
},
|
739 |
{
|
740 |
"epoch": 0.9371428571428572,
|
741 |
+
"grad_norm": 21.80106996482806,
|
742 |
"learning_rate": 5.8005019731033615e-09,
|
743 |
+
"logits/chosen": -0.4477500021457672,
|
744 |
+
"logits/rejected": 0.4871234893798828,
|
745 |
+
"logps/chosen": -340.3522033691406,
|
746 |
+
"logps/rejected": -401.26629638671875,
|
747 |
+
"loss": 0.5153,
|
748 |
+
"rewards/accuracies": 0.7875000238418579,
|
749 |
+
"rewards/chosen": -0.6757135391235352,
|
750 |
+
"rewards/margins": 0.9294708371162415,
|
751 |
+
"rewards/rejected": -1.6051843166351318,
|
752 |
"step": 410
|
753 |
},
|
754 |
{
|
755 |
"epoch": 0.96,
|
756 |
+
"grad_norm": 23.103516202896476,
|
757 |
"learning_rate": 2.3049103053431886e-09,
|
758 |
+
"logits/chosen": -0.3995040953159332,
|
759 |
+
"logits/rejected": 0.3316659927368164,
|
760 |
+
"logps/chosen": -386.6695861816406,
|
761 |
+
"logps/rejected": -383.6183776855469,
|
762 |
+
"loss": 0.5348,
|
763 |
+
"rewards/accuracies": 0.6812499761581421,
|
764 |
+
"rewards/chosen": -0.7931796312332153,
|
765 |
+
"rewards/margins": 0.6191332936286926,
|
766 |
+
"rewards/rejected": -1.4123131036758423,
|
767 |
"step": 420
|
768 |
},
|
769 |
{
|
770 |
"epoch": 0.9828571428571429,
|
771 |
+
"grad_norm": 22.91261742161674,
|
772 |
"learning_rate": 3.9129780600541397e-10,
|
773 |
+
"logits/chosen": -0.2376430332660675,
|
774 |
+
"logits/rejected": 0.6147540807723999,
|
775 |
+
"logps/chosen": -368.62451171875,
|
776 |
+
"logps/rejected": -394.5757751464844,
|
777 |
+
"loss": 0.5369,
|
778 |
+
"rewards/accuracies": 0.7124999761581421,
|
779 |
+
"rewards/chosen": -0.8315681219100952,
|
780 |
+
"rewards/margins": 0.7056232690811157,
|
781 |
+
"rewards/rejected": -1.53719162940979,
|
782 |
"step": 430
|
783 |
},
|
784 |
{
|
785 |
"epoch": 0.9988571428571429,
|
786 |
"step": 437,
|
787 |
"total_flos": 0.0,
|
788 |
+
"train_loss": 0.5600041279258117,
|
789 |
+
"train_runtime": 11271.3049,
|
790 |
+
"train_samples_per_second": 4.968,
|
791 |
"train_steps_per_second": 0.039
|
792 |
}
|
793 |
],
|
training_args.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 7608
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:c6bf17a6b11cca2cbecdb0160512e7fcc2ca49ba133f6a4311e37f4633e4bf58
|
3 |
size 7608
|