| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.3066557778685177, | |
| "eval_steps": 500, | |
| "global_step": 400, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.016333197223356473, | |
| "grad_norm": 1.1593419313430786, | |
| "learning_rate": 4.9996340229869763e-05, | |
| "logits/chosen": -1.0346699953079224, | |
| "logits/rejected": -2.6695446968078613, | |
| "logps/chosen": -1.101169466972351, | |
| "logps/rejected": -1.0235556364059448, | |
| "loss": 1.178, | |
| "odds_ratio_loss": 0.768240213394165, | |
| "rewards/accuracies": 0.3499999940395355, | |
| "rewards/chosen": -0.11011694371700287, | |
| "rewards/margins": -0.007761377841234207, | |
| "rewards/rejected": -0.10235557705163956, | |
| "sft_loss": 1.101169466972351, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.032666394446712946, | |
| "grad_norm": 1.1402696371078491, | |
| "learning_rate": 4.9985361990992455e-05, | |
| "logits/chosen": -0.8193758726119995, | |
| "logits/rejected": -2.842833995819092, | |
| "logps/chosen": -1.0217273235321045, | |
| "logps/rejected": -1.0100481510162354, | |
| "loss": 1.0927, | |
| "odds_ratio_loss": 0.7092453837394714, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.10217274725437164, | |
| "rewards/margins": -0.0011679243762046099, | |
| "rewards/rejected": -0.10100481659173965, | |
| "sft_loss": 1.0217273235321045, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.04899959167006942, | |
| "grad_norm": 1.3554140329360962, | |
| "learning_rate": 4.996706849759453e-05, | |
| "logits/chosen": -0.7696830034255981, | |
| "logits/rejected": -2.703421115875244, | |
| "logps/chosen": -1.0294106006622314, | |
| "logps/rejected": -1.0599125623703003, | |
| "loss": 1.0993, | |
| "odds_ratio_loss": 0.6984036564826965, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.10294107347726822, | |
| "rewards/margins": 0.003050185739994049, | |
| "rewards/rejected": -0.10599125921726227, | |
| "sft_loss": 1.0294106006622314, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.06533278889342589, | |
| "grad_norm": 1.406853199005127, | |
| "learning_rate": 4.9941465105674435e-05, | |
| "logits/chosen": -0.698488712310791, | |
| "logits/rejected": -2.808516263961792, | |
| "logps/chosen": -1.0651928186416626, | |
| "logps/rejected": -0.996777355670929, | |
| "loss": 1.1402, | |
| "odds_ratio_loss": 0.7504132390022278, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.10651928186416626, | |
| "rewards/margins": -0.006841552909463644, | |
| "rewards/rejected": -0.09967772662639618, | |
| "sft_loss": 1.0651928186416626, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.08166598611678236, | |
| "grad_norm": 1.238513469696045, | |
| "learning_rate": 4.99085593114345e-05, | |
| "logits/chosen": -0.6306881904602051, | |
| "logits/rejected": -2.698362350463867, | |
| "logps/chosen": -1.0242538452148438, | |
| "logps/rejected": -1.0320734977722168, | |
| "loss": 1.0937, | |
| "odds_ratio_loss": 0.6945221424102783, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.10242538154125214, | |
| "rewards/margins": 0.0007819652673788369, | |
| "rewards/rejected": -0.10320734977722168, | |
| "sft_loss": 1.0242538452148438, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.09799918334013884, | |
| "grad_norm": 1.9588699340820312, | |
| "learning_rate": 4.986836074908616e-05, | |
| "logits/chosen": -0.5534936189651489, | |
| "logits/rejected": -2.678889036178589, | |
| "logps/chosen": -1.1003000736236572, | |
| "logps/rejected": -1.0425399541854858, | |
| "loss": 1.1745, | |
| "odds_ratio_loss": 0.7416313290596008, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.11003003269433975, | |
| "rewards/margins": -0.005776026751846075, | |
| "rewards/rejected": -0.10425399243831635, | |
| "sft_loss": 1.1003000736236572, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.1143323805634953, | |
| "grad_norm": 1.1857821941375732, | |
| "learning_rate": 4.982088118802922e-05, | |
| "logits/chosen": -0.6129628419876099, | |
| "logits/rejected": -2.876364231109619, | |
| "logps/chosen": -0.9628928303718567, | |
| "logps/rejected": -0.952060341835022, | |
| "loss": 1.0336, | |
| "odds_ratio_loss": 0.7068274617195129, | |
| "rewards/accuracies": 0.42500001192092896, | |
| "rewards/chosen": -0.09628929197788239, | |
| "rewards/margins": -0.001083258306607604, | |
| "rewards/rejected": -0.09520602226257324, | |
| "sft_loss": 0.9628928303718567, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.13066557778685178, | |
| "grad_norm": 1.33924400806427, | |
| "learning_rate": 4.976613452940604e-05, | |
| "logits/chosen": -0.7091814279556274, | |
| "logits/rejected": -2.6595664024353027, | |
| "logps/chosen": -0.89775550365448, | |
| "logps/rejected": -0.935952365398407, | |
| "loss": 0.9642, | |
| "odds_ratio_loss": 0.6644185781478882, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.08977555483579636, | |
| "rewards/margins": 0.003819683101028204, | |
| "rewards/rejected": -0.0935952290892601, | |
| "sft_loss": 0.89775550365448, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.14699877501020825, | |
| "grad_norm": 1.136667013168335, | |
| "learning_rate": 4.9704136802031485e-05, | |
| "logits/chosen": -0.7233263850212097, | |
| "logits/rejected": -2.6673388481140137, | |
| "logps/chosen": -0.9036429524421692, | |
| "logps/rejected": -0.9419530630111694, | |
| "loss": 0.9699, | |
| "odds_ratio_loss": 0.662254273891449, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.09036429226398468, | |
| "rewards/margins": 0.0038310133386403322, | |
| "rewards/rejected": -0.09419531375169754, | |
| "sft_loss": 0.9036429524421692, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.16333197223356472, | |
| "grad_norm": 1.4290399551391602, | |
| "learning_rate": 4.9634906157700036e-05, | |
| "logits/chosen": -0.6113058924674988, | |
| "logits/rejected": -2.705373764038086, | |
| "logps/chosen": -0.9550830125808716, | |
| "logps/rejected": -0.9817631840705872, | |
| "loss": 1.0223, | |
| "odds_ratio_loss": 0.6725824475288391, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.09550830721855164, | |
| "rewards/margins": 0.002668013097718358, | |
| "rewards/rejected": -0.09817632287740707, | |
| "sft_loss": 0.9550830125808716, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.17966516945692118, | |
| "grad_norm": 1.4945589303970337, | |
| "learning_rate": 4.955846286587122e-05, | |
| "logits/chosen": -0.647710919380188, | |
| "logits/rejected": -2.7208316326141357, | |
| "logps/chosen": -0.9462896585464478, | |
| "logps/rejected": -0.9507497549057007, | |
| "loss": 1.0148, | |
| "odds_ratio_loss": 0.6852728128433228, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.0946289673447609, | |
| "rewards/margins": 0.0004460157942958176, | |
| "rewards/rejected": -0.09507497400045395, | |
| "sft_loss": 0.9462896585464478, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.19599836668027767, | |
| "grad_norm": 1.3553156852722168, | |
| "learning_rate": 4.9474829307735115e-05, | |
| "logits/chosen": -0.6279743909835815, | |
| "logits/rejected": -2.6854496002197266, | |
| "logps/chosen": -0.8896121978759766, | |
| "logps/rejected": -0.9993942975997925, | |
| "loss": 0.9509, | |
| "odds_ratio_loss": 0.6129180192947388, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.08896122127771378, | |
| "rewards/margins": 0.010978218168020248, | |
| "rewards/rejected": -0.09993942826986313, | |
| "sft_loss": 0.8896121978759766, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.21233156390363414, | |
| "grad_norm": 1.4247411489486694, | |
| "learning_rate": 4.9384029969659573e-05, | |
| "logits/chosen": -0.698381781578064, | |
| "logits/rejected": -2.7967753410339355, | |
| "logps/chosen": -0.9455701112747192, | |
| "logps/rejected": -0.9564310908317566, | |
| "loss": 1.0147, | |
| "odds_ratio_loss": 0.6911824941635132, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0945570170879364, | |
| "rewards/margins": 0.0010860899928957224, | |
| "rewards/rejected": -0.09564311057329178, | |
| "sft_loss": 0.9455701112747192, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.2286647611269906, | |
| "grad_norm": 1.6154508590698242, | |
| "learning_rate": 4.9286091436021015e-05, | |
| "logits/chosen": -0.595032274723053, | |
| "logits/rejected": -2.740784168243408, | |
| "logps/chosen": -0.9214372634887695, | |
| "logps/rejected": -1.0210167169570923, | |
| "loss": 0.9836, | |
| "odds_ratio_loss": 0.6212525963783264, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.09214373677968979, | |
| "rewards/margins": 0.00995794590562582, | |
| "rewards/rejected": -0.10210166871547699, | |
| "sft_loss": 0.9214372634887695, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.24499795835034707, | |
| "grad_norm": 1.8231102228164673, | |
| "learning_rate": 4.918104238142104e-05, | |
| "logits/chosen": -0.5186889171600342, | |
| "logits/rejected": -2.7576358318328857, | |
| "logps/chosen": -0.9789311289787292, | |
| "logps/rejected": -1.0169849395751953, | |
| "loss": 1.0454, | |
| "odds_ratio_loss": 0.6644267439842224, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.09789310395717621, | |
| "rewards/margins": 0.0038053765892982483, | |
| "rewards/rejected": -0.10169849544763565, | |
| "sft_loss": 0.9789311289787292, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.26133115557370357, | |
| "grad_norm": 2.05936336517334, | |
| "learning_rate": 4.906891356229103e-05, | |
| "logits/chosen": -0.47254711389541626, | |
| "logits/rejected": -2.68280029296875, | |
| "logps/chosen": -0.9620391726493835, | |
| "logps/rejected": -1.0026928186416626, | |
| "loss": 1.0294, | |
| "odds_ratio_loss": 0.6731926202774048, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09620390832424164, | |
| "rewards/margins": 0.004065364599227905, | |
| "rewards/rejected": -0.10026928037405014, | |
| "sft_loss": 0.9620391726493835, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.27766435279706003, | |
| "grad_norm": 1.6302626132965088, | |
| "learning_rate": 4.894973780788722e-05, | |
| "logits/chosen": -0.5315309762954712, | |
| "logits/rejected": -2.7227425575256348, | |
| "logps/chosen": -0.9423310160636902, | |
| "logps/rejected": -1.071781039237976, | |
| "loss": 1.0032, | |
| "odds_ratio_loss": 0.6090766191482544, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.09423311054706573, | |
| "rewards/margins": 0.012944999150931835, | |
| "rewards/rejected": -0.10717810690402985, | |
| "sft_loss": 0.9423310160636902, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.2939975500204165, | |
| "grad_norm": 1.7891793251037598, | |
| "learning_rate": 4.882355001067892e-05, | |
| "logits/chosen": -0.5275685787200928, | |
| "logits/rejected": -2.6127567291259766, | |
| "logps/chosen": -0.9351356625556946, | |
| "logps/rejected": -1.0083874464035034, | |
| "loss": 0.9995, | |
| "odds_ratio_loss": 0.6435801386833191, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.09351354837417603, | |
| "rewards/margins": 0.0073251971043646336, | |
| "rewards/rejected": -0.10083875805139542, | |
| "sft_loss": 0.9351356625556946, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.31033074724377296, | |
| "grad_norm": 1.852216124534607, | |
| "learning_rate": 4.869038711613259e-05, | |
| "logits/chosen": -0.5760647058486938, | |
| "logits/rejected": -2.722364664077759, | |
| "logps/chosen": -0.9301847219467163, | |
| "logps/rejected": -1.0749822854995728, | |
| "loss": 0.9903, | |
| "odds_ratio_loss": 0.601297914981842, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.09301847964525223, | |
| "rewards/margins": 0.014479748904705048, | |
| "rewards/rejected": -0.10749821364879608, | |
| "sft_loss": 0.9301847219467163, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.32666394446712943, | |
| "grad_norm": 1.8597307205200195, | |
| "learning_rate": 4.855028811189496e-05, | |
| "logits/chosen": -0.5871164202690125, | |
| "logits/rejected": -2.6969285011291504, | |
| "logps/chosen": -1.0701930522918701, | |
| "logps/rejected": -1.0556358098983765, | |
| "loss": 1.1427, | |
| "odds_ratio_loss": 0.7247555255889893, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.10701930522918701, | |
| "rewards/margins": -0.0014557272661477327, | |
| "rewards/rejected": -0.10556356608867645, | |
| "sft_loss": 1.0701930522918701, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.3429971416904859, | |
| "grad_norm": 1.6869758367538452, | |
| "learning_rate": 4.8403294016378095e-05, | |
| "logits/chosen": -0.4466473460197449, | |
| "logits/rejected": -2.501394510269165, | |
| "logps/chosen": -0.880301833152771, | |
| "logps/rejected": -1.0003767013549805, | |
| "loss": 0.941, | |
| "odds_ratio_loss": 0.60676509141922, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.08803018927574158, | |
| "rewards/margins": 0.012007484212517738, | |
| "rewards/rejected": -0.10003767162561417, | |
| "sft_loss": 0.880301833152771, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 0.35933033891384236, | |
| "grad_norm": 1.8963792324066162, | |
| "learning_rate": 4.8249447866750025e-05, | |
| "logits/chosen": -0.41199809312820435, | |
| "logits/rejected": -2.594975471496582, | |
| "logps/chosen": -0.8528003692626953, | |
| "logps/rejected": -0.9772283434867859, | |
| "loss": 0.9128, | |
| "odds_ratio_loss": 0.6004671454429626, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.08528003096580505, | |
| "rewards/margins": 0.012442806735634804, | |
| "rewards/rejected": -0.0977228432893753, | |
| "sft_loss": 0.8528003692626953, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.3756635361371989, | |
| "grad_norm": 2.6958725452423096, | |
| "learning_rate": 4.808879470633417e-05, | |
| "logits/chosen": -0.43985629081726074, | |
| "logits/rejected": -2.6999452114105225, | |
| "logps/chosen": -0.9486901164054871, | |
| "logps/rejected": -1.0987169742584229, | |
| "loss": 1.0076, | |
| "odds_ratio_loss": 0.5889819264411926, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.09486902505159378, | |
| "rewards/margins": 0.015002680011093616, | |
| "rewards/rejected": -0.10987170040607452, | |
| "sft_loss": 0.9486901164054871, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 0.39199673336055535, | |
| "grad_norm": 1.6014004945755005, | |
| "learning_rate": 4.792138157142158e-05, | |
| "logits/chosen": -0.43066906929016113, | |
| "logits/rejected": -2.634669065475464, | |
| "logps/chosen": -0.8806023597717285, | |
| "logps/rejected": -1.0933107137680054, | |
| "loss": 0.9352, | |
| "odds_ratio_loss": 0.5455727577209473, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.08806024491786957, | |
| "rewards/margins": 0.021270830184221268, | |
| "rewards/rejected": -0.10933107137680054, | |
| "sft_loss": 0.8806023597717285, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.4083299305839118, | |
| "grad_norm": 3.4042723178863525, | |
| "learning_rate": 4.7747257477499474e-05, | |
| "logits/chosen": -0.4240710735321045, | |
| "logits/rejected": -2.664668560028076, | |
| "logps/chosen": -0.8993316888809204, | |
| "logps/rejected": -1.0890860557556152, | |
| "loss": 0.9553, | |
| "odds_ratio_loss": 0.5601183772087097, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.08993317186832428, | |
| "rewards/margins": 0.018975432962179184, | |
| "rewards/rejected": -0.10890860855579376, | |
| "sft_loss": 0.8993316888809204, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 0.4246631278072683, | |
| "grad_norm": 1.724076509475708, | |
| "learning_rate": 4.75664734049005e-05, | |
| "logits/chosen": -0.3910972476005554, | |
| "logits/rejected": -2.571296215057373, | |
| "logps/chosen": -0.9113162755966187, | |
| "logps/rejected": -1.094262719154358, | |
| "loss": 0.9682, | |
| "odds_ratio_loss": 0.5683911442756653, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.09113162755966187, | |
| "rewards/margins": 0.018294643610715866, | |
| "rewards/rejected": -0.10942627489566803, | |
| "sft_loss": 0.9113162755966187, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.44099632503062475, | |
| "grad_norm": 1.6363219022750854, | |
| "learning_rate": 4.7379082283876566e-05, | |
| "logits/chosen": -0.5021435022354126, | |
| "logits/rejected": -2.6572070121765137, | |
| "logps/chosen": -0.8906953930854797, | |
| "logps/rejected": -1.0344655513763428, | |
| "loss": 0.9498, | |
| "odds_ratio_loss": 0.5914202928543091, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.08906953036785126, | |
| "rewards/margins": 0.014377007260918617, | |
| "rewards/rejected": -0.10344654321670532, | |
| "sft_loss": 0.8906953930854797, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 0.4573295222539812, | |
| "grad_norm": 1.7812143564224243, | |
| "learning_rate": 4.7185138979101864e-05, | |
| "logits/chosen": -0.507682740688324, | |
| "logits/rejected": -2.593475580215454, | |
| "logps/chosen": -0.9167561531066895, | |
| "logps/rejected": -1.1608246564865112, | |
| "loss": 0.9698, | |
| "odds_ratio_loss": 0.5303933024406433, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.09167560935020447, | |
| "rewards/margins": 0.024406855925917625, | |
| "rewards/rejected": -0.11608246713876724, | |
| "sft_loss": 0.9167561531066895, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.4736627194773377, | |
| "grad_norm": 1.823789119720459, | |
| "learning_rate": 4.6984700273609495e-05, | |
| "logits/chosen": -0.3446227014064789, | |
| "logits/rejected": -2.5670173168182373, | |
| "logps/chosen": -0.8484543561935425, | |
| "logps/rejected": -1.0493099689483643, | |
| "loss": 0.903, | |
| "odds_ratio_loss": 0.5457809567451477, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.08484544605016708, | |
| "rewards/margins": 0.020085560157895088, | |
| "rewards/rejected": -0.10493101179599762, | |
| "sft_loss": 0.8484543561935425, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 0.48999591670069415, | |
| "grad_norm": 1.889367699623108, | |
| "learning_rate": 4.677782485216644e-05, | |
| "logits/chosen": -0.44083428382873535, | |
| "logits/rejected": -2.573638439178467, | |
| "logps/chosen": -0.833287239074707, | |
| "logps/rejected": -1.1050159931182861, | |
| "loss": 0.8838, | |
| "odds_ratio_loss": 0.5049694180488586, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.0833287239074707, | |
| "rewards/margins": 0.02717287465929985, | |
| "rewards/rejected": -0.11050160229206085, | |
| "sft_loss": 0.833287239074707, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.5063291139240507, | |
| "grad_norm": 1.9978969097137451, | |
| "learning_rate": 4.6564573284091765e-05, | |
| "logits/chosen": -0.37475016713142395, | |
| "logits/rejected": -2.530078887939453, | |
| "logps/chosen": -0.9055852890014648, | |
| "logps/rejected": -1.3165007829666138, | |
| "loss": 0.9498, | |
| "odds_ratio_loss": 0.4424561858177185, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.09055852890014648, | |
| "rewards/margins": 0.04109155014157295, | |
| "rewards/rejected": -0.13165007531642914, | |
| "sft_loss": 0.9055852890014648, | |
| "step": 155 | |
| }, | |
| { | |
| "epoch": 0.5226623111474071, | |
| "grad_norm": 2.2659225463867188, | |
| "learning_rate": 4.6345008005522966e-05, | |
| "logits/chosen": -0.27824097871780396, | |
| "logits/rejected": -2.1937553882598877, | |
| "logps/chosen": -0.7978348731994629, | |
| "logps/rejected": -1.7242333889007568, | |
| "loss": 0.8243, | |
| "odds_ratio_loss": 0.26432332396507263, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07978348433971405, | |
| "rewards/margins": 0.09263985604047775, | |
| "rewards/rejected": -0.1724233478307724, | |
| "sft_loss": 0.7978348731994629, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.5389955083707636, | |
| "grad_norm": 2.497178792953491, | |
| "learning_rate": 4.611919330113592e-05, | |
| "logits/chosen": -0.1297352910041809, | |
| "logits/rejected": -1.7525418996810913, | |
| "logps/chosen": -0.9057181477546692, | |
| "logps/rejected": -2.926887035369873, | |
| "loss": 0.9163, | |
| "odds_ratio_loss": 0.10600633919239044, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0905718058347702, | |
| "rewards/margins": 0.20211689174175262, | |
| "rewards/rejected": -0.2926887273788452, | |
| "sft_loss": 0.9057181477546692, | |
| "step": 165 | |
| }, | |
| { | |
| "epoch": 0.5553287055941201, | |
| "grad_norm": 1.9597887992858887, | |
| "learning_rate": 4.588719528532342e-05, | |
| "logits/chosen": -0.20320752263069153, | |
| "logits/rejected": -1.4767993688583374, | |
| "logps/chosen": -0.9343756437301636, | |
| "logps/rejected": -3.7731080055236816, | |
| "loss": 0.9406, | |
| "odds_ratio_loss": 0.062325619161129, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0934375673532486, | |
| "rewards/margins": 0.28387323021888733, | |
| "rewards/rejected": -0.3773108124732971, | |
| "sft_loss": 0.9343756437301636, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.5716619028174765, | |
| "grad_norm": 1.9177374839782715, | |
| "learning_rate": 4.564908188283814e-05, | |
| "logits/chosen": -0.16985416412353516, | |
| "logits/rejected": -1.3144887685775757, | |
| "logps/chosen": -0.8834988474845886, | |
| "logps/rejected": -4.147261619567871, | |
| "loss": 0.8871, | |
| "odds_ratio_loss": 0.03579668328166008, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08834987133741379, | |
| "rewards/margins": 0.3263762891292572, | |
| "rewards/rejected": -0.414726197719574, | |
| "sft_loss": 0.8834988474845886, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 0.587995100040833, | |
| "grad_norm": 1.9115794897079468, | |
| "learning_rate": 4.540492280890555e-05, | |
| "logits/chosen": -0.34701991081237793, | |
| "logits/rejected": -1.1767631769180298, | |
| "logps/chosen": -0.8725343942642212, | |
| "logps/rejected": -4.597090721130371, | |
| "loss": 0.8753, | |
| "odds_ratio_loss": 0.027656957507133484, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08725343644618988, | |
| "rewards/margins": 0.3724556267261505, | |
| "rewards/rejected": -0.4597090780735016, | |
| "sft_loss": 0.8725343942642212, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.6043282972641895, | |
| "grad_norm": 1.7303317785263062, | |
| "learning_rate": 4.515478954881248e-05, | |
| "logits/chosen": -0.37866610288619995, | |
| "logits/rejected": -1.20633065700531, | |
| "logps/chosen": -0.8602287173271179, | |
| "logps/rejected": -4.4016923904418945, | |
| "loss": 0.8625, | |
| "odds_ratio_loss": 0.022269519045948982, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08602287620306015, | |
| "rewards/margins": 0.35414639115333557, | |
| "rewards/rejected": -0.4401692748069763, | |
| "sft_loss": 0.8602287173271179, | |
| "step": 185 | |
| }, | |
| { | |
| "epoch": 0.6206614944875459, | |
| "grad_norm": 5.377653121948242, | |
| "learning_rate": 4.4898755336977673e-05, | |
| "logits/chosen": -0.32614630460739136, | |
| "logits/rejected": -1.1425362825393677, | |
| "logps/chosen": -0.9115999937057495, | |
| "logps/rejected": -4.731467247009277, | |
| "loss": 0.9149, | |
| "odds_ratio_loss": 0.0329320952296257, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09115998446941376, | |
| "rewards/margins": 0.38198670744895935, | |
| "rewards/rejected": -0.4731467366218567, | |
| "sft_loss": 0.9115999937057495, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.6369946917109024, | |
| "grad_norm": 1.6315754652023315, | |
| "learning_rate": 4.463689513550997e-05, | |
| "logits/chosen": -0.3550247550010681, | |
| "logits/rejected": -1.241008996963501, | |
| "logps/chosen": -0.8556908369064331, | |
| "logps/rejected": -4.392051696777344, | |
| "loss": 0.8582, | |
| "odds_ratio_loss": 0.024758270010352135, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08556908369064331, | |
| "rewards/margins": 0.35363608598709106, | |
| "rewards/rejected": -0.4392051696777344, | |
| "sft_loss": 0.8556908369064331, | |
| "step": 195 | |
| }, | |
| { | |
| "epoch": 0.6533278889342589, | |
| "grad_norm": 1.8591830730438232, | |
| "learning_rate": 4.436928561226087e-05, | |
| "logits/chosen": -0.3138996660709381, | |
| "logits/rejected": -1.2280880212783813, | |
| "logps/chosen": -0.901713490486145, | |
| "logps/rejected": -4.212186336517334, | |
| "loss": 0.9045, | |
| "odds_ratio_loss": 0.027884572744369507, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09017134457826614, | |
| "rewards/margins": 0.3310473561286926, | |
| "rewards/rejected": -0.4212186932563782, | |
| "sft_loss": 0.901713490486145, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.6696610861576153, | |
| "grad_norm": 1.8346513509750366, | |
| "learning_rate": 4.409600511837756e-05, | |
| "logits/chosen": -0.23156340420246124, | |
| "logits/rejected": -1.138501524925232, | |
| "logps/chosen": -0.9389227032661438, | |
| "logps/rejected": -4.50938606262207, | |
| "loss": 0.9416, | |
| "odds_ratio_loss": 0.02684590592980385, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09389226883649826, | |
| "rewards/margins": 0.35704633593559265, | |
| "rewards/rejected": -0.4509386122226715, | |
| "sft_loss": 0.9389227032661438, | |
| "step": 205 | |
| }, | |
| { | |
| "epoch": 0.6859942833809718, | |
| "grad_norm": 1.5812890529632568, | |
| "learning_rate": 4.381713366536311e-05, | |
| "logits/chosen": -0.24586188793182373, | |
| "logits/rejected": -1.1232062578201294, | |
| "logps/chosen": -0.859118640422821, | |
| "logps/rejected": -4.883942127227783, | |
| "loss": 0.8609, | |
| "odds_ratio_loss": 0.018183354288339615, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08591187745332718, | |
| "rewards/margins": 0.4024823307991028, | |
| "rewards/rejected": -0.48839420080184937, | |
| "sft_loss": 0.859118640422821, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.7023274806043283, | |
| "grad_norm": 1.8375471830368042, | |
| "learning_rate": 4.353275290165066e-05, | |
| "logits/chosen": -0.15520325303077698, | |
| "logits/rejected": -0.985150933265686, | |
| "logps/chosen": -0.8214719891548157, | |
| "logps/rejected": -4.9630866050720215, | |
| "loss": 0.823, | |
| "odds_ratio_loss": 0.014944888651371002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08214719593524933, | |
| "rewards/margins": 0.4141615033149719, | |
| "rewards/rejected": -0.49630871415138245, | |
| "sft_loss": 0.8214719891548157, | |
| "step": 215 | |
| }, | |
| { | |
| "epoch": 0.7186606778276847, | |
| "grad_norm": 2.556330919265747, | |
| "learning_rate": 4.324294608869817e-05, | |
| "logits/chosen": -0.3221624791622162, | |
| "logits/rejected": -0.9542757272720337, | |
| "logps/chosen": -0.9679568409919739, | |
| "logps/rejected": -5.273629188537598, | |
| "loss": 0.9693, | |
| "odds_ratio_loss": 0.013113953173160553, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0967956930398941, | |
| "rewards/margins": 0.4305671751499176, | |
| "rewards/rejected": -0.5273628830909729, | |
| "sft_loss": 0.9679568409919739, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.7349938750510412, | |
| "grad_norm": 2.346754312515259, | |
| "learning_rate": 4.294779807661105e-05, | |
| "logits/chosen": -0.2690557539463043, | |
| "logits/rejected": -1.0382682085037231, | |
| "logps/chosen": -0.8765946626663208, | |
| "logps/rejected": -5.27601432800293, | |
| "loss": 0.8783, | |
| "odds_ratio_loss": 0.016597110778093338, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08765946328639984, | |
| "rewards/margins": 0.43994197249412537, | |
| "rewards/rejected": -0.527601420879364, | |
| "sft_loss": 0.8765946626663208, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 0.7513270722743978, | |
| "grad_norm": 1.543803095817566, | |
| "learning_rate": 4.270789202181204e-05, | |
| "logits/chosen": -0.3406456410884857, | |
| "logits/rejected": -1.0243726968765259, | |
| "logps/chosen": -0.938999354839325, | |
| "logps/rejected": -5.179265022277832, | |
| "loss": 0.9411, | |
| "odds_ratio_loss": 0.021242566406726837, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09389995038509369, | |
| "rewards/margins": 0.4240265488624573, | |
| "rewards/rejected": -0.5179265141487122, | |
| "sft_loss": 0.938999354839325, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.7676602694977542, | |
| "grad_norm": 1.8233140707015991, | |
| "learning_rate": 4.2403348649073174e-05, | |
| "logits/chosen": -0.2546311020851135, | |
| "logits/rejected": -0.9541198015213013, | |
| "logps/chosen": -0.7934538722038269, | |
| "logps/rejected": -5.206686973571777, | |
| "loss": 0.7944, | |
| "odds_ratio_loss": 0.00939333625137806, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07934539020061493, | |
| "rewards/margins": 0.4413233697414398, | |
| "rewards/rejected": -0.5206687450408936, | |
| "sft_loss": 0.7934538722038269, | |
| "step": 235 | |
| }, | |
| { | |
| "epoch": 0.7839934667211107, | |
| "grad_norm": 1.5795395374298096, | |
| "learning_rate": 4.209370989589013e-05, | |
| "logits/chosen": -0.3499365448951721, | |
| "logits/rejected": -1.0148460865020752, | |
| "logps/chosen": -0.7999550104141235, | |
| "logps/rejected": -5.092876434326172, | |
| "loss": 0.8013, | |
| "odds_ratio_loss": 0.013519616797566414, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07999551296234131, | |
| "rewards/margins": 0.4292922019958496, | |
| "rewards/rejected": -0.5092877149581909, | |
| "sft_loss": 0.7999550104141235, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.8003266639444672, | |
| "grad_norm": 1.8533421754837036, | |
| "learning_rate": 4.177906641879575e-05, | |
| "logits/chosen": -0.3404558598995209, | |
| "logits/rejected": -1.0349606275558472, | |
| "logps/chosen": -0.9274562001228333, | |
| "logps/rejected": -4.853649616241455, | |
| "loss": 0.9298, | |
| "odds_ratio_loss": 0.023337265476584435, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09274561703205109, | |
| "rewards/margins": 0.3926192820072174, | |
| "rewards/rejected": -0.4853649139404297, | |
| "sft_loss": 0.9274562001228333, | |
| "step": 245 | |
| }, | |
| { | |
| "epoch": 0.8166598611678236, | |
| "grad_norm": 1.792864441871643, | |
| "learning_rate": 4.145951033961395e-05, | |
| "logits/chosen": -0.39237070083618164, | |
| "logits/rejected": -1.20218825340271, | |
| "logps/chosen": -0.8956495523452759, | |
| "logps/rejected": -4.416346549987793, | |
| "loss": 0.8988, | |
| "odds_ratio_loss": 0.03134030103683472, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08956494927406311, | |
| "rewards/margins": 0.3520697355270386, | |
| "rewards/rejected": -0.4416346549987793, | |
| "sft_loss": 0.8956495523452759, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.8329930583911801, | |
| "grad_norm": 1.388491153717041, | |
| "learning_rate": 4.1135135218488206e-05, | |
| "logits/chosen": -0.4966661036014557, | |
| "logits/rejected": -1.301102638244629, | |
| "logps/chosen": -0.822670578956604, | |
| "logps/rejected": -4.420852184295654, | |
| "loss": 0.8251, | |
| "odds_ratio_loss": 0.02416636422276497, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08226706087589264, | |
| "rewards/margins": 0.3598182201385498, | |
| "rewards/rejected": -0.4420852065086365, | |
| "sft_loss": 0.822670578956604, | |
| "step": 255 | |
| }, | |
| { | |
| "epoch": 0.8493262556145366, | |
| "grad_norm": 1.7927485704421997, | |
| "learning_rate": 4.080603602648887e-05, | |
| "logits/chosen": -0.38711199164390564, | |
| "logits/rejected": -1.1762360334396362, | |
| "logps/chosen": -0.8747900128364563, | |
| "logps/rejected": -4.844551086425781, | |
| "loss": 0.8772, | |
| "odds_ratio_loss": 0.024142403155565262, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08747899532318115, | |
| "rewards/margins": 0.3969760835170746, | |
| "rewards/rejected": -0.4844551086425781, | |
| "sft_loss": 0.8747900128364563, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.865659452837893, | |
| "grad_norm": 1.763637661933899, | |
| "learning_rate": 4.047230911780737e-05, | |
| "logits/chosen": -0.4611227512359619, | |
| "logits/rejected": -1.092810034751892, | |
| "logps/chosen": -0.8380341529846191, | |
| "logps/rejected": -4.902915000915527, | |
| "loss": 0.8414, | |
| "odds_ratio_loss": 0.03330729901790619, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08380343019962311, | |
| "rewards/margins": 0.4064881205558777, | |
| "rewards/rejected": -0.4902915358543396, | |
| "sft_loss": 0.8380341529846191, | |
| "step": 265 | |
| }, | |
| { | |
| "epoch": 0.8819926500612495, | |
| "grad_norm": 1.7203322649002075, | |
| "learning_rate": 4.0134052201545447e-05, | |
| "logits/chosen": -0.24241454899311066, | |
| "logits/rejected": -0.9308720827102661, | |
| "logps/chosen": -0.8675628900527954, | |
| "logps/rejected": -5.390278339385986, | |
| "loss": 0.8685, | |
| "odds_ratio_loss": 0.009246011264622211, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08675628900527954, | |
| "rewards/margins": 0.4522715210914612, | |
| "rewards/rejected": -0.5390278100967407, | |
| "sft_loss": 0.8675628900527954, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.898325847284606, | |
| "grad_norm": 1.9116899967193604, | |
| "learning_rate": 3.9791364313107804e-05, | |
| "logits/chosen": -0.401073157787323, | |
| "logits/rejected": -0.9959608912467957, | |
| "logps/chosen": -0.8706862330436707, | |
| "logps/rejected": -5.204241752624512, | |
| "loss": 0.872, | |
| "odds_ratio_loss": 0.0133589506149292, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08706863224506378, | |
| "rewards/margins": 0.4333556294441223, | |
| "rewards/rejected": -0.5204241871833801, | |
| "sft_loss": 0.8706862330436707, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 0.9146590445079624, | |
| "grad_norm": 1.9296520948410034, | |
| "learning_rate": 3.9444345785206285e-05, | |
| "logits/chosen": -0.40887147188186646, | |
| "logits/rejected": -0.9652082324028015, | |
| "logps/chosen": -0.8309628367424011, | |
| "logps/rejected": -5.662296295166016, | |
| "loss": 0.8318, | |
| "odds_ratio_loss": 0.008708246983587742, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08309628069400787, | |
| "rewards/margins": 0.48313331604003906, | |
| "rewards/rejected": -0.5662296414375305, | |
| "sft_loss": 0.8309628367424011, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.9309922417313189, | |
| "grad_norm": 3.055554151535034, | |
| "learning_rate": 3.909309821848432e-05, | |
| "logits/chosen": -0.3440246284008026, | |
| "logits/rejected": -0.8914414644241333, | |
| "logps/chosen": -0.8914461135864258, | |
| "logps/rejected": -5.75334358215332, | |
| "loss": 0.8921, | |
| "odds_ratio_loss": 0.0068351454101502895, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08914461731910706, | |
| "rewards/margins": 0.48618975281715393, | |
| "rewards/rejected": -0.575334370136261, | |
| "sft_loss": 0.8914461135864258, | |
| "step": 285 | |
| }, | |
| { | |
| "epoch": 0.9473254389546754, | |
| "grad_norm": 1.5384926795959473, | |
| "learning_rate": 3.873772445177015e-05, | |
| "logits/chosen": -0.37930798530578613, | |
| "logits/rejected": -0.9451101422309875, | |
| "logps/chosen": -0.8741447329521179, | |
| "logps/rejected": -5.84790563583374, | |
| "loss": 0.8751, | |
| "odds_ratio_loss": 0.009465084411203861, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08741448074579239, | |
| "rewards/margins": 0.49737611413002014, | |
| "rewards/rejected": -0.5847905874252319, | |
| "sft_loss": 0.8741447329521179, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.9636586361780318, | |
| "grad_norm": 2.4715216159820557, | |
| "learning_rate": 3.837832853196751e-05, | |
| "logits/chosen": -0.3153218626976013, | |
| "logits/rejected": -0.9612942934036255, | |
| "logps/chosen": -0.8935874700546265, | |
| "logps/rejected": -5.140650749206543, | |
| "loss": 0.8956, | |
| "odds_ratio_loss": 0.01970650628209114, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08935876190662384, | |
| "rewards/margins": 0.4247063100337982, | |
| "rewards/rejected": -0.514065146446228, | |
| "sft_loss": 0.8935874700546265, | |
| "step": 295 | |
| }, | |
| { | |
| "epoch": 0.9799918334013883, | |
| "grad_norm": 1.9313322305679321, | |
| "learning_rate": 3.8015015683592575e-05, | |
| "logits/chosen": -0.30432718992233276, | |
| "logits/rejected": -1.099277377128601, | |
| "logps/chosen": -0.8711298108100891, | |
| "logps/rejected": -4.722751140594482, | |
| "loss": 0.873, | |
| "odds_ratio_loss": 0.018964512273669243, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08711298555135727, | |
| "rewards/margins": 0.3851621747016907, | |
| "rewards/rejected": -0.47227516770362854, | |
| "sft_loss": 0.8711298108100891, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.9963250306247448, | |
| "grad_norm": 2.2331244945526123, | |
| "learning_rate": 3.764789227796616e-05, | |
| "logits/chosen": -0.3758210837841034, | |
| "logits/rejected": -1.1826483011245728, | |
| "logps/chosen": -0.8142495155334473, | |
| "logps/rejected": -4.7738471031188965, | |
| "loss": 0.816, | |
| "odds_ratio_loss": 0.017485955730080605, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08142495155334473, | |
| "rewards/margins": 0.3959597945213318, | |
| "rewards/rejected": -0.4773847460746765, | |
| "sft_loss": 0.8142495155334473, | |
| "step": 305 | |
| }, | |
| { | |
| "epoch": 1.0126582278481013, | |
| "grad_norm": 1.7077336311340332, | |
| "learning_rate": 3.727706580207021e-05, | |
| "logits/chosen": -0.38093143701553345, | |
| "logits/rejected": -1.096470594406128, | |
| "logps/chosen": -0.8485842943191528, | |
| "logps/rejected": -4.818546295166016, | |
| "loss": 0.8503, | |
| "odds_ratio_loss": 0.01746557094156742, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08485842496156693, | |
| "rewards/margins": 0.3969961702823639, | |
| "rewards/rejected": -0.4818546175956726, | |
| "sft_loss": 0.8485842943191528, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 1.0289914250714578, | |
| "grad_norm": 1.8160737752914429, | |
| "learning_rate": 3.69026448270775e-05, | |
| "logits/chosen": -0.37943610548973083, | |
| "logits/rejected": -1.0238980054855347, | |
| "logps/chosen": -0.8509234189987183, | |
| "logps/rejected": -5.104341506958008, | |
| "loss": 0.8521, | |
| "odds_ratio_loss": 0.011288631707429886, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08509235084056854, | |
| "rewards/margins": 0.4253418445587158, | |
| "rewards/rejected": -0.5104341506958008, | |
| "sft_loss": 0.8509234189987183, | |
| "step": 315 | |
| }, | |
| { | |
| "epoch": 1.0453246222948143, | |
| "grad_norm": 2.9576852321624756, | |
| "learning_rate": 3.652473897656409e-05, | |
| "logits/chosen": -0.30405062437057495, | |
| "logits/rejected": -0.9463305473327637, | |
| "logps/chosen": -0.9074727296829224, | |
| "logps/rejected": -5.486522674560547, | |
| "loss": 0.9089, | |
| "odds_ratio_loss": 0.013948513194918633, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09074728190898895, | |
| "rewards/margins": 0.45790499448776245, | |
| "rewards/rejected": -0.5486522912979126, | |
| "sft_loss": 0.9074727296829224, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 1.0616578195181707, | |
| "grad_norm": 1.9609084129333496, | |
| "learning_rate": 3.6143458894413465e-05, | |
| "logits/chosen": -0.39042356610298157, | |
| "logits/rejected": -1.0480425357818604, | |
| "logps/chosen": -0.7837303876876831, | |
| "logps/rejected": -5.29976749420166, | |
| "loss": 0.7846, | |
| "odds_ratio_loss": 0.008555177599191666, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0783730298280716, | |
| "rewards/margins": 0.45160380005836487, | |
| "rewards/rejected": -0.5299768447875977, | |
| "sft_loss": 0.7837303876876831, | |
| "step": 325 | |
| }, | |
| { | |
| "epoch": 1.0779910167415272, | |
| "grad_norm": 2.609851360321045, | |
| "learning_rate": 3.575891621242209e-05, | |
| "logits/chosen": -0.35802769660949707, | |
| "logits/rejected": -0.9912473559379578, | |
| "logps/chosen": -0.8010882139205933, | |
| "logps/rejected": -5.255009651184082, | |
| "loss": 0.8022, | |
| "odds_ratio_loss": 0.010739787481725216, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08010882139205933, | |
| "rewards/margins": 0.4453921914100647, | |
| "rewards/rejected": -0.525501012802124, | |
| "sft_loss": 0.8010882139205933, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 1.0943242139648837, | |
| "grad_norm": 1.9164317846298218, | |
| "learning_rate": 3.537122351761568e-05, | |
| "logits/chosen": -0.29361772537231445, | |
| "logits/rejected": -0.9894407987594604, | |
| "logps/chosen": -0.8663752675056458, | |
| "logps/rejected": -5.1811041831970215, | |
| "loss": 0.8682, | |
| "odds_ratio_loss": 0.01779218763113022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08663753420114517, | |
| "rewards/margins": 0.43147286772727966, | |
| "rewards/rejected": -0.5181103944778442, | |
| "sft_loss": 0.8663752675056458, | |
| "step": 335 | |
| }, | |
| { | |
| "epoch": 1.1106574111882401, | |
| "grad_norm": 1.8551751375198364, | |
| "learning_rate": 3.4980494319285775e-05, | |
| "logits/chosen": -0.2934134900569916, | |
| "logits/rejected": -0.9298968315124512, | |
| "logps/chosen": -0.8729950189590454, | |
| "logps/rejected": -5.402509689331055, | |
| "loss": 0.8739, | |
| "odds_ratio_loss": 0.009193124249577522, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08729950338602066, | |
| "rewards/margins": 0.45295149087905884, | |
| "rewards/rejected": -0.5402510166168213, | |
| "sft_loss": 0.8729950189590454, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 1.1269906084115966, | |
| "grad_norm": 1.8746849298477173, | |
| "learning_rate": 3.45868430157563e-05, | |
| "logits/chosen": -0.27233654260635376, | |
| "logits/rejected": -0.9365426301956177, | |
| "logps/chosen": -0.8479824066162109, | |
| "logps/rejected": -5.225064754486084, | |
| "loss": 0.8493, | |
| "odds_ratio_loss": 0.013331398367881775, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08479823917150497, | |
| "rewards/margins": 0.43770831823349, | |
| "rewards/rejected": -0.522506594657898, | |
| "sft_loss": 0.8479824066162109, | |
| "step": 345 | |
| }, | |
| { | |
| "epoch": 1.143323805634953, | |
| "grad_norm": 1.7111650705337524, | |
| "learning_rate": 3.4190384860889824e-05, | |
| "logits/chosen": -0.366852343082428, | |
| "logits/rejected": -1.0381531715393066, | |
| "logps/chosen": -0.8182533383369446, | |
| "logps/rejected": -5.278301239013672, | |
| "loss": 0.8204, | |
| "odds_ratio_loss": 0.02150268480181694, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08182532340288162, | |
| "rewards/margins": 0.4460047781467438, | |
| "rewards/rejected": -0.5278300642967224, | |
| "sft_loss": 0.8182533383369446, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 1.1596570028583095, | |
| "grad_norm": 1.9346224069595337, | |
| "learning_rate": 3.379123593034342e-05, | |
| "logits/chosen": -0.2924169898033142, | |
| "logits/rejected": -1.028192400932312, | |
| "logps/chosen": -0.9392467737197876, | |
| "logps/rejected": -5.247817516326904, | |
| "loss": 0.9409, | |
| "odds_ratio_loss": 0.0163104310631752, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09392467886209488, | |
| "rewards/margins": 0.4308570921421051, | |
| "rewards/rejected": -0.5247817635536194, | |
| "sft_loss": 0.9392467737197876, | |
| "step": 355 | |
| }, | |
| { | |
| "epoch": 1.175990200081666, | |
| "grad_norm": 1.9083410501480103, | |
| "learning_rate": 3.3389513087583754e-05, | |
| "logits/chosen": -0.1519567221403122, | |
| "logits/rejected": -1.0742218494415283, | |
| "logps/chosen": -0.8435864448547363, | |
| "logps/rejected": -4.644381523132324, | |
| "loss": 0.8465, | |
| "odds_ratio_loss": 0.028982138261198997, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08435864001512527, | |
| "rewards/margins": 0.3800795078277588, | |
| "rewards/rejected": -0.4644381105899811, | |
| "sft_loss": 0.8435864448547363, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 1.1923233973050225, | |
| "grad_norm": 1.8590240478515625, | |
| "learning_rate": 3.2985333949671685e-05, | |
| "logits/chosen": -0.43951839208602905, | |
| "logits/rejected": -1.1476421356201172, | |
| "logps/chosen": -0.8573881387710571, | |
| "logps/rejected": -4.641860485076904, | |
| "loss": 0.8605, | |
| "odds_ratio_loss": 0.030726104974746704, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08573882281780243, | |
| "rewards/margins": 0.3784472346305847, | |
| "rewards/rejected": -0.4641861021518707, | |
| "sft_loss": 0.8573881387710571, | |
| "step": 365 | |
| }, | |
| { | |
| "epoch": 1.208656594528379, | |
| "grad_norm": 2.064635753631592, | |
| "learning_rate": 3.257881685282609e-05, | |
| "logits/chosen": -0.2586503028869629, | |
| "logits/rejected": -1.1886835098266602, | |
| "logps/chosen": -0.7855366468429565, | |
| "logps/rejected": -4.374009132385254, | |
| "loss": 0.7886, | |
| "odds_ratio_loss": 0.03049846924841404, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07855366170406342, | |
| "rewards/margins": 0.3588472604751587, | |
| "rewards/rejected": -0.4374009668827057, | |
| "sft_loss": 0.7855366468429565, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 1.2249897917517354, | |
| "grad_norm": 1.986666202545166, | |
| "learning_rate": 3.217008081777726e-05, | |
| "logits/chosen": -0.3716577887535095, | |
| "logits/rejected": -1.2067687511444092, | |
| "logps/chosen": -0.8298207521438599, | |
| "logps/rejected": -4.59980583190918, | |
| "loss": 0.8315, | |
| "odds_ratio_loss": 0.016955239698290825, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08298207074403763, | |
| "rewards/margins": 0.3769984841346741, | |
| "rewards/rejected": -0.4599805772304535, | |
| "sft_loss": 0.8298207521438599, | |
| "step": 375 | |
| }, | |
| { | |
| "epoch": 1.2413229889750919, | |
| "grad_norm": 1.8263311386108398, | |
| "learning_rate": 3.175924551491977e-05, | |
| "logits/chosen": -0.3963642716407776, | |
| "logits/rejected": -1.1271188259124756, | |
| "logps/chosen": -0.8212121725082397, | |
| "logps/rejected": -4.6982741355896, | |
| "loss": 0.8234, | |
| "odds_ratio_loss": 0.02219826728105545, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08212121576070786, | |
| "rewards/margins": 0.3877061605453491, | |
| "rewards/rejected": -0.4698273539543152, | |
| "sft_loss": 0.8212121725082397, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 1.2576561861984483, | |
| "grad_norm": 1.946442723274231, | |
| "learning_rate": 3.13464312292752e-05, | |
| "logits/chosen": -0.343479186296463, | |
| "logits/rejected": -0.9710591435432434, | |
| "logps/chosen": -0.8622458577156067, | |
| "logps/rejected": -5.370631217956543, | |
| "loss": 0.8646, | |
| "odds_ratio_loss": 0.023645440116524696, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08622457832098007, | |
| "rewards/margins": 0.450838565826416, | |
| "rewards/rejected": -0.5370631217956543, | |
| "sft_loss": 0.8622458577156067, | |
| "step": 385 | |
| }, | |
| { | |
| "epoch": 1.2739893834218048, | |
| "grad_norm": 1.7212151288986206, | |
| "learning_rate": 3.093175882527491e-05, | |
| "logits/chosen": -0.3060234785079956, | |
| "logits/rejected": -1.022698163986206, | |
| "logps/chosen": -0.8830709457397461, | |
| "logps/rejected": -5.35630464553833, | |
| "loss": 0.8848, | |
| "odds_ratio_loss": 0.01716751977801323, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08830709010362625, | |
| "rewards/margins": 0.44732338190078735, | |
| "rewards/rejected": -0.535630464553833, | |
| "sft_loss": 0.8830709457397461, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 1.2903225806451613, | |
| "grad_norm": 2.029407024383545, | |
| "learning_rate": 3.0515349711373146e-05, | |
| "logits/chosen": -0.13591887056827545, | |
| "logits/rejected": -0.9809386134147644, | |
| "logps/chosen": -0.880872368812561, | |
| "logps/rejected": -5.28397798538208, | |
| "loss": 0.8826, | |
| "odds_ratio_loss": 0.01720813475549221, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08808723837137222, | |
| "rewards/margins": 0.4403105676174164, | |
| "rewards/rejected": -0.5283978581428528, | |
| "sft_loss": 0.880872368812561, | |
| "step": 395 | |
| }, | |
| { | |
| "epoch": 1.3066557778685177, | |
| "grad_norm": 1.8320516347885132, | |
| "learning_rate": 3.0097325804500864e-05, | |
| "logits/chosen": -0.3814406991004944, | |
| "logits/rejected": -1.0283949375152588, | |
| "logps/chosen": -0.7927986979484558, | |
| "logps/rejected": -5.100050926208496, | |
| "loss": 0.7945, | |
| "odds_ratio_loss": 0.016741547733545303, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07927986979484558, | |
| "rewards/margins": 0.4307252764701843, | |
| "rewards/rejected": -0.5100051164627075, | |
| "sft_loss": 0.7927986979484558, | |
| "step": 400 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 918, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 100, | |
| "total_flos": 1.663517229609124e+17, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |