| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.9599836668027768, | |
| "eval_steps": 500, | |
| "global_step": 600, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.016333197223356473, | |
| "grad_norm": 1.1593419313430786, | |
| "learning_rate": 4.9996340229869763e-05, | |
| "logits/chosen": -1.0346699953079224, | |
| "logits/rejected": -2.6695446968078613, | |
| "logps/chosen": -1.101169466972351, | |
| "logps/rejected": -1.0235556364059448, | |
| "loss": 1.178, | |
| "odds_ratio_loss": 0.768240213394165, | |
| "rewards/accuracies": 0.3499999940395355, | |
| "rewards/chosen": -0.11011694371700287, | |
| "rewards/margins": -0.007761377841234207, | |
| "rewards/rejected": -0.10235557705163956, | |
| "sft_loss": 1.101169466972351, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.032666394446712946, | |
| "grad_norm": 1.1402696371078491, | |
| "learning_rate": 4.9985361990992455e-05, | |
| "logits/chosen": -0.8193758726119995, | |
| "logits/rejected": -2.842833995819092, | |
| "logps/chosen": -1.0217273235321045, | |
| "logps/rejected": -1.0100481510162354, | |
| "loss": 1.0927, | |
| "odds_ratio_loss": 0.7092453837394714, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.10217274725437164, | |
| "rewards/margins": -0.0011679243762046099, | |
| "rewards/rejected": -0.10100481659173965, | |
| "sft_loss": 1.0217273235321045, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.04899959167006942, | |
| "grad_norm": 1.3554140329360962, | |
| "learning_rate": 4.996706849759453e-05, | |
| "logits/chosen": -0.7696830034255981, | |
| "logits/rejected": -2.703421115875244, | |
| "logps/chosen": -1.0294106006622314, | |
| "logps/rejected": -1.0599125623703003, | |
| "loss": 1.0993, | |
| "odds_ratio_loss": 0.6984036564826965, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.10294107347726822, | |
| "rewards/margins": 0.003050185739994049, | |
| "rewards/rejected": -0.10599125921726227, | |
| "sft_loss": 1.0294106006622314, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.06533278889342589, | |
| "grad_norm": 1.406853199005127, | |
| "learning_rate": 4.9941465105674435e-05, | |
| "logits/chosen": -0.698488712310791, | |
| "logits/rejected": -2.808516263961792, | |
| "logps/chosen": -1.0651928186416626, | |
| "logps/rejected": -0.996777355670929, | |
| "loss": 1.1402, | |
| "odds_ratio_loss": 0.7504132390022278, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.10651928186416626, | |
| "rewards/margins": -0.006841552909463644, | |
| "rewards/rejected": -0.09967772662639618, | |
| "sft_loss": 1.0651928186416626, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.08166598611678236, | |
| "grad_norm": 1.238513469696045, | |
| "learning_rate": 4.99085593114345e-05, | |
| "logits/chosen": -0.6306881904602051, | |
| "logits/rejected": -2.698362350463867, | |
| "logps/chosen": -1.0242538452148438, | |
| "logps/rejected": -1.0320734977722168, | |
| "loss": 1.0937, | |
| "odds_ratio_loss": 0.6945221424102783, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.10242538154125214, | |
| "rewards/margins": 0.0007819652673788369, | |
| "rewards/rejected": -0.10320734977722168, | |
| "sft_loss": 1.0242538452148438, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.09799918334013884, | |
| "grad_norm": 1.9588699340820312, | |
| "learning_rate": 4.986836074908616e-05, | |
| "logits/chosen": -0.5534936189651489, | |
| "logits/rejected": -2.678889036178589, | |
| "logps/chosen": -1.1003000736236572, | |
| "logps/rejected": -1.0425399541854858, | |
| "loss": 1.1745, | |
| "odds_ratio_loss": 0.7416313290596008, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.11003003269433975, | |
| "rewards/margins": -0.005776026751846075, | |
| "rewards/rejected": -0.10425399243831635, | |
| "sft_loss": 1.1003000736236572, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.1143323805634953, | |
| "grad_norm": 1.1857821941375732, | |
| "learning_rate": 4.982088118802922e-05, | |
| "logits/chosen": -0.6129628419876099, | |
| "logits/rejected": -2.876364231109619, | |
| "logps/chosen": -0.9628928303718567, | |
| "logps/rejected": -0.952060341835022, | |
| "loss": 1.0336, | |
| "odds_ratio_loss": 0.7068274617195129, | |
| "rewards/accuracies": 0.42500001192092896, | |
| "rewards/chosen": -0.09628929197788239, | |
| "rewards/margins": -0.001083258306607604, | |
| "rewards/rejected": -0.09520602226257324, | |
| "sft_loss": 0.9628928303718567, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.13066557778685178, | |
| "grad_norm": 1.33924400806427, | |
| "learning_rate": 4.976613452940604e-05, | |
| "logits/chosen": -0.7091814279556274, | |
| "logits/rejected": -2.6595664024353027, | |
| "logps/chosen": -0.89775550365448, | |
| "logps/rejected": -0.935952365398407, | |
| "loss": 0.9642, | |
| "odds_ratio_loss": 0.6644185781478882, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.08977555483579636, | |
| "rewards/margins": 0.003819683101028204, | |
| "rewards/rejected": -0.0935952290892601, | |
| "sft_loss": 0.89775550365448, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.14699877501020825, | |
| "grad_norm": 1.136667013168335, | |
| "learning_rate": 4.9704136802031485e-05, | |
| "logits/chosen": -0.7233263850212097, | |
| "logits/rejected": -2.6673388481140137, | |
| "logps/chosen": -0.9036429524421692, | |
| "logps/rejected": -0.9419530630111694, | |
| "loss": 0.9699, | |
| "odds_ratio_loss": 0.662254273891449, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.09036429226398468, | |
| "rewards/margins": 0.0038310133386403322, | |
| "rewards/rejected": -0.09419531375169754, | |
| "sft_loss": 0.9036429524421692, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.16333197223356472, | |
| "grad_norm": 1.4290399551391602, | |
| "learning_rate": 4.9634906157700036e-05, | |
| "logits/chosen": -0.6113058924674988, | |
| "logits/rejected": -2.705373764038086, | |
| "logps/chosen": -0.9550830125808716, | |
| "logps/rejected": -0.9817631840705872, | |
| "loss": 1.0223, | |
| "odds_ratio_loss": 0.6725824475288391, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.09550830721855164, | |
| "rewards/margins": 0.002668013097718358, | |
| "rewards/rejected": -0.09817632287740707, | |
| "sft_loss": 0.9550830125808716, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.17966516945692118, | |
| "grad_norm": 1.4945589303970337, | |
| "learning_rate": 4.955846286587122e-05, | |
| "logits/chosen": -0.647710919380188, | |
| "logits/rejected": -2.7208316326141357, | |
| "logps/chosen": -0.9462896585464478, | |
| "logps/rejected": -0.9507497549057007, | |
| "loss": 1.0148, | |
| "odds_ratio_loss": 0.6852728128433228, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.0946289673447609, | |
| "rewards/margins": 0.0004460157942958176, | |
| "rewards/rejected": -0.09507497400045395, | |
| "sft_loss": 0.9462896585464478, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.19599836668027767, | |
| "grad_norm": 1.3553156852722168, | |
| "learning_rate": 4.9474829307735115e-05, | |
| "logits/chosen": -0.6279743909835815, | |
| "logits/rejected": -2.6854496002197266, | |
| "logps/chosen": -0.8896121978759766, | |
| "logps/rejected": -0.9993942975997925, | |
| "loss": 0.9509, | |
| "odds_ratio_loss": 0.6129180192947388, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.08896122127771378, | |
| "rewards/margins": 0.010978218168020248, | |
| "rewards/rejected": -0.09993942826986313, | |
| "sft_loss": 0.8896121978759766, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.21233156390363414, | |
| "grad_norm": 1.4247411489486694, | |
| "learning_rate": 4.9384029969659573e-05, | |
| "logits/chosen": -0.698381781578064, | |
| "logits/rejected": -2.7967753410339355, | |
| "logps/chosen": -0.9455701112747192, | |
| "logps/rejected": -0.9564310908317566, | |
| "loss": 1.0147, | |
| "odds_ratio_loss": 0.6911824941635132, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0945570170879364, | |
| "rewards/margins": 0.0010860899928957224, | |
| "rewards/rejected": -0.09564311057329178, | |
| "sft_loss": 0.9455701112747192, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.2286647611269906, | |
| "grad_norm": 1.6154508590698242, | |
| "learning_rate": 4.9286091436021015e-05, | |
| "logits/chosen": -0.595032274723053, | |
| "logits/rejected": -2.740784168243408, | |
| "logps/chosen": -0.9214372634887695, | |
| "logps/rejected": -1.0210167169570923, | |
| "loss": 0.9836, | |
| "odds_ratio_loss": 0.6212525963783264, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.09214373677968979, | |
| "rewards/margins": 0.00995794590562582, | |
| "rewards/rejected": -0.10210166871547699, | |
| "sft_loss": 0.9214372634887695, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.24499795835034707, | |
| "grad_norm": 1.8231102228164673, | |
| "learning_rate": 4.918104238142104e-05, | |
| "logits/chosen": -0.5186889171600342, | |
| "logits/rejected": -2.7576358318328857, | |
| "logps/chosen": -0.9789311289787292, | |
| "logps/rejected": -1.0169849395751953, | |
| "loss": 1.0454, | |
| "odds_ratio_loss": 0.6644267439842224, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.09789310395717621, | |
| "rewards/margins": 0.0038053765892982483, | |
| "rewards/rejected": -0.10169849544763565, | |
| "sft_loss": 0.9789311289787292, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.26133115557370357, | |
| "grad_norm": 2.05936336517334, | |
| "learning_rate": 4.906891356229103e-05, | |
| "logits/chosen": -0.47254711389541626, | |
| "logits/rejected": -2.68280029296875, | |
| "logps/chosen": -0.9620391726493835, | |
| "logps/rejected": -1.0026928186416626, | |
| "loss": 1.0294, | |
| "odds_ratio_loss": 0.6731926202774048, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09620390832424164, | |
| "rewards/margins": 0.004065364599227905, | |
| "rewards/rejected": -0.10026928037405014, | |
| "sft_loss": 0.9620391726493835, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.27766435279706003, | |
| "grad_norm": 1.6302626132965088, | |
| "learning_rate": 4.894973780788722e-05, | |
| "logits/chosen": -0.5315309762954712, | |
| "logits/rejected": -2.7227425575256348, | |
| "logps/chosen": -0.9423310160636902, | |
| "logps/rejected": -1.071781039237976, | |
| "loss": 1.0032, | |
| "odds_ratio_loss": 0.6090766191482544, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.09423311054706573, | |
| "rewards/margins": 0.012944999150931835, | |
| "rewards/rejected": -0.10717810690402985, | |
| "sft_loss": 0.9423310160636902, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.2939975500204165, | |
| "grad_norm": 1.7891793251037598, | |
| "learning_rate": 4.882355001067892e-05, | |
| "logits/chosen": -0.5275685787200928, | |
| "logits/rejected": -2.6127567291259766, | |
| "logps/chosen": -0.9351356625556946, | |
| "logps/rejected": -1.0083874464035034, | |
| "loss": 0.9995, | |
| "odds_ratio_loss": 0.6435801386833191, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.09351354837417603, | |
| "rewards/margins": 0.0073251971043646336, | |
| "rewards/rejected": -0.10083875805139542, | |
| "sft_loss": 0.9351356625556946, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.31033074724377296, | |
| "grad_norm": 1.852216124534607, | |
| "learning_rate": 4.869038711613259e-05, | |
| "logits/chosen": -0.5760647058486938, | |
| "logits/rejected": -2.722364664077759, | |
| "logps/chosen": -0.9301847219467163, | |
| "logps/rejected": -1.0749822854995728, | |
| "loss": 0.9903, | |
| "odds_ratio_loss": 0.601297914981842, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.09301847964525223, | |
| "rewards/margins": 0.014479748904705048, | |
| "rewards/rejected": -0.10749821364879608, | |
| "sft_loss": 0.9301847219467163, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.32666394446712943, | |
| "grad_norm": 1.8597307205200195, | |
| "learning_rate": 4.855028811189496e-05, | |
| "logits/chosen": -0.5871164202690125, | |
| "logits/rejected": -2.6969285011291504, | |
| "logps/chosen": -1.0701930522918701, | |
| "logps/rejected": -1.0556358098983765, | |
| "loss": 1.1427, | |
| "odds_ratio_loss": 0.7247555255889893, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.10701930522918701, | |
| "rewards/margins": -0.0014557272661477327, | |
| "rewards/rejected": -0.10556356608867645, | |
| "sft_loss": 1.0701930522918701, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.3429971416904859, | |
| "grad_norm": 1.6869758367538452, | |
| "learning_rate": 4.8403294016378095e-05, | |
| "logits/chosen": -0.4466473460197449, | |
| "logits/rejected": -2.501394510269165, | |
| "logps/chosen": -0.880301833152771, | |
| "logps/rejected": -1.0003767013549805, | |
| "loss": 0.941, | |
| "odds_ratio_loss": 0.60676509141922, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.08803018927574158, | |
| "rewards/margins": 0.012007484212517738, | |
| "rewards/rejected": -0.10003767162561417, | |
| "sft_loss": 0.880301833152771, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 0.35933033891384236, | |
| "grad_norm": 1.8963792324066162, | |
| "learning_rate": 4.8249447866750025e-05, | |
| "logits/chosen": -0.41199809312820435, | |
| "logits/rejected": -2.594975471496582, | |
| "logps/chosen": -0.8528003692626953, | |
| "logps/rejected": -0.9772283434867859, | |
| "loss": 0.9128, | |
| "odds_ratio_loss": 0.6004671454429626, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.08528003096580505, | |
| "rewards/margins": 0.012442806735634804, | |
| "rewards/rejected": -0.0977228432893753, | |
| "sft_loss": 0.8528003692626953, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.3756635361371989, | |
| "grad_norm": 2.6958725452423096, | |
| "learning_rate": 4.808879470633417e-05, | |
| "logits/chosen": -0.43985629081726074, | |
| "logits/rejected": -2.6999452114105225, | |
| "logps/chosen": -0.9486901164054871, | |
| "logps/rejected": -1.0987169742584229, | |
| "loss": 1.0076, | |
| "odds_ratio_loss": 0.5889819264411926, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.09486902505159378, | |
| "rewards/margins": 0.015002680011093616, | |
| "rewards/rejected": -0.10987170040607452, | |
| "sft_loss": 0.9486901164054871, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 0.39199673336055535, | |
| "grad_norm": 1.6014004945755005, | |
| "learning_rate": 4.792138157142158e-05, | |
| "logits/chosen": -0.43066906929016113, | |
| "logits/rejected": -2.634669065475464, | |
| "logps/chosen": -0.8806023597717285, | |
| "logps/rejected": -1.0933107137680054, | |
| "loss": 0.9352, | |
| "odds_ratio_loss": 0.5455727577209473, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.08806024491786957, | |
| "rewards/margins": 0.021270830184221268, | |
| "rewards/rejected": -0.10933107137680054, | |
| "sft_loss": 0.8806023597717285, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.4083299305839118, | |
| "grad_norm": 3.4042723178863525, | |
| "learning_rate": 4.7747257477499474e-05, | |
| "logits/chosen": -0.4240710735321045, | |
| "logits/rejected": -2.664668560028076, | |
| "logps/chosen": -0.8993316888809204, | |
| "logps/rejected": -1.0890860557556152, | |
| "loss": 0.9553, | |
| "odds_ratio_loss": 0.5601183772087097, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.08993317186832428, | |
| "rewards/margins": 0.018975432962179184, | |
| "rewards/rejected": -0.10890860855579376, | |
| "sft_loss": 0.8993316888809204, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 0.4246631278072683, | |
| "grad_norm": 1.724076509475708, | |
| "learning_rate": 4.75664734049005e-05, | |
| "logits/chosen": -0.3910972476005554, | |
| "logits/rejected": -2.571296215057373, | |
| "logps/chosen": -0.9113162755966187, | |
| "logps/rejected": -1.094262719154358, | |
| "loss": 0.9682, | |
| "odds_ratio_loss": 0.5683911442756653, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.09113162755966187, | |
| "rewards/margins": 0.018294643610715866, | |
| "rewards/rejected": -0.10942627489566803, | |
| "sft_loss": 0.9113162755966187, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.44099632503062475, | |
| "grad_norm": 1.6363219022750854, | |
| "learning_rate": 4.7379082283876566e-05, | |
| "logits/chosen": -0.5021435022354126, | |
| "logits/rejected": -2.6572070121765137, | |
| "logps/chosen": -0.8906953930854797, | |
| "logps/rejected": -1.0344655513763428, | |
| "loss": 0.9498, | |
| "odds_ratio_loss": 0.5914202928543091, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.08906953036785126, | |
| "rewards/margins": 0.014377007260918617, | |
| "rewards/rejected": -0.10344654321670532, | |
| "sft_loss": 0.8906953930854797, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 0.4573295222539812, | |
| "grad_norm": 1.7812143564224243, | |
| "learning_rate": 4.7185138979101864e-05, | |
| "logits/chosen": -0.507682740688324, | |
| "logits/rejected": -2.593475580215454, | |
| "logps/chosen": -0.9167561531066895, | |
| "logps/rejected": -1.1608246564865112, | |
| "loss": 0.9698, | |
| "odds_ratio_loss": 0.5303933024406433, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.09167560935020447, | |
| "rewards/margins": 0.024406855925917625, | |
| "rewards/rejected": -0.11608246713876724, | |
| "sft_loss": 0.9167561531066895, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.4736627194773377, | |
| "grad_norm": 1.823789119720459, | |
| "learning_rate": 4.6984700273609495e-05, | |
| "logits/chosen": -0.3446227014064789, | |
| "logits/rejected": -2.5670173168182373, | |
| "logps/chosen": -0.8484543561935425, | |
| "logps/rejected": -1.0493099689483643, | |
| "loss": 0.903, | |
| "odds_ratio_loss": 0.5457809567451477, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.08484544605016708, | |
| "rewards/margins": 0.020085560157895088, | |
| "rewards/rejected": -0.10493101179599762, | |
| "sft_loss": 0.8484543561935425, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 0.48999591670069415, | |
| "grad_norm": 1.889367699623108, | |
| "learning_rate": 4.677782485216644e-05, | |
| "logits/chosen": -0.44083428382873535, | |
| "logits/rejected": -2.573638439178467, | |
| "logps/chosen": -0.833287239074707, | |
| "logps/rejected": -1.1050159931182861, | |
| "loss": 0.8838, | |
| "odds_ratio_loss": 0.5049694180488586, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.0833287239074707, | |
| "rewards/margins": 0.02717287465929985, | |
| "rewards/rejected": -0.11050160229206085, | |
| "sft_loss": 0.833287239074707, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.5063291139240507, | |
| "grad_norm": 1.9978969097137451, | |
| "learning_rate": 4.6564573284091765e-05, | |
| "logits/chosen": -0.37475016713142395, | |
| "logits/rejected": -2.530078887939453, | |
| "logps/chosen": -0.9055852890014648, | |
| "logps/rejected": -1.3165007829666138, | |
| "loss": 0.9498, | |
| "odds_ratio_loss": 0.4424561858177185, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.09055852890014648, | |
| "rewards/margins": 0.04109155014157295, | |
| "rewards/rejected": -0.13165007531642914, | |
| "sft_loss": 0.9055852890014648, | |
| "step": 155 | |
| }, | |
| { | |
| "epoch": 0.5226623111474071, | |
| "grad_norm": 2.2659225463867188, | |
| "learning_rate": 4.6345008005522966e-05, | |
| "logits/chosen": -0.27824097871780396, | |
| "logits/rejected": -2.1937553882598877, | |
| "logps/chosen": -0.7978348731994629, | |
| "logps/rejected": -1.7242333889007568, | |
| "loss": 0.8243, | |
| "odds_ratio_loss": 0.26432332396507263, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07978348433971405, | |
| "rewards/margins": 0.09263985604047775, | |
| "rewards/rejected": -0.1724233478307724, | |
| "sft_loss": 0.7978348731994629, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.5389955083707636, | |
| "grad_norm": 2.497178792953491, | |
| "learning_rate": 4.611919330113592e-05, | |
| "logits/chosen": -0.1297352910041809, | |
| "logits/rejected": -1.7525418996810913, | |
| "logps/chosen": -0.9057181477546692, | |
| "logps/rejected": -2.926887035369873, | |
| "loss": 0.9163, | |
| "odds_ratio_loss": 0.10600633919239044, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0905718058347702, | |
| "rewards/margins": 0.20211689174175262, | |
| "rewards/rejected": -0.2926887273788452, | |
| "sft_loss": 0.9057181477546692, | |
| "step": 165 | |
| }, | |
| { | |
| "epoch": 0.5553287055941201, | |
| "grad_norm": 1.9597887992858887, | |
| "learning_rate": 4.588719528532342e-05, | |
| "logits/chosen": -0.20320752263069153, | |
| "logits/rejected": -1.4767993688583374, | |
| "logps/chosen": -0.9343756437301636, | |
| "logps/rejected": -3.7731080055236816, | |
| "loss": 0.9406, | |
| "odds_ratio_loss": 0.062325619161129, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0934375673532486, | |
| "rewards/margins": 0.28387323021888733, | |
| "rewards/rejected": -0.3773108124732971, | |
| "sft_loss": 0.9343756437301636, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.5716619028174765, | |
| "grad_norm": 1.9177374839782715, | |
| "learning_rate": 4.564908188283814e-05, | |
| "logits/chosen": -0.16985416412353516, | |
| "logits/rejected": -1.3144887685775757, | |
| "logps/chosen": -0.8834988474845886, | |
| "logps/rejected": -4.147261619567871, | |
| "loss": 0.8871, | |
| "odds_ratio_loss": 0.03579668328166008, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08834987133741379, | |
| "rewards/margins": 0.3263762891292572, | |
| "rewards/rejected": -0.414726197719574, | |
| "sft_loss": 0.8834988474845886, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 0.587995100040833, | |
| "grad_norm": 1.9115794897079468, | |
| "learning_rate": 4.540492280890555e-05, | |
| "logits/chosen": -0.34701991081237793, | |
| "logits/rejected": -1.1767631769180298, | |
| "logps/chosen": -0.8725343942642212, | |
| "logps/rejected": -4.597090721130371, | |
| "loss": 0.8753, | |
| "odds_ratio_loss": 0.027656957507133484, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08725343644618988, | |
| "rewards/margins": 0.3724556267261505, | |
| "rewards/rejected": -0.4597090780735016, | |
| "sft_loss": 0.8725343942642212, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.6043282972641895, | |
| "grad_norm": 1.7303317785263062, | |
| "learning_rate": 4.515478954881248e-05, | |
| "logits/chosen": -0.37866610288619995, | |
| "logits/rejected": -1.20633065700531, | |
| "logps/chosen": -0.8602287173271179, | |
| "logps/rejected": -4.4016923904418945, | |
| "loss": 0.8625, | |
| "odds_ratio_loss": 0.022269519045948982, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08602287620306015, | |
| "rewards/margins": 0.35414639115333557, | |
| "rewards/rejected": -0.4401692748069763, | |
| "sft_loss": 0.8602287173271179, | |
| "step": 185 | |
| }, | |
| { | |
| "epoch": 0.6206614944875459, | |
| "grad_norm": 5.377653121948242, | |
| "learning_rate": 4.4898755336977673e-05, | |
| "logits/chosen": -0.32614630460739136, | |
| "logits/rejected": -1.1425362825393677, | |
| "logps/chosen": -0.9115999937057495, | |
| "logps/rejected": -4.731467247009277, | |
| "loss": 0.9149, | |
| "odds_ratio_loss": 0.0329320952296257, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09115998446941376, | |
| "rewards/margins": 0.38198670744895935, | |
| "rewards/rejected": -0.4731467366218567, | |
| "sft_loss": 0.9115999937057495, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.6369946917109024, | |
| "grad_norm": 1.6315754652023315, | |
| "learning_rate": 4.463689513550997e-05, | |
| "logits/chosen": -0.3550247550010681, | |
| "logits/rejected": -1.241008996963501, | |
| "logps/chosen": -0.8556908369064331, | |
| "logps/rejected": -4.392051696777344, | |
| "loss": 0.8582, | |
| "odds_ratio_loss": 0.024758270010352135, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08556908369064331, | |
| "rewards/margins": 0.35363608598709106, | |
| "rewards/rejected": -0.4392051696777344, | |
| "sft_loss": 0.8556908369064331, | |
| "step": 195 | |
| }, | |
| { | |
| "epoch": 0.6533278889342589, | |
| "grad_norm": 1.8591830730438232, | |
| "learning_rate": 4.436928561226087e-05, | |
| "logits/chosen": -0.3138996660709381, | |
| "logits/rejected": -1.2280880212783813, | |
| "logps/chosen": -0.901713490486145, | |
| "logps/rejected": -4.212186336517334, | |
| "loss": 0.9045, | |
| "odds_ratio_loss": 0.027884572744369507, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09017134457826614, | |
| "rewards/margins": 0.3310473561286926, | |
| "rewards/rejected": -0.4212186932563782, | |
| "sft_loss": 0.901713490486145, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.6696610861576153, | |
| "grad_norm": 1.8346513509750366, | |
| "learning_rate": 4.409600511837756e-05, | |
| "logits/chosen": -0.23156340420246124, | |
| "logits/rejected": -1.138501524925232, | |
| "logps/chosen": -0.9389227032661438, | |
| "logps/rejected": -4.50938606262207, | |
| "loss": 0.9416, | |
| "odds_ratio_loss": 0.02684590592980385, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09389226883649826, | |
| "rewards/margins": 0.35704633593559265, | |
| "rewards/rejected": -0.4509386122226715, | |
| "sft_loss": 0.9389227032661438, | |
| "step": 205 | |
| }, | |
| { | |
| "epoch": 0.6859942833809718, | |
| "grad_norm": 1.5812890529632568, | |
| "learning_rate": 4.381713366536311e-05, | |
| "logits/chosen": -0.24586188793182373, | |
| "logits/rejected": -1.1232062578201294, | |
| "logps/chosen": -0.859118640422821, | |
| "logps/rejected": -4.883942127227783, | |
| "loss": 0.8609, | |
| "odds_ratio_loss": 0.018183354288339615, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08591187745332718, | |
| "rewards/margins": 0.4024823307991028, | |
| "rewards/rejected": -0.48839420080184937, | |
| "sft_loss": 0.859118640422821, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.7023274806043283, | |
| "grad_norm": 1.8375471830368042, | |
| "learning_rate": 4.353275290165066e-05, | |
| "logits/chosen": -0.15520325303077698, | |
| "logits/rejected": -0.985150933265686, | |
| "logps/chosen": -0.8214719891548157, | |
| "logps/rejected": -4.9630866050720215, | |
| "loss": 0.823, | |
| "odds_ratio_loss": 0.014944888651371002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08214719593524933, | |
| "rewards/margins": 0.4141615033149719, | |
| "rewards/rejected": -0.49630871415138245, | |
| "sft_loss": 0.8214719891548157, | |
| "step": 215 | |
| }, | |
| { | |
| "epoch": 0.7186606778276847, | |
| "grad_norm": 2.556330919265747, | |
| "learning_rate": 4.324294608869817e-05, | |
| "logits/chosen": -0.3221624791622162, | |
| "logits/rejected": -0.9542757272720337, | |
| "logps/chosen": -0.9679568409919739, | |
| "logps/rejected": -5.273629188537598, | |
| "loss": 0.9693, | |
| "odds_ratio_loss": 0.013113953173160553, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0967956930398941, | |
| "rewards/margins": 0.4305671751499176, | |
| "rewards/rejected": -0.5273628830909729, | |
| "sft_loss": 0.9679568409919739, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.7349938750510412, | |
| "grad_norm": 2.346754312515259, | |
| "learning_rate": 4.294779807661105e-05, | |
| "logits/chosen": -0.2690557539463043, | |
| "logits/rejected": -1.0382682085037231, | |
| "logps/chosen": -0.8765946626663208, | |
| "logps/rejected": -5.27601432800293, | |
| "loss": 0.8783, | |
| "odds_ratio_loss": 0.016597110778093338, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08765946328639984, | |
| "rewards/margins": 0.43994197249412537, | |
| "rewards/rejected": -0.527601420879364, | |
| "sft_loss": 0.8765946626663208, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 0.7513270722743978, | |
| "grad_norm": 1.543803095817566, | |
| "learning_rate": 4.270789202181204e-05, | |
| "logits/chosen": -0.3406456410884857, | |
| "logits/rejected": -1.0243726968765259, | |
| "logps/chosen": -0.938999354839325, | |
| "logps/rejected": -5.179265022277832, | |
| "loss": 0.9411, | |
| "odds_ratio_loss": 0.021242566406726837, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09389995038509369, | |
| "rewards/margins": 0.4240265488624573, | |
| "rewards/rejected": -0.5179265141487122, | |
| "sft_loss": 0.938999354839325, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.7676602694977542, | |
| "grad_norm": 1.8233140707015991, | |
| "learning_rate": 4.2403348649073174e-05, | |
| "logits/chosen": -0.2546311020851135, | |
| "logits/rejected": -0.9541198015213013, | |
| "logps/chosen": -0.7934538722038269, | |
| "logps/rejected": -5.206686973571777, | |
| "loss": 0.7944, | |
| "odds_ratio_loss": 0.00939333625137806, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07934539020061493, | |
| "rewards/margins": 0.4413233697414398, | |
| "rewards/rejected": -0.5206687450408936, | |
| "sft_loss": 0.7934538722038269, | |
| "step": 235 | |
| }, | |
| { | |
| "epoch": 0.7839934667211107, | |
| "grad_norm": 1.5795395374298096, | |
| "learning_rate": 4.209370989589013e-05, | |
| "logits/chosen": -0.3499365448951721, | |
| "logits/rejected": -1.0148460865020752, | |
| "logps/chosen": -0.7999550104141235, | |
| "logps/rejected": -5.092876434326172, | |
| "loss": 0.8013, | |
| "odds_ratio_loss": 0.013519616797566414, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07999551296234131, | |
| "rewards/margins": 0.4292922019958496, | |
| "rewards/rejected": -0.5092877149581909, | |
| "sft_loss": 0.7999550104141235, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.8003266639444672, | |
| "grad_norm": 1.8533421754837036, | |
| "learning_rate": 4.177906641879575e-05, | |
| "logits/chosen": -0.3404558598995209, | |
| "logits/rejected": -1.0349606275558472, | |
| "logps/chosen": -0.9274562001228333, | |
| "logps/rejected": -4.853649616241455, | |
| "loss": 0.9298, | |
| "odds_ratio_loss": 0.023337265476584435, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09274561703205109, | |
| "rewards/margins": 0.3926192820072174, | |
| "rewards/rejected": -0.4853649139404297, | |
| "sft_loss": 0.9274562001228333, | |
| "step": 245 | |
| }, | |
| { | |
| "epoch": 0.8166598611678236, | |
| "grad_norm": 1.792864441871643, | |
| "learning_rate": 4.145951033961395e-05, | |
| "logits/chosen": -0.39237070083618164, | |
| "logits/rejected": -1.20218825340271, | |
| "logps/chosen": -0.8956495523452759, | |
| "logps/rejected": -4.416346549987793, | |
| "loss": 0.8988, | |
| "odds_ratio_loss": 0.03134030103683472, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08956494927406311, | |
| "rewards/margins": 0.3520697355270386, | |
| "rewards/rejected": -0.4416346549987793, | |
| "sft_loss": 0.8956495523452759, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.8329930583911801, | |
| "grad_norm": 1.388491153717041, | |
| "learning_rate": 4.1135135218488206e-05, | |
| "logits/chosen": -0.4966661036014557, | |
| "logits/rejected": -1.301102638244629, | |
| "logps/chosen": -0.822670578956604, | |
| "logps/rejected": -4.420852184295654, | |
| "loss": 0.8251, | |
| "odds_ratio_loss": 0.02416636422276497, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08226706087589264, | |
| "rewards/margins": 0.3598182201385498, | |
| "rewards/rejected": -0.4420852065086365, | |
| "sft_loss": 0.822670578956604, | |
| "step": 255 | |
| }, | |
| { | |
| "epoch": 0.8493262556145366, | |
| "grad_norm": 1.7927485704421997, | |
| "learning_rate": 4.080603602648887e-05, | |
| "logits/chosen": -0.38711199164390564, | |
| "logits/rejected": -1.1762360334396362, | |
| "logps/chosen": -0.8747900128364563, | |
| "logps/rejected": -4.844551086425781, | |
| "loss": 0.8772, | |
| "odds_ratio_loss": 0.024142403155565262, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08747899532318115, | |
| "rewards/margins": 0.3969760835170746, | |
| "rewards/rejected": -0.4844551086425781, | |
| "sft_loss": 0.8747900128364563, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.865659452837893, | |
| "grad_norm": 1.763637661933899, | |
| "learning_rate": 4.047230911780737e-05, | |
| "logits/chosen": -0.4611227512359619, | |
| "logits/rejected": -1.092810034751892, | |
| "logps/chosen": -0.8380341529846191, | |
| "logps/rejected": -4.902915000915527, | |
| "loss": 0.8414, | |
| "odds_ratio_loss": 0.03330729901790619, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08380343019962311, | |
| "rewards/margins": 0.4064881205558777, | |
| "rewards/rejected": -0.4902915358543396, | |
| "sft_loss": 0.8380341529846191, | |
| "step": 265 | |
| }, | |
| { | |
| "epoch": 0.8819926500612495, | |
| "grad_norm": 1.7203322649002075, | |
| "learning_rate": 4.0134052201545447e-05, | |
| "logits/chosen": -0.24241454899311066, | |
| "logits/rejected": -0.9308720827102661, | |
| "logps/chosen": -0.8675628900527954, | |
| "logps/rejected": -5.390278339385986, | |
| "loss": 0.8685, | |
| "odds_ratio_loss": 0.009246011264622211, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08675628900527954, | |
| "rewards/margins": 0.4522715210914612, | |
| "rewards/rejected": -0.5390278100967407, | |
| "sft_loss": 0.8675628900527954, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.898325847284606, | |
| "grad_norm": 1.9116899967193604, | |
| "learning_rate": 3.9791364313107804e-05, | |
| "logits/chosen": -0.401073157787323, | |
| "logits/rejected": -0.9959608912467957, | |
| "logps/chosen": -0.8706862330436707, | |
| "logps/rejected": -5.204241752624512, | |
| "loss": 0.872, | |
| "odds_ratio_loss": 0.0133589506149292, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08706863224506378, | |
| "rewards/margins": 0.4333556294441223, | |
| "rewards/rejected": -0.5204241871833801, | |
| "sft_loss": 0.8706862330436707, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 0.9146590445079624, | |
| "grad_norm": 1.9296520948410034, | |
| "learning_rate": 3.9444345785206285e-05, | |
| "logits/chosen": -0.40887147188186646, | |
| "logits/rejected": -0.9652082324028015, | |
| "logps/chosen": -0.8309628367424011, | |
| "logps/rejected": -5.662296295166016, | |
| "loss": 0.8318, | |
| "odds_ratio_loss": 0.008708246983587742, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08309628069400787, | |
| "rewards/margins": 0.48313331604003906, | |
| "rewards/rejected": -0.5662296414375305, | |
| "sft_loss": 0.8309628367424011, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.9309922417313189, | |
| "grad_norm": 3.055554151535034, | |
| "learning_rate": 3.909309821848432e-05, | |
| "logits/chosen": -0.3440246284008026, | |
| "logits/rejected": -0.8914414644241333, | |
| "logps/chosen": -0.8914461135864258, | |
| "logps/rejected": -5.75334358215332, | |
| "loss": 0.8921, | |
| "odds_ratio_loss": 0.0068351454101502895, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08914461731910706, | |
| "rewards/margins": 0.48618975281715393, | |
| "rewards/rejected": -0.575334370136261, | |
| "sft_loss": 0.8914461135864258, | |
| "step": 285 | |
| }, | |
| { | |
| "epoch": 0.9473254389546754, | |
| "grad_norm": 1.5384926795959473, | |
| "learning_rate": 3.873772445177015e-05, | |
| "logits/chosen": -0.37930798530578613, | |
| "logits/rejected": -0.9451101422309875, | |
| "logps/chosen": -0.8741447329521179, | |
| "logps/rejected": -5.84790563583374, | |
| "loss": 0.8751, | |
| "odds_ratio_loss": 0.009465084411203861, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08741448074579239, | |
| "rewards/margins": 0.49737611413002014, | |
| "rewards/rejected": -0.5847905874252319, | |
| "sft_loss": 0.8741447329521179, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.9636586361780318, | |
| "grad_norm": 2.4715216159820557, | |
| "learning_rate": 3.837832853196751e-05, | |
| "logits/chosen": -0.3153218626976013, | |
| "logits/rejected": -0.9612942934036255, | |
| "logps/chosen": -0.8935874700546265, | |
| "logps/rejected": -5.140650749206543, | |
| "loss": 0.8956, | |
| "odds_ratio_loss": 0.01970650628209114, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08935876190662384, | |
| "rewards/margins": 0.4247063100337982, | |
| "rewards/rejected": -0.514065146446228, | |
| "sft_loss": 0.8935874700546265, | |
| "step": 295 | |
| }, | |
| { | |
| "epoch": 0.9799918334013883, | |
| "grad_norm": 1.9313322305679321, | |
| "learning_rate": 3.8015015683592575e-05, | |
| "logits/chosen": -0.30432718992233276, | |
| "logits/rejected": -1.099277377128601, | |
| "logps/chosen": -0.8711298108100891, | |
| "logps/rejected": -4.722751140594482, | |
| "loss": 0.873, | |
| "odds_ratio_loss": 0.018964512273669243, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08711298555135727, | |
| "rewards/margins": 0.3851621747016907, | |
| "rewards/rejected": -0.47227516770362854, | |
| "sft_loss": 0.8711298108100891, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.9963250306247448, | |
| "grad_norm": 2.2331244945526123, | |
| "learning_rate": 3.764789227796616e-05, | |
| "logits/chosen": -0.3758210837841034, | |
| "logits/rejected": -1.1826483011245728, | |
| "logps/chosen": -0.8142495155334473, | |
| "logps/rejected": -4.7738471031188965, | |
| "loss": 0.816, | |
| "odds_ratio_loss": 0.017485955730080605, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08142495155334473, | |
| "rewards/margins": 0.3959597945213318, | |
| "rewards/rejected": -0.4773847460746765, | |
| "sft_loss": 0.8142495155334473, | |
| "step": 305 | |
| }, | |
| { | |
| "epoch": 1.0126582278481013, | |
| "grad_norm": 1.7077336311340332, | |
| "learning_rate": 3.727706580207021e-05, | |
| "logits/chosen": -0.38093143701553345, | |
| "logits/rejected": -1.096470594406128, | |
| "logps/chosen": -0.8485842943191528, | |
| "logps/rejected": -4.818546295166016, | |
| "loss": 0.8503, | |
| "odds_ratio_loss": 0.01746557094156742, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08485842496156693, | |
| "rewards/margins": 0.3969961702823639, | |
| "rewards/rejected": -0.4818546175956726, | |
| "sft_loss": 0.8485842943191528, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 1.0289914250714578, | |
| "grad_norm": 1.8160737752914429, | |
| "learning_rate": 3.69026448270775e-05, | |
| "logits/chosen": -0.37943610548973083, | |
| "logits/rejected": -1.0238980054855347, | |
| "logps/chosen": -0.8509234189987183, | |
| "logps/rejected": -5.104341506958008, | |
| "loss": 0.8521, | |
| "odds_ratio_loss": 0.011288631707429886, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08509235084056854, | |
| "rewards/margins": 0.4253418445587158, | |
| "rewards/rejected": -0.5104341506958008, | |
| "sft_loss": 0.8509234189987183, | |
| "step": 315 | |
| }, | |
| { | |
| "epoch": 1.0453246222948143, | |
| "grad_norm": 2.9576852321624756, | |
| "learning_rate": 3.652473897656409e-05, | |
| "logits/chosen": -0.30405062437057495, | |
| "logits/rejected": -0.9463305473327637, | |
| "logps/chosen": -0.9074727296829224, | |
| "logps/rejected": -5.486522674560547, | |
| "loss": 0.9089, | |
| "odds_ratio_loss": 0.013948513194918633, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09074728190898895, | |
| "rewards/margins": 0.45790499448776245, | |
| "rewards/rejected": -0.5486522912979126, | |
| "sft_loss": 0.9074727296829224, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 1.0616578195181707, | |
| "grad_norm": 1.9609084129333496, | |
| "learning_rate": 3.6143458894413465e-05, | |
| "logits/chosen": -0.39042356610298157, | |
| "logits/rejected": -1.0480425357818604, | |
| "logps/chosen": -0.7837303876876831, | |
| "logps/rejected": -5.29976749420166, | |
| "loss": 0.7846, | |
| "odds_ratio_loss": 0.008555177599191666, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0783730298280716, | |
| "rewards/margins": 0.45160380005836487, | |
| "rewards/rejected": -0.5299768447875977, | |
| "sft_loss": 0.7837303876876831, | |
| "step": 325 | |
| }, | |
| { | |
| "epoch": 1.0779910167415272, | |
| "grad_norm": 2.609851360321045, | |
| "learning_rate": 3.575891621242209e-05, | |
| "logits/chosen": -0.35802769660949707, | |
| "logits/rejected": -0.9912473559379578, | |
| "logps/chosen": -0.8010882139205933, | |
| "logps/rejected": -5.255009651184082, | |
| "loss": 0.8022, | |
| "odds_ratio_loss": 0.010739787481725216, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08010882139205933, | |
| "rewards/margins": 0.4453921914100647, | |
| "rewards/rejected": -0.525501012802124, | |
| "sft_loss": 0.8010882139205933, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 1.0943242139648837, | |
| "grad_norm": 1.9164317846298218, | |
| "learning_rate": 3.537122351761568e-05, | |
| "logits/chosen": -0.29361772537231445, | |
| "logits/rejected": -0.9894407987594604, | |
| "logps/chosen": -0.8663752675056458, | |
| "logps/rejected": -5.1811041831970215, | |
| "loss": 0.8682, | |
| "odds_ratio_loss": 0.01779218763113022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08663753420114517, | |
| "rewards/margins": 0.43147286772727966, | |
| "rewards/rejected": -0.5181103944778442, | |
| "sft_loss": 0.8663752675056458, | |
| "step": 335 | |
| }, | |
| { | |
| "epoch": 1.1106574111882401, | |
| "grad_norm": 1.8551751375198364, | |
| "learning_rate": 3.4980494319285775e-05, | |
| "logits/chosen": -0.2934134900569916, | |
| "logits/rejected": -0.9298968315124512, | |
| "logps/chosen": -0.8729950189590454, | |
| "logps/rejected": -5.402509689331055, | |
| "loss": 0.8739, | |
| "odds_ratio_loss": 0.009193124249577522, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08729950338602066, | |
| "rewards/margins": 0.45295149087905884, | |
| "rewards/rejected": -0.5402510166168213, | |
| "sft_loss": 0.8729950189590454, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 1.1269906084115966, | |
| "grad_norm": 1.8746849298477173, | |
| "learning_rate": 3.45868430157563e-05, | |
| "logits/chosen": -0.27233654260635376, | |
| "logits/rejected": -0.9365426301956177, | |
| "logps/chosen": -0.8479824066162109, | |
| "logps/rejected": -5.225064754486084, | |
| "loss": 0.8493, | |
| "odds_ratio_loss": 0.013331398367881775, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08479823917150497, | |
| "rewards/margins": 0.43770831823349, | |
| "rewards/rejected": -0.522506594657898, | |
| "sft_loss": 0.8479824066162109, | |
| "step": 345 | |
| }, | |
| { | |
| "epoch": 1.143323805634953, | |
| "grad_norm": 1.7111650705337524, | |
| "learning_rate": 3.4190384860889824e-05, | |
| "logits/chosen": -0.366852343082428, | |
| "logits/rejected": -1.0381531715393066, | |
| "logps/chosen": -0.8182533383369446, | |
| "logps/rejected": -5.278301239013672, | |
| "loss": 0.8204, | |
| "odds_ratio_loss": 0.02150268480181694, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08182532340288162, | |
| "rewards/margins": 0.4460047781467438, | |
| "rewards/rejected": -0.5278300642967224, | |
| "sft_loss": 0.8182533383369446, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 1.1596570028583095, | |
| "grad_norm": 1.9346224069595337, | |
| "learning_rate": 3.379123593034342e-05, | |
| "logits/chosen": -0.2924169898033142, | |
| "logits/rejected": -1.028192400932312, | |
| "logps/chosen": -0.9392467737197876, | |
| "logps/rejected": -5.247817516326904, | |
| "loss": 0.9409, | |
| "odds_ratio_loss": 0.0163104310631752, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09392467886209488, | |
| "rewards/margins": 0.4308570921421051, | |
| "rewards/rejected": -0.5247817635536194, | |
| "sft_loss": 0.9392467737197876, | |
| "step": 355 | |
| }, | |
| { | |
| "epoch": 1.175990200081666, | |
| "grad_norm": 1.9083410501480103, | |
| "learning_rate": 3.3389513087583754e-05, | |
| "logits/chosen": -0.1519567221403122, | |
| "logits/rejected": -1.0742218494415283, | |
| "logps/chosen": -0.8435864448547363, | |
| "logps/rejected": -4.644381523132324, | |
| "loss": 0.8465, | |
| "odds_ratio_loss": 0.028982138261198997, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08435864001512527, | |
| "rewards/margins": 0.3800795078277588, | |
| "rewards/rejected": -0.4644381105899811, | |
| "sft_loss": 0.8435864448547363, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 1.1923233973050225, | |
| "grad_norm": 1.8590240478515625, | |
| "learning_rate": 3.2985333949671685e-05, | |
| "logits/chosen": -0.43951839208602905, | |
| "logits/rejected": -1.1476421356201172, | |
| "logps/chosen": -0.8573881387710571, | |
| "logps/rejected": -4.641860485076904, | |
| "loss": 0.8605, | |
| "odds_ratio_loss": 0.030726104974746704, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08573882281780243, | |
| "rewards/margins": 0.3784472346305847, | |
| "rewards/rejected": -0.4641861021518707, | |
| "sft_loss": 0.8573881387710571, | |
| "step": 365 | |
| }, | |
| { | |
| "epoch": 1.208656594528379, | |
| "grad_norm": 2.064635753631592, | |
| "learning_rate": 3.257881685282609e-05, | |
| "logits/chosen": -0.2586503028869629, | |
| "logits/rejected": -1.1886835098266602, | |
| "logps/chosen": -0.7855366468429565, | |
| "logps/rejected": -4.374009132385254, | |
| "loss": 0.7886, | |
| "odds_ratio_loss": 0.03049846924841404, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07855366170406342, | |
| "rewards/margins": 0.3588472604751587, | |
| "rewards/rejected": -0.4374009668827057, | |
| "sft_loss": 0.7855366468429565, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 1.2249897917517354, | |
| "grad_norm": 1.986666202545166, | |
| "learning_rate": 3.217008081777726e-05, | |
| "logits/chosen": -0.3716577887535095, | |
| "logits/rejected": -1.2067687511444092, | |
| "logps/chosen": -0.8298207521438599, | |
| "logps/rejected": -4.59980583190918, | |
| "loss": 0.8315, | |
| "odds_ratio_loss": 0.016955239698290825, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08298207074403763, | |
| "rewards/margins": 0.3769984841346741, | |
| "rewards/rejected": -0.4599805772304535, | |
| "sft_loss": 0.8298207521438599, | |
| "step": 375 | |
| }, | |
| { | |
| "epoch": 1.2413229889750919, | |
| "grad_norm": 1.8263311386108398, | |
| "learning_rate": 3.175924551491977e-05, | |
| "logits/chosen": -0.3963642716407776, | |
| "logits/rejected": -1.1271188259124756, | |
| "logps/chosen": -0.8212121725082397, | |
| "logps/rejected": -4.6982741355896, | |
| "loss": 0.8234, | |
| "odds_ratio_loss": 0.02219826728105545, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08212121576070786, | |
| "rewards/margins": 0.3877061605453491, | |
| "rewards/rejected": -0.4698273539543152, | |
| "sft_loss": 0.8212121725082397, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 1.2576561861984483, | |
| "grad_norm": 1.946442723274231, | |
| "learning_rate": 3.13464312292752e-05, | |
| "logits/chosen": -0.343479186296463, | |
| "logits/rejected": -0.9710591435432434, | |
| "logps/chosen": -0.8622458577156067, | |
| "logps/rejected": -5.370631217956543, | |
| "loss": 0.8646, | |
| "odds_ratio_loss": 0.023645440116524696, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08622457832098007, | |
| "rewards/margins": 0.450838565826416, | |
| "rewards/rejected": -0.5370631217956543, | |
| "sft_loss": 0.8622458577156067, | |
| "step": 385 | |
| }, | |
| { | |
| "epoch": 1.2739893834218048, | |
| "grad_norm": 1.7212151288986206, | |
| "learning_rate": 3.093175882527491e-05, | |
| "logits/chosen": -0.3060234785079956, | |
| "logits/rejected": -1.022698163986206, | |
| "logps/chosen": -0.8830709457397461, | |
| "logps/rejected": -5.35630464553833, | |
| "loss": 0.8848, | |
| "odds_ratio_loss": 0.01716751977801323, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08830709010362625, | |
| "rewards/margins": 0.44732338190078735, | |
| "rewards/rejected": -0.535630464553833, | |
| "sft_loss": 0.8830709457397461, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 1.2903225806451613, | |
| "grad_norm": 2.029407024383545, | |
| "learning_rate": 3.0515349711373146e-05, | |
| "logits/chosen": -0.13591887056827545, | |
| "logits/rejected": -0.9809386134147644, | |
| "logps/chosen": -0.880872368812561, | |
| "logps/rejected": -5.28397798538208, | |
| "loss": 0.8826, | |
| "odds_ratio_loss": 0.01720813475549221, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08808723837137222, | |
| "rewards/margins": 0.4403105676174164, | |
| "rewards/rejected": -0.5283978581428528, | |
| "sft_loss": 0.880872368812561, | |
| "step": 395 | |
| }, | |
| { | |
| "epoch": 1.3066557778685177, | |
| "grad_norm": 1.8320516347885132, | |
| "learning_rate": 3.0097325804500864e-05, | |
| "logits/chosen": -0.3814406991004944, | |
| "logits/rejected": -1.0283949375152588, | |
| "logps/chosen": -0.7927986979484558, | |
| "logps/rejected": -5.100050926208496, | |
| "loss": 0.7945, | |
| "odds_ratio_loss": 0.016741547733545303, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07927986979484558, | |
| "rewards/margins": 0.4307252764701843, | |
| "rewards/rejected": -0.5100051164627075, | |
| "sft_loss": 0.7927986979484558, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 1.3229889750918742, | |
| "grad_norm": 10.229716300964355, | |
| "learning_rate": 2.9677809494370712e-05, | |
| "logits/chosen": -0.3555116057395935, | |
| "logits/rejected": -0.966505229473114, | |
| "logps/chosen": -0.9298495054244995, | |
| "logps/rejected": -5.674726963043213, | |
| "loss": 0.9316, | |
| "odds_ratio_loss": 0.017632385715842247, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09298495203256607, | |
| "rewards/margins": 0.4744877219200134, | |
| "rewards/rejected": -0.5674726963043213, | |
| "sft_loss": 0.9298495054244995, | |
| "step": 405 | |
| }, | |
| { | |
| "epoch": 1.3393221723152307, | |
| "grad_norm": 2.0804049968719482, | |
| "learning_rate": 2.925692360764356e-05, | |
| "logits/chosen": -0.2548617124557495, | |
| "logits/rejected": -1.0827587842941284, | |
| "logps/chosen": -0.8656631708145142, | |
| "logps/rejected": -4.993069648742676, | |
| "loss": 0.8675, | |
| "odds_ratio_loss": 0.018203863874077797, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08656631410121918, | |
| "rewards/margins": 0.41274064779281616, | |
| "rewards/rejected": -0.4993070662021637, | |
| "sft_loss": 0.8656631708145142, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 1.3556553695385871, | |
| "grad_norm": 2.4393317699432373, | |
| "learning_rate": 2.8834791371967142e-05, | |
| "logits/chosen": -0.18482331931591034, | |
| "logits/rejected": -1.0267010927200317, | |
| "logps/chosen": -0.868569016456604, | |
| "logps/rejected": -5.216423511505127, | |
| "loss": 0.8702, | |
| "odds_ratio_loss": 0.01657196693122387, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0868569016456604, | |
| "rewards/margins": 0.4347854554653168, | |
| "rewards/rejected": -0.5216423869132996, | |
| "sft_loss": 0.868569016456604, | |
| "step": 415 | |
| }, | |
| { | |
| "epoch": 1.3719885667619436, | |
| "grad_norm": 1.7740427255630493, | |
| "learning_rate": 2.8411536379897203e-05, | |
| "logits/chosen": -0.27949684858322144, | |
| "logits/rejected": -1.0097931623458862, | |
| "logps/chosen": -0.8230732083320618, | |
| "logps/rejected": -5.000334739685059, | |
| "loss": 0.8254, | |
| "odds_ratio_loss": 0.023418055847287178, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08230730891227722, | |
| "rewards/margins": 0.4177261292934418, | |
| "rewards/rejected": -0.5000334978103638, | |
| "sft_loss": 0.8230732083320618, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 1.3883217639853, | |
| "grad_norm": 1.8371398448944092, | |
| "learning_rate": 2.7987282552711973e-05, | |
| "logits/chosen": -0.32712841033935547, | |
| "logits/rejected": -1.0544841289520264, | |
| "logps/chosen": -0.8350850939750671, | |
| "logps/rejected": -5.021151065826416, | |
| "loss": 0.837, | |
| "odds_ratio_loss": 0.01911192014813423, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08350850641727448, | |
| "rewards/margins": 0.41860657930374146, | |
| "rewards/rejected": -0.5021151304244995, | |
| "sft_loss": 0.8350850939750671, | |
| "step": 425 | |
| }, | |
| { | |
| "epoch": 1.4046549612086565, | |
| "grad_norm": 2.2567172050476074, | |
| "learning_rate": 2.7562154104130178e-05, | |
| "logits/chosen": -0.39351749420166016, | |
| "logits/rejected": -1.124728798866272, | |
| "logps/chosen": -0.7841497659683228, | |
| "logps/rejected": -5.364629745483398, | |
| "loss": 0.7859, | |
| "odds_ratio_loss": 0.017894607037305832, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07841498404741287, | |
| "rewards/margins": 0.458048015832901, | |
| "rewards/rejected": -0.5364629626274109, | |
| "sft_loss": 0.7841497659683228, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 1.4209881584320132, | |
| "grad_norm": 2.822150945663452, | |
| "learning_rate": 2.7136275503943626e-05, | |
| "logits/chosen": -0.14898474514484406, | |
| "logits/rejected": -0.778265655040741, | |
| "logps/chosen": -0.8347780108451843, | |
| "logps/rejected": -5.595808506011963, | |
| "loss": 0.8356, | |
| "odds_ratio_loss": 0.007863040082156658, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08347781002521515, | |
| "rewards/margins": 0.4761030673980713, | |
| "rewards/rejected": -0.55958092212677, | |
| "sft_loss": 0.8347780108451843, | |
| "step": 435 | |
| }, | |
| { | |
| "epoch": 1.4373213556553694, | |
| "grad_norm": 1.9301971197128296, | |
| "learning_rate": 2.670977144157475e-05, | |
| "logits/chosen": -0.3294993042945862, | |
| "logits/rejected": -0.8833429217338562, | |
| "logps/chosen": -0.8435837030410767, | |
| "logps/rejected": -5.550654888153076, | |
| "loss": 0.845, | |
| "odds_ratio_loss": 0.013694020919501781, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08435836434364319, | |
| "rewards/margins": 0.47070708870887756, | |
| "rewards/rejected": -0.5550654530525208, | |
| "sft_loss": 0.8435837030410767, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 1.4536545528787261, | |
| "grad_norm": 1.864906907081604, | |
| "learning_rate": 2.628276678956974e-05, | |
| "logits/chosen": -0.38186246156692505, | |
| "logits/rejected": -0.9432054758071899, | |
| "logps/chosen": -0.7808457612991333, | |
| "logps/rejected": -5.465119361877441, | |
| "loss": 0.7817, | |
| "odds_ratio_loss": 0.008780199103057384, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07808457314968109, | |
| "rewards/margins": 0.4684273600578308, | |
| "rewards/rejected": -0.5465119481086731, | |
| "sft_loss": 0.7808457612991333, | |
| "step": 445 | |
| }, | |
| { | |
| "epoch": 1.4699877501020824, | |
| "grad_norm": 1.652700424194336, | |
| "learning_rate": 2.5855386567038304e-05, | |
| "logits/chosen": -0.2546243369579315, | |
| "logits/rejected": -0.8346729278564453, | |
| "logps/chosen": -0.8080782890319824, | |
| "logps/rejected": -5.827437400817871, | |
| "loss": 0.8087, | |
| "odds_ratio_loss": 0.005900132469832897, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08080783486366272, | |
| "rewards/margins": 0.5019358992576599, | |
| "rewards/rejected": -0.582743763923645, | |
| "sft_loss": 0.8080782890319824, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 1.486320947325439, | |
| "grad_norm": 1.7943625450134277, | |
| "learning_rate": 2.5427755903050232e-05, | |
| "logits/chosen": -0.3007858395576477, | |
| "logits/rejected": -0.751599907875061, | |
| "logps/chosen": -0.8181053400039673, | |
| "logps/rejected": -5.938103199005127, | |
| "loss": 0.8187, | |
| "odds_ratio_loss": 0.005451642908155918, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08181053400039673, | |
| "rewards/margins": 0.511999785900116, | |
| "rewards/rejected": -0.5938103199005127, | |
| "sft_loss": 0.8181053400039673, | |
| "step": 455 | |
| }, | |
| { | |
| "epoch": 1.5026541445487953, | |
| "grad_norm": 1.9405500888824463, | |
| "learning_rate": 2.5e-05, | |
| "logits/chosen": -0.3018740713596344, | |
| "logits/rejected": -0.8147015571594238, | |
| "logps/chosen": -0.8223198652267456, | |
| "logps/rejected": -6.007750034332275, | |
| "loss": 0.8231, | |
| "odds_ratio_loss": 0.007952779531478882, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08223198354244232, | |
| "rewards/margins": 0.518543004989624, | |
| "rewards/rejected": -0.6007749438285828, | |
| "sft_loss": 0.8223198652267456, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 1.518987341772152, | |
| "grad_norm": 29.944835662841797, | |
| "learning_rate": 2.4572244096949774e-05, | |
| "logits/chosen": -0.3322751522064209, | |
| "logits/rejected": -0.8299162983894348, | |
| "logps/chosen": -0.8652318120002747, | |
| "logps/rejected": -5.8731489181518555, | |
| "loss": 0.8665, | |
| "odds_ratio_loss": 0.012552870437502861, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08652317523956299, | |
| "rewards/margins": 0.5007917881011963, | |
| "rewards/rejected": -0.5873149633407593, | |
| "sft_loss": 0.8652318120002747, | |
| "step": 465 | |
| }, | |
| { | |
| "epoch": 1.5353205389955082, | |
| "grad_norm": 4.769914150238037, | |
| "learning_rate": 2.41446134329617e-05, | |
| "logits/chosen": -0.30929869413375854, | |
| "logits/rejected": -0.7486665844917297, | |
| "logps/chosen": -0.8595030903816223, | |
| "logps/rejected": -6.054928779602051, | |
| "loss": 0.8602, | |
| "odds_ratio_loss": 0.007209992501884699, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08595031499862671, | |
| "rewards/margins": 0.5195425748825073, | |
| "rewards/rejected": -0.6054929494857788, | |
| "sft_loss": 0.8595030903816223, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 1.551653736218865, | |
| "grad_norm": 2.072922468185425, | |
| "learning_rate": 2.3717233210430256e-05, | |
| "logits/chosen": -0.3498794436454773, | |
| "logits/rejected": -0.9254905581474304, | |
| "logps/chosen": -0.836539626121521, | |
| "logps/rejected": -5.841341495513916, | |
| "loss": 0.8373, | |
| "odds_ratio_loss": 0.007826647721230984, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08365396410226822, | |
| "rewards/margins": 0.5004801750183105, | |
| "rewards/rejected": -0.5841341018676758, | |
| "sft_loss": 0.836539626121521, | |
| "step": 475 | |
| }, | |
| { | |
| "epoch": 1.5679869334422212, | |
| "grad_norm": 2.006734609603882, | |
| "learning_rate": 2.3290228558425264e-05, | |
| "logits/chosen": -0.21343150734901428, | |
| "logits/rejected": -0.997966468334198, | |
| "logps/chosen": -0.8389822244644165, | |
| "logps/rejected": -5.127771854400635, | |
| "loss": 0.8408, | |
| "odds_ratio_loss": 0.018000155687332153, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08389821648597717, | |
| "rewards/margins": 0.4288790225982666, | |
| "rewards/rejected": -0.5127772092819214, | |
| "sft_loss": 0.8389822244644165, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 1.5843201306655779, | |
| "grad_norm": 2.0465636253356934, | |
| "learning_rate": 2.2863724496056377e-05, | |
| "logits/chosen": -0.36094948649406433, | |
| "logits/rejected": -1.0251775979995728, | |
| "logps/chosen": -0.8593729138374329, | |
| "logps/rejected": -5.164275169372559, | |
| "loss": 0.8608, | |
| "odds_ratio_loss": 0.014115150086581707, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08593729883432388, | |
| "rewards/margins": 0.43049025535583496, | |
| "rewards/rejected": -0.5164275169372559, | |
| "sft_loss": 0.8593729138374329, | |
| "step": 485 | |
| }, | |
| { | |
| "epoch": 1.600653327888934, | |
| "grad_norm": 1.6602107286453247, | |
| "learning_rate": 2.2437845895869828e-05, | |
| "logits/chosen": -0.35683882236480713, | |
| "logits/rejected": -1.1014772653579712, | |
| "logps/chosen": -0.8006218671798706, | |
| "logps/rejected": -4.8285603523254395, | |
| "loss": 0.8022, | |
| "odds_ratio_loss": 0.01606990396976471, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08006219565868378, | |
| "rewards/margins": 0.402793824672699, | |
| "rewards/rejected": -0.48285597562789917, | |
| "sft_loss": 0.8006218671798706, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 1.6169865251122908, | |
| "grad_norm": 2.5427048206329346, | |
| "learning_rate": 2.2012717447288033e-05, | |
| "logits/chosen": -0.3278926610946655, | |
| "logits/rejected": -1.0268025398254395, | |
| "logps/chosen": -0.8360313177108765, | |
| "logps/rejected": -4.992895126342773, | |
| "loss": 0.8377, | |
| "odds_ratio_loss": 0.017158646136522293, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0836031436920166, | |
| "rewards/margins": 0.41568630933761597, | |
| "rewards/rejected": -0.49928945302963257, | |
| "sft_loss": 0.8360313177108765, | |
| "step": 495 | |
| }, | |
| { | |
| "epoch": 1.633319722335647, | |
| "grad_norm": 2.533067226409912, | |
| "learning_rate": 2.1588463620102796e-05, | |
| "logits/chosen": -0.3250111937522888, | |
| "logits/rejected": -0.9380800127983093, | |
| "logps/chosen": -0.7811130881309509, | |
| "logps/rejected": -5.476039409637451, | |
| "loss": 0.7821, | |
| "odds_ratio_loss": 0.009653602726757526, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07811131328344345, | |
| "rewards/margins": 0.46949267387390137, | |
| "rewards/rejected": -0.547603964805603, | |
| "sft_loss": 0.7811130881309509, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 1.6496529195590037, | |
| "grad_norm": 2.0488734245300293, | |
| "learning_rate": 2.116520862803286e-05, | |
| "logits/chosen": -0.21985015273094177, | |
| "logits/rejected": -0.9144424200057983, | |
| "logps/chosen": -0.7239242792129517, | |
| "logps/rejected": -5.2707695960998535, | |
| "loss": 0.726, | |
| "odds_ratio_loss": 0.0210262518376112, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07239242643117905, | |
| "rewards/margins": 0.4546845555305481, | |
| "rewards/rejected": -0.5270770192146301, | |
| "sft_loss": 0.7239242792129517, | |
| "step": 505 | |
| }, | |
| { | |
| "epoch": 1.6659861167823602, | |
| "grad_norm": 1.8625127077102661, | |
| "learning_rate": 2.074307639235645e-05, | |
| "logits/chosen": -0.16022160649299622, | |
| "logits/rejected": -0.7577518224716187, | |
| "logps/chosen": -0.8772943615913391, | |
| "logps/rejected": -5.823426246643066, | |
| "loss": 0.8781, | |
| "odds_ratio_loss": 0.007620103657245636, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08772943168878555, | |
| "rewards/margins": 0.49461308121681213, | |
| "rewards/rejected": -0.5823425650596619, | |
| "sft_loss": 0.8772943615913391, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 1.6823193140057167, | |
| "grad_norm": 2.045032024383545, | |
| "learning_rate": 2.0322190505629297e-05, | |
| "logits/chosen": -0.3373351991176605, | |
| "logits/rejected": -0.8180400729179382, | |
| "logps/chosen": -0.7550835013389587, | |
| "logps/rejected": -6.03420877456665, | |
| "loss": 0.7556, | |
| "odds_ratio_loss": 0.004965050611644983, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07550835609436035, | |
| "rewards/margins": 0.5279124975204468, | |
| "rewards/rejected": -0.6034208536148071, | |
| "sft_loss": 0.7550835013389587, | |
| "step": 515 | |
| }, | |
| { | |
| "epoch": 1.6986525112290731, | |
| "grad_norm": 2.0969460010528564, | |
| "learning_rate": 1.9902674195499145e-05, | |
| "logits/chosen": -0.16994231939315796, | |
| "logits/rejected": -0.7098522782325745, | |
| "logps/chosen": -0.8867838978767395, | |
| "logps/rejected": -6.680023193359375, | |
| "loss": 0.8871, | |
| "odds_ratio_loss": 0.0027744858525693417, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08867839723825455, | |
| "rewards/margins": 0.5793240070343018, | |
| "rewards/rejected": -0.6680023670196533, | |
| "sft_loss": 0.8867838978767395, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 1.7149857084524296, | |
| "grad_norm": 2.6975438594818115, | |
| "learning_rate": 1.948465028862686e-05, | |
| "logits/chosen": -0.2635698616504669, | |
| "logits/rejected": -0.7826798558235168, | |
| "logps/chosen": -0.8126001358032227, | |
| "logps/rejected": -6.0560736656188965, | |
| "loss": 0.8132, | |
| "odds_ratio_loss": 0.005659271962940693, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08126001060009003, | |
| "rewards/margins": 0.5243473052978516, | |
| "rewards/rejected": -0.6056073904037476, | |
| "sft_loss": 0.8126001358032227, | |
| "step": 525 | |
| }, | |
| { | |
| "epoch": 1.731318905675786, | |
| "grad_norm": 2.2599716186523438, | |
| "learning_rate": 1.906824117472509e-05, | |
| "logits/chosen": -0.3497006297111511, | |
| "logits/rejected": -0.8874444961547852, | |
| "logps/chosen": -0.8235942721366882, | |
| "logps/rejected": -5.622652530670166, | |
| "loss": 0.8246, | |
| "odds_ratio_loss": 0.009956205263733864, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0823594257235527, | |
| "rewards/margins": 0.47990575432777405, | |
| "rewards/rejected": -0.5622652173042297, | |
| "sft_loss": 0.8235942721366882, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 1.7476521028991425, | |
| "grad_norm": 2.015336275100708, | |
| "learning_rate": 1.8653568770724806e-05, | |
| "logits/chosen": -0.2165544331073761, | |
| "logits/rejected": -0.780961811542511, | |
| "logps/chosen": -0.8200712203979492, | |
| "logps/rejected": -6.040628910064697, | |
| "loss": 0.8208, | |
| "odds_ratio_loss": 0.006871505174785852, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08200712502002716, | |
| "rewards/margins": 0.5220558643341064, | |
| "rewards/rejected": -0.6040629148483276, | |
| "sft_loss": 0.8200712203979492, | |
| "step": 535 | |
| }, | |
| { | |
| "epoch": 1.763985300122499, | |
| "grad_norm": 2.2979736328125, | |
| "learning_rate": 1.8240754485080242e-05, | |
| "logits/chosen": -0.32368141412734985, | |
| "logits/rejected": -0.7435051798820496, | |
| "logps/chosen": -0.864768385887146, | |
| "logps/rejected": -6.1721649169921875, | |
| "loss": 0.8654, | |
| "odds_ratio_loss": 0.006089936941862106, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08647684752941132, | |
| "rewards/margins": 0.5307397246360779, | |
| "rewards/rejected": -0.6172165274620056, | |
| "sft_loss": 0.864768385887146, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 1.7803184973458555, | |
| "grad_norm": 1.8173251152038574, | |
| "learning_rate": 1.7829919182222752e-05, | |
| "logits/chosen": -0.1074652448296547, | |
| "logits/rejected": -0.6679927706718445, | |
| "logps/chosen": -0.8761852383613586, | |
| "logps/rejected": -6.271355628967285, | |
| "loss": 0.8768, | |
| "odds_ratio_loss": 0.005977124907076359, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08761852234601974, | |
| "rewards/margins": 0.5395171046257019, | |
| "rewards/rejected": -0.6271356344223022, | |
| "sft_loss": 0.8761852383613586, | |
| "step": 545 | |
| }, | |
| { | |
| "epoch": 1.796651694569212, | |
| "grad_norm": 2.2009291648864746, | |
| "learning_rate": 1.7421183147173915e-05, | |
| "logits/chosen": -0.27911990880966187, | |
| "logits/rejected": -0.7374569177627563, | |
| "logps/chosen": -0.843457043170929, | |
| "logps/rejected": -6.139245986938477, | |
| "loss": 0.8441, | |
| "odds_ratio_loss": 0.006457020528614521, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08434571325778961, | |
| "rewards/margins": 0.5295789241790771, | |
| "rewards/rejected": -0.6139246225357056, | |
| "sft_loss": 0.843457043170929, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 1.8129848917925684, | |
| "grad_norm": 1.7380826473236084, | |
| "learning_rate": 1.7014666050328325e-05, | |
| "logits/chosen": -0.3113337457180023, | |
| "logits/rejected": -0.786532998085022, | |
| "logps/chosen": -0.7784915566444397, | |
| "logps/rejected": -5.818180084228516, | |
| "loss": 0.7793, | |
| "odds_ratio_loss": 0.007720579393208027, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07784914970397949, | |
| "rewards/margins": 0.5039688944816589, | |
| "rewards/rejected": -0.5818179845809937, | |
| "sft_loss": 0.7784915566444397, | |
| "step": 555 | |
| }, | |
| { | |
| "epoch": 1.8293180890159249, | |
| "grad_norm": 2.6397993564605713, | |
| "learning_rate": 1.661048691241625e-05, | |
| "logits/chosen": -0.27005431056022644, | |
| "logits/rejected": -0.7860145568847656, | |
| "logps/chosen": -0.9139931797981262, | |
| "logps/rejected": -6.222220420837402, | |
| "loss": 0.9148, | |
| "odds_ratio_loss": 0.007696300745010376, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.09139931201934814, | |
| "rewards/margins": 0.53082275390625, | |
| "rewards/rejected": -0.6222220659255981, | |
| "sft_loss": 0.9139931797981262, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 1.8456512862392813, | |
| "grad_norm": 2.7970798015594482, | |
| "learning_rate": 1.620876406965658e-05, | |
| "logits/chosen": -0.2970743775367737, | |
| "logits/rejected": -0.8503972291946411, | |
| "logps/chosen": -0.8780502080917358, | |
| "logps/rejected": -6.108642578125, | |
| "loss": 0.8797, | |
| "odds_ratio_loss": 0.016725538298487663, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08780501782894135, | |
| "rewards/margins": 0.5230591893196106, | |
| "rewards/rejected": -0.6108641624450684, | |
| "sft_loss": 0.8780502080917358, | |
| "step": 565 | |
| }, | |
| { | |
| "epoch": 1.8619844834626378, | |
| "grad_norm": 2.1944961547851562, | |
| "learning_rate": 1.5809615139110172e-05, | |
| "logits/chosen": -0.30784982442855835, | |
| "logits/rejected": -0.7813681364059448, | |
| "logps/chosen": -0.8596214056015015, | |
| "logps/rejected": -6.107638835906982, | |
| "loss": 0.8601, | |
| "odds_ratio_loss": 0.0052293105982244015, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08596214652061462, | |
| "rewards/margins": 0.5248017907142639, | |
| "rewards/rejected": -0.6107639074325562, | |
| "sft_loss": 0.8596214056015015, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 1.8783176806859943, | |
| "grad_norm": 2.0301718711853027, | |
| "learning_rate": 1.5413156984243716e-05, | |
| "logits/chosen": -0.3718656003475189, | |
| "logits/rejected": -0.8364275097846985, | |
| "logps/chosen": -0.8916218876838684, | |
| "logps/rejected": -5.7218194007873535, | |
| "loss": 0.893, | |
| "odds_ratio_loss": 0.013976702466607094, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0891621857881546, | |
| "rewards/margins": 0.48301976919174194, | |
| "rewards/rejected": -0.5721819996833801, | |
| "sft_loss": 0.8916218876838684, | |
| "step": 575 | |
| }, | |
| { | |
| "epoch": 1.8946508779093507, | |
| "grad_norm": 2.0284810066223145, | |
| "learning_rate": 1.5019505680714233e-05, | |
| "logits/chosen": -0.2422557771205902, | |
| "logits/rejected": -0.9137754440307617, | |
| "logps/chosen": -0.819043755531311, | |
| "logps/rejected": -5.523108005523682, | |
| "loss": 0.82, | |
| "odds_ratio_loss": 0.009329824708402157, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08190438896417618, | |
| "rewards/margins": 0.4704064428806305, | |
| "rewards/rejected": -0.5523108243942261, | |
| "sft_loss": 0.819043755531311, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 1.9109840751327072, | |
| "grad_norm": 1.9808765649795532, | |
| "learning_rate": 1.462877648238432e-05, | |
| "logits/chosen": -0.4207962453365326, | |
| "logits/rejected": -0.9078248739242554, | |
| "logps/chosen": -0.7534202337265015, | |
| "logps/rejected": -5.3436737060546875, | |
| "loss": 0.7547, | |
| "odds_ratio_loss": 0.012538093142211437, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.07534202188253403, | |
| "rewards/margins": 0.4590253233909607, | |
| "rewards/rejected": -0.5343673825263977, | |
| "sft_loss": 0.7534202337265015, | |
| "step": 585 | |
| }, | |
| { | |
| "epoch": 1.9273172723560636, | |
| "grad_norm": 2.0298492908477783, | |
| "learning_rate": 1.4241083787577923e-05, | |
| "logits/chosen": -0.3211058974266052, | |
| "logits/rejected": -0.8285148739814758, | |
| "logps/chosen": -0.8483121991157532, | |
| "logps/rejected": -5.782259941101074, | |
| "loss": 0.85, | |
| "odds_ratio_loss": 0.01721656322479248, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08483122289180756, | |
| "rewards/margins": 0.49339476227760315, | |
| "rewards/rejected": -0.5782259702682495, | |
| "sft_loss": 0.8483121991157532, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 1.9436504695794201, | |
| "grad_norm": 2.123544454574585, | |
| "learning_rate": 1.3856541105586545e-05, | |
| "logits/chosen": -0.3325098752975464, | |
| "logits/rejected": -0.9181059002876282, | |
| "logps/chosen": -0.8079050779342651, | |
| "logps/rejected": -5.831791877746582, | |
| "loss": 0.8089, | |
| "odds_ratio_loss": 0.010058022104203701, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.08079050481319427, | |
| "rewards/margins": 0.5023886561393738, | |
| "rewards/rejected": -0.5831791758537292, | |
| "sft_loss": 0.8079050779342651, | |
| "step": 595 | |
| }, | |
| { | |
| "epoch": 1.9599836668027768, | |
| "grad_norm": 1.670109748840332, | |
| "learning_rate": 1.3475261023435909e-05, | |
| "logits/chosen": -0.2341674566268921, | |
| "logits/rejected": -0.7504159808158875, | |
| "logps/chosen": -0.8234028816223145, | |
| "logps/rejected": -6.169764041900635, | |
| "loss": 0.8238, | |
| "odds_ratio_loss": 0.004405100364238024, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0823402926325798, | |
| "rewards/margins": 0.5346360206604004, | |
| "rewards/rejected": -0.6169763803482056, | |
| "sft_loss": 0.8234028816223145, | |
| "step": 600 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 918, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 100, | |
| "total_flos": 2.496716870021284e+17, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |