|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 2.981333333333333, |
|
"eval_steps": 500, |
|
"global_step": 351, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.042666666666666665, |
|
"grad_norm": 1961.1512451171875, |
|
"learning_rate": 6.944444444444445e-06, |
|
"log_odds_chosen": 3.233767032623291, |
|
"log_odds_ratio": -7.3191938400268555, |
|
"logps/chosen": -23.426687240600586, |
|
"logps/rejected": -26.660680770874023, |
|
"loss": 425.3708, |
|
"nll_loss": 9.728386878967285, |
|
"po_loss": 3.5644500255584717, |
|
"rewards/accuracies": 0.515625, |
|
"rewards/chosen": -11.713343620300293, |
|
"rewards/margins": 1.6169971227645874, |
|
"rewards/rejected": -13.330340385437012, |
|
"step": 5 |
|
}, |
|
{ |
|
"epoch": 0.08533333333333333, |
|
"grad_norm": 500.5514831542969, |
|
"learning_rate": 1.388888888888889e-05, |
|
"log_odds_chosen": 3.315504550933838, |
|
"log_odds_ratio": -6.866879463195801, |
|
"logps/chosen": -23.438692092895508, |
|
"logps/rejected": -26.755935668945312, |
|
"loss": 393.0612, |
|
"nll_loss": 8.955938339233398, |
|
"po_loss": 3.3272223472595215, |
|
"rewards/accuracies": 0.48124998807907104, |
|
"rewards/chosen": -11.719346046447754, |
|
"rewards/margins": 1.658623456954956, |
|
"rewards/rejected": -13.377967834472656, |
|
"step": 10 |
|
}, |
|
{ |
|
"epoch": 0.128, |
|
"grad_norm": 514.0573120117188, |
|
"learning_rate": 2.0833333333333336e-05, |
|
"log_odds_chosen": 7.634564399719238, |
|
"log_odds_ratio": -6.296060562133789, |
|
"logps/chosen": -22.5439395904541, |
|
"logps/rejected": -30.179019927978516, |
|
"loss": 378.1973, |
|
"nll_loss": 8.766839981079102, |
|
"po_loss": 3.0518250465393066, |
|
"rewards/accuracies": 0.53125, |
|
"rewards/chosen": -11.27196979522705, |
|
"rewards/margins": 3.8175387382507324, |
|
"rewards/rejected": -15.089509963989258, |
|
"step": 15 |
|
}, |
|
{ |
|
"epoch": 0.17066666666666666, |
|
"grad_norm": 3820.703125, |
|
"learning_rate": 2.777777777777778e-05, |
|
"log_odds_chosen": -1.3400392532348633, |
|
"log_odds_ratio": -8.175825119018555, |
|
"logps/chosen": -18.689817428588867, |
|
"logps/rejected": -17.35008430480957, |
|
"loss": 363.51, |
|
"nll_loss": 7.390625953674316, |
|
"po_loss": 3.9690635204315186, |
|
"rewards/accuracies": 0.484375, |
|
"rewards/chosen": -9.344908714294434, |
|
"rewards/margins": -0.6698670983314514, |
|
"rewards/rejected": -8.675042152404785, |
|
"step": 20 |
|
}, |
|
{ |
|
"epoch": 0.21333333333333335, |
|
"grad_norm": 417.3852844238281, |
|
"learning_rate": 3.472222222222222e-05, |
|
"log_odds_chosen": -0.37303870916366577, |
|
"log_odds_ratio": -2.343538284301758, |
|
"logps/chosen": -5.883327484130859, |
|
"logps/rejected": -5.492794036865234, |
|
"loss": 134.8965, |
|
"nll_loss": 3.241476535797119, |
|
"po_loss": 0.9740395545959473, |
|
"rewards/accuracies": 0.5562499761581421, |
|
"rewards/chosen": -2.9416637420654297, |
|
"rewards/margins": -0.19526681303977966, |
|
"rewards/rejected": -2.746397018432617, |
|
"step": 25 |
|
}, |
|
{ |
|
"epoch": 0.256, |
|
"grad_norm": 330.9832458496094, |
|
"learning_rate": 4.166666666666667e-05, |
|
"log_odds_chosen": 0.3532889187335968, |
|
"log_odds_ratio": -0.7218837141990662, |
|
"logps/chosen": -1.7468492984771729, |
|
"logps/rejected": -2.0471103191375732, |
|
"loss": 70.938, |
|
"nll_loss": 2.1013693809509277, |
|
"po_loss": 0.11544491350650787, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -0.8734246492385864, |
|
"rewards/margins": 0.15013054013252258, |
|
"rewards/rejected": -1.0235551595687866, |
|
"step": 30 |
|
}, |
|
{ |
|
"epoch": 0.2986666666666667, |
|
"grad_norm": 284.9355163574219, |
|
"learning_rate": 4.8611111111111115e-05, |
|
"log_odds_chosen": 0.4371984004974365, |
|
"log_odds_ratio": -0.6356752514839172, |
|
"logps/chosen": -1.627811074256897, |
|
"logps/rejected": -1.993390679359436, |
|
"loss": 63.8055, |
|
"nll_loss": 1.9257453680038452, |
|
"po_loss": 0.06817711889743805, |
|
"rewards/accuracies": 0.6937500238418579, |
|
"rewards/chosen": -0.8139055371284485, |
|
"rewards/margins": 0.18278983235359192, |
|
"rewards/rejected": -0.996695339679718, |
|
"step": 35 |
|
}, |
|
{ |
|
"epoch": 0.3413333333333333, |
|
"grad_norm": 126.80204772949219, |
|
"learning_rate": 4.998010925565448e-05, |
|
"log_odds_chosen": 0.36739838123321533, |
|
"log_odds_ratio": -0.6945838928222656, |
|
"logps/chosen": -1.696966528892517, |
|
"logps/rejected": -2.019742488861084, |
|
"loss": 66.0958, |
|
"nll_loss": 1.972928762435913, |
|
"po_loss": 0.0925649031996727, |
|
"rewards/accuracies": 0.637499988079071, |
|
"rewards/chosen": -0.8484832644462585, |
|
"rewards/margins": 0.16138817369937897, |
|
"rewards/rejected": -1.009871244430542, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 0.384, |
|
"grad_norm": 734.975830078125, |
|
"learning_rate": 4.989935734988098e-05, |
|
"log_odds_chosen": 0.2258051633834839, |
|
"log_odds_ratio": -0.6970190405845642, |
|
"logps/chosen": -1.5770127773284912, |
|
"logps/rejected": -1.7601944208145142, |
|
"loss": 61.9986, |
|
"nll_loss": 1.8443195819854736, |
|
"po_loss": 0.09313799440860748, |
|
"rewards/accuracies": 0.6000000238418579, |
|
"rewards/chosen": -0.7885063886642456, |
|
"rewards/margins": 0.09159073978662491, |
|
"rewards/rejected": -0.8800972104072571, |
|
"step": 45 |
|
}, |
|
{ |
|
"epoch": 0.4266666666666667, |
|
"grad_norm": 98.6443862915039, |
|
"learning_rate": 4.975670171853926e-05, |
|
"log_odds_chosen": 0.3061736226081848, |
|
"log_odds_ratio": -0.6558084487915039, |
|
"logps/chosen": -1.410742998123169, |
|
"logps/rejected": -1.6612850427627563, |
|
"loss": 56.1342, |
|
"nll_loss": 1.6897376775741577, |
|
"po_loss": 0.06445512920618057, |
|
"rewards/accuracies": 0.6343749761581421, |
|
"rewards/chosen": -0.7053714990615845, |
|
"rewards/margins": 0.1252710521221161, |
|
"rewards/rejected": -0.8306425213813782, |
|
"step": 50 |
|
}, |
|
{ |
|
"epoch": 0.4693333333333333, |
|
"grad_norm": 138.6815948486328, |
|
"learning_rate": 4.9552497026005974e-05, |
|
"log_odds_chosen": 0.21871152520179749, |
|
"log_odds_ratio": -0.7383303046226501, |
|
"logps/chosen": -1.4634255170822144, |
|
"logps/rejected": -1.6147739887237549, |
|
"loss": 59.4312, |
|
"nll_loss": 1.7589048147201538, |
|
"po_loss": 0.09831906855106354, |
|
"rewards/accuracies": 0.581250011920929, |
|
"rewards/chosen": -0.7317127585411072, |
|
"rewards/margins": 0.07567427307367325, |
|
"rewards/rejected": -0.8073869943618774, |
|
"step": 55 |
|
}, |
|
{ |
|
"epoch": 0.512, |
|
"grad_norm": 102.89739227294922, |
|
"learning_rate": 4.928725095732169e-05, |
|
"log_odds_chosen": 0.401327520608902, |
|
"log_odds_ratio": -0.6062948107719421, |
|
"logps/chosen": -1.2786072492599487, |
|
"logps/rejected": -1.585089921951294, |
|
"loss": 51.5199, |
|
"nll_loss": 1.5874063968658447, |
|
"po_loss": 0.02259092405438423, |
|
"rewards/accuracies": 0.6812499761581421, |
|
"rewards/chosen": -0.6393036246299744, |
|
"rewards/margins": 0.15324126183986664, |
|
"rewards/rejected": -0.792544960975647, |
|
"step": 60 |
|
}, |
|
{ |
|
"epoch": 0.5546666666666666, |
|
"grad_norm": 100.74688720703125, |
|
"learning_rate": 4.896162295600589e-05, |
|
"log_odds_chosen": 0.33866989612579346, |
|
"log_odds_ratio": -0.6249087452888489, |
|
"logps/chosen": -1.3379658460617065, |
|
"logps/rejected": -1.6066213846206665, |
|
"loss": 53.1437, |
|
"nll_loss": 1.6211912631988525, |
|
"po_loss": 0.03954792767763138, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -0.6689829230308533, |
|
"rewards/margins": 0.13432787358760834, |
|
"rewards/rejected": -0.8033106923103333, |
|
"step": 65 |
|
}, |
|
{ |
|
"epoch": 0.5973333333333334, |
|
"grad_norm": 107.1102523803711, |
|
"learning_rate": 4.8576422584576514e-05, |
|
"log_odds_chosen": 0.28569236397743225, |
|
"log_odds_ratio": -0.6388062238693237, |
|
"logps/chosen": -1.2627140283584595, |
|
"logps/rejected": -1.478611707687378, |
|
"loss": 51.3954, |
|
"nll_loss": 1.5717823505401611, |
|
"po_loss": 0.034325726330280304, |
|
"rewards/accuracies": 0.628125011920929, |
|
"rewards/chosen": -0.6313570141792297, |
|
"rewards/margins": 0.10794880241155624, |
|
"rewards/rejected": -0.739305853843689, |
|
"step": 70 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"grad_norm": 87.37519073486328, |
|
"learning_rate": 4.813260751184992e-05, |
|
"log_odds_chosen": 0.4192368984222412, |
|
"log_odds_ratio": -0.6039353609085083, |
|
"logps/chosen": -1.2572083473205566, |
|
"logps/rejected": -1.5851625204086304, |
|
"loss": 50.7092, |
|
"nll_loss": 1.5785938501358032, |
|
"po_loss": 0.00606974633410573, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": -0.6286041736602783, |
|
"rewards/margins": 0.1639770269393921, |
|
"rewards/rejected": -0.7925812602043152, |
|
"step": 75 |
|
}, |
|
{ |
|
"epoch": 0.6826666666666666, |
|
"grad_norm": 84.41800689697266, |
|
"learning_rate": 4.763128113202537e-05, |
|
"log_odds_chosen": 0.3978249132633209, |
|
"log_odds_ratio": -0.6257617473602295, |
|
"logps/chosen": -1.1973122358322144, |
|
"logps/rejected": -1.5119242668151855, |
|
"loss": 49.6894, |
|
"nll_loss": 1.5280753374099731, |
|
"po_loss": 0.024718867614865303, |
|
"rewards/accuracies": 0.6031249761581421, |
|
"rewards/chosen": -0.5986561179161072, |
|
"rewards/margins": 0.1573060303926468, |
|
"rewards/rejected": -0.7559621334075928, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 0.7253333333333334, |
|
"grad_norm": 217.4127655029297, |
|
"learning_rate": 4.707368982147318e-05, |
|
"log_odds_chosen": 0.9764216542243958, |
|
"log_odds_ratio": -0.5953387022018433, |
|
"logps/chosen": -1.2593096494674683, |
|
"logps/rejected": -2.1358559131622314, |
|
"loss": 50.7417, |
|
"nll_loss": 1.5718034505844116, |
|
"po_loss": 0.013875389471650124, |
|
"rewards/accuracies": 0.628125011920929, |
|
"rewards/chosen": -0.6296548247337341, |
|
"rewards/margins": 0.43827304244041443, |
|
"rewards/rejected": -1.0679279565811157, |
|
"step": 85 |
|
}, |
|
{ |
|
"epoch": 0.768, |
|
"grad_norm": 74.66828918457031, |
|
"learning_rate": 4.6461219840046654e-05, |
|
"log_odds_chosen": 0.7273199558258057, |
|
"log_odds_ratio": -0.6018415093421936, |
|
"logps/chosen": -1.213512897491455, |
|
"logps/rejected": -1.8320105075836182, |
|
"loss": 48.283, |
|
"nll_loss": 1.5248548984527588, |
|
"po_loss": -0.016011018306016922, |
|
"rewards/accuracies": 0.6625000238418579, |
|
"rewards/chosen": -0.6067564487457275, |
|
"rewards/margins": 0.30924877524375916, |
|
"rewards/rejected": -0.9160052537918091, |
|
"step": 90 |
|
}, |
|
{ |
|
"epoch": 0.8106666666666666, |
|
"grad_norm": 96.17576599121094, |
|
"learning_rate": 4.579539388462173e-05, |
|
"log_odds_chosen": 0.968981921672821, |
|
"log_odds_ratio": -0.5852179527282715, |
|
"logps/chosen": -1.1610548496246338, |
|
"logps/rejected": -2.0138120651245117, |
|
"loss": 46.2603, |
|
"nll_loss": 1.5272886753082275, |
|
"po_loss": -0.08165426552295685, |
|
"rewards/accuracies": 0.637499988079071, |
|
"rewards/chosen": -0.5805274248123169, |
|
"rewards/margins": 0.4263785481452942, |
|
"rewards/rejected": -1.0069060325622559, |
|
"step": 95 |
|
}, |
|
{ |
|
"epoch": 0.8533333333333334, |
|
"grad_norm": 1037.95703125, |
|
"learning_rate": 4.5077867303432546e-05, |
|
"log_odds_chosen": 2.305642604827881, |
|
"log_odds_ratio": -0.6378474831581116, |
|
"logps/chosen": -1.2965327501296997, |
|
"logps/rejected": -3.484023332595825, |
|
"loss": 40.3825, |
|
"nll_loss": 1.5268816947937012, |
|
"po_loss": -0.2649289071559906, |
|
"rewards/accuracies": 0.671875, |
|
"rewards/chosen": -0.6482663750648499, |
|
"rewards/margins": 1.093745231628418, |
|
"rewards/rejected": -1.7420116662979126, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.896, |
|
"grad_norm": 120.91621398925781, |
|
"learning_rate": 4.431042398061499e-05, |
|
"log_odds_chosen": 2.5239200592041016, |
|
"log_odds_ratio": -0.7032963633537292, |
|
"logps/chosen": -2.863506317138672, |
|
"logps/rejected": -5.296839714050293, |
|
"loss": 85.1499, |
|
"nll_loss": 2.635256767272949, |
|
"po_loss": 0.025676274672150612, |
|
"rewards/accuracies": 0.6937500238418579, |
|
"rewards/chosen": -1.431753158569336, |
|
"rewards/margins": 1.2166664600372314, |
|
"rewards/rejected": -2.6484198570251465, |
|
"step": 105 |
|
}, |
|
{ |
|
"epoch": 0.9386666666666666, |
|
"grad_norm": 80.25611114501953, |
|
"learning_rate": 4.34949719011896e-05, |
|
"log_odds_chosen": 0.859009861946106, |
|
"log_odds_ratio": -0.5764500498771667, |
|
"logps/chosen": -1.229062557220459, |
|
"logps/rejected": -1.9831807613372803, |
|
"loss": 47.4796, |
|
"nll_loss": 1.5179859399795532, |
|
"po_loss": -0.03424710035324097, |
|
"rewards/accuracies": 0.6625000238418579, |
|
"rewards/chosen": -0.6145312786102295, |
|
"rewards/margins": 0.37705910205841064, |
|
"rewards/rejected": -0.9915903806686401, |
|
"step": 110 |
|
}, |
|
{ |
|
"epoch": 0.9813333333333333, |
|
"grad_norm": 271.3537902832031, |
|
"learning_rate": 4.263353840751022e-05, |
|
"log_odds_chosen": 1.4788315296173096, |
|
"log_odds_ratio": -0.5846825242042542, |
|
"logps/chosen": -1.1537017822265625, |
|
"logps/rejected": -2.52790904045105, |
|
"loss": 44.319, |
|
"nll_loss": 1.4921709299087524, |
|
"po_loss": -0.10720314830541611, |
|
"rewards/accuracies": 0.6781250238418579, |
|
"rewards/chosen": -0.5768508911132812, |
|
"rewards/margins": 0.6871034502983093, |
|
"rewards/rejected": -1.263954520225525, |
|
"step": 115 |
|
}, |
|
{ |
|
"epoch": 1.0170666666666666, |
|
"grad_norm": 146.54196166992188, |
|
"learning_rate": 4.172826515897146e-05, |
|
"log_odds_chosen": 2.831665515899658, |
|
"log_odds_ratio": -0.4861010015010834, |
|
"logps/chosen": -1.089508056640625, |
|
"logps/rejected": -3.7528676986694336, |
|
"loss": 33.7801, |
|
"nll_loss": 1.4263149499893188, |
|
"po_loss": -0.1658623069524765, |
|
"rewards/accuracies": 0.7723880410194397, |
|
"rewards/chosen": -0.5447540283203125, |
|
"rewards/margins": 1.3316797018051147, |
|
"rewards/rejected": -1.8764338493347168, |
|
"step": 120 |
|
}, |
|
{ |
|
"epoch": 1.0597333333333334, |
|
"grad_norm": 1590.6824951171875, |
|
"learning_rate": 4.078140280750597e-05, |
|
"log_odds_chosen": 4.709661483764648, |
|
"log_odds_ratio": -0.41537848114967346, |
|
"logps/chosen": -1.5830024480819702, |
|
"logps/rejected": -6.075448036193848, |
|
"loss": 46.6655, |
|
"nll_loss": 1.6417958736419678, |
|
"po_loss": -0.18349842727184296, |
|
"rewards/accuracies": 0.8531249761581421, |
|
"rewards/chosen": -0.7915012240409851, |
|
"rewards/margins": 2.246222496032715, |
|
"rewards/rejected": -3.037724018096924, |
|
"step": 125 |
|
}, |
|
{ |
|
"epoch": 1.1024, |
|
"grad_norm": 321.93255615234375, |
|
"learning_rate": 3.9795305402109195e-05, |
|
"log_odds_chosen": 3.6606578826904297, |
|
"log_odds_ratio": -0.4261302947998047, |
|
"logps/chosen": -1.2628165483474731, |
|
"logps/rejected": -4.708868980407715, |
|
"loss": 36.688, |
|
"nll_loss": 1.4487800598144531, |
|
"po_loss": -0.3022807240486145, |
|
"rewards/accuracies": 0.8218749761581421, |
|
"rewards/chosen": -0.6314082741737366, |
|
"rewards/margins": 1.7230265140533447, |
|
"rewards/rejected": -2.3544344902038574, |
|
"step": 130 |
|
}, |
|
{ |
|
"epoch": 1.1450666666666667, |
|
"grad_norm": 156.33656311035156, |
|
"learning_rate": 3.8772424536302564e-05, |
|
"log_odds_chosen": 3.3216488361358643, |
|
"log_odds_ratio": -0.45871955156326294, |
|
"logps/chosen": -1.054512619972229, |
|
"logps/rejected": -4.166798114776611, |
|
"loss": 37.5147, |
|
"nll_loss": 1.3529837131500244, |
|
"po_loss": -0.18065081536769867, |
|
"rewards/accuracies": 0.778124988079071, |
|
"rewards/chosen": -0.5272563099861145, |
|
"rewards/margins": 1.5561425685882568, |
|
"rewards/rejected": -2.0833990573883057, |
|
"step": 135 |
|
}, |
|
{ |
|
"epoch": 1.1877333333333333, |
|
"grad_norm": 126.58124542236328, |
|
"learning_rate": 3.771530325308579e-05, |
|
"log_odds_chosen": 3.002664089202881, |
|
"log_odds_ratio": -0.3991411030292511, |
|
"logps/chosen": -0.99451744556427, |
|
"logps/rejected": -3.7320716381073, |
|
"loss": 33.4168, |
|
"nll_loss": 1.2971866130828857, |
|
"po_loss": -0.2529115378856659, |
|
"rewards/accuracies": 0.862500011920929, |
|
"rewards/chosen": -0.497258722782135, |
|
"rewards/margins": 1.3687771558761597, |
|
"rewards/rejected": -1.86603581905365, |
|
"step": 140 |
|
}, |
|
{ |
|
"epoch": 1.2304, |
|
"grad_norm": 144.6908416748047, |
|
"learning_rate": 3.662656972253127e-05, |
|
"log_odds_chosen": 3.620790958404541, |
|
"log_odds_ratio": -0.39333289861679077, |
|
"logps/chosen": -1.1291520595550537, |
|
"logps/rejected": -4.468079090118408, |
|
"loss": 31.6104, |
|
"nll_loss": 1.2871015071868896, |
|
"po_loss": -0.2992752194404602, |
|
"rewards/accuracies": 0.875, |
|
"rewards/chosen": -0.5645760297775269, |
|
"rewards/margins": 1.6694637537002563, |
|
"rewards/rejected": -2.234039545059204, |
|
"step": 145 |
|
}, |
|
{ |
|
"epoch": 1.2730666666666668, |
|
"grad_norm": 317.3659362792969, |
|
"learning_rate": 3.550893070773914e-05, |
|
"log_odds_chosen": 2.5045361518859863, |
|
"log_odds_ratio": -0.5062359571456909, |
|
"logps/chosen": -1.01883864402771, |
|
"logps/rejected": -3.3051059246063232, |
|
"loss": 35.2667, |
|
"nll_loss": 1.3030840158462524, |
|
"po_loss": -0.20100060105323792, |
|
"rewards/accuracies": 0.831250011920929, |
|
"rewards/chosen": -0.509419322013855, |
|
"rewards/margins": 1.143133521080017, |
|
"rewards/rejected": -1.6525529623031616, |
|
"step": 150 |
|
}, |
|
{ |
|
"epoch": 1.3157333333333332, |
|
"grad_norm": 99.71307373046875, |
|
"learning_rate": 3.436516483539781e-05, |
|
"log_odds_chosen": 2.3835015296936035, |
|
"log_odds_ratio": -0.4490797519683838, |
|
"logps/chosen": -0.961607813835144, |
|
"logps/rejected": -3.094728946685791, |
|
"loss": 36.1188, |
|
"nll_loss": 1.2786829471588135, |
|
"po_loss": -0.1499709188938141, |
|
"rewards/accuracies": 0.8343750238418579, |
|
"rewards/chosen": -0.480803906917572, |
|
"rewards/margins": 1.0665608644485474, |
|
"rewards/rejected": -1.5473644733428955, |
|
"step": 155 |
|
}, |
|
{ |
|
"epoch": 1.3584, |
|
"grad_norm": 176.96144104003906, |
|
"learning_rate": 3.3198115687680115e-05, |
|
"log_odds_chosen": 2.804508924484253, |
|
"log_odds_ratio": -0.4551823139190674, |
|
"logps/chosen": -0.9649540781974792, |
|
"logps/rejected": -3.5283005237579346, |
|
"loss": 31.9084, |
|
"nll_loss": 1.2769290208816528, |
|
"po_loss": -0.2797916531562805, |
|
"rewards/accuracies": 0.7906249761581421, |
|
"rewards/chosen": -0.4824770390987396, |
|
"rewards/margins": 1.2816731929779053, |
|
"rewards/rejected": -1.7641502618789673, |
|
"step": 160 |
|
}, |
|
{ |
|
"epoch": 1.4010666666666667, |
|
"grad_norm": 86.9504165649414, |
|
"learning_rate": 3.201068473265007e-05, |
|
"log_odds_chosen": 4.039158821105957, |
|
"log_odds_ratio": -0.41200175881385803, |
|
"logps/chosen": -1.1530981063842773, |
|
"logps/rejected": -4.936544418334961, |
|
"loss": 33.9241, |
|
"nll_loss": 1.339659571647644, |
|
"po_loss": -0.27953004837036133, |
|
"rewards/accuracies": 0.840624988079071, |
|
"rewards/chosen": -0.5765490531921387, |
|
"rewards/margins": 1.8917229175567627, |
|
"rewards/rejected": -2.4682722091674805, |
|
"step": 165 |
|
}, |
|
{ |
|
"epoch": 1.4437333333333333, |
|
"grad_norm": 67.8962631225586, |
|
"learning_rate": 3.0805824110756064e-05, |
|
"log_odds_chosen": 2.8494575023651123, |
|
"log_odds_ratio": -0.4309585690498352, |
|
"logps/chosen": -1.0003385543823242, |
|
"logps/rejected": -3.5997538566589355, |
|
"loss": 32.9378, |
|
"nll_loss": 1.320733904838562, |
|
"po_loss": -0.2914286255836487, |
|
"rewards/accuracies": 0.84375, |
|
"rewards/chosen": -0.5001692771911621, |
|
"rewards/margins": 1.2997074127197266, |
|
"rewards/rejected": -1.7998769283294678, |
|
"step": 170 |
|
}, |
|
{ |
|
"epoch": 1.4864, |
|
"grad_norm": 332.5802307128906, |
|
"learning_rate": 2.958652929534456e-05, |
|
"log_odds_chosen": 4.219790458679199, |
|
"log_odds_ratio": -0.3872655928134918, |
|
"logps/chosen": -0.9488153457641602, |
|
"logps/rejected": -4.870639324188232, |
|
"loss": 24.5, |
|
"nll_loss": 1.2430005073547363, |
|
"po_loss": -0.47737568616867065, |
|
"rewards/accuracies": 0.8656250238418579, |
|
"rewards/chosen": -0.4744076728820801, |
|
"rewards/margins": 1.9609121084213257, |
|
"rewards/rejected": -2.435319662094116, |
|
"step": 175 |
|
}, |
|
{ |
|
"epoch": 1.5290666666666666, |
|
"grad_norm": 120.37972259521484, |
|
"learning_rate": 2.8355831645441388e-05, |
|
"log_odds_chosen": 3.8075263500213623, |
|
"log_odds_ratio": -0.4285067617893219, |
|
"logps/chosen": -1.0132713317871094, |
|
"logps/rejected": -4.545207500457764, |
|
"loss": 30.9088, |
|
"nll_loss": 1.2752387523651123, |
|
"po_loss": -0.3093392252922058, |
|
"rewards/accuracies": 0.840624988079071, |
|
"rewards/chosen": -0.5066356658935547, |
|
"rewards/margins": 1.7659679651260376, |
|
"rewards/rejected": -2.272603750228882, |
|
"step": 180 |
|
}, |
|
{ |
|
"epoch": 1.5717333333333334, |
|
"grad_norm": 74.23609924316406, |
|
"learning_rate": 2.7116790869315582e-05, |
|
"log_odds_chosen": 3.5458972454071045, |
|
"log_odds_ratio": -0.3724256157875061, |
|
"logps/chosen": -0.9797577857971191, |
|
"logps/rejected": -4.209129810333252, |
|
"loss": 27.577, |
|
"nll_loss": 1.2894318103790283, |
|
"po_loss": -0.42764997482299805, |
|
"rewards/accuracies": 0.887499988079071, |
|
"rewards/chosen": -0.48987889289855957, |
|
"rewards/margins": 1.6146857738494873, |
|
"rewards/rejected": -2.104564905166626, |
|
"step": 185 |
|
}, |
|
{ |
|
"epoch": 1.6143999999999998, |
|
"grad_norm": 1912.4134521484375, |
|
"learning_rate": 2.587248741756253e-05, |
|
"log_odds_chosen": 4.569626808166504, |
|
"log_odds_ratio": -0.6492961049079895, |
|
"logps/chosen": -1.6978082656860352, |
|
"logps/rejected": -5.967839241027832, |
|
"loss": 47.8645, |
|
"nll_loss": 1.7877342700958252, |
|
"po_loss": -0.2919684648513794, |
|
"rewards/accuracies": 0.840624988079071, |
|
"rewards/chosen": -0.8489041328430176, |
|
"rewards/margins": 2.1350154876708984, |
|
"rewards/rejected": -2.983919620513916, |
|
"step": 190 |
|
}, |
|
{ |
|
"epoch": 1.6570666666666667, |
|
"grad_norm": 246.9727020263672, |
|
"learning_rate": 2.4626014824618415e-05, |
|
"log_odds_chosen": 3.452259063720703, |
|
"log_odds_ratio": -0.4340329170227051, |
|
"logps/chosen": -1.348034143447876, |
|
"logps/rejected": -4.516729831695557, |
|
"loss": 39.5128, |
|
"nll_loss": 1.5523712635040283, |
|
"po_loss": -0.3175976276397705, |
|
"rewards/accuracies": 0.8218749761581421, |
|
"rewards/chosen": -0.674017071723938, |
|
"rewards/margins": 1.5843479633331299, |
|
"rewards/rejected": -2.2583649158477783, |
|
"step": 195 |
|
}, |
|
{ |
|
"epoch": 1.6997333333333333, |
|
"grad_norm": 249.69097900390625, |
|
"learning_rate": 2.3380472017746202e-05, |
|
"log_odds_chosen": 4.080594539642334, |
|
"log_odds_ratio": -0.41809386014938354, |
|
"logps/chosen": -1.1225640773773193, |
|
"logps/rejected": -4.919617652893066, |
|
"loss": 22.4016, |
|
"nll_loss": 1.3972797393798828, |
|
"po_loss": -0.6972296237945557, |
|
"rewards/accuracies": 0.862500011920929, |
|
"rewards/chosen": -0.5612820386886597, |
|
"rewards/margins": 1.8985265493392944, |
|
"rewards/rejected": -2.459808826446533, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 1.7424, |
|
"grad_norm": 70.95199584960938, |
|
"learning_rate": 2.2138955612614207e-05, |
|
"log_odds_chosen": 3.4808452129364014, |
|
"log_odds_ratio": -0.4113942086696625, |
|
"logps/chosen": -0.9912225604057312, |
|
"logps/rejected": -4.186377048492432, |
|
"loss": 30.6187, |
|
"nll_loss": 1.2635384798049927, |
|
"po_loss": -0.30670270323753357, |
|
"rewards/accuracies": 0.862500011920929, |
|
"rewards/chosen": -0.4956112802028656, |
|
"rewards/margins": 1.5975773334503174, |
|
"rewards/rejected": -2.093188524246216, |
|
"step": 205 |
|
}, |
|
{ |
|
"epoch": 1.7850666666666668, |
|
"grad_norm": 77.6159896850586, |
|
"learning_rate": 2.090455221462156e-05, |
|
"log_odds_chosen": 3.126645803451538, |
|
"log_odds_ratio": -0.4194856286048889, |
|
"logps/chosen": -0.976054847240448, |
|
"logps/rejected": -3.822575807571411, |
|
"loss": 30.5635, |
|
"nll_loss": 1.2749359607696533, |
|
"po_loss": -0.319825679063797, |
|
"rewards/accuracies": 0.8500000238418579, |
|
"rewards/chosen": -0.488027423620224, |
|
"rewards/margins": 1.4232604503631592, |
|
"rewards/rejected": -1.9112879037857056, |
|
"step": 210 |
|
}, |
|
{ |
|
"epoch": 1.8277333333333332, |
|
"grad_norm": 191.34779357910156, |
|
"learning_rate": 1.9680330745110954e-05, |
|
"log_odds_chosen": 4.950188636779785, |
|
"log_odds_ratio": -0.3956086039543152, |
|
"logps/chosen": -0.9748937487602234, |
|
"logps/rejected": -5.643929481506348, |
|
"loss": 23.1695, |
|
"nll_loss": 1.2857718467712402, |
|
"po_loss": -0.5617233514785767, |
|
"rewards/accuracies": 0.84375, |
|
"rewards/chosen": -0.4874468743801117, |
|
"rewards/margins": 2.33451771736145, |
|
"rewards/rejected": -2.821964740753174, |
|
"step": 215 |
|
}, |
|
{ |
|
"epoch": 1.8704, |
|
"grad_norm": 327.7476501464844, |
|
"learning_rate": 1.8469334811546542e-05, |
|
"log_odds_chosen": 4.540267467498779, |
|
"log_odds_ratio": -0.39753258228302, |
|
"logps/chosen": -0.9387453198432922, |
|
"logps/rejected": -5.1981964111328125, |
|
"loss": 28.4212, |
|
"nll_loss": 1.252907633781433, |
|
"po_loss": -0.36474356055259705, |
|
"rewards/accuracies": 0.859375, |
|
"rewards/chosen": -0.4693726599216461, |
|
"rewards/margins": 2.129725217819214, |
|
"rewards/rejected": -2.5990982055664062, |
|
"step": 220 |
|
}, |
|
{ |
|
"epoch": 1.9130666666666667, |
|
"grad_norm": 540.2483520507812, |
|
"learning_rate": 1.7274575140626318e-05, |
|
"log_odds_chosen": 5.64566707611084, |
|
"log_odds_ratio": -0.4446595311164856, |
|
"logps/chosen": -1.22698974609375, |
|
"logps/rejected": -6.585175514221191, |
|
"loss": 34.7211, |
|
"nll_loss": 1.3462177515029907, |
|
"po_loss": -0.26118165254592896, |
|
"rewards/accuracies": 0.8656250238418579, |
|
"rewards/chosen": -0.613494873046875, |
|
"rewards/margins": 2.6790928840637207, |
|
"rewards/rejected": -3.2925877571105957, |
|
"step": 225 |
|
}, |
|
{ |
|
"epoch": 1.9557333333333333, |
|
"grad_norm": 401.98248291015625, |
|
"learning_rate": 1.609902209314108e-05, |
|
"log_odds_chosen": 4.817292213439941, |
|
"log_odds_ratio": -0.42455047369003296, |
|
"logps/chosen": -1.0640500783920288, |
|
"logps/rejected": -5.626768589019775, |
|
"loss": 31.062, |
|
"nll_loss": 1.342284917831421, |
|
"po_loss": -0.37159785628318787, |
|
"rewards/accuracies": 0.8187500238418579, |
|
"rewards/chosen": -0.5320250391960144, |
|
"rewards/margins": 2.2813591957092285, |
|
"rewards/rejected": -2.8133842945098877, |
|
"step": 230 |
|
}, |
|
{ |
|
"epoch": 1.9984, |
|
"grad_norm": 91.74356079101562, |
|
"learning_rate": 1.4945598279189565e-05, |
|
"log_odds_chosen": 5.049031734466553, |
|
"log_odds_ratio": -0.406465619802475, |
|
"logps/chosen": -0.9556134939193726, |
|
"logps/rejected": -5.697612762451172, |
|
"loss": 25.4086, |
|
"nll_loss": 1.2838685512542725, |
|
"po_loss": -0.48984843492507935, |
|
"rewards/accuracies": 0.840624988079071, |
|
"rewards/chosen": -0.4778067469596863, |
|
"rewards/margins": 2.370999813079834, |
|
"rewards/rejected": -2.848806381225586, |
|
"step": 235 |
|
}, |
|
{ |
|
"epoch": 2.034133333333333, |
|
"grad_norm": 84.14781188964844, |
|
"learning_rate": 1.3817171292109183e-05, |
|
"log_odds_chosen": 4.798027992248535, |
|
"log_odds_ratio": -0.3244837820529938, |
|
"logps/chosen": -0.8924720287322998, |
|
"logps/rejected": -5.292112827301025, |
|
"loss": 14.7486, |
|
"nll_loss": 1.1619552373886108, |
|
"po_loss": -0.6116347908973694, |
|
"rewards/accuracies": 0.9365671873092651, |
|
"rewards/chosen": -0.4462360143661499, |
|
"rewards/margins": 2.1998205184936523, |
|
"rewards/rejected": -2.6460564136505127, |
|
"step": 240 |
|
}, |
|
{ |
|
"epoch": 2.0768, |
|
"grad_norm": 265.51214599609375, |
|
"learning_rate": 1.271654657918722e-05, |
|
"log_odds_chosen": 3.9462997913360596, |
|
"log_odds_ratio": -0.29830431938171387, |
|
"logps/chosen": -0.8560145497322083, |
|
"logps/rejected": -4.366959571838379, |
|
"loss": 21.2586, |
|
"nll_loss": 1.1650712490081787, |
|
"po_loss": -0.5007405281066895, |
|
"rewards/accuracies": 0.9437500238418579, |
|
"rewards/chosen": -0.4280072748661041, |
|
"rewards/margins": 1.7554724216461182, |
|
"rewards/rejected": -2.1834797859191895, |
|
"step": 245 |
|
}, |
|
{ |
|
"epoch": 2.119466666666667, |
|
"grad_norm": 189.54098510742188, |
|
"learning_rate": 1.1646460466876783e-05, |
|
"log_odds_chosen": 5.587141036987305, |
|
"log_odds_ratio": -0.23089304566383362, |
|
"logps/chosen": -0.941892147064209, |
|
"logps/rejected": -6.023226737976074, |
|
"loss": 15.3802, |
|
"nll_loss": 1.1797395944595337, |
|
"po_loss": -0.6991080641746521, |
|
"rewards/accuracies": 0.9437500238418579, |
|
"rewards/chosen": -0.4709460735321045, |
|
"rewards/margins": 2.5406672954559326, |
|
"rewards/rejected": -3.011613368988037, |
|
"step": 250 |
|
}, |
|
{ |
|
"epoch": 2.1621333333333332, |
|
"grad_norm": 198.67486572265625, |
|
"learning_rate": 1.0609573357858166e-05, |
|
"log_odds_chosen": 6.2472615242004395, |
|
"log_odds_ratio": -0.2708757519721985, |
|
"logps/chosen": -1.0158169269561768, |
|
"logps/rejected": -6.834556579589844, |
|
"loss": 14.1929, |
|
"nll_loss": 1.237670660018921, |
|
"po_loss": -0.7941441535949707, |
|
"rewards/accuracies": 0.925000011920929, |
|
"rewards/chosen": -0.5079084634780884, |
|
"rewards/margins": 2.909369945526123, |
|
"rewards/rejected": -3.417278289794922, |
|
"step": 255 |
|
}, |
|
{ |
|
"epoch": 2.2048, |
|
"grad_norm": 758.944580078125, |
|
"learning_rate": 9.608463116858542e-06, |
|
"log_odds_chosen": 6.66796350479126, |
|
"log_odds_ratio": -0.23688967525959015, |
|
"logps/chosen": -0.9843616485595703, |
|
"logps/rejected": -7.208956718444824, |
|
"loss": 11.9915, |
|
"nll_loss": 1.2095975875854492, |
|
"po_loss": -0.8348617553710938, |
|
"rewards/accuracies": 0.9593750238418579, |
|
"rewards/chosen": -0.49218082427978516, |
|
"rewards/margins": 3.112297534942627, |
|
"rewards/rejected": -3.604478359222412, |
|
"step": 260 |
|
}, |
|
{ |
|
"epoch": 2.2474666666666665, |
|
"grad_norm": 504.9031982421875, |
|
"learning_rate": 8.645618661674142e-06, |
|
"log_odds_chosen": 6.688530921936035, |
|
"log_odds_ratio": -0.2643258273601532, |
|
"logps/chosen": -1.105387806892395, |
|
"logps/rejected": -7.340773582458496, |
|
"loss": 11.084, |
|
"nll_loss": 1.2117825746536255, |
|
"po_loss": -0.8654062151908875, |
|
"rewards/accuracies": 0.953125, |
|
"rewards/chosen": -0.5526939034461975, |
|
"rewards/margins": 3.1176929473876953, |
|
"rewards/rejected": -3.670386791229248, |
|
"step": 265 |
|
}, |
|
{ |
|
"epoch": 2.2901333333333334, |
|
"grad_norm": 250.27699279785156, |
|
"learning_rate": 7.723433775328384e-06, |
|
"log_odds_chosen": 6.858447074890137, |
|
"log_odds_ratio": -0.31602945923805237, |
|
"logps/chosen": -1.1505261659622192, |
|
"logps/rejected": -7.580657005310059, |
|
"loss": 20.3187, |
|
"nll_loss": 1.2361539602279663, |
|
"po_loss": -0.6011937260627747, |
|
"rewards/accuracies": 0.925000011920929, |
|
"rewards/chosen": -0.5752630829811096, |
|
"rewards/margins": 3.2150657176971436, |
|
"rewards/rejected": -3.7903285026550293, |
|
"step": 270 |
|
}, |
|
{ |
|
"epoch": 2.3327999999999998, |
|
"grad_norm": 3240.22802734375, |
|
"learning_rate": 6.844201154750177e-06, |
|
"log_odds_chosen": 6.9127960205078125, |
|
"log_odds_ratio": -0.28268226981163025, |
|
"logps/chosen": -1.3622684478759766, |
|
"logps/rejected": -7.855978965759277, |
|
"loss": 22.8393, |
|
"nll_loss": 1.4035627841949463, |
|
"po_loss": -0.6898348927497864, |
|
"rewards/accuracies": 0.9437500238418579, |
|
"rewards/chosen": -0.6811342239379883, |
|
"rewards/margins": 3.2468554973602295, |
|
"rewards/rejected": -3.9279894828796387, |
|
"step": 275 |
|
}, |
|
{ |
|
"epoch": 2.3754666666666666, |
|
"grad_norm": 650.8209228515625, |
|
"learning_rate": 6.010106710768052e-06, |
|
"log_odds_chosen": 6.493264675140381, |
|
"log_odds_ratio": -0.34161344170570374, |
|
"logps/chosen": -1.386515498161316, |
|
"logps/rejected": -7.452689170837402, |
|
"loss": 19.5164, |
|
"nll_loss": 1.3803019523620605, |
|
"po_loss": -0.7704136967658997, |
|
"rewards/accuracies": 0.90625, |
|
"rewards/chosen": -0.693257749080658, |
|
"rewards/margins": 3.0330867767333984, |
|
"rewards/rejected": -3.726344585418701, |
|
"step": 280 |
|
}, |
|
{ |
|
"epoch": 2.4181333333333335, |
|
"grad_norm": 507.415283203125, |
|
"learning_rate": 5.223224133591476e-06, |
|
"log_odds_chosen": 6.825043678283691, |
|
"log_odds_ratio": -0.2423882931470871, |
|
"logps/chosen": -1.1171207427978516, |
|
"logps/rejected": -7.483119964599609, |
|
"loss": 19.53, |
|
"nll_loss": 1.2847764492034912, |
|
"po_loss": -0.6744629144668579, |
|
"rewards/accuracies": 0.9468749761581421, |
|
"rewards/chosen": -0.5585603713989258, |
|
"rewards/margins": 3.1829993724823, |
|
"rewards/rejected": -3.7415599822998047, |
|
"step": 285 |
|
}, |
|
{ |
|
"epoch": 2.4608, |
|
"grad_norm": 789.3963623046875, |
|
"learning_rate": 4.4855097372902135e-06, |
|
"log_odds_chosen": 7.850038051605225, |
|
"log_odds_ratio": -0.26526254415512085, |
|
"logps/chosen": -1.4054704904556274, |
|
"logps/rejected": -8.81872844696045, |
|
"loss": 18.6059, |
|
"nll_loss": 1.3957431316375732, |
|
"po_loss": -0.8143098950386047, |
|
"rewards/accuracies": 0.9312499761581421, |
|
"rewards/chosen": -0.7027352452278137, |
|
"rewards/margins": 3.7066292762756348, |
|
"rewards/rejected": -4.409364223480225, |
|
"step": 290 |
|
}, |
|
{ |
|
"epoch": 2.5034666666666667, |
|
"grad_norm": 1187.58837890625, |
|
"learning_rate": 3.798797596089351e-06, |
|
"log_odds_chosen": 8.998720169067383, |
|
"log_odds_ratio": -0.218130424618721, |
|
"logps/chosen": -1.247434377670288, |
|
"logps/rejected": -9.765419960021973, |
|
"loss": 10.3413, |
|
"nll_loss": 1.355464220046997, |
|
"po_loss": -1.032299280166626, |
|
"rewards/accuracies": 0.9593750238418579, |
|
"rewards/chosen": -0.623717188835144, |
|
"rewards/margins": 4.2589921951293945, |
|
"rewards/rejected": -4.882709980010986, |
|
"step": 295 |
|
}, |
|
{ |
|
"epoch": 2.5461333333333336, |
|
"grad_norm": 593.267578125, |
|
"learning_rate": 3.164794984571759e-06, |
|
"log_odds_chosen": 6.977963924407959, |
|
"log_odds_ratio": -0.3458719849586487, |
|
"logps/chosen": -1.6929105520248413, |
|
"logps/rejected": -8.215627670288086, |
|
"loss": 27.4312, |
|
"nll_loss": 1.5623621940612793, |
|
"po_loss": -0.7051381468772888, |
|
"rewards/accuracies": 0.921875, |
|
"rewards/chosen": -0.8464552760124207, |
|
"rewards/margins": 3.2613587379455566, |
|
"rewards/rejected": -4.107813835144043, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 2.5888, |
|
"grad_norm": 938.1863403320312, |
|
"learning_rate": 2.58507813312448e-06, |
|
"log_odds_chosen": 6.914087772369385, |
|
"log_odds_ratio": -0.28258031606674194, |
|
"logps/chosen": -1.2223470211029053, |
|
"logps/rejected": -7.694061279296875, |
|
"loss": 21.9591, |
|
"nll_loss": 1.3453009128570557, |
|
"po_loss": -0.6590788960456848, |
|
"rewards/accuracies": 0.909375011920929, |
|
"rewards/chosen": -0.6111735105514526, |
|
"rewards/margins": 3.2358570098876953, |
|
"rewards/rejected": -3.8470306396484375, |
|
"step": 305 |
|
}, |
|
{ |
|
"epoch": 2.6314666666666664, |
|
"grad_norm": 221.02281188964844, |
|
"learning_rate": 2.0610883091816525e-06, |
|
"log_odds_chosen": 5.710585594177246, |
|
"log_odds_ratio": -0.2850475609302521, |
|
"logps/chosen": -1.021317720413208, |
|
"logps/rejected": -6.243259906768799, |
|
"loss": 16.8653, |
|
"nll_loss": 1.247992753982544, |
|
"po_loss": -0.7209508419036865, |
|
"rewards/accuracies": 0.9281250238418579, |
|
"rewards/chosen": -0.510658860206604, |
|
"rewards/margins": 2.610970973968506, |
|
"rewards/rejected": -3.1216299533843994, |
|
"step": 310 |
|
}, |
|
{ |
|
"epoch": 2.6741333333333333, |
|
"grad_norm": 475.2201843261719, |
|
"learning_rate": 1.59412823400657e-06, |
|
"log_odds_chosen": 5.025318145751953, |
|
"log_odds_ratio": -0.25998514890670776, |
|
"logps/chosen": -1.0306951999664307, |
|
"logps/rejected": -5.601127624511719, |
|
"loss": 14.5714, |
|
"nll_loss": 1.1835569143295288, |
|
"po_loss": -0.7282010316848755, |
|
"rewards/accuracies": 0.925000011920929, |
|
"rewards/chosen": -0.5153475999832153, |
|
"rewards/margins": 2.2852160930633545, |
|
"rewards/rejected": -2.8005638122558594, |
|
"step": 315 |
|
}, |
|
{ |
|
"epoch": 2.7168, |
|
"grad_norm": 337.47796630859375, |
|
"learning_rate": 1.1853588439213442e-06, |
|
"log_odds_chosen": 6.4345574378967285, |
|
"log_odds_ratio": -0.23376791179180145, |
|
"logps/chosen": -1.014891266822815, |
|
"logps/rejected": -7.0141282081604, |
|
"loss": 16.8373, |
|
"nll_loss": 1.2330230474472046, |
|
"po_loss": -0.7068589329719543, |
|
"rewards/accuracies": 0.9437500238418579, |
|
"rewards/chosen": -0.5074456334114075, |
|
"rewards/margins": 2.9996187686920166, |
|
"rewards/rejected": -3.5070641040802, |
|
"step": 320 |
|
}, |
|
{ |
|
"epoch": 2.7594666666666665, |
|
"grad_norm": 251.52114868164062, |
|
"learning_rate": 8.357964040363209e-07, |
|
"log_odds_chosen": 7.006105899810791, |
|
"log_odds_ratio": -0.3340076804161072, |
|
"logps/chosen": -1.0801838636398315, |
|
"logps/rejected": -7.584368705749512, |
|
"loss": 7.9524, |
|
"nll_loss": 1.2427351474761963, |
|
"po_loss": -0.9942227602005005, |
|
"rewards/accuracies": 0.9281250238418579, |
|
"rewards/chosen": -0.5400919318199158, |
|
"rewards/margins": 3.2520930767059326, |
|
"rewards/rejected": -3.792184352874756, |
|
"step": 325 |
|
}, |
|
{ |
|
"epoch": 2.8021333333333334, |
|
"grad_norm": 500.19580078125, |
|
"learning_rate": 5.463099816548579e-07, |
|
"log_odds_chosen": 7.333872318267822, |
|
"log_odds_ratio": -0.251155823469162, |
|
"logps/chosen": -1.1577497720718384, |
|
"logps/rejected": -8.027981758117676, |
|
"loss": 19.0267, |
|
"nll_loss": 1.2874268293380737, |
|
"po_loss": -0.6928423047065735, |
|
"rewards/accuracies": 0.9312499761581421, |
|
"rewards/chosen": -0.5788748860359192, |
|
"rewards/margins": 3.4351158142089844, |
|
"rewards/rejected": -4.013990879058838, |
|
"step": 330 |
|
}, |
|
{ |
|
"epoch": 2.8448, |
|
"grad_norm": 827.9666748046875, |
|
"learning_rate": 3.1761928563510955e-07, |
|
"log_odds_chosen": 5.787890434265137, |
|
"log_odds_ratio": -0.2869322896003723, |
|
"logps/chosen": -1.0971204042434692, |
|
"logps/rejected": -6.4105658531188965, |
|
"loss": 12.9881, |
|
"nll_loss": 1.2217962741851807, |
|
"po_loss": -0.8159177899360657, |
|
"rewards/accuracies": 0.9125000238418579, |
|
"rewards/chosen": -0.5485602021217346, |
|
"rewards/margins": 2.6567225456237793, |
|
"rewards/rejected": -3.2052829265594482, |
|
"step": 335 |
|
}, |
|
{ |
|
"epoch": 2.8874666666666666, |
|
"grad_norm": 292.38897705078125, |
|
"learning_rate": 1.5029287708036854e-07, |
|
"log_odds_chosen": 6.05283260345459, |
|
"log_odds_ratio": -0.24598774313926697, |
|
"logps/chosen": -1.0574235916137695, |
|
"logps/rejected": -6.646490573883057, |
|
"loss": 7.9611, |
|
"nll_loss": 1.1931092739105225, |
|
"po_loss": -0.9443262219429016, |
|
"rewards/accuracies": 0.956250011920929, |
|
"rewards/chosen": -0.5287117958068848, |
|
"rewards/margins": 2.7945332527160645, |
|
"rewards/rejected": -3.3232452869415283, |
|
"step": 340 |
|
}, |
|
{ |
|
"epoch": 2.9301333333333335, |
|
"grad_norm": 387.1372985839844, |
|
"learning_rate": 4.474675580662113e-08, |
|
"log_odds_chosen": 6.334747791290283, |
|
"log_odds_ratio": -0.32026782631874084, |
|
"logps/chosen": -1.1003960371017456, |
|
"logps/rejected": -6.976574897766113, |
|
"loss": 12.2009, |
|
"nll_loss": 1.235642671585083, |
|
"po_loss": -0.8543656468391418, |
|
"rewards/accuracies": 0.9156249761581421, |
|
"rewards/chosen": -0.5501980185508728, |
|
"rewards/margins": 2.938089370727539, |
|
"rewards/rejected": -3.4882874488830566, |
|
"step": 345 |
|
}, |
|
{ |
|
"epoch": 2.9728, |
|
"grad_norm": 1250.0882568359375, |
|
"learning_rate": 1.2433261014244136e-09, |
|
"log_odds_chosen": 7.907639980316162, |
|
"log_odds_ratio": -0.22040143609046936, |
|
"logps/chosen": -1.2740892171859741, |
|
"logps/rejected": -8.710322380065918, |
|
"loss": 9.5303, |
|
"nll_loss": 1.3264214992523193, |
|
"po_loss": -1.028598666191101, |
|
"rewards/accuracies": 0.9281250238418579, |
|
"rewards/chosen": -0.6370446085929871, |
|
"rewards/margins": 3.7181167602539062, |
|
"rewards/rejected": -4.355161190032959, |
|
"step": 350 |
|
}, |
|
{ |
|
"epoch": 2.981333333333333, |
|
"step": 351, |
|
"total_flos": 0.0, |
|
"train_loss": 54.80183921716152, |
|
"train_runtime": 10133.5503, |
|
"train_samples_per_second": 2.22, |
|
"train_steps_per_second": 0.035 |
|
} |
|
], |
|
"logging_steps": 5, |
|
"max_steps": 351, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 3, |
|
"save_steps": 100000, |
|
"stateful_callbacks": { |
|
"TrainerControl": { |
|
"args": { |
|
"should_epoch_stop": false, |
|
"should_evaluate": false, |
|
"should_log": false, |
|
"should_save": true, |
|
"should_training_stop": true |
|
}, |
|
"attributes": {} |
|
} |
|
}, |
|
"total_flos": 0.0, |
|
"train_batch_size": 1, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|