silviasapora's picture
Model save
0f03fb5 verified
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 2.981333333333333,
"eval_steps": 500,
"global_step": 351,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.042666666666666665,
"grad_norm": 1961.1512451171875,
"learning_rate": 6.944444444444445e-06,
"log_odds_chosen": 3.233767032623291,
"log_odds_ratio": -7.3191938400268555,
"logps/chosen": -23.426687240600586,
"logps/rejected": -26.660680770874023,
"loss": 425.3708,
"nll_loss": 9.728386878967285,
"po_loss": 3.5644500255584717,
"rewards/accuracies": 0.515625,
"rewards/chosen": -11.713343620300293,
"rewards/margins": 1.6169971227645874,
"rewards/rejected": -13.330340385437012,
"step": 5
},
{
"epoch": 0.08533333333333333,
"grad_norm": 500.5514831542969,
"learning_rate": 1.388888888888889e-05,
"log_odds_chosen": 3.315504550933838,
"log_odds_ratio": -6.866879463195801,
"logps/chosen": -23.438692092895508,
"logps/rejected": -26.755935668945312,
"loss": 393.0612,
"nll_loss": 8.955938339233398,
"po_loss": 3.3272223472595215,
"rewards/accuracies": 0.48124998807907104,
"rewards/chosen": -11.719346046447754,
"rewards/margins": 1.658623456954956,
"rewards/rejected": -13.377967834472656,
"step": 10
},
{
"epoch": 0.128,
"grad_norm": 514.0573120117188,
"learning_rate": 2.0833333333333336e-05,
"log_odds_chosen": 7.634564399719238,
"log_odds_ratio": -6.296060562133789,
"logps/chosen": -22.5439395904541,
"logps/rejected": -30.179019927978516,
"loss": 378.1973,
"nll_loss": 8.766839981079102,
"po_loss": 3.0518250465393066,
"rewards/accuracies": 0.53125,
"rewards/chosen": -11.27196979522705,
"rewards/margins": 3.8175387382507324,
"rewards/rejected": -15.089509963989258,
"step": 15
},
{
"epoch": 0.17066666666666666,
"grad_norm": 3820.703125,
"learning_rate": 2.777777777777778e-05,
"log_odds_chosen": -1.3400392532348633,
"log_odds_ratio": -8.175825119018555,
"logps/chosen": -18.689817428588867,
"logps/rejected": -17.35008430480957,
"loss": 363.51,
"nll_loss": 7.390625953674316,
"po_loss": 3.9690635204315186,
"rewards/accuracies": 0.484375,
"rewards/chosen": -9.344908714294434,
"rewards/margins": -0.6698670983314514,
"rewards/rejected": -8.675042152404785,
"step": 20
},
{
"epoch": 0.21333333333333335,
"grad_norm": 417.3852844238281,
"learning_rate": 3.472222222222222e-05,
"log_odds_chosen": -0.37303870916366577,
"log_odds_ratio": -2.343538284301758,
"logps/chosen": -5.883327484130859,
"logps/rejected": -5.492794036865234,
"loss": 134.8965,
"nll_loss": 3.241476535797119,
"po_loss": 0.9740395545959473,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": -2.9416637420654297,
"rewards/margins": -0.19526681303977966,
"rewards/rejected": -2.746397018432617,
"step": 25
},
{
"epoch": 0.256,
"grad_norm": 330.9832458496094,
"learning_rate": 4.166666666666667e-05,
"log_odds_chosen": 0.3532889187335968,
"log_odds_ratio": -0.7218837141990662,
"logps/chosen": -1.7468492984771729,
"logps/rejected": -2.0471103191375732,
"loss": 70.938,
"nll_loss": 2.1013693809509277,
"po_loss": 0.11544491350650787,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.8734246492385864,
"rewards/margins": 0.15013054013252258,
"rewards/rejected": -1.0235551595687866,
"step": 30
},
{
"epoch": 0.2986666666666667,
"grad_norm": 284.9355163574219,
"learning_rate": 4.8611111111111115e-05,
"log_odds_chosen": 0.4371984004974365,
"log_odds_ratio": -0.6356752514839172,
"logps/chosen": -1.627811074256897,
"logps/rejected": -1.993390679359436,
"loss": 63.8055,
"nll_loss": 1.9257453680038452,
"po_loss": 0.06817711889743805,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -0.8139055371284485,
"rewards/margins": 0.18278983235359192,
"rewards/rejected": -0.996695339679718,
"step": 35
},
{
"epoch": 0.3413333333333333,
"grad_norm": 126.80204772949219,
"learning_rate": 4.998010925565448e-05,
"log_odds_chosen": 0.36739838123321533,
"log_odds_ratio": -0.6945838928222656,
"logps/chosen": -1.696966528892517,
"logps/rejected": -2.019742488861084,
"loss": 66.0958,
"nll_loss": 1.972928762435913,
"po_loss": 0.0925649031996727,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.8484832644462585,
"rewards/margins": 0.16138817369937897,
"rewards/rejected": -1.009871244430542,
"step": 40
},
{
"epoch": 0.384,
"grad_norm": 734.975830078125,
"learning_rate": 4.989935734988098e-05,
"log_odds_chosen": 0.2258051633834839,
"log_odds_ratio": -0.6970190405845642,
"logps/chosen": -1.5770127773284912,
"logps/rejected": -1.7601944208145142,
"loss": 61.9986,
"nll_loss": 1.8443195819854736,
"po_loss": 0.09313799440860748,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.7885063886642456,
"rewards/margins": 0.09159073978662491,
"rewards/rejected": -0.8800972104072571,
"step": 45
},
{
"epoch": 0.4266666666666667,
"grad_norm": 98.6443862915039,
"learning_rate": 4.975670171853926e-05,
"log_odds_chosen": 0.3061736226081848,
"log_odds_ratio": -0.6558084487915039,
"logps/chosen": -1.410742998123169,
"logps/rejected": -1.6612850427627563,
"loss": 56.1342,
"nll_loss": 1.6897376775741577,
"po_loss": 0.06445512920618057,
"rewards/accuracies": 0.6343749761581421,
"rewards/chosen": -0.7053714990615845,
"rewards/margins": 0.1252710521221161,
"rewards/rejected": -0.8306425213813782,
"step": 50
},
{
"epoch": 0.4693333333333333,
"grad_norm": 138.6815948486328,
"learning_rate": 4.9552497026005974e-05,
"log_odds_chosen": 0.21871152520179749,
"log_odds_ratio": -0.7383303046226501,
"logps/chosen": -1.4634255170822144,
"logps/rejected": -1.6147739887237549,
"loss": 59.4312,
"nll_loss": 1.7589048147201538,
"po_loss": 0.09831906855106354,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -0.7317127585411072,
"rewards/margins": 0.07567427307367325,
"rewards/rejected": -0.8073869943618774,
"step": 55
},
{
"epoch": 0.512,
"grad_norm": 102.89739227294922,
"learning_rate": 4.928725095732169e-05,
"log_odds_chosen": 0.401327520608902,
"log_odds_ratio": -0.6062948107719421,
"logps/chosen": -1.2786072492599487,
"logps/rejected": -1.585089921951294,
"loss": 51.5199,
"nll_loss": 1.5874063968658447,
"po_loss": 0.02259092405438423,
"rewards/accuracies": 0.6812499761581421,
"rewards/chosen": -0.6393036246299744,
"rewards/margins": 0.15324126183986664,
"rewards/rejected": -0.792544960975647,
"step": 60
},
{
"epoch": 0.5546666666666666,
"grad_norm": 100.74688720703125,
"learning_rate": 4.896162295600589e-05,
"log_odds_chosen": 0.33866989612579346,
"log_odds_ratio": -0.6249087452888489,
"logps/chosen": -1.3379658460617065,
"logps/rejected": -1.6066213846206665,
"loss": 53.1437,
"nll_loss": 1.6211912631988525,
"po_loss": 0.03954792767763138,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.6689829230308533,
"rewards/margins": 0.13432787358760834,
"rewards/rejected": -0.8033106923103333,
"step": 65
},
{
"epoch": 0.5973333333333334,
"grad_norm": 107.1102523803711,
"learning_rate": 4.8576422584576514e-05,
"log_odds_chosen": 0.28569236397743225,
"log_odds_ratio": -0.6388062238693237,
"logps/chosen": -1.2627140283584595,
"logps/rejected": -1.478611707687378,
"loss": 51.3954,
"nll_loss": 1.5717823505401611,
"po_loss": 0.034325726330280304,
"rewards/accuracies": 0.628125011920929,
"rewards/chosen": -0.6313570141792297,
"rewards/margins": 0.10794880241155624,
"rewards/rejected": -0.739305853843689,
"step": 70
},
{
"epoch": 0.64,
"grad_norm": 87.37519073486328,
"learning_rate": 4.813260751184992e-05,
"log_odds_chosen": 0.4192368984222412,
"log_odds_ratio": -0.6039353609085083,
"logps/chosen": -1.2572083473205566,
"logps/rejected": -1.5851625204086304,
"loss": 50.7092,
"nll_loss": 1.5785938501358032,
"po_loss": 0.00606974633410573,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.6286041736602783,
"rewards/margins": 0.1639770269393921,
"rewards/rejected": -0.7925812602043152,
"step": 75
},
{
"epoch": 0.6826666666666666,
"grad_norm": 84.41800689697266,
"learning_rate": 4.763128113202537e-05,
"log_odds_chosen": 0.3978249132633209,
"log_odds_ratio": -0.6257617473602295,
"logps/chosen": -1.1973122358322144,
"logps/rejected": -1.5119242668151855,
"loss": 49.6894,
"nll_loss": 1.5280753374099731,
"po_loss": 0.024718867614865303,
"rewards/accuracies": 0.6031249761581421,
"rewards/chosen": -0.5986561179161072,
"rewards/margins": 0.1573060303926468,
"rewards/rejected": -0.7559621334075928,
"step": 80
},
{
"epoch": 0.7253333333333334,
"grad_norm": 217.4127655029297,
"learning_rate": 4.707368982147318e-05,
"log_odds_chosen": 0.9764216542243958,
"log_odds_ratio": -0.5953387022018433,
"logps/chosen": -1.2593096494674683,
"logps/rejected": -2.1358559131622314,
"loss": 50.7417,
"nll_loss": 1.5718034505844116,
"po_loss": 0.013875389471650124,
"rewards/accuracies": 0.628125011920929,
"rewards/chosen": -0.6296548247337341,
"rewards/margins": 0.43827304244041443,
"rewards/rejected": -1.0679279565811157,
"step": 85
},
{
"epoch": 0.768,
"grad_norm": 74.66828918457031,
"learning_rate": 4.6461219840046654e-05,
"log_odds_chosen": 0.7273199558258057,
"log_odds_ratio": -0.6018415093421936,
"logps/chosen": -1.213512897491455,
"logps/rejected": -1.8320105075836182,
"loss": 48.283,
"nll_loss": 1.5248548984527588,
"po_loss": -0.016011018306016922,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.6067564487457275,
"rewards/margins": 0.30924877524375916,
"rewards/rejected": -0.9160052537918091,
"step": 90
},
{
"epoch": 0.8106666666666666,
"grad_norm": 96.17576599121094,
"learning_rate": 4.579539388462173e-05,
"log_odds_chosen": 0.968981921672821,
"log_odds_ratio": -0.5852179527282715,
"logps/chosen": -1.1610548496246338,
"logps/rejected": -2.0138120651245117,
"loss": 46.2603,
"nll_loss": 1.5272886753082275,
"po_loss": -0.08165426552295685,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -0.5805274248123169,
"rewards/margins": 0.4263785481452942,
"rewards/rejected": -1.0069060325622559,
"step": 95
},
{
"epoch": 0.8533333333333334,
"grad_norm": 1037.95703125,
"learning_rate": 4.5077867303432546e-05,
"log_odds_chosen": 2.305642604827881,
"log_odds_ratio": -0.6378474831581116,
"logps/chosen": -1.2965327501296997,
"logps/rejected": -3.484023332595825,
"loss": 40.3825,
"nll_loss": 1.5268816947937012,
"po_loss": -0.2649289071559906,
"rewards/accuracies": 0.671875,
"rewards/chosen": -0.6482663750648499,
"rewards/margins": 1.093745231628418,
"rewards/rejected": -1.7420116662979126,
"step": 100
},
{
"epoch": 0.896,
"grad_norm": 120.91621398925781,
"learning_rate": 4.431042398061499e-05,
"log_odds_chosen": 2.5239200592041016,
"log_odds_ratio": -0.7032963633537292,
"logps/chosen": -2.863506317138672,
"logps/rejected": -5.296839714050293,
"loss": 85.1499,
"nll_loss": 2.635256767272949,
"po_loss": 0.025676274672150612,
"rewards/accuracies": 0.6937500238418579,
"rewards/chosen": -1.431753158569336,
"rewards/margins": 1.2166664600372314,
"rewards/rejected": -2.6484198570251465,
"step": 105
},
{
"epoch": 0.9386666666666666,
"grad_norm": 80.25611114501953,
"learning_rate": 4.34949719011896e-05,
"log_odds_chosen": 0.859009861946106,
"log_odds_ratio": -0.5764500498771667,
"logps/chosen": -1.229062557220459,
"logps/rejected": -1.9831807613372803,
"loss": 47.4796,
"nll_loss": 1.5179859399795532,
"po_loss": -0.03424710035324097,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": -0.6145312786102295,
"rewards/margins": 0.37705910205841064,
"rewards/rejected": -0.9915903806686401,
"step": 110
},
{
"epoch": 0.9813333333333333,
"grad_norm": 271.3537902832031,
"learning_rate": 4.263353840751022e-05,
"log_odds_chosen": 1.4788315296173096,
"log_odds_ratio": -0.5846825242042542,
"logps/chosen": -1.1537017822265625,
"logps/rejected": -2.52790904045105,
"loss": 44.319,
"nll_loss": 1.4921709299087524,
"po_loss": -0.10720314830541611,
"rewards/accuracies": 0.6781250238418579,
"rewards/chosen": -0.5768508911132812,
"rewards/margins": 0.6871034502983093,
"rewards/rejected": -1.263954520225525,
"step": 115
},
{
"epoch": 1.0170666666666666,
"grad_norm": 146.54196166992188,
"learning_rate": 4.172826515897146e-05,
"log_odds_chosen": 2.831665515899658,
"log_odds_ratio": -0.4861010015010834,
"logps/chosen": -1.089508056640625,
"logps/rejected": -3.7528676986694336,
"loss": 33.7801,
"nll_loss": 1.4263149499893188,
"po_loss": -0.1658623069524765,
"rewards/accuracies": 0.7723880410194397,
"rewards/chosen": -0.5447540283203125,
"rewards/margins": 1.3316797018051147,
"rewards/rejected": -1.8764338493347168,
"step": 120
},
{
"epoch": 1.0597333333333334,
"grad_norm": 1590.6824951171875,
"learning_rate": 4.078140280750597e-05,
"log_odds_chosen": 4.709661483764648,
"log_odds_ratio": -0.41537848114967346,
"logps/chosen": -1.5830024480819702,
"logps/rejected": -6.075448036193848,
"loss": 46.6655,
"nll_loss": 1.6417958736419678,
"po_loss": -0.18349842727184296,
"rewards/accuracies": 0.8531249761581421,
"rewards/chosen": -0.7915012240409851,
"rewards/margins": 2.246222496032715,
"rewards/rejected": -3.037724018096924,
"step": 125
},
{
"epoch": 1.1024,
"grad_norm": 321.93255615234375,
"learning_rate": 3.9795305402109195e-05,
"log_odds_chosen": 3.6606578826904297,
"log_odds_ratio": -0.4261302947998047,
"logps/chosen": -1.2628165483474731,
"logps/rejected": -4.708868980407715,
"loss": 36.688,
"nll_loss": 1.4487800598144531,
"po_loss": -0.3022807240486145,
"rewards/accuracies": 0.8218749761581421,
"rewards/chosen": -0.6314082741737366,
"rewards/margins": 1.7230265140533447,
"rewards/rejected": -2.3544344902038574,
"step": 130
},
{
"epoch": 1.1450666666666667,
"grad_norm": 156.33656311035156,
"learning_rate": 3.8772424536302564e-05,
"log_odds_chosen": 3.3216488361358643,
"log_odds_ratio": -0.45871955156326294,
"logps/chosen": -1.054512619972229,
"logps/rejected": -4.166798114776611,
"loss": 37.5147,
"nll_loss": 1.3529837131500244,
"po_loss": -0.18065081536769867,
"rewards/accuracies": 0.778124988079071,
"rewards/chosen": -0.5272563099861145,
"rewards/margins": 1.5561425685882568,
"rewards/rejected": -2.0833990573883057,
"step": 135
},
{
"epoch": 1.1877333333333333,
"grad_norm": 126.58124542236328,
"learning_rate": 3.771530325308579e-05,
"log_odds_chosen": 3.002664089202881,
"log_odds_ratio": -0.3991411030292511,
"logps/chosen": -0.99451744556427,
"logps/rejected": -3.7320716381073,
"loss": 33.4168,
"nll_loss": 1.2971866130828857,
"po_loss": -0.2529115378856659,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": -0.497258722782135,
"rewards/margins": 1.3687771558761597,
"rewards/rejected": -1.86603581905365,
"step": 140
},
{
"epoch": 1.2304,
"grad_norm": 144.6908416748047,
"learning_rate": 3.662656972253127e-05,
"log_odds_chosen": 3.620790958404541,
"log_odds_ratio": -0.39333289861679077,
"logps/chosen": -1.1291520595550537,
"logps/rejected": -4.468079090118408,
"loss": 31.6104,
"nll_loss": 1.2871015071868896,
"po_loss": -0.2992752194404602,
"rewards/accuracies": 0.875,
"rewards/chosen": -0.5645760297775269,
"rewards/margins": 1.6694637537002563,
"rewards/rejected": -2.234039545059204,
"step": 145
},
{
"epoch": 1.2730666666666668,
"grad_norm": 317.3659362792969,
"learning_rate": 3.550893070773914e-05,
"log_odds_chosen": 2.5045361518859863,
"log_odds_ratio": -0.5062359571456909,
"logps/chosen": -1.01883864402771,
"logps/rejected": -3.3051059246063232,
"loss": 35.2667,
"nll_loss": 1.3030840158462524,
"po_loss": -0.20100060105323792,
"rewards/accuracies": 0.831250011920929,
"rewards/chosen": -0.509419322013855,
"rewards/margins": 1.143133521080017,
"rewards/rejected": -1.6525529623031616,
"step": 150
},
{
"epoch": 1.3157333333333332,
"grad_norm": 99.71307373046875,
"learning_rate": 3.436516483539781e-05,
"log_odds_chosen": 2.3835015296936035,
"log_odds_ratio": -0.4490797519683838,
"logps/chosen": -0.961607813835144,
"logps/rejected": -3.094728946685791,
"loss": 36.1188,
"nll_loss": 1.2786829471588135,
"po_loss": -0.1499709188938141,
"rewards/accuracies": 0.8343750238418579,
"rewards/chosen": -0.480803906917572,
"rewards/margins": 1.0665608644485474,
"rewards/rejected": -1.5473644733428955,
"step": 155
},
{
"epoch": 1.3584,
"grad_norm": 176.96144104003906,
"learning_rate": 3.3198115687680115e-05,
"log_odds_chosen": 2.804508924484253,
"log_odds_ratio": -0.4551823139190674,
"logps/chosen": -0.9649540781974792,
"logps/rejected": -3.5283005237579346,
"loss": 31.9084,
"nll_loss": 1.2769290208816528,
"po_loss": -0.2797916531562805,
"rewards/accuracies": 0.7906249761581421,
"rewards/chosen": -0.4824770390987396,
"rewards/margins": 1.2816731929779053,
"rewards/rejected": -1.7641502618789673,
"step": 160
},
{
"epoch": 1.4010666666666667,
"grad_norm": 86.9504165649414,
"learning_rate": 3.201068473265007e-05,
"log_odds_chosen": 4.039158821105957,
"log_odds_ratio": -0.41200175881385803,
"logps/chosen": -1.1530981063842773,
"logps/rejected": -4.936544418334961,
"loss": 33.9241,
"nll_loss": 1.339659571647644,
"po_loss": -0.27953004837036133,
"rewards/accuracies": 0.840624988079071,
"rewards/chosen": -0.5765490531921387,
"rewards/margins": 1.8917229175567627,
"rewards/rejected": -2.4682722091674805,
"step": 165
},
{
"epoch": 1.4437333333333333,
"grad_norm": 67.8962631225586,
"learning_rate": 3.0805824110756064e-05,
"log_odds_chosen": 2.8494575023651123,
"log_odds_ratio": -0.4309585690498352,
"logps/chosen": -1.0003385543823242,
"logps/rejected": -3.5997538566589355,
"loss": 32.9378,
"nll_loss": 1.320733904838562,
"po_loss": -0.2914286255836487,
"rewards/accuracies": 0.84375,
"rewards/chosen": -0.5001692771911621,
"rewards/margins": 1.2997074127197266,
"rewards/rejected": -1.7998769283294678,
"step": 170
},
{
"epoch": 1.4864,
"grad_norm": 332.5802307128906,
"learning_rate": 2.958652929534456e-05,
"log_odds_chosen": 4.219790458679199,
"log_odds_ratio": -0.3872655928134918,
"logps/chosen": -0.9488153457641602,
"logps/rejected": -4.870639324188232,
"loss": 24.5,
"nll_loss": 1.2430005073547363,
"po_loss": -0.47737568616867065,
"rewards/accuracies": 0.8656250238418579,
"rewards/chosen": -0.4744076728820801,
"rewards/margins": 1.9609121084213257,
"rewards/rejected": -2.435319662094116,
"step": 175
},
{
"epoch": 1.5290666666666666,
"grad_norm": 120.37972259521484,
"learning_rate": 2.8355831645441388e-05,
"log_odds_chosen": 3.8075263500213623,
"log_odds_ratio": -0.4285067617893219,
"logps/chosen": -1.0132713317871094,
"logps/rejected": -4.545207500457764,
"loss": 30.9088,
"nll_loss": 1.2752387523651123,
"po_loss": -0.3093392252922058,
"rewards/accuracies": 0.840624988079071,
"rewards/chosen": -0.5066356658935547,
"rewards/margins": 1.7659679651260376,
"rewards/rejected": -2.272603750228882,
"step": 180
},
{
"epoch": 1.5717333333333334,
"grad_norm": 74.23609924316406,
"learning_rate": 2.7116790869315582e-05,
"log_odds_chosen": 3.5458972454071045,
"log_odds_ratio": -0.3724256157875061,
"logps/chosen": -0.9797577857971191,
"logps/rejected": -4.209129810333252,
"loss": 27.577,
"nll_loss": 1.2894318103790283,
"po_loss": -0.42764997482299805,
"rewards/accuracies": 0.887499988079071,
"rewards/chosen": -0.48987889289855957,
"rewards/margins": 1.6146857738494873,
"rewards/rejected": -2.104564905166626,
"step": 185
},
{
"epoch": 1.6143999999999998,
"grad_norm": 1912.4134521484375,
"learning_rate": 2.587248741756253e-05,
"log_odds_chosen": 4.569626808166504,
"log_odds_ratio": -0.6492961049079895,
"logps/chosen": -1.6978082656860352,
"logps/rejected": -5.967839241027832,
"loss": 47.8645,
"nll_loss": 1.7877342700958252,
"po_loss": -0.2919684648513794,
"rewards/accuracies": 0.840624988079071,
"rewards/chosen": -0.8489041328430176,
"rewards/margins": 2.1350154876708984,
"rewards/rejected": -2.983919620513916,
"step": 190
},
{
"epoch": 1.6570666666666667,
"grad_norm": 246.9727020263672,
"learning_rate": 2.4626014824618415e-05,
"log_odds_chosen": 3.452259063720703,
"log_odds_ratio": -0.4340329170227051,
"logps/chosen": -1.348034143447876,
"logps/rejected": -4.516729831695557,
"loss": 39.5128,
"nll_loss": 1.5523712635040283,
"po_loss": -0.3175976276397705,
"rewards/accuracies": 0.8218749761581421,
"rewards/chosen": -0.674017071723938,
"rewards/margins": 1.5843479633331299,
"rewards/rejected": -2.2583649158477783,
"step": 195
},
{
"epoch": 1.6997333333333333,
"grad_norm": 249.69097900390625,
"learning_rate": 2.3380472017746202e-05,
"log_odds_chosen": 4.080594539642334,
"log_odds_ratio": -0.41809386014938354,
"logps/chosen": -1.1225640773773193,
"logps/rejected": -4.919617652893066,
"loss": 22.4016,
"nll_loss": 1.3972797393798828,
"po_loss": -0.6972296237945557,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": -0.5612820386886597,
"rewards/margins": 1.8985265493392944,
"rewards/rejected": -2.459808826446533,
"step": 200
},
{
"epoch": 1.7424,
"grad_norm": 70.95199584960938,
"learning_rate": 2.2138955612614207e-05,
"log_odds_chosen": 3.4808452129364014,
"log_odds_ratio": -0.4113942086696625,
"logps/chosen": -0.9912225604057312,
"logps/rejected": -4.186377048492432,
"loss": 30.6187,
"nll_loss": 1.2635384798049927,
"po_loss": -0.30670270323753357,
"rewards/accuracies": 0.862500011920929,
"rewards/chosen": -0.4956112802028656,
"rewards/margins": 1.5975773334503174,
"rewards/rejected": -2.093188524246216,
"step": 205
},
{
"epoch": 1.7850666666666668,
"grad_norm": 77.6159896850586,
"learning_rate": 2.090455221462156e-05,
"log_odds_chosen": 3.126645803451538,
"log_odds_ratio": -0.4194856286048889,
"logps/chosen": -0.976054847240448,
"logps/rejected": -3.822575807571411,
"loss": 30.5635,
"nll_loss": 1.2749359607696533,
"po_loss": -0.319825679063797,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.488027423620224,
"rewards/margins": 1.4232604503631592,
"rewards/rejected": -1.9112879037857056,
"step": 210
},
{
"epoch": 1.8277333333333332,
"grad_norm": 191.34779357910156,
"learning_rate": 1.9680330745110954e-05,
"log_odds_chosen": 4.950188636779785,
"log_odds_ratio": -0.3956086039543152,
"logps/chosen": -0.9748937487602234,
"logps/rejected": -5.643929481506348,
"loss": 23.1695,
"nll_loss": 1.2857718467712402,
"po_loss": -0.5617233514785767,
"rewards/accuracies": 0.84375,
"rewards/chosen": -0.4874468743801117,
"rewards/margins": 2.33451771736145,
"rewards/rejected": -2.821964740753174,
"step": 215
},
{
"epoch": 1.8704,
"grad_norm": 327.7476501464844,
"learning_rate": 1.8469334811546542e-05,
"log_odds_chosen": 4.540267467498779,
"log_odds_ratio": -0.39753258228302,
"logps/chosen": -0.9387453198432922,
"logps/rejected": -5.1981964111328125,
"loss": 28.4212,
"nll_loss": 1.252907633781433,
"po_loss": -0.36474356055259705,
"rewards/accuracies": 0.859375,
"rewards/chosen": -0.4693726599216461,
"rewards/margins": 2.129725217819214,
"rewards/rejected": -2.5990982055664062,
"step": 220
},
{
"epoch": 1.9130666666666667,
"grad_norm": 540.2483520507812,
"learning_rate": 1.7274575140626318e-05,
"log_odds_chosen": 5.64566707611084,
"log_odds_ratio": -0.4446595311164856,
"logps/chosen": -1.22698974609375,
"logps/rejected": -6.585175514221191,
"loss": 34.7211,
"nll_loss": 1.3462177515029907,
"po_loss": -0.26118165254592896,
"rewards/accuracies": 0.8656250238418579,
"rewards/chosen": -0.613494873046875,
"rewards/margins": 2.6790928840637207,
"rewards/rejected": -3.2925877571105957,
"step": 225
},
{
"epoch": 1.9557333333333333,
"grad_norm": 401.98248291015625,
"learning_rate": 1.609902209314108e-05,
"log_odds_chosen": 4.817292213439941,
"log_odds_ratio": -0.42455047369003296,
"logps/chosen": -1.0640500783920288,
"logps/rejected": -5.626768589019775,
"loss": 31.062,
"nll_loss": 1.342284917831421,
"po_loss": -0.37159785628318787,
"rewards/accuracies": 0.8187500238418579,
"rewards/chosen": -0.5320250391960144,
"rewards/margins": 2.2813591957092285,
"rewards/rejected": -2.8133842945098877,
"step": 230
},
{
"epoch": 1.9984,
"grad_norm": 91.74356079101562,
"learning_rate": 1.4945598279189565e-05,
"log_odds_chosen": 5.049031734466553,
"log_odds_ratio": -0.406465619802475,
"logps/chosen": -0.9556134939193726,
"logps/rejected": -5.697612762451172,
"loss": 25.4086,
"nll_loss": 1.2838685512542725,
"po_loss": -0.48984843492507935,
"rewards/accuracies": 0.840624988079071,
"rewards/chosen": -0.4778067469596863,
"rewards/margins": 2.370999813079834,
"rewards/rejected": -2.848806381225586,
"step": 235
},
{
"epoch": 2.034133333333333,
"grad_norm": 84.14781188964844,
"learning_rate": 1.3817171292109183e-05,
"log_odds_chosen": 4.798027992248535,
"log_odds_ratio": -0.3244837820529938,
"logps/chosen": -0.8924720287322998,
"logps/rejected": -5.292112827301025,
"loss": 14.7486,
"nll_loss": 1.1619552373886108,
"po_loss": -0.6116347908973694,
"rewards/accuracies": 0.9365671873092651,
"rewards/chosen": -0.4462360143661499,
"rewards/margins": 2.1998205184936523,
"rewards/rejected": -2.6460564136505127,
"step": 240
},
{
"epoch": 2.0768,
"grad_norm": 265.51214599609375,
"learning_rate": 1.271654657918722e-05,
"log_odds_chosen": 3.9462997913360596,
"log_odds_ratio": -0.29830431938171387,
"logps/chosen": -0.8560145497322083,
"logps/rejected": -4.366959571838379,
"loss": 21.2586,
"nll_loss": 1.1650712490081787,
"po_loss": -0.5007405281066895,
"rewards/accuracies": 0.9437500238418579,
"rewards/chosen": -0.4280072748661041,
"rewards/margins": 1.7554724216461182,
"rewards/rejected": -2.1834797859191895,
"step": 245
},
{
"epoch": 2.119466666666667,
"grad_norm": 189.54098510742188,
"learning_rate": 1.1646460466876783e-05,
"log_odds_chosen": 5.587141036987305,
"log_odds_ratio": -0.23089304566383362,
"logps/chosen": -0.941892147064209,
"logps/rejected": -6.023226737976074,
"loss": 15.3802,
"nll_loss": 1.1797395944595337,
"po_loss": -0.6991080641746521,
"rewards/accuracies": 0.9437500238418579,
"rewards/chosen": -0.4709460735321045,
"rewards/margins": 2.5406672954559326,
"rewards/rejected": -3.011613368988037,
"step": 250
},
{
"epoch": 2.1621333333333332,
"grad_norm": 198.67486572265625,
"learning_rate": 1.0609573357858166e-05,
"log_odds_chosen": 6.2472615242004395,
"log_odds_ratio": -0.2708757519721985,
"logps/chosen": -1.0158169269561768,
"logps/rejected": -6.834556579589844,
"loss": 14.1929,
"nll_loss": 1.237670660018921,
"po_loss": -0.7941441535949707,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -0.5079084634780884,
"rewards/margins": 2.909369945526123,
"rewards/rejected": -3.417278289794922,
"step": 255
},
{
"epoch": 2.2048,
"grad_norm": 758.944580078125,
"learning_rate": 9.608463116858542e-06,
"log_odds_chosen": 6.66796350479126,
"log_odds_ratio": -0.23688967525959015,
"logps/chosen": -0.9843616485595703,
"logps/rejected": -7.208956718444824,
"loss": 11.9915,
"nll_loss": 1.2095975875854492,
"po_loss": -0.8348617553710938,
"rewards/accuracies": 0.9593750238418579,
"rewards/chosen": -0.49218082427978516,
"rewards/margins": 3.112297534942627,
"rewards/rejected": -3.604478359222412,
"step": 260
},
{
"epoch": 2.2474666666666665,
"grad_norm": 504.9031982421875,
"learning_rate": 8.645618661674142e-06,
"log_odds_chosen": 6.688530921936035,
"log_odds_ratio": -0.2643258273601532,
"logps/chosen": -1.105387806892395,
"logps/rejected": -7.340773582458496,
"loss": 11.084,
"nll_loss": 1.2117825746536255,
"po_loss": -0.8654062151908875,
"rewards/accuracies": 0.953125,
"rewards/chosen": -0.5526939034461975,
"rewards/margins": 3.1176929473876953,
"rewards/rejected": -3.670386791229248,
"step": 265
},
{
"epoch": 2.2901333333333334,
"grad_norm": 250.27699279785156,
"learning_rate": 7.723433775328384e-06,
"log_odds_chosen": 6.858447074890137,
"log_odds_ratio": -0.31602945923805237,
"logps/chosen": -1.1505261659622192,
"logps/rejected": -7.580657005310059,
"loss": 20.3187,
"nll_loss": 1.2361539602279663,
"po_loss": -0.6011937260627747,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -0.5752630829811096,
"rewards/margins": 3.2150657176971436,
"rewards/rejected": -3.7903285026550293,
"step": 270
},
{
"epoch": 2.3327999999999998,
"grad_norm": 3240.22802734375,
"learning_rate": 6.844201154750177e-06,
"log_odds_chosen": 6.9127960205078125,
"log_odds_ratio": -0.28268226981163025,
"logps/chosen": -1.3622684478759766,
"logps/rejected": -7.855978965759277,
"loss": 22.8393,
"nll_loss": 1.4035627841949463,
"po_loss": -0.6898348927497864,
"rewards/accuracies": 0.9437500238418579,
"rewards/chosen": -0.6811342239379883,
"rewards/margins": 3.2468554973602295,
"rewards/rejected": -3.9279894828796387,
"step": 275
},
{
"epoch": 2.3754666666666666,
"grad_norm": 650.8209228515625,
"learning_rate": 6.010106710768052e-06,
"log_odds_chosen": 6.493264675140381,
"log_odds_ratio": -0.34161344170570374,
"logps/chosen": -1.386515498161316,
"logps/rejected": -7.452689170837402,
"loss": 19.5164,
"nll_loss": 1.3803019523620605,
"po_loss": -0.7704136967658997,
"rewards/accuracies": 0.90625,
"rewards/chosen": -0.693257749080658,
"rewards/margins": 3.0330867767333984,
"rewards/rejected": -3.726344585418701,
"step": 280
},
{
"epoch": 2.4181333333333335,
"grad_norm": 507.415283203125,
"learning_rate": 5.223224133591476e-06,
"log_odds_chosen": 6.825043678283691,
"log_odds_ratio": -0.2423882931470871,
"logps/chosen": -1.1171207427978516,
"logps/rejected": -7.483119964599609,
"loss": 19.53,
"nll_loss": 1.2847764492034912,
"po_loss": -0.6744629144668579,
"rewards/accuracies": 0.9468749761581421,
"rewards/chosen": -0.5585603713989258,
"rewards/margins": 3.1829993724823,
"rewards/rejected": -3.7415599822998047,
"step": 285
},
{
"epoch": 2.4608,
"grad_norm": 789.3963623046875,
"learning_rate": 4.4855097372902135e-06,
"log_odds_chosen": 7.850038051605225,
"log_odds_ratio": -0.26526254415512085,
"logps/chosen": -1.4054704904556274,
"logps/rejected": -8.81872844696045,
"loss": 18.6059,
"nll_loss": 1.3957431316375732,
"po_loss": -0.8143098950386047,
"rewards/accuracies": 0.9312499761581421,
"rewards/chosen": -0.7027352452278137,
"rewards/margins": 3.7066292762756348,
"rewards/rejected": -4.409364223480225,
"step": 290
},
{
"epoch": 2.5034666666666667,
"grad_norm": 1187.58837890625,
"learning_rate": 3.798797596089351e-06,
"log_odds_chosen": 8.998720169067383,
"log_odds_ratio": -0.218130424618721,
"logps/chosen": -1.247434377670288,
"logps/rejected": -9.765419960021973,
"loss": 10.3413,
"nll_loss": 1.355464220046997,
"po_loss": -1.032299280166626,
"rewards/accuracies": 0.9593750238418579,
"rewards/chosen": -0.623717188835144,
"rewards/margins": 4.2589921951293945,
"rewards/rejected": -4.882709980010986,
"step": 295
},
{
"epoch": 2.5461333333333336,
"grad_norm": 593.267578125,
"learning_rate": 3.164794984571759e-06,
"log_odds_chosen": 6.977963924407959,
"log_odds_ratio": -0.3458719849586487,
"logps/chosen": -1.6929105520248413,
"logps/rejected": -8.215627670288086,
"loss": 27.4312,
"nll_loss": 1.5623621940612793,
"po_loss": -0.7051381468772888,
"rewards/accuracies": 0.921875,
"rewards/chosen": -0.8464552760124207,
"rewards/margins": 3.2613587379455566,
"rewards/rejected": -4.107813835144043,
"step": 300
},
{
"epoch": 2.5888,
"grad_norm": 938.1863403320312,
"learning_rate": 2.58507813312448e-06,
"log_odds_chosen": 6.914087772369385,
"log_odds_ratio": -0.28258031606674194,
"logps/chosen": -1.2223470211029053,
"logps/rejected": -7.694061279296875,
"loss": 21.9591,
"nll_loss": 1.3453009128570557,
"po_loss": -0.6590788960456848,
"rewards/accuracies": 0.909375011920929,
"rewards/chosen": -0.6111735105514526,
"rewards/margins": 3.2358570098876953,
"rewards/rejected": -3.8470306396484375,
"step": 305
},
{
"epoch": 2.6314666666666664,
"grad_norm": 221.02281188964844,
"learning_rate": 2.0610883091816525e-06,
"log_odds_chosen": 5.710585594177246,
"log_odds_ratio": -0.2850475609302521,
"logps/chosen": -1.021317720413208,
"logps/rejected": -6.243259906768799,
"loss": 16.8653,
"nll_loss": 1.247992753982544,
"po_loss": -0.7209508419036865,
"rewards/accuracies": 0.9281250238418579,
"rewards/chosen": -0.510658860206604,
"rewards/margins": 2.610970973968506,
"rewards/rejected": -3.1216299533843994,
"step": 310
},
{
"epoch": 2.6741333333333333,
"grad_norm": 475.2201843261719,
"learning_rate": 1.59412823400657e-06,
"log_odds_chosen": 5.025318145751953,
"log_odds_ratio": -0.25998514890670776,
"logps/chosen": -1.0306951999664307,
"logps/rejected": -5.601127624511719,
"loss": 14.5714,
"nll_loss": 1.1835569143295288,
"po_loss": -0.7282010316848755,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -0.5153475999832153,
"rewards/margins": 2.2852160930633545,
"rewards/rejected": -2.8005638122558594,
"step": 315
},
{
"epoch": 2.7168,
"grad_norm": 337.47796630859375,
"learning_rate": 1.1853588439213442e-06,
"log_odds_chosen": 6.4345574378967285,
"log_odds_ratio": -0.23376791179180145,
"logps/chosen": -1.014891266822815,
"logps/rejected": -7.0141282081604,
"loss": 16.8373,
"nll_loss": 1.2330230474472046,
"po_loss": -0.7068589329719543,
"rewards/accuracies": 0.9437500238418579,
"rewards/chosen": -0.5074456334114075,
"rewards/margins": 2.9996187686920166,
"rewards/rejected": -3.5070641040802,
"step": 320
},
{
"epoch": 2.7594666666666665,
"grad_norm": 251.52114868164062,
"learning_rate": 8.357964040363209e-07,
"log_odds_chosen": 7.006105899810791,
"log_odds_ratio": -0.3340076804161072,
"logps/chosen": -1.0801838636398315,
"logps/rejected": -7.584368705749512,
"loss": 7.9524,
"nll_loss": 1.2427351474761963,
"po_loss": -0.9942227602005005,
"rewards/accuracies": 0.9281250238418579,
"rewards/chosen": -0.5400919318199158,
"rewards/margins": 3.2520930767059326,
"rewards/rejected": -3.792184352874756,
"step": 325
},
{
"epoch": 2.8021333333333334,
"grad_norm": 500.19580078125,
"learning_rate": 5.463099816548579e-07,
"log_odds_chosen": 7.333872318267822,
"log_odds_ratio": -0.251155823469162,
"logps/chosen": -1.1577497720718384,
"logps/rejected": -8.027981758117676,
"loss": 19.0267,
"nll_loss": 1.2874268293380737,
"po_loss": -0.6928423047065735,
"rewards/accuracies": 0.9312499761581421,
"rewards/chosen": -0.5788748860359192,
"rewards/margins": 3.4351158142089844,
"rewards/rejected": -4.013990879058838,
"step": 330
},
{
"epoch": 2.8448,
"grad_norm": 827.9666748046875,
"learning_rate": 3.1761928563510955e-07,
"log_odds_chosen": 5.787890434265137,
"log_odds_ratio": -0.2869322896003723,
"logps/chosen": -1.0971204042434692,
"logps/rejected": -6.4105658531188965,
"loss": 12.9881,
"nll_loss": 1.2217962741851807,
"po_loss": -0.8159177899360657,
"rewards/accuracies": 0.9125000238418579,
"rewards/chosen": -0.5485602021217346,
"rewards/margins": 2.6567225456237793,
"rewards/rejected": -3.2052829265594482,
"step": 335
},
{
"epoch": 2.8874666666666666,
"grad_norm": 292.38897705078125,
"learning_rate": 1.5029287708036854e-07,
"log_odds_chosen": 6.05283260345459,
"log_odds_ratio": -0.24598774313926697,
"logps/chosen": -1.0574235916137695,
"logps/rejected": -6.646490573883057,
"loss": 7.9611,
"nll_loss": 1.1931092739105225,
"po_loss": -0.9443262219429016,
"rewards/accuracies": 0.956250011920929,
"rewards/chosen": -0.5287117958068848,
"rewards/margins": 2.7945332527160645,
"rewards/rejected": -3.3232452869415283,
"step": 340
},
{
"epoch": 2.9301333333333335,
"grad_norm": 387.1372985839844,
"learning_rate": 4.474675580662113e-08,
"log_odds_chosen": 6.334747791290283,
"log_odds_ratio": -0.32026782631874084,
"logps/chosen": -1.1003960371017456,
"logps/rejected": -6.976574897766113,
"loss": 12.2009,
"nll_loss": 1.235642671585083,
"po_loss": -0.8543656468391418,
"rewards/accuracies": 0.9156249761581421,
"rewards/chosen": -0.5501980185508728,
"rewards/margins": 2.938089370727539,
"rewards/rejected": -3.4882874488830566,
"step": 345
},
{
"epoch": 2.9728,
"grad_norm": 1250.0882568359375,
"learning_rate": 1.2433261014244136e-09,
"log_odds_chosen": 7.907639980316162,
"log_odds_ratio": -0.22040143609046936,
"logps/chosen": -1.2740892171859741,
"logps/rejected": -8.710322380065918,
"loss": 9.5303,
"nll_loss": 1.3264214992523193,
"po_loss": -1.028598666191101,
"rewards/accuracies": 0.9281250238418579,
"rewards/chosen": -0.6370446085929871,
"rewards/margins": 3.7181167602539062,
"rewards/rejected": -4.355161190032959,
"step": 350
},
{
"epoch": 2.981333333333333,
"step": 351,
"total_flos": 0.0,
"train_loss": 54.80183921716152,
"train_runtime": 10133.5503,
"train_samples_per_second": 2.22,
"train_steps_per_second": 0.035
}
],
"logging_steps": 5,
"max_steps": 351,
"num_input_tokens_seen": 0,
"num_train_epochs": 3,
"save_steps": 100000,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 1,
"trial_name": null,
"trial_params": null
}