falcon-7b-dpo-lora / trainer_state.json
Mastane's picture
Model save
3bae7d1
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 2.9992254066615027,
"eval_steps": 100,
"global_step": 363,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.01,
"learning_rate": 1.3513513513513514e-08,
"logits/chosen": -15.172547340393066,
"logits/rejected": -15.251230239868164,
"logps/chosen": -103.97029113769531,
"logps/rejected": -79.94886016845703,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.08,
"learning_rate": 1.3513513513513515e-07,
"logits/chosen": -15.22812557220459,
"logits/rejected": -15.30654525756836,
"logps/chosen": -123.74452209472656,
"logps/rejected": -93.8215560913086,
"loss": 0.6931,
"rewards/accuracies": 0.2309027761220932,
"rewards/chosen": 0.0003177134203724563,
"rewards/margins": 0.00015111881657503545,
"rewards/rejected": 0.000166594676556997,
"step": 10
},
{
"epoch": 0.17,
"learning_rate": 2.702702702702703e-07,
"logits/chosen": -15.262018203735352,
"logits/rejected": -15.291465759277344,
"logps/chosen": -123.9498291015625,
"logps/rejected": -96.19405364990234,
"loss": 0.6931,
"rewards/accuracies": 0.29374998807907104,
"rewards/chosen": 0.0009166754898615181,
"rewards/margins": 0.0013821485918015242,
"rewards/rejected": -0.00046547286910936236,
"step": 20
},
{
"epoch": 0.25,
"learning_rate": 4.054054054054054e-07,
"logits/chosen": -15.272542953491211,
"logits/rejected": -15.305673599243164,
"logps/chosen": -127.54842376708984,
"logps/rejected": -105.3931884765625,
"loss": 0.6933,
"rewards/accuracies": 0.2796874940395355,
"rewards/chosen": 0.00042984067113138735,
"rewards/margins": -0.00048730484559200704,
"rewards/rejected": 0.0009171456913463771,
"step": 30
},
{
"epoch": 0.33,
"learning_rate": 4.95398773006135e-07,
"logits/chosen": -15.2285737991333,
"logits/rejected": -15.296609878540039,
"logps/chosen": -124.9699935913086,
"logps/rejected": -97.49568176269531,
"loss": 0.6931,
"rewards/accuracies": 0.26249998807907104,
"rewards/chosen": 0.0005645907949656248,
"rewards/margins": 0.00030518617131747305,
"rewards/rejected": 0.00025940462364815176,
"step": 40
},
{
"epoch": 0.41,
"learning_rate": 4.800613496932515e-07,
"logits/chosen": -15.285560607910156,
"logits/rejected": -15.345190048217773,
"logps/chosen": -129.31256103515625,
"logps/rejected": -99.52166748046875,
"loss": 0.6931,
"rewards/accuracies": 0.2796874940395355,
"rewards/chosen": 0.00027664678054861724,
"rewards/margins": 5.752297511207871e-05,
"rewards/rejected": 0.00021912386000622064,
"step": 50
},
{
"epoch": 0.5,
"learning_rate": 4.647239263803681e-07,
"logits/chosen": -15.253962516784668,
"logits/rejected": -15.33386516571045,
"logps/chosen": -119.763427734375,
"logps/rejected": -102.88087463378906,
"loss": 0.6932,
"rewards/accuracies": 0.2796874940395355,
"rewards/chosen": 0.000503324088640511,
"rewards/margins": 8.147591142915189e-05,
"rewards/rejected": 0.0004218483227305114,
"step": 60
},
{
"epoch": 0.58,
"learning_rate": 4.4938650306748465e-07,
"logits/chosen": -15.288751602172852,
"logits/rejected": -15.326438903808594,
"logps/chosen": -127.03987121582031,
"logps/rejected": -96.76243591308594,
"loss": 0.6932,
"rewards/accuracies": 0.2890625,
"rewards/chosen": 0.0017284611240029335,
"rewards/margins": 0.0012586858356371522,
"rewards/rejected": 0.0004697750264313072,
"step": 70
},
{
"epoch": 0.66,
"learning_rate": 4.340490797546012e-07,
"logits/chosen": -15.288309097290039,
"logits/rejected": -15.354669570922852,
"logps/chosen": -129.9634552001953,
"logps/rejected": -101.72119140625,
"loss": 0.693,
"rewards/accuracies": 0.28125,
"rewards/chosen": 0.0008187053608708084,
"rewards/margins": 0.0004409000393934548,
"rewards/rejected": 0.0003778053796850145,
"step": 80
},
{
"epoch": 0.74,
"learning_rate": 4.187116564417178e-07,
"logits/chosen": -15.274989128112793,
"logits/rejected": -15.232473373413086,
"logps/chosen": -121.2137451171875,
"logps/rejected": -113.5948486328125,
"loss": 0.6931,
"rewards/accuracies": 0.296875,
"rewards/chosen": 0.001987897325307131,
"rewards/margins": 0.0013456062879413366,
"rewards/rejected": 0.0006422909209504724,
"step": 90
},
{
"epoch": 0.83,
"learning_rate": 4.0337423312883434e-07,
"logits/chosen": -15.281481742858887,
"logits/rejected": -15.288961410522461,
"logps/chosen": -113.20597839355469,
"logps/rejected": -97.28117370605469,
"loss": 0.693,
"rewards/accuracies": 0.2874999940395355,
"rewards/chosen": 0.0011146818287670612,
"rewards/margins": 0.00026254478143528104,
"rewards/rejected": 0.000852137163747102,
"step": 100
},
{
"epoch": 0.91,
"learning_rate": 3.880368098159509e-07,
"logits/chosen": -15.232090950012207,
"logits/rejected": -15.308767318725586,
"logps/chosen": -131.0157012939453,
"logps/rejected": -104.84590911865234,
"loss": 0.6929,
"rewards/accuracies": 0.28125,
"rewards/chosen": 0.0027348899748176336,
"rewards/margins": 0.0014507302548736334,
"rewards/rejected": 0.0012841597199440002,
"step": 110
},
{
"epoch": 0.99,
"learning_rate": 3.7269938650306747e-07,
"logits/chosen": -15.231793403625488,
"logits/rejected": -15.243997573852539,
"logps/chosen": -120.3092269897461,
"logps/rejected": -101.40711975097656,
"loss": 0.6927,
"rewards/accuracies": 0.26875001192092896,
"rewards/chosen": 0.0026273976545780897,
"rewards/margins": 0.0005600537406280637,
"rewards/rejected": 0.002067344030365348,
"step": 120
},
{
"epoch": 1.0,
"eval_logits/chosen": -14.84730052947998,
"eval_logits/rejected": -14.93464183807373,
"eval_logps/chosen": -110.10073852539062,
"eval_logps/rejected": -88.51683044433594,
"eval_loss": 0.6929101943969727,
"eval_rewards/accuracies": 0.2857142984867096,
"eval_rewards/chosen": 0.0035249004140496254,
"eval_rewards/margins": 0.0020474623888731003,
"eval_rewards/rejected": 0.001477438141591847,
"eval_runtime": 230.1482,
"eval_samples_per_second": 8.69,
"eval_steps_per_second": 0.274,
"step": 121
},
{
"epoch": 1.07,
"learning_rate": 3.5736196319018404e-07,
"logits/chosen": -15.243385314941406,
"logits/rejected": -15.28660774230957,
"logps/chosen": -117.83067321777344,
"logps/rejected": -105.49190521240234,
"loss": 0.6927,
"rewards/accuracies": 0.3031249940395355,
"rewards/chosen": 0.0028103243093937635,
"rewards/margins": 0.0009912283858284354,
"rewards/rejected": 0.0018190961563959718,
"step": 130
},
{
"epoch": 1.16,
"learning_rate": 3.420245398773006e-07,
"logits/chosen": -15.298578262329102,
"logits/rejected": -15.308378219604492,
"logps/chosen": -117.3888168334961,
"logps/rejected": -95.77253723144531,
"loss": 0.6928,
"rewards/accuracies": 0.28125,
"rewards/chosen": 0.003832989139482379,
"rewards/margins": 0.0002880497486330569,
"rewards/rejected": 0.0035449396818876266,
"step": 140
},
{
"epoch": 1.24,
"learning_rate": 3.2668711656441716e-07,
"logits/chosen": -15.269906997680664,
"logits/rejected": -15.330175399780273,
"logps/chosen": -121.6659927368164,
"logps/rejected": -98.5518569946289,
"loss": 0.6927,
"rewards/accuracies": 0.28593748807907104,
"rewards/chosen": 0.004216608125716448,
"rewards/margins": 0.0014314096188172698,
"rewards/rejected": 0.0027851988561451435,
"step": 150
},
{
"epoch": 1.32,
"learning_rate": 3.1134969325153373e-07,
"logits/chosen": -15.28375244140625,
"logits/rejected": -15.307037353515625,
"logps/chosen": -117.98995208740234,
"logps/rejected": -101.3696517944336,
"loss": 0.6925,
"rewards/accuracies": 0.2874999940395355,
"rewards/chosen": 0.004791011102497578,
"rewards/margins": 0.0008280017646029592,
"rewards/rejected": 0.00396300945430994,
"step": 160
},
{
"epoch": 1.4,
"learning_rate": 2.960122699386503e-07,
"logits/chosen": -15.264017105102539,
"logits/rejected": -15.2572603225708,
"logps/chosen": -125.29273986816406,
"logps/rejected": -112.38765716552734,
"loss": 0.6923,
"rewards/accuracies": 0.3499999940395355,
"rewards/chosen": 0.006073893513530493,
"rewards/margins": 0.0020425966940820217,
"rewards/rejected": 0.004031297750771046,
"step": 170
},
{
"epoch": 1.49,
"learning_rate": 2.8067484662576686e-07,
"logits/chosen": -15.275349617004395,
"logits/rejected": -15.314315795898438,
"logps/chosen": -120.6910400390625,
"logps/rejected": -92.7549057006836,
"loss": 0.6921,
"rewards/accuracies": 0.3343749940395355,
"rewards/chosen": 0.006351888179779053,
"rewards/margins": 0.002958633005619049,
"rewards/rejected": 0.0033932551741600037,
"step": 180
},
{
"epoch": 1.57,
"learning_rate": 2.653374233128834e-07,
"logits/chosen": -15.27843189239502,
"logits/rejected": -15.3081693649292,
"logps/chosen": -115.19669342041016,
"logps/rejected": -92.90242004394531,
"loss": 0.6921,
"rewards/accuracies": 0.3218750059604645,
"rewards/chosen": 0.0064470237120985985,
"rewards/margins": 0.0023960156831890345,
"rewards/rejected": 0.004051008261740208,
"step": 190
},
{
"epoch": 1.65,
"learning_rate": 2.5e-07,
"logits/chosen": -15.239352226257324,
"logits/rejected": -15.272363662719727,
"logps/chosen": -121.6275634765625,
"logps/rejected": -93.08324432373047,
"loss": 0.6919,
"rewards/accuracies": 0.2734375,
"rewards/chosen": 0.0069998181425035,
"rewards/margins": 0.0022552732843905687,
"rewards/rejected": 0.004744545556604862,
"step": 200
},
{
"epoch": 1.74,
"learning_rate": 2.3466257668711655e-07,
"logits/chosen": -15.199145317077637,
"logits/rejected": -15.267333984375,
"logps/chosen": -124.52889251708984,
"logps/rejected": -97.74082946777344,
"loss": 0.6921,
"rewards/accuracies": 0.2593750059604645,
"rewards/chosen": 0.0072901747189462185,
"rewards/margins": 0.0013283558655530214,
"rewards/rejected": 0.005961819086223841,
"step": 210
},
{
"epoch": 1.82,
"learning_rate": 2.1932515337423312e-07,
"logits/chosen": -15.27015495300293,
"logits/rejected": -15.279447555541992,
"logps/chosen": -118.91484069824219,
"logps/rejected": -90.32121276855469,
"loss": 0.6918,
"rewards/accuracies": 0.30156248807907104,
"rewards/chosen": 0.00907520018517971,
"rewards/margins": 0.0032898869831115007,
"rewards/rejected": 0.005785313434898853,
"step": 220
},
{
"epoch": 1.9,
"learning_rate": 2.0398773006134968e-07,
"logits/chosen": -15.252248764038086,
"logits/rejected": -15.287099838256836,
"logps/chosen": -117.89067077636719,
"logps/rejected": -100.09212493896484,
"loss": 0.6918,
"rewards/accuracies": 0.3187499940395355,
"rewards/chosen": 0.00985044240951538,
"rewards/margins": 0.004119374789297581,
"rewards/rejected": 0.0057310680858790874,
"step": 230
},
{
"epoch": 1.98,
"learning_rate": 1.8865030674846625e-07,
"logits/chosen": -15.214279174804688,
"logits/rejected": -15.315710067749023,
"logps/chosen": -121.98006439208984,
"logps/rejected": -98.08609008789062,
"loss": 0.6915,
"rewards/accuracies": 0.3125,
"rewards/chosen": 0.010013025254011154,
"rewards/margins": 0.0034062224440276623,
"rewards/rejected": 0.00660680141299963,
"step": 240
},
{
"epoch": 2.0,
"eval_logits/chosen": -14.846867561340332,
"eval_logits/rejected": -14.935086250305176,
"eval_logps/chosen": -110.01992797851562,
"eval_logps/rejected": -88.46048736572266,
"eval_loss": 0.6916756629943848,
"eval_rewards/accuracies": 0.3055555522441864,
"eval_rewards/chosen": 0.011606148444116116,
"eval_rewards/margins": 0.00449267216026783,
"eval_rewards/rejected": 0.007113476749509573,
"eval_runtime": 229.7499,
"eval_samples_per_second": 8.705,
"eval_steps_per_second": 0.274,
"step": 242
},
{
"epoch": 2.07,
"learning_rate": 1.733128834355828e-07,
"logits/chosen": -15.271708488464355,
"logits/rejected": -15.325927734375,
"logps/chosen": -120.98309326171875,
"logps/rejected": -108.2812728881836,
"loss": 0.6916,
"rewards/accuracies": 0.296875,
"rewards/chosen": 0.009962974116206169,
"rewards/margins": 0.0022991253063082695,
"rewards/rejected": 0.007663849741220474,
"step": 250
},
{
"epoch": 2.15,
"learning_rate": 1.5797546012269938e-07,
"logits/chosen": -15.266347885131836,
"logits/rejected": -15.258427619934082,
"logps/chosen": -123.05375671386719,
"logps/rejected": -98.55216979980469,
"loss": 0.6914,
"rewards/accuracies": 0.2984375059604645,
"rewards/chosen": 0.011418156325817108,
"rewards/margins": 0.0035811502020806074,
"rewards/rejected": 0.007837006822228432,
"step": 260
},
{
"epoch": 2.23,
"learning_rate": 1.4263803680981594e-07,
"logits/chosen": -15.247785568237305,
"logits/rejected": -15.258626937866211,
"logps/chosen": -115.1404037475586,
"logps/rejected": -93.46723175048828,
"loss": 0.6912,
"rewards/accuracies": 0.3375000059604645,
"rewards/chosen": 0.011196712031960487,
"rewards/margins": 0.004130281042307615,
"rewards/rejected": 0.007066431455314159,
"step": 270
},
{
"epoch": 2.31,
"learning_rate": 1.273006134969325e-07,
"logits/chosen": -15.278572082519531,
"logits/rejected": -15.311563491821289,
"logps/chosen": -128.5188446044922,
"logps/rejected": -100.33396911621094,
"loss": 0.6911,
"rewards/accuracies": 0.3046875,
"rewards/chosen": 0.012510362081229687,
"rewards/margins": 0.004159414209425449,
"rewards/rejected": 0.008350946940481663,
"step": 280
},
{
"epoch": 2.4,
"learning_rate": 1.1196319018404908e-07,
"logits/chosen": -15.250155448913574,
"logits/rejected": -15.272921562194824,
"logps/chosen": -119.62606048583984,
"logps/rejected": -103.21083068847656,
"loss": 0.6911,
"rewards/accuracies": 0.296875,
"rewards/chosen": 0.013957071118056774,
"rewards/margins": 0.004067812114953995,
"rewards/rejected": 0.00988925900310278,
"step": 290
},
{
"epoch": 2.48,
"learning_rate": 9.662576687116564e-08,
"logits/chosen": -15.27168083190918,
"logits/rejected": -15.364572525024414,
"logps/chosen": -119.32635498046875,
"logps/rejected": -98.04741668701172,
"loss": 0.691,
"rewards/accuracies": 0.29218751192092896,
"rewards/chosen": 0.012033398263156414,
"rewards/margins": 0.0029196850955486298,
"rewards/rejected": 0.009113713167607784,
"step": 300
},
{
"epoch": 2.56,
"learning_rate": 8.12883435582822e-08,
"logits/chosen": -15.267038345336914,
"logits/rejected": -15.33867359161377,
"logps/chosen": -125.45201110839844,
"logps/rejected": -98.39042663574219,
"loss": 0.6911,
"rewards/accuracies": 0.32499998807907104,
"rewards/chosen": 0.013260738924145699,
"rewards/margins": 0.004113315604627132,
"rewards/rejected": 0.00914742425084114,
"step": 310
},
{
"epoch": 2.64,
"learning_rate": 6.595092024539877e-08,
"logits/chosen": -15.279388427734375,
"logits/rejected": -15.33612060546875,
"logps/chosen": -123.69743347167969,
"logps/rejected": -101.3545150756836,
"loss": 0.6911,
"rewards/accuracies": 0.32343751192092896,
"rewards/chosen": 0.013454603031277657,
"rewards/margins": 0.00448086392134428,
"rewards/rejected": 0.008973738178610802,
"step": 320
},
{
"epoch": 2.73,
"learning_rate": 5.061349693251534e-08,
"logits/chosen": -15.248504638671875,
"logits/rejected": -15.330392837524414,
"logps/chosen": -123.80655670166016,
"logps/rejected": -98.1585922241211,
"loss": 0.691,
"rewards/accuracies": 0.3187499940395355,
"rewards/chosen": 0.014067019335925579,
"rewards/margins": 0.005277669057250023,
"rewards/rejected": 0.008789352141320705,
"step": 330
},
{
"epoch": 2.81,
"learning_rate": 3.5276073619631896e-08,
"logits/chosen": -15.255121231079102,
"logits/rejected": -15.33825397491455,
"logps/chosen": -123.04536437988281,
"logps/rejected": -98.64664459228516,
"loss": 0.6908,
"rewards/accuracies": 0.34375,
"rewards/chosen": 0.014080156572163105,
"rewards/margins": 0.005763668101280928,
"rewards/rejected": 0.008316488936543465,
"step": 340
},
{
"epoch": 2.89,
"learning_rate": 1.9938650306748464e-08,
"logits/chosen": -15.204339981079102,
"logits/rejected": -15.2611722946167,
"logps/chosen": -127.34835052490234,
"logps/rejected": -111.3030014038086,
"loss": 0.6911,
"rewards/accuracies": 0.33281248807907104,
"rewards/chosen": 0.015759991481900215,
"rewards/margins": 0.0036518643610179424,
"rewards/rejected": 0.012108126655220985,
"step": 350
},
{
"epoch": 2.97,
"learning_rate": 4.601226993865031e-09,
"logits/chosen": -15.213580131530762,
"logits/rejected": -15.218714714050293,
"logps/chosen": -105.95635986328125,
"logps/rejected": -87.81764221191406,
"loss": 0.6913,
"rewards/accuracies": 0.2828125059604645,
"rewards/chosen": 0.012317867949604988,
"rewards/margins": 0.002315809251740575,
"rewards/rejected": 0.010002057068049908,
"step": 360
},
{
"epoch": 3.0,
"eval_logits/chosen": -14.8499116897583,
"eval_logits/rejected": -14.938364028930664,
"eval_logps/chosen": -109.97705078125,
"eval_logps/rejected": -88.43526458740234,
"eval_loss": 0.6909456849098206,
"eval_rewards/accuracies": 0.3174603283405304,
"eval_rewards/chosen": 0.015893306583166122,
"eval_rewards/margins": 0.006259124726057053,
"eval_rewards/rejected": 0.00963417999446392,
"eval_runtime": 227.7514,
"eval_samples_per_second": 8.782,
"eval_steps_per_second": 0.277,
"step": 363
},
{
"epoch": 3.0,
"step": 363,
"total_flos": 0.0,
"train_loss": 0.692123620142293,
"train_runtime": 31275.9572,
"train_samples_per_second": 5.944,
"train_steps_per_second": 0.012
}
],
"logging_steps": 10,
"max_steps": 363,
"num_train_epochs": 3,
"save_steps": 500,
"total_flos": 0.0,
"trial_name": null,
"trial_params": null
}