{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-08,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2820512820512818e-07,
      "logits/chosen": -1.8662172555923462,
      "logits/rejected": -1.8705354928970337,
      "logps/chosen": -37.00023651123047,
      "logps/rejected": -33.65288162231445,
      "loss": 0.6868,
      "rewards/accuracies": 0.4861111044883728,
      "rewards/chosen": 0.0031818184070289135,
      "rewards/margins": 0.014585819095373154,
      "rewards/rejected": -0.011404001154005527,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5641025641025636e-07,
      "logits/chosen": -1.997016191482544,
      "logits/rejected": -1.9996668100357056,
      "logps/chosen": -29.636394500732422,
      "logps/rejected": -29.053604125976562,
      "loss": 0.6951,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0028971880674362183,
      "rewards/margins": -0.002024129033088684,
      "rewards/rejected": 0.004921317100524902,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.8461538461538463e-07,
      "logits/chosen": -1.9197543859481812,
      "logits/rejected": -1.9170539379119873,
      "logps/chosen": -31.429040908813477,
      "logps/rejected": -33.19585418701172,
      "loss": 0.7032,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.006471562199294567,
      "rewards/margins": -0.017771439626812935,
      "rewards/rejected": 0.011299876496195793,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438433e-07,
      "logits/chosen": -2.016721248626709,
      "logits/rejected": -2.007981300354004,
      "logps/chosen": -32.594520568847656,
      "logps/rejected": -32.49314498901367,
      "loss": 0.7034,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.00885552167892456,
      "rewards/margins": -0.018619000911712646,
      "rewards/rejected": 0.009763479232788086,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542186e-07,
      "logits/chosen": -1.8646072149276733,
      "logits/rejected": -1.8538135290145874,
      "logps/chosen": -33.57071304321289,
      "logps/rejected": -35.4249267578125,
      "loss": 0.7044,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.006614250130951405,
      "rewards/margins": -0.020322056487202644,
      "rewards/rejected": 0.01370780635625124,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941118e-07,
      "logits/chosen": -1.9454749822616577,
      "logits/rejected": -1.9473978281021118,
      "logps/chosen": -32.56868362426758,
      "logps/rejected": -33.174442291259766,
      "loss": 0.6856,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.015751874074339867,
      "rewards/margins": 0.01728220283985138,
      "rewards/rejected": -0.0015303284162655473,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413548e-07,
      "logits/chosen": -2.079904794692993,
      "logits/rejected": -2.0848889350891113,
      "logps/chosen": -34.005821228027344,
      "logps/rejected": -36.594722747802734,
      "loss": 0.6925,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.008668524213135242,
      "rewards/margins": 0.004255795385688543,
      "rewards/rejected": -0.012924319133162498,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-07,
      "logits/chosen": -1.9423730373382568,
      "logits/rejected": -1.9455206394195557,
      "logps/chosen": -34.40643310546875,
      "logps/rejected": -34.57245635986328,
      "loss": 0.6941,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01068954449146986,
      "rewards/margins": 0.000712524342816323,
      "rewards/rejected": 0.00997702032327652,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.736716601303429e-07,
      "logits/chosen": -1.9506025314331055,
      "logits/rejected": -1.955121636390686,
      "logps/chosen": -32.45918655395508,
      "logps/rejected": -32.362247467041016,
      "loss": 0.6914,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.004118821118026972,
      "rewards/margins": 0.005789563059806824,
      "rewards/rejected": -0.0016707420581951737,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.62624545834521e-07,
      "logits/chosen": -2.0490012168884277,
      "logits/rejected": -2.0470049381256104,
      "logps/chosen": -32.25275802612305,
      "logps/rejected": -31.29152488708496,
      "loss": 0.6909,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.005962538532912731,
      "rewards/margins": 0.006695576012134552,
      "rewards/rejected": -0.012658114545047283,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.2439146041870117,
      "eval_logits/rejected": -2.2390429973602295,
      "eval_logps/chosen": -34.02096939086914,
      "eval_logps/rejected": -37.521278381347656,
      "eval_loss": 0.6897847056388855,
      "eval_rewards/accuracies": 0.5627076625823975,
      "eval_rewards/chosen": 0.006789046805351973,
      "eval_rewards/margins": 0.009119975380599499,
      "eval_rewards/rejected": -0.002330928109586239,
      "eval_runtime": 146.2765,
      "eval_samples_per_second": 2.345,
      "eval_steps_per_second": 0.294,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4982572012636904e-07,
      "logits/chosen": -2.0054855346679688,
      "logits/rejected": -2.0030674934387207,
      "logps/chosen": -33.233333587646484,
      "logps/rejected": -34.01185989379883,
      "loss": 0.6959,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.005242079496383667,
      "rewards/margins": -0.003514623735100031,
      "rewards/rejected": 0.00875670276582241,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777677e-07,
      "logits/chosen": -2.0167009830474854,
      "logits/rejected": -2.008331537246704,
      "logps/chosen": -32.438194274902344,
      "logps/rejected": -32.18104934692383,
      "loss": 0.6877,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0033566683996468782,
      "rewards/margins": 0.013127696700394154,
      "rewards/rejected": -0.009771028533577919,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.194082707715275e-07,
      "logits/chosen": -2.0462117195129395,
      "logits/rejected": -2.0381579399108887,
      "logps/chosen": -30.49118995666504,
      "logps/rejected": -32.039451599121094,
      "loss": 0.7014,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0077363611198961735,
      "rewards/margins": -0.014514627866446972,
      "rewards/rejected": 0.006778267212212086,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.020402418666621e-07,
      "logits/chosen": -1.9772098064422607,
      "logits/rejected": -1.9874696731567383,
      "logps/chosen": -31.401031494140625,
      "logps/rejected": -32.54351043701172,
      "loss": 0.6877,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.009314531460404396,
      "rewards/margins": 0.013629269786179066,
      "rewards/rejected": -0.00431473832577467,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8341962650351185e-07,
      "logits/chosen": -1.8905785083770752,
      "logits/rejected": -1.8916677236557007,
      "logps/chosen": -34.182621002197266,
      "logps/rejected": -34.750953674316406,
      "loss": 0.6937,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.005035269074141979,
      "rewards/margins": 0.0018822342390194535,
      "rewards/rejected": 0.0031530349515378475,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800572e-07,
      "logits/chosen": -1.9425489902496338,
      "logits/rejected": -1.939056634902954,
      "logps/chosen": -36.16254425048828,
      "logps/rejected": -32.711036682128906,
      "loss": 0.6949,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.0045667411759495735,
      "rewards/margins": -0.0012899183202534914,
      "rewards/rejected": 0.005856659263372421,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.430433172111807e-07,
      "logits/chosen": -2.041844129562378,
      "logits/rejected": -2.034442186355591,
      "logps/chosen": -33.78926467895508,
      "logps/rejected": -31.351390838623047,
      "loss": 0.692,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004301992245018482,
      "rewards/margins": 0.004073685500770807,
      "rewards/rejected": 0.00022830665693618357,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.216202642830543e-07,
      "logits/chosen": -2.047542095184326,
      "logits/rejected": -2.0527994632720947,
      "logps/chosen": -32.531822204589844,
      "logps/rejected": -32.49492645263672,
      "loss": 0.6887,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.00648072361946106,
      "rewards/margins": 0.01137510221451521,
      "rewards/rejected": -0.00489437859505415,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.9960716642946403e-07,
      "logits/chosen": -2.048457622528076,
      "logits/rejected": -2.0456814765930176,
      "logps/chosen": -31.491973876953125,
      "logps/rejected": -31.334081649780273,
      "loss": 0.6912,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.00151053664740175,
      "rewards/margins": 0.006422492675483227,
      "rewards/rejected": -0.007933029904961586,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.771853789806683e-07,
      "logits/chosen": -1.9186420440673828,
      "logits/rejected": -1.9233074188232422,
      "logps/chosen": -31.597881317138672,
      "logps/rejected": -32.79570770263672,
      "loss": 0.691,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.004199606366455555,
      "rewards/margins": 0.006588342599570751,
      "rewards/rejected": -0.0023887366987764835,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.243356466293335,
      "eval_logits/rejected": -2.2384772300720215,
      "eval_logps/chosen": -34.025508880615234,
      "eval_logps/rejected": -37.50304412841797,
      "eval_loss": 0.6954899430274963,
      "eval_rewards/accuracies": 0.4779900312423706,
      "eval_rewards/chosen": 0.004520992282778025,
      "eval_rewards/margins": -0.0022661893162876368,
      "eval_rewards/rejected": 0.006787182297557592,
      "eval_runtime": 145.936,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402e-07,
      "logits/chosen": -2.0320427417755127,
      "logits/rejected": -2.0427355766296387,
      "logps/chosen": -31.94189453125,
      "logps/rejected": -33.89322280883789,
      "loss": 0.6796,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.014702295884490013,
      "rewards/margins": 0.029963891953229904,
      "rewards/rejected": -0.015261595137417316,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.318564697655179e-07,
      "logits/chosen": -1.9252369403839111,
      "logits/rejected": -1.9400920867919922,
      "logps/chosen": -30.095117568969727,
      "logps/rejected": -31.542316436767578,
      "loss": 0.6945,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.008213011547923088,
      "rewards/margins": 0.0002824708935804665,
      "rewards/rejected": 0.007930541411042213,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.093227910899832e-07,
      "logits/chosen": -1.9828424453735352,
      "logits/rejected": -1.9868125915527344,
      "logps/chosen": -33.39949417114258,
      "logps/rejected": -31.549747467041016,
      "loss": 0.6899,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.00898725725710392,
      "rewards/margins": 0.009209835901856422,
      "rewards/rejected": -0.00022257864475250244,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279356e-07,
      "logits/chosen": -1.982708215713501,
      "logits/rejected": -1.9607412815093994,
      "logps/chosen": -34.14110565185547,
      "logps/rejected": -34.958885192871094,
      "loss": 0.6892,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006322857923805714,
      "rewards/margins": 0.01036941446363926,
      "rewards/rejected": -0.004046556539833546,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.654436768970182e-07,
      "logits/chosen": -2.0246503353118896,
      "logits/rejected": -2.0213494300842285,
      "logps/chosen": -32.90447998046875,
      "logps/rejected": -36.232059478759766,
      "loss": 0.6874,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010292356833815575,
      "rewards/margins": 0.013860592618584633,
      "rewards/rejected": -0.0035682350862771273,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.444597403062196e-07,
      "logits/chosen": -1.8912708759307861,
      "logits/rejected": -1.8888260126113892,
      "logps/chosen": -34.188419342041016,
      "logps/rejected": -35.51929473876953,
      "loss": 0.6906,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -9.60230863711331e-06,
      "rewards/margins": 0.0073286681436002254,
      "rewards/rejected": -0.007338270545005798,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2434529917578887e-07,
      "logits/chosen": -1.8759148120880127,
      "logits/rejected": -1.8733787536621094,
      "logps/chosen": -34.3978157043457,
      "logps/rejected": -31.752635955810547,
      "loss": 0.6988,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00010652244236553088,
      "rewards/margins": -0.009455609135329723,
      "rewards/rejected": 0.009349087253212929,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603521e-07,
      "logits/chosen": -1.9801543951034546,
      "logits/rejected": -1.9695091247558594,
      "logps/chosen": -35.32206344604492,
      "logps/rejected": -31.858333587646484,
      "loss": 0.6816,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.017289450392127037,
      "rewards/margins": 0.025958454236388206,
      "rewards/rejected": -0.00866900384426117,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071453e-08,
      "logits/chosen": -2.07523775100708,
      "logits/rejected": -2.0602152347564697,
      "logps/chosen": -30.91782569885254,
      "logps/rejected": -32.631629943847656,
      "loss": 0.6977,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0031501606572419405,
      "rewards/margins": -0.006952463183552027,
      "rewards/rejected": 0.010102624073624611,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-08,
      "logits/chosen": -1.9465487003326416,
      "logits/rejected": -1.9440209865570068,
      "logps/chosen": -32.89127731323242,
      "logps/rejected": -30.83047866821289,
      "loss": 0.6819,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.017916660755872726,
      "rewards/margins": 0.024751421064138412,
      "rewards/rejected": -0.006834760308265686,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.243638753890991,
      "eval_logits/rejected": -2.238759994506836,
      "eval_logps/chosen": -34.016868591308594,
      "eval_logps/rejected": -37.50496292114258,
      "eval_loss": 0.6929702162742615,
      "eval_rewards/accuracies": 0.5045680999755859,
      "eval_rewards/chosen": 0.008841208182275295,
      "eval_rewards/margins": 0.0030128166545182467,
      "eval_rewards/rejected": 0.005828391760587692,
      "eval_runtime": 145.9617,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.576113578589034e-08,
      "logits/chosen": -1.928741216659546,
      "logits/rejected": -1.9254982471466064,
      "logps/chosen": -31.564533233642578,
      "logps/rejected": -33.7502555847168,
      "loss": 0.6827,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019408607855439186,
      "rewards/margins": 0.0234612338244915,
      "rewards/rejected": -0.004052626900374889,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.229036944380912e-08,
      "logits/chosen": -1.9805328845977783,
      "logits/rejected": -1.968227744102478,
      "logps/chosen": -34.543617248535156,
      "logps/rejected": -33.547271728515625,
      "loss": 0.6797,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.02622898854315281,
      "rewards/margins": 0.03044353798031807,
      "rewards/rejected": -0.004214548971503973,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.053082288996112e-08,
      "logits/chosen": -2.016136884689331,
      "logits/rejected": -2.014674663543701,
      "logps/chosen": -33.480918884277344,
      "logps/rejected": -32.46858215332031,
      "loss": 0.6958,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0024364828132092953,
      "rewards/margins": -0.0030642927158623934,
      "rewards/rejected": 0.005500775761902332,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.05793773749158e-08,
      "logits/chosen": -2.1030609607696533,
      "logits/rejected": -2.0872530937194824,
      "logps/chosen": -34.16389083862305,
      "logps/rejected": -33.10190963745117,
      "loss": 0.695,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.007748591713607311,
      "rewards/margins": -0.000315183395287022,
      "rewards/rejected": 0.00806377548724413,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.251801807404168e-08,
      "logits/chosen": -1.9751472473144531,
      "logits/rejected": -1.9742006063461304,
      "logps/chosen": -33.24718475341797,
      "logps/rejected": -32.467201232910156,
      "loss": 0.6853,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.017456945031881332,
      "rewards/margins": 0.01848423480987549,
      "rewards/rejected": -0.001027289079502225,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.41315865106129e-09,
      "logits/chosen": -1.9307329654693604,
      "logits/rejected": -1.9411048889160156,
      "logps/chosen": -32.18671798706055,
      "logps/rejected": -35.27191162109375,
      "loss": 0.6942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008985924534499645,
      "rewards/margins": 0.0005646973731927574,
      "rewards/rejected": 0.00842122733592987,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3150941078050324e-09,
      "logits/chosen": -2.0697169303894043,
      "logits/rejected": -2.0631613731384277,
      "logps/chosen": -33.653404235839844,
      "logps/rejected": -29.2293701171875,
      "loss": 0.6939,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.000717848539352417,
      "rewards/margins": 0.0007542222738265991,
      "rewards/rejected": -0.0014720708131790161,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.575864278703266e-10,
      "logits/chosen": -1.9293102025985718,
      "logits/rejected": -1.931481122970581,
      "logps/chosen": -34.22270965576172,
      "logps/rejected": -30.89593505859375,
      "loss": 0.6846,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.008169981651008129,
      "rewards/margins": 0.019822364673018456,
      "rewards/rejected": -0.011652383022010326,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.6914243283209862,
      "train_runtime": 3256.6129,
      "train_samples_per_second": 0.945,
      "train_steps_per_second": 0.118
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}