{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-08,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2820512820512818e-07,
      "logits/chosen": -1.866300106048584,
      "logits/rejected": -1.870603084564209,
      "logps/chosen": -36.973060607910156,
      "logps/rejected": -33.65780258178711,
      "loss": 0.6843,
      "rewards/accuracies": 0.5138888955116272,
      "rewards/chosen": 0.010061310604214668,
      "rewards/margins": 0.018380172550678253,
      "rewards/rejected": -0.008318860083818436,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5641025641025636e-07,
      "logits/chosen": -1.9976227283477783,
      "logits/rejected": -2.0002670288085938,
      "logps/chosen": -29.652048110961914,
      "logps/rejected": -29.051151275634766,
      "loss": 0.6969,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0029565312433987856,
      "rewards/margins": -0.006644421722739935,
      "rewards/rejected": 0.003687891410663724,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.8461538461538463e-07,
      "logits/chosen": -1.9202871322631836,
      "logits/rejected": -1.9175989627838135,
      "logps/chosen": -31.403018951416016,
      "logps/rejected": -33.22191619873047,
      "loss": 0.6911,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0039227609522640705,
      "rewards/margins": 0.004958181641995907,
      "rewards/rejected": -0.0010354205733165145,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438433e-07,
      "logits/chosen": -2.016842842102051,
      "logits/rejected": -2.008082151412964,
      "logps/chosen": -32.58789825439453,
      "logps/rejected": -32.494510650634766,
      "loss": 0.6978,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.003327915444970131,
      "rewards/margins": -0.008775560185313225,
      "rewards/rejected": 0.005447645206004381,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542186e-07,
      "logits/chosen": -1.8639650344848633,
      "logits/rejected": -1.8531978130340576,
      "logps/chosen": -33.55598831176758,
      "logps/rejected": -35.437034606933594,
      "loss": 0.6956,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0004471995052881539,
      "rewards/margins": -0.004145150538533926,
      "rewards/rejected": 0.00459235068410635,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941118e-07,
      "logits/chosen": -1.9451839923858643,
      "logits/rejected": -1.94712233543396,
      "logps/chosen": -32.599273681640625,
      "logps/rejected": -33.205684661865234,
      "loss": 0.6882,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.000273245939752087,
      "rewards/margins": 0.010566400364041328,
      "rewards/rejected": -0.010293153114616871,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413548e-07,
      "logits/chosen": -2.0796267986297607,
      "logits/rejected": -2.0845961570739746,
      "logps/chosen": -33.98247528076172,
      "logps/rejected": -36.595970153808594,
      "loss": 0.6887,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.0018027916084975004,
      "rewards/margins": 0.009932766668498516,
      "rewards/rejected": -0.008129975758492947,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-07,
      "logits/chosen": -1.9422763586044312,
      "logits/rejected": -1.9454386234283447,
      "logps/chosen": -34.4161491394043,
      "logps/rejected": -34.59540939331055,
      "loss": 0.6915,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.0034980489872395992,
      "rewards/margins": 0.004399437457323074,
      "rewards/rejected": -0.0009013883536681533,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.736716601303429e-07,
      "logits/chosen": -1.950871467590332,
      "logits/rejected": -1.9553781747817993,
      "logps/chosen": -32.455718994140625,
      "logps/rejected": -32.37923812866211,
      "loss": 0.6888,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003510843263939023,
      "rewards/margins": 0.009610405191779137,
      "rewards/rejected": -0.00609956169500947,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.62624545834521e-07,
      "logits/chosen": -2.0493884086608887,
      "logits/rejected": -2.0473878383636475,
      "logps/chosen": -32.24263381958008,
      "logps/rejected": -31.301403045654297,
      "loss": 0.6886,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.0005389736033976078,
      "rewards/margins": 0.010018542408943176,
      "rewards/rejected": -0.010557514615356922,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.244236707687378,
      "eval_logits/rejected": -2.2393476963043213,
      "eval_logps/chosen": -34.027488708496094,
      "eval_logps/rejected": -37.500125885009766,
      "eval_loss": 0.6949825882911682,
      "eval_rewards/accuracies": 0.48172760009765625,
      "eval_rewards/chosen": 0.0021188759710639715,
      "eval_rewards/margins": -0.0028304157312959433,
      "eval_rewards/rejected": 0.0049492912366986275,
      "eval_runtime": 146.0369,
      "eval_samples_per_second": 2.349,
      "eval_steps_per_second": 0.294,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4982572012636904e-07,
      "logits/chosen": -2.005858898162842,
      "logits/rejected": -2.0034613609313965,
      "logps/chosen": -33.22675323486328,
      "logps/rejected": -34.024375915527344,
      "loss": 0.6918,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.0051184119656682014,
      "rewards/margins": 0.0036206122022122145,
      "rewards/rejected": 0.0014977991813793778,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777677e-07,
      "logits/chosen": -2.0165371894836426,
      "logits/rejected": -2.0081915855407715,
      "logps/chosen": -32.43837356567383,
      "logps/rejected": -32.1854362487793,
      "loss": 0.689,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.0019600242376327515,
      "rewards/margins": 0.009139470756053925,
      "rewards/rejected": -0.0071794455870985985,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.194082707715275e-07,
      "logits/chosen": -2.046520709991455,
      "logits/rejected": -2.0384721755981445,
      "logps/chosen": -30.485082626342773,
      "logps/rejected": -32.0345344543457,
      "loss": 0.6976,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.002810357604175806,
      "rewards/margins": -0.008352747187018394,
      "rewards/rejected": 0.005542389117181301,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.020402418666621e-07,
      "logits/chosen": -1.976843237876892,
      "logits/rejected": -1.9871089458465576,
      "logps/chosen": -31.4160099029541,
      "logps/rejected": -32.535850524902344,
      "loss": 0.6929,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0010961029911413789,
      "rewards/margins": 0.0013871133560314775,
      "rewards/rejected": -0.0002910103357862681,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8341962650351185e-07,
      "logits/chosen": -1.890434980392456,
      "logits/rejected": -1.8915313482284546,
      "logps/chosen": -34.165489196777344,
      "logps/rejected": -34.74839782714844,
      "loss": 0.691,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.008160250261425972,
      "rewards/margins": 0.005502746906131506,
      "rewards/rejected": 0.0026575028896331787,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800572e-07,
      "logits/chosen": -1.942751169204712,
      "logits/rejected": -1.9392738342285156,
      "logps/chosen": -36.15436553955078,
      "logps/rejected": -32.734371185302734,
      "loss": 0.6893,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.005194104742258787,
      "rewards/margins": 0.008680562488734722,
      "rewards/rejected": -0.0034864575136452913,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.430433172111807e-07,
      "logits/chosen": -2.042160749435425,
      "logits/rejected": -2.0347483158111572,
      "logps/chosen": -33.785484313964844,
      "logps/rejected": -31.344274520874023,
      "loss": 0.6928,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.003714872058480978,
      "rewards/margins": 0.0014427527785301208,
      "rewards/rejected": 0.002272119279950857,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.216202642830543e-07,
      "logits/chosen": -2.0473780632019043,
      "logits/rejected": -2.0526490211486816,
      "logps/chosen": -32.52016830444336,
      "logps/rejected": -32.495323181152344,
      "loss": 0.6883,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.007385374512523413,
      "rewards/margins": 0.010440578684210777,
      "rewards/rejected": -0.0030552041716873646,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.9960716642946403e-07,
      "logits/chosen": -2.0484580993652344,
      "logits/rejected": -2.045682191848755,
      "logps/chosen": -31.508682250976562,
      "logps/rejected": -31.3348388671875,
      "loss": 0.694,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.005918826907873154,
      "rewards/margins": -0.0009324293350800872,
      "rewards/rejected": -0.004986398387700319,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.771853789806683e-07,
      "logits/chosen": -1.9187612533569336,
      "logits/rejected": -1.9234310388565063,
      "logps/chosen": -31.59661865234375,
      "logps/rejected": -32.798248291015625,
      "loss": 0.6909,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.002899137092754245,
      "rewards/margins": 0.00509423715993762,
      "rewards/rejected": -0.002195100300014019,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.2439279556274414,
      "eval_logits/rejected": -2.239055633544922,
      "eval_logps/chosen": -34.03278350830078,
      "eval_logps/rejected": -37.50389099121094,
      "eval_loss": 0.6952112317085266,
      "eval_rewards/accuracies": 0.5191029906272888,
      "eval_rewards/chosen": 0.0005306191742420197,
      "eval_rewards/margins": -0.0032888990826904774,
      "eval_rewards/rejected": 0.0038195180241018534,
      "eval_runtime": 145.8142,
      "eval_samples_per_second": 2.352,
      "eval_steps_per_second": 0.295,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402e-07,
      "logits/chosen": -2.0318386554718018,
      "logits/rejected": -2.0425145626068115,
      "logps/chosen": -31.961750030517578,
      "logps/rejected": -33.86240005493164,
      "loss": 0.6923,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0028654958587139845,
      "rewards/margins": 0.002774373395368457,
      "rewards/rejected": 9.112273983191699e-05,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.318564697655179e-07,
      "logits/chosen": -1.9253981113433838,
      "logits/rejected": -1.9402506351470947,
      "logps/chosen": -30.093517303466797,
      "logps/rejected": -31.571548461914062,
      "loss": 0.6889,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.005405673757195473,
      "rewards/margins": 0.009417025372385979,
      "rewards/rejected": -0.004011353012174368,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.093227910899832e-07,
      "logits/chosen": -1.9839330911636353,
      "logits/rejected": -1.9879001379013062,
      "logps/chosen": -33.4061279296875,
      "logps/rejected": -31.564762115478516,
      "loss": 0.6896,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0034024070482701063,
      "rewards/margins": 0.008040683344006538,
      "rewards/rejected": -0.004638276062905788,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279356e-07,
      "logits/chosen": -1.983189344406128,
      "logits/rejected": -1.9612159729003906,
      "logps/chosen": -34.15366744995117,
      "logps/rejected": -34.95070266723633,
      "loss": 0.6937,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 2.4808850866975263e-05,
      "rewards/margins": -1.6275793086606427e-06,
      "rewards/rejected": 2.643577681737952e-05,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.654436768970182e-07,
      "logits/chosen": -2.0243947505950928,
      "logits/rejected": -2.021087169647217,
      "logps/chosen": -32.91602325439453,
      "logps/rejected": -36.221763610839844,
      "loss": 0.6928,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.002715445589274168,
      "rewards/margins": 0.001767707639373839,
      "rewards/rejected": 0.0009477367857471108,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.444597403062196e-07,
      "logits/chosen": -1.8911654949188232,
      "logits/rejected": -1.8887317180633545,
      "logps/chosen": -34.18123245239258,
      "logps/rejected": -35.515174865722656,
      "loss": 0.691,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.0021498873829841614,
      "rewards/margins": 0.005315869115293026,
      "rewards/rejected": -0.0031659831292927265,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2434529917578887e-07,
      "logits/chosen": -1.875580072402954,
      "logits/rejected": -1.8730493783950806,
      "logps/chosen": -34.37896728515625,
      "logps/rejected": -31.76068687438965,
      "loss": 0.6925,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.005589136388152838,
      "rewards/margins": 0.0023957251105457544,
      "rewards/rejected": 0.0031934112776070833,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603521e-07,
      "logits/chosen": -1.9798663854599,
      "logits/rejected": -1.9692500829696655,
      "logps/chosen": -35.32219696044922,
      "logps/rejected": -31.857141494750977,
      "loss": 0.6861,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010332418605685234,
      "rewards/margins": 0.015176350250840187,
      "rewards/rejected": -0.004843929782509804,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071453e-08,
      "logits/chosen": -2.076137065887451,
      "logits/rejected": -2.061100721359253,
      "logps/chosen": -30.907434463500977,
      "logps/rejected": -32.638729095458984,
      "loss": 0.693,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.00500696524977684,
      "rewards/margins": 0.0010755129624158144,
      "rewards/rejected": 0.0039314525201916695,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-08,
      "logits/chosen": -1.947422981262207,
      "logits/rejected": -1.9448877573013306,
      "logps/chosen": -32.89393997192383,
      "logps/rejected": -30.828500747680664,
      "loss": 0.6869,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.009951007552444935,
      "rewards/margins": 0.013458291999995708,
      "rewards/rejected": -0.003507285611703992,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.244764566421509,
      "eval_logits/rejected": -2.2398788928985596,
      "eval_logps/chosen": -34.038692474365234,
      "eval_logps/rejected": -37.512168884277344,
      "eval_loss": 0.6949278712272644,
      "eval_rewards/accuracies": 0.5161960124969482,
      "eval_rewards/chosen": -0.0012418677797541022,
      "eval_rewards/margins": -0.0025783225428313017,
      "eval_rewards/rejected": 0.0013364545302465558,
      "eval_runtime": 145.8034,
      "eval_samples_per_second": 2.352,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.576113578589034e-08,
      "logits/chosen": -1.9299108982086182,
      "logits/rejected": -1.9266481399536133,
      "logps/chosen": -31.572988510131836,
      "logps/rejected": -33.727210998535156,
      "loss": 0.6912,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.009108476340770721,
      "rewards/margins": 0.004626707639545202,
      "rewards/rejected": 0.0044817691668868065,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.229036944380912e-08,
      "logits/chosen": -1.9814649820327759,
      "logits/rejected": -1.9691375494003296,
      "logps/chosen": -34.551979064941406,
      "logps/rejected": -33.541648864746094,
      "loss": 0.6866,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.013229632750153542,
      "rewards/margins": 0.014072793535888195,
      "rewards/rejected": -0.0008431615424342453,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.053082288996112e-08,
      "logits/chosen": -2.017097234725952,
      "logits/rejected": -2.0156359672546387,
      "logps/chosen": -33.49595260620117,
      "logps/rejected": -32.49913787841797,
      "loss": 0.6922,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.003048144979402423,
      "rewards/margins": 0.002819380722939968,
      "rewards/rejected": -0.00586752500385046,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.05793773749158e-08,
      "logits/chosen": -2.103210926055908,
      "logits/rejected": -2.0874040126800537,
      "logps/chosen": -34.16447830200195,
      "logps/rejected": -33.07390594482422,
      "loss": 0.6981,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.004473570734262466,
      "rewards/margins": -0.008766504004597664,
      "rewards/rejected": 0.01324007473886013,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.251801807404168e-08,
      "logits/chosen": -1.9749380350112915,
      "logits/rejected": -1.9740018844604492,
      "logps/chosen": -33.25703048706055,
      "logps/rejected": -32.47133255004883,
      "loss": 0.689,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00751916179433465,
      "rewards/margins": 0.009374773129820824,
      "rewards/rejected": -0.0018556114519014955,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.41315865106129e-09,
      "logits/chosen": -1.9304084777832031,
      "logits/rejected": -1.9407765865325928,
      "logps/chosen": -32.21118927001953,
      "logps/rejected": -35.31554412841797,
      "loss": 0.6905,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0019482700154185295,
      "rewards/margins": 0.0060885134153068066,
      "rewards/rejected": -0.008036783896386623,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3150941078050324e-09,
      "logits/chosen": -2.06982421875,
      "logits/rejected": -2.0632567405700684,
      "logps/chosen": -33.641204833984375,
      "logps/rejected": -29.2253475189209,
      "loss": 0.6921,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003229249268770218,
      "rewards/margins": 0.0029065334238111973,
      "rewards/rejected": 0.000322715932270512,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.575864278703266e-10,
      "logits/chosen": -1.929220199584961,
      "logits/rejected": -1.9313905239105225,
      "logps/chosen": -34.26762008666992,
      "logps/rejected": -30.9031982421875,
      "loss": 0.6934,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.00857237633317709,
      "rewards/margins": 0.0005976069951429963,
      "rewards/rejected": -0.009169982746243477,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.691356255791404,
      "train_runtime": 3254.7157,
      "train_samples_per_second": 0.946,
      "train_steps_per_second": 0.118
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}