{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-08,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 1.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2820512820512818e-07,
      "logits/chosen": -1.8661272525787354,
      "logits/rejected": -1.8704373836517334,
      "logps/chosen": -36.993247985839844,
      "logps/rejected": -33.67000961303711,
      "loss": 0.952,
      "rewards/accuracies": 0.5138888955116272,
      "rewards/chosen": 0.012012440711259842,
      "rewards/margins": 0.047955650836229324,
      "rewards/rejected": -0.035943206399679184,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5641025641025636e-07,
      "logits/chosen": -1.9975776672363281,
      "logits/rejected": -2.000213146209717,
      "logps/chosen": -29.647293090820312,
      "logps/rejected": -29.056921005249023,
      "loss": 1.0105,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00459072832018137,
      "rewards/margins": -0.010462107136845589,
      "rewards/rejected": 0.005871377885341644,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.8461538461538463e-07,
      "logits/chosen": -1.920013189315796,
      "logits/rejected": -1.917337417602539,
      "logps/chosen": -31.4055233001709,
      "logps/rejected": -33.22685623168945,
      "loss": 0.9829,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.009516814723610878,
      "rewards/margins": 0.01707320474088192,
      "rewards/rejected": -0.007556394673883915,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438433e-07,
      "logits/chosen": -2.0167124271392822,
      "logits/rejected": -2.0079712867736816,
      "logps/chosen": -32.590545654296875,
      "logps/rejected": -32.499671936035156,
      "loss": 1.0241,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01236567273736,
      "rewards/margins": -0.024063030257821083,
      "rewards/rejected": 0.011697360314428806,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542186e-07,
      "logits/chosen": -1.8641105890274048,
      "logits/rejected": -1.8533456325531006,
      "logps/chosen": -33.56208038330078,
      "logps/rejected": -35.437400817871094,
      "loss": 1.0176,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004139312542974949,
      "rewards/margins": -0.017586929723620415,
      "rewards/rejected": 0.013447612524032593,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941118e-07,
      "logits/chosen": -1.9452192783355713,
      "logits/rejected": -1.9471514225006104,
      "logps/chosen": -32.587867736816406,
      "logps/rejected": -33.20810317993164,
      "loss": 0.9559,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.01108462829142809,
      "rewards/margins": 0.04413585737347603,
      "rewards/rejected": -0.033051230013370514,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413548e-07,
      "logits/chosen": -2.0798697471618652,
      "logits/rejected": -2.084836483001709,
      "logps/chosen": -33.99953079223633,
      "logps/rejected": -36.59130859375,
      "loss": 0.9897,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.009940117597579956,
      "rewards/margins": 0.010251833125948906,
      "rewards/rejected": -0.020191945135593414,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-07,
      "logits/chosen": -1.9422235488891602,
      "logits/rejected": -1.9453849792480469,
      "logps/chosen": -34.411861419677734,
      "logps/rejected": -34.599830627441406,
      "loss": 0.979,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.014353242702782154,
      "rewards/margins": 0.02102992869913578,
      "rewards/rejected": -0.006676685996353626,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.736716601303429e-07,
      "logits/chosen": -1.950791358947754,
      "logits/rejected": -1.9553003311157227,
      "logps/chosen": -32.45661163330078,
      "logps/rejected": -32.35947799682617,
      "loss": 0.9898,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.009730497375130653,
      "rewards/margins": 0.010243801400065422,
      "rewards/rejected": -0.0005133040249347687,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.62624545834521e-07,
      "logits/chosen": -2.0489749908447266,
      "logits/rejected": -2.046968460083008,
      "logps/chosen": -32.233360290527344,
      "logps/rejected": -31.278451919555664,
      "loss": 0.9823,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.006729811429977417,
      "rewards/margins": 0.01774778962135315,
      "rewards/rejected": -0.011017980054020882,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.243974447250366,
      "eval_logits/rejected": -2.239091634750366,
      "eval_logps/chosen": -34.021610260009766,
      "eval_logps/rejected": -37.50578689575195,
      "eval_loss": 0.9981797933578491,
      "eval_rewards/accuracies": 0.5103820562362671,
      "eval_rewards/chosen": 0.011647163890302181,
      "eval_rewards/margins": 0.0018998426385223866,
      "eval_rewards/rejected": 0.009747318923473358,
      "eval_runtime": 146.0735,
      "eval_samples_per_second": 2.348,
      "eval_steps_per_second": 0.294,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4982572012636904e-07,
      "logits/chosen": -2.0054612159729004,
      "logits/rejected": -2.003042221069336,
      "logps/chosen": -33.2550048828125,
      "logps/rejected": -34.02202224731445,
      "loss": 1.0167,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.010068392381072044,
      "rewards/margins": -0.016683781519532204,
      "rewards/rejected": 0.006615391466766596,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777677e-07,
      "logits/chosen": -2.016511917114258,
      "logits/rejected": -2.008146047592163,
      "logps/chosen": -32.45395278930664,
      "logps/rejected": -32.17681121826172,
      "loss": 0.9944,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.008141009137034416,
      "rewards/margins": 0.00563241308555007,
      "rewards/rejected": -0.013773423619568348,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.194082707715275e-07,
      "logits/chosen": -2.0463106632232666,
      "logits/rejected": -2.0382590293884277,
      "logps/chosen": -30.476207733154297,
      "logps/rejected": -32.020484924316406,
      "loss": 1.0297,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.00044600124238058925,
      "rewards/margins": -0.029714178293943405,
      "rewards/rejected": 0.02926817536354065,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.020402418666621e-07,
      "logits/chosen": -1.9767357110977173,
      "logits/rejected": -1.986999273300171,
      "logps/chosen": -31.381155014038086,
      "logps/rejected": -32.54108428955078,
      "loss": 0.9598,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.034655503928661346,
      "rewards/margins": 0.04024083539843559,
      "rewards/rejected": -0.005585335195064545,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8341962650351185e-07,
      "logits/chosen": -1.8906068801879883,
      "logits/rejected": -1.8916860818862915,
      "logps/chosen": -34.20075225830078,
      "logps/rejected": -34.7618522644043,
      "loss": 1.0031,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.007252261973917484,
      "rewards/margins": -0.0031175222247838974,
      "rewards/rejected": -0.004134741611778736,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800572e-07,
      "logits/chosen": -1.942251443862915,
      "logits/rejected": -1.9387743473052979,
      "logps/chosen": -36.145790100097656,
      "logps/rejected": -32.73196029663086,
      "loss": 0.9684,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.023301539942622185,
      "rewards/margins": 0.031586963683366776,
      "rewards/rejected": -0.008285422809422016,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.430433172111807e-07,
      "logits/chosen": -2.041952610015869,
      "logits/rejected": -2.0345559120178223,
      "logps/chosen": -33.81856155395508,
      "logps/rejected": -31.359323501586914,
      "loss": 1.0119,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.018632039427757263,
      "rewards/margins": -0.011905943043529987,
      "rewards/rejected": -0.006726093590259552,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.216202642830543e-07,
      "logits/chosen": -2.0473170280456543,
      "logits/rejected": -2.0525729656219482,
      "logps/chosen": -32.52764129638672,
      "logps/rejected": -32.50192642211914,
      "loss": 0.9695,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.015429094433784485,
      "rewards/margins": 0.030534446239471436,
      "rewards/rejected": -0.015105349011719227,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.9960716642946403e-07,
      "logits/chosen": -2.048530340194702,
      "logits/rejected": -2.045767307281494,
      "logps/chosen": -31.485219955444336,
      "logps/rejected": -31.34262466430664,
      "loss": 0.9747,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.003360496135428548,
      "rewards/margins": 0.02533094584941864,
      "rewards/rejected": -0.02197045087814331,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.771853789806683e-07,
      "logits/chosen": -1.918745756149292,
      "logits/rejected": -1.9234097003936768,
      "logps/chosen": -31.576425552368164,
      "logps/rejected": -32.79480743408203,
      "loss": 0.9696,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.026870254427194595,
      "rewards/margins": 0.03035963699221611,
      "rewards/rejected": -0.0034893869888037443,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.243912696838379,
      "eval_logits/rejected": -2.2390317916870117,
      "eval_logps/chosen": -34.021690368652344,
      "eval_logps/rejected": -37.502418518066406,
      "eval_loss": 1.001371145248413,
      "eval_rewards/accuracies": 0.4692690968513489,
      "eval_rewards/chosen": 0.011576101183891296,
      "eval_rewards/margins": -0.0012021720176562667,
      "eval_rewards/rejected": 0.012778270989656448,
      "eval_runtime": 145.8493,
      "eval_samples_per_second": 2.352,
      "eval_steps_per_second": 0.295,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402e-07,
      "logits/chosen": -2.0318515300750732,
      "logits/rejected": -2.0425400733947754,
      "logps/chosen": -31.96026039123535,
      "logps/rejected": -33.8691291809082,
      "loss": 0.9843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009936656802892685,
      "rewards/margins": 0.015721982344985008,
      "rewards/rejected": -0.005785329267382622,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.318564697655179e-07,
      "logits/chosen": -1.9256188869476318,
      "logits/rejected": -1.9404585361480713,
      "logps/chosen": -30.080001831054688,
      "logps/rejected": -31.53633689880371,
      "loss": 0.9913,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.028384273871779442,
      "rewards/margins": 0.00873211957514286,
      "rewards/rejected": 0.01965215802192688,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.093227910899832e-07,
      "logits/chosen": -1.9830671548843384,
      "logits/rejected": -1.9870269298553467,
      "logps/chosen": -33.386474609375,
      "logps/rejected": -31.55719566345215,
      "loss": 0.965,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.02789575420320034,
      "rewards/margins": 0.03499894589185715,
      "rewards/rejected": -0.007103185169398785,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279356e-07,
      "logits/chosen": -1.9833042621612549,
      "logits/rejected": -1.96132493019104,
      "logps/chosen": -34.13536834716797,
      "logps/rejected": -34.952049255371094,
      "loss": 0.9823,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.016542484983801842,
      "rewards/margins": 0.01767394319176674,
      "rewards/rejected": -0.0011314575094729662,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.654436768970182e-07,
      "logits/chosen": -2.0247886180877686,
      "logits/rejected": -2.0214812755584717,
      "logps/chosen": -32.913856506347656,
      "logps/rejected": -36.23151397705078,
      "loss": 0.984,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010092089883983135,
      "rewards/margins": 0.016022875905036926,
      "rewards/rejected": -0.0059307836927473545,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.444597403062196e-07,
      "logits/chosen": -1.8914467096328735,
      "logits/rejected": -1.8890043497085571,
      "logps/chosen": -34.18690872192383,
      "logps/rejected": -35.524742126464844,
      "loss": 0.9806,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.0013398710871115327,
      "rewards/margins": 0.019445054233074188,
      "rewards/rejected": -0.01810518093407154,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2434529917578887e-07,
      "logits/chosen": -1.8761389255523682,
      "logits/rejected": -1.8735923767089844,
      "logps/chosen": -34.396785736083984,
      "logps/rejected": -31.74139976501465,
      "loss": 1.0262,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.000734941684640944,
      "rewards/margins": -0.026202131062746048,
      "rewards/rejected": 0.02693706750869751,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603521e-07,
      "logits/chosen": -1.9801757335662842,
      "logits/rejected": -1.9695546627044678,
      "logps/chosen": -35.34246063232422,
      "logps/rejected": -31.823230743408203,
      "loss": 1.0032,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.012763315811753273,
      "rewards/margins": -0.003222113475203514,
      "rewards/rejected": 0.015985429286956787,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071453e-08,
      "logits/chosen": -2.075316905975342,
      "logits/rejected": -2.0602917671203613,
      "logps/chosen": -30.897375106811523,
      "logps/rejected": -32.64093780517578,
      "loss": 0.9857,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.024072837084531784,
      "rewards/margins": 0.014264687895774841,
      "rewards/rejected": 0.009808152914047241,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-08,
      "logits/chosen": -1.946758508682251,
      "logits/rejected": -1.9442226886749268,
      "logps/chosen": -32.89502716064453,
      "logps/rejected": -30.831274032592773,
      "loss": 0.9581,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0288736280053854,
      "rewards/margins": 0.04189068451523781,
      "rewards/rejected": -0.013017055578529835,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.2437047958374023,
      "eval_logits/rejected": -2.2388339042663574,
      "eval_logps/chosen": -34.03062438964844,
      "eval_logps/rejected": -37.499412536621094,
      "eval_loss": 1.0120128393173218,
      "eval_rewards/accuracies": 0.472591370344162,
      "eval_rewards/chosen": 0.003532935632392764,
      "eval_rewards/margins": -0.011953731998801231,
      "eval_rewards/rejected": 0.015486669726669788,
      "eval_runtime": 145.7744,
      "eval_samples_per_second": 2.353,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.576113578589034e-08,
      "logits/chosen": -1.928391695022583,
      "logits/rejected": -1.925132393836975,
      "logps/chosen": -31.573719024658203,
      "logps/rejected": -33.733333587646484,
      "loss": 0.9813,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.026669049635529518,
      "rewards/margins": 0.01873381994664669,
      "rewards/rejected": 0.007935228757560253,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.229036944380912e-08,
      "logits/chosen": -1.9808565378189087,
      "logits/rejected": -1.9685465097427368,
      "logps/chosen": -34.569026947021484,
      "logps/rejected": -33.5385856628418,
      "loss": 0.9759,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.02433854714035988,
      "rewards/margins": 0.024108227342367172,
      "rewards/rejected": 0.00023032091849017888,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.053082288996112e-08,
      "logits/chosen": -2.0160164833068848,
      "logits/rejected": -2.0145697593688965,
      "logps/chosen": -33.470794677734375,
      "logps/rejected": -32.465301513671875,
      "loss": 0.9994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013503548689186573,
      "rewards/margins": 0.0006495133275166154,
      "rewards/rejected": 0.012854036875069141,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.05793773749158e-08,
      "logits/chosen": -2.1030516624450684,
      "logits/rejected": -2.087252140045166,
      "logps/chosen": -34.154754638671875,
      "logps/rejected": -33.087013244628906,
      "loss": 1.0057,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.02217467688024044,
      "rewards/margins": -0.005742859095335007,
      "rewards/rejected": 0.027917543426156044,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.251801807404168e-08,
      "logits/chosen": -1.9748103618621826,
      "logits/rejected": -1.9738750457763672,
      "logps/chosen": -33.23518371582031,
      "logps/rejected": -32.46066665649414,
      "loss": 0.9618,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.04222174361348152,
      "rewards/margins": 0.03818847984075546,
      "rewards/rejected": 0.004033264704048634,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.41315865106129e-09,
      "logits/chosen": -1.9303611516952515,
      "logits/rejected": -1.940708875656128,
      "logps/chosen": -32.200523376464844,
      "logps/rejected": -35.27861785888672,
      "loss": 1.0054,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.0037514485884457827,
      "rewards/margins": -0.005373493768274784,
      "rewards/rejected": 0.009124943986535072,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3150941078050324e-09,
      "logits/chosen": -2.069342613220215,
      "logits/rejected": -2.062788486480713,
      "logps/chosen": -33.65803527832031,
      "logps/rejected": -29.199758529663086,
      "loss": 1.0295,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005459528882056475,
      "rewards/margins": -0.029461050406098366,
      "rewards/rejected": 0.02400151826441288,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.575864278703266e-10,
      "logits/chosen": -1.9290788173675537,
      "logits/rejected": -1.931239366531372,
      "logps/chosen": -34.229915618896484,
      "logps/rejected": -30.90093994140625,
      "loss": 0.9663,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00822196900844574,
      "rewards/margins": 0.03370029479265213,
      "rewards/rejected": -0.02547832392156124,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.9887851405453373,
      "train_runtime": 3254.6365,
      "train_samples_per_second": 0.946,
      "train_steps_per_second": 0.118
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}