{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-08,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2820512820512818e-07,
      "logits/chosen": -1.866014003753662,
      "logits/rejected": -1.8703253269195557,
      "logps/chosen": -36.98957061767578,
      "logps/rejected": -33.655086517333984,
      "loss": 0.6802,
      "rewards/accuracies": 0.5555555820465088,
      "rewards/chosen": 0.01192101463675499,
      "rewards/margins": 0.029430679976940155,
      "rewards/rejected": -0.017509667202830315,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5641025641025636e-07,
      "logits/chosen": -1.9975181818008423,
      "logits/rejected": -2.000157356262207,
      "logps/chosen": -29.643035888671875,
      "logps/rejected": -29.068897247314453,
      "loss": 0.6931,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.0005901128170080483,
      "rewards/margins": 0.003226220142096281,
      "rewards/rejected": -0.003816334530711174,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.8461538461538463e-07,
      "logits/chosen": -1.919750452041626,
      "logits/rejected": -1.9170730113983154,
      "logps/chosen": -31.426767349243164,
      "logps/rejected": -33.24191665649414,
      "loss": 0.6908,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.00747058168053627,
      "rewards/margins": 0.008947307243943214,
      "rewards/rejected": -0.016417888924479485,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438433e-07,
      "logits/chosen": -2.0170185565948486,
      "logits/rejected": -2.008249282836914,
      "logps/chosen": -32.58369827270508,
      "logps/rejected": -32.508426666259766,
      "loss": 0.6989,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.004824736155569553,
      "rewards/margins": -0.007793365977704525,
      "rewards/rejected": 0.0029686305206269026,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542186e-07,
      "logits/chosen": -1.864098310470581,
      "logits/rejected": -1.8533239364624023,
      "logps/chosen": -33.5538444519043,
      "logps/rejected": -35.4467658996582,
      "loss": 0.6956,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.002545009134337306,
      "rewards/margins": -0.0013612977927550673,
      "rewards/rejected": 0.003906308673322201,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941118e-07,
      "logits/chosen": -1.9450995922088623,
      "logits/rejected": -1.9470382928848267,
      "logps/chosen": -32.56639862060547,
      "logps/rejected": -33.19929122924805,
      "loss": 0.6743,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02365189418196678,
      "rewards/margins": 0.043189071118831635,
      "rewards/rejected": -0.019537178799510002,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413548e-07,
      "logits/chosen": -2.0796637535095215,
      "logits/rejected": -2.084643602371216,
      "logps/chosen": -33.98316955566406,
      "logps/rejected": -36.59341049194336,
      "loss": 0.686,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.00371909374371171,
      "rewards/margins": 0.020896537229418755,
      "rewards/rejected": -0.01717744767665863,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-07,
      "logits/chosen": -1.9420992136001587,
      "logits/rejected": -1.9452590942382812,
      "logps/chosen": -34.40605545043945,
      "logps/rejected": -34.58306884765625,
      "loss": 0.6914,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.015227687545120716,
      "rewards/margins": 0.008690384216606617,
      "rewards/rejected": 0.006537304259836674,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.736716601303429e-07,
      "logits/chosen": -1.950547456741333,
      "logits/rejected": -1.9550635814666748,
      "logps/chosen": -32.4520378112793,
      "logps/rejected": -32.369686126708984,
      "loss": 0.6861,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010769412852823734,
      "rewards/margins": 0.018315035849809647,
      "rewards/rejected": -0.007545621134340763,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.62624545834521e-07,
      "logits/chosen": -2.0487563610076904,
      "logits/rejected": -2.046765089035034,
      "logps/chosen": -32.21302032470703,
      "logps/rejected": -31.275714874267578,
      "loss": 0.6821,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.019469190388917923,
      "rewards/margins": 0.02612192928791046,
      "rewards/rejected": -0.006652742624282837,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.244154930114746,
      "eval_logits/rejected": -2.2392725944519043,
      "eval_logps/chosen": -34.02579116821289,
      "eval_logps/rejected": -37.495243072509766,
      "eval_loss": 0.6996207237243652,
      "eval_rewards/accuracies": 0.48048171401023865,
      "eval_rewards/chosen": 0.006134189199656248,
      "eval_rewards/margins": -0.008827367797493935,
      "eval_rewards/rejected": 0.014961558394134045,
      "eval_runtime": 146.1618,
      "eval_samples_per_second": 2.347,
      "eval_steps_per_second": 0.294,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4982572012636904e-07,
      "logits/chosen": -2.005396604537964,
      "logits/rejected": -2.00298810005188,
      "logps/chosen": -33.235660552978516,
      "logps/rejected": -34.023067474365234,
      "loss": 0.6946,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0057089305482804775,
      "rewards/margins": 0.001296796603128314,
      "rewards/rejected": 0.0044121332466602325,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777677e-07,
      "logits/chosen": -2.0165247917175293,
      "logits/rejected": -2.0081515312194824,
      "logps/chosen": -32.45998764038086,
      "logps/rejected": -32.18000793457031,
      "loss": 0.6936,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.010559305548667908,
      "rewards/margins": 0.0023918983060866594,
      "rewards/rejected": -0.012951202690601349,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.194082707715275e-07,
      "logits/chosen": -2.0464999675750732,
      "logits/rejected": -2.0384459495544434,
      "logps/chosen": -30.482324600219727,
      "logps/rejected": -32.024261474609375,
      "loss": 0.7073,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.004625456873327494,
      "rewards/margins": -0.02474728412926197,
      "rewards/rejected": 0.020121825858950615,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.020402418666621e-07,
      "logits/chosen": -1.976978063583374,
      "logits/rejected": -1.9872522354125977,
      "logps/chosen": -31.402069091796875,
      "logps/rejected": -32.54044723510742,
      "loss": 0.6869,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.012314395979046822,
      "rewards/margins": 0.01621088758111,
      "rewards/rejected": -0.0038964927662163973,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8341962650351185e-07,
      "logits/chosen": -1.8904860019683838,
      "logits/rejected": -1.8915693759918213,
      "logps/chosen": -34.20112609863281,
      "logps/rejected": -34.765201568603516,
      "loss": 0.6961,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.005904030986130238,
      "rewards/margins": -0.0003432026132941246,
      "rewards/rejected": -0.005560829304158688,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800572e-07,
      "logits/chosen": -1.9423534870147705,
      "logits/rejected": -1.938856840133667,
      "logps/chosen": -36.14204025268555,
      "logps/rejected": -32.7347412109375,
      "loss": 0.6817,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.020746376365423203,
      "rewards/margins": 0.029139723628759384,
      "rewards/rejected": -0.008393346332013607,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.430433172111807e-07,
      "logits/chosen": -2.0420148372650146,
      "logits/rejected": -2.034619092941284,
      "logps/chosen": -33.79521942138672,
      "logps/rejected": -31.3667049407959,
      "loss": 0.6897,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0018501380691304803,
      "rewards/margins": 0.012249214574694633,
      "rewards/rejected": -0.010399078950285912,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.216202642830543e-07,
      "logits/chosen": -2.0474741458892822,
      "logits/rejected": -2.052746295928955,
      "logps/chosen": -32.533199310302734,
      "logps/rejected": -32.49622344970703,
      "loss": 0.6872,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.008109848015010357,
      "rewards/margins": 0.015868691727519035,
      "rewards/rejected": -0.007758840918540955,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.9960716642946403e-07,
      "logits/chosen": -2.048755407333374,
      "logits/rejected": -2.045989513397217,
      "logps/chosen": -31.495113372802734,
      "logps/rejected": -31.329687118530273,
      "loss": 0.6936,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.004311963450163603,
      "rewards/margins": 0.003718080697581172,
      "rewards/rejected": -0.008030043914914131,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.771853789806683e-07,
      "logits/chosen": -1.9183790683746338,
      "logits/rejected": -1.9230560064315796,
      "logps/chosen": -31.61056137084961,
      "logps/rejected": -32.79776382446289,
      "loss": 0.6938,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.002994548762217164,
      "rewards/margins": 0.001788289868272841,
      "rewards/rejected": -0.004782836884260178,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.2438013553619385,
      "eval_logits/rejected": -2.2389278411865234,
      "eval_logps/chosen": -34.02748107910156,
      "eval_logps/rejected": -37.502662658691406,
      "eval_loss": 0.6977797150611877,
      "eval_rewards/accuracies": 0.48172760009765625,
      "eval_rewards/chosen": 0.004952840972691774,
      "eval_rewards/margins": -0.004817771725356579,
      "eval_rewards/rejected": 0.009770614095032215,
      "eval_runtime": 146.0018,
      "eval_samples_per_second": 2.349,
      "eval_steps_per_second": 0.295,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402e-07,
      "logits/chosen": -2.0318925380706787,
      "logits/rejected": -2.0425820350646973,
      "logps/chosen": -31.96123695373535,
      "logps/rejected": -33.87626647949219,
      "loss": 0.6874,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.007046692073345184,
      "rewards/margins": 0.01654236949980259,
      "rewards/rejected": -0.009495675563812256,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.318564697655179e-07,
      "logits/chosen": -1.9252264499664307,
      "logits/rejected": -1.940086007118225,
      "logps/chosen": -30.087350845336914,
      "logps/rejected": -31.552490234375,
      "loss": 0.6894,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.01693141832947731,
      "rewards/margins": 0.012951460666954517,
      "rewards/rejected": 0.003979954868555069,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.093227910899832e-07,
      "logits/chosen": -1.9830553531646729,
      "logits/rejected": -1.9870340824127197,
      "logps/chosen": -33.37425231933594,
      "logps/rejected": -31.55866050720215,
      "loss": 0.6771,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.030249694362282753,
      "rewards/margins": 0.036800943315029144,
      "rewards/rejected": -0.006551248021423817,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279356e-07,
      "logits/chosen": -1.9825870990753174,
      "logits/rejected": -1.9606201648712158,
      "logps/chosen": -34.172298431396484,
      "logps/rejected": -34.95369338989258,
      "loss": 0.7012,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.012983955442905426,
      "rewards/margins": -0.010954022407531738,
      "rewards/rejected": -0.002029931638389826,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.654436768970182e-07,
      "logits/chosen": -2.0240674018859863,
      "logits/rejected": -2.020759105682373,
      "logps/chosen": -32.91376495361328,
      "logps/rejected": -36.22763442993164,
      "loss": 0.6907,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.007910441607236862,
      "rewards/margins": 0.009807790629565716,
      "rewards/rejected": -0.0018973512342199683,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.444597403062196e-07,
      "logits/chosen": -1.891227126121521,
      "logits/rejected": -1.8887770175933838,
      "logps/chosen": -34.196754455566406,
      "logps/rejected": -35.51189422607422,
      "loss": 0.6959,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005844797007739544,
      "rewards/margins": -0.0007547728600911796,
      "rewards/rejected": -0.005090024787932634,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2434529917578887e-07,
      "logits/chosen": -1.875758409500122,
      "logits/rejected": -1.8732105493545532,
      "logps/chosen": -34.38095474243164,
      "logps/rejected": -31.74618148803711,
      "loss": 0.6979,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.011651447974145412,
      "rewards/margins": -0.005950801074504852,
      "rewards/rejected": 0.01760224997997284,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603521e-07,
      "logits/chosen": -1.9794175624847412,
      "logits/rejected": -1.968788504600525,
      "logps/chosen": -35.32674026489258,
      "logps/rejected": -31.84674072265625,
      "loss": 0.6832,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.020933207124471664,
      "rewards/margins": 0.024957498535513878,
      "rewards/rejected": -0.004024294205009937,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071453e-08,
      "logits/chosen": -2.075899600982666,
      "logits/rejected": -2.0608580112457275,
      "logps/chosen": -30.886249542236328,
      "logps/rejected": -32.66474533081055,
      "loss": 0.6773,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.026513904333114624,
      "rewards/margins": 0.03555149585008621,
      "rewards/rejected": -0.009037593379616737,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-08,
      "logits/chosen": -1.9468847513198853,
      "logits/rejected": -1.9443557262420654,
      "logps/chosen": -32.885986328125,
      "logps/rejected": -30.816707611083984,
      "loss": 0.681,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02878658100962639,
      "rewards/margins": 0.028715472668409348,
      "rewards/rejected": 7.110489241313189e-05,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.244056224822998,
      "eval_logits/rejected": -2.2391812801361084,
      "eval_logps/chosen": -34.01231384277344,
      "eval_logps/rejected": -37.509132385253906,
      "eval_loss": 0.6901772618293762,
      "eval_rewards/accuracies": 0.5132890343666077,
      "eval_rewards/chosen": 0.015566298738121986,
      "eval_rewards/margins": 0.01032568421214819,
      "eval_rewards/rejected": 0.005240614525973797,
      "eval_runtime": 145.9901,
      "eval_samples_per_second": 2.349,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.576113578589034e-08,
      "logits/chosen": -1.9292223453521729,
      "logits/rejected": -1.9259742498397827,
      "logps/chosen": -31.571924209594727,
      "logps/rejected": -33.74208450317383,
      "loss": 0.6842,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.021998558193445206,
      "rewards/margins": 0.02195030450820923,
      "rewards/rejected": 4.825703945243731e-05,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.229036944380912e-08,
      "logits/chosen": -1.980800986289978,
      "logits/rejected": -1.9684827327728271,
      "logps/chosen": -34.57365417480469,
      "logps/rejected": -33.5623779296875,
      "loss": 0.6796,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.015695111826062202,
      "rewards/margins": 0.032171379774808884,
      "rewards/rejected": -0.01647626794874668,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.053082288996112e-08,
      "logits/chosen": -2.016021490097046,
      "logits/rejected": -2.0145699977874756,
      "logps/chosen": -33.47764205932617,
      "logps/rejected": -32.47804260253906,
      "loss": 0.6932,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.005706763360649347,
      "rewards/margins": 0.00462835980579257,
      "rewards/rejected": 0.0010784054175019264,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.05793773749158e-08,
      "logits/chosen": -2.1033365726470947,
      "logits/rejected": -2.087541103363037,
      "logps/chosen": -34.160888671875,
      "logps/rejected": -33.08522415161133,
      "loss": 0.7012,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.012950187548995018,
      "rewards/margins": -0.010019468143582344,
      "rewards/rejected": 0.022969653829932213,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.251801807404168e-08,
      "logits/chosen": -1.974867582321167,
      "logits/rejected": -1.9739230871200562,
      "logps/chosen": -33.23981857299805,
      "logps/rejected": -32.456180572509766,
      "loss": 0.6846,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.029593482613563538,
      "rewards/margins": 0.023315051570534706,
      "rewards/rejected": 0.006278430111706257,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.41315865106129e-09,
      "logits/chosen": -1.9302256107330322,
      "logits/rejected": -1.940580129623413,
      "logps/chosen": -32.2147102355957,
      "logps/rejected": -35.29616165161133,
      "loss": 0.6963,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.007009805645793676,
      "rewards/margins": -0.0018279661890119314,
      "rewards/rejected": -0.005181840155273676,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3150941078050324e-09,
      "logits/chosen": -2.069504737854004,
      "logits/rejected": -2.062945604324341,
      "logps/chosen": -33.646629333496094,
      "logps/rejected": -29.217721939086914,
      "loss": 0.6968,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.003734806552529335,
      "rewards/margins": -0.0023581120185554028,
      "rewards/rejected": 0.0060929167084395885,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.575864278703266e-10,
      "logits/chosen": -1.9288103580474854,
      "logits/rejected": -1.9309613704681396,
      "logps/chosen": -34.23542022705078,
      "logps/rejected": -30.89495277404785,
      "loss": 0.6866,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0025402153842151165,
      "rewards/margins": 0.018168382346630096,
      "rewards/rejected": -0.015628164634108543,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.6898305428492558,
      "train_runtime": 3256.6227,
      "train_samples_per_second": 0.945,
      "train_steps_per_second": 0.118
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}