{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 31.625,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": -2.7358343601226807,
      "logits/rejected": -2.7480404376983643,
      "logps/chosen": -27.35565757751465,
      "logps/rejected": -21.06114387512207,
      "loss": 1.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "grad_norm": 53.5,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": -3.0091097354888916,
      "logits/rejected": -2.998112678527832,
      "logps/chosen": -33.19804763793945,
      "logps/rejected": -31.97610092163086,
      "loss": 1.002,
      "rewards/accuracies": 0.4722222089767456,
      "rewards/chosen": -0.006841929163783789,
      "rewards/margins": -0.0019655984360724688,
      "rewards/rejected": -0.004876333754509687,
      "step": 10
    },
    {
      "epoch": 0.05,
      "grad_norm": 36.75,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": -2.899024248123169,
      "logits/rejected": -2.8939948081970215,
      "logps/chosen": -32.47606658935547,
      "logps/rejected": -28.96236228942871,
      "loss": 1.0082,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.0003868088242597878,
      "rewards/margins": -0.008188480511307716,
      "rewards/rejected": 0.007801672909408808,
      "step": 20
    },
    {
      "epoch": 0.08,
      "grad_norm": 35.25,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": -3.097607135772705,
      "logits/rejected": -3.109447956085205,
      "logps/chosen": -32.788970947265625,
      "logps/rejected": -30.134746551513672,
      "loss": 0.9766,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.047886691987514496,
      "rewards/margins": 0.023363908752799034,
      "rewards/rejected": 0.02452278696000576,
      "step": 30
    },
    {
      "epoch": 0.1,
      "grad_norm": 33.0,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": -2.8632471561431885,
      "logits/rejected": -2.853929042816162,
      "logps/chosen": -31.501651763916016,
      "logps/rejected": -32.37067413330078,
      "loss": 0.8747,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.1314682960510254,
      "rewards/margins": 0.13675658404827118,
      "rewards/rejected": -0.005288311745971441,
      "step": 40
    },
    {
      "epoch": 0.13,
      "grad_norm": 22.5,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": -2.881110191345215,
      "logits/rejected": -2.878807783126831,
      "logps/chosen": -29.374563217163086,
      "logps/rejected": -30.105758666992188,
      "loss": 0.886,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.1462380588054657,
      "rewards/margins": 0.15296456217765808,
      "rewards/rejected": -0.006726491264998913,
      "step": 50
    },
    {
      "epoch": 0.16,
      "grad_norm": 32.75,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": -2.9093446731567383,
      "logits/rejected": -2.9112706184387207,
      "logps/chosen": -29.828176498413086,
      "logps/rejected": -28.021154403686523,
      "loss": 0.8701,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.12063708156347275,
      "rewards/margins": 0.14682583510875702,
      "rewards/rejected": -0.026188766583800316,
      "step": 60
    },
    {
      "epoch": 0.18,
      "grad_norm": 43.5,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": -2.9932944774627686,
      "logits/rejected": -2.999823808670044,
      "logps/chosen": -29.177806854248047,
      "logps/rejected": -30.96217918395996,
      "loss": 0.9306,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.041265495121479034,
      "rewards/margins": 0.06941740214824677,
      "rewards/rejected": -0.028151914477348328,
      "step": 70
    },
    {
      "epoch": 0.21,
      "grad_norm": 37.75,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": -2.8079781532287598,
      "logits/rejected": -2.8230159282684326,
      "logps/chosen": -29.30885887145996,
      "logps/rejected": -29.878082275390625,
      "loss": 0.8332,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.09274748712778091,
      "rewards/margins": 0.1742672175168991,
      "rewards/rejected": -0.08151975274085999,
      "step": 80
    },
    {
      "epoch": 0.23,
      "grad_norm": 28.125,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": -2.897887945175171,
      "logits/rejected": -2.8805594444274902,
      "logps/chosen": -32.75906753540039,
      "logps/rejected": -30.377796173095703,
      "loss": 0.8621,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.030946413055062294,
      "rewards/margins": 0.17340537905693054,
      "rewards/rejected": -0.1424589455127716,
      "step": 90
    },
    {
      "epoch": 0.26,
      "grad_norm": 27.625,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": -3.0043857097625732,
      "logits/rejected": -3.005366086959839,
      "logps/chosen": -31.849411010742188,
      "logps/rejected": -31.039392471313477,
      "loss": 0.8436,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0829862430691719,
      "rewards/margins": 0.1722743958234787,
      "rewards/rejected": -0.08928816020488739,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.812446117401123,
      "eval_logits/rejected": -2.8096587657928467,
      "eval_logps/chosen": -31.28770637512207,
      "eval_logps/rejected": -34.89251708984375,
      "eval_loss": 0.9278174042701721,
      "eval_rewards/accuracies": 0.5984219312667847,
      "eval_rewards/chosen": -0.0021020672284066677,
      "eval_rewards/margins": 0.07564153522253036,
      "eval_rewards/rejected": -0.07774360477924347,
      "eval_runtime": 113.4025,
      "eval_samples_per_second": 3.025,
      "eval_steps_per_second": 0.379,
      "step": 100
    },
    {
      "epoch": 0.29,
      "grad_norm": 42.0,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": -2.961785078048706,
      "logits/rejected": -2.9373226165771484,
      "logps/chosen": -31.922677993774414,
      "logps/rejected": -31.55402183532715,
      "loss": 0.7753,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.11764738708734512,
      "rewards/margins": 0.2757166028022766,
      "rewards/rejected": -0.1580692082643509,
      "step": 110
    },
    {
      "epoch": 0.31,
      "grad_norm": 35.5,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": -3.0439839363098145,
      "logits/rejected": -3.073005199432373,
      "logps/chosen": -28.63894271850586,
      "logps/rejected": -34.55387878417969,
      "loss": 0.7242,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.20998422801494598,
      "rewards/margins": 0.372516393661499,
      "rewards/rejected": -0.16253218054771423,
      "step": 120
    },
    {
      "epoch": 0.34,
      "grad_norm": 33.75,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": -2.7482008934020996,
      "logits/rejected": -2.7447853088378906,
      "logps/chosen": -28.648107528686523,
      "logps/rejected": -30.5201358795166,
      "loss": 0.7756,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.13062894344329834,
      "rewards/margins": 0.2851604223251343,
      "rewards/rejected": -0.15453146398067474,
      "step": 130
    },
    {
      "epoch": 0.36,
      "grad_norm": 26.25,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": -3.0264618396759033,
      "logits/rejected": -3.023954391479492,
      "logps/chosen": -27.322336196899414,
      "logps/rejected": -32.06721496582031,
      "loss": 0.8404,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.06555580347776413,
      "rewards/margins": 0.22973506152629852,
      "rewards/rejected": -0.1641792207956314,
      "step": 140
    },
    {
      "epoch": 0.39,
      "grad_norm": 27.625,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": -2.823460817337036,
      "logits/rejected": -2.818603038787842,
      "logps/chosen": -27.322635650634766,
      "logps/rejected": -31.723133087158203,
      "loss": 0.711,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2011496126651764,
      "rewards/margins": 0.4009442925453186,
      "rewards/rejected": -0.1997946947813034,
      "step": 150
    },
    {
      "epoch": 0.42,
      "grad_norm": 40.25,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": -3.140930414199829,
      "logits/rejected": -3.123213291168213,
      "logps/chosen": -31.732044219970703,
      "logps/rejected": -29.67775535583496,
      "loss": 0.5884,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3038697838783264,
      "rewards/margins": 0.5830621123313904,
      "rewards/rejected": -0.27919238805770874,
      "step": 160
    },
    {
      "epoch": 0.44,
      "grad_norm": 33.5,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": -2.9524331092834473,
      "logits/rejected": -2.960047960281372,
      "logps/chosen": -29.4462947845459,
      "logps/rejected": -31.882787704467773,
      "loss": 0.6552,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.1793280392885208,
      "rewards/margins": 0.49264994263648987,
      "rewards/rejected": -0.31332188844680786,
      "step": 170
    },
    {
      "epoch": 0.47,
      "grad_norm": 28.375,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": -2.8057327270507812,
      "logits/rejected": -2.803337812423706,
      "logps/chosen": -29.322677612304688,
      "logps/rejected": -30.268985748291016,
      "loss": 0.7292,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.1388693004846573,
      "rewards/margins": 0.3391944468021393,
      "rewards/rejected": -0.2003251612186432,
      "step": 180
    },
    {
      "epoch": 0.49,
      "grad_norm": 23.625,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": -2.9184248447418213,
      "logits/rejected": -2.9148693084716797,
      "logps/chosen": -29.80621910095215,
      "logps/rejected": -28.81686782836914,
      "loss": 0.7753,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.1521972417831421,
      "rewards/margins": 0.33593088388442993,
      "rewards/rejected": -0.18373365700244904,
      "step": 190
    },
    {
      "epoch": 0.52,
      "grad_norm": 28.125,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": -2.985816478729248,
      "logits/rejected": -2.9742205142974854,
      "logps/chosen": -32.966896057128906,
      "logps/rejected": -30.653575897216797,
      "loss": 0.7459,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.29284805059432983,
      "rewards/margins": 0.38064295053482056,
      "rewards/rejected": -0.08779484033584595,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.825000762939453,
      "eval_logits/rejected": -2.8226914405822754,
      "eval_logps/chosen": -31.455589294433594,
      "eval_logps/rejected": -35.09489822387695,
      "eval_loss": 0.9159321784973145,
      "eval_rewards/accuracies": 0.5834717750549316,
      "eval_rewards/chosen": -0.06925402581691742,
      "eval_rewards/margins": 0.0894429013133049,
      "eval_rewards/rejected": -0.15869693458080292,
      "eval_runtime": 113.1231,
      "eval_samples_per_second": 3.032,
      "eval_steps_per_second": 0.38,
      "step": 200
    },
    {
      "epoch": 0.55,
      "grad_norm": 34.0,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": -2.9240100383758545,
      "logits/rejected": -2.9254937171936035,
      "logps/chosen": -32.46898651123047,
      "logps/rejected": -34.43748092651367,
      "loss": 0.6514,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.22576837241649628,
      "rewards/margins": 0.4561736583709717,
      "rewards/rejected": -0.2304052859544754,
      "step": 210
    },
    {
      "epoch": 0.57,
      "grad_norm": 32.25,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": -2.9049904346466064,
      "logits/rejected": -2.9205589294433594,
      "logps/chosen": -29.369049072265625,
      "logps/rejected": -29.028148651123047,
      "loss": 0.6606,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.3235965967178345,
      "rewards/margins": 0.523547351360321,
      "rewards/rejected": -0.19995081424713135,
      "step": 220
    },
    {
      "epoch": 0.6,
      "grad_norm": 28.75,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": -2.950502395629883,
      "logits/rejected": -2.9541735649108887,
      "logps/chosen": -30.678882598876953,
      "logps/rejected": -32.15251922607422,
      "loss": 0.7668,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.18919439613819122,
      "rewards/margins": 0.3469431698322296,
      "rewards/rejected": -0.1577487289905548,
      "step": 230
    },
    {
      "epoch": 0.62,
      "grad_norm": 34.0,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": -3.00547456741333,
      "logits/rejected": -3.013357400894165,
      "logps/chosen": -30.189453125,
      "logps/rejected": -30.650592803955078,
      "loss": 0.6128,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.2861887216567993,
      "rewards/margins": 0.4523293375968933,
      "rewards/rejected": -0.1661405861377716,
      "step": 240
    },
    {
      "epoch": 0.65,
      "grad_norm": 33.0,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": -2.838496208190918,
      "logits/rejected": -2.828334093093872,
      "logps/chosen": -26.336994171142578,
      "logps/rejected": -29.769458770751953,
      "loss": 0.7551,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.24688871204853058,
      "rewards/margins": 0.3420138955116272,
      "rewards/rejected": -0.09512520581483841,
      "step": 250
    },
    {
      "epoch": 0.68,
      "grad_norm": 29.625,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": -2.8198904991149902,
      "logits/rejected": -2.8399436473846436,
      "logps/chosen": -29.0089168548584,
      "logps/rejected": -34.85820388793945,
      "loss": 0.5813,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.32841864228248596,
      "rewards/margins": 0.6020644903182983,
      "rewards/rejected": -0.27364593744277954,
      "step": 260
    },
    {
      "epoch": 0.7,
      "grad_norm": 36.75,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": -2.962282657623291,
      "logits/rejected": -2.969045639038086,
      "logps/chosen": -30.215042114257812,
      "logps/rejected": -30.566543579101562,
      "loss": 0.5913,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.22652491927146912,
      "rewards/margins": 0.5070374608039856,
      "rewards/rejected": -0.2805125117301941,
      "step": 270
    },
    {
      "epoch": 0.73,
      "grad_norm": 29.5,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": -2.9766674041748047,
      "logits/rejected": -2.9624428749084473,
      "logps/chosen": -30.22548484802246,
      "logps/rejected": -28.989456176757812,
      "loss": 0.7584,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.19847916066646576,
      "rewards/margins": 0.34172323346138,
      "rewards/rejected": -0.14324405789375305,
      "step": 280
    },
    {
      "epoch": 0.75,
      "grad_norm": 30.0,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": -2.9045450687408447,
      "logits/rejected": -2.887028932571411,
      "logps/chosen": -31.592443466186523,
      "logps/rejected": -31.358936309814453,
      "loss": 0.4878,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.4175439774990082,
      "rewards/margins": 0.751793384552002,
      "rewards/rejected": -0.3342495858669281,
      "step": 290
    },
    {
      "epoch": 0.78,
      "grad_norm": 24.75,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": -2.9053683280944824,
      "logits/rejected": -2.9024596214294434,
      "logps/chosen": -31.347707748413086,
      "logps/rejected": -28.053564071655273,
      "loss": 0.665,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.34997984766960144,
      "rewards/margins": 0.529971182346344,
      "rewards/rejected": -0.17999130487442017,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.8241891860961914,
      "eval_logits/rejected": -2.8221466541290283,
      "eval_logps/chosen": -31.476619720458984,
      "eval_logps/rejected": -35.105430603027344,
      "eval_loss": 0.9201633334159851,
      "eval_rewards/accuracies": 0.565614640712738,
      "eval_rewards/chosen": -0.07766681164503098,
      "eval_rewards/margins": 0.08524217456579208,
      "eval_rewards/rejected": -0.16290898621082306,
      "eval_runtime": 113.1247,
      "eval_samples_per_second": 3.032,
      "eval_steps_per_second": 0.38,
      "step": 300
    },
    {
      "epoch": 0.81,
      "grad_norm": 32.5,
      "learning_rate": 5.576113578589035e-07,
      "logits/chosen": -2.7875466346740723,
      "logits/rejected": -2.8046317100524902,
      "logps/chosen": -28.552978515625,
      "logps/rejected": -31.452463150024414,
      "loss": 0.5777,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.27057594060897827,
      "rewards/margins": 0.5587579011917114,
      "rewards/rejected": -0.28818196058273315,
      "step": 310
    },
    {
      "epoch": 0.83,
      "grad_norm": 26.75,
      "learning_rate": 4.229036944380913e-07,
      "logits/chosen": -3.0314865112304688,
      "logits/rejected": -3.0174806118011475,
      "logps/chosen": -29.399667739868164,
      "logps/rejected": -28.924419403076172,
      "loss": 0.5701,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.32476717233657837,
      "rewards/margins": 0.567025363445282,
      "rewards/rejected": -0.24225828051567078,
      "step": 320
    },
    {
      "epoch": 0.86,
      "grad_norm": 18.25,
      "learning_rate": 3.053082288996112e-07,
      "logits/chosen": -2.946519374847412,
      "logits/rejected": -2.9290707111358643,
      "logps/chosen": -27.334081649780273,
      "logps/rejected": -31.206268310546875,
      "loss": 0.4948,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.4111965298652649,
      "rewards/margins": 0.7370456457138062,
      "rewards/rejected": -0.32584911584854126,
      "step": 330
    },
    {
      "epoch": 0.88,
      "grad_norm": 35.0,
      "learning_rate": 2.0579377374915805e-07,
      "logits/chosen": -3.16451096534729,
      "logits/rejected": -3.1703600883483887,
      "logps/chosen": -30.927570343017578,
      "logps/rejected": -33.580810546875,
      "loss": 0.5586,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.2846537232398987,
      "rewards/margins": 0.6385756731033325,
      "rewards/rejected": -0.35392194986343384,
      "step": 340
    },
    {
      "epoch": 0.91,
      "grad_norm": 23.875,
      "learning_rate": 1.2518018074041684e-07,
      "logits/chosen": -3.040112018585205,
      "logits/rejected": -3.0432353019714355,
      "logps/chosen": -29.85060691833496,
      "logps/rejected": -32.070159912109375,
      "loss": 0.5751,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.3825104236602783,
      "rewards/margins": 0.6052514314651489,
      "rewards/rejected": -0.22274093329906464,
      "step": 350
    },
    {
      "epoch": 0.94,
      "grad_norm": 35.0,
      "learning_rate": 6.41315865106129e-08,
      "logits/chosen": -2.8801891803741455,
      "logits/rejected": -2.8814785480499268,
      "logps/chosen": -27.602718353271484,
      "logps/rejected": -30.264202117919922,
      "loss": 0.5555,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.41880473494529724,
      "rewards/margins": 0.5921100974082947,
      "rewards/rejected": -0.17330540716648102,
      "step": 360
    },
    {
      "epoch": 0.96,
      "grad_norm": 41.0,
      "learning_rate": 2.3150941078050325e-08,
      "logits/chosen": -2.9562087059020996,
      "logits/rejected": -2.9534800052642822,
      "logps/chosen": -29.909082412719727,
      "logps/rejected": -32.4835205078125,
      "loss": 0.5884,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.3510281443595886,
      "rewards/margins": 0.5732189416885376,
      "rewards/rejected": -0.22219082713127136,
      "step": 370
    },
    {
      "epoch": 0.99,
      "grad_norm": 30.75,
      "learning_rate": 2.575864278703266e-09,
      "logits/chosen": -2.9113378524780273,
      "logits/rejected": -2.894592523574829,
      "logps/chosen": -28.256174087524414,
      "logps/rejected": -28.505691528320312,
      "loss": 0.5793,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.275543749332428,
      "rewards/margins": 0.521973192691803,
      "rewards/rejected": -0.24642948806285858,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.7203964704042906,
      "train_runtime": 2720.5927,
      "train_samples_per_second": 1.132,
      "train_steps_per_second": 0.142
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}