diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,4267 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9999018549416037,
+  "eval_steps": 100,
+  "global_step": 2547,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.000392580233585239,
+      "grad_norm": 0.7404606342315674,
+      "learning_rate": 1.9607843137254902e-08,
+      "logits/chosen": -2.3771963119506836,
+      "logits/rejected": -2.050717353820801,
+      "logps/chosen": -501.02435302734375,
+      "logps/rejected": -350.531494140625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.00392580233585239,
+      "grad_norm": 0.7286408543586731,
+      "learning_rate": 1.9607843137254904e-07,
+      "logits/chosen": -2.0385897159576416,
+      "logits/rejected": -2.0307867527008057,
+      "logps/chosen": -340.7717590332031,
+      "logps/rejected": -311.61895751953125,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4305555522441864,
+      "rewards/chosen": -0.0002206952776759863,
+      "rewards/margins": -0.00035487255081534386,
+      "rewards/rejected": 0.00013417721493169665,
+      "step": 10
+    },
+    {
+      "epoch": 0.00785160467170478,
+      "grad_norm": 0.7240912914276123,
+      "learning_rate": 3.921568627450981e-07,
+      "logits/chosen": -1.9975738525390625,
+      "logits/rejected": -1.930702805519104,
+      "logps/chosen": -323.7059631347656,
+      "logps/rejected": -305.4819030761719,
+      "loss": 0.6935,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0005853104521520436,
+      "rewards/margins": -0.00014178038691170514,
+      "rewards/rejected": 0.0007270908099599183,
+      "step": 20
+    },
+    {
+      "epoch": 0.01177740700755717,
+      "grad_norm": 0.6032301783561707,
+      "learning_rate": 5.882352941176471e-07,
+      "logits/chosen": -1.8970115184783936,
+      "logits/rejected": -1.8781898021697998,
+      "logps/chosen": -297.4019470214844,
+      "logps/rejected": -315.6284484863281,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.002337783109396696,
+      "rewards/margins": -6.183599907672033e-05,
+      "rewards/rejected": 0.0023996192030608654,
+      "step": 30
+    },
+    {
+      "epoch": 0.01570320934340956,
+      "grad_norm": 0.6545844078063965,
+      "learning_rate": 7.843137254901962e-07,
+      "logits/chosen": -2.0507616996765137,
+      "logits/rejected": -1.9800456762313843,
+      "logps/chosen": -360.62908935546875,
+      "logps/rejected": -325.87310791015625,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0027643158100545406,
+      "rewards/margins": 0.0012084465706720948,
+      "rewards/rejected": 0.0015558695886284113,
+      "step": 40
+    },
+    {
+      "epoch": 0.01962901167926195,
+      "grad_norm": 0.787367045879364,
+      "learning_rate": 9.80392156862745e-07,
+      "logits/chosen": -2.029198169708252,
+      "logits/rejected": -1.9147459268569946,
+      "logps/chosen": -416.381591796875,
+      "logps/rejected": -357.2953186035156,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.004883971530944109,
+      "rewards/margins": 0.002786849159747362,
+      "rewards/rejected": 0.002097122836858034,
+      "step": 50
+    },
+    {
+      "epoch": 0.02355481401511434,
+      "grad_norm": 0.6833757162094116,
+      "learning_rate": 1.1764705882352942e-06,
+      "logits/chosen": -1.9757764339447021,
+      "logits/rejected": -1.8583787679672241,
+      "logps/chosen": -327.52886962890625,
+      "logps/rejected": -303.41583251953125,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.006807367317378521,
+      "rewards/margins": 0.004317972343415022,
+      "rewards/rejected": 0.002489394275471568,
+      "step": 60
+    },
+    {
+      "epoch": 0.027480616350966727,
+      "grad_norm": 0.7148961424827576,
+      "learning_rate": 1.3725490196078434e-06,
+      "logits/chosen": -1.9287532567977905,
+      "logits/rejected": -1.8990862369537354,
+      "logps/chosen": -443.18682861328125,
+      "logps/rejected": -342.8143005371094,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.010740823112428188,
+      "rewards/margins": 0.007196782622486353,
+      "rewards/rejected": 0.0035440411884337664,
+      "step": 70
+    },
+    {
+      "epoch": 0.03140641868681912,
+      "grad_norm": 0.702590823173523,
+      "learning_rate": 1.5686274509803923e-06,
+      "logits/chosen": -1.9715759754180908,
+      "logits/rejected": -1.9125280380249023,
+      "logps/chosen": -369.59326171875,
+      "logps/rejected": -292.1513366699219,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.014338649809360504,
+      "rewards/margins": 0.010924512520432472,
+      "rewards/rejected": 0.003414137288928032,
+      "step": 80
+    },
+    {
+      "epoch": 0.03533222102267151,
+      "grad_norm": 0.6938802599906921,
+      "learning_rate": 1.7647058823529414e-06,
+      "logits/chosen": -1.9071309566497803,
+      "logits/rejected": -1.8129125833511353,
+      "logps/chosen": -382.6009826660156,
+      "logps/rejected": -336.3795471191406,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.02006966806948185,
+      "rewards/margins": 0.009984933771193027,
+      "rewards/rejected": 0.010084734298288822,
+      "step": 90
+    },
+    {
+      "epoch": 0.0392580233585239,
+      "grad_norm": 0.7240464091300964,
+      "learning_rate": 1.96078431372549e-06,
+      "logits/chosen": -1.9107997417449951,
+      "logits/rejected": -1.8976436853408813,
+      "logps/chosen": -371.04547119140625,
+      "logps/rejected": -302.9181823730469,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03604495897889137,
+      "rewards/margins": 0.015005680732429028,
+      "rewards/rejected": 0.02103927731513977,
+      "step": 100
+    },
+    {
+      "epoch": 0.0392580233585239,
+      "eval_logits/chosen": -1.9903417825698853,
+      "eval_logits/rejected": -1.9288501739501953,
+      "eval_logps/chosen": -343.2012634277344,
+      "eval_logps/rejected": -301.8898620605469,
+      "eval_loss": 0.6861926913261414,
+      "eval_rewards/accuracies": 0.658682644367218,
+      "eval_rewards/chosen": 0.04213272035121918,
+      "eval_rewards/margins": 0.014640606939792633,
+      "eval_rewards/rejected": 0.027492111548781395,
+      "eval_runtime": 297.7911,
+      "eval_samples_per_second": 6.716,
+      "eval_steps_per_second": 1.122,
+      "step": 100
+    },
+    {
+      "epoch": 0.04318382569437629,
+      "grad_norm": 0.5929064154624939,
+      "learning_rate": 2.1568627450980393e-06,
+      "logits/chosen": -1.8939924240112305,
+      "logits/rejected": -1.8193858861923218,
+      "logps/chosen": -303.50201416015625,
+      "logps/rejected": -315.70758056640625,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.039821166545152664,
+      "rewards/margins": 0.009396327659487724,
+      "rewards/rejected": 0.03042483702301979,
+      "step": 110
+    },
+    {
+      "epoch": 0.04710962803022868,
+      "grad_norm": 0.6487820744514465,
+      "learning_rate": 2.3529411764705885e-06,
+      "logits/chosen": -1.9013248682022095,
+      "logits/rejected": -1.8881601095199585,
+      "logps/chosen": -309.12939453125,
+      "logps/rejected": -280.3207092285156,
+      "loss": 0.687,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.0327104851603508,
+      "rewards/margins": 0.015539437532424927,
+      "rewards/rejected": 0.017171043902635574,
+      "step": 120
+    },
+    {
+      "epoch": 0.05103543036608107,
+      "grad_norm": 0.7229335904121399,
+      "learning_rate": 2.549019607843137e-06,
+      "logits/chosen": -1.964864730834961,
+      "logits/rejected": -1.8627372980117798,
+      "logps/chosen": -335.9957580566406,
+      "logps/rejected": -308.3912658691406,
+      "loss": 0.681,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0357685461640358,
+      "rewards/margins": 0.02001311257481575,
+      "rewards/rejected": 0.015755433589220047,
+      "step": 130
+    },
+    {
+      "epoch": 0.054961232701933455,
+      "grad_norm": 0.7636377811431885,
+      "learning_rate": 2.7450980392156867e-06,
+      "logits/chosen": -1.9330470561981201,
+      "logits/rejected": -1.9024755954742432,
+      "logps/chosen": -357.3898620605469,
+      "logps/rejected": -290.8785400390625,
+      "loss": 0.676,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.03444816544651985,
+      "rewards/margins": 0.04734820872545242,
+      "rewards/rejected": -0.012900042347609997,
+      "step": 140
+    },
+    {
+      "epoch": 0.05888703503778585,
+      "grad_norm": 0.7752945423126221,
+      "learning_rate": 2.9411764705882355e-06,
+      "logits/chosen": -1.9442981481552124,
+      "logits/rejected": -1.905564308166504,
+      "logps/chosen": -325.9679260253906,
+      "logps/rejected": -301.0154113769531,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.027961844578385353,
+      "rewards/margins": 0.03049735352396965,
+      "rewards/rejected": -0.0025355082470923662,
+      "step": 150
+    },
+    {
+      "epoch": 0.06281283737363824,
+      "grad_norm": 0.7767668962478638,
+      "learning_rate": 3.1372549019607846e-06,
+      "logits/chosen": -1.9174308776855469,
+      "logits/rejected": -1.8516910076141357,
+      "logps/chosen": -411.96575927734375,
+      "logps/rejected": -349.1806640625,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.020252307876944542,
+      "rewards/margins": 0.0418366976082325,
+      "rewards/rejected": -0.021584395319223404,
+      "step": 160
+    },
+    {
+      "epoch": 0.06673863970949062,
+      "grad_norm": 1.8108171224594116,
+      "learning_rate": 3.3333333333333333e-06,
+      "logits/chosen": -2.028966188430786,
+      "logits/rejected": -1.9510780572891235,
+      "logps/chosen": -346.4346618652344,
+      "logps/rejected": -315.243896484375,
+      "loss": 0.6711,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.014916600659489632,
+      "rewards/margins": 0.0593319833278656,
+      "rewards/rejected": -0.07424857467412949,
+      "step": 170
+    },
+    {
+      "epoch": 0.07066444204534301,
+      "grad_norm": 1.4104039669036865,
+      "learning_rate": 3.529411764705883e-06,
+      "logits/chosen": -1.9167848825454712,
+      "logits/rejected": -1.8759437799453735,
+      "logps/chosen": -358.4148864746094,
+      "logps/rejected": -279.017578125,
+      "loss": 0.6605,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.033157967031002045,
+      "rewards/margins": 0.08327115327119827,
+      "rewards/rejected": -0.05011318251490593,
+      "step": 180
+    },
+    {
+      "epoch": 0.07459024438119541,
+      "grad_norm": 1.1098579168319702,
+      "learning_rate": 3.7254901960784316e-06,
+      "logits/chosen": -1.9406001567840576,
+      "logits/rejected": -1.8640397787094116,
+      "logps/chosen": -304.02008056640625,
+      "logps/rejected": -283.79364013671875,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.020353609696030617,
+      "rewards/margins": 0.05034954100847244,
+      "rewards/rejected": -0.029995933175086975,
+      "step": 190
+    },
+    {
+      "epoch": 0.0785160467170478,
+      "grad_norm": 2.6454787254333496,
+      "learning_rate": 3.92156862745098e-06,
+      "logits/chosen": -1.9150803089141846,
+      "logits/rejected": -1.79108464717865,
+      "logps/chosen": -324.07611083984375,
+      "logps/rejected": -283.6548156738281,
+      "loss": 0.6613,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.061957359313964844,
+      "rewards/margins": 0.09046927094459534,
+      "rewards/rejected": -0.15242663025856018,
+      "step": 200
+    },
+    {
+      "epoch": 0.0785160467170478,
+      "eval_logits/chosen": -1.9594751596450806,
+      "eval_logits/rejected": -1.901289701461792,
+      "eval_logps/chosen": -352.16546630859375,
+      "eval_logps/rejected": -318.214599609375,
+      "eval_loss": 0.6586626172065735,
+      "eval_rewards/accuracies": 0.6706587076187134,
+      "eval_rewards/chosen": -0.04750930890440941,
+      "eval_rewards/margins": 0.0882464051246643,
+      "eval_rewards/rejected": -0.135755717754364,
+      "eval_runtime": 295.0464,
+      "eval_samples_per_second": 6.779,
+      "eval_steps_per_second": 1.132,
+      "step": 200
+    },
+    {
+      "epoch": 0.08244184905290018,
+      "grad_norm": 1.3472086191177368,
+      "learning_rate": 4.11764705882353e-06,
+      "logits/chosen": -1.9669873714447021,
+      "logits/rejected": -1.8843181133270264,
+      "logps/chosen": -418.80450439453125,
+      "logps/rejected": -333.00909423828125,
+      "loss": 0.6513,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.08746848255395889,
+      "rewards/margins": 0.12608177959918976,
+      "rewards/rejected": -0.21355025470256805,
+      "step": 210
+    },
+    {
+      "epoch": 0.08636765138875258,
+      "grad_norm": 1.0999418497085571,
+      "learning_rate": 4.313725490196079e-06,
+      "logits/chosen": -1.9091193675994873,
+      "logits/rejected": -1.8638063669204712,
+      "logps/chosen": -326.2521667480469,
+      "logps/rejected": -333.1636047363281,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.23567886650562286,
+      "rewards/margins": 0.06013675406575203,
+      "rewards/rejected": -0.295815646648407,
+      "step": 220
+    },
+    {
+      "epoch": 0.09029345372460497,
+      "grad_norm": 1.2177945375442505,
+      "learning_rate": 4.509803921568628e-06,
+      "logits/chosen": -1.8844512701034546,
+      "logits/rejected": -1.9035418033599854,
+      "logps/chosen": -388.63201904296875,
+      "logps/rejected": -347.8082275390625,
+      "loss": 0.6519,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.18088065087795258,
+      "rewards/margins": 0.07500191032886505,
+      "rewards/rejected": -0.2558825612068176,
+      "step": 230
+    },
+    {
+      "epoch": 0.09421925606045736,
+      "grad_norm": 2.058034658432007,
+      "learning_rate": 4.705882352941177e-06,
+      "logits/chosen": -1.9320697784423828,
+      "logits/rejected": -1.8659961223602295,
+      "logps/chosen": -335.31964111328125,
+      "logps/rejected": -339.7356262207031,
+      "loss": 0.6618,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.05537889152765274,
+      "rewards/margins": 0.07215271145105362,
+      "rewards/rejected": -0.12753158807754517,
+      "step": 240
+    },
+    {
+      "epoch": 0.09814505839630974,
+      "grad_norm": 1.5527970790863037,
+      "learning_rate": 4.901960784313726e-06,
+      "logits/chosen": -1.9350112676620483,
+      "logits/rejected": -1.8908573389053345,
+      "logps/chosen": -409.8229064941406,
+      "logps/rejected": -358.0290222167969,
+      "loss": 0.6357,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.07777851819992065,
+      "rewards/margins": 0.1180642619729042,
+      "rewards/rejected": -0.19584277272224426,
+      "step": 250
+    },
+    {
+      "epoch": 0.10207086073216214,
+      "grad_norm": 1.945937991142273,
+      "learning_rate": 4.999941289086112e-06,
+      "logits/chosen": -1.96586012840271,
+      "logits/rejected": -1.8799537420272827,
+      "logps/chosen": -365.5788879394531,
+      "logps/rejected": -333.016845703125,
+      "loss": 0.6147,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.07585909962654114,
+      "rewards/margins": 0.22610175609588623,
+      "rewards/rejected": -0.30196088552474976,
+      "step": 260
+    },
+    {
+      "epoch": 0.10599666306801453,
+      "grad_norm": 2.1760644912719727,
+      "learning_rate": 4.999471618320339e-06,
+      "logits/chosen": -1.952376365661621,
+      "logits/rejected": -1.9244153499603271,
+      "logps/chosen": -322.3785095214844,
+      "logps/rejected": -295.06182861328125,
+      "loss": 0.6573,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08501241356134415,
+      "rewards/margins": 0.05953482538461685,
+      "rewards/rejected": -0.1445472538471222,
+      "step": 270
+    },
+    {
+      "epoch": 0.10992246540386691,
+      "grad_norm": 2.6156668663024902,
+      "learning_rate": 4.998532365027117e-06,
+      "logits/chosen": -1.8586006164550781,
+      "logits/rejected": -1.8407413959503174,
+      "logps/chosen": -370.5046691894531,
+      "logps/rejected": -317.41546630859375,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.26402169466018677,
+      "rewards/margins": 0.09567264467477798,
+      "rewards/rejected": -0.35969430208206177,
+      "step": 280
+    },
+    {
+      "epoch": 0.1138482677397193,
+      "grad_norm": 3.405456781387329,
+      "learning_rate": 4.997123705666514e-06,
+      "logits/chosen": -1.781801462173462,
+      "logits/rejected": -1.7918872833251953,
+      "logps/chosen": -408.0342102050781,
+      "logps/rejected": -392.877197265625,
+      "loss": 0.6465,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3006923794746399,
+      "rewards/margins": 0.13740524649620056,
+      "rewards/rejected": -0.43809765577316284,
+      "step": 290
+    },
+    {
+      "epoch": 0.1177740700755717,
+      "grad_norm": 2.0584757328033447,
+      "learning_rate": 4.995245904887195e-06,
+      "logits/chosen": -1.8385387659072876,
+      "logits/rejected": -1.8264141082763672,
+      "logps/chosen": -394.44403076171875,
+      "logps/rejected": -386.1804504394531,
+      "loss": 0.6358,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.11758632957935333,
+      "rewards/margins": 0.15193092823028564,
+      "rewards/rejected": -0.26951727271080017,
+      "step": 300
+    },
+    {
+      "epoch": 0.1177740700755717,
+      "eval_logits/chosen": -1.8743999004364014,
+      "eval_logits/rejected": -1.8153735399246216,
+      "eval_logps/chosen": -371.9539489746094,
+      "eval_logps/rejected": -344.5503234863281,
+      "eval_loss": 0.6395125389099121,
+      "eval_rewards/accuracies": 0.6871257424354553,
+      "eval_rewards/chosen": -0.2453942596912384,
+      "eval_rewards/margins": 0.15371853113174438,
+      "eval_rewards/rejected": -0.3991128206253052,
+      "eval_runtime": 294.7104,
+      "eval_samples_per_second": 6.786,
+      "eval_steps_per_second": 1.133,
+      "step": 300
+    },
+    {
+      "epoch": 0.12169987241142409,
+      "grad_norm": 1.7917004823684692,
+      "learning_rate": 4.992899315476696e-06,
+      "logits/chosen": -1.8489936590194702,
+      "logits/rejected": -1.803655982017517,
+      "logps/chosen": -342.4612121582031,
+      "logps/rejected": -312.39984130859375,
+      "loss": 0.6338,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3183877766132355,
+      "rewards/margins": 0.19239942729473114,
+      "rewards/rejected": -0.5107872486114502,
+      "step": 310
+    },
+    {
+      "epoch": 0.12562567474727648,
+      "grad_norm": 2.1951587200164795,
+      "learning_rate": 4.990084378295148e-06,
+      "logits/chosen": -1.8730294704437256,
+      "logits/rejected": -1.8302888870239258,
+      "logps/chosen": -368.7808837890625,
+      "logps/rejected": -365.14361572265625,
+      "loss": 0.6392,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.4258648753166199,
+      "rewards/margins": 0.14107713103294373,
+      "rewards/rejected": -0.5669419765472412,
+      "step": 320
+    },
+    {
+      "epoch": 0.12955147708312886,
+      "grad_norm": 1.792680263519287,
+      "learning_rate": 4.986801622192453e-06,
+      "logits/chosen": -1.8786293268203735,
+      "logits/rejected": -1.7592655420303345,
+      "logps/chosen": -414.01483154296875,
+      "logps/rejected": -352.684326171875,
+      "loss": 0.6222,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.34100285172462463,
+      "rewards/margins": 0.18358322978019714,
+      "rewards/rejected": -0.5245860815048218,
+      "step": 330
+    },
+    {
+      "epoch": 0.13347727941898124,
+      "grad_norm": 1.965647578239441,
+      "learning_rate": 4.9830516639089226e-06,
+      "logits/chosen": -1.8507169485092163,
+      "logits/rejected": -1.8387380838394165,
+      "logps/chosen": -405.011474609375,
+      "logps/rejected": -370.067138671875,
+      "loss": 0.614,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.3230993151664734,
+      "rewards/margins": 0.21295097470283508,
+      "rewards/rejected": -0.5360503196716309,
+      "step": 340
+    },
+    {
+      "epoch": 0.13740308175483365,
+      "grad_norm": 2.2121853828430176,
+      "learning_rate": 4.978835207959414e-06,
+      "logits/chosen": -1.8559701442718506,
+      "logits/rejected": -1.874185562133789,
+      "logps/chosen": -412.4024353027344,
+      "logps/rejected": -412.3687438964844,
+      "loss": 0.6539,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.3071586489677429,
+      "rewards/margins": 0.1814052164554596,
+      "rewards/rejected": -0.4885639250278473,
+      "step": 350
+    },
+    {
+      "epoch": 0.14132888409068603,
+      "grad_norm": 2.0039916038513184,
+      "learning_rate": 4.9741530465009665e-06,
+      "logits/chosen": -1.83514404296875,
+      "logits/rejected": -1.764387845993042,
+      "logps/chosen": -404.0180969238281,
+      "logps/rejected": -377.49688720703125,
+      "loss": 0.5782,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.4407728612422943,
+      "rewards/margins": 0.3137550354003906,
+      "rewards/rejected": -0.7545278668403625,
+      "step": 360
+    },
+    {
+      "epoch": 0.14525468642653844,
+      "grad_norm": 1.9982868432998657,
+      "learning_rate": 4.969006059183984e-06,
+      "logits/chosen": -1.7599804401397705,
+      "logits/rejected": -1.707847237586975,
+      "logps/chosen": -369.6714782714844,
+      "logps/rejected": -368.0159912109375,
+      "loss": 0.6501,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.37968629598617554,
+      "rewards/margins": 0.16202610731124878,
+      "rewards/rejected": -0.5417124032974243,
+      "step": 370
+    },
+    {
+      "epoch": 0.14918048876239082,
+      "grad_norm": 2.460099220275879,
+      "learning_rate": 4.963395212986964e-06,
+      "logits/chosen": -1.8493001461029053,
+      "logits/rejected": -1.7587206363677979,
+      "logps/chosen": -465.4515075683594,
+      "logps/rejected": -376.24359130859375,
+      "loss": 0.6398,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3220883309841156,
+      "rewards/margins": 0.31719639897346497,
+      "rewards/rejected": -0.6392847895622253,
+      "step": 380
+    },
+    {
+      "epoch": 0.1531062910982432,
+      "grad_norm": 2.4347827434539795,
+      "learning_rate": 4.957321562034833e-06,
+      "logits/chosen": -1.8540918827056885,
+      "logits/rejected": -1.8075182437896729,
+      "logps/chosen": -419.6316833496094,
+      "logps/rejected": -392.05084228515625,
+      "loss": 0.625,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.44284310936927795,
+      "rewards/margins": 0.26629263162612915,
+      "rewards/rejected": -0.7091357111930847,
+      "step": 390
+    },
+    {
+      "epoch": 0.1570320934340956,
+      "grad_norm": 2.085526466369629,
+      "learning_rate": 4.950786247400908e-06,
+      "logits/chosen": -1.753591537475586,
+      "logits/rejected": -1.7399803400039673,
+      "logps/chosen": -380.83868408203125,
+      "logps/rejected": -356.3904724121094,
+      "loss": 0.6277,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.4947834610939026,
+      "rewards/margins": 0.25253739953041077,
+      "rewards/rejected": -0.7473207712173462,
+      "step": 400
+    },
+    {
+      "epoch": 0.1570320934340956,
+      "eval_logits/chosen": -1.8698549270629883,
+      "eval_logits/rejected": -1.81108558177948,
+      "eval_logps/chosen": -399.4671936035156,
+      "eval_logps/rejected": -378.91021728515625,
+      "eval_loss": 0.6237406730651855,
+      "eval_rewards/accuracies": 0.697604775428772,
+      "eval_rewards/chosen": -0.5205269455909729,
+      "eval_rewards/margins": 0.22218464314937592,
+      "eval_rewards/rejected": -0.74271160364151,
+      "eval_runtime": 296.0017,
+      "eval_samples_per_second": 6.757,
+      "eval_steps_per_second": 1.128,
+      "step": 400
+    },
+    {
+      "epoch": 0.16095789576994798,
+      "grad_norm": 3.25508975982666,
+      "learning_rate": 4.943790496892513e-06,
+      "logits/chosen": -1.8466389179229736,
+      "logits/rejected": -1.7645518779754639,
+      "logps/chosen": -378.880859375,
+      "logps/rejected": -356.48114013671875,
+      "loss": 0.6234,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.44048675894737244,
+      "rewards/margins": 0.1955534666776657,
+      "rewards/rejected": -0.6360402703285217,
+      "step": 410
+    },
+    {
+      "epoch": 0.16488369810580036,
+      "grad_norm": 2.1383957862854004,
+      "learning_rate": 4.936335624820313e-06,
+      "logits/chosen": -1.8428977727890015,
+      "logits/rejected": -1.751543641090393,
+      "logps/chosen": -351.134521484375,
+      "logps/rejected": -298.6847839355469,
+      "loss": 0.5932,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.23920945823192596,
+      "rewards/margins": 0.2866617441177368,
+      "rewards/rejected": -0.525871217250824,
+      "step": 420
+    },
+    {
+      "epoch": 0.16880950044165277,
+      "grad_norm": 4.538311958312988,
+      "learning_rate": 4.9284230317513906e-06,
+      "logits/chosen": -1.7963340282440186,
+      "logits/rejected": -1.7333629131317139,
+      "logps/chosen": -382.993896484375,
+      "logps/rejected": -394.06365966796875,
+      "loss": 0.6085,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.48227494955062866,
+      "rewards/margins": 0.30000603199005127,
+      "rewards/rejected": -0.7822809219360352,
+      "step": 430
+    },
+    {
+      "epoch": 0.17273530277750515,
+      "grad_norm": 2.0562307834625244,
+      "learning_rate": 4.920054204246116e-06,
+      "logits/chosen": -1.901017189025879,
+      "logits/rejected": -1.7471482753753662,
+      "logps/chosen": -394.3864440917969,
+      "logps/rejected": -361.3542785644531,
+      "loss": 0.619,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.45573440194129944,
+      "rewards/margins": 0.25341400504112244,
+      "rewards/rejected": -0.7091482877731323,
+      "step": 440
+    },
+    {
+      "epoch": 0.17666110511335753,
+      "grad_norm": 5.015087604522705,
+      "learning_rate": 4.911230714578858e-06,
+      "logits/chosen": -1.8501704931259155,
+      "logits/rejected": -1.7716137170791626,
+      "logps/chosen": -360.9024658203125,
+      "logps/rejected": -365.27008056640625,
+      "loss": 0.6084,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.7083210945129395,
+      "rewards/margins": 0.29572638869285583,
+      "rewards/rejected": -1.0040475130081177,
+      "step": 450
+    },
+    {
+      "epoch": 0.18058690744920994,
+      "grad_norm": 2.6905391216278076,
+      "learning_rate": 4.90195422044261e-06,
+      "logits/chosen": -1.7778104543685913,
+      "logits/rejected": -1.7667028903961182,
+      "logps/chosen": -383.0188293457031,
+      "logps/rejected": -387.96551513671875,
+      "loss": 0.5636,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.7167367339134216,
+      "rewards/margins": 0.3017016649246216,
+      "rewards/rejected": -1.018438458442688,
+      "step": 460
+    },
+    {
+      "epoch": 0.18451270978506232,
+      "grad_norm": 3.832167625427246,
+      "learning_rate": 4.89222646463754e-06,
+      "logits/chosen": -1.8416099548339844,
+      "logits/rejected": -1.8041002750396729,
+      "logps/chosen": -418.85076904296875,
+      "logps/rejected": -434.2696838378906,
+      "loss": 0.6116,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.687454342842102,
+      "rewards/margins": 0.35261327028274536,
+      "rewards/rejected": -1.0400676727294922,
+      "step": 470
+    },
+    {
+      "epoch": 0.18843851212091473,
+      "grad_norm": 3.0037221908569336,
+      "learning_rate": 4.882049274743578e-06,
+      "logits/chosen": -1.8012994527816772,
+      "logits/rejected": -1.7601085901260376,
+      "logps/chosen": -409.33221435546875,
+      "logps/rejected": -424.42108154296875,
+      "loss": 0.6101,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5494194626808167,
+      "rewards/margins": 0.33693990111351013,
+      "rewards/rejected": -0.8863593339920044,
+      "step": 480
+    },
+    {
+      "epoch": 0.1923643144567671,
+      "grad_norm": 2.0838212966918945,
+      "learning_rate": 4.8714245627770515e-06,
+      "logits/chosen": -1.9366506338119507,
+      "logits/rejected": -1.7997667789459229,
+      "logps/chosen": -434.9437561035156,
+      "logps/rejected": -394.2633361816406,
+      "loss": 0.5896,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.6098533272743225,
+      "rewards/margins": 0.47047552466392517,
+      "rewards/rejected": -1.0803288221359253,
+      "step": 490
+    },
+    {
+      "epoch": 0.19629011679261948,
+      "grad_norm": 2.7533013820648193,
+      "learning_rate": 4.860354324831482e-06,
+      "logits/chosen": -1.7913497686386108,
+      "logits/rejected": -1.7294028997421265,
+      "logps/chosen": -349.739013671875,
+      "logps/rejected": -334.67132568359375,
+      "loss": 0.5933,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.5735246539115906,
+      "rewards/margins": 0.30554503202438354,
+      "rewards/rejected": -0.8790696859359741,
+      "step": 500
+    },
+    {
+      "epoch": 0.19629011679261948,
+      "eval_logits/chosen": -1.8287168741226196,
+      "eval_logits/rejected": -1.7721105813980103,
+      "eval_logps/chosen": -417.0301208496094,
+      "eval_logps/rejected": -408.3534240722656,
+      "eval_loss": 0.6017878651618958,
+      "eval_rewards/accuracies": 0.6931137442588806,
+      "eval_rewards/chosen": -0.6961557269096375,
+      "eval_rewards/margins": 0.3409880995750427,
+      "eval_rewards/rejected": -1.0371438264846802,
+      "eval_runtime": 295.7116,
+      "eval_samples_per_second": 6.763,
+      "eval_steps_per_second": 1.129,
+      "step": 500
+    },
+    {
+      "epoch": 0.2002159191284719,
+      "grad_norm": 2.4620580673217773,
+      "learning_rate": 4.848840640702565e-06,
+      "logits/chosen": -1.8078361749649048,
+      "logits/rejected": -1.765496850013733,
+      "logps/chosen": -361.7633056640625,
+      "logps/rejected": -421.6953125,
+      "loss": 0.5983,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5555116534233093,
+      "rewards/margins": 0.3170231282711029,
+      "rewards/rejected": -0.8725348711013794,
+      "step": 510
+    },
+    {
+      "epoch": 0.20414172146432427,
+      "grad_norm": 2.9025681018829346,
+      "learning_rate": 4.836885673497435e-06,
+      "logits/chosen": -1.7657276391983032,
+      "logits/rejected": -1.6786730289459229,
+      "logps/chosen": -426.5061950683594,
+      "logps/rejected": -419.21014404296875,
+      "loss": 0.6015,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5616867542266846,
+      "rewards/margins": 0.322202205657959,
+      "rewards/rejected": -0.883888840675354,
+      "step": 520
+    },
+    {
+      "epoch": 0.20806752380017665,
+      "grad_norm": 3.0112686157226562,
+      "learning_rate": 4.824491669228279e-06,
+      "logits/chosen": -1.7685273885726929,
+      "logits/rejected": -1.7272237539291382,
+      "logps/chosen": -415.84503173828125,
+      "logps/rejected": -408.3431701660156,
+      "loss": 0.5856,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9017496109008789,
+      "rewards/margins": 0.3433293104171753,
+      "rewards/rejected": -1.2450790405273438,
+      "step": 530
+    },
+    {
+      "epoch": 0.21199332613602906,
+      "grad_norm": 2.767817258834839,
+      "learning_rate": 4.811660956390372e-06,
+      "logits/chosen": -1.7965497970581055,
+      "logits/rejected": -1.7540159225463867,
+      "logps/chosen": -451.7413635253906,
+      "logps/rejected": -438.45013427734375,
+      "loss": 0.6087,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.8332487344741821,
+      "rewards/margins": 0.2144457995891571,
+      "rewards/rejected": -1.047694444656372,
+      "step": 540
+    },
+    {
+      "epoch": 0.21591912847188144,
+      "grad_norm": 4.630012035369873,
+      "learning_rate": 4.798395945524615e-06,
+      "logits/chosen": -1.8445618152618408,
+      "logits/rejected": -1.823258399963379,
+      "logps/chosen": -375.31024169921875,
+      "logps/rejected": -437.65911865234375,
+      "loss": 0.6403,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3766379654407501,
+      "rewards/margins": 0.1970234364271164,
+      "rewards/rejected": -0.5736614465713501,
+      "step": 550
+    },
+    {
+      "epoch": 0.21984493080773382,
+      "grad_norm": 3.3371288776397705,
+      "learning_rate": 4.784699128764654e-06,
+      "logits/chosen": -1.8190743923187256,
+      "logits/rejected": -1.7443910837173462,
+      "logps/chosen": -377.7322692871094,
+      "logps/rejected": -371.0264587402344,
+      "loss": 0.5934,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.48190051317214966,
+      "rewards/margins": 0.25160467624664307,
+      "rewards/rejected": -0.733505129814148,
+      "step": 560
+    },
+    {
+      "epoch": 0.22377073314358623,
+      "grad_norm": 4.315640449523926,
+      "learning_rate": 4.770573079368691e-06,
+      "logits/chosen": -1.8440895080566406,
+      "logits/rejected": -1.8170363903045654,
+      "logps/chosen": -423.81134033203125,
+      "logps/rejected": -431.7996520996094,
+      "loss": 0.6057,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8013380765914917,
+      "rewards/margins": 0.3498791754245758,
+      "rewards/rejected": -1.1512172222137451,
+      "step": 570
+    },
+    {
+      "epoch": 0.2276965354794386,
+      "grad_norm": 3.7494423389434814,
+      "learning_rate": 4.756020451236025e-06,
+      "logits/chosen": -1.758657693862915,
+      "logits/rejected": -1.7444913387298584,
+      "logps/chosen": -433.5675354003906,
+      "logps/rejected": -456.9940490722656,
+      "loss": 0.5995,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.0486401319503784,
+      "rewards/margins": 0.29404133558273315,
+      "rewards/rejected": -1.342681646347046,
+      "step": 580
+    },
+    {
+      "epoch": 0.231622337815291,
+      "grad_norm": 2.9106571674346924,
+      "learning_rate": 4.741043978408463e-06,
+      "logits/chosen": -1.8087266683578491,
+      "logits/rejected": -1.7474292516708374,
+      "logps/chosen": -430.044921875,
+      "logps/rejected": -481.80743408203125,
+      "loss": 0.5858,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.9068681597709656,
+      "rewards/margins": 0.42691636085510254,
+      "rewards/rejected": -1.3337844610214233,
+      "step": 590
+    },
+    {
+      "epoch": 0.2355481401511434,
+      "grad_norm": 4.692925453186035,
+      "learning_rate": 4.725646474556666e-06,
+      "logits/chosen": -1.7093442678451538,
+      "logits/rejected": -1.6595290899276733,
+      "logps/chosen": -370.7290344238281,
+      "logps/rejected": -417.06585693359375,
+      "loss": 0.5665,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.772936999797821,
+      "rewards/margins": 0.5391116142272949,
+      "rewards/rejected": -1.3120487928390503,
+      "step": 600
+    },
+    {
+      "epoch": 0.2355481401511434,
+      "eval_logits/chosen": -1.8241028785705566,
+      "eval_logits/rejected": -1.7701058387756348,
+      "eval_logps/chosen": -410.8185729980469,
+      "eval_logps/rejected": -407.9361572265625,
+      "eval_loss": 0.5955410599708557,
+      "eval_rewards/accuracies": 0.6931137442588806,
+      "eval_rewards/chosen": -0.6340402364730835,
+      "eval_rewards/margins": 0.3989306092262268,
+      "eval_rewards/rejected": -1.032970905303955,
+      "eval_runtime": 297.5237,
+      "eval_samples_per_second": 6.722,
+      "eval_steps_per_second": 1.123,
+      "step": 600
+    },
+    {
+      "epoch": 0.23947394248699577,
+      "grad_norm": 2.3394923210144043,
+      "learning_rate": 4.709830832451538e-06,
+      "logits/chosen": -1.8074710369110107,
+      "logits/rejected": -1.7653526067733765,
+      "logps/chosen": -400.6386413574219,
+      "logps/rejected": -363.33245849609375,
+      "loss": 0.626,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4516834318637848,
+      "rewards/margins": 0.353360116481781,
+      "rewards/rejected": -0.8050435781478882,
+      "step": 610
+    },
+    {
+      "epoch": 0.24339974482284818,
+      "grad_norm": 2.561375379562378,
+      "learning_rate": 4.693600023420758e-06,
+      "logits/chosen": -1.7490384578704834,
+      "logits/rejected": -1.7471377849578857,
+      "logps/chosen": -389.53424072265625,
+      "logps/rejected": -460.62548828125,
+      "loss": 0.5745,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.45052212476730347,
+      "rewards/margins": 0.42988190054893494,
+      "rewards/rejected": -0.8804041147232056,
+      "step": 620
+    },
+    {
+      "epoch": 0.24732554715870056,
+      "grad_norm": 3.367673397064209,
+      "learning_rate": 4.676957096790536e-06,
+      "logits/chosen": -1.6660236120224,
+      "logits/rejected": -1.6637165546417236,
+      "logps/chosen": -390.4853515625,
+      "logps/rejected": -455.03240966796875,
+      "loss": 0.5586,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.562824010848999,
+      "rewards/margins": 0.3832342326641083,
+      "rewards/rejected": -0.9460583925247192,
+      "step": 630
+    },
+    {
+      "epoch": 0.25125134949455297,
+      "grad_norm": 2.9658846855163574,
+      "learning_rate": 4.659905179312743e-06,
+      "logits/chosen": -1.8020744323730469,
+      "logits/rejected": -1.7609078884124756,
+      "logps/chosen": -420.54638671875,
+      "logps/rejected": -474.72930908203125,
+      "loss": 0.5785,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.7424986362457275,
+      "rewards/margins": 0.5096467733383179,
+      "rewards/rejected": -1.252145528793335,
+      "step": 640
+    },
+    {
+      "epoch": 0.25517715183040535,
+      "grad_norm": 2.326108694076538,
+      "learning_rate": 4.642447474577466e-06,
+      "logits/chosen": -1.6250314712524414,
+      "logits/rejected": -1.6034454107284546,
+      "logps/chosen": -428.2274475097656,
+      "logps/rejected": -430.71063232421875,
+      "loss": 0.526,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.6683988571166992,
+      "rewards/margins": 0.525662899017334,
+      "rewards/rejected": -1.1940616369247437,
+      "step": 650
+    },
+    {
+      "epoch": 0.2591029541662577,
+      "grad_norm": 3.0301599502563477,
+      "learning_rate": 4.6245872624111535e-06,
+      "logits/chosen": -1.7263195514678955,
+      "logits/rejected": -1.688970923423767,
+      "logps/chosen": -421.3147888183594,
+      "logps/rejected": -406.07672119140625,
+      "loss": 0.57,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.8159321546554565,
+      "rewards/margins": 0.5035170316696167,
+      "rewards/rejected": -1.3194490671157837,
+      "step": 660
+    },
+    {
+      "epoch": 0.2630287565021101,
+      "grad_norm": 2.7172772884368896,
+      "learning_rate": 4.606327898260413e-06,
+      "logits/chosen": -1.774688482284546,
+      "logits/rejected": -1.6726405620574951,
+      "logps/chosen": -510.53741455078125,
+      "logps/rejected": -463.9078674316406,
+      "loss": 0.5903,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.8305042386054993,
+      "rewards/margins": 0.32598876953125,
+      "rewards/rejected": -1.1564929485321045,
+      "step": 670
+    },
+    {
+      "epoch": 0.2669545588379625,
+      "grad_norm": 2.2783751487731934,
+      "learning_rate": 4.587672812561626e-06,
+      "logits/chosen": -1.6800400018692017,
+      "logits/rejected": -1.6490108966827393,
+      "logps/chosen": -331.3873596191406,
+      "logps/rejected": -348.4686584472656,
+      "loss": 0.5947,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.7024337649345398,
+      "rewards/margins": 0.24449098110198975,
+      "rewards/rejected": -0.9469248056411743,
+      "step": 680
+    },
+    {
+      "epoch": 0.2708803611738149,
+      "grad_norm": 2.0253257751464844,
+      "learning_rate": 4.5686255100964535e-06,
+      "logits/chosen": -1.7758172750473022,
+      "logits/rejected": -1.7389147281646729,
+      "logps/chosen": -400.5079040527344,
+      "logps/rejected": -379.46380615234375,
+      "loss": 0.5841,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.4625607132911682,
+      "rewards/margins": 0.33417630195617676,
+      "rewards/rejected": -0.796737015247345,
+      "step": 690
+    },
+    {
+      "epoch": 0.2748061635096673,
+      "grad_norm": 2.86422061920166,
+      "learning_rate": 4.549189569333387e-06,
+      "logits/chosen": -1.7560211420059204,
+      "logits/rejected": -1.6849863529205322,
+      "logps/chosen": -423.73388671875,
+      "logps/rejected": -413.3765563964844,
+      "loss": 0.5322,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.5607272982597351,
+      "rewards/margins": 0.4947907328605652,
+      "rewards/rejected": -1.0555180311203003,
+      "step": 700
+    },
+    {
+      "epoch": 0.2748061635096673,
+      "eval_logits/chosen": -1.765041708946228,
+      "eval_logits/rejected": -1.7115814685821533,
+      "eval_logps/chosen": -421.46533203125,
+      "eval_logps/rejected": -426.0079650878906,
+      "eval_loss": 0.5794752240180969,
+      "eval_rewards/accuracies": 0.711077868938446,
+      "eval_rewards/chosen": -0.740507960319519,
+      "eval_rewards/margins": 0.47318097949028015,
+      "eval_rewards/rejected": -1.2136889696121216,
+      "eval_runtime": 295.0983,
+      "eval_samples_per_second": 6.777,
+      "eval_steps_per_second": 1.132,
+      "step": 700
+    },
+    {
+      "epoch": 0.2787319658455197,
+      "grad_norm": 2.876797676086426,
+      "learning_rate": 4.529368641755453e-06,
+      "logits/chosen": -1.6831592321395874,
+      "logits/rejected": -1.6914005279541016,
+      "logps/chosen": -441.3789978027344,
+      "logps/rejected": -408.1473083496094,
+      "loss": 0.5799,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.8866918683052063,
+      "rewards/margins": 0.39884892106056213,
+      "rewards/rejected": -1.2855409383773804,
+      "step": 710
+    },
+    {
+      "epoch": 0.28265776818137206,
+      "grad_norm": 4.033863544464111,
+      "learning_rate": 4.509166451174194e-06,
+      "logits/chosen": -1.7658096551895142,
+      "logits/rejected": -1.6881206035614014,
+      "logps/chosen": -451.5057678222656,
+      "logps/rejected": -490.4564514160156,
+      "loss": 0.5576,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.9588688015937805,
+      "rewards/margins": 0.562650740146637,
+      "rewards/rejected": -1.5215195417404175,
+      "step": 720
+    },
+    {
+      "epoch": 0.28658357051722444,
+      "grad_norm": 2.9548211097717285,
+      "learning_rate": 4.488586793030075e-06,
+      "logits/chosen": -1.7060620784759521,
+      "logits/rejected": -1.7342453002929688,
+      "logps/chosen": -448.1982421875,
+      "logps/rejected": -481.39068603515625,
+      "loss": 0.6177,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1785500049591064,
+      "rewards/margins": 0.31686827540397644,
+      "rewards/rejected": -1.4954183101654053,
+      "step": 730
+    },
+    {
+      "epoch": 0.2905093728530769,
+      "grad_norm": 3.627689838409424,
+      "learning_rate": 4.4676335336794125e-06,
+      "logits/chosen": -1.8308156728744507,
+      "logits/rejected": -1.7847893238067627,
+      "logps/chosen": -430.2438049316406,
+      "logps/rejected": -452.053955078125,
+      "loss": 0.5959,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.9821481704711914,
+      "rewards/margins": 0.309665709733963,
+      "rewards/rejected": -1.2918137311935425,
+      "step": 740
+    },
+    {
+      "epoch": 0.29443517518892925,
+      "grad_norm": 3.5429649353027344,
+      "learning_rate": 4.446310609668001e-06,
+      "logits/chosen": -1.6525977849960327,
+      "logits/rejected": -1.6022307872772217,
+      "logps/chosen": -393.2485656738281,
+      "logps/rejected": -453.4503479003906,
+      "loss": 0.5847,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5448301434516907,
+      "rewards/margins": 0.5748300552368164,
+      "rewards/rejected": -1.1196601390838623,
+      "step": 750
+    },
+    {
+      "epoch": 0.29836097752478163,
+      "grad_norm": 3.4878909587860107,
+      "learning_rate": 4.424622026991536e-06,
+      "logits/chosen": -1.7291042804718018,
+      "logits/rejected": -1.6743202209472656,
+      "logps/chosen": -392.51007080078125,
+      "logps/rejected": -383.07281494140625,
+      "loss": 0.5896,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.49430790543556213,
+      "rewards/margins": 0.3120374083518982,
+      "rewards/rejected": -0.8063452839851379,
+      "step": 760
+    },
+    {
+      "epoch": 0.302286779860634,
+      "grad_norm": 3.364633798599243,
+      "learning_rate": 4.402571860343006e-06,
+      "logits/chosen": -1.7037136554718018,
+      "logits/rejected": -1.6506502628326416,
+      "logps/chosen": -358.54351806640625,
+      "logps/rejected": -395.66741943359375,
+      "loss": 0.5745,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.7334325313568115,
+      "rewards/margins": 0.388921320438385,
+      "rewards/rejected": -1.1223537921905518,
+      "step": 770
+    },
+    {
+      "epoch": 0.3062125821964864,
+      "grad_norm": 2.9582035541534424,
+      "learning_rate": 4.3801642523471585e-06,
+      "logits/chosen": -1.7445404529571533,
+      "logits/rejected": -1.7248423099517822,
+      "logps/chosen": -449.40673828125,
+      "logps/rejected": -455.3212890625,
+      "loss": 0.5993,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.8387622833251953,
+      "rewards/margins": 0.42324042320251465,
+      "rewards/rejected": -1.26200270652771,
+      "step": 780
+    },
+    {
+      "epoch": 0.3101383845323388,
+      "grad_norm": 3.2264795303344727,
+      "learning_rate": 4.35740341278222e-06,
+      "logits/chosen": -1.6737827062606812,
+      "logits/rejected": -1.6733373403549194,
+      "logps/chosen": -426.0733337402344,
+      "logps/rejected": -478.4773864746094,
+      "loss": 0.5703,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.077840805053711,
+      "rewards/margins": 0.3981647193431854,
+      "rewards/rejected": -1.4760054349899292,
+      "step": 790
+    },
+    {
+      "epoch": 0.3140641868681912,
+      "grad_norm": 3.7072577476501465,
+      "learning_rate": 4.334293617788992e-06,
+      "logits/chosen": -1.6208372116088867,
+      "logits/rejected": -1.654841661453247,
+      "logps/chosen": -414.236083984375,
+      "logps/rejected": -447.9951171875,
+      "loss": 0.616,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9234563708305359,
+      "rewards/margins": 0.37732914090156555,
+      "rewards/rejected": -1.3007854223251343,
+      "step": 800
+    },
+    {
+      "epoch": 0.3140641868681912,
+      "eval_logits/chosen": -1.6827820539474487,
+      "eval_logits/rejected": -1.6309748888015747,
+      "eval_logps/chosen": -423.0748596191406,
+      "eval_logps/rejected": -429.31488037109375,
+      "eval_loss": 0.5719860792160034,
+      "eval_rewards/accuracies": 0.71856290102005,
+      "eval_rewards/chosen": -0.7566031813621521,
+      "eval_rewards/margins": 0.4901549220085144,
+      "eval_rewards/rejected": -1.2467581033706665,
+      "eval_runtime": 294.7339,
+      "eval_samples_per_second": 6.786,
+      "eval_steps_per_second": 1.133,
+      "step": 800
+    },
+    {
+      "epoch": 0.3179899892040436,
+      "grad_norm": 3.104132652282715,
+      "learning_rate": 4.310839209067482e-06,
+      "logits/chosen": -1.6490373611450195,
+      "logits/rejected": -1.5513606071472168,
+      "logps/chosen": -374.03839111328125,
+      "logps/rejected": -440.2606506347656,
+      "loss": 0.5776,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.6420900225639343,
+      "rewards/margins": 0.44617852568626404,
+      "rewards/rejected": -1.0882686376571655,
+      "step": 810
+    },
+    {
+      "epoch": 0.32191579153989597,
+      "grad_norm": 2.9992995262145996,
+      "learning_rate": 4.2870445930612135e-06,
+      "logits/chosen": -1.7541091442108154,
+      "logits/rejected": -1.6079908609390259,
+      "logps/chosen": -496.16400146484375,
+      "logps/rejected": -463.64544677734375,
+      "loss": 0.5969,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.49080151319503784,
+      "rewards/margins": 0.3940032422542572,
+      "rewards/rejected": -0.8848048448562622,
+      "step": 820
+    },
+    {
+      "epoch": 0.32584159387574835,
+      "grad_norm": 3.0423057079315186,
+      "learning_rate": 4.262914240129379e-06,
+      "logits/chosen": -1.6219298839569092,
+      "logits/rejected": -1.5714378356933594,
+      "logps/chosen": -417.99237060546875,
+      "logps/rejected": -401.5321960449219,
+      "loss": 0.5686,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.6151792407035828,
+      "rewards/margins": 0.5490989685058594,
+      "rewards/rejected": -1.164278268814087,
+      "step": 830
+    },
+    {
+      "epoch": 0.3297673962116007,
+      "grad_norm": 2.3737690448760986,
+      "learning_rate": 4.238452683706979e-06,
+      "logits/chosen": -1.6418819427490234,
+      "logits/rejected": -1.5690456628799438,
+      "logps/chosen": -453.8641662597656,
+      "logps/rejected": -417.9911193847656,
+      "loss": 0.6124,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.8284245729446411,
+      "rewards/margins": 0.4460185468196869,
+      "rewards/rejected": -1.2744431495666504,
+      "step": 840
+    },
+    {
+      "epoch": 0.33369319854745316,
+      "grad_norm": 2.6307549476623535,
+      "learning_rate": 4.213664519453115e-06,
+      "logits/chosen": -1.6348683834075928,
+      "logits/rejected": -1.6477634906768799,
+      "logps/chosen": -418.5729064941406,
+      "logps/rejected": -453.28594970703125,
+      "loss": 0.5938,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.9299256205558777,
+      "rewards/margins": 0.33246856927871704,
+      "rewards/rejected": -1.2623941898345947,
+      "step": 850
+    },
+    {
+      "epoch": 0.33761900088330554,
+      "grad_norm": 3.0583863258361816,
+      "learning_rate": 4.188554404387588e-06,
+      "logits/chosen": -1.6888093948364258,
+      "logits/rejected": -1.6553878784179688,
+      "logps/chosen": -428.0262145996094,
+      "logps/rejected": -403.271728515625,
+      "loss": 0.5902,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8281693458557129,
+      "rewards/margins": 0.3182961344718933,
+      "rewards/rejected": -1.146465539932251,
+      "step": 860
+    },
+    {
+      "epoch": 0.3415448032191579,
+      "grad_norm": 2.7328920364379883,
+      "learning_rate": 4.163127056015975e-06,
+      "logits/chosen": -1.7795034646987915,
+      "logits/rejected": -1.7467601299285889,
+      "logps/chosen": -388.02850341796875,
+      "logps/rejected": -402.0059814453125,
+      "loss": 0.578,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.7644864320755005,
+      "rewards/margins": 0.4680708944797516,
+      "rewards/rejected": -1.2325572967529297,
+      "step": 870
+    },
+    {
+      "epoch": 0.3454706055550103,
+      "grad_norm": 2.6769022941589355,
+      "learning_rate": 4.137387251443335e-06,
+      "logits/chosen": -1.7767242193222046,
+      "logits/rejected": -1.7529585361480713,
+      "logps/chosen": -404.7185363769531,
+      "logps/rejected": -407.1458435058594,
+      "loss": 0.5763,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5854518413543701,
+      "rewards/margins": 0.435697466135025,
+      "rewards/rejected": -1.0211492776870728,
+      "step": 880
+    },
+    {
+      "epoch": 0.3493964078908627,
+      "grad_norm": 2.73026967048645,
+      "learning_rate": 4.111339826476725e-06,
+      "logits/chosen": -1.7634567022323608,
+      "logits/rejected": -1.718462347984314,
+      "logps/chosen": -388.9556884765625,
+      "logps/rejected": -417.4217834472656,
+      "loss": 0.6018,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.4032491147518158,
+      "rewards/margins": 0.33838939666748047,
+      "rewards/rejected": -0.7416385412216187,
+      "step": 890
+    },
+    {
+      "epoch": 0.35332221022671506,
+      "grad_norm": 3.116485357284546,
+      "learning_rate": 4.084989674716679e-06,
+      "logits/chosen": -1.6566784381866455,
+      "logits/rejected": -1.6071946620941162,
+      "logps/chosen": -407.0699157714844,
+      "logps/rejected": -372.7525329589844,
+      "loss": 0.6129,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.45755815505981445,
+      "rewards/margins": 0.2521994113922119,
+      "rewards/rejected": -0.7097574472427368,
+      "step": 900
+    },
+    {
+      "epoch": 0.35332221022671506,
+      "eval_logits/chosen": -1.6991018056869507,
+      "eval_logits/rejected": -1.6470667123794556,
+      "eval_logps/chosen": -397.11492919921875,
+      "eval_logps/rejected": -401.11444091796875,
+      "eval_loss": 0.5754956007003784,
+      "eval_rewards/accuracies": 0.7290419340133667,
+      "eval_rewards/chosen": -0.49700385332107544,
+      "eval_rewards/margins": 0.4677499532699585,
+      "eval_rewards/rejected": -0.9647537469863892,
+      "eval_runtime": 293.9153,
+      "eval_samples_per_second": 6.805,
+      "eval_steps_per_second": 1.136,
+      "step": 900
+    },
+    {
+      "epoch": 0.3572480125625675,
+      "grad_norm": 2.6833512783050537,
+      "learning_rate": 4.05834174663784e-06,
+      "logits/chosen": -1.6667829751968384,
+      "logits/rejected": -1.667248010635376,
+      "logps/chosen": -387.62255859375,
+      "logps/rejected": -423.0357971191406,
+      "loss": 0.547,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.6865302920341492,
+      "rewards/margins": 0.47197800874710083,
+      "rewards/rejected": -1.15850830078125,
+      "step": 910
+    },
+    {
+      "epoch": 0.3611738148984199,
+      "grad_norm": 3.9031505584716797,
+      "learning_rate": 4.031401048658892e-06,
+      "logits/chosen": -1.6854177713394165,
+      "logits/rejected": -1.6338930130004883,
+      "logps/chosen": -497.706787109375,
+      "logps/rejected": -516.3783569335938,
+      "loss": 0.5658,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.9667622447013855,
+      "rewards/margins": 0.6015533804893494,
+      "rewards/rejected": -1.5683156251907349,
+      "step": 920
+    },
+    {
+      "epoch": 0.36509961723427226,
+      "grad_norm": 4.924274444580078,
+      "learning_rate": 4.004172642202002e-06,
+      "logits/chosen": -1.5499565601348877,
+      "logits/rejected": -1.4909119606018066,
+      "logps/chosen": -449.03564453125,
+      "logps/rejected": -427.6610412597656,
+      "loss": 0.5682,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9110291600227356,
+      "rewards/margins": 0.5169877409934998,
+      "rewards/rejected": -1.4280169010162354,
+      "step": 930
+    },
+    {
+      "epoch": 0.36902541957012464,
+      "grad_norm": 2.352947235107422,
+      "learning_rate": 3.976661642741908e-06,
+      "logits/chosen": -1.6199270486831665,
+      "logits/rejected": -1.5865291357040405,
+      "logps/chosen": -380.2807922363281,
+      "logps/rejected": -411.0011291503906,
+      "loss": 0.5344,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.5013939142227173,
+      "rewards/margins": 0.548475980758667,
+      "rewards/rejected": -1.0498700141906738,
+      "step": 940
+    },
+    {
+      "epoch": 0.372951221905977,
+      "grad_norm": 4.017947196960449,
+      "learning_rate": 3.948873218844863e-06,
+      "logits/chosen": -1.6051380634307861,
+      "logits/rejected": -1.5401808023452759,
+      "logps/chosen": -443.994384765625,
+      "logps/rejected": -412.3163146972656,
+      "loss": 0.5819,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.6204164624214172,
+      "rewards/margins": 0.710503876209259,
+      "rewards/rejected": -1.3309203386306763,
+      "step": 950
+    },
+    {
+      "epoch": 0.37687702424182945,
+      "grad_norm": 3.388486385345459,
+      "learning_rate": 3.920812591197604e-06,
+      "logits/chosen": -1.6786155700683594,
+      "logits/rejected": -1.6055418252944946,
+      "logps/chosen": -445.1845703125,
+      "logps/rejected": -464.0609436035156,
+      "loss": 0.5628,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.8863388895988464,
+      "rewards/margins": 0.6250656843185425,
+      "rewards/rejected": -1.5114046335220337,
+      "step": 960
+    },
+    {
+      "epoch": 0.38080282657768183,
+      "grad_norm": 2.9201085567474365,
+      "learning_rate": 3.892485031626527e-06,
+      "logits/chosen": -1.6259195804595947,
+      "logits/rejected": -1.538879632949829,
+      "logps/chosen": -467.6678771972656,
+      "logps/rejected": -452.05517578125,
+      "loss": 0.5985,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.9320256114006042,
+      "rewards/margins": 0.5447267293930054,
+      "rewards/rejected": -1.4767522811889648,
+      "step": 970
+    },
+    {
+      "epoch": 0.3847286289135342,
+      "grad_norm": 2.0337541103363037,
+      "learning_rate": 3.863895862107255e-06,
+      "logits/chosen": -1.620653510093689,
+      "logits/rejected": -1.5558189153671265,
+      "logps/chosen": -402.3785705566406,
+      "logps/rejected": -429.40643310546875,
+      "loss": 0.5265,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.5818737745285034,
+      "rewards/margins": 0.7111320495605469,
+      "rewards/rejected": -1.2930058240890503,
+      "step": 980
+    },
+    {
+      "epoch": 0.3886544312493866,
+      "grad_norm": 2.0163416862487793,
+      "learning_rate": 3.835050453764779e-06,
+      "logits/chosen": -1.6193315982818604,
+      "logits/rejected": -1.530425786972046,
+      "logps/chosen": -479.17352294921875,
+      "logps/rejected": -442.89501953125,
+      "loss": 0.5668,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9273340106010437,
+      "rewards/margins": 0.5894566774368286,
+      "rewards/rejected": -1.5167906284332275,
+      "step": 990
+    },
+    {
+      "epoch": 0.39258023358523897,
+      "grad_norm": 2.8958888053894043,
+      "learning_rate": 3.80595422586438e-06,
+      "logits/chosen": -1.634264588356018,
+      "logits/rejected": -1.5909229516983032,
+      "logps/chosen": -471.4090270996094,
+      "logps/rejected": -486.3736267089844,
+      "loss": 0.5308,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0152546167373657,
+      "rewards/margins": 0.6755619049072266,
+      "rewards/rejected": -1.6908165216445923,
+      "step": 1000
+    },
+    {
+      "epoch": 0.39258023358523897,
+      "eval_logits/chosen": -1.6002135276794434,
+      "eval_logits/rejected": -1.550965428352356,
+      "eval_logps/chosen": -460.9561767578125,
+      "eval_logps/rejected": -474.817138671875,
+      "eval_loss": 0.5657364130020142,
+      "eval_rewards/accuracies": 0.71856290102005,
+      "eval_rewards/chosen": -1.1354165077209473,
+      "eval_rewards/margins": 0.5663644671440125,
+      "eval_rewards/rejected": -1.701780915260315,
+      "eval_runtime": 293.0409,
+      "eval_samples_per_second": 6.825,
+      "eval_steps_per_second": 1.14,
+      "step": 1000
+    },
+    {
+      "epoch": 0.39650603592109135,
+      "grad_norm": 3.303689956665039,
+      "learning_rate": 3.7766126447934857e-06,
+      "logits/chosen": -1.4985648393630981,
+      "logits/rejected": -1.479155421257019,
+      "logps/chosen": -444.32293701171875,
+      "logps/rejected": -487.92767333984375,
+      "loss": 0.6283,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2959802150726318,
+      "rewards/margins": 0.27502721548080444,
+      "rewards/rejected": -1.571007490158081,
+      "step": 1010
+    },
+    {
+      "epoch": 0.4004318382569438,
+      "grad_norm": 3.359537363052368,
+      "learning_rate": 3.7470312230346955e-06,
+      "logits/chosen": -1.530688762664795,
+      "logits/rejected": -1.5040019750595093,
+      "logps/chosen": -460.97906494140625,
+      "logps/rejected": -479.7684020996094,
+      "loss": 0.6032,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9211111068725586,
+      "rewards/margins": 0.5467273592948914,
+      "rewards/rejected": -1.4678385257720947,
+      "step": 1020
+    },
+    {
+      "epoch": 0.40435764059279616,
+      "grad_norm": 2.5796802043914795,
+      "learning_rate": 3.717215518130127e-06,
+      "logits/chosen": -1.5627844333648682,
+      "logits/rejected": -1.5050039291381836,
+      "logps/chosen": -429.95379638671875,
+      "logps/rejected": -466.34100341796875,
+      "loss": 0.5826,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.9197722673416138,
+      "rewards/margins": 0.487074077129364,
+      "rewards/rejected": -1.406846284866333,
+      "step": 1030
+    },
+    {
+      "epoch": 0.40828344292864854,
+      "grad_norm": 3.103071451187134,
+      "learning_rate": 3.687171131637314e-06,
+      "logits/chosen": -1.569985032081604,
+      "logits/rejected": -1.5849441289901733,
+      "logps/chosen": -504.1112365722656,
+      "logps/rejected": -485.5889587402344,
+      "loss": 0.5678,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9759623408317566,
+      "rewards/margins": 0.45774656534194946,
+      "rewards/rejected": -1.4337090253829956,
+      "step": 1040
+    },
+    {
+      "epoch": 0.4122092452645009,
+      "grad_norm": 3.31626296043396,
+      "learning_rate": 3.6569037080768153e-06,
+      "logits/chosen": -1.4381552934646606,
+      "logits/rejected": -1.4639440774917603,
+      "logps/chosen": -430.3623046875,
+      "logps/rejected": -468.56304931640625,
+      "loss": 0.5766,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9098421335220337,
+      "rewards/margins": 0.5434075593948364,
+      "rewards/rejected": -1.4532496929168701,
+      "step": 1050
+    },
+    {
+      "epoch": 0.4161350476003533,
+      "grad_norm": 2.4118287563323975,
+      "learning_rate": 3.6264189338717766e-06,
+      "logits/chosen": -1.6304969787597656,
+      "logits/rejected": -1.6439449787139893,
+      "logps/chosen": -423.7255859375,
+      "logps/rejected": -496.8777770996094,
+      "loss": 0.5451,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8576018214225769,
+      "rewards/margins": 0.4397641122341156,
+      "rewards/rejected": -1.2973659038543701,
+      "step": 1060
+    },
+    {
+      "epoch": 0.42006084993620574,
+      "grad_norm": 3.1058785915374756,
+      "learning_rate": 3.595722536279595e-06,
+      "logits/chosen": -1.553431510925293,
+      "logits/rejected": -1.5193746089935303,
+      "logps/chosen": -407.58941650390625,
+      "logps/rejected": -415.96331787109375,
+      "loss": 0.6103,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.7842639684677124,
+      "rewards/margins": 0.47796306014060974,
+      "rewards/rejected": -1.262226939201355,
+      "step": 1070
+    },
+    {
+      "epoch": 0.4239866522720581,
+      "grad_norm": 4.273246765136719,
+      "learning_rate": 3.5648202823159317e-06,
+      "logits/chosen": -1.6235965490341187,
+      "logits/rejected": -1.5664032697677612,
+      "logps/chosen": -399.4711608886719,
+      "logps/rejected": -445.1605529785156,
+      "loss": 0.5607,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.0092414617538452,
+      "rewards/margins": 0.4340530037879944,
+      "rewards/rejected": -1.443294644355774,
+      "step": 1080
+    },
+    {
+      "epoch": 0.4279124546079105,
+      "grad_norm": 3.8848683834075928,
+      "learning_rate": 3.5337179776712427e-06,
+      "logits/chosen": -1.6012245416641235,
+      "logits/rejected": -1.6299206018447876,
+      "logps/chosen": -390.93463134765625,
+      "logps/rejected": -421.5492248535156,
+      "loss": 0.6212,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.0607120990753174,
+      "rewards/margins": 0.3177862763404846,
+      "rewards/rejected": -1.3784983158111572,
+      "step": 1090
+    },
+    {
+      "epoch": 0.4318382569437629,
+      "grad_norm": 3.0826752185821533,
+      "learning_rate": 3.5024214656200497e-06,
+      "logits/chosen": -1.6045929193496704,
+      "logits/rejected": -1.5481618642807007,
+      "logps/chosen": -513.1831665039062,
+      "logps/rejected": -486.402587890625,
+      "loss": 0.589,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.2744470834732056,
+      "rewards/margins": 0.6375499963760376,
+      "rewards/rejected": -1.9119970798492432,
+      "step": 1100
+    },
+    {
+      "epoch": 0.4318382569437629,
+      "eval_logits/chosen": -1.593135952949524,
+      "eval_logits/rejected": -1.544421672821045,
+      "eval_logps/chosen": -462.1784362792969,
+      "eval_logps/rejected": -477.9910583496094,
+      "eval_loss": 0.5631109476089478,
+      "eval_rewards/accuracies": 0.720059871673584,
+      "eval_rewards/chosen": -1.1476389169692993,
+      "eval_rewards/margins": 0.5858806371688843,
+      "eval_rewards/rejected": -1.7335195541381836,
+      "eval_runtime": 292.7864,
+      "eval_samples_per_second": 6.831,
+      "eval_steps_per_second": 1.141,
+      "step": 1100
+    },
+    {
+      "epoch": 0.43576405927961526,
+      "grad_norm": 3.3952105045318604,
+      "learning_rate": 3.4709366259231468e-06,
+      "logits/chosen": -1.570192575454712,
+      "logits/rejected": -1.5551464557647705,
+      "logps/chosen": -465.5960388183594,
+      "logps/rejected": -451.34051513671875,
+      "loss": 0.5983,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2366596460342407,
+      "rewards/margins": 0.3722042441368103,
+      "rewards/rejected": -1.6088638305664062,
+      "step": 1110
+    },
+    {
+      "epoch": 0.43968986161546764,
+      "grad_norm": 3.588393449783325,
+      "learning_rate": 3.439269373722957e-06,
+      "logits/chosen": -1.5961639881134033,
+      "logits/rejected": -1.5418142080307007,
+      "logps/chosen": -487.3839416503906,
+      "logps/rejected": -499.2284240722656,
+      "loss": 0.5692,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.0747463703155518,
+      "rewards/margins": 0.48480862379074097,
+      "rewards/rejected": -1.5595548152923584,
+      "step": 1120
+    },
+    {
+      "epoch": 0.44361566395132007,
+      "grad_norm": 2.92950701713562,
+      "learning_rate": 3.4074256584322336e-06,
+      "logits/chosen": -1.5175635814666748,
+      "logits/rejected": -1.473159909248352,
+      "logps/chosen": -448.197509765625,
+      "logps/rejected": -434.1591796875,
+      "loss": 0.606,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0764273405075073,
+      "rewards/margins": 0.4142221510410309,
+      "rewards/rejected": -1.490649700164795,
+      "step": 1130
+    },
+    {
+      "epoch": 0.44754146628717245,
+      "grad_norm": 2.5090534687042236,
+      "learning_rate": 3.375411462616332e-06,
+      "logits/chosen": -1.5814701318740845,
+      "logits/rejected": -1.5614421367645264,
+      "logps/chosen": -434.6560974121094,
+      "logps/rejected": -488.8587951660156,
+      "loss": 0.577,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1506696939468384,
+      "rewards/margins": 0.32148224115371704,
+      "rewards/rejected": -1.4721519947052002,
+      "step": 1140
+    },
+    {
+      "epoch": 0.45146726862302483,
+      "grad_norm": 3.282829523086548,
+      "learning_rate": 3.343232800869247e-06,
+      "logits/chosen": -1.633130669593811,
+      "logits/rejected": -1.6071325540542603,
+      "logps/chosen": -458.73529052734375,
+      "logps/rejected": -480.19268798828125,
+      "loss": 0.5598,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1532833576202393,
+      "rewards/margins": 0.5188425183296204,
+      "rewards/rejected": -1.672126054763794,
+      "step": 1150
+    },
+    {
+      "epoch": 0.4553930709588772,
+      "grad_norm": 2.593871593475342,
+      "learning_rate": 3.310895718683635e-06,
+      "logits/chosen": -1.6055257320404053,
+      "logits/rejected": -1.576418399810791,
+      "logps/chosen": -443.70697021484375,
+      "logps/rejected": -430.3130798339844,
+      "loss": 0.5869,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.9588342905044556,
+      "rewards/margins": 0.3664381206035614,
+      "rewards/rejected": -1.3252723217010498,
+      "step": 1160
+    },
+    {
+      "epoch": 0.4593188732947296,
+      "grad_norm": 2.250080108642578,
+      "learning_rate": 3.27840629131503e-06,
+      "logits/chosen": -1.604743242263794,
+      "logits/rejected": -1.6042743921279907,
+      "logps/chosen": -454.82049560546875,
+      "logps/rejected": -452.29962158203125,
+      "loss": 0.577,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.920699417591095,
+      "rewards/margins": 0.32778987288475037,
+      "rewards/rejected": -1.2484893798828125,
+      "step": 1170
+    },
+    {
+      "epoch": 0.463244675630582,
+      "grad_norm": 2.582003116607666,
+      "learning_rate": 3.2457706226404715e-06,
+      "logits/chosen": -1.4776430130004883,
+      "logits/rejected": -1.4150015115737915,
+      "logps/chosen": -396.91497802734375,
+      "logps/rejected": -442.81231689453125,
+      "loss": 0.5427,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.7093313336372375,
+      "rewards/margins": 0.7621004581451416,
+      "rewards/rejected": -1.471431851387024,
+      "step": 1180
+    },
+    {
+      "epoch": 0.4671704779664344,
+      "grad_norm": 3.7171318531036377,
+      "learning_rate": 3.2129948440117487e-06,
+      "logits/chosen": -1.576615571975708,
+      "logits/rejected": -1.523790955543518,
+      "logps/chosen": -401.42669677734375,
+      "logps/rejected": -461.55938720703125,
+      "loss": 0.575,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.880218505859375,
+      "rewards/margins": 0.6095125675201416,
+      "rewards/rejected": -1.4897310733795166,
+      "step": 1190
+    },
+    {
+      "epoch": 0.4710962803022868,
+      "grad_norm": 3.0626087188720703,
+      "learning_rate": 3.1800851131034904e-06,
+      "logits/chosen": -1.488353967666626,
+      "logits/rejected": -1.467444658279419,
+      "logps/chosen": -479.37939453125,
+      "logps/rejected": -543.6526489257812,
+      "loss": 0.5694,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.183075189590454,
+      "rewards/margins": 0.6361004114151001,
+      "rewards/rejected": -1.8191754817962646,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4710962803022868,
+      "eval_logits/chosen": -1.5827584266662598,
+      "eval_logits/rejected": -1.5333001613616943,
+      "eval_logps/chosen": -451.9159851074219,
+      "eval_logps/rejected": -466.84356689453125,
+      "eval_loss": 0.5629106760025024,
+      "eval_rewards/accuracies": 0.7245509028434753,
+      "eval_rewards/chosen": -1.0450141429901123,
+      "eval_rewards/margins": 0.5770303606987,
+      "eval_rewards/rejected": -1.622044563293457,
+      "eval_runtime": 295.237,
+      "eval_samples_per_second": 6.774,
+      "eval_steps_per_second": 1.131,
+      "step": 1200
+    },
+    {
+      "epoch": 0.47502208263813916,
+      "grad_norm": 4.683452129364014,
+      "learning_rate": 3.147047612756302e-06,
+      "logits/chosen": -1.4236586093902588,
+      "logits/rejected": -1.3944308757781982,
+      "logps/chosen": -439.0166931152344,
+      "logps/rejected": -465.93963623046875,
+      "loss": 0.5642,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.2439113855361938,
+      "rewards/margins": 0.3991193175315857,
+      "rewards/rejected": -1.6430307626724243,
+      "step": 1210
+    },
+    {
+      "epoch": 0.47894788497399154,
+      "grad_norm": 3.1946606636047363,
+      "learning_rate": 3.1138885498151843e-06,
+      "logits/chosen": -1.5926167964935303,
+      "logits/rejected": -1.5980358123779297,
+      "logps/chosen": -503.7044372558594,
+      "logps/rejected": -501.81597900390625,
+      "loss": 0.6095,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.162198781967163,
+      "rewards/margins": 0.5470144748687744,
+      "rewards/rejected": -1.7092134952545166,
+      "step": 1220
+    },
+    {
+      "epoch": 0.4828736873098439,
+      "grad_norm": 3.0145528316497803,
+      "learning_rate": 3.0806141539634294e-06,
+      "logits/chosen": -1.5540027618408203,
+      "logits/rejected": -1.5151664018630981,
+      "logps/chosen": -519.291259765625,
+      "logps/rejected": -516.1077880859375,
+      "loss": 0.5593,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1783506870269775,
+      "rewards/margins": 0.5996993780136108,
+      "rewards/rejected": -1.7780498266220093,
+      "step": 1230
+    },
+    {
+      "epoch": 0.48679948964569636,
+      "grad_norm": 2.868039608001709,
+      "learning_rate": 3.0472306765522393e-06,
+      "logits/chosen": -1.4834442138671875,
+      "logits/rejected": -1.4614264965057373,
+      "logps/chosen": -495.28033447265625,
+      "logps/rejected": -493.601318359375,
+      "loss": 0.5497,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.886478066444397,
+      "rewards/margins": 0.5514383316040039,
+      "rewards/rejected": -1.4379163980484009,
+      "step": 1240
+    },
+    {
+      "epoch": 0.49072529198154874,
+      "grad_norm": 2.698237180709839,
+      "learning_rate": 3.0137443894262634e-06,
+      "logits/chosen": -1.4641456604003906,
+      "logits/rejected": -1.4282712936401367,
+      "logps/chosen": -425.768798828125,
+      "logps/rejected": -456.55615234375,
+      "loss": 0.5054,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.9318915605545044,
+      "rewards/margins": 0.6572065353393555,
+      "rewards/rejected": -1.5890980958938599,
+      "step": 1250
+    },
+    {
+      "epoch": 0.4946510943174011,
+      "grad_norm": 3.1201820373535156,
+      "learning_rate": 2.980161583745294e-06,
+      "logits/chosen": -1.6448482275009155,
+      "logits/rejected": -1.5536471605300903,
+      "logps/chosen": -496.49346923828125,
+      "logps/rejected": -471.482421875,
+      "loss": 0.5766,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.113041639328003,
+      "rewards/margins": 0.5853625535964966,
+      "rewards/rejected": -1.698404312133789,
+      "step": 1260
+    },
+    {
+      "epoch": 0.4985768966532535,
+      "grad_norm": 3.22414493560791,
+      "learning_rate": 2.946488568802324e-06,
+      "logits/chosen": -1.5524537563323975,
+      "logits/rejected": -1.527789831161499,
+      "logps/chosen": -483.74786376953125,
+      "logps/rejected": -498.4803771972656,
+      "loss": 0.5517,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9959393739700317,
+      "rewards/margins": 0.44567403197288513,
+      "rewards/rejected": -1.4416133165359497,
+      "step": 1270
+    },
+    {
+      "epoch": 0.5025026989891059,
+      "grad_norm": 2.6703150272369385,
+      "learning_rate": 2.912731670838207e-06,
+      "logits/chosen": -1.4681823253631592,
+      "logits/rejected": -1.4283071756362915,
+      "logps/chosen": -446.96112060546875,
+      "logps/rejected": -492.69488525390625,
+      "loss": 0.5495,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.0284326076507568,
+      "rewards/margins": 0.5881555676460266,
+      "rewards/rejected": -1.6165879964828491,
+      "step": 1280
+    },
+    {
+      "epoch": 0.5064285013249583,
+      "grad_norm": 3.062986135482788,
+      "learning_rate": 2.8788972318531272e-06,
+      "logits/chosen": -1.5675195455551147,
+      "logits/rejected": -1.4750767946243286,
+      "logps/chosen": -405.035400390625,
+      "logps/rejected": -424.39764404296875,
+      "loss": 0.5291,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.9429351687431335,
+      "rewards/margins": 0.7298198938369751,
+      "rewards/rejected": -1.6727550029754639,
+      "step": 1290
+    },
+    {
+      "epoch": 0.5103543036608107,
+      "grad_norm": 3.3106205463409424,
+      "learning_rate": 2.844991608415113e-06,
+      "logits/chosen": -1.500427007675171,
+      "logits/rejected": -1.4527623653411865,
+      "logps/chosen": -442.693603515625,
+      "logps/rejected": -503.65057373046875,
+      "loss": 0.5809,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.0279461145401,
+      "rewards/margins": 0.6742316484451294,
+      "rewards/rejected": -1.70217764377594,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5103543036608107,
+      "eval_logits/chosen": -1.5488569736480713,
+      "eval_logits/rejected": -1.4997485876083374,
+      "eval_logps/chosen": -444.8670959472656,
+      "eval_logps/rejected": -463.78662109375,
+      "eval_loss": 0.5586755871772766,
+      "eval_rewards/accuracies": 0.727544903755188,
+      "eval_rewards/chosen": -0.9745256900787354,
+      "eval_rewards/margins": 0.6169502139091492,
+      "eval_rewards/rejected": -1.5914759635925293,
+      "eval_runtime": 297.1031,
+      "eval_samples_per_second": 6.732,
+      "eval_steps_per_second": 1.124,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5142801059966631,
+      "grad_norm": 2.650071382522583,
+      "learning_rate": 2.8110211704658073e-06,
+      "logits/chosen": -1.5513745546340942,
+      "logits/rejected": -1.4511005878448486,
+      "logps/chosen": -467.90277099609375,
+      "logps/rejected": -534.8497314453125,
+      "loss": 0.5412,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1337813138961792,
+      "rewards/margins": 0.7278991937637329,
+      "rewards/rejected": -1.8616803884506226,
+      "step": 1310
+    },
+    {
+      "epoch": 0.5182059083325155,
+      "grad_norm": 2.893183708190918,
+      "learning_rate": 2.776992300123732e-06,
+      "logits/chosen": -1.4944512844085693,
+      "logits/rejected": -1.4401706457138062,
+      "logps/chosen": -432.7562561035156,
+      "logps/rejected": -509.317626953125,
+      "loss": 0.5698,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.2368332147598267,
+      "rewards/margins": 0.5617424249649048,
+      "rewards/rejected": -1.7985754013061523,
+      "step": 1320
+    },
+    {
+      "epoch": 0.5221317106683678,
+      "grad_norm": 3.467410087585449,
+      "learning_rate": 2.742911390485262e-06,
+      "logits/chosen": -1.551828384399414,
+      "logits/rejected": -1.4773415327072144,
+      "logps/chosen": -510.3999938964844,
+      "logps/rejected": -479.6830139160156,
+      "loss": 0.5865,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.320778489112854,
+      "rewards/margins": 0.3750554919242859,
+      "rewards/rejected": -1.6958339214324951,
+      "step": 1330
+    },
+    {
+      "epoch": 0.5260575130042202,
+      "grad_norm": 2.826223611831665,
+      "learning_rate": 2.7087848444235354e-06,
+      "logits/chosen": -1.6097875833511353,
+      "logits/rejected": -1.5896812677383423,
+      "logps/chosen": -493.05792236328125,
+      "logps/rejected": -493.2828674316406,
+      "loss": 0.5408,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1781175136566162,
+      "rewards/margins": 0.651949405670166,
+      "rewards/rejected": -1.8300669193267822,
+      "step": 1340
+    },
+    {
+      "epoch": 0.5299833153400726,
+      "grad_norm": 2.8154220581054688,
+      "learning_rate": 2.674619073385531e-06,
+      "logits/chosen": -1.610780119895935,
+      "logits/rejected": -1.6012611389160156,
+      "logps/chosen": -489.55047607421875,
+      "logps/rejected": -460.4015197753906,
+      "loss": 0.5414,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.9827660322189331,
+      "rewards/margins": 0.49178582429885864,
+      "rewards/rejected": -1.474551796913147,
+      "step": 1350
+    },
+    {
+      "epoch": 0.533909117675925,
+      "grad_norm": 2.5804426670074463,
+      "learning_rate": 2.640420496187528e-06,
+      "logits/chosen": -1.6268459558486938,
+      "logits/rejected": -1.5926978588104248,
+      "logps/chosen": -454.572998046875,
+      "logps/rejected": -445.8119201660156,
+      "loss": 0.5567,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.9476795196533203,
+      "rewards/margins": 0.4974881708621979,
+      "rewards/rejected": -1.4451675415039062,
+      "step": 1360
+    },
+    {
+      "epoch": 0.5378349200117775,
+      "grad_norm": 2.428431749343872,
+      "learning_rate": 2.6061955378091896e-06,
+      "logits/chosen": -1.5578778982162476,
+      "logits/rejected": -1.4766862392425537,
+      "logps/chosen": -433.17669677734375,
+      "logps/rejected": -438.27178955078125,
+      "loss": 0.5458,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.0543098449707031,
+      "rewards/margins": 0.5402029752731323,
+      "rewards/rejected": -1.5945125818252563,
+      "step": 1370
+    },
+    {
+      "epoch": 0.5417607223476298,
+      "grad_norm": 3.3964052200317383,
+      "learning_rate": 2.5719506281864838e-06,
+      "logits/chosen": -1.5066369771957397,
+      "logits/rejected": -1.452606439590454,
+      "logps/chosen": -466.7701721191406,
+      "logps/rejected": -503.127197265625,
+      "loss": 0.5643,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1856281757354736,
+      "rewards/margins": 0.5961874723434448,
+      "rewards/rejected": -1.781815528869629,
+      "step": 1380
+    },
+    {
+      "epoch": 0.5456865246834822,
+      "grad_norm": 3.363842010498047,
+      "learning_rate": 2.537692201003671e-06,
+      "logits/chosen": -1.5076158046722412,
+      "logits/rejected": -1.4413753747940063,
+      "logps/chosen": -380.2969055175781,
+      "logps/rejected": -434.618896484375,
+      "loss": 0.5904,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.1435441970825195,
+      "rewards/margins": 0.4523565173149109,
+      "rewards/rejected": -1.5959007740020752,
+      "step": 1390
+    },
+    {
+      "epoch": 0.5496123270193346,
+      "grad_norm": 3.692343235015869,
+      "learning_rate": 2.503426692484594e-06,
+      "logits/chosen": -1.443113088607788,
+      "logits/rejected": -1.4232076406478882,
+      "logps/chosen": -536.1734619140625,
+      "logps/rejected": -561.1747436523438,
+      "loss": 0.5597,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.2561962604522705,
+      "rewards/margins": 0.6204220652580261,
+      "rewards/rejected": -1.8766181468963623,
+      "step": 1400
+    },
+    {
+      "epoch": 0.5496123270193346,
+      "eval_logits/chosen": -1.5439138412475586,
+      "eval_logits/rejected": -1.496781349182129,
+      "eval_logps/chosen": -459.4293518066406,
+      "eval_logps/rejected": -477.03887939453125,
+      "eval_loss": 0.553538978099823,
+      "eval_rewards/accuracies": 0.7380239367485046,
+      "eval_rewards/chosen": -1.1201481819152832,
+      "eval_rewards/margins": 0.6038500666618347,
+      "eval_rewards/rejected": -1.7239980697631836,
+      "eval_runtime": 294.7597,
+      "eval_samples_per_second": 6.785,
+      "eval_steps_per_second": 1.133,
+      "step": 1400
+    },
+    {
+      "epoch": 0.553538129355187,
+      "grad_norm": 2.692399263381958,
+      "learning_rate": 2.4691605401834843e-06,
+      "logits/chosen": -1.4821271896362305,
+      "logits/rejected": -1.465431571006775,
+      "logps/chosen": -434.5506896972656,
+      "logps/rejected": -492.6884765625,
+      "loss": 0.5527,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1405826807022095,
+      "rewards/margins": 0.5877465009689331,
+      "rewards/rejected": -1.7283293008804321,
+      "step": 1410
+    },
+    {
+      "epoch": 0.5574639316910394,
+      "grad_norm": 3.5580761432647705,
+      "learning_rate": 2.434900181775524e-06,
+      "logits/chosen": -1.5584666728973389,
+      "logits/rejected": -1.5402156114578247,
+      "logps/chosen": -460.17767333984375,
+      "logps/rejected": -440.67236328125,
+      "loss": 0.5572,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.0293381214141846,
+      "rewards/margins": 0.4713987410068512,
+      "rewards/rejected": -1.500736951828003,
+      "step": 1420
+    },
+    {
+      "epoch": 0.5613897340268917,
+      "grad_norm": 2.9644525051116943,
+      "learning_rate": 2.40065205384738e-06,
+      "logits/chosen": -1.5250346660614014,
+      "logits/rejected": -1.4797368049621582,
+      "logps/chosen": -460.33648681640625,
+      "logps/rejected": -440.6958923339844,
+      "loss": 0.5725,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.8073607683181763,
+      "rewards/margins": 0.5154079794883728,
+      "rewards/rejected": -1.3227688074111938,
+      "step": 1430
+    },
+    {
+      "epoch": 0.5653155363627441,
+      "grad_norm": 4.129930019378662,
+      "learning_rate": 2.3664225906879452e-06,
+      "logits/chosen": -1.513352632522583,
+      "logits/rejected": -1.4991798400878906,
+      "logps/chosen": -412.8216857910156,
+      "logps/rejected": -412.7459411621094,
+      "loss": 0.5674,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7217937707901001,
+      "rewards/margins": 0.46254992485046387,
+      "rewards/rejected": -1.1843438148498535,
+      "step": 1440
+    },
+    {
+      "epoch": 0.5692413386985965,
+      "grad_norm": 3.431654691696167,
+      "learning_rate": 2.3322182230795127e-06,
+      "logits/chosen": -1.4814140796661377,
+      "logits/rejected": -1.4627304077148438,
+      "logps/chosen": -399.53497314453125,
+      "logps/rejected": -441.78533935546875,
+      "loss": 0.5414,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8845251202583313,
+      "rewards/margins": 0.6136487722396851,
+      "rewards/rejected": -1.4981739521026611,
+      "step": 1450
+    },
+    {
+      "epoch": 0.5731671410344489,
+      "grad_norm": 3.988335371017456,
+      "learning_rate": 2.298045377089604e-06,
+      "logits/chosen": -1.6010191440582275,
+      "logits/rejected": -1.5490202903747559,
+      "logps/chosen": -495.11407470703125,
+      "logps/rejected": -465.97283935546875,
+      "loss": 0.6213,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.0170643329620361,
+      "rewards/margins": 0.37314504384994507,
+      "rewards/rejected": -1.3902093172073364,
+      "step": 1460
+    },
+    {
+      "epoch": 0.5770929433703013,
+      "grad_norm": 4.615077495574951,
+      "learning_rate": 2.2639104728636915e-06,
+      "logits/chosen": -1.5407696962356567,
+      "logits/rejected": -1.4720336198806763,
+      "logps/chosen": -423.0731506347656,
+      "logps/rejected": -420.0899963378906,
+      "loss": 0.5511,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8140323758125305,
+      "rewards/margins": 0.5152740478515625,
+      "rewards/rejected": -1.3293063640594482,
+      "step": 1470
+    },
+    {
+      "epoch": 0.5810187457061537,
+      "grad_norm": 3.13749361038208,
+      "learning_rate": 2.2298199234190236e-06,
+      "logits/chosen": -1.5636894702911377,
+      "logits/rejected": -1.55384361743927,
+      "logps/chosen": -475.68865966796875,
+      "logps/rejected": -496.26300048828125,
+      "loss": 0.6122,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.9036370515823364,
+      "rewards/margins": 0.5218679308891296,
+      "rewards/rejected": -1.4255050420761108,
+      "step": 1480
+    },
+    {
+      "epoch": 0.5849445480420061,
+      "grad_norm": 2.8493857383728027,
+      "learning_rate": 2.195780133439794e-06,
+      "logits/chosen": -1.4807980060577393,
+      "logits/rejected": -1.4771836996078491,
+      "logps/chosen": -432.133544921875,
+      "logps/rejected": -457.8267517089844,
+      "loss": 0.533,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.7880655527114868,
+      "rewards/margins": 0.7504501938819885,
+      "rewards/rejected": -1.5385158061981201,
+      "step": 1490
+    },
+    {
+      "epoch": 0.5888703503778585,
+      "grad_norm": 3.1914541721343994,
+      "learning_rate": 2.1617974980738814e-06,
+      "logits/chosen": -1.5311092138290405,
+      "logits/rejected": -1.47171950340271,
+      "logps/chosen": -456.5677795410156,
+      "logps/rejected": -454.6976013183594,
+      "loss": 0.5964,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8986822962760925,
+      "rewards/margins": 0.3416224420070648,
+      "rewards/rejected": -1.240304708480835,
+      "step": 1500
+    },
+    {
+      "epoch": 0.5888703503778585,
+      "eval_logits/chosen": -1.5311331748962402,
+      "eval_logits/rejected": -1.4828494787216187,
+      "eval_logps/chosen": -436.41461181640625,
+      "eval_logps/rejected": -452.6323547363281,
+      "eval_loss": 0.556452751159668,
+      "eval_rewards/accuracies": 0.735029935836792,
+      "eval_rewards/chosen": -0.8900012969970703,
+      "eval_rewards/margins": 0.5899319052696228,
+      "eval_rewards/rejected": -1.479933261871338,
+      "eval_runtime": 293.188,
+      "eval_samples_per_second": 6.822,
+      "eval_steps_per_second": 1.139,
+      "step": 1500
+    },
+    {
+      "epoch": 0.5927961527137109,
+      "grad_norm": 2.8875980377197266,
+      "learning_rate": 2.1278784017313688e-06,
+      "logits/chosen": -1.582453966140747,
+      "logits/rejected": -1.537969946861267,
+      "logps/chosen": -454.04364013671875,
+      "logps/rejected": -456.50445556640625,
+      "loss": 0.5785,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9088457226753235,
+      "rewards/margins": 0.5641152858734131,
+      "rewards/rejected": -1.4729610681533813,
+      "step": 1510
+    },
+    {
+      "epoch": 0.5967219550495633,
+      "grad_norm": 3.405135154724121,
+      "learning_rate": 2.0940292168850913e-06,
+      "logits/chosen": -1.457522988319397,
+      "logits/rejected": -1.4143173694610596,
+      "logps/chosen": -511.7796936035156,
+      "logps/rejected": -472.60882568359375,
+      "loss": 0.5505,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0364328622817993,
+      "rewards/margins": 0.681273877620697,
+      "rewards/rejected": -1.7177066802978516,
+      "step": 1520
+    },
+    {
+      "epoch": 0.6006477573854156,
+      "grad_norm": 3.9621615409851074,
+      "learning_rate": 2.060256302873421e-06,
+      "logits/chosen": -1.513451337814331,
+      "logits/rejected": -1.4876306056976318,
+      "logps/chosen": -437.97576904296875,
+      "logps/rejected": -466.766357421875,
+      "loss": 0.5671,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1324082612991333,
+      "rewards/margins": 0.5312899351119995,
+      "rewards/rejected": -1.6636981964111328,
+      "step": 1530
+    },
+    {
+      "epoch": 0.604573559721268,
+      "grad_norm": 3.5171825885772705,
+      "learning_rate": 2.02656600470552e-06,
+      "logits/chosen": -1.4176056385040283,
+      "logits/rejected": -1.4338403940200806,
+      "logps/chosen": -372.839111328125,
+      "logps/rejected": -401.86334228515625,
+      "loss": 0.5829,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9607407450675964,
+      "rewards/margins": 0.5128813982009888,
+      "rewards/rejected": -1.4736220836639404,
+      "step": 1540
+    },
+    {
+      "epoch": 0.6084993620571204,
+      "grad_norm": 3.5174639225006104,
+      "learning_rate": 1.99296465186929e-06,
+      "logits/chosen": -1.5188140869140625,
+      "logits/rejected": -1.5075643062591553,
+      "logps/chosen": -427.47747802734375,
+      "logps/rejected": -443.4689025878906,
+      "loss": 0.5886,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9522552490234375,
+      "rewards/margins": 0.519784152507782,
+      "rewards/rejected": -1.4720392227172852,
+      "step": 1550
+    },
+    {
+      "epoch": 0.6124251643929728,
+      "grad_norm": 3.5703771114349365,
+      "learning_rate": 1.959458557142228e-06,
+      "logits/chosen": -1.3921113014221191,
+      "logits/rejected": -1.3389828205108643,
+      "logps/chosen": -454.9217224121094,
+      "logps/rejected": -448.68145751953125,
+      "loss": 0.5329,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.1833871603012085,
+      "rewards/margins": 0.59441077709198,
+      "rewards/rejected": -1.777798056602478,
+      "step": 1560
+    },
+    {
+      "epoch": 0.6163509667288252,
+      "grad_norm": 2.4494590759277344,
+      "learning_rate": 1.9260540154054317e-06,
+      "logits/chosen": -1.5239397287368774,
+      "logits/rejected": -1.4449244737625122,
+      "logps/chosen": -434.91668701171875,
+      "logps/rejected": -473.08941650390625,
+      "loss": 0.5221,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.147991418838501,
+      "rewards/margins": 0.6677936315536499,
+      "rewards/rejected": -1.8157850503921509,
+      "step": 1570
+    },
+    {
+      "epoch": 0.6202767690646775,
+      "grad_norm": 2.9639182090759277,
+      "learning_rate": 1.8927573024609666e-06,
+      "logits/chosen": -1.5022614002227783,
+      "logits/rejected": -1.476105809211731,
+      "logps/chosen": -501.60137939453125,
+      "logps/rejected": -555.4838256835938,
+      "loss": 0.5129,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.2436672449111938,
+      "rewards/margins": 0.6114083528518677,
+      "rewards/rejected": -1.8550755977630615,
+      "step": 1580
+    },
+    {
+      "epoch": 0.62420257140053,
+      "grad_norm": 3.5787737369537354,
+      "learning_rate": 1.8595746738528045e-06,
+      "logits/chosen": -1.4589451551437378,
+      "logits/rejected": -1.4299007654190063,
+      "logps/chosen": -424.246337890625,
+      "logps/rejected": -459.80767822265625,
+      "loss": 0.5722,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1273281574249268,
+      "rewards/margins": 0.5254072546958923,
+      "rewards/rejected": -1.6527354717254639,
+      "step": 1590
+    },
+    {
+      "epoch": 0.6281283737363824,
+      "grad_norm": 3.5686867237091064,
+      "learning_rate": 1.826512363691586e-06,
+      "logits/chosen": -1.5050278902053833,
+      "logits/rejected": -1.4701493978500366,
+      "logps/chosen": -469.849609375,
+      "logps/rejected": -480.487060546875,
+      "loss": 0.5329,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9026687741279602,
+      "rewards/margins": 0.666443407535553,
+      "rewards/rejected": -1.5691120624542236,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6281283737363824,
+      "eval_logits/chosen": -1.5115160942077637,
+      "eval_logits/rejected": -1.4628429412841797,
+      "eval_logps/chosen": -457.0048522949219,
+      "eval_logps/rejected": -478.6323547363281,
+      "eval_loss": 0.5533447861671448,
+      "eval_rewards/accuracies": 0.7365269660949707,
+      "eval_rewards/chosen": -1.0959036350250244,
+      "eval_rewards/margins": 0.6440290212631226,
+      "eval_rewards/rejected": -1.739932656288147,
+      "eval_runtime": 293.1399,
+      "eval_samples_per_second": 6.823,
+      "eval_steps_per_second": 1.139,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6320541760722348,
+      "grad_norm": 3.2954156398773193,
+      "learning_rate": 1.7935765834833966e-06,
+      "logits/chosen": -1.519984483718872,
+      "logits/rejected": -1.4952067136764526,
+      "logps/chosen": -431.4827575683594,
+      "logps/rejected": -437.34686279296875,
+      "loss": 0.5769,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0700024366378784,
+      "rewards/margins": 0.5090070962905884,
+      "rewards/rejected": -1.5790095329284668,
+      "step": 1610
+    },
+    {
+      "epoch": 0.6359799784080872,
+      "grad_norm": 3.408398389816284,
+      "learning_rate": 1.7607735209627953e-06,
+      "logits/chosen": -1.5044435262680054,
+      "logits/rejected": -1.415461778640747,
+      "logps/chosen": -432.9769592285156,
+      "logps/rejected": -445.25885009765625,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.076953411102295,
+      "rewards/margins": 0.48173418641090393,
+      "rewards/rejected": -1.5586875677108765,
+      "step": 1620
+    },
+    {
+      "epoch": 0.6399057807439396,
+      "grad_norm": 3.0543644428253174,
+      "learning_rate": 1.7281093389303105e-06,
+      "logits/chosen": -1.5522346496582031,
+      "logits/rejected": -1.4349719285964966,
+      "logps/chosen": -500.4112243652344,
+      "logps/rejected": -495.94873046875,
+      "loss": 0.5616,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.1183501482009888,
+      "rewards/margins": 0.7165015935897827,
+      "rewards/rejected": -1.834851861000061,
+      "step": 1630
+    },
+    {
+      "epoch": 0.6438315830797919,
+      "grad_norm": 2.5816094875335693,
+      "learning_rate": 1.6955901740946136e-06,
+      "logits/chosen": -1.571271538734436,
+      "logits/rejected": -1.5148191452026367,
+      "logps/chosen": -512.9193115234375,
+      "logps/rejected": -462.5840759277344,
+      "loss": 0.5292,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.148260235786438,
+      "rewards/margins": 0.5933681726455688,
+      "rewards/rejected": -1.7416284084320068,
+      "step": 1640
+    },
+    {
+      "epoch": 0.6477573854156443,
+      "grad_norm": 2.953559398651123,
+      "learning_rate": 1.663222135919601e-06,
+      "logits/chosen": -1.413680076599121,
+      "logits/rejected": -1.3973824977874756,
+      "logps/chosen": -415.37164306640625,
+      "logps/rejected": -459.9598693847656,
+      "loss": 0.5713,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.2905628681182861,
+      "rewards/margins": 0.3814533054828644,
+      "rewards/rejected": -1.6720163822174072,
+      "step": 1650
+    },
+    {
+      "epoch": 0.6516831877514967,
+      "grad_norm": 3.10904860496521,
+      "learning_rate": 1.6310113054765947e-06,
+      "logits/chosen": -1.4589123725891113,
+      "logits/rejected": -1.4665824174880981,
+      "logps/chosen": -448.07598876953125,
+      "logps/rejected": -464.26776123046875,
+      "loss": 0.5175,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1187959909439087,
+      "rewards/margins": 0.6980869770050049,
+      "rewards/rejected": -1.816882848739624,
+      "step": 1660
+    },
+    {
+      "epoch": 0.6556089900873491,
+      "grad_norm": 2.5229618549346924,
+      "learning_rate": 1.5989637343018705e-06,
+      "logits/chosen": -1.4531184434890747,
+      "logits/rejected": -1.443787693977356,
+      "logps/chosen": -419.0887145996094,
+      "logps/rejected": -479.41790771484375,
+      "loss": 0.5455,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.9844555854797363,
+      "rewards/margins": 0.6616882085800171,
+      "rewards/rejected": -1.6461436748504639,
+      "step": 1670
+    },
+    {
+      "epoch": 0.6595347924232015,
+      "grad_norm": 3.533864974975586,
+      "learning_rate": 1.5670854432597433e-06,
+      "logits/chosen": -1.4726221561431885,
+      "logits/rejected": -1.4184763431549072,
+      "logps/chosen": -526.8727416992188,
+      "logps/rejected": -481.2684020996094,
+      "loss": 0.5243,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.0825868844985962,
+      "rewards/margins": 0.6506483554840088,
+      "rewards/rejected": -1.7332350015640259,
+      "step": 1680
+    },
+    {
+      "epoch": 0.6634605947590538,
+      "grad_norm": 3.316822052001953,
+      "learning_rate": 1.5353824214114075e-06,
+      "logits/chosen": -1.464456558227539,
+      "logits/rejected": -1.3914768695831299,
+      "logps/chosen": -411.2251892089844,
+      "logps/rejected": -467.06170654296875,
+      "loss": 0.5235,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.0634043216705322,
+      "rewards/margins": 0.7228339314460754,
+      "rewards/rejected": -1.7862380743026733,
+      "step": 1690
+    },
+    {
+      "epoch": 0.6673863970949063,
+      "grad_norm": 3.873871088027954,
+      "learning_rate": 1.5038606248897586e-06,
+      "logits/chosen": -1.4639793634414673,
+      "logits/rejected": -1.3958394527435303,
+      "logps/chosen": -396.053955078125,
+      "logps/rejected": -427.2110290527344,
+      "loss": 0.5701,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.0697581768035889,
+      "rewards/margins": 0.47869449853897095,
+      "rewards/rejected": -1.548452615737915,
+      "step": 1700
+    },
+    {
+      "epoch": 0.6673863970949063,
+      "eval_logits/chosen": -1.5060590505599976,
+      "eval_logits/rejected": -1.4577833414077759,
+      "eval_logps/chosen": -458.0072937011719,
+      "eval_logps/rejected": -481.96514892578125,
+      "eval_loss": 0.5519984364509583,
+      "eval_rewards/accuracies": 0.742514967918396,
+      "eval_rewards/chosen": -1.1059272289276123,
+      "eval_rewards/margins": 0.6673337817192078,
+      "eval_rewards/rejected": -1.7732609510421753,
+      "eval_runtime": 294.0853,
+      "eval_samples_per_second": 6.801,
+      "eval_steps_per_second": 1.136,
+      "step": 1700
+    },
+    {
+      "epoch": 0.6713121994307587,
+      "grad_norm": 4.115907192230225,
+      "learning_rate": 1.4725259757803983e-06,
+      "logits/chosen": -1.5623457431793213,
+      "logits/rejected": -1.5087127685546875,
+      "logps/chosen": -560.9649047851562,
+      "logps/rejected": -505.98321533203125,
+      "loss": 0.5669,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.1059812307357788,
+      "rewards/margins": 0.6403836011886597,
+      "rewards/rejected": -1.746364951133728,
+      "step": 1710
+    },
+    {
+      "epoch": 0.6752380017666111,
+      "grad_norm": 2.6118557453155518,
+      "learning_rate": 1.4413843610090342e-06,
+      "logits/chosen": -1.5451549291610718,
+      "logits/rejected": -1.473829984664917,
+      "logps/chosen": -464.28717041015625,
+      "logps/rejected": -516.9815673828125,
+      "loss": 0.5467,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0781443119049072,
+      "rewards/margins": 0.6509603261947632,
+      "rewards/rejected": -1.7291046380996704,
+      "step": 1720
+    },
+    {
+      "epoch": 0.6791638041024635,
+      "grad_norm": 3.4120659828186035,
+      "learning_rate": 1.410441631235487e-06,
+      "logits/chosen": -1.5216848850250244,
+      "logits/rejected": -1.5256197452545166,
+      "logps/chosen": -387.226806640625,
+      "logps/rejected": -409.0061340332031,
+      "loss": 0.5415,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0608408451080322,
+      "rewards/margins": 0.5060892105102539,
+      "rewards/rejected": -1.5669299364089966,
+      "step": 1730
+    },
+    {
+      "epoch": 0.6830896064383158,
+      "grad_norm": 3.4134604930877686,
+      "learning_rate": 1.3797035997545144e-06,
+      "logits/chosen": -1.4526493549346924,
+      "logits/rejected": -1.4065980911254883,
+      "logps/chosen": -453.49237060546875,
+      "logps/rejected": -489.89080810546875,
+      "loss": 0.5524,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0430675745010376,
+      "rewards/margins": 0.8062189221382141,
+      "rewards/rejected": -1.849286437034607,
+      "step": 1740
+    },
+    {
+      "epoch": 0.6870154087741682,
+      "grad_norm": 3.8936967849731445,
+      "learning_rate": 1.3491760414036478e-06,
+      "logits/chosen": -1.4090884923934937,
+      "logits/rejected": -1.3523749113082886,
+      "logps/chosen": -452.6044921875,
+      "logps/rejected": -462.2596740722656,
+      "loss": 0.5505,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.2693541049957275,
+      "rewards/margins": 0.6317874193191528,
+      "rewards/rejected": -1.9011415243148804,
+      "step": 1750
+    },
+    {
+      "epoch": 0.6909412111100206,
+      "grad_norm": 3.299311637878418,
+      "learning_rate": 1.3188646914782616e-06,
+      "logits/chosen": -1.3984262943267822,
+      "logits/rejected": -1.3706824779510498,
+      "logps/chosen": -449.51409912109375,
+      "logps/rejected": -463.17852783203125,
+      "loss": 0.5377,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.2553191184997559,
+      "rewards/margins": 0.5780755877494812,
+      "rewards/rejected": -1.8333946466445923,
+      "step": 1760
+    },
+    {
+      "epoch": 0.694867013445873,
+      "grad_norm": 4.783185005187988,
+      "learning_rate": 1.288775244654062e-06,
+      "logits/chosen": -1.504199743270874,
+      "logits/rejected": -1.4564763307571411,
+      "logps/chosen": -481.9203186035156,
+      "logps/rejected": -502.52777099609375,
+      "loss": 0.5145,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.0656607151031494,
+      "rewards/margins": 1.0003831386566162,
+      "rewards/rejected": -2.0660440921783447,
+      "step": 1770
+    },
+    {
+      "epoch": 0.6987928157817254,
+      "grad_norm": 3.704270124435425,
+      "learning_rate": 1.2589133539172193e-06,
+      "logits/chosen": -1.5180060863494873,
+      "logits/rejected": -1.4053322076797485,
+      "logps/chosen": -450.322021484375,
+      "logps/rejected": -503.0769958496094,
+      "loss": 0.5397,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.103685736656189,
+      "rewards/margins": 0.7220897674560547,
+      "rewards/rejected": -1.8257755041122437,
+      "step": 1780
+    },
+    {
+      "epoch": 0.7027186181175777,
+      "grad_norm": 3.3040413856506348,
+      "learning_rate": 1.2292846295023222e-06,
+      "logits/chosen": -1.518012285232544,
+      "logits/rejected": -1.5066865682601929,
+      "logps/chosen": -450.42767333984375,
+      "logps/rejected": -465.0337829589844,
+      "loss": 0.5911,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0928590297698975,
+      "rewards/margins": 0.5593289136886597,
+      "rewards/rejected": -1.6521879434585571,
+      "step": 1790
+    },
+    {
+      "epoch": 0.7066444204534301,
+      "grad_norm": 3.3465688228607178,
+      "learning_rate": 1.19989463783837e-06,
+      "logits/chosen": -1.415979027748108,
+      "logits/rejected": -1.3558562994003296,
+      "logps/chosen": -464.41876220703125,
+      "logps/rejected": -511.23553466796875,
+      "loss": 0.5522,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.156848669052124,
+      "rewards/margins": 0.6018009185791016,
+      "rewards/rejected": -1.7586495876312256,
+      "step": 1800
+    },
+    {
+      "epoch": 0.7066444204534301,
+      "eval_logits/chosen": -1.495127558708191,
+      "eval_logits/rejected": -1.446120262145996,
+      "eval_logps/chosen": -452.52667236328125,
+      "eval_logps/rejected": -476.23040771484375,
+      "eval_loss": 0.5523282885551453,
+      "eval_rewards/accuracies": 0.7380239367485046,
+      "eval_rewards/chosen": -1.0511209964752197,
+      "eval_rewards/margins": 0.6647923588752747,
+      "eval_rewards/rejected": -1.7159132957458496,
+      "eval_runtime": 294.0354,
+      "eval_samples_per_second": 6.802,
+      "eval_steps_per_second": 1.136,
+      "step": 1800
+    },
+    {
+      "epoch": 0.7105702227892826,
+      "grad_norm": 4.849062919616699,
+      "learning_rate": 1.1707489005029877e-06,
+      "logits/chosen": -1.4643745422363281,
+      "logits/rejected": -1.4222137928009033,
+      "logps/chosen": -434.943603515625,
+      "logps/rejected": -453.41143798828125,
+      "loss": 0.5783,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.957152247428894,
+      "rewards/margins": 0.6891154646873474,
+      "rewards/rejected": -1.6462681293487549,
+      "step": 1810
+    },
+    {
+      "epoch": 0.714496025125135,
+      "grad_norm": 3.1195292472839355,
+      "learning_rate": 1.1418528931850781e-06,
+      "logits/chosen": -1.384520411491394,
+      "logits/rejected": -1.3301794528961182,
+      "logps/chosen": -419.5125427246094,
+      "logps/rejected": -459.73822021484375,
+      "loss": 0.5078,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9871832132339478,
+      "rewards/margins": 0.7259316444396973,
+      "rewards/rejected": -1.7131149768829346,
+      "step": 1820
+    },
+    {
+      "epoch": 0.7184218274609874,
+      "grad_norm": 3.167475938796997,
+      "learning_rate": 1.113212044656087e-06,
+      "logits/chosen": -1.4200365543365479,
+      "logits/rejected": -1.38724684715271,
+      "logps/chosen": -492.0989685058594,
+      "logps/rejected": -546.4012451171875,
+      "loss": 0.5334,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.0492780208587646,
+      "rewards/margins": 0.7958940863609314,
+      "rewards/rejected": -1.8451721668243408,
+      "step": 1830
+    },
+    {
+      "epoch": 0.7223476297968398,
+      "grad_norm": 5.2765302658081055,
+      "learning_rate": 1.0848317357500854e-06,
+      "logits/chosen": -1.486344337463379,
+      "logits/rejected": -1.4261810779571533,
+      "logps/chosen": -442.8671875,
+      "logps/rejected": -479.9161682128906,
+      "loss": 0.5186,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.0721113681793213,
+      "rewards/margins": 0.7349880337715149,
+      "rewards/rejected": -1.8070993423461914,
+      "step": 1840
+    },
+    {
+      "epoch": 0.7262734321326921,
+      "grad_norm": 3.378751754760742,
+      "learning_rate": 1.0567172983528534e-06,
+      "logits/chosen": -1.5136101245880127,
+      "logits/rejected": -1.4122953414916992,
+      "logps/chosen": -522.0455322265625,
+      "logps/rejected": -550.4339599609375,
+      "loss": 0.5356,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.1462188959121704,
+      "rewards/margins": 0.6934286952018738,
+      "rewards/rejected": -1.8396475315093994,
+      "step": 1850
+    },
+    {
+      "epoch": 0.7301992344685445,
+      "grad_norm": 5.7028656005859375,
+      "learning_rate": 1.0288740144001722e-06,
+      "logits/chosen": -1.4152565002441406,
+      "logits/rejected": -1.3890769481658936,
+      "logps/chosen": -452.39752197265625,
+      "logps/rejected": -471.009033203125,
+      "loss": 0.507,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0435569286346436,
+      "rewards/margins": 0.6164140105247498,
+      "rewards/rejected": -1.659970998764038,
+      "step": 1860
+    },
+    {
+      "epoch": 0.7341250368043969,
+      "grad_norm": 2.870126962661743,
+      "learning_rate": 1.0013071148854861e-06,
+      "logits/chosen": -1.4720185995101929,
+      "logits/rejected": -1.4466623067855835,
+      "logps/chosen": -409.47943115234375,
+      "logps/rejected": -497.20947265625,
+      "loss": 0.5667,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0388069152832031,
+      "rewards/margins": 0.595206081867218,
+      "rewards/rejected": -1.6340129375457764,
+      "step": 1870
+    },
+    {
+      "epoch": 0.7380508391402493,
+      "grad_norm": 3.388361930847168,
+      "learning_rate": 9.740217788771453e-07,
+      "logits/chosen": -1.467180848121643,
+      "logits/rejected": -1.3647053241729736,
+      "logps/chosen": -412.0126953125,
+      "logps/rejected": -443.46868896484375,
+      "loss": 0.5949,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.990081787109375,
+      "rewards/margins": 0.6179161071777344,
+      "rewards/rejected": -1.6079976558685303,
+      "step": 1880
+    },
+    {
+      "epoch": 0.7419766414761017,
+      "grad_norm": 3.0970327854156494,
+      "learning_rate": 9.470231325453958e-07,
+      "logits/chosen": -1.4770920276641846,
+      "logits/rejected": -1.4540226459503174,
+      "logps/chosen": -402.5322570800781,
+      "logps/rejected": -438.1739807128906,
+      "loss": 0.581,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.0194588899612427,
+      "rewards/margins": 0.5488156080245972,
+      "rewards/rejected": -1.5682743787765503,
+      "step": 1890
+    },
+    {
+      "epoch": 0.745902443811954,
+      "grad_norm": 2.47135853767395,
+      "learning_rate": 9.203162481993175e-07,
+      "logits/chosen": -1.4647947549819946,
+      "logits/rejected": -1.4300190210342407,
+      "logps/chosen": -447.2220764160156,
+      "logps/rejected": -472.08526611328125,
+      "loss": 0.5659,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0458368062973022,
+      "rewards/margins": 0.5263345241546631,
+      "rewards/rejected": -1.5721712112426758,
+      "step": 1900
+    },
+    {
+      "epoch": 0.745902443811954,
+      "eval_logits/chosen": -1.4980415105819702,
+      "eval_logits/rejected": -1.449204921722412,
+      "eval_logps/chosen": -440.41302490234375,
+      "eval_logps/rejected": -461.88916015625,
+      "eval_loss": 0.5553146004676819,
+      "eval_rewards/accuracies": 0.7365269660949707,
+      "eval_rewards/chosen": -0.9299851655960083,
+      "eval_rewards/margins": 0.642516016960144,
+      "eval_rewards/rejected": -1.5725011825561523,
+      "eval_runtime": 292.7238,
+      "eval_samples_per_second": 6.832,
+      "eval_steps_per_second": 1.141,
+      "step": 1900
+    },
+    {
+      "epoch": 0.7498282461478064,
+      "grad_norm": 4.308950901031494,
+      "learning_rate": 8.939061433338722e-07,
+      "logits/chosen": -1.5734995603561401,
+      "logits/rejected": -1.5377581119537354,
+      "logps/chosen": -481.59814453125,
+      "logps/rejected": -504.37628173828125,
+      "loss": 0.5329,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.7615936994552612,
+      "rewards/margins": 0.7267706990242004,
+      "rewards/rejected": -1.4883644580841064,
+      "step": 1910
+    },
+    {
+      "epoch": 0.7537540484836589,
+      "grad_norm": 2.9517087936401367,
+      "learning_rate": 8.677977796872541e-07,
+      "logits/chosen": -1.4577100276947021,
+      "logits/rejected": -1.4005043506622314,
+      "logps/chosen": -380.909423828125,
+      "logps/rejected": -415.84912109375,
+      "loss": 0.6019,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.953689694404602,
+      "rewards/margins": 0.6682855486869812,
+      "rewards/rejected": -1.621975302696228,
+      "step": 1920
+    },
+    {
+      "epoch": 0.7576798508195113,
+      "grad_norm": 4.669971942901611,
+      "learning_rate": 8.419960623087129e-07,
+      "logits/chosen": -1.3476511240005493,
+      "logits/rejected": -1.3028614521026611,
+      "logps/chosen": -469.5086975097656,
+      "logps/rejected": -444.77734375,
+      "loss": 0.5833,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.1782282590866089,
+      "rewards/margins": 0.5026072263717651,
+      "rewards/rejected": -1.680835485458374,
+      "step": 1930
+    },
+    {
+      "epoch": 0.7616056531553637,
+      "grad_norm": 2.7053141593933105,
+      "learning_rate": 8.165058386370314e-07,
+      "logits/chosen": -1.3782241344451904,
+      "logits/rejected": -1.3696388006210327,
+      "logps/chosen": -427.6993713378906,
+      "logps/rejected": -455.4295349121094,
+      "loss": 0.5407,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.0545322895050049,
+      "rewards/margins": 0.5850927233695984,
+      "rewards/rejected": -1.6396249532699585,
+      "step": 1940
+    },
+    {
+      "epoch": 0.765531455491216,
+      "grad_norm": 3.4946670532226562,
+      "learning_rate": 7.913318975898238e-07,
+      "logits/chosen": -1.4812867641448975,
+      "logits/rejected": -1.405998945236206,
+      "logps/chosen": -514.7869873046875,
+      "logps/rejected": -525.06201171875,
+      "loss": 0.5789,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.002190351486206,
+      "rewards/margins": 0.7534323334693909,
+      "rewards/rejected": -1.7556226253509521,
+      "step": 1950
+    },
+    {
+      "epoch": 0.7694572578270684,
+      "grad_norm": 4.065061569213867,
+      "learning_rate": 7.664789686638272e-07,
+      "logits/chosen": -1.4213950634002686,
+      "logits/rejected": -1.357412576675415,
+      "logps/chosen": -499.64111328125,
+      "logps/rejected": -460.8352966308594,
+      "loss": 0.5315,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9986658096313477,
+      "rewards/margins": 0.7509936094284058,
+      "rewards/rejected": -1.7496591806411743,
+      "step": 1960
+    },
+    {
+      "epoch": 0.7733830601629208,
+      "grad_norm": 2.915386915206909,
+      "learning_rate": 7.41951721046357e-07,
+      "logits/chosen": -1.4658726453781128,
+      "logits/rejected": -1.400398850440979,
+      "logps/chosen": -448.7250061035156,
+      "logps/rejected": -445.1400451660156,
+      "loss": 0.5394,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1403663158416748,
+      "rewards/margins": 0.6125134229660034,
+      "rewards/rejected": -1.7528797388076782,
+      "step": 1970
+    },
+    {
+      "epoch": 0.7773088624987732,
+      "grad_norm": 3.1753182411193848,
+      "learning_rate": 7.177547627380987e-07,
+      "logits/chosen": -1.4184179306030273,
+      "logits/rejected": -1.3604623079299927,
+      "logps/chosen": -423.7699279785156,
+      "logps/rejected": -498.23016357421875,
+      "loss": 0.5411,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0760539770126343,
+      "rewards/margins": 0.7359834909439087,
+      "rewards/rejected": -1.8120372295379639,
+      "step": 1980
+    },
+    {
+      "epoch": 0.7812346648346256,
+      "grad_norm": 3.0003552436828613,
+      "learning_rate": 6.93892639687386e-07,
+      "logits/chosen": -1.477745532989502,
+      "logits/rejected": -1.4782835245132446,
+      "logps/chosen": -419.5091857910156,
+      "logps/rejected": -446.32421875,
+      "loss": 0.5322,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0432634353637695,
+      "rewards/margins": 0.5055574178695679,
+      "rewards/rejected": -1.5488207340240479,
+      "step": 1990
+    },
+    {
+      "epoch": 0.7851604671704779,
+      "grad_norm": 3.448577880859375,
+      "learning_rate": 6.703698349361437e-07,
+      "logits/chosen": -1.4692994356155396,
+      "logits/rejected": -1.4076192378997803,
+      "logps/chosen": -430.3692932128906,
+      "logps/rejected": -472.45770263671875,
+      "loss": 0.5375,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.9906840324401855,
+      "rewards/margins": 0.8723163604736328,
+      "rewards/rejected": -1.863000512123108,
+      "step": 2000
+    },
+    {
+      "epoch": 0.7851604671704779,
+      "eval_logits/chosen": -1.476766586303711,
+      "eval_logits/rejected": -1.4278054237365723,
+      "eval_logps/chosen": -458.3736572265625,
+      "eval_logps/rejected": -481.2357177734375,
+      "eval_loss": 0.5503016710281372,
+      "eval_rewards/accuracies": 0.7440119981765747,
+      "eval_rewards/chosen": -1.1095911264419556,
+      "eval_rewards/margins": 0.6563754677772522,
+      "eval_rewards/rejected": -1.765966534614563,
+      "eval_runtime": 293.1552,
+      "eval_samples_per_second": 6.822,
+      "eval_steps_per_second": 1.139,
+      "step": 2000
+    },
+    {
+      "epoch": 0.7890862695063303,
+      "grad_norm": 3.277191638946533,
+      "learning_rate": 6.471907677776426e-07,
+      "logits/chosen": -1.4940061569213867,
+      "logits/rejected": -1.411240816116333,
+      "logps/chosen": -450.0491638183594,
+      "logps/rejected": -493.69110107421875,
+      "loss": 0.5533,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1073410511016846,
+      "rewards/margins": 0.6493383646011353,
+      "rewards/rejected": -1.7566792964935303,
+      "step": 2010
+    },
+    {
+      "epoch": 0.7930120718421827,
+      "grad_norm": 4.704875946044922,
+      "learning_rate": 6.243597929262404e-07,
+      "logits/chosen": -1.5036876201629639,
+      "logits/rejected": -1.4907557964324951,
+      "logps/chosen": -433.5472106933594,
+      "logps/rejected": -468.02642822265625,
+      "loss": 0.5754,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.2662441730499268,
+      "rewards/margins": 0.553165078163147,
+      "rewards/rejected": -1.8194093704223633,
+      "step": 2020
+    },
+    {
+      "epoch": 0.7969378741780352,
+      "grad_norm": 5.0611891746521,
+      "learning_rate": 6.018811996992455e-07,
+      "logits/chosen": -1.4318408966064453,
+      "logits/rejected": -1.3853800296783447,
+      "logps/chosen": -454.83026123046875,
+      "logps/rejected": -457.53350830078125,
+      "loss": 0.5387,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0497772693634033,
+      "rewards/margins": 0.6260667443275452,
+      "rewards/rejected": -1.6758439540863037,
+      "step": 2030
+    },
+    {
+      "epoch": 0.8008636765138876,
+      "grad_norm": 4.334845542907715,
+      "learning_rate": 5.797592112110734e-07,
+      "logits/chosen": -1.4254496097564697,
+      "logits/rejected": -1.3859260082244873,
+      "logps/chosen": -429.9593811035156,
+      "logps/rejected": -492.8987731933594,
+      "loss": 0.5258,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1567713022232056,
+      "rewards/margins": 0.4895525574684143,
+      "rewards/rejected": -1.646323800086975,
+      "step": 2040
+    },
+    {
+      "epoch": 0.80478947884974,
+      "grad_norm": 3.2099335193634033,
+      "learning_rate": 5.579979835798361e-07,
+      "logits/chosen": -1.5321909189224243,
+      "logits/rejected": -1.5133243799209595,
+      "logps/chosen": -517.7656860351562,
+      "logps/rejected": -497.2415466308594,
+      "loss": 0.5798,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0302859544754028,
+      "rewards/margins": 0.6422498226165771,
+      "rewards/rejected": -1.6725358963012695,
+      "step": 2050
+    },
+    {
+      "epoch": 0.8087152811855923,
+      "grad_norm": 2.798924446105957,
+      "learning_rate": 5.366016051465245e-07,
+      "logits/chosen": -1.3858851194381714,
+      "logits/rejected": -1.367538332939148,
+      "logps/chosen": -460.4857482910156,
+      "logps/rejected": -503.989501953125,
+      "loss": 0.4947,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1764501333236694,
+      "rewards/margins": 0.7395855188369751,
+      "rewards/rejected": -1.9160358905792236,
+      "step": 2060
+    },
+    {
+      "epoch": 0.8126410835214447,
+      "grad_norm": 2.9203500747680664,
+      "learning_rate": 5.155740957069186e-07,
+      "logits/chosen": -1.5713320970535278,
+      "logits/rejected": -1.5787502527236938,
+      "logps/chosen": -474.79931640625,
+      "logps/rejected": -505.1097106933594,
+      "loss": 0.5449,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.0999139547348022,
+      "rewards/margins": 0.6679128408432007,
+      "rewards/rejected": -1.767827033996582,
+      "step": 2070
+    },
+    {
+      "epoch": 0.8165668858572971,
+      "grad_norm": 3.334927797317505,
+      "learning_rate": 4.949194057563783e-07,
+      "logits/chosen": -1.427640676498413,
+      "logits/rejected": -1.4084398746490479,
+      "logps/chosen": -469.8573303222656,
+      "logps/rejected": -547.0362548828125,
+      "loss": 0.5286,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.3319743871688843,
+      "rewards/margins": 0.6832603812217712,
+      "rewards/rejected": -2.0152347087860107,
+      "step": 2080
+    },
+    {
+      "epoch": 0.8204926881931495,
+      "grad_norm": 4.331505298614502,
+      "learning_rate": 4.746414157476506e-07,
+      "logits/chosen": -1.5367690324783325,
+      "logits/rejected": -1.541548490524292,
+      "logps/chosen": -422.27459716796875,
+      "logps/rejected": -508.6590881347656,
+      "loss": 0.5515,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.2113606929779053,
+      "rewards/margins": 0.6534624099731445,
+      "rewards/rejected": -1.8648231029510498,
+      "step": 2090
+    },
+    {
+      "epoch": 0.8244184905290018,
+      "grad_norm": 3.515759229660034,
+      "learning_rate": 4.5474393536184214e-07,
+      "logits/chosen": -1.450679898262024,
+      "logits/rejected": -1.3933296203613281,
+      "logps/chosen": -411.50439453125,
+      "logps/rejected": -428.12890625,
+      "loss": 0.5836,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1269792318344116,
+      "rewards/margins": 0.5875270962715149,
+      "rewards/rejected": -1.7145063877105713,
+      "step": 2100
+    },
+    {
+      "epoch": 0.8244184905290018,
+      "eval_logits/chosen": -1.4632281064987183,
+      "eval_logits/rejected": -1.4142000675201416,
+      "eval_logps/chosen": -462.6366882324219,
+      "eval_logps/rejected": -486.8011169433594,
+      "eval_loss": 0.5494332313537598,
+      "eval_rewards/accuracies": 0.7395209670066833,
+      "eval_rewards/chosen": -1.152221441268921,
+      "eval_rewards/margins": 0.6693998575210571,
+      "eval_rewards/rejected": -1.821621060371399,
+      "eval_runtime": 293.2554,
+      "eval_samples_per_second": 6.82,
+      "eval_steps_per_second": 1.139,
+      "step": 2100
+    },
+    {
+      "epoch": 0.8283442928648542,
+      "grad_norm": 3.203925132751465,
+      "learning_rate": 4.352307027926828e-07,
+      "logits/chosen": -1.4488589763641357,
+      "logits/rejected": -1.397608757019043,
+      "logps/chosen": -487.95263671875,
+      "logps/rejected": -509.68670654296875,
+      "loss": 0.5371,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.2453491687774658,
+      "rewards/margins": 0.662249743938446,
+      "rewards/rejected": -1.907598853111267,
+      "step": 2110
+    },
+    {
+      "epoch": 0.8322700952007066,
+      "grad_norm": 3.6506752967834473,
+      "learning_rate": 4.1610538404421837e-07,
+      "logits/chosen": -1.4884932041168213,
+      "logits/rejected": -1.422940969467163,
+      "logps/chosen": -440.89898681640625,
+      "logps/rejected": -531.4912719726562,
+      "loss": 0.5689,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1230660676956177,
+      "rewards/margins": 0.7992385625839233,
+      "rewards/rejected": -1.9223047494888306,
+      "step": 2120
+    },
+    {
+      "epoch": 0.836195897536559,
+      "grad_norm": 4.26902437210083,
+      "learning_rate": 3.9737157224207265e-07,
+      "logits/chosen": -1.4173133373260498,
+      "logits/rejected": -1.3832111358642578,
+      "logps/chosen": -448.400390625,
+      "logps/rejected": -505.0480041503906,
+      "loss": 0.5167,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.0978434085845947,
+      "rewards/margins": 0.7672898173332214,
+      "rewards/rejected": -1.865133285522461,
+      "step": 2130
+    },
+    {
+      "epoch": 0.8401216998724115,
+      "grad_norm": 2.9841227531433105,
+      "learning_rate": 3.7903278695839456e-07,
+      "logits/chosen": -1.4843977689743042,
+      "logits/rejected": -1.5173877477645874,
+      "logps/chosen": -472.466796875,
+      "logps/rejected": -531.3754272460938,
+      "loss": 0.5963,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.120090126991272,
+      "rewards/margins": 0.5689474940299988,
+      "rewards/rejected": -1.689037561416626,
+      "step": 2140
+    },
+    {
+      "epoch": 0.8440475022082639,
+      "grad_norm": 2.9768121242523193,
+      "learning_rate": 3.610924735506274e-07,
+      "logits/chosen": -1.3902521133422852,
+      "logits/rejected": -1.3804384469985962,
+      "logps/chosen": -495.37237548828125,
+      "logps/rejected": -493.58453369140625,
+      "loss": 0.5508,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.079955816268921,
+      "rewards/margins": 0.6100202202796936,
+      "rewards/rejected": -1.6899760961532593,
+      "step": 2150
+    },
+    {
+      "epoch": 0.8479733045441162,
+      "grad_norm": 2.7421867847442627,
+      "learning_rate": 3.4355400251421977e-07,
+      "logits/chosen": -1.4071491956710815,
+      "logits/rejected": -1.3936915397644043,
+      "logps/chosen": -432.0848083496094,
+      "logps/rejected": -462.03271484375,
+      "loss": 0.5576,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.0723979473114014,
+      "rewards/margins": 0.4340779185295105,
+      "rewards/rejected": -1.506475806236267,
+      "step": 2160
+    },
+    {
+      "epoch": 0.8518991068799686,
+      "grad_norm": 3.633836507797241,
+      "learning_rate": 3.2642066884940064e-07,
+      "logits/chosen": -1.4596307277679443,
+      "logits/rejected": -1.4016952514648438,
+      "logps/chosen": -488.4602966308594,
+      "logps/rejected": -520.9617919921875,
+      "loss": 0.5422,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.147655963897705,
+      "rewards/margins": 0.5640686750411987,
+      "rewards/rejected": -1.711724877357483,
+      "step": 2170
+    },
+    {
+      "epoch": 0.855824909215821,
+      "grad_norm": 4.396763801574707,
+      "learning_rate": 3.0969569144214147e-07,
+      "logits/chosen": -1.511970043182373,
+      "logits/rejected": -1.4692320823669434,
+      "logps/chosen": -450.92376708984375,
+      "logps/rejected": -491.7442321777344,
+      "loss": 0.5461,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1887543201446533,
+      "rewards/margins": 0.6474616527557373,
+      "rewards/rejected": -1.8362159729003906,
+      "step": 2180
+    },
+    {
+      "epoch": 0.8597507115516734,
+      "grad_norm": 4.131274223327637,
+      "learning_rate": 2.933822124594124e-07,
+      "logits/chosen": -1.4287583827972412,
+      "logits/rejected": -1.4037811756134033,
+      "logps/chosen": -498.95379638671875,
+      "logps/rejected": -517.1318359375,
+      "loss": 0.6189,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.3147928714752197,
+      "rewards/margins": 0.38747531175613403,
+      "rewards/rejected": -1.7022680044174194,
+      "step": 2190
+    },
+    {
+      "epoch": 0.8636765138875258,
+      "grad_norm": 3.2338428497314453,
+      "learning_rate": 2.774832967588556e-07,
+      "logits/chosen": -1.4276396036148071,
+      "logits/rejected": -1.3732216358184814,
+      "logps/chosen": -471.07952880859375,
+      "logps/rejected": -532.7431640625,
+      "loss": 0.5282,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.3066188097000122,
+      "rewards/margins": 0.6181976199150085,
+      "rewards/rejected": -1.924816370010376,
+      "step": 2200
+    },
+    {
+      "epoch": 0.8636765138875258,
+      "eval_logits/chosen": -1.4607295989990234,
+      "eval_logits/rejected": -1.4116848707199097,
+      "eval_logps/chosen": -463.6923522949219,
+      "eval_logps/rejected": -486.9384460449219,
+      "eval_loss": 0.5488107800483704,
+      "eval_rewards/accuracies": 0.7365269660949707,
+      "eval_rewards/chosen": -1.1627787351608276,
+      "eval_rewards/margins": 0.660214900970459,
+      "eval_rewards/rejected": -1.8229936361312866,
+      "eval_runtime": 293.9852,
+      "eval_samples_per_second": 6.803,
+      "eval_steps_per_second": 1.136,
+      "step": 2200
+    },
+    {
+      "epoch": 0.8676023162233781,
+      "grad_norm": 3.42141056060791,
+      "learning_rate": 2.6200193131298376e-07,
+      "logits/chosen": -1.5293169021606445,
+      "logits/rejected": -1.4347726106643677,
+      "logps/chosen": -423.95233154296875,
+      "logps/rejected": -411.8409118652344,
+      "loss": 0.5597,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0945215225219727,
+      "rewards/margins": 0.62129145860672,
+      "rewards/rejected": -1.7158129215240479,
+      "step": 2210
+    },
+    {
+      "epoch": 0.8715281185592305,
+      "grad_norm": 4.3379926681518555,
+      "learning_rate": 2.469410246480067e-07,
+      "logits/chosen": -1.4881136417388916,
+      "logits/rejected": -1.4418436288833618,
+      "logps/chosen": -517.5466918945312,
+      "logps/rejected": -468.6542053222656,
+      "loss": 0.5762,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.2271819114685059,
+      "rewards/margins": 0.44314274191856384,
+      "rewards/rejected": -1.670324683189392,
+      "step": 2220
+    },
+    {
+      "epoch": 0.8754539208950829,
+      "grad_norm": 3.3948209285736084,
+      "learning_rate": 2.3230340629740166e-07,
+      "logits/chosen": -1.3410069942474365,
+      "logits/rejected": -1.264758586883545,
+      "logps/chosen": -457.53509521484375,
+      "logps/rejected": -478.69891357421875,
+      "loss": 0.5119,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.2337532043457031,
+      "rewards/margins": 0.6139904260635376,
+      "rewards/rejected": -1.8477437496185303,
+      "step": 2230
+    },
+    {
+      "epoch": 0.8793797232309353,
+      "grad_norm": 2.92060923576355,
+      "learning_rate": 2.1809182627031883e-07,
+      "logits/chosen": -1.3494832515716553,
+      "logits/rejected": -1.3096946477890015,
+      "logps/chosen": -491.5668029785156,
+      "logps/rejected": -571.541748046875,
+      "loss": 0.5087,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1234079599380493,
+      "rewards/margins": 0.8506112098693848,
+      "rewards/rejected": -1.9740190505981445,
+      "step": 2240
+    },
+    {
+      "epoch": 0.8833055255667878,
+      "grad_norm": 4.318138122558594,
+      "learning_rate": 2.0430895453492944e-07,
+      "logits/chosen": -1.4682143926620483,
+      "logits/rejected": -1.4394243955612183,
+      "logps/chosen": -450.42138671875,
+      "logps/rejected": -455.76336669921875,
+      "loss": 0.5255,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2331966161727905,
+      "rewards/margins": 0.40833601355552673,
+      "rewards/rejected": -1.6415326595306396,
+      "step": 2250
+    },
+    {
+      "epoch": 0.8872313279026401,
+      "grad_norm": 3.434488534927368,
+      "learning_rate": 1.9095738051681412e-07,
+      "logits/chosen": -1.519504189491272,
+      "logits/rejected": -1.4198932647705078,
+      "logps/chosen": -492.00213623046875,
+      "logps/rejected": -465.56121826171875,
+      "loss": 0.5586,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.1482508182525635,
+      "rewards/margins": 0.6950271129608154,
+      "rewards/rejected": -1.843278169631958,
+      "step": 2260
+    },
+    {
+      "epoch": 0.8911571302384925,
+      "grad_norm": 4.2272820472717285,
+      "learning_rate": 1.7803961261247864e-07,
+      "logits/chosen": -1.4441721439361572,
+      "logits/rejected": -1.4162051677703857,
+      "logps/chosen": -527.7174682617188,
+      "logps/rejected": -531.0477294921875,
+      "loss": 0.5511,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.13276207447052,
+      "rewards/margins": 0.5703519582748413,
+      "rewards/rejected": -1.7031141519546509,
+      "step": 2270
+    },
+    {
+      "epoch": 0.8950829325743449,
+      "grad_norm": 3.41951060295105,
+      "learning_rate": 1.6555807771809375e-07,
+      "logits/chosen": -1.5005133152008057,
+      "logits/rejected": -1.468090295791626,
+      "logps/chosen": -538.9075927734375,
+      "logps/rejected": -545.8330688476562,
+      "loss": 0.519,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1071159839630127,
+      "rewards/margins": 0.8658310174942017,
+      "rewards/rejected": -1.972947120666504,
+      "step": 2280
+    },
+    {
+      "epoch": 0.8990087349101973,
+      "grad_norm": 3.4647693634033203,
+      "learning_rate": 1.5351512077355024e-07,
+      "logits/chosen": -1.3801993131637573,
+      "logits/rejected": -1.301965355873108,
+      "logps/chosen": -487.7762145996094,
+      "logps/rejected": -480.733642578125,
+      "loss": 0.5217,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.189225435256958,
+      "rewards/margins": 0.5852234363555908,
+      "rewards/rejected": -1.774449110031128,
+      "step": 2290
+    },
+    {
+      "epoch": 0.9029345372460497,
+      "grad_norm": 4.317968845367432,
+      "learning_rate": 1.4191300432190634e-07,
+      "logits/chosen": -1.530280351638794,
+      "logits/rejected": -1.5285309553146362,
+      "logps/chosen": -451.9605407714844,
+      "logps/rejected": -483.2005310058594,
+      "loss": 0.5604,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.038931965827942,
+      "rewards/margins": 0.36412283778190613,
+      "rewards/rejected": -1.40305495262146,
+      "step": 2300
+    },
+    {
+      "epoch": 0.9029345372460497,
+      "eval_logits/chosen": -1.4634631872177124,
+      "eval_logits/rejected": -1.4144114255905151,
+      "eval_logps/chosen": -460.8885803222656,
+      "eval_logps/rejected": -484.3240051269531,
+      "eval_loss": 0.5487485527992249,
+      "eval_rewards/accuracies": 0.7380239367485046,
+      "eval_rewards/chosen": -1.1347408294677734,
+      "eval_rewards/margins": 0.6621092557907104,
+      "eval_rewards/rejected": -1.7968502044677734,
+      "eval_runtime": 293.2531,
+      "eval_samples_per_second": 6.82,
+      "eval_steps_per_second": 1.139,
+      "step": 2300
+    },
+    {
+      "epoch": 0.906860339581902,
+      "grad_norm": 3.551766872406006,
+      "learning_rate": 1.3075390808431897e-07,
+      "logits/chosen": -1.42119562625885,
+      "logits/rejected": -1.3684390783309937,
+      "logps/chosen": -474.9652404785156,
+      "logps/rejected": -484.1065368652344,
+      "loss": 0.5128,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.0246719121932983,
+      "rewards/margins": 0.6398465037345886,
+      "rewards/rejected": -1.6645185947418213,
+      "step": 2310
+    },
+    {
+      "epoch": 0.9107861419177544,
+      "grad_norm": 3.1574060916900635,
+      "learning_rate": 1.2003992855053326e-07,
+      "logits/chosen": -1.4762132167816162,
+      "logits/rejected": -1.3953925371170044,
+      "logps/chosen": -459.08251953125,
+      "logps/rejected": -501.9891052246094,
+      "loss": 0.5774,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1319857835769653,
+      "rewards/margins": 0.7239108681678772,
+      "rewards/rejected": -1.8558965921401978,
+      "step": 2320
+    },
+    {
+      "epoch": 0.9147119442536068,
+      "grad_norm": 3.0150318145751953,
+      "learning_rate": 1.0977307858500818e-07,
+      "logits/chosen": -1.4380375146865845,
+      "logits/rejected": -1.382394790649414,
+      "logps/chosen": -474.906494140625,
+      "logps/rejected": -441.0604553222656,
+      "loss": 0.569,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.278412103652954,
+      "rewards/margins": 0.39085739850997925,
+      "rewards/rejected": -1.6692695617675781,
+      "step": 2330
+    },
+    {
+      "epoch": 0.9186377465894592,
+      "grad_norm": 2.777505397796631,
+      "learning_rate": 9.995528704875635e-08,
+      "logits/chosen": -1.4099693298339844,
+      "logits/rejected": -1.3602485656738281,
+      "logps/chosen": -507.23553466796875,
+      "logps/rejected": -485.8119201660156,
+      "loss": 0.5188,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1641985177993774,
+      "rewards/margins": 0.6164953112602234,
+      "rewards/rejected": -1.7806940078735352,
+      "step": 2340
+    },
+    {
+      "epoch": 0.9225635489253116,
+      "grad_norm": 3.3079113960266113,
+      "learning_rate": 9.058839843696237e-08,
+      "logits/chosen": -1.4558384418487549,
+      "logits/rejected": -1.391679048538208,
+      "logps/chosen": -469.39007568359375,
+      "logps/rejected": -524.3314208984375,
+      "loss": 0.5417,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.111272931098938,
+      "rewards/margins": 0.6209251284599304,
+      "rewards/rejected": -1.7321981191635132,
+      "step": 2350
+    },
+    {
+      "epoch": 0.926489351261164,
+      "grad_norm": 3.4808218479156494,
+      "learning_rate": 8.167417253245213e-08,
+      "logits/chosen": -1.4440009593963623,
+      "logits/rejected": -1.4202438592910767,
+      "logps/chosen": -467.93585205078125,
+      "logps/rejected": -526.765869140625,
+      "loss": 0.5431,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2468875646591187,
+      "rewards/margins": 0.504252552986145,
+      "rewards/rejected": -1.7511402368545532,
+      "step": 2360
+    },
+    {
+      "epoch": 0.9304151535970164,
+      "grad_norm": 3.7339236736297607,
+      "learning_rate": 7.32142840750788e-08,
+      "logits/chosen": -1.4894109964370728,
+      "logits/rejected": -1.466700792312622,
+      "logps/chosen": -477.885009765625,
+      "logps/rejected": -517.4005126953125,
+      "loss": 0.5903,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.1919804811477661,
+      "rewards/margins": 0.47159361839294434,
+      "rewards/rejected": -1.6635738611221313,
+      "step": 2370
+    },
+    {
+      "epoch": 0.9343409559328688,
+      "grad_norm": 2.9417037963867188,
+      "learning_rate": 6.521032244708375e-08,
+      "logits/chosen": -1.4463917016983032,
+      "logits/rejected": -1.411002278327942,
+      "logps/chosen": -484.45086669921875,
+      "logps/rejected": -494.3321228027344,
+      "loss": 0.5324,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1160385608673096,
+      "rewards/margins": 0.7087076902389526,
+      "rewards/rejected": -1.8247463703155518,
+      "step": 2380
+    },
+    {
+      "epoch": 0.9382667582687212,
+      "grad_norm": 3.4771625995635986,
+      "learning_rate": 5.766379137449624e-08,
+      "logits/chosen": -1.4637922048568726,
+      "logits/rejected": -1.424060583114624,
+      "logps/chosen": -486.72796630859375,
+      "logps/rejected": -531.3692626953125,
+      "loss": 0.5275,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1237993240356445,
+      "rewards/margins": 0.7095829844474792,
+      "rewards/rejected": -1.8333823680877686,
+      "step": 2390
+    },
+    {
+      "epoch": 0.9421925606045736,
+      "grad_norm": 3.7362887859344482,
+      "learning_rate": 5.0576108644623536e-08,
+      "logits/chosen": -1.4107905626296997,
+      "logits/rejected": -1.3791674375534058,
+      "logps/chosen": -429.25390625,
+      "logps/rejected": -452.63916015625,
+      "loss": 0.5365,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1171209812164307,
+      "rewards/margins": 0.5699509382247925,
+      "rewards/rejected": -1.6870720386505127,
+      "step": 2400
+    },
+    {
+      "epoch": 0.9421925606045736,
+      "eval_logits/chosen": -1.4632999897003174,
+      "eval_logits/rejected": -1.4142227172851562,
+      "eval_logps/chosen": -459.3744812011719,
+      "eval_logps/rejected": -482.7509460449219,
+      "eval_loss": 0.5488057732582092,
+      "eval_rewards/accuracies": 0.7380239367485046,
+      "eval_rewards/chosen": -1.11959969997406,
+      "eval_rewards/margins": 0.6615188121795654,
+      "eval_rewards/rejected": -1.781118631362915,
+      "eval_runtime": 294.198,
+      "eval_samples_per_second": 6.798,
+      "eval_steps_per_second": 1.135,
+      "step": 2400
+    },
+    {
+      "epoch": 0.946118362940426,
+      "grad_norm": 3.9023168087005615,
+      "learning_rate": 4.394860583968624e-08,
+      "logits/chosen": -1.4483604431152344,
+      "logits/rejected": -1.390679121017456,
+      "logps/chosen": -463.2852478027344,
+      "logps/rejected": -480.2080078125,
+      "loss": 0.5888,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.2475261688232422,
+      "rewards/margins": 0.4811752438545227,
+      "rewards/rejected": -1.7287015914916992,
+      "step": 2410
+    },
+    {
+      "epoch": 0.9500441652762783,
+      "grad_norm": 3.1474504470825195,
+      "learning_rate": 3.778252808665284e-08,
+      "logits/chosen": -1.4969747066497803,
+      "logits/rejected": -1.4146597385406494,
+      "logps/chosen": -442.46875,
+      "logps/rejected": -471.52490234375,
+      "loss": 0.5503,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0809533596038818,
+      "rewards/margins": 0.7569353580474854,
+      "rewards/rejected": -1.837888479232788,
+      "step": 2420
+    },
+    {
+      "epoch": 0.9539699676121307,
+      "grad_norm": 3.268937110900879,
+      "learning_rate": 3.207903382331262e-08,
+      "logits/chosen": -1.3833338022232056,
+      "logits/rejected": -1.3094943761825562,
+      "logps/chosen": -445.5309143066406,
+      "logps/rejected": -491.1158752441406,
+      "loss": 0.5392,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0812087059020996,
+      "rewards/margins": 0.73862224817276,
+      "rewards/rejected": -1.8198306560516357,
+      "step": 2430
+    },
+    {
+      "epoch": 0.9578957699479831,
+      "grad_norm": 3.46282696723938,
+      "learning_rate": 2.683919458063705e-08,
+      "logits/chosen": -1.363900899887085,
+      "logits/rejected": -1.3325783014297485,
+      "logps/chosen": -508.6698303222656,
+      "logps/rejected": -464.42449951171875,
+      "loss": 0.5527,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2314481735229492,
+      "rewards/margins": 0.43883267045021057,
+      "rewards/rejected": -1.6702810525894165,
+      "step": 2440
+    },
+    {
+      "epoch": 0.9618215722838355,
+      "grad_norm": 3.474923610687256,
+      "learning_rate": 2.2063994781468256e-08,
+      "logits/chosen": -1.4222522974014282,
+      "logits/rejected": -1.3719159364700317,
+      "logps/chosen": -484.34368896484375,
+      "logps/rejected": -467.943115234375,
+      "loss": 0.5762,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.2819271087646484,
+      "rewards/margins": 0.5020255446434021,
+      "rewards/rejected": -1.7839524745941162,
+      "step": 2450
+    },
+    {
+      "epoch": 0.9657473746196878,
+      "grad_norm": 2.6612024307250977,
+      "learning_rate": 1.7754331555573656e-08,
+      "logits/chosen": -1.4514880180358887,
+      "logits/rejected": -1.4274556636810303,
+      "logps/chosen": -507.3514709472656,
+      "logps/rejected": -517.5911254882812,
+      "loss": 0.5175,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.1000603437423706,
+      "rewards/margins": 0.7943276762962341,
+      "rewards/rejected": -1.89438796043396,
+      "step": 2460
+    },
+    {
+      "epoch": 0.9696731769555403,
+      "grad_norm": 3.3850739002227783,
+      "learning_rate": 1.3911014571098835e-08,
+      "logits/chosen": -1.4218047857284546,
+      "logits/rejected": -1.413437843322754,
+      "logps/chosen": -498.19927978515625,
+      "logps/rejected": -560.646484375,
+      "loss": 0.5299,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.2489643096923828,
+      "rewards/margins": 0.6791669726371765,
+      "rewards/rejected": -1.928131341934204,
+      "step": 2470
+    },
+    {
+      "epoch": 0.9735989792913927,
+      "grad_norm": 3.580686569213867,
+      "learning_rate": 1.0534765882453113e-08,
+      "logits/chosen": -1.373701572418213,
+      "logits/rejected": -1.329279899597168,
+      "logps/chosen": -417.05108642578125,
+      "logps/rejected": -472.8899841308594,
+      "loss": 0.5223,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1596735715866089,
+      "rewards/margins": 0.683711051940918,
+      "rewards/rejected": -1.8433845043182373,
+      "step": 2480
+    },
+    {
+      "epoch": 0.9775247816272451,
+      "grad_norm": 3.0793566703796387,
+      "learning_rate": 7.626219794655553e-09,
+      "logits/chosen": -1.410854458808899,
+      "logits/rejected": -1.342854619026184,
+      "logps/chosen": -453.941162109375,
+      "logps/rejected": -482.8081970214844,
+      "loss": 0.5217,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.124043345451355,
+      "rewards/margins": 0.6700025796890259,
+      "rewards/rejected": -1.7940460443496704,
+      "step": 2490
+    },
+    {
+      "epoch": 0.9814505839630975,
+      "grad_norm": 2.9402501583099365,
+      "learning_rate": 5.185922744166128e-09,
+      "logits/chosen": -1.3898409605026245,
+      "logits/rejected": -1.3367985486984253,
+      "logps/chosen": -445.125244140625,
+      "logps/rejected": -459.73358154296875,
+      "loss": 0.5135,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.1596896648406982,
+      "rewards/margins": 0.6667672991752625,
+      "rewards/rejected": -1.8264570236206055,
+      "step": 2500
+    },
+    {
+      "epoch": 0.9814505839630975,
+      "eval_logits/chosen": -1.4625605344772339,
+      "eval_logits/rejected": -1.4134557247161865,
+      "eval_logps/chosen": -460.12322998046875,
+      "eval_logps/rejected": -483.52081298828125,
+      "eval_loss": 0.5487865209579468,
+      "eval_rewards/accuracies": 0.7380239367485046,
+      "eval_rewards/chosen": -1.1270867586135864,
+      "eval_rewards/margins": 0.6617311239242554,
+      "eval_rewards/rejected": -1.7888180017471313,
+      "eval_runtime": 293.242,
+      "eval_samples_per_second": 6.82,
+      "eval_steps_per_second": 1.139,
+      "step": 2500
+    },
+    {
+      "epoch": 0.9853763862989499,
+      "grad_norm": 3.156381845474243,
+      "learning_rate": 3.2143331962256053e-09,
+      "logits/chosen": -1.4826661348342896,
+      "logits/rejected": -1.417670488357544,
+      "logps/chosen": -481.48675537109375,
+      "logps/rejected": -475.925048828125,
+      "loss": 0.5722,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.1260427236557007,
+      "rewards/margins": 0.5149827003479004,
+      "rewards/rejected": -1.6410255432128906,
+      "step": 2510
+    },
+    {
+      "epoch": 0.9893021886348022,
+      "grad_norm": 4.953946590423584,
+      "learning_rate": 1.711821558721405e-09,
+      "logits/chosen": -1.4517848491668701,
+      "logits/rejected": -1.4465982913970947,
+      "logps/chosen": -509.51727294921875,
+      "logps/rejected": -568.0440063476562,
+      "loss": 0.57,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.219585657119751,
+      "rewards/margins": 0.5374733209609985,
+      "rewards/rejected": -1.757059097290039,
+      "step": 2520
+    },
+    {
+      "epoch": 0.9932279909706546,
+      "grad_norm": 3.313225746154785,
+      "learning_rate": 6.786701125999218e-10,
+      "logits/chosen": -1.4304636716842651,
+      "logits/rejected": -1.4257185459136963,
+      "logps/chosen": -499.6388244628906,
+      "logps/rejected": -531.0709228515625,
+      "loss": 0.5115,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0990434885025024,
+      "rewards/margins": 0.6478461027145386,
+      "rewards/rejected": -1.7468894720077515,
+      "step": 2530
+    },
+    {
+      "epoch": 0.997153793306507,
+      "grad_norm": 3.157412528991699,
+      "learning_rate": 1.1507295883145253e-10,
+      "logits/chosen": -1.4119558334350586,
+      "logits/rejected": -1.3239681720733643,
+      "logps/chosen": -454.55078125,
+      "logps/rejected": -458.1592712402344,
+      "loss": 0.5388,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.09469735622406,
+      "rewards/margins": 0.6515146493911743,
+      "rewards/rejected": -1.7462117671966553,
+      "step": 2540
+    },
+    {
+      "epoch": 0.9999018549416037,
+      "step": 2547,
+      "total_flos": 0.0,
+      "train_loss": 0.5787653771390342,
+      "train_runtime": 28047.1443,
+      "train_samples_per_second": 2.18,
+      "train_steps_per_second": 0.091
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2547,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}