diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,9244 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9997333028536595,
+  "eval_steps": 500,
+  "global_step": 3280,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5243902439024392e-07,
+      "logits/chosen": -1.6215482950210571,
+      "logits/rejected": -1.4746919870376587,
+      "logps/chosen": -188.31854248046875,
+      "logps/rejected": -214.3458709716797,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.621951219512196e-07,
+      "logits/chosen": -1.6072877645492554,
+      "logits/rejected": -1.5261101722717285,
+      "logps/chosen": -266.4974365234375,
+      "logps/rejected": -276.2115478515625,
+      "loss": 0.6935,
+      "rewards/accuracies": 0.3203125,
+      "rewards/chosen": -0.0012238634517416358,
+      "rewards/margins": 0.0034746606834232807,
+      "rewards/rejected": -0.004698523320257664,
+      "step": 5
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5243902439024391e-06,
+      "logits/chosen": -1.5626871585845947,
+      "logits/rejected": -1.499194860458374,
+      "logps/chosen": -200.6118621826172,
+      "logps/rejected": -229.2737579345703,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.3687500059604645,
+      "rewards/chosen": -0.001540867961011827,
+      "rewards/margins": 0.0018269469728693366,
+      "rewards/rejected": -0.003367815865203738,
+      "step": 10
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.2865853658536584e-06,
+      "logits/chosen": -1.528067708015442,
+      "logits/rejected": -1.425481915473938,
+      "logps/chosen": -230.9717254638672,
+      "logps/rejected": -243.3507537841797,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.019894156605005264,
+      "rewards/margins": 0.011482590809464455,
+      "rewards/rejected": -0.03137674927711487,
+      "step": 15
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.0487804878048782e-06,
+      "logits/chosen": -1.5903682708740234,
+      "logits/rejected": -1.508452296257019,
+      "logps/chosen": -220.4499053955078,
+      "logps/rejected": -250.4200897216797,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": -0.04624359309673309,
+      "rewards/margins": 0.019922306761145592,
+      "rewards/rejected": -0.06616590172052383,
+      "step": 20
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.8109756097560976e-06,
+      "logits/chosen": -1.542128324508667,
+      "logits/rejected": -1.4916765689849854,
+      "logps/chosen": -214.9829864501953,
+      "logps/rejected": -228.2103271484375,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -0.07907415926456451,
+      "rewards/margins": 0.03622515872120857,
+      "rewards/rejected": -0.11529930680990219,
+      "step": 25
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.573170731707317e-06,
+      "logits/chosen": -1.5964621305465698,
+      "logits/rejected": -1.4607049226760864,
+      "logps/chosen": -226.9277801513672,
+      "logps/rejected": -208.7239227294922,
+      "loss": 0.6677,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": -0.15189151465892792,
+      "rewards/margins": 0.0342349037528038,
+      "rewards/rejected": -0.18612642586231232,
+      "step": 30
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.335365853658537e-06,
+      "logits/chosen": -1.608441948890686,
+      "logits/rejected": -1.538400411605835,
+      "logps/chosen": -241.2967071533203,
+      "logps/rejected": -258.8771057128906,
+      "loss": 0.6601,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.2652124762535095,
+      "rewards/margins": 0.08378251641988754,
+      "rewards/rejected": -0.3489949703216553,
+      "step": 35
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.0975609756097564e-06,
+      "logits/chosen": -1.5152992010116577,
+      "logits/rejected": -1.3842694759368896,
+      "logps/chosen": -265.1772155761719,
+      "logps/rejected": -267.61614990234375,
+      "loss": 0.6425,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.3918093740940094,
+      "rewards/margins": 0.12125066667795181,
+      "rewards/rejected": -0.5130600333213806,
+      "step": 40
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.859756097560977e-06,
+      "logits/chosen": -1.4685131311416626,
+      "logits/rejected": -1.374135136604309,
+      "logps/chosen": -221.75424194335938,
+      "logps/rejected": -250.12954711914062,
+      "loss": 0.6139,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.4650735855102539,
+      "rewards/margins": 0.23447296023368835,
+      "rewards/rejected": -0.6995465755462646,
+      "step": 45
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 7.621951219512195e-06,
+      "logits/chosen": -1.4883078336715698,
+      "logits/rejected": -1.3383334875106812,
+      "logps/chosen": -247.33468627929688,
+      "logps/rejected": -261.386962890625,
+      "loss": 0.63,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.6402769088745117,
+      "rewards/margins": 0.2748766839504242,
+      "rewards/rejected": -0.9151536822319031,
+      "step": 50
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 8.384146341463415e-06,
+      "logits/chosen": -1.5034300088882446,
+      "logits/rejected": -1.4040600061416626,
+      "logps/chosen": -233.43331909179688,
+      "logps/rejected": -245.99453735351562,
+      "loss": 0.5996,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.7301857471466064,
+      "rewards/margins": 0.21929316222667694,
+      "rewards/rejected": -0.949478805065155,
+      "step": 55
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.146341463414634e-06,
+      "logits/chosen": -1.438262701034546,
+      "logits/rejected": -1.3356047868728638,
+      "logps/chosen": -258.9648742675781,
+      "logps/rejected": -276.9036560058594,
+      "loss": 0.5881,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.9311250448226929,
+      "rewards/margins": 0.41171926259994507,
+      "rewards/rejected": -1.3428443670272827,
+      "step": 60
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.908536585365854e-06,
+      "logits/chosen": -1.5152195692062378,
+      "logits/rejected": -1.4204599857330322,
+      "logps/chosen": -232.12216186523438,
+      "logps/rejected": -245.5828094482422,
+      "loss": 0.5892,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.841234028339386,
+      "rewards/margins": 0.3476036489009857,
+      "rewards/rejected": -1.1888377666473389,
+      "step": 65
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0670731707317074e-05,
+      "logits/chosen": -1.4773343801498413,
+      "logits/rejected": -1.371441125869751,
+      "logps/chosen": -264.8576965332031,
+      "logps/rejected": -297.8639831542969,
+      "loss": 0.5599,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.033233880996704,
+      "rewards/margins": 0.5808243155479431,
+      "rewards/rejected": -1.6140581369400024,
+      "step": 70
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.1432926829268294e-05,
+      "logits/chosen": -1.5124752521514893,
+      "logits/rejected": -1.4127274751663208,
+      "logps/chosen": -265.6245422363281,
+      "logps/rejected": -263.59515380859375,
+      "loss": 0.5978,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.9992305040359497,
+      "rewards/margins": 0.321955144405365,
+      "rewards/rejected": -1.32118558883667,
+      "step": 75
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.2195121951219513e-05,
+      "logits/chosen": -1.3964924812316895,
+      "logits/rejected": -1.356090784072876,
+      "logps/chosen": -246.69619750976562,
+      "logps/rejected": -292.822265625,
+      "loss": 0.5646,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.9433499574661255,
+      "rewards/margins": 0.6655504703521729,
+      "rewards/rejected": -1.6089003086090088,
+      "step": 80
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.2957317073170733e-05,
+      "logits/chosen": -1.4542206525802612,
+      "logits/rejected": -1.3648021221160889,
+      "logps/chosen": -261.1116638183594,
+      "logps/rejected": -290.764404296875,
+      "loss": 0.5359,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.813259482383728,
+      "rewards/margins": 0.595789909362793,
+      "rewards/rejected": -1.4090495109558105,
+      "step": 85
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3719512195121953e-05,
+      "logits/chosen": -1.5060111284255981,
+      "logits/rejected": -1.4409078359603882,
+      "logps/chosen": -243.53173828125,
+      "logps/rejected": -266.01361083984375,
+      "loss": 0.5356,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.7168170809745789,
+      "rewards/margins": 0.6658238172531128,
+      "rewards/rejected": -1.3826408386230469,
+      "step": 90
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.448170731707317e-05,
+      "logits/chosen": -1.377535104751587,
+      "logits/rejected": -1.3736878633499146,
+      "logps/chosen": -227.3628387451172,
+      "logps/rejected": -285.86334228515625,
+      "loss": 0.4957,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.6789393424987793,
+      "rewards/margins": 0.7634402513504028,
+      "rewards/rejected": -1.4423797130584717,
+      "step": 95
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.524390243902439e-05,
+      "logits/chosen": -1.4626977443695068,
+      "logits/rejected": -1.3379249572753906,
+      "logps/chosen": -254.5047607421875,
+      "logps/rejected": -249.72500610351562,
+      "loss": 0.4935,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.8015538454055786,
+      "rewards/margins": 0.6757813692092896,
+      "rewards/rejected": -1.4773352146148682,
+      "step": 100
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.600609756097561e-05,
+      "logits/chosen": -1.5257985591888428,
+      "logits/rejected": -1.380997657775879,
+      "logps/chosen": -232.9523162841797,
+      "logps/rejected": -260.36492919921875,
+      "loss": 0.4978,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.6900753974914551,
+      "rewards/margins": 0.927207350730896,
+      "rewards/rejected": -1.6172831058502197,
+      "step": 105
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.676829268292683e-05,
+      "logits/chosen": -1.4730074405670166,
+      "logits/rejected": -1.47112238407135,
+      "logps/chosen": -212.0960235595703,
+      "logps/rejected": -271.15191650390625,
+      "loss": 0.483,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.7050459980964661,
+      "rewards/margins": 0.937295138835907,
+      "rewards/rejected": -1.6423410177230835,
+      "step": 110
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.7530487804878047e-05,
+      "logits/chosen": -1.4949665069580078,
+      "logits/rejected": -1.5023237466812134,
+      "logps/chosen": -231.5373992919922,
+      "logps/rejected": -283.80023193359375,
+      "loss": 0.4846,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.8734248280525208,
+      "rewards/margins": 1.1210492849349976,
+      "rewards/rejected": -1.994474172592163,
+      "step": 115
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8292682926829268e-05,
+      "logits/chosen": -1.4580038785934448,
+      "logits/rejected": -1.3469959497451782,
+      "logps/chosen": -220.4951934814453,
+      "logps/rejected": -260.50958251953125,
+      "loss": 0.4338,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.8012296557426453,
+      "rewards/margins": 1.2397785186767578,
+      "rewards/rejected": -2.041008234024048,
+      "step": 120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9054878048780488e-05,
+      "logits/chosen": -1.4429595470428467,
+      "logits/rejected": -1.3944337368011475,
+      "logps/chosen": -272.3973388671875,
+      "logps/rejected": -293.87725830078125,
+      "loss": 0.4562,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.9875411987304688,
+      "rewards/margins": 1.1865813732147217,
+      "rewards/rejected": -2.1741225719451904,
+      "step": 125
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9817073170731708e-05,
+      "logits/chosen": -1.5181801319122314,
+      "logits/rejected": -1.4099066257476807,
+      "logps/chosen": -211.18594360351562,
+      "logps/rejected": -239.5623321533203,
+      "loss": 0.4578,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.5671035051345825,
+      "rewards/margins": 1.078687310218811,
+      "rewards/rejected": -1.6457910537719727,
+      "step": 130
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0579268292682928e-05,
+      "logits/chosen": -1.4463402032852173,
+      "logits/rejected": -1.33974289894104,
+      "logps/chosen": -241.71322631835938,
+      "logps/rejected": -276.1986389160156,
+      "loss": 0.4639,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.8435440063476562,
+      "rewards/margins": 1.1942956447601318,
+      "rewards/rejected": -2.037839651107788,
+      "step": 135
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.134146341463415e-05,
+      "logits/chosen": -1.4488505125045776,
+      "logits/rejected": -1.2843577861785889,
+      "logps/chosen": -256.20452880859375,
+      "logps/rejected": -275.42095947265625,
+      "loss": 0.4569,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.6826750636100769,
+      "rewards/margins": 1.3818552494049072,
+      "rewards/rejected": -2.06453013420105,
+      "step": 140
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.210365853658537e-05,
+      "logits/chosen": -1.471332311630249,
+      "logits/rejected": -1.4009536504745483,
+      "logps/chosen": -260.52374267578125,
+      "logps/rejected": -292.8216247558594,
+      "loss": 0.4397,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.5928641557693481,
+      "rewards/margins": 1.5231386423110962,
+      "rewards/rejected": -2.1160027980804443,
+      "step": 145
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.286585365853659e-05,
+      "logits/chosen": -1.5144442319869995,
+      "logits/rejected": -1.4480407238006592,
+      "logps/chosen": -251.3614959716797,
+      "logps/rejected": -273.5283508300781,
+      "loss": 0.4372,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.3241608738899231,
+      "rewards/margins": 1.3152064085006714,
+      "rewards/rejected": -1.6393673419952393,
+      "step": 150
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.3628048780487806e-05,
+      "logits/chosen": -1.4405564069747925,
+      "logits/rejected": -1.388474941253662,
+      "logps/chosen": -214.82778930664062,
+      "logps/rejected": -269.74468994140625,
+      "loss": 0.4043,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.21781139075756073,
+      "rewards/margins": 1.6215749979019165,
+      "rewards/rejected": -1.8393863439559937,
+      "step": 155
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.4390243902439026e-05,
+      "logits/chosen": -1.5152744054794312,
+      "logits/rejected": -1.3840240240097046,
+      "logps/chosen": -260.8744201660156,
+      "logps/rejected": -284.11175537109375,
+      "loss": 0.4132,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.3614785969257355,
+      "rewards/margins": 1.4981237649917603,
+      "rewards/rejected": -1.8596023321151733,
+      "step": 160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.5152439024390246e-05,
+      "logits/chosen": -1.4719091653823853,
+      "logits/rejected": -1.3672441244125366,
+      "logps/chosen": -236.2136993408203,
+      "logps/rejected": -263.2491455078125,
+      "loss": 0.4243,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.6757379770278931,
+      "rewards/margins": 1.6337473392486572,
+      "rewards/rejected": -2.3094851970672607,
+      "step": 165
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.5914634146341466e-05,
+      "logits/chosen": -1.3289225101470947,
+      "logits/rejected": -1.2671074867248535,
+      "logps/chosen": -257.1202392578125,
+      "logps/rejected": -296.96026611328125,
+      "loss": 0.4002,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.0305023193359375,
+      "rewards/margins": 1.9576082229614258,
+      "rewards/rejected": -2.988110303878784,
+      "step": 170
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.6676829268292686e-05,
+      "logits/chosen": -1.4129236936569214,
+      "logits/rejected": -1.2899045944213867,
+      "logps/chosen": -241.03207397460938,
+      "logps/rejected": -279.87408447265625,
+      "loss": 0.4107,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.7252627611160278,
+      "rewards/margins": 2.208683967590332,
+      "rewards/rejected": -2.9339470863342285,
+      "step": 175
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.7439024390243906e-05,
+      "logits/chosen": -1.4490526914596558,
+      "logits/rejected": -1.3659610748291016,
+      "logps/chosen": -224.96658325195312,
+      "logps/rejected": -268.126953125,
+      "loss": 0.3825,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.39545050263404846,
+      "rewards/margins": 1.7132034301757812,
+      "rewards/rejected": -2.108654022216797,
+      "step": 180
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.820121951219512e-05,
+      "logits/chosen": -1.4639991521835327,
+      "logits/rejected": -1.351359248161316,
+      "logps/chosen": -263.8490295410156,
+      "logps/rejected": -305.0282287597656,
+      "loss": 0.4509,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6279351115226746,
+      "rewards/margins": 1.842530608177185,
+      "rewards/rejected": -2.470465898513794,
+      "step": 185
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.896341463414634e-05,
+      "logits/chosen": -1.437174677848816,
+      "logits/rejected": -1.3693865537643433,
+      "logps/chosen": -212.9979248046875,
+      "logps/rejected": -263.60546875,
+      "loss": 0.4206,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.675702691078186,
+      "rewards/margins": 1.771080732345581,
+      "rewards/rejected": -2.4467835426330566,
+      "step": 190
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.972560975609756e-05,
+      "logits/chosen": -1.4669255018234253,
+      "logits/rejected": -1.3856886625289917,
+      "logps/chosen": -258.96734619140625,
+      "logps/rejected": -292.31195068359375,
+      "loss": 0.3976,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.35433077812194824,
+      "rewards/margins": 1.9425218105316162,
+      "rewards/rejected": -2.2968528270721436,
+      "step": 195
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.048780487804878e-05,
+      "logits/chosen": -1.5489139556884766,
+      "logits/rejected": -1.4419410228729248,
+      "logps/chosen": -247.7865447998047,
+      "logps/rejected": -272.13140869140625,
+      "loss": 0.4203,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.44836997985839844,
+      "rewards/margins": 1.8278976678848267,
+      "rewards/rejected": -2.2762677669525146,
+      "step": 200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.125e-05,
+      "logits/chosen": -1.5211713314056396,
+      "logits/rejected": -1.5042860507965088,
+      "logps/chosen": -234.41732788085938,
+      "logps/rejected": -296.26666259765625,
+      "loss": 0.3832,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.4585009217262268,
+      "rewards/margins": 1.8909355401992798,
+      "rewards/rejected": -2.3494365215301514,
+      "step": 205
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.201219512195122e-05,
+      "logits/chosen": -1.4364079236984253,
+      "logits/rejected": -1.3636162281036377,
+      "logps/chosen": -252.7544403076172,
+      "logps/rejected": -289.1746826171875,
+      "loss": 0.3609,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3245674967765808,
+      "rewards/margins": 2.125056028366089,
+      "rewards/rejected": -2.4496235847473145,
+      "step": 210
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.277439024390244e-05,
+      "logits/chosen": -1.4644180536270142,
+      "logits/rejected": -1.3949480056762695,
+      "logps/chosen": -234.929443359375,
+      "logps/rejected": -279.88995361328125,
+      "loss": 0.4598,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.40384259819984436,
+      "rewards/margins": 1.5408689975738525,
+      "rewards/rejected": -1.944711685180664,
+      "step": 215
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.353658536585366e-05,
+      "logits/chosen": -1.4723870754241943,
+      "logits/rejected": -1.353324294090271,
+      "logps/chosen": -233.2500762939453,
+      "logps/rejected": -264.16302490234375,
+      "loss": 0.3909,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.6612171530723572,
+      "rewards/margins": 2.2186388969421387,
+      "rewards/rejected": -2.8798558712005615,
+      "step": 220
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.429878048780488e-05,
+      "logits/chosen": -1.5066630840301514,
+      "logits/rejected": -1.442996621131897,
+      "logps/chosen": -242.844970703125,
+      "logps/rejected": -285.40301513671875,
+      "loss": 0.4212,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.8180769681930542,
+      "rewards/margins": 1.7058541774749756,
+      "rewards/rejected": -2.5239310264587402,
+      "step": 225
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.5060975609756095e-05,
+      "logits/chosen": -1.397632122039795,
+      "logits/rejected": -1.309533715248108,
+      "logps/chosen": -271.36676025390625,
+      "logps/rejected": -294.4072265625,
+      "loss": 0.395,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.6913538575172424,
+      "rewards/margins": 1.8501994609832764,
+      "rewards/rejected": -2.5415537357330322,
+      "step": 230
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.5823170731707315e-05,
+      "logits/chosen": -1.3661539554595947,
+      "logits/rejected": -1.2781219482421875,
+      "logps/chosen": -234.1933135986328,
+      "logps/rejected": -266.4363708496094,
+      "loss": 0.3969,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.8205755949020386,
+      "rewards/margins": 2.130988597869873,
+      "rewards/rejected": -2.951564311981201,
+      "step": 235
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.6585365853658535e-05,
+      "logits/chosen": -1.429700493812561,
+      "logits/rejected": -1.3226317167282104,
+      "logps/chosen": -260.207763671875,
+      "logps/rejected": -288.5872802734375,
+      "loss": 0.3711,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.6312899589538574,
+      "rewards/margins": 2.1224799156188965,
+      "rewards/rejected": -2.7537693977355957,
+      "step": 240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.7347560975609755e-05,
+      "logits/chosen": -1.414186954498291,
+      "logits/rejected": -1.2793656587600708,
+      "logps/chosen": -269.228515625,
+      "logps/rejected": -310.0638732910156,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.8959183692932129,
+      "rewards/margins": 2.4896857738494873,
+      "rewards/rejected": -3.3856041431427,
+      "step": 245
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.8109756097560976e-05,
+      "logits/chosen": -1.3947970867156982,
+      "logits/rejected": -1.3410694599151611,
+      "logps/chosen": -225.885009765625,
+      "logps/rejected": -293.60003662109375,
+      "loss": 0.4103,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.6115488409996033,
+      "rewards/margins": 2.3446762561798096,
+      "rewards/rejected": -2.9562251567840576,
+      "step": 250
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.8871951219512196e-05,
+      "logits/chosen": -1.428045392036438,
+      "logits/rejected": -1.3080122470855713,
+      "logps/chosen": -253.37646484375,
+      "logps/rejected": -262.73150634765625,
+      "loss": 0.4013,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.43573275208473206,
+      "rewards/margins": 2.107551097869873,
+      "rewards/rejected": -2.5432839393615723,
+      "step": 255
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.9634146341463416e-05,
+      "logits/chosen": -1.5183364152908325,
+      "logits/rejected": -1.4998283386230469,
+      "logps/chosen": -224.19851684570312,
+      "logps/rejected": -268.945556640625,
+      "loss": 0.4347,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.19424612820148468,
+      "rewards/margins": 1.7188571691513062,
+      "rewards/rejected": -1.913103461265564,
+      "step": 260
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.0396341463414636e-05,
+      "logits/chosen": -1.447766900062561,
+      "logits/rejected": -1.3362300395965576,
+      "logps/chosen": -202.533935546875,
+      "logps/rejected": -243.1204071044922,
+      "loss": 0.3827,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.07845243811607361,
+      "rewards/margins": 1.7323980331420898,
+      "rewards/rejected": -1.8108505010604858,
+      "step": 265
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.1158536585365856e-05,
+      "logits/chosen": -1.4217723608016968,
+      "logits/rejected": -1.3448355197906494,
+      "logps/chosen": -207.6378631591797,
+      "logps/rejected": -260.37933349609375,
+      "loss": 0.4343,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.5095584988594055,
+      "rewards/margins": 1.8481013774871826,
+      "rewards/rejected": -2.3576598167419434,
+      "step": 270
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.1920731707317077e-05,
+      "logits/chosen": -1.3837717771530151,
+      "logits/rejected": -1.3595670461654663,
+      "logps/chosen": -227.80239868164062,
+      "logps/rejected": -270.43463134765625,
+      "loss": 0.4198,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.9875295758247375,
+      "rewards/margins": 2.1480934619903564,
+      "rewards/rejected": -3.135622978210449,
+      "step": 275
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.26829268292683e-05,
+      "logits/chosen": -1.3282934427261353,
+      "logits/rejected": -1.2439639568328857,
+      "logps/chosen": -259.5870056152344,
+      "logps/rejected": -298.34307861328125,
+      "loss": 0.3466,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.8122395277023315,
+      "rewards/margins": 2.514617443084717,
+      "rewards/rejected": -3.326857089996338,
+      "step": 280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.344512195121952e-05,
+      "logits/chosen": -1.3736869096755981,
+      "logits/rejected": -1.3249460458755493,
+      "logps/chosen": -213.5878143310547,
+      "logps/rejected": -260.0492248535156,
+      "loss": 0.4065,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.6954141855239868,
+      "rewards/margins": 2.317218065261841,
+      "rewards/rejected": -3.012632369995117,
+      "step": 285
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.420731707317074e-05,
+      "logits/chosen": -1.3154847621917725,
+      "logits/rejected": -1.2118194103240967,
+      "logps/chosen": -266.9904479980469,
+      "logps/rejected": -295.3359375,
+      "loss": 0.416,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.930508017539978,
+      "rewards/margins": 2.233632802963257,
+      "rewards/rejected": -3.1641409397125244,
+      "step": 290
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.496951219512196e-05,
+      "logits/chosen": -1.4980214834213257,
+      "logits/rejected": -1.3795270919799805,
+      "logps/chosen": -224.4019317626953,
+      "logps/rejected": -240.95443725585938,
+      "loss": 0.419,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.718636691570282,
+      "rewards/margins": 1.8446115255355835,
+      "rewards/rejected": -2.5632483959198,
+      "step": 295
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.573170731707318e-05,
+      "logits/chosen": -1.388089895248413,
+      "logits/rejected": -1.3248611688613892,
+      "logps/chosen": -223.11434936523438,
+      "logps/rejected": -272.0300598144531,
+      "loss": 0.4197,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.6024666428565979,
+      "rewards/margins": 2.1911685466766357,
+      "rewards/rejected": -2.793635129928589,
+      "step": 300
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.64939024390244e-05,
+      "logits/chosen": -1.4117351770401,
+      "logits/rejected": -1.3508259057998657,
+      "logps/chosen": -225.8909912109375,
+      "logps/rejected": -276.47314453125,
+      "loss": 0.44,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.6590641140937805,
+      "rewards/margins": 1.815882921218872,
+      "rewards/rejected": -2.474947452545166,
+      "step": 305
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.725609756097561e-05,
+      "logits/chosen": -1.4158817529678345,
+      "logits/rejected": -1.3631826639175415,
+      "logps/chosen": -241.3859405517578,
+      "logps/rejected": -298.5724182128906,
+      "loss": 0.4038,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.1089242696762085,
+      "rewards/margins": 2.054959535598755,
+      "rewards/rejected": -3.163883924484253,
+      "step": 310
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.801829268292683e-05,
+      "logits/chosen": -1.3394591808319092,
+      "logits/rejected": -1.2597063779830933,
+      "logps/chosen": -236.72244262695312,
+      "logps/rejected": -318.73687744140625,
+      "loss": 0.3443,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.473099708557129,
+      "rewards/margins": 2.826732873916626,
+      "rewards/rejected": -4.299832344055176,
+      "step": 315
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.878048780487805e-05,
+      "logits/chosen": -1.2992823123931885,
+      "logits/rejected": -1.1666510105133057,
+      "logps/chosen": -272.9997863769531,
+      "logps/rejected": -309.48681640625,
+      "loss": 0.509,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7227243185043335,
+      "rewards/margins": 2.872715473175049,
+      "rewards/rejected": -4.595439910888672,
+      "step": 320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.954268292682927e-05,
+      "logits/chosen": -1.4183080196380615,
+      "logits/rejected": -1.3151204586029053,
+      "logps/chosen": -260.7613220214844,
+      "logps/rejected": -307.1131286621094,
+      "loss": 0.412,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.0861679315567017,
+      "rewards/margins": 2.3004274368286133,
+      "rewards/rejected": -3.3865954875946045,
+      "step": 325
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.9999943371262496e-05,
+      "logits/chosen": -1.5749884843826294,
+      "logits/rejected": -1.4654959440231323,
+      "logps/chosen": -234.9315643310547,
+      "logps/rejected": -249.64013671875,
+      "loss": 0.3938,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.14985708892345428,
+      "rewards/margins": 1.9599332809448242,
+      "rewards/rejected": -2.109790325164795,
+      "step": 330
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.9999306300911826e-05,
+      "logits/chosen": -1.4392597675323486,
+      "logits/rejected": -1.3621281385421753,
+      "logps/chosen": -214.2711181640625,
+      "logps/rejected": -261.72552490234375,
+      "loss": 0.4055,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.43718934059143066,
+      "rewards/margins": 1.943355917930603,
+      "rewards/rejected": -2.380545139312744,
+      "step": 335
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999796139238694e-05,
+      "logits/chosen": -1.518296480178833,
+      "logits/rejected": -1.4602665901184082,
+      "logps/chosen": -242.5746307373047,
+      "logps/rejected": -296.28204345703125,
+      "loss": 0.5931,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.0693089962005615,
+      "rewards/margins": 2.0596704483032227,
+      "rewards/rejected": -3.128979444503784,
+      "step": 340
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.9995908683767986e-05,
+      "logits/chosen": -1.4445441961288452,
+      "logits/rejected": -1.3583850860595703,
+      "logps/chosen": -238.28549194335938,
+      "logps/rejected": -276.1322021484375,
+      "loss": 0.418,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.0093810558319092,
+      "rewards/margins": 2.0990090370178223,
+      "rewards/rejected": -3.1083903312683105,
+      "step": 345
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999314823317602e-05,
+      "logits/chosen": -1.4120949506759644,
+      "logits/rejected": -1.3273041248321533,
+      "logps/chosen": -234.6574249267578,
+      "logps/rejected": -289.5172424316406,
+      "loss": 0.4384,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5543441772460938,
+      "rewards/margins": 2.0190649032592773,
+      "rewards/rejected": -2.573408842086792,
+      "step": 350
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.9989680118771284e-05,
+      "logits/chosen": -1.4652230739593506,
+      "logits/rejected": -1.295188069343567,
+      "logps/chosen": -271.06207275390625,
+      "logps/rejected": -289.2547302246094,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.5600930452346802,
+      "rewards/margins": 2.303740978240967,
+      "rewards/rejected": -2.8638339042663574,
+      "step": 355
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.9985504438751075e-05,
+      "logits/chosen": -1.5906970500946045,
+      "logits/rejected": -1.433538556098938,
+      "logps/chosen": -252.5865936279297,
+      "logps/rejected": -284.74139404296875,
+      "loss": 0.365,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3220791220664978,
+      "rewards/margins": 1.9737313985824585,
+      "rewards/rejected": -2.2958106994628906,
+      "step": 360
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998062131134687e-05,
+      "logits/chosen": -1.4737342596054077,
+      "logits/rejected": -1.3808215856552124,
+      "logps/chosen": -225.4361114501953,
+      "logps/rejected": -269.8510437011719,
+      "loss": 0.4805,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.5886635184288025,
+      "rewards/margins": 2.1054704189300537,
+      "rewards/rejected": -2.694133996963501,
+      "step": 365
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.99750308748211e-05,
+      "logits/chosen": -1.3672006130218506,
+      "logits/rejected": -1.2418177127838135,
+      "logps/chosen": -252.7670135498047,
+      "logps/rejected": -298.70159912109375,
+      "loss": 0.429,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.6679830551147461,
+      "rewards/margins": 2.3387298583984375,
+      "rewards/rejected": -3.0067131519317627,
+      "step": 370
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.996873328746311e-05,
+      "logits/chosen": -1.444612741470337,
+      "logits/rejected": -1.3134263753890991,
+      "logps/chosen": -258.3940734863281,
+      "logps/rejected": -270.0694885253906,
+      "loss": 0.4651,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.7545816898345947,
+      "rewards/margins": 2.0772323608398438,
+      "rewards/rejected": -2.8318140506744385,
+      "step": 375
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.9961728727584764e-05,
+      "logits/chosen": -1.4437062740325928,
+      "logits/rejected": -1.3258285522460938,
+      "logps/chosen": -248.73953247070312,
+      "logps/rejected": -284.25653076171875,
+      "loss": 0.385,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.1430978775024414,
+      "rewards/margins": 2.3336520195007324,
+      "rewards/rejected": -2.4767496585845947,
+      "step": 380
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995401739351536e-05,
+      "logits/chosen": -1.5048315525054932,
+      "logits/rejected": -1.4178255796432495,
+      "logps/chosen": -204.1002655029297,
+      "logps/rejected": -228.6428680419922,
+      "loss": 0.4414,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.21167969703674316,
+      "rewards/margins": 1.6109260320663452,
+      "rewards/rejected": -1.822605848312378,
+      "step": 385
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.994559950359603e-05,
+      "logits/chosen": -1.521078109741211,
+      "logits/rejected": -1.431056022644043,
+      "logps/chosen": -211.77392578125,
+      "logps/rejected": -271.99530029296875,
+      "loss": 0.4005,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.32148247957229614,
+      "rewards/margins": 2.0455245971679688,
+      "rewards/rejected": -2.36700701713562,
+      "step": 390
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.9936475296173524e-05,
+      "logits/chosen": -1.4915629625320435,
+      "logits/rejected": -1.4468661546707153,
+      "logps/chosen": -220.66238403320312,
+      "logps/rejected": -274.36212158203125,
+      "loss": 0.398,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.4649580121040344,
+      "rewards/margins": 2.0459415912628174,
+      "rewards/rejected": -2.510899543762207,
+      "step": 395
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.992664502959351e-05,
+      "logits/chosen": -1.5382647514343262,
+      "logits/rejected": -1.4424117803573608,
+      "logps/chosen": -237.43264770507812,
+      "logps/rejected": -265.6887512207031,
+      "loss": 0.4285,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.7469789981842041,
+      "rewards/margins": 2.0618503093719482,
+      "rewards/rejected": -2.8088290691375732,
+      "step": 400
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.9916108982193246e-05,
+      "logits/chosen": -1.4920802116394043,
+      "logits/rejected": -1.3715362548828125,
+      "logps/chosen": -248.9379425048828,
+      "logps/rejected": -275.0431213378906,
+      "loss": 0.3805,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.266801118850708,
+      "rewards/margins": 1.8889585733413696,
+      "rewards/rejected": -2.155759811401367,
+      "step": 405
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.990486745229364e-05,
+      "logits/chosen": -1.4824012517929077,
+      "logits/rejected": -1.41977858543396,
+      "logps/chosen": -220.0709991455078,
+      "logps/rejected": -247.99560546875,
+      "loss": 0.4792,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.18558254837989807,
+      "rewards/margins": 1.6866681575775146,
+      "rewards/rejected": -1.501085638999939,
+      "step": 410
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.9892920758190907e-05,
+      "logits/chosen": -1.4227807521820068,
+      "logits/rejected": -1.3182239532470703,
+      "logps/chosen": -237.0270233154297,
+      "logps/rejected": -266.37872314453125,
+      "loss": 0.4349,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.0361756794154644,
+      "rewards/margins": 1.9962981939315796,
+      "rewards/rejected": -2.0324740409851074,
+      "step": 415
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.988026923814748e-05,
+      "logits/chosen": -1.5704119205474854,
+      "logits/rejected": -1.472022294998169,
+      "logps/chosen": -251.1329803466797,
+      "logps/rejected": -289.251953125,
+      "loss": 0.3614,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.0848822221159935,
+      "rewards/margins": 2.3444581031799316,
+      "rewards/rejected": -2.429340362548828,
+      "step": 420
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.986691325038244e-05,
+      "logits/chosen": -1.5436654090881348,
+      "logits/rejected": -1.4181368350982666,
+      "logps/chosen": -235.8417205810547,
+      "logps/rejected": -269.031005859375,
+      "loss": 0.4396,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.306237131357193,
+      "rewards/margins": 2.005009889602661,
+      "rewards/rejected": -2.3112473487854004,
+      "step": 425
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.985285317306141e-05,
+      "logits/chosen": -1.4109728336334229,
+      "logits/rejected": -1.3263506889343262,
+      "logps/chosen": -242.7462615966797,
+      "logps/rejected": -278.8143005371094,
+      "loss": 0.3797,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.6337249279022217,
+      "rewards/margins": 2.4727721214294434,
+      "rewards/rejected": -3.106497049331665,
+      "step": 430
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.9838089404285807e-05,
+      "logits/chosen": -1.4374382495880127,
+      "logits/rejected": -1.3346731662750244,
+      "logps/chosen": -229.8583984375,
+      "logps/rejected": -258.11395263671875,
+      "loss": 0.5004,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.9803594350814819,
+      "rewards/margins": 2.253356695175171,
+      "rewards/rejected": -3.2337162494659424,
+      "step": 435
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.9822622362081594e-05,
+      "logits/chosen": -1.449986219406128,
+      "logits/rejected": -1.3739886283874512,
+      "logps/chosen": -240.986328125,
+      "logps/rejected": -296.00531005859375,
+      "loss": 0.4265,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.41646942496299744,
+      "rewards/margins": 2.698214292526245,
+      "rewards/rejected": -3.1146836280822754,
+      "step": 440
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.980645248438745e-05,
+      "logits/chosen": -1.5218524932861328,
+      "logits/rejected": -1.4302794933319092,
+      "logps/chosen": -214.82852172851562,
+      "logps/rejected": -265.3507995605469,
+      "loss": 0.3939,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.3738623261451721,
+      "rewards/margins": 2.2887752056121826,
+      "rewards/rejected": -1.9149129390716553,
+      "step": 445
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.978958022904235e-05,
+      "logits/chosen": -1.5862996578216553,
+      "logits/rejected": -1.4788892269134521,
+      "logps/chosen": -234.18478393554688,
+      "logps/rejected": -263.1192932128906,
+      "loss": 0.4216,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.3776335120201111,
+      "rewards/margins": 2.2993741035461426,
+      "rewards/rejected": -1.9217407703399658,
+      "step": 450
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.977200607377259e-05,
+      "logits/chosen": -1.5885207653045654,
+      "logits/rejected": -1.5190343856811523,
+      "logps/chosen": -228.0282745361328,
+      "logps/rejected": -270.5809326171875,
+      "loss": 0.4147,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.047803785651922226,
+      "rewards/margins": 1.674283742904663,
+      "rewards/rejected": -1.7220878601074219,
+      "step": 455
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.9753730516178313e-05,
+      "logits/chosen": -1.5095998048782349,
+      "logits/rejected": -1.4479546546936035,
+      "logps/chosen": -242.82656860351562,
+      "logps/rejected": -279.6665344238281,
+      "loss": 0.3911,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.3711318373680115,
+      "rewards/margins": 2.2145276069641113,
+      "rewards/rejected": -2.5856597423553467,
+      "step": 460
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.9734754073719355e-05,
+      "logits/chosen": -1.5498822927474976,
+      "logits/rejected": -1.4521539211273193,
+      "logps/chosen": -237.3514404296875,
+      "logps/rejected": -279.55035400390625,
+      "loss": 0.4096,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.7321179509162903,
+      "rewards/margins": 2.069248676300049,
+      "rewards/rejected": -2.8013663291931152,
+      "step": 465
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.971507728370066e-05,
+      "logits/chosen": -1.4880825281143188,
+      "logits/rejected": -1.414366602897644,
+      "logps/chosen": -244.0042266845703,
+      "logps/rejected": -303.25506591796875,
+      "loss": 0.3607,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.46353158354759216,
+      "rewards/margins": 2.542686939239502,
+      "rewards/rejected": -3.006218671798706,
+      "step": 470
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.969470070325699e-05,
+      "logits/chosen": -1.546096682548523,
+      "logits/rejected": -1.4253944158554077,
+      "logps/chosen": -225.0137481689453,
+      "logps/rejected": -271.5694885253906,
+      "loss": 0.4059,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.3330061137676239,
+      "rewards/margins": 2.4641575813293457,
+      "rewards/rejected": -2.797163724899292,
+      "step": 475
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.967362490933723e-05,
+      "logits/chosen": -1.37833571434021,
+      "logits/rejected": -1.2442013025283813,
+      "logps/chosen": -227.9774627685547,
+      "logps/rejected": -260.406982421875,
+      "loss": 0.3492,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.4235529899597168,
+      "rewards/margins": 2.632197618484497,
+      "rewards/rejected": -3.055750608444214,
+      "step": 480
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.9651850498688e-05,
+      "logits/chosen": -1.5022382736206055,
+      "logits/rejected": -1.3960180282592773,
+      "logps/chosen": -244.43344116210938,
+      "logps/rejected": -302.40570068359375,
+      "loss": 0.4167,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.21162764728069305,
+      "rewards/margins": 2.4925358295440674,
+      "rewards/rejected": -2.7041635513305664,
+      "step": 485
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.962937808783675e-05,
+      "logits/chosen": -1.4933425188064575,
+      "logits/rejected": -1.441125512123108,
+      "logps/chosen": -241.4817352294922,
+      "logps/rejected": -303.1959533691406,
+      "loss": 0.3826,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2536941468715668,
+      "rewards/margins": 2.397275447845459,
+      "rewards/rejected": -2.6509695053100586,
+      "step": 490
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.960620831307436e-05,
+      "logits/chosen": -1.4081984758377075,
+      "logits/rejected": -1.3692537546157837,
+      "logps/chosen": -205.82504272460938,
+      "logps/rejected": -246.38015747070312,
+      "loss": 0.4207,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.6011780500411987,
+      "rewards/margins": 1.9714361429214478,
+      "rewards/rejected": -2.5726141929626465,
+      "step": 495
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.9582341830437085e-05,
+      "logits/chosen": -1.4795446395874023,
+      "logits/rejected": -1.3015785217285156,
+      "logps/chosen": -238.45947265625,
+      "logps/rejected": -233.8590850830078,
+      "loss": 0.436,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.386214941740036,
+      "rewards/margins": 2.216794490814209,
+      "rewards/rejected": -2.6030097007751465,
+      "step": 500
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.955777931568797e-05,
+      "logits/chosen": -1.3686350584030151,
+      "logits/rejected": -1.3235373497009277,
+      "logps/chosen": -217.8348388671875,
+      "logps/rejected": -280.8824768066406,
+      "loss": 0.4044,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.014941399917006493,
+      "rewards/margins": 2.3758702278137207,
+      "rewards/rejected": -2.360928773880005,
+      "step": 505
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.953252146429772e-05,
+      "logits/chosen": -1.51080322265625,
+      "logits/rejected": -1.4593368768692017,
+      "logps/chosen": -200.56521606445312,
+      "logps/rejected": -246.88388061523438,
+      "loss": 0.4536,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.2722179889678955,
+      "rewards/margins": 1.7458912134170532,
+      "rewards/rejected": -1.4736731052398682,
+      "step": 510
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.9506568991425065e-05,
+      "logits/chosen": -1.4120771884918213,
+      "logits/rejected": -1.3655294179916382,
+      "logps/chosen": -221.00479125976562,
+      "logps/rejected": -252.9807586669922,
+      "loss": 0.4058,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.4606713652610779,
+      "rewards/margins": 1.9229342937469482,
+      "rewards/rejected": -1.462262749671936,
+      "step": 515
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.9479922631896405e-05,
+      "logits/chosen": -1.523662805557251,
+      "logits/rejected": -1.4615800380706787,
+      "logps/chosen": -216.89169311523438,
+      "logps/rejected": -274.2598571777344,
+      "loss": 0.3706,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.594303548336029,
+      "rewards/margins": 2.0781185626983643,
+      "rewards/rejected": -1.4838149547576904,
+      "step": 520
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.945258314018511e-05,
+      "logits/chosen": -1.523301124572754,
+      "logits/rejected": -1.373157262802124,
+      "logps/chosen": -245.4419403076172,
+      "logps/rejected": -268.5808410644531,
+      "loss": 0.3833,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.17819438874721527,
+      "rewards/margins": 2.2116293907165527,
+      "rewards/rejected": -2.033435106277466,
+      "step": 525
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.942455129039011e-05,
+      "logits/chosen": -1.455971360206604,
+      "logits/rejected": -1.3837201595306396,
+      "logps/chosen": -233.8610382080078,
+      "logps/rejected": -282.94891357421875,
+      "loss": 0.3602,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3101358711719513,
+      "rewards/margins": 2.8269975185394287,
+      "rewards/rejected": -3.1371333599090576,
+      "step": 530
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.9395827876213936e-05,
+      "logits/chosen": -1.4602159261703491,
+      "logits/rejected": -1.357772707939148,
+      "logps/chosen": -238.6614227294922,
+      "logps/rejected": -281.5059814453125,
+      "loss": 0.4362,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.42020535469055176,
+      "rewards/margins": 2.5654282569885254,
+      "rewards/rejected": -2.9856338500976562,
+      "step": 535
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.936641371094033e-05,
+      "logits/chosen": -1.5019209384918213,
+      "logits/rejected": -1.5190550088882446,
+      "logps/chosen": -197.41287231445312,
+      "logps/rejected": -252.25137329101562,
+      "loss": 0.4469,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.11766906827688217,
+      "rewards/margins": 2.03245210647583,
+      "rewards/rejected": -2.1501212120056152,
+      "step": 540
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.9336309627411163e-05,
+      "logits/chosen": -1.423473834991455,
+      "logits/rejected": -1.4443773031234741,
+      "logps/chosen": -217.05453491210938,
+      "logps/rejected": -282.62164306640625,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.06701436638832092,
+      "rewards/margins": 2.2761709690093994,
+      "rewards/rejected": -2.3431851863861084,
+      "step": 545
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.9305516478002865e-05,
+      "logits/chosen": -1.4173814058303833,
+      "logits/rejected": -1.3098132610321045,
+      "logps/chosen": -249.5691375732422,
+      "logps/rejected": -292.90435791015625,
+      "loss": 0.4061,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.5416631102561951,
+      "rewards/margins": 2.6680707931518555,
+      "rewards/rejected": -3.2097339630126953,
+      "step": 550
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.92740351346023e-05,
+      "logits/chosen": -1.3292713165283203,
+      "logits/rejected": -1.2327873706817627,
+      "logps/chosen": -229.86007690429688,
+      "logps/rejected": -264.585693359375,
+      "loss": 0.3875,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.7183946371078491,
+      "rewards/margins": 2.537214517593384,
+      "rewards/rejected": -3.2556090354919434,
+      "step": 555
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.924186648858207e-05,
+      "logits/chosen": -1.3974854946136475,
+      "logits/rejected": -1.2770755290985107,
+      "logps/chosen": -228.25625610351562,
+      "logps/rejected": -271.54052734375,
+      "loss": 0.398,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.6100394129753113,
+      "rewards/margins": 2.393568992614746,
+      "rewards/rejected": -3.003608226776123,
+      "step": 560
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.920901145077527e-05,
+      "logits/chosen": -1.5996006727218628,
+      "logits/rejected": -1.5182517766952515,
+      "logps/chosen": -213.2860565185547,
+      "logps/rejected": -258.7254943847656,
+      "loss": 0.4096,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1525319218635559,
+      "rewards/margins": 1.8957157135009766,
+      "rewards/rejected": -2.0482475757598877,
+      "step": 565
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.917547095144971e-05,
+      "logits/chosen": -1.492539644241333,
+      "logits/rejected": -1.4047850370407104,
+      "logps/chosen": -239.7179412841797,
+      "logps/rejected": -278.13507080078125,
+      "loss": 0.3826,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.23858800530433655,
+      "rewards/margins": 2.1294798851013184,
+      "rewards/rejected": -2.368067979812622,
+      "step": 570
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.914124594028157e-05,
+      "logits/chosen": -1.4673938751220703,
+      "logits/rejected": -1.3367671966552734,
+      "logps/chosen": -265.41009521484375,
+      "logps/rejected": -313.9954528808594,
+      "loss": 0.4158,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5570913553237915,
+      "rewards/margins": 2.5719971656799316,
+      "rewards/rejected": -3.1290886402130127,
+      "step": 575
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.9106337386328524e-05,
+      "logits/chosen": -1.4329808950424194,
+      "logits/rejected": -1.3196234703063965,
+      "logps/chosen": -249.49081420898438,
+      "logps/rejected": -286.67352294921875,
+      "loss": 0.4113,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6757558584213257,
+      "rewards/margins": 2.547755241394043,
+      "rewards/rejected": -3.2235107421875,
+      "step": 580
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.907074627800229e-05,
+      "logits/chosen": -1.5212651491165161,
+      "logits/rejected": -1.376366376876831,
+      "logps/chosen": -263.5170593261719,
+      "logps/rejected": -291.48876953125,
+      "loss": 0.4057,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6802183389663696,
+      "rewards/margins": 2.613145112991333,
+      "rewards/rejected": -3.293363094329834,
+      "step": 585
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.903447362304061e-05,
+      "logits/chosen": -1.5662963390350342,
+      "logits/rejected": -1.4853650331497192,
+      "logps/chosen": -226.19937133789062,
+      "logps/rejected": -273.17620849609375,
+      "loss": 0.3983,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.4168701767921448,
+      "rewards/margins": 2.3373947143554688,
+      "rewards/rejected": -2.7542648315429688,
+      "step": 590
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.899752044847881e-05,
+      "logits/chosen": -1.5506370067596436,
+      "logits/rejected": -1.4166381359100342,
+      "logps/chosen": -239.8184814453125,
+      "logps/rejected": -272.5811462402344,
+      "loss": 0.4064,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.8294523358345032,
+      "rewards/margins": 2.456678867340088,
+      "rewards/rejected": -3.2861316204071045,
+      "step": 595
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.895988780062059e-05,
+      "logits/chosen": -1.3921419382095337,
+      "logits/rejected": -1.3240123987197876,
+      "logps/chosen": -232.83157348632812,
+      "logps/rejected": -274.8085021972656,
+      "loss": 0.4231,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.0233221054077148,
+      "rewards/margins": 2.417581081390381,
+      "rewards/rejected": -3.440903425216675,
+      "step": 600
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.8921576745008544e-05,
+      "logits/chosen": -1.551561713218689,
+      "logits/rejected": -1.423801302909851,
+      "logps/chosen": -262.9552917480469,
+      "logps/rejected": -295.07269287109375,
+      "loss": 0.4166,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0641061067581177,
+      "rewards/margins": 2.3552744388580322,
+      "rewards/rejected": -3.4193801879882812,
+      "step": 605
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.888258836639386e-05,
+      "logits/chosen": -1.3881410360336304,
+      "logits/rejected": -1.3344438076019287,
+      "logps/chosen": -239.0188751220703,
+      "logps/rejected": -303.75640869140625,
+      "loss": 0.4113,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.2905619144439697,
+      "rewards/margins": 2.735004186630249,
+      "rewards/rejected": -4.025566577911377,
+      "step": 610
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.884292376870567e-05,
+      "logits/chosen": -1.3135260343551636,
+      "logits/rejected": -1.2955373525619507,
+      "logps/chosen": -240.2941436767578,
+      "logps/rejected": -308.3316345214844,
+      "loss": 0.5103,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.0934088230133057,
+      "rewards/margins": 2.280285358428955,
+      "rewards/rejected": -3.3736941814422607,
+      "step": 615
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.880258407501982e-05,
+      "logits/chosen": -1.4538220167160034,
+      "logits/rejected": -1.3662500381469727,
+      "logps/chosen": -253.21591186523438,
+      "logps/rejected": -297.54193115234375,
+      "loss": 0.4718,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.6241488456726074,
+      "rewards/margins": 2.2601757049560547,
+      "rewards/rejected": -2.884324550628662,
+      "step": 620
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8761570427526973e-05,
+      "logits/chosen": -1.5741875171661377,
+      "logits/rejected": -1.4919278621673584,
+      "logps/chosen": -232.79373168945312,
+      "logps/rejected": -269.60089111328125,
+      "loss": 0.4284,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.1897115260362625,
+      "rewards/margins": 1.9414294958114624,
+      "rewards/rejected": -1.7517179250717163,
+      "step": 625
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.87198839875004e-05,
+      "logits/chosen": -1.5609136819839478,
+      "logits/rejected": -1.45805025100708,
+      "logps/chosen": -208.7440643310547,
+      "logps/rejected": -234.986083984375,
+      "loss": 0.4536,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.25724244117736816,
+      "rewards/margins": 1.7011772394180298,
+      "rewards/rejected": -1.443934679031372,
+      "step": 630
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.867752593526297e-05,
+      "logits/chosen": -1.5343798398971558,
+      "logits/rejected": -1.4163181781768799,
+      "logps/chosen": -227.30960083007812,
+      "logps/rejected": -285.6361083984375,
+      "loss": 0.3958,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.07786116749048233,
+      "rewards/margins": 2.143432855606079,
+      "rewards/rejected": -2.0655717849731445,
+      "step": 635
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.863449747015384e-05,
+      "logits/chosen": -1.4224778413772583,
+      "logits/rejected": -1.3595422506332397,
+      "logps/chosen": -240.3905029296875,
+      "logps/rejected": -296.36712646484375,
+      "loss": 0.4179,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.4491243362426758,
+      "rewards/margins": 2.387755870819092,
+      "rewards/rejected": -2.8368804454803467,
+      "step": 640
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.8590799810494405e-05,
+      "logits/chosen": -1.4686813354492188,
+      "logits/rejected": -1.373623013496399,
+      "logps/chosen": -196.8067626953125,
+      "logps/rejected": -231.6805877685547,
+      "loss": 0.3975,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.22348129749298096,
+      "rewards/margins": 2.002708911895752,
+      "rewards/rejected": -2.2261900901794434,
+      "step": 645
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.854643419355387e-05,
+      "logits/chosen": -1.3826911449432373,
+      "logits/rejected": -1.2899580001831055,
+      "logps/chosen": -208.8357391357422,
+      "logps/rejected": -274.06304931640625,
+      "loss": 0.3487,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.2312304526567459,
+      "rewards/margins": 2.2438297271728516,
+      "rewards/rejected": -2.475059986114502,
+      "step": 650
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.850140187551417e-05,
+      "logits/chosen": -1.4895564317703247,
+      "logits/rejected": -1.4108827114105225,
+      "logps/chosen": -220.452392578125,
+      "logps/rejected": -257.15142822265625,
+      "loss": 0.3977,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.1533854752779007,
+      "rewards/margins": 2.6028971672058105,
+      "rewards/rejected": -2.756282329559326,
+      "step": 655
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.8455704131434463e-05,
+      "logits/chosen": -1.402146339416504,
+      "logits/rejected": -1.3426183462142944,
+      "logps/chosen": -210.43310546875,
+      "logps/rejected": -261.40826416015625,
+      "loss": 0.4083,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.7025829553604126,
+      "rewards/margins": 2.593606472015381,
+      "rewards/rejected": -3.296189069747925,
+      "step": 660
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.840934225521495e-05,
+      "logits/chosen": -1.3444569110870361,
+      "logits/rejected": -1.2977235317230225,
+      "logps/chosen": -234.8567657470703,
+      "logps/rejected": -276.48529052734375,
+      "loss": 0.3982,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.500916600227356,
+      "rewards/margins": 2.0213747024536133,
+      "rewards/rejected": -2.522291421890259,
+      "step": 665
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.8362317559560274e-05,
+      "logits/chosen": -1.4623371362686157,
+      "logits/rejected": -1.3292256593704224,
+      "logps/chosen": -227.4473876953125,
+      "logps/rejected": -257.6618957519531,
+      "loss": 0.3551,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.2117508351802826,
+      "rewards/margins": 2.53009033203125,
+      "rewards/rejected": -2.7418415546417236,
+      "step": 670
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.8314631375942385e-05,
+      "logits/chosen": -1.495482087135315,
+      "logits/rejected": -1.4167420864105225,
+      "logps/chosen": -230.7625274658203,
+      "logps/rejected": -274.5197448730469,
+      "loss": 0.3984,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.09574685990810394,
+      "rewards/margins": 2.2370810508728027,
+      "rewards/rejected": -2.3328278064727783,
+      "step": 675
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.8266285054562794e-05,
+      "logits/chosen": -1.5286659002304077,
+      "logits/rejected": -1.4208118915557861,
+      "logps/chosen": -238.05770874023438,
+      "logps/rejected": -278.955078125,
+      "loss": 0.3855,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.17491035163402557,
+      "rewards/margins": 2.2506463527679443,
+      "rewards/rejected": -2.0757360458374023,
+      "step": 680
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.821727996431435e-05,
+      "logits/chosen": -1.4394538402557373,
+      "logits/rejected": -1.3951141834259033,
+      "logps/chosen": -227.53298950195312,
+      "logps/rejected": -279.4712219238281,
+      "loss": 0.3826,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.11384101212024689,
+      "rewards/margins": 2.3539175987243652,
+      "rewards/rejected": -2.2400765419006348,
+      "step": 685
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.816761749274251e-05,
+      "logits/chosen": -1.4274007081985474,
+      "logits/rejected": -1.4132310152053833,
+      "logps/chosen": -217.4455108642578,
+      "logps/rejected": -278.5160827636719,
+      "loss": 0.3496,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.09646536409854889,
+      "rewards/margins": 2.620297908782959,
+      "rewards/rejected": -2.7167630195617676,
+      "step": 690
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.8117299046006e-05,
+      "logits/chosen": -1.5871320962905884,
+      "logits/rejected": -1.4668903350830078,
+      "logps/chosen": -237.6433868408203,
+      "logps/rejected": -277.51123046875,
+      "loss": 0.3578,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2201867550611496,
+      "rewards/margins": 2.6030189990997314,
+      "rewards/rejected": -2.8232059478759766,
+      "step": 695
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.806632604883708e-05,
+      "logits/chosen": -1.492653489112854,
+      "logits/rejected": -1.3919525146484375,
+      "logps/chosen": -248.25741577148438,
+      "logps/rejected": -313.47393798828125,
+      "loss": 0.3474,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.24769659340381622,
+      "rewards/margins": 2.886350154876709,
+      "rewards/rejected": -3.134047031402588,
+      "step": 700
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.801469994450111e-05,
+      "logits/chosen": -1.5104761123657227,
+      "logits/rejected": -1.3608448505401611,
+      "logps/chosen": -250.60995483398438,
+      "logps/rejected": -254.9477996826172,
+      "loss": 0.4383,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.2116759717464447,
+      "rewards/margins": 2.409428596496582,
+      "rewards/rejected": -2.6211047172546387,
+      "step": 705
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.796242219475575e-05,
+      "logits/chosen": -1.4535516500473022,
+      "logits/rejected": -1.391486644744873,
+      "logps/chosen": -223.2052764892578,
+      "logps/rejected": -280.236083984375,
+      "loss": 0.3887,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.05361563712358475,
+      "rewards/margins": 2.583587646484375,
+      "rewards/rejected": -2.6372032165527344,
+      "step": 710
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.790949427980956e-05,
+      "logits/chosen": -1.4059816598892212,
+      "logits/rejected": -1.3338401317596436,
+      "logps/chosen": -249.96279907226562,
+      "logps/rejected": -290.481689453125,
+      "loss": 0.4033,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4107217788696289,
+      "rewards/margins": 2.9012093544006348,
+      "rewards/rejected": -3.3119311332702637,
+      "step": 715
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.7855917698280054e-05,
+      "logits/chosen": -1.4610540866851807,
+      "logits/rejected": -1.317604660987854,
+      "logps/chosen": -248.26956176757812,
+      "logps/rejected": -262.8702697753906,
+      "loss": 0.4896,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.37552323937416077,
+      "rewards/margins": 2.372670888900757,
+      "rewards/rejected": -2.7481942176818848,
+      "step": 720
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.780169396715133e-05,
+      "logits/chosen": -1.5573104619979858,
+      "logits/rejected": -1.4791498184204102,
+      "logps/chosen": -219.789794921875,
+      "logps/rejected": -264.08685302734375,
+      "loss": 0.4026,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.15070626139640808,
+      "rewards/margins": 2.185694932937622,
+      "rewards/rejected": -2.0349888801574707,
+      "step": 725
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.774682462173105e-05,
+      "logits/chosen": -1.551232099533081,
+      "logits/rejected": -1.407405138015747,
+      "logps/chosen": -245.1878662109375,
+      "logps/rejected": -269.4394226074219,
+      "loss": 0.389,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.07436565309762955,
+      "rewards/margins": 2.136859178543091,
+      "rewards/rejected": -2.2112247943878174,
+      "step": 730
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.769131121560701e-05,
+      "logits/chosen": -1.5128840208053589,
+      "logits/rejected": -1.4270654916763306,
+      "logps/chosen": -246.69100952148438,
+      "logps/rejected": -283.3271484375,
+      "loss": 0.4394,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.672562837600708,
+      "rewards/margins": 2.451646327972412,
+      "rewards/rejected": -3.1242096424102783,
+      "step": 735
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.763515532060316e-05,
+      "logits/chosen": -1.4596669673919678,
+      "logits/rejected": -1.420090913772583,
+      "logps/chosen": -211.2493133544922,
+      "logps/rejected": -274.11322021484375,
+      "loss": 0.4276,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.5362947583198547,
+      "rewards/margins": 2.3526790142059326,
+      "rewards/rejected": -2.8889739513397217,
+      "step": 740
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7578358526735065e-05,
+      "logits/chosen": -1.5720094442367554,
+      "logits/rejected": -1.4286072254180908,
+      "logps/chosen": -254.8545379638672,
+      "logps/rejected": -267.1537170410156,
+      "loss": 0.4106,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07587162405252457,
+      "rewards/margins": 2.3346734046936035,
+      "rewards/rejected": -2.4105448722839355,
+      "step": 745
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7520922442164894e-05,
+      "logits/chosen": -1.5144745111465454,
+      "logits/rejected": -1.4029021263122559,
+      "logps/chosen": -212.46533203125,
+      "logps/rejected": -231.5972900390625,
+      "loss": 0.4307,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.12235695123672485,
+      "rewards/margins": 1.6954295635223389,
+      "rewards/rejected": -1.5730727910995483,
+      "step": 750
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.74628486931559e-05,
+      "logits/chosen": -1.5446897745132446,
+      "logits/rejected": -1.4459664821624756,
+      "logps/chosen": -241.0093536376953,
+      "logps/rejected": -268.1150207519531,
+      "loss": 0.3903,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.28674596548080444,
+      "rewards/margins": 2.2632548809051514,
+      "rewards/rejected": -1.9765087366104126,
+      "step": 755
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.740413892402639e-05,
+      "logits/chosen": -1.4135468006134033,
+      "logits/rejected": -1.3602290153503418,
+      "logps/chosen": -253.1114044189453,
+      "logps/rejected": -305.5167541503906,
+      "loss": 0.4179,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.12774869799613953,
+      "rewards/margins": 2.793480634689331,
+      "rewards/rejected": -2.6657321453094482,
+      "step": 760
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.734479479710311e-05,
+      "logits/chosen": -1.5195525884628296,
+      "logits/rejected": -1.446173906326294,
+      "logps/chosen": -244.97616577148438,
+      "logps/rejected": -279.899658203125,
+      "loss": 0.3906,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.19458039104938507,
+      "rewards/margins": 2.4705350399017334,
+      "rewards/rejected": -2.6651155948638916,
+      "step": 765
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.728481799267421e-05,
+      "logits/chosen": -1.4518150091171265,
+      "logits/rejected": -1.3336080312728882,
+      "logps/chosen": -266.017822265625,
+      "logps/rejected": -296.8194274902344,
+      "loss": 0.3854,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.503818154335022,
+      "rewards/margins": 2.678640365600586,
+      "rewards/rejected": -3.1824586391448975,
+      "step": 770
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.722421020894169e-05,
+      "logits/chosen": -1.4531335830688477,
+      "logits/rejected": -1.3481992483139038,
+      "logps/chosen": -247.59585571289062,
+      "logps/rejected": -294.21722412109375,
+      "loss": 0.3923,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7091779708862305,
+      "rewards/margins": 3.397832155227661,
+      "rewards/rejected": -4.107010364532471,
+      "step": 775
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.71629731619733e-05,
+      "logits/chosen": -1.4069632291793823,
+      "logits/rejected": -1.3233740329742432,
+      "logps/chosen": -251.7724151611328,
+      "logps/rejected": -308.6446838378906,
+      "loss": 0.387,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.1772078275680542,
+      "rewards/margins": 2.8983986377716064,
+      "rewards/rejected": -4.075606346130371,
+      "step": 780
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.7101108585653905e-05,
+      "logits/chosen": -1.4547842741012573,
+      "logits/rejected": -1.313291311264038,
+      "logps/chosen": -258.6347961425781,
+      "logps/rejected": -288.6201171875,
+      "loss": 0.3736,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.9564958810806274,
+      "rewards/margins": 2.8389906883239746,
+      "rewards/rejected": -3.7954864501953125,
+      "step": 785
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.703861823163649e-05,
+      "logits/chosen": -1.5221706628799438,
+      "logits/rejected": -1.4411219358444214,
+      "logps/chosen": -226.8802032470703,
+      "logps/rejected": -279.59564208984375,
+      "loss": 0.4248,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.6723430752754211,
+      "rewards/margins": 2.194626569747925,
+      "rewards/rejected": -2.866969585418701,
+      "step": 790
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.697550386929246e-05,
+      "logits/chosen": -1.3913832902908325,
+      "logits/rejected": -1.274837613105774,
+      "logps/chosen": -244.55783081054688,
+      "logps/rejected": -282.39422607421875,
+      "loss": 0.3963,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.37080082297325134,
+      "rewards/margins": 2.579662561416626,
+      "rewards/rejected": -2.9504635334014893,
+      "step": 795
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.691176728566159e-05,
+      "logits/chosen": -1.4640603065490723,
+      "logits/rejected": -1.405020833015442,
+      "logps/chosen": -220.28701782226562,
+      "logps/rejected": -271.06011962890625,
+      "loss": 0.3549,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.5138343572616577,
+      "rewards/margins": 2.453996181488037,
+      "rewards/rejected": -2.9678304195404053,
+      "step": 800
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.684741028540146e-05,
+      "logits/chosen": -1.3809168338775635,
+      "logits/rejected": -1.29449462890625,
+      "logps/chosen": -220.06051635742188,
+      "logps/rejected": -277.65325927734375,
+      "loss": 0.361,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.5629986524581909,
+      "rewards/margins": 2.7898635864257812,
+      "rewards/rejected": -3.3528621196746826,
+      "step": 805
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.6782434690736274e-05,
+      "logits/chosen": -1.455427885055542,
+      "logits/rejected": -1.315850019454956,
+      "logps/chosen": -265.70391845703125,
+      "logps/rejected": -298.28924560546875,
+      "loss": 0.4675,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.49216946959495544,
+      "rewards/margins": 3.0039374828338623,
+      "rewards/rejected": -3.4961071014404297,
+      "step": 810
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.671684234140535e-05,
+      "logits/chosen": -1.4259642362594604,
+      "logits/rejected": -1.296662449836731,
+      "logps/chosen": -237.97695922851562,
+      "logps/rejected": -260.7139892578125,
+      "loss": 0.4043,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.25738030672073364,
+      "rewards/margins": 2.880896806716919,
+      "rewards/rejected": -3.1382765769958496,
+      "step": 815
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.665063509461097e-05,
+      "logits/chosen": -1.4097397327423096,
+      "logits/rejected": -1.3623135089874268,
+      "logps/chosen": -224.6775665283203,
+      "logps/rejected": -264.73846435546875,
+      "loss": 0.4093,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.17778576910495758,
+      "rewards/margins": 2.099287986755371,
+      "rewards/rejected": -1.9215021133422852,
+      "step": 820
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.6583814824965805e-05,
+      "logits/chosen": -1.5525894165039062,
+      "logits/rejected": -1.4566829204559326,
+      "logps/chosen": -222.94430541992188,
+      "logps/rejected": -271.26556396484375,
+      "loss": 0.3995,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.313865065574646,
+      "rewards/margins": 2.3322434425354004,
+      "rewards/rejected": -2.018378496170044,
+      "step": 825
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.651638342443987e-05,
+      "logits/chosen": -1.5715656280517578,
+      "logits/rejected": -1.554890751838684,
+      "logps/chosen": -224.2897186279297,
+      "logps/rejected": -272.0341796875,
+      "loss": 0.4459,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.030287206172943115,
+      "rewards/margins": 1.9702775478363037,
+      "rewards/rejected": -1.9399904012680054,
+      "step": 830
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.644834280230692e-05,
+      "logits/chosen": -1.5804816484451294,
+      "logits/rejected": -1.4860570430755615,
+      "logps/chosen": -200.5336456298828,
+      "logps/rejected": -246.2007598876953,
+      "loss": 0.4214,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.011333741247653961,
+      "rewards/margins": 2.026169776916504,
+      "rewards/rejected": -2.014835834503174,
+      "step": 835
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6379694885090405e-05,
+      "logits/chosen": -1.4862781763076782,
+      "logits/rejected": -1.4114696979522705,
+      "logps/chosen": -252.9210205078125,
+      "logps/rejected": -303.67584228515625,
+      "loss": 0.3977,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.4330156445503235,
+      "rewards/margins": 2.5075771808624268,
+      "rewards/rejected": -2.9405925273895264,
+      "step": 840
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6310441616508914e-05,
+      "logits/chosen": -1.3494175672531128,
+      "logits/rejected": -1.2918002605438232,
+      "logps/chosen": -236.6602783203125,
+      "logps/rejected": -294.8357238769531,
+      "loss": 0.3647,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.5964530110359192,
+      "rewards/margins": 2.777496814727783,
+      "rewards/rejected": -3.3739497661590576,
+      "step": 845
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.624058495742114e-05,
+      "logits/chosen": -1.4458904266357422,
+      "logits/rejected": -1.3650354146957397,
+      "logps/chosen": -261.0570983886719,
+      "logps/rejected": -323.8190002441406,
+      "loss": 0.3724,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.723595917224884,
+      "rewards/margins": 2.936446189880371,
+      "rewards/rejected": -3.6600422859191895,
+      "step": 850
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.617012688577036e-05,
+      "logits/chosen": -1.4270175695419312,
+      "logits/rejected": -1.3395029306411743,
+      "logps/chosen": -234.1733856201172,
+      "logps/rejected": -280.4658203125,
+      "loss": 0.3788,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4625687599182129,
+      "rewards/margins": 2.767164707183838,
+      "rewards/rejected": -3.2297332286834717,
+      "step": 855
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.609906939652846e-05,
+      "logits/chosen": -1.4226279258728027,
+      "logits/rejected": -1.3623218536376953,
+      "logps/chosen": -195.22999572753906,
+      "logps/rejected": -245.10183715820312,
+      "loss": 0.3799,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.3807176947593689,
+      "rewards/margins": 2.3278114795684814,
+      "rewards/rejected": -2.708528995513916,
+      "step": 860
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.60274145016394e-05,
+      "logits/chosen": -1.4433257579803467,
+      "logits/rejected": -1.3707187175750732,
+      "logps/chosen": -241.7848358154297,
+      "logps/rejected": -267.00213623046875,
+      "loss": 0.3711,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.17411458492279053,
+      "rewards/margins": 2.5103354454040527,
+      "rewards/rejected": -2.6844499111175537,
+      "step": 865
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.595516422996227e-05,
+      "logits/chosen": -1.4536000490188599,
+      "logits/rejected": -1.3923813104629517,
+      "logps/chosen": -204.7315216064453,
+      "logps/rejected": -269.3832092285156,
+      "loss": 0.3762,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.10245015472173691,
+      "rewards/margins": 3.06905198097229,
+      "rewards/rejected": -2.966601848602295,
+      "step": 870
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.588232062721385e-05,
+      "logits/chosen": -1.506850004196167,
+      "logits/rejected": -1.417551875114441,
+      "logps/chosen": -226.9280242919922,
+      "logps/rejected": -282.6661071777344,
+      "loss": 0.3891,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.23108553886413574,
+      "rewards/margins": 2.9151499271392822,
+      "rewards/rejected": -3.1462349891662598,
+      "step": 875
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.580888575591068e-05,
+      "logits/chosen": -1.432558298110962,
+      "logits/rejected": -1.3910208940505981,
+      "logps/chosen": -224.8462677001953,
+      "logps/rejected": -275.9529724121094,
+      "loss": 0.4098,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.4546372890472412,
+      "rewards/margins": 2.7860777378082275,
+      "rewards/rejected": -3.2407150268554688,
+      "step": 880
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.573486169531068e-05,
+      "logits/chosen": -1.3392664194107056,
+      "logits/rejected": -1.2887176275253296,
+      "logps/chosen": -228.43896484375,
+      "logps/rejected": -280.51556396484375,
+      "loss": 0.3702,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.29709941148757935,
+      "rewards/margins": 3.137378692626953,
+      "rewards/rejected": -3.434478282928467,
+      "step": 885
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5660250541354224e-05,
+      "logits/chosen": -1.484899878501892,
+      "logits/rejected": -1.381151556968689,
+      "logps/chosen": -244.162841796875,
+      "logps/rejected": -285.1504821777344,
+      "loss": 0.3823,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2389202117919922,
+      "rewards/margins": 3.0065085887908936,
+      "rewards/rejected": -3.245429277420044,
+      "step": 890
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5585054406604864e-05,
+      "logits/chosen": -1.5870790481567383,
+      "logits/rejected": -1.535390019416809,
+      "logps/chosen": -222.4707489013672,
+      "logps/rejected": -274.00555419921875,
+      "loss": 0.4039,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.4108337461948395,
+      "rewards/margins": 2.4919819831848145,
+      "rewards/rejected": -2.902815580368042,
+      "step": 895
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.550927542018947e-05,
+      "logits/chosen": -1.3818638324737549,
+      "logits/rejected": -1.3315644264221191,
+      "logps/chosen": -225.2039337158203,
+      "logps/rejected": -255.9619140625,
+      "loss": 0.4276,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.7841989398002625,
+      "rewards/margins": 2.3433711528778076,
+      "rewards/rejected": -3.127570152282715,
+      "step": 900
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.5432915727737936e-05,
+      "logits/chosen": -1.424290418624878,
+      "logits/rejected": -1.3178008794784546,
+      "logps/chosen": -248.19631958007812,
+      "logps/rejected": -290.7337341308594,
+      "loss": 0.4093,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.01188063621521,
+      "rewards/margins": 2.751819610595703,
+      "rewards/rejected": -3.763700008392334,
+      "step": 905
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.5355977491322485e-05,
+      "logits/chosen": -1.4348728656768799,
+      "logits/rejected": -1.380027413368225,
+      "logps/chosen": -258.47039794921875,
+      "logps/rejected": -330.335693359375,
+      "loss": 0.3708,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1714378595352173,
+      "rewards/margins": 3.1479644775390625,
+      "rewards/rejected": -4.31940221786499,
+      "step": 910
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.527846288939639e-05,
+      "logits/chosen": -1.5116336345672607,
+      "logits/rejected": -1.37888503074646,
+      "logps/chosen": -240.5450439453125,
+      "logps/rejected": -272.766357421875,
+      "loss": 0.4085,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.6231032013893127,
+      "rewards/margins": 2.247542381286621,
+      "rewards/rejected": -2.870645761489868,
+      "step": 915
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.5200374116732325e-05,
+      "logits/chosen": -1.4633252620697021,
+      "logits/rejected": -1.3567806482315063,
+      "logps/chosen": -251.53158569335938,
+      "logps/rejected": -293.6622009277344,
+      "loss": 0.3777,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4417695105075836,
+      "rewards/margins": 2.9817652702331543,
+      "rewards/rejected": -3.423534870147705,
+      "step": 920
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.5121713384360215e-05,
+      "logits/chosen": -1.4623820781707764,
+      "logits/rejected": -1.3391244411468506,
+      "logps/chosen": -227.37808227539062,
+      "logps/rejected": -264.84442138671875,
+      "loss": 0.3827,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.28840094804763794,
+      "rewards/margins": 2.5161187648773193,
+      "rewards/rejected": -2.8045194149017334,
+      "step": 925
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.504248291950462e-05,
+      "logits/chosen": -1.540131688117981,
+      "logits/rejected": -1.4444448947906494,
+      "logps/chosen": -199.7274932861328,
+      "logps/rejected": -243.74893188476562,
+      "loss": 0.3956,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.33068814873695374,
+      "rewards/margins": 2.3045554161071777,
+      "rewards/rejected": -2.6352434158325195,
+      "step": 930
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.4962684965521695e-05,
+      "logits/chosen": -1.4449470043182373,
+      "logits/rejected": -1.335399866104126,
+      "logps/chosen": -231.919677734375,
+      "logps/rejected": -284.73236083984375,
+      "loss": 0.3636,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.4564400315284729,
+      "rewards/margins": 3.0215699672698975,
+      "rewards/rejected": -3.4780101776123047,
+      "step": 935
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.488232178183567e-05,
+      "logits/chosen": -1.284208059310913,
+      "logits/rejected": -1.2242339849472046,
+      "logps/chosen": -247.7313232421875,
+      "logps/rejected": -297.8804931640625,
+      "loss": 0.3918,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.9177427291870117,
+      "rewards/margins": 3.033815383911133,
+      "rewards/rejected": -3.9515578746795654,
+      "step": 940
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.480139564387482e-05,
+      "logits/chosen": -1.3877151012420654,
+      "logits/rejected": -1.3023748397827148,
+      "logps/chosen": -224.7671356201172,
+      "logps/rejected": -260.46551513671875,
+      "loss": 0.3879,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.9650894403457642,
+      "rewards/margins": 2.5788512229919434,
+      "rewards/rejected": -3.543941020965576,
+      "step": 945
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.471990884300715e-05,
+      "logits/chosen": -1.4054934978485107,
+      "logits/rejected": -1.3035714626312256,
+      "logps/chosen": -245.2154541015625,
+      "logps/rejected": -302.49969482421875,
+      "loss": 0.3624,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.588658332824707,
+      "rewards/margins": 3.1204638481140137,
+      "rewards/rejected": -3.7091221809387207,
+      "step": 950
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.46378636864754e-05,
+      "logits/chosen": -1.3097150325775146,
+      "logits/rejected": -1.226994276046753,
+      "logps/chosen": -236.5945281982422,
+      "logps/rejected": -296.509765625,
+      "loss": 0.4001,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7584825158119202,
+      "rewards/margins": 3.3817715644836426,
+      "rewards/rejected": -4.140254020690918,
+      "step": 955
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.455526249733178e-05,
+      "logits/chosen": -1.4240782260894775,
+      "logits/rejected": -1.3734047412872314,
+      "logps/chosen": -230.6123046875,
+      "logps/rejected": -291.6025390625,
+      "loss": 0.4944,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2551751732826233,
+      "rewards/margins": 2.7103066444396973,
+      "rewards/rejected": -2.965481996536255,
+      "step": 960
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.447210761437219e-05,
+      "logits/chosen": -1.5501660108566284,
+      "logits/rejected": -1.4900107383728027,
+      "logps/chosen": -229.24978637695312,
+      "logps/rejected": -278.9122314453125,
+      "loss": 0.4433,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.038763850927352905,
+      "rewards/margins": 2.4731106758117676,
+      "rewards/rejected": -2.5118744373321533,
+      "step": 965
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.4388401392069975e-05,
+      "logits/chosen": -1.5285327434539795,
+      "logits/rejected": -1.370157241821289,
+      "logps/chosen": -245.3329315185547,
+      "logps/rejected": -271.7296447753906,
+      "loss": 0.3852,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.16751372814178467,
+      "rewards/margins": 2.6887688636779785,
+      "rewards/rejected": -2.5212550163269043,
+      "step": 970
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.430414620050929e-05,
+      "logits/chosen": -1.5238043069839478,
+      "logits/rejected": -1.4335300922393799,
+      "logps/chosen": -219.3772430419922,
+      "logps/rejected": -284.52667236328125,
+      "loss": 0.3898,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.2622632384300232,
+      "rewards/margins": 2.474923610687256,
+      "rewards/rejected": -2.212660551071167,
+      "step": 975
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.421934442531796e-05,
+      "logits/chosen": -1.4353379011154175,
+      "logits/rejected": -1.4253833293914795,
+      "logps/chosen": -226.8037567138672,
+      "logps/rejected": -284.70269775390625,
+      "loss": 0.3631,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.3952658772468567,
+      "rewards/margins": 2.801473617553711,
+      "rewards/rejected": -3.196739673614502,
+      "step": 980
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.413399846759998e-05,
+      "logits/chosen": -1.4747555255889893,
+      "logits/rejected": -1.3409960269927979,
+      "logps/chosen": -261.8158264160156,
+      "logps/rejected": -304.100830078125,
+      "loss": 0.4125,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.8937687873840332,
+      "rewards/margins": 3.569911241531372,
+      "rewards/rejected": -4.463679790496826,
+      "step": 985
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.4048110743867455e-05,
+      "logits/chosen": -1.4302071332931519,
+      "logits/rejected": -1.298626184463501,
+      "logps/chosen": -241.70767211914062,
+      "logps/rejected": -302.16937255859375,
+      "loss": 0.3641,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.8468769788742065,
+      "rewards/margins": 2.884181499481201,
+      "rewards/rejected": -3.7310585975646973,
+      "step": 990
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.396168368597226e-05,
+      "logits/chosen": -1.410563588142395,
+      "logits/rejected": -1.296134352684021,
+      "logps/chosen": -247.36376953125,
+      "logps/rejected": -289.80047607421875,
+      "loss": 0.3609,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.35360056161880493,
+      "rewards/margins": 2.7350914478302,
+      "rewards/rejected": -3.0886917114257812,
+      "step": 995
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.387471974103713e-05,
+      "logits/chosen": -1.5295279026031494,
+      "logits/rejected": -1.3939071893692017,
+      "logps/chosen": -225.24356079101562,
+      "logps/rejected": -259.98565673828125,
+      "loss": 0.4028,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.06099366396665573,
+      "rewards/margins": 2.6358046531677246,
+      "rewards/rejected": -2.696798086166382,
+      "step": 1000
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3787221371386384e-05,
+      "logits/chosen": -1.5613595247268677,
+      "logits/rejected": -1.516898274421692,
+      "logps/chosen": -248.61074829101562,
+      "logps/rejected": -296.6068115234375,
+      "loss": 0.4068,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.038737304508686066,
+      "rewards/margins": 2.5149483680725098,
+      "rewards/rejected": -2.476210832595825,
+      "step": 1005
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.369919105447622e-05,
+      "logits/chosen": -1.5018450021743774,
+      "logits/rejected": -1.4464499950408936,
+      "logps/chosen": -215.8905029296875,
+      "logps/rejected": -250.96121215820312,
+      "loss": 0.4027,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.006156214978545904,
+      "rewards/margins": 2.1807312965393066,
+      "rewards/rejected": -2.174575090408325,
+      "step": 1010
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3610631282824556e-05,
+      "logits/chosen": -1.487079381942749,
+      "logits/rejected": -1.3799813985824585,
+      "logps/chosen": -220.0702362060547,
+      "logps/rejected": -256.46270751953125,
+      "loss": 0.4356,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.10034330934286118,
+      "rewards/margins": 2.6790289878845215,
+      "rewards/rejected": -2.779372453689575,
+      "step": 1015
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.352154456394045e-05,
+      "logits/chosen": -1.4811842441558838,
+      "logits/rejected": -1.3606802225112915,
+      "logps/chosen": -235.79672241210938,
+      "logps/rejected": -265.3165588378906,
+      "loss": 0.3825,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.42521244287490845,
+      "rewards/margins": 2.6016688346862793,
+      "rewards/rejected": -3.026881456375122,
+      "step": 1020
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.34319334202531e-05,
+      "logits/chosen": -1.490431547164917,
+      "logits/rejected": -1.380516767501831,
+      "logps/chosen": -245.6602325439453,
+      "logps/rejected": -304.4634094238281,
+      "loss": 0.3552,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.8394023776054382,
+      "rewards/margins": 2.91302490234375,
+      "rewards/rejected": -3.752427339553833,
+      "step": 1025
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.334180038904046e-05,
+      "logits/chosen": -1.3724013566970825,
+      "logits/rejected": -1.2893320322036743,
+      "logps/chosen": -224.7459259033203,
+      "logps/rejected": -275.4010314941406,
+      "loss": 0.3907,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.885975182056427,
+      "rewards/margins": 2.789304733276367,
+      "rewards/rejected": -3.6752796173095703,
+      "step": 1030
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.3251148022357355e-05,
+      "logits/chosen": -1.4460914134979248,
+      "logits/rejected": -1.3820542097091675,
+      "logps/chosen": -262.4001159667969,
+      "logps/rejected": -319.7551574707031,
+      "loss": 0.4385,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1028560400009155,
+      "rewards/margins": 2.362874746322632,
+      "rewards/rejected": -3.465731143951416,
+      "step": 1035
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.3159978886963226e-05,
+      "logits/chosen": -1.4794714450836182,
+      "logits/rejected": -1.3599398136138916,
+      "logps/chosen": -277.8358459472656,
+      "logps/rejected": -306.3834533691406,
+      "loss": 0.4237,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.0677882432937622,
+      "rewards/margins": 3.120405673980713,
+      "rewards/rejected": -4.188194274902344,
+      "step": 1040
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.306829556424948e-05,
+      "logits/chosen": -1.4093915224075317,
+      "logits/rejected": -1.271278977394104,
+      "logps/chosen": -287.8832702636719,
+      "logps/rejected": -332.91485595703125,
+      "loss": 0.3677,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.1396197080612183,
+      "rewards/margins": 3.1220901012420654,
+      "rewards/rejected": -4.261710166931152,
+      "step": 1045
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.2976100650166387e-05,
+      "logits/chosen": -1.4229646921157837,
+      "logits/rejected": -1.39849853515625,
+      "logps/chosen": -228.4208984375,
+      "logps/rejected": -291.3751525878906,
+      "loss": 0.3707,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9911971092224121,
+      "rewards/margins": 2.7032597064971924,
+      "rewards/rejected": -3.6944565773010254,
+      "step": 1050
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.288339675514954e-05,
+      "logits/chosen": -1.3454296588897705,
+      "logits/rejected": -1.302362322807312,
+      "logps/chosen": -255.00906372070312,
+      "logps/rejected": -313.9950256347656,
+      "loss": 0.4064,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.0980554819107056,
+      "rewards/margins": 3.0446298122406006,
+      "rewards/rejected": -4.142685890197754,
+      "step": 1055
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.279018650404604e-05,
+      "logits/chosen": -1.3721091747283936,
+      "logits/rejected": -1.36617112159729,
+      "logps/chosen": -248.34432983398438,
+      "logps/rejected": -322.83502197265625,
+      "loss": 0.3819,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.2138582468032837,
+      "rewards/margins": 2.7490146160125732,
+      "rewards/rejected": -3.9628729820251465,
+      "step": 1060
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.2696472536040054e-05,
+      "logits/chosen": -1.3144346475601196,
+      "logits/rejected": -1.199225664138794,
+      "logps/chosen": -266.83746337890625,
+      "logps/rejected": -312.2145080566406,
+      "loss": 0.3602,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.659379005432129,
+      "rewards/margins": 3.416776180267334,
+      "rewards/rejected": -5.076155185699463,
+      "step": 1065
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.260225750457818e-05,
+      "logits/chosen": -1.4334145784378052,
+      "logits/rejected": -1.3207252025604248,
+      "logps/chosen": -258.37725830078125,
+      "logps/rejected": -299.75164794921875,
+      "loss": 0.34,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7483505010604858,
+      "rewards/margins": 2.753929615020752,
+      "rewards/rejected": -4.502279758453369,
+      "step": 1070
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.250754407729428e-05,
+      "logits/chosen": -1.327194333076477,
+      "logits/rejected": -1.2377169132232666,
+      "logps/chosen": -279.93280029296875,
+      "logps/rejected": -334.41094970703125,
+      "loss": 0.3752,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0792298316955566,
+      "rewards/margins": 3.3054840564727783,
+      "rewards/rejected": -5.384713649749756,
+      "step": 1075
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.241233493593393e-05,
+      "logits/chosen": -1.2953674793243408,
+      "logits/rejected": -1.263270378112793,
+      "logps/chosen": -241.9004364013672,
+      "logps/rejected": -313.4391784667969,
+      "loss": 0.4028,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5718278884887695,
+      "rewards/margins": 3.0327353477478027,
+      "rewards/rejected": -4.6045637130737305,
+      "step": 1080
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2316632776278525e-05,
+      "logits/chosen": -1.3943222761154175,
+      "logits/rejected": -1.2810288667678833,
+      "logps/chosen": -232.25814819335938,
+      "logps/rejected": -280.0733947753906,
+      "loss": 0.3978,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.7831779718399048,
+      "rewards/margins": 2.5116419792175293,
+      "rewards/rejected": -3.2948200702667236,
+      "step": 1085
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.222044030806894e-05,
+      "logits/chosen": -1.3264403343200684,
+      "logits/rejected": -1.3240474462509155,
+      "logps/chosen": -215.8976287841797,
+      "logps/rejected": -287.00128173828125,
+      "loss": 0.3795,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.7215701341629028,
+      "rewards/margins": 2.7273683547973633,
+      "rewards/rejected": -3.4489383697509766,
+      "step": 1090
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.21237602549288e-05,
+      "logits/chosen": -1.434257984161377,
+      "logits/rejected": -1.4035327434539795,
+      "logps/chosen": -201.57369995117188,
+      "logps/rejected": -253.14108276367188,
+      "loss": 0.3946,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.4502865672111511,
+      "rewards/margins": 2.529646396636963,
+      "rewards/rejected": -2.979933023452759,
+      "step": 1095
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.2026595354287334e-05,
+      "logits/chosen": -1.3879592418670654,
+      "logits/rejected": -1.3350013494491577,
+      "logps/chosen": -241.10311889648438,
+      "logps/rejected": -302.80865478515625,
+      "loss": 0.3825,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.7047330737113953,
+      "rewards/margins": 2.6905667781829834,
+      "rewards/rejected": -3.3953003883361816,
+      "step": 1100
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.192894835730193e-05,
+      "logits/chosen": -1.3509743213653564,
+      "logits/rejected": -1.248357892036438,
+      "logps/chosen": -253.964111328125,
+      "logps/rejected": -293.91632080078125,
+      "loss": 0.3919,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8855097889900208,
+      "rewards/margins": 2.823207139968872,
+      "rewards/rejected": -3.7087173461914062,
+      "step": 1105
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.1830822028780194e-05,
+      "logits/chosen": -1.447584867477417,
+      "logits/rejected": -1.3725135326385498,
+      "logps/chosen": -240.1465606689453,
+      "logps/rejected": -290.83099365234375,
+      "loss": 0.4142,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9225603342056274,
+      "rewards/margins": 2.983272075653076,
+      "rewards/rejected": -3.905831813812256,
+      "step": 1110
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.173221914710165e-05,
+      "logits/chosen": -1.404601812362671,
+      "logits/rejected": -1.2846992015838623,
+      "logps/chosen": -221.6542510986328,
+      "logps/rejected": -249.8398895263672,
+      "loss": 0.4451,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.8987483978271484,
+      "rewards/margins": 2.412379503250122,
+      "rewards/rejected": -3.3111279010772705,
+      "step": 1115
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.163314250413913e-05,
+      "logits/chosen": -1.4802556037902832,
+      "logits/rejected": -1.3953096866607666,
+      "logps/chosen": -230.58865356445312,
+      "logps/rejected": -278.18951416015625,
+      "loss": 0.384,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.9633780717849731,
+      "rewards/margins": 2.581531047821045,
+      "rewards/rejected": -3.5449092388153076,
+      "step": 1120
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.153359490517969e-05,
+      "logits/chosen": -1.4657261371612549,
+      "logits/rejected": -1.377966284751892,
+      "logps/chosen": -228.0840606689453,
+      "logps/rejected": -270.4330749511719,
+      "loss": 0.378,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.8458096385002136,
+      "rewards/margins": 2.5918195247650146,
+      "rewards/rejected": -3.437628984451294,
+      "step": 1125
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.143357916884514e-05,
+      "logits/chosen": -1.4898326396942139,
+      "logits/rejected": -1.3859083652496338,
+      "logps/chosen": -256.3155212402344,
+      "logps/rejected": -299.3856506347656,
+      "loss": 0.4363,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.0702875852584839,
+      "rewards/margins": 2.819854497909546,
+      "rewards/rejected": -3.8901419639587402,
+      "step": 1130
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.1333098127012326e-05,
+      "logits/chosen": -1.491857886314392,
+      "logits/rejected": -1.4556844234466553,
+      "logps/chosen": -254.2758331298828,
+      "logps/rejected": -287.3238220214844,
+      "loss": 0.4276,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.9167013168334961,
+      "rewards/margins": 2.7353272438049316,
+      "rewards/rejected": -3.6520285606384277,
+      "step": 1135
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.123215462473287e-05,
+      "logits/chosen": -1.4471662044525146,
+      "logits/rejected": -1.3652303218841553,
+      "logps/chosen": -257.15521240234375,
+      "logps/rejected": -321.16632080078125,
+      "loss": 0.3993,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.5140289068222046,
+      "rewards/margins": 3.106577157974243,
+      "rewards/rejected": -3.620605945587158,
+      "step": 1140
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.113075152015267e-05,
+      "logits/chosen": -1.4940803050994873,
+      "logits/rejected": -1.4113094806671143,
+      "logps/chosen": -231.76272583007812,
+      "logps/rejected": -276.2952575683594,
+      "loss": 0.3857,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.4637375473976135,
+      "rewards/margins": 2.7834160327911377,
+      "rewards/rejected": -3.2471535205841064,
+      "step": 1145
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.102889168443091e-05,
+      "logits/chosen": -1.4232820272445679,
+      "logits/rejected": -1.3385612964630127,
+      "logps/chosen": -220.4342041015625,
+      "logps/rejected": -273.684326171875,
+      "loss": 0.3638,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5296335220336914,
+      "rewards/margins": 2.962373733520508,
+      "rewards/rejected": -3.49200701713562,
+      "step": 1150
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.092657800165883e-05,
+      "logits/chosen": -1.3860952854156494,
+      "logits/rejected": -1.2572487592697144,
+      "logps/chosen": -238.58279418945312,
+      "logps/rejected": -299.2005310058594,
+      "loss": 0.3959,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.492671400308609,
+      "rewards/margins": 3.1469099521636963,
+      "rewards/rejected": -3.6395816802978516,
+      "step": 1155
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.082381336877805e-05,
+      "logits/chosen": -1.4538966417312622,
+      "logits/rejected": -1.3704473972320557,
+      "logps/chosen": -232.91757202148438,
+      "logps/rejected": -289.62396240234375,
+      "loss": 0.4583,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.3003008961677551,
+      "rewards/margins": 3.076662540435791,
+      "rewards/rejected": -3.3769633769989014,
+      "step": 1160
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.0720600695498486e-05,
+      "logits/chosen": -1.5047948360443115,
+      "logits/rejected": -1.3999977111816406,
+      "logps/chosen": -216.4929962158203,
+      "logps/rejected": -261.029296875,
+      "loss": 0.3941,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.28260135650634766,
+      "rewards/margins": 2.698638677597046,
+      "rewards/rejected": -2.981240749359131,
+      "step": 1165
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.061694290421604e-05,
+      "logits/chosen": -1.5519943237304688,
+      "logits/rejected": -1.4196147918701172,
+      "logps/chosen": -237.73605346679688,
+      "logps/rejected": -281.0528259277344,
+      "loss": 0.3755,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.3319626450538635,
+      "rewards/margins": 2.4680209159851074,
+      "rewards/rejected": -2.799983501434326,
+      "step": 1170
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.051284292992984e-05,
+      "logits/chosen": -1.3771086931228638,
+      "logits/rejected": -1.3074411153793335,
+      "logps/chosen": -251.0067596435547,
+      "logps/rejected": -294.7237243652344,
+      "loss": 0.3995,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.2037571370601654,
+      "rewards/margins": 3.2541725635528564,
+      "rewards/rejected": -3.4579296112060547,
+      "step": 1175
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.040830372015909e-05,
+      "logits/chosen": -1.475381851196289,
+      "logits/rejected": -1.3613998889923096,
+      "logps/chosen": -246.23281860351562,
+      "logps/rejected": -300.03497314453125,
+      "loss": 0.3781,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.10555452108383179,
+      "rewards/margins": 3.167620897293091,
+      "rewards/rejected": -3.273175001144409,
+      "step": 1180
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.0303328234859665e-05,
+      "logits/chosen": -1.4284617900848389,
+      "logits/rejected": -1.2940706014633179,
+      "logps/chosen": -268.23077392578125,
+      "logps/rejected": -295.5655517578125,
+      "loss": 0.3795,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.13254739344120026,
+      "rewards/margins": 3.2897098064422607,
+      "rewards/rejected": -3.4222571849823,
+      "step": 1185
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.019791944634027e-05,
+      "logits/chosen": -1.4546287059783936,
+      "logits/rejected": -1.4235341548919678,
+      "logps/chosen": -225.8767547607422,
+      "logps/rejected": -300.689697265625,
+      "loss": 0.3781,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.15003997087478638,
+      "rewards/margins": 2.844740390777588,
+      "rewards/rejected": -2.9947803020477295,
+      "step": 1190
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.00920803391783e-05,
+      "logits/chosen": -1.4976985454559326,
+      "logits/rejected": -1.436232089996338,
+      "logps/chosen": -216.7958526611328,
+      "logps/rejected": -249.0604705810547,
+      "loss": 0.38,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.038081564009189606,
+      "rewards/margins": 2.4354381561279297,
+      "rewards/rejected": -2.473519802093506,
+      "step": 1195
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9985813910135304e-05,
+      "logits/chosen": -1.488646149635315,
+      "logits/rejected": -1.4349124431610107,
+      "logps/chosen": -235.86972045898438,
+      "logps/rejected": -304.09039306640625,
+      "loss": 0.3782,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.26524874567985535,
+      "rewards/margins": 3.0398926734924316,
+      "rewards/rejected": -3.305140972137451,
+      "step": 1200
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9879123168072206e-05,
+      "logits/chosen": -1.4791837930679321,
+      "logits/rejected": -1.4347190856933594,
+      "logps/chosen": -243.2008819580078,
+      "logps/rejected": -321.9309997558594,
+      "loss": 0.4218,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.11531716585159302,
+      "rewards/margins": 2.624474048614502,
+      "rewards/rejected": -2.73979115486145,
+      "step": 1205
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.977201113386402e-05,
+      "logits/chosen": -1.5107253789901733,
+      "logits/rejected": -1.3714876174926758,
+      "logps/chosen": -253.50497436523438,
+      "logps/rejected": -301.078125,
+      "loss": 0.3595,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.07889306545257568,
+      "rewards/margins": 2.69875431060791,
+      "rewards/rejected": -2.7776474952697754,
+      "step": 1210
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.966448084031437e-05,
+      "logits/chosen": -1.3860998153686523,
+      "logits/rejected": -1.3501628637313843,
+      "logps/chosen": -210.1219940185547,
+      "logps/rejected": -260.87872314453125,
+      "loss": 0.3633,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.19702157378196716,
+      "rewards/margins": 2.5177106857299805,
+      "rewards/rejected": -2.7147319316864014,
+      "step": 1215
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.955653533206959e-05,
+      "logits/chosen": -1.4096229076385498,
+      "logits/rejected": -1.2972562313079834,
+      "logps/chosen": -219.9523162841797,
+      "logps/rejected": -269.67333984375,
+      "loss": 0.3453,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.26071828603744507,
+      "rewards/margins": 3.1086888313293457,
+      "rewards/rejected": -3.3694069385528564,
+      "step": 1220
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9448177665532574e-05,
+      "logits/chosen": -1.469242811203003,
+      "logits/rejected": -1.3070530891418457,
+      "logps/chosen": -230.3858184814453,
+      "logps/rejected": -264.9944763183594,
+      "loss": 0.3727,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.5432360172271729,
+      "rewards/margins": 2.8007569313049316,
+      "rewards/rejected": -3.3439929485321045,
+      "step": 1225
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.933941090877615e-05,
+      "logits/chosen": -1.4690440893173218,
+      "logits/rejected": -1.3958766460418701,
+      "logps/chosen": -232.1632843017578,
+      "logps/rejected": -284.0280456542969,
+      "loss": 0.382,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.4447177052497864,
+      "rewards/margins": 2.9213266372680664,
+      "rewards/rejected": -3.3660449981689453,
+      "step": 1230
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.923023814145629e-05,
+      "logits/chosen": -1.5301823616027832,
+      "logits/rejected": -1.376138687133789,
+      "logps/chosen": -245.1982879638672,
+      "logps/rejected": -270.8854675292969,
+      "loss": 0.4028,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.6281201243400574,
+      "rewards/margins": 2.786634922027588,
+      "rewards/rejected": -3.414755344390869,
+      "step": 1235
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.9120662454724836e-05,
+      "logits/chosen": -1.4721466302871704,
+      "logits/rejected": -1.4076852798461914,
+      "logps/chosen": -237.6669158935547,
+      "logps/rejected": -292.71478271484375,
+      "loss": 0.3709,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.0528624057769775,
+      "rewards/margins": 3.1795761585235596,
+      "rewards/rejected": -4.232438087463379,
+      "step": 1240
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.901068695114206e-05,
+      "logits/chosen": -1.4836117029190063,
+      "logits/rejected": -1.366645097732544,
+      "logps/chosen": -242.7480010986328,
+      "logps/rejected": -301.66607666015625,
+      "loss": 0.4359,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.254749059677124,
+      "rewards/margins": 3.5557892322540283,
+      "rewards/rejected": -4.810537815093994,
+      "step": 1245
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.890031474458874e-05,
+      "logits/chosen": -1.4514219760894775,
+      "logits/rejected": -1.3249971866607666,
+      "logps/chosen": -275.0703430175781,
+      "logps/rejected": -320.9544372558594,
+      "loss": 0.4112,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.782173752784729,
+      "rewards/margins": 3.3930366039276123,
+      "rewards/rejected": -5.175210475921631,
+      "step": 1250
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.878954896017804e-05,
+      "logits/chosen": -1.3831149339675903,
+      "logits/rejected": -1.261541724205017,
+      "logps/chosen": -270.8629455566406,
+      "logps/rejected": -328.1438903808594,
+      "loss": 0.3622,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6734645366668701,
+      "rewards/margins": 3.397109270095825,
+      "rewards/rejected": -5.070573806762695,
+      "step": 1255
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.867839273416701e-05,
+      "logits/chosen": -1.375957727432251,
+      "logits/rejected": -1.244816541671753,
+      "logps/chosen": -237.8169403076172,
+      "logps/rejected": -267.6765441894531,
+      "loss": 0.3726,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1279919147491455,
+      "rewards/margins": 3.1958394050598145,
+      "rewards/rejected": -4.323831081390381,
+      "step": 1260
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8566849213867795e-05,
+      "logits/chosen": -1.3469189405441284,
+      "logits/rejected": -1.3233760595321655,
+      "logps/chosen": -215.3714141845703,
+      "logps/rejected": -297.8151550292969,
+      "loss": 0.3699,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6543620824813843,
+      "rewards/margins": 2.7842297554016113,
+      "rewards/rejected": -4.438591957092285,
+      "step": 1265
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8454921557558476e-05,
+      "logits/chosen": -1.4500279426574707,
+      "logits/rejected": -1.343481183052063,
+      "logps/chosen": -246.06689453125,
+      "logps/rejected": -302.48260498046875,
+      "loss": 0.3783,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4052373170852661,
+      "rewards/margins": 3.2777016162872314,
+      "rewards/rejected": -4.6829400062561035,
+      "step": 1270
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.834261293439374e-05,
+      "logits/chosen": -1.2790629863739014,
+      "logits/rejected": -1.1767776012420654,
+      "logps/chosen": -233.415771484375,
+      "logps/rejected": -286.8774719238281,
+      "loss": 0.4056,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6317148208618164,
+      "rewards/margins": 2.9100582599639893,
+      "rewards/rejected": -4.541773319244385,
+      "step": 1275
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8229926524315016e-05,
+      "logits/chosen": -1.4536702632904053,
+      "logits/rejected": -1.3482431173324585,
+      "logps/chosen": -238.40676879882812,
+      "logps/rejected": -278.3370666503906,
+      "loss": 0.3956,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.226927638053894,
+      "rewards/margins": 3.155203342437744,
+      "rewards/rejected": -4.3821306228637695,
+      "step": 1280
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8116865517960585e-05,
+      "logits/chosen": -1.4348114728927612,
+      "logits/rejected": -1.318174123764038,
+      "logps/chosen": -228.39523315429688,
+      "logps/rejected": -275.8654479980469,
+      "loss": 0.4036,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2049471139907837,
+      "rewards/margins": 3.1574530601501465,
+      "rewards/rejected": -4.362399578094482,
+      "step": 1285
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.800343311657509e-05,
+      "logits/chosen": -1.4712860584259033,
+      "logits/rejected": -1.3852875232696533,
+      "logps/chosen": -236.8831024169922,
+      "logps/rejected": -282.43572998046875,
+      "loss": 0.4184,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1031112670898438,
+      "rewards/margins": 3.026742458343506,
+      "rewards/rejected": -4.12985372543335,
+      "step": 1290
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.788963253191905e-05,
+      "logits/chosen": -1.4327385425567627,
+      "logits/rejected": -1.3884919881820679,
+      "logps/chosen": -256.63848876953125,
+      "logps/rejected": -309.84759521484375,
+      "loss": 0.4386,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.7931209206581116,
+      "rewards/margins": 2.4091200828552246,
+      "rewards/rejected": -3.2022411823272705,
+      "step": 1295
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.777546698617776e-05,
+      "logits/chosen": -1.4254684448242188,
+      "logits/rejected": -1.3687108755111694,
+      "logps/chosen": -224.0829315185547,
+      "logps/rejected": -275.72247314453125,
+      "loss": 0.3798,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.6869279146194458,
+      "rewards/margins": 2.486288547515869,
+      "rewards/rejected": -3.1732163429260254,
+      "step": 1300
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.766093971187019e-05,
+      "logits/chosen": -1.452755331993103,
+      "logits/rejected": -1.3947112560272217,
+      "logps/chosen": -239.74575805664062,
+      "logps/rejected": -277.12017822265625,
+      "loss": 0.3977,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.5879044532775879,
+      "rewards/margins": 2.501594066619873,
+      "rewards/rejected": -3.089498519897461,
+      "step": 1305
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.75460539517574e-05,
+      "logits/chosen": -1.4273698329925537,
+      "logits/rejected": -1.3977384567260742,
+      "logps/chosen": -231.1275634765625,
+      "logps/rejected": -284.3058776855469,
+      "loss": 0.3925,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.20015409588813782,
+      "rewards/margins": 2.5172126293182373,
+      "rewards/rejected": -2.7173666954040527,
+      "step": 1310
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.743081295875069e-05,
+      "logits/chosen": -1.5619311332702637,
+      "logits/rejected": -1.4386647939682007,
+      "logps/chosen": -235.97891235351562,
+      "logps/rejected": -289.92694091796875,
+      "loss": 0.3512,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.166568323969841,
+      "rewards/margins": 2.67271089553833,
+      "rewards/rejected": -2.8392791748046875,
+      "step": 1315
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7315219995819594e-05,
+      "logits/chosen": -1.5064969062805176,
+      "logits/rejected": -1.3810298442840576,
+      "logps/chosen": -266.1444091796875,
+      "logps/rejected": -311.23199462890625,
+      "loss": 0.3587,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.7214463353157043,
+      "rewards/margins": 3.029715061187744,
+      "rewards/rejected": -3.7511610984802246,
+      "step": 1320
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.719927833589939e-05,
+      "logits/chosen": -1.358946442604065,
+      "logits/rejected": -1.2690280675888062,
+      "logps/chosen": -235.3449249267578,
+      "logps/rejected": -287.78924560546875,
+      "loss": 0.3899,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.8808122873306274,
+      "rewards/margins": 2.6421303749084473,
+      "rewards/rejected": -3.5229427814483643,
+      "step": 1325
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.708299126179847e-05,
+      "logits/chosen": -1.4998195171356201,
+      "logits/rejected": -1.3532516956329346,
+      "logps/chosen": -251.736572265625,
+      "logps/rejected": -285.5415954589844,
+      "loss": 0.3396,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.8132502436637878,
+      "rewards/margins": 3.2861030101776123,
+      "rewards/rejected": -4.099352836608887,
+      "step": 1330
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6966362066105435e-05,
+      "logits/chosen": -1.4474642276763916,
+      "logits/rejected": -1.3161416053771973,
+      "logps/chosen": -244.8996124267578,
+      "logps/rejected": -285.0702209472656,
+      "loss": 0.3514,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.6026067733764648,
+      "rewards/margins": 3.359797954559326,
+      "rewards/rejected": -3.96240496635437,
+      "step": 1335
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.684939405109577e-05,
+      "logits/chosen": -1.4846141338348389,
+      "logits/rejected": -1.3937715291976929,
+      "logps/chosen": -234.49697875976562,
+      "logps/rejected": -277.59503173828125,
+      "loss": 0.3958,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.3563997745513916,
+      "rewards/margins": 2.4923202991485596,
+      "rewards/rejected": -2.848719835281372,
+      "step": 1340
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.673209052863843e-05,
+      "logits/chosen": -1.51144540309906,
+      "logits/rejected": -1.347617268562317,
+      "logps/chosen": -239.48681640625,
+      "logps/rejected": -275.97760009765625,
+      "loss": 0.3446,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.024043012410402298,
+      "rewards/margins": 2.835794687271118,
+      "rewards/rejected": -2.8117516040802,
+      "step": 1345
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6614454820102017e-05,
+      "logits/chosen": -1.47091543674469,
+      "logits/rejected": -1.385925531387329,
+      "logps/chosen": -257.21575927734375,
+      "logps/rejected": -294.6009521484375,
+      "loss": 0.3907,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.17883895337581635,
+      "rewards/margins": 2.582345485687256,
+      "rewards/rejected": -2.7611842155456543,
+      "step": 1350
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6496490256260777e-05,
+      "logits/chosen": -1.5150299072265625,
+      "logits/rejected": -1.4086599349975586,
+      "logps/chosen": -237.181640625,
+      "logps/rejected": -292.00518798828125,
+      "loss": 0.3889,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.24234585464000702,
+      "rewards/margins": 2.8969500064849854,
+      "rewards/rejected": -3.1392955780029297,
+      "step": 1355
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6378200177200224e-05,
+      "logits/chosen": -1.4277657270431519,
+      "logits/rejected": -1.350029706954956,
+      "logps/chosen": -230.1826629638672,
+      "logps/rejected": -300.61090087890625,
+      "loss": 0.3423,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.406125545501709,
+      "rewards/margins": 3.0230190753936768,
+      "rewards/rejected": -3.4291443824768066,
+      "step": 1360
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.625958793222265e-05,
+      "logits/chosen": -1.4115116596221924,
+      "logits/rejected": -1.2951580286026,
+      "logps/chosen": -203.2592315673828,
+      "logps/rejected": -257.6768798828125,
+      "loss": 0.4226,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.45247992873191833,
+      "rewards/margins": 2.8937525749206543,
+      "rewards/rejected": -3.3462326526641846,
+      "step": 1365
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.614065687975225e-05,
+      "logits/chosen": -1.3729918003082275,
+      "logits/rejected": -1.274886131286621,
+      "logps/chosen": -236.5909423828125,
+      "logps/rejected": -300.1371154785156,
+      "loss": 0.3832,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.6886480450630188,
+      "rewards/margins": 3.3041484355926514,
+      "rewards/rejected": -3.9927964210510254,
+      "step": 1370
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.602141038724001e-05,
+      "logits/chosen": -1.445521593093872,
+      "logits/rejected": -1.3185946941375732,
+      "logps/chosen": -251.41104125976562,
+      "logps/rejected": -301.2436218261719,
+      "loss": 0.3619,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.5627816319465637,
+      "rewards/margins": 3.646247386932373,
+      "rewards/rejected": -4.209029197692871,
+      "step": 1375
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.590185183106842e-05,
+      "logits/chosen": -1.4172786474227905,
+      "logits/rejected": -1.3537501096725464,
+      "logps/chosen": -225.7064666748047,
+      "logps/rejected": -291.16998291015625,
+      "loss": 0.3889,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.118973508477211,
+      "rewards/margins": 2.9702212810516357,
+      "rewards/rejected": -3.0891947746276855,
+      "step": 1380
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.578198459645579e-05,
+      "logits/chosen": -1.4852367639541626,
+      "logits/rejected": -1.3799657821655273,
+      "logps/chosen": -253.7847442626953,
+      "logps/rejected": -293.2589111328125,
+      "loss": 0.3236,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0412837415933609,
+      "rewards/margins": 3.353567123413086,
+      "rewards/rejected": -3.394850969314575,
+      "step": 1385
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.56618120773605e-05,
+      "logits/chosen": -1.4363138675689697,
+      "logits/rejected": -1.2875852584838867,
+      "logps/chosen": -235.5326385498047,
+      "logps/rejected": -260.58050537109375,
+      "loss": 0.4108,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.2815939784049988,
+      "rewards/margins": 2.755204677581787,
+      "rewards/rejected": -3.0367987155914307,
+      "step": 1390
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.55413376763848e-05,
+      "logits/chosen": -1.434983491897583,
+      "logits/rejected": -1.3754985332489014,
+      "logps/chosen": -251.4075164794922,
+      "logps/rejected": -303.62640380859375,
+      "loss": 0.3982,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3561423420906067,
+      "rewards/margins": 2.8273160457611084,
+      "rewards/rejected": -3.1834583282470703,
+      "step": 1395
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.542056480467858e-05,
+      "logits/chosen": -1.3716362714767456,
+      "logits/rejected": -1.308511734008789,
+      "logps/chosen": -207.52523803710938,
+      "logps/rejected": -277.71246337890625,
+      "loss": 0.3433,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.49458226561546326,
+      "rewards/margins": 3.2204792499542236,
+      "rewards/rejected": -3.7150611877441406,
+      "step": 1400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.529949688184265e-05,
+      "logits/chosen": -1.357021450996399,
+      "logits/rejected": -1.2644864320755005,
+      "logps/chosen": -252.494384765625,
+      "logps/rejected": -286.4941101074219,
+      "loss": 0.3908,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.7067984342575073,
+      "rewards/margins": 2.705132484436035,
+      "rewards/rejected": -3.411930799484253,
+      "step": 1405
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5178137335832045e-05,
+      "logits/chosen": -1.4006474018096924,
+      "logits/rejected": -1.3148066997528076,
+      "logps/chosen": -220.7507781982422,
+      "logps/rejected": -300.52197265625,
+      "loss": 0.4377,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.8004969358444214,
+      "rewards/margins": 2.8967666625976562,
+      "rewards/rejected": -3.697263240814209,
+      "step": 1410
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.50564896028589e-05,
+      "logits/chosen": -1.4328795671463013,
+      "logits/rejected": -1.2834830284118652,
+      "logps/chosen": -248.32144165039062,
+      "logps/rejected": -283.7814025878906,
+      "loss": 0.3956,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.0089048147201538,
+      "rewards/margins": 3.0403881072998047,
+      "rewards/rejected": -4.049293041229248,
+      "step": 1415
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.493455712729514e-05,
+      "logits/chosen": -1.4717390537261963,
+      "logits/rejected": -1.4287965297698975,
+      "logps/chosen": -233.8995819091797,
+      "logps/rejected": -281.0123596191406,
+      "loss": 0.403,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.6191913485527039,
+      "rewards/margins": 2.783186435699463,
+      "rewards/rejected": -3.4023776054382324,
+      "step": 1420
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.4812343361575e-05,
+      "logits/chosen": -1.4601266384124756,
+      "logits/rejected": -1.4116663932800293,
+      "logps/chosen": -239.9668731689453,
+      "logps/rejected": -297.328125,
+      "loss": 0.3886,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.38425880670547485,
+      "rewards/margins": 2.980128049850464,
+      "rewards/rejected": -3.364386796951294,
+      "step": 1425
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.468985176609726e-05,
+      "logits/chosen": -1.425545334815979,
+      "logits/rejected": -1.3163349628448486,
+      "logps/chosen": -251.3730010986328,
+      "logps/rejected": -292.0272216796875,
+      "loss": 0.321,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.007286679930984974,
+      "rewards/margins": 2.743786573410034,
+      "rewards/rejected": -2.7510733604431152,
+      "step": 1430
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.456708580912725e-05,
+      "logits/chosen": -1.448166012763977,
+      "logits/rejected": -1.3342589139938354,
+      "logps/chosen": -245.2729034423828,
+      "logps/rejected": -287.38189697265625,
+      "loss": 0.361,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.09365560114383698,
+      "rewards/margins": 2.8862528800964355,
+      "rewards/rejected": -2.7925972938537598,
+      "step": 1435
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.444404896669865e-05,
+      "logits/chosen": -1.4818215370178223,
+      "logits/rejected": -1.3631136417388916,
+      "logps/chosen": -257.03533935546875,
+      "logps/rejected": -275.7957763671875,
+      "loss": 0.3723,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.19895866513252258,
+      "rewards/margins": 2.7059268951416016,
+      "rewards/rejected": -2.9048852920532227,
+      "step": 1440
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.432074472251508e-05,
+      "logits/chosen": -1.3858647346496582,
+      "logits/rejected": -1.305906057357788,
+      "logps/chosen": -243.6377410888672,
+      "logps/rejected": -286.6610107421875,
+      "loss": 0.3676,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.13256962597370148,
+      "rewards/margins": 3.50431752204895,
+      "rewards/rejected": -3.6368870735168457,
+      "step": 1445
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.419717656785146e-05,
+      "logits/chosen": -1.3872106075286865,
+      "logits/rejected": -1.2487151622772217,
+      "logps/chosen": -209.8594512939453,
+      "logps/rejected": -236.38931274414062,
+      "loss": 0.3655,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.3276270925998688,
+      "rewards/margins": 2.7453348636627197,
+      "rewards/rejected": -3.0729620456695557,
+      "step": 1450
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4073348001455164e-05,
+      "logits/chosen": -1.4358813762664795,
+      "logits/rejected": -1.3491919040679932,
+      "logps/chosen": -253.41952514648438,
+      "logps/rejected": -297.0237731933594,
+      "loss": 0.3833,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3080201745033264,
+      "rewards/margins": 3.321065902709961,
+      "rewards/rejected": -3.6290860176086426,
+      "step": 1455
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3949262529446915e-05,
+      "logits/chosen": -1.394351601600647,
+      "logits/rejected": -1.3554754257202148,
+      "logps/chosen": -228.14852905273438,
+      "logps/rejected": -290.87591552734375,
+      "loss": 0.3869,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.8205677270889282,
+      "rewards/margins": 2.6521365642547607,
+      "rewards/rejected": -3.4727044105529785,
+      "step": 1460
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.382492366522158e-05,
+      "logits/chosen": -1.4379384517669678,
+      "logits/rejected": -1.292317509651184,
+      "logps/chosen": -235.61386108398438,
+      "logps/rejected": -263.2154541015625,
+      "loss": 0.3953,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.6492370367050171,
+      "rewards/margins": 2.9269096851348877,
+      "rewards/rejected": -3.5761466026306152,
+      "step": 1465
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.370033492934862e-05,
+      "logits/chosen": -1.366807222366333,
+      "logits/rejected": -1.2599581480026245,
+      "logps/chosen": -271.9466857910156,
+      "logps/rejected": -321.1268005371094,
+      "loss": 0.332,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7314456105232239,
+      "rewards/margins": 3.1656384468078613,
+      "rewards/rejected": -3.8970837593078613,
+      "step": 1470
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.357549984947246e-05,
+      "logits/chosen": -1.392762303352356,
+      "logits/rejected": -1.2771762609481812,
+      "logps/chosen": -245.3613739013672,
+      "logps/rejected": -282.22467041015625,
+      "loss": 0.3886,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.8663581013679504,
+      "rewards/margins": 2.6135199069976807,
+      "rewards/rejected": -3.4798779487609863,
+      "step": 1475
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3450421960212566e-05,
+      "logits/chosen": -1.4894797801971436,
+      "logits/rejected": -1.3859410285949707,
+      "logps/chosen": -243.93490600585938,
+      "logps/rejected": -270.7559814453125,
+      "loss": 0.3777,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.4715906083583832,
+      "rewards/margins": 2.5874104499816895,
+      "rewards/rejected": -3.0590012073516846,
+      "step": 1480
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.332510480306342e-05,
+      "logits/chosen": -1.4027369022369385,
+      "logits/rejected": -1.281185269355774,
+      "logps/chosen": -239.8651580810547,
+      "logps/rejected": -269.84600830078125,
+      "loss": 0.4071,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.3220774233341217,
+      "rewards/margins": 2.7993996143341064,
+      "rewards/rejected": -3.121476650238037,
+      "step": 1485
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.319955192629417e-05,
+      "logits/chosen": -1.4315681457519531,
+      "logits/rejected": -1.3057044744491577,
+      "logps/chosen": -248.1969757080078,
+      "logps/rejected": -286.1350402832031,
+      "loss": 0.3744,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.40552300214767456,
+      "rewards/margins": 2.5983104705810547,
+      "rewards/rejected": -3.003833293914795,
+      "step": 1490
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.3073766884848234e-05,
+      "logits/chosen": -1.3912522792816162,
+      "logits/rejected": -1.3030383586883545,
+      "logps/chosen": -223.78085327148438,
+      "logps/rejected": -273.02410888671875,
+      "loss": 0.3343,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.3259705901145935,
+      "rewards/margins": 2.883284091949463,
+      "rewards/rejected": -3.2092552185058594,
+      "step": 1495
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.294775324024259e-05,
+      "logits/chosen": -1.4088590145111084,
+      "logits/rejected": -1.3673789501190186,
+      "logps/chosen": -222.629150390625,
+      "logps/rejected": -287.8419189453125,
+      "loss": 0.3654,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3555578589439392,
+      "rewards/margins": 3.0522375106811523,
+      "rewards/rejected": -3.4077954292297363,
+      "step": 1500
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2821514560466965e-05,
+      "logits/chosen": -1.3416802883148193,
+      "logits/rejected": -1.2942748069763184,
+      "logps/chosen": -256.21875,
+      "logps/rejected": -308.81402587890625,
+      "loss": 0.4083,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5183025002479553,
+      "rewards/margins": 3.0418663024902344,
+      "rewards/rejected": -3.560168743133545,
+      "step": 1505
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.269505441988281e-05,
+      "logits/chosen": -1.408935308456421,
+      "logits/rejected": -1.2729170322418213,
+      "logps/chosen": -265.832275390625,
+      "logps/rejected": -287.1328430175781,
+      "loss": 0.354,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.49197083711624146,
+      "rewards/margins": 3.098872661590576,
+      "rewards/rejected": -3.5908432006835938,
+      "step": 1510
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.256837639912208e-05,
+      "logits/chosen": -1.4301097393035889,
+      "logits/rejected": -1.3893522024154663,
+      "logps/chosen": -226.5152587890625,
+      "logps/rejected": -269.6661682128906,
+      "loss": 0.3423,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.36981138586997986,
+      "rewards/margins": 2.82721209526062,
+      "rewards/rejected": -3.197023391723633,
+      "step": 1515
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2441484084985865e-05,
+      "logits/chosen": -1.408756971359253,
+      "logits/rejected": -1.3597663640975952,
+      "logps/chosen": -246.8629913330078,
+      "logps/rejected": -291.6884765625,
+      "loss": 0.4077,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.5306534767150879,
+      "rewards/margins": 2.8845298290252686,
+      "rewards/rejected": -3.4151833057403564,
+      "step": 1520
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.231438107034281e-05,
+      "logits/chosen": -1.457080602645874,
+      "logits/rejected": -1.37287437915802,
+      "logps/chosen": -251.12301635742188,
+      "logps/rejected": -291.7841796875,
+      "loss": 0.4085,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5804392099380493,
+      "rewards/margins": 2.6093997955322266,
+      "rewards/rejected": -3.1898388862609863,
+      "step": 1525
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.218707095402741e-05,
+      "logits/chosen": -1.4033076763153076,
+      "logits/rejected": -1.305397868156433,
+      "logps/chosen": -225.1595916748047,
+      "logps/rejected": -280.4457702636719,
+      "loss": 0.3649,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.493985116481781,
+      "rewards/margins": 2.9174463748931885,
+      "rewards/rejected": -3.4114317893981934,
+      "step": 1530
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.20595573407381e-05,
+      "logits/chosen": -1.4854360818862915,
+      "logits/rejected": -1.3654184341430664,
+      "logps/chosen": -251.3857879638672,
+      "logps/rejected": -296.96356201171875,
+      "loss": 0.3963,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.7684920430183411,
+      "rewards/margins": 2.8625669479370117,
+      "rewards/rejected": -3.631059169769287,
+      "step": 1535
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.19318438409352e-05,
+      "logits/chosen": -1.4596775770187378,
+      "logits/rejected": -1.3616211414337158,
+      "logps/chosen": -219.89013671875,
+      "logps/rejected": -271.64605712890625,
+      "loss": 0.4047,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.1834189891815186,
+      "rewards/margins": 2.471865653991699,
+      "rewards/rejected": -3.655284881591797,
+      "step": 1540
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.180393407073866e-05,
+      "logits/chosen": -1.3478964567184448,
+      "logits/rejected": -1.266242265701294,
+      "logps/chosen": -259.29766845703125,
+      "logps/rejected": -302.2203369140625,
+      "loss": 0.4261,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.5068633556365967,
+      "rewards/margins": 2.906827449798584,
+      "rewards/rejected": -4.413690567016602,
+      "step": 1545
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1675831651825704e-05,
+      "logits/chosen": -1.2607046365737915,
+      "logits/rejected": -1.12994384765625,
+      "logps/chosen": -280.3153381347656,
+      "logps/rejected": -326.66900634765625,
+      "loss": 0.3582,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.910593032836914,
+      "rewards/margins": 3.522291898727417,
+      "rewards/rejected": -5.43288516998291,
+      "step": 1550
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.154754021132827e-05,
+      "logits/chosen": -1.4051783084869385,
+      "logits/rejected": -1.296360969543457,
+      "logps/chosen": -274.32733154296875,
+      "logps/rejected": -321.9216003417969,
+      "loss": 0.403,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.8257957696914673,
+      "rewards/margins": 3.0792431831359863,
+      "rewards/rejected": -4.905039310455322,
+      "step": 1555
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1419063381730317e-05,
+      "logits/chosen": -1.3628051280975342,
+      "logits/rejected": -1.2559598684310913,
+      "logps/chosen": -234.3201141357422,
+      "logps/rejected": -262.8561706542969,
+      "loss": 0.4669,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.6719917058944702,
+      "rewards/margins": 2.5399348735809326,
+      "rewards/rejected": -4.211926460266113,
+      "step": 1560
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.129040480076496e-05,
+      "logits/chosen": -1.427811861038208,
+      "logits/rejected": -1.2909691333770752,
+      "logps/chosen": -259.02105712890625,
+      "logps/rejected": -307.50006103515625,
+      "loss": 0.3602,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5834705829620361,
+      "rewards/margins": 3.394528865814209,
+      "rewards/rejected": -4.977999687194824,
+      "step": 1565
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.116156811131148e-05,
+      "logits/chosen": -1.404813528060913,
+      "logits/rejected": -1.2708321809768677,
+      "logps/chosen": -235.78720092773438,
+      "logps/rejected": -253.49172973632812,
+      "loss": 0.3823,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.3320645093917847,
+      "rewards/margins": 2.766334056854248,
+      "rewards/rejected": -4.098398685455322,
+      "step": 1570
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1032556961292194e-05,
+      "logits/chosen": -1.457380771636963,
+      "logits/rejected": -1.3725563287734985,
+      "logps/chosen": -235.90377807617188,
+      "logps/rejected": -280.8343200683594,
+      "loss": 0.3649,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.9432939291000366,
+      "rewards/margins": 2.720813512802124,
+      "rewards/rejected": -3.664107084274292,
+      "step": 1575
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0903375003569124e-05,
+      "logits/chosen": -1.3895059823989868,
+      "logits/rejected": -1.3300002813339233,
+      "logps/chosen": -250.08218383789062,
+      "logps/rejected": -312.3929748535156,
+      "loss": 0.3683,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0983926057815552,
+      "rewards/margins": 3.0330488681793213,
+      "rewards/rejected": -4.131441116333008,
+      "step": 1580
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.077402589584061e-05,
+      "logits/chosen": -1.4692285060882568,
+      "logits/rejected": -1.3498972654342651,
+      "logps/chosen": -269.5802917480469,
+      "logps/rejected": -317.4629211425781,
+      "loss": 0.3919,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.9154409170150757,
+      "rewards/margins": 3.027033805847168,
+      "rewards/rejected": -3.942474842071533,
+      "step": 1585
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.064451330053773e-05,
+      "logits/chosen": -1.358955979347229,
+      "logits/rejected": -1.2798420190811157,
+      "logps/chosen": -221.613525390625,
+      "logps/rejected": -278.844970703125,
+      "loss": 0.354,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.8618852496147156,
+      "rewards/margins": 3.097717761993408,
+      "rewards/rejected": -3.9596030712127686,
+      "step": 1590
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0514840884720598e-05,
+      "logits/chosen": -1.5032036304473877,
+      "logits/rejected": -1.3345158100128174,
+      "logps/chosen": -291.01507568359375,
+      "logps/rejected": -325.32574462890625,
+      "loss": 0.409,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.3084170818328857,
+      "rewards/margins": 2.8849997520446777,
+      "rewards/rejected": -4.193417549133301,
+      "step": 1595
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0385012319974537e-05,
+      "logits/chosen": -1.4529359340667725,
+      "logits/rejected": -1.3749693632125854,
+      "logps/chosen": -238.7993927001953,
+      "logps/rejected": -302.42303466796875,
+      "loss": 0.3772,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.1509284973144531,
+      "rewards/margins": 3.0289180278778076,
+      "rewards/rejected": -4.17984676361084,
+      "step": 1600
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0255031282306106e-05,
+      "logits/chosen": -1.3734673261642456,
+      "logits/rejected": -1.2939527034759521,
+      "logps/chosen": -236.4968719482422,
+      "logps/rejected": -294.1763000488281,
+      "loss": 0.3329,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.4785823822021484,
+      "rewards/margins": 3.626276731491089,
+      "rewards/rejected": -5.104858875274658,
+      "step": 1605
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.012490145203906e-05,
+      "logits/chosen": -1.396791696548462,
+      "logits/rejected": -1.3902806043624878,
+      "logps/chosen": -227.6318817138672,
+      "logps/rejected": -299.10784912109375,
+      "loss": 0.3666,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.4431589841842651,
+      "rewards/margins": 3.2994017601013184,
+      "rewards/rejected": -4.742560863494873,
+      "step": 1610
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.9994626513710084e-05,
+      "logits/chosen": -1.3043615818023682,
+      "logits/rejected": -1.1662665605545044,
+      "logps/chosen": -263.1172180175781,
+      "logps/rejected": -311.5401916503906,
+      "loss": 0.3786,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4172508716583252,
+      "rewards/margins": 4.293739318847656,
+      "rewards/rejected": -5.710989952087402,
+      "step": 1615
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.9864210155964507e-05,
+      "logits/chosen": -1.3513799905776978,
+      "logits/rejected": -1.226161241531372,
+      "logps/chosen": -235.1254119873047,
+      "logps/rejected": -298.6861267089844,
+      "loss": 0.3383,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1513350009918213,
+      "rewards/margins": 3.736593723297119,
+      "rewards/rejected": -4.887928485870361,
+      "step": 1620
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9733656071451867e-05,
+      "logits/chosen": -1.3315715789794922,
+      "logits/rejected": -1.280723214149475,
+      "logps/chosen": -248.2561492919922,
+      "logps/rejected": -322.32025146484375,
+      "loss": 0.3294,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.2765705585479736,
+      "rewards/margins": 3.39424204826355,
+      "rewards/rejected": -4.670812606811523,
+      "step": 1625
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9602967956721316e-05,
+      "logits/chosen": -1.3882957696914673,
+      "logits/rejected": -1.284002661705017,
+      "logps/chosen": -276.1745910644531,
+      "logps/rejected": -322.44061279296875,
+      "loss": 0.4215,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.0892524719238281,
+      "rewards/margins": 3.2803215980529785,
+      "rewards/rejected": -4.369574069976807,
+      "step": 1630
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.947214951211701e-05,
+      "logits/chosen": -1.4538739919662476,
+      "logits/rejected": -1.3563892841339111,
+      "logps/chosen": -234.4571533203125,
+      "logps/rejected": -294.8569030761719,
+      "loss": 0.3709,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.6589905619621277,
+      "rewards/margins": 3.601499080657959,
+      "rewards/rejected": -4.260489463806152,
+      "step": 1635
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9341204441673266e-05,
+      "logits/chosen": -1.4792182445526123,
+      "logits/rejected": -1.377165675163269,
+      "logps/chosen": -239.96060180664062,
+      "logps/rejected": -283.60491943359375,
+      "loss": 0.3906,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.825871467590332,
+      "rewards/margins": 3.0817301273345947,
+      "rewards/rejected": -3.9076011180877686,
+      "step": 1640
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.921013645300975e-05,
+      "logits/chosen": -1.4524424076080322,
+      "logits/rejected": -1.401745080947876,
+      "logps/chosen": -234.1028289794922,
+      "logps/rejected": -295.9302673339844,
+      "loss": 0.4504,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.8630698919296265,
+      "rewards/margins": 2.638474702835083,
+      "rewards/rejected": -3.50154447555542,
+      "step": 1645
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.907894925722648e-05,
+      "logits/chosen": -1.3984206914901733,
+      "logits/rejected": -1.3711296319961548,
+      "logps/chosen": -251.3262481689453,
+      "logps/rejected": -302.9779968261719,
+      "loss": 0.422,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.8784204721450806,
+      "rewards/margins": 2.59228515625,
+      "rewards/rejected": -3.470705509185791,
+      "step": 1650
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.894764656879873e-05,
+      "logits/chosen": -1.5556986331939697,
+      "logits/rejected": -1.48770010471344,
+      "logps/chosen": -241.54751586914062,
+      "logps/rejected": -301.2930603027344,
+      "loss": 0.374,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6194090247154236,
+      "rewards/margins": 2.9210126399993896,
+      "rewards/rejected": -3.540421962738037,
+      "step": 1655
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8816232105471863e-05,
+      "logits/chosen": -1.5585861206054688,
+      "logits/rejected": -1.4013144969940186,
+      "logps/chosen": -277.42864990234375,
+      "logps/rejected": -293.63787841796875,
+      "loss": 0.3893,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6244452595710754,
+      "rewards/margins": 2.798342227935791,
+      "rewards/rejected": -3.4227874279022217,
+      "step": 1660
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8684709588156085e-05,
+      "logits/chosen": -1.4837238788604736,
+      "logits/rejected": -1.3633246421813965,
+      "logps/chosen": -267.8808898925781,
+      "logps/rejected": -308.5547180175781,
+      "loss": 0.3612,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.40743985772132874,
+      "rewards/margins": 3.1635591983795166,
+      "rewards/rejected": -3.5709991455078125,
+      "step": 1665
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8553082740821057e-05,
+      "logits/chosen": -1.5174936056137085,
+      "logits/rejected": -1.3714519739151,
+      "logps/chosen": -262.1058349609375,
+      "logps/rejected": -278.0890197753906,
+      "loss": 0.4465,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.555295467376709,
+      "rewards/margins": 2.108398675918579,
+      "rewards/rejected": -2.663693904876709,
+      "step": 1670
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8421355290390506e-05,
+      "logits/chosen": -1.4402861595153809,
+      "logits/rejected": -1.3555018901824951,
+      "logps/chosen": -259.9664001464844,
+      "logps/rejected": -304.7403259277344,
+      "loss": 0.3857,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.537945568561554,
+      "rewards/margins": 2.5592586994171143,
+      "rewards/rejected": -3.0972039699554443,
+      "step": 1675
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8289530966636625e-05,
+      "logits/chosen": -1.4750789403915405,
+      "logits/rejected": -1.4176933765411377,
+      "logps/chosen": -247.0699920654297,
+      "logps/rejected": -291.6591796875,
+      "loss": 0.4013,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.47404319047927856,
+      "rewards/margins": 2.721088171005249,
+      "rewards/rejected": -3.195131301879883,
+      "step": 1680
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8157613502074543e-05,
+      "logits/chosen": -1.3425180912017822,
+      "logits/rejected": -1.2664659023284912,
+      "logps/chosen": -229.84848022460938,
+      "logps/rejected": -275.348388671875,
+      "loss": 0.386,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.45674949884414673,
+      "rewards/margins": 2.6647157669067383,
+      "rewards/rejected": -3.1214652061462402,
+      "step": 1685
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.8025606631856578e-05,
+      "logits/chosen": -1.4296760559082031,
+      "logits/rejected": -1.3889728784561157,
+      "logps/chosen": -240.1858673095703,
+      "logps/rejected": -287.5098571777344,
+      "loss": 0.3997,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.3761466145515442,
+      "rewards/margins": 2.935863971710205,
+      "rewards/rejected": -3.3120105266571045,
+      "step": 1690
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7893514093666538e-05,
+      "logits/chosen": -1.5364658832550049,
+      "logits/rejected": -1.4733527898788452,
+      "logps/chosen": -221.28564453125,
+      "logps/rejected": -300.366943359375,
+      "loss": 0.3334,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.08093585073947906,
+      "rewards/margins": 3.043505907058716,
+      "rewards/rejected": -2.9625699520111084,
+      "step": 1695
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7761339627613848e-05,
+      "logits/chosen": -1.5357977151870728,
+      "logits/rejected": -1.3645578622817993,
+      "logps/chosen": -251.7772979736328,
+      "logps/rejected": -281.92376708984375,
+      "loss": 0.364,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3491308391094208,
+      "rewards/margins": 2.4752745628356934,
+      "rewards/rejected": -2.8244051933288574,
+      "step": 1700
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.762908697612765e-05,
+      "logits/chosen": -1.5145528316497803,
+      "logits/rejected": -1.4855618476867676,
+      "logps/chosen": -197.80935668945312,
+      "logps/rejected": -249.3860321044922,
+      "loss": 0.3845,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.19136330485343933,
+      "rewards/margins": 2.256312608718872,
+      "rewards/rejected": -2.447675943374634,
+      "step": 1705
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.749675988385087e-05,
+      "logits/chosen": -1.5334855318069458,
+      "logits/rejected": -1.4892711639404297,
+      "logps/chosen": -212.96484375,
+      "logps/rejected": -259.4864196777344,
+      "loss": 0.3969,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.04548867046833038,
+      "rewards/margins": 2.449012279510498,
+      "rewards/rejected": -2.4945008754730225,
+      "step": 1710
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7364362097534165e-05,
+      "logits/chosen": -1.3821698427200317,
+      "logits/rejected": -1.2841250896453857,
+      "logps/chosen": -239.5942840576172,
+      "logps/rejected": -286.0035095214844,
+      "loss": 0.3596,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.20363537967205048,
+      "rewards/margins": 3.127345561981201,
+      "rewards/rejected": -3.3309807777404785,
+      "step": 1715
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.723189736592986e-05,
+      "logits/chosen": -1.4247050285339355,
+      "logits/rejected": -1.3530040979385376,
+      "logps/chosen": -229.03988647460938,
+      "logps/rejected": -294.17706298828125,
+      "loss": 0.3672,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.4444514811038971,
+      "rewards/margins": 2.9286928176879883,
+      "rewards/rejected": -3.3731446266174316,
+      "step": 1720
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.709936943968577e-05,
+      "logits/chosen": -1.4718440771102905,
+      "logits/rejected": -1.3845016956329346,
+      "logps/chosen": -238.27865600585938,
+      "logps/rejected": -283.81939697265625,
+      "loss": 0.411,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5444830656051636,
+      "rewards/margins": 2.8488547801971436,
+      "rewards/rejected": -3.3933379650115967,
+      "step": 1725
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6966782071239027e-05,
+      "logits/chosen": -1.4738774299621582,
+      "logits/rejected": -1.3779428005218506,
+      "logps/chosen": -231.0215606689453,
+      "logps/rejected": -272.0970764160156,
+      "loss": 0.3721,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.35090774297714233,
+      "rewards/margins": 2.7793362140655518,
+      "rewards/rejected": -3.1302435398101807,
+      "step": 1730
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6834139014709843e-05,
+      "logits/chosen": -1.37410569190979,
+      "logits/rejected": -1.3108699321746826,
+      "logps/chosen": -239.4388885498047,
+      "logps/rejected": -282.64971923828125,
+      "loss": 0.3667,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3109089732170105,
+      "rewards/margins": 3.148824691772461,
+      "rewards/rejected": -3.459733486175537,
+      "step": 1735
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.670144402579518e-05,
+      "logits/chosen": -1.4222691059112549,
+      "logits/rejected": -1.3661749362945557,
+      "logps/chosen": -251.8944091796875,
+      "logps/rejected": -308.817138671875,
+      "loss": 0.3653,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.46417436003685,
+      "rewards/margins": 2.9680099487304688,
+      "rewards/rejected": -3.4321842193603516,
+      "step": 1740
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6568700861662445e-05,
+      "logits/chosen": -1.4371557235717773,
+      "logits/rejected": -1.395452857017517,
+      "logps/chosen": -233.2667236328125,
+      "logps/rejected": -296.88897705078125,
+      "loss": 0.3752,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.42657342553138733,
+      "rewards/margins": 2.4918549060821533,
+      "rewards/rejected": -2.9184281826019287,
+      "step": 1745
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.643591328084309e-05,
+      "logits/chosen": -1.3927974700927734,
+      "logits/rejected": -1.3720002174377441,
+      "logps/chosen": -216.7613983154297,
+      "logps/rejected": -285.96746826171875,
+      "loss": 0.3812,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.2734035849571228,
+      "rewards/margins": 2.81980562210083,
+      "rewards/rejected": -3.093209743499756,
+      "step": 1750
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6303085043126176e-05,
+      "logits/chosen": -1.4500024318695068,
+      "logits/rejected": -1.35175359249115,
+      "logps/chosen": -269.2599182128906,
+      "logps/rejected": -313.74639892578125,
+      "loss": 0.3278,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.4534524977207184,
+      "rewards/margins": 3.014010190963745,
+      "rewards/rejected": -3.467463254928589,
+      "step": 1755
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.617021990945197e-05,
+      "logits/chosen": -1.627383828163147,
+      "logits/rejected": -1.5457924604415894,
+      "logps/chosen": -228.83285522460938,
+      "logps/rejected": -268.08148193359375,
+      "loss": 0.4364,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.3291838467121124,
+      "rewards/margins": 2.480336904525757,
+      "rewards/rejected": -2.809520721435547,
+      "step": 1760
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.603732164180539e-05,
+      "logits/chosen": -1.3406190872192383,
+      "logits/rejected": -1.321590542793274,
+      "logps/chosen": -204.06773376464844,
+      "logps/rejected": -272.1907653808594,
+      "loss": 0.3289,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.5110599398612976,
+      "rewards/margins": 2.9445903301239014,
+      "rewards/rejected": -3.4556503295898438,
+      "step": 1765
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5904394003109555e-05,
+      "logits/chosen": -1.407454252243042,
+      "logits/rejected": -1.3605538606643677,
+      "logps/chosen": -241.2417755126953,
+      "logps/rejected": -289.2109680175781,
+      "loss": 0.3726,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.7598351836204529,
+      "rewards/margins": 2.538835287094116,
+      "rewards/rejected": -3.2986702919006348,
+      "step": 1770
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5771440757119165e-05,
+      "logits/chosen": -1.3968393802642822,
+      "logits/rejected": -1.3065111637115479,
+      "logps/chosen": -258.99139404296875,
+      "logps/rejected": -323.99920654296875,
+      "loss": 0.3912,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.9822736978530884,
+      "rewards/margins": 3.3598105907440186,
+      "rewards/rejected": -4.3420844078063965,
+      "step": 1775
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5638465668314006e-05,
+      "logits/chosen": -1.40274178981781,
+      "logits/rejected": -1.3181835412979126,
+      "logps/chosen": -232.64627075195312,
+      "logps/rejected": -276.2817077636719,
+      "loss": 0.3353,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.7937625646591187,
+      "rewards/margins": 2.890428066253662,
+      "rewards/rejected": -3.684190273284912,
+      "step": 1780
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5505472501792298e-05,
+      "logits/chosen": -1.41157066822052,
+      "logits/rejected": -1.3127758502960205,
+      "logps/chosen": -240.01473999023438,
+      "logps/rejected": -264.02313232421875,
+      "loss": 0.4051,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.821036159992218,
+      "rewards/margins": 2.6046383380889893,
+      "rewards/rejected": -3.4256744384765625,
+      "step": 1785
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5372465023164148e-05,
+      "logits/chosen": -1.5105726718902588,
+      "logits/rejected": -1.3530454635620117,
+      "logps/chosen": -232.69873046875,
+      "logps/rejected": -260.64453125,
+      "loss": 0.4211,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.6090893745422363,
+      "rewards/margins": 2.594888210296631,
+      "rewards/rejected": -3.2039780616760254,
+      "step": 1790
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5239446998444898e-05,
+      "logits/chosen": -1.4365472793579102,
+      "logits/rejected": -1.3337024450302124,
+      "logps/chosen": -217.64816284179688,
+      "logps/rejected": -258.85650634765625,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.6081379652023315,
+      "rewards/margins": 2.89850115776062,
+      "rewards/rejected": -3.5066394805908203,
+      "step": 1795
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.510642219394847e-05,
+      "logits/chosen": -1.3360542058944702,
+      "logits/rejected": -1.2949392795562744,
+      "logps/chosen": -221.05899047851562,
+      "logps/rejected": -275.3384704589844,
+      "loss": 0.4155,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.44868984818458557,
+      "rewards/margins": 3.002562999725342,
+      "rewards/rejected": -3.4512531757354736,
+      "step": 1800
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4973394376180773e-05,
+      "logits/chosen": -1.4750487804412842,
+      "logits/rejected": -1.392292857170105,
+      "logps/chosen": -215.1530303955078,
+      "logps/rejected": -286.1130065917969,
+      "loss": 0.3386,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.04450890049338341,
+      "rewards/margins": 3.0319771766662598,
+      "rewards/rejected": -3.076486110687256,
+      "step": 1805
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4840367311733024e-05,
+      "logits/chosen": -1.4934360980987549,
+      "logits/rejected": -1.4029854536056519,
+      "logps/chosen": -238.88021850585938,
+      "logps/rejected": -284.1180114746094,
+      "loss": 0.3991,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.19598393142223358,
+      "rewards/margins": 2.753159999847412,
+      "rewards/rejected": -2.949143886566162,
+      "step": 1810
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4707344767175118e-05,
+      "logits/chosen": -1.4552268981933594,
+      "logits/rejected": -1.3600109815597534,
+      "logps/chosen": -244.9551239013672,
+      "logps/rejected": -293.6857604980469,
+      "loss": 0.3665,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.16339322924613953,
+      "rewards/margins": 2.704094409942627,
+      "rewards/rejected": -2.867487668991089,
+      "step": 1815
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.457433050894896e-05,
+      "logits/chosen": -1.4229071140289307,
+      "logits/rejected": -1.3520857095718384,
+      "logps/chosen": -234.4111785888672,
+      "logps/rejected": -274.68487548828125,
+      "loss": 0.4018,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.19696488976478577,
+      "rewards/margins": 3.0029873847961426,
+      "rewards/rejected": -3.1999526023864746,
+      "step": 1820
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4441328303261867e-05,
+      "logits/chosen": -1.365638017654419,
+      "logits/rejected": -1.2584137916564941,
+      "logps/chosen": -267.7275085449219,
+      "logps/rejected": -299.4195251464844,
+      "loss": 0.3461,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.6673690676689148,
+      "rewards/margins": 2.8428685665130615,
+      "rewards/rejected": -3.5102379322052,
+      "step": 1825
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4308341915979838e-05,
+      "logits/chosen": -1.501835823059082,
+      "logits/rejected": -1.4059853553771973,
+      "logps/chosen": -221.58859252929688,
+      "logps/rejected": -267.6972961425781,
+      "loss": 0.3689,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.6144279837608337,
+      "rewards/margins": 2.702439785003662,
+      "rewards/rejected": -3.3168678283691406,
+      "step": 1830
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.417537511252105e-05,
+      "logits/chosen": -1.4447147846221924,
+      "logits/rejected": -1.3833913803100586,
+      "logps/chosen": -236.3400115966797,
+      "logps/rejected": -290.0586853027344,
+      "loss": 0.3494,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.6023090481758118,
+      "rewards/margins": 3.214966297149658,
+      "rewards/rejected": -3.817275285720825,
+      "step": 1835
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4042431657749117e-05,
+      "logits/chosen": -1.4315189123153687,
+      "logits/rejected": -1.3628207445144653,
+      "logps/chosen": -207.71719360351562,
+      "logps/rejected": -248.83102416992188,
+      "loss": 0.3966,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.6046200394630432,
+      "rewards/margins": 2.3360109329223633,
+      "rewards/rejected": -2.9406309127807617,
+      "step": 1840
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3909515315866605e-05,
+      "logits/chosen": -1.333467960357666,
+      "logits/rejected": -1.236061692237854,
+      "logps/chosen": -257.92413330078125,
+      "logps/rejected": -294.17901611328125,
+      "loss": 0.3582,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.8103699684143066,
+      "rewards/margins": 3.270451784133911,
+      "rewards/rejected": -4.0808210372924805,
+      "step": 1845
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3776629850308354e-05,
+      "logits/chosen": -1.3392812013626099,
+      "logits/rejected": -1.299647331237793,
+      "logps/chosen": -239.8131103515625,
+      "logps/rejected": -303.4067077636719,
+      "loss": 0.3613,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.6897146105766296,
+      "rewards/margins": 3.181128978729248,
+      "rewards/rejected": -3.8708438873291016,
+      "step": 1850
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.364377902363499e-05,
+      "logits/chosen": -1.4630589485168457,
+      "logits/rejected": -1.3817940950393677,
+      "logps/chosen": -224.0995635986328,
+      "logps/rejected": -267.346435546875,
+      "loss": 0.4036,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.8380252122879028,
+      "rewards/margins": 2.4955790042877197,
+      "rewards/rejected": -3.333604097366333,
+      "step": 1855
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3510966597426354e-05,
+      "logits/chosen": -1.4201809167861938,
+      "logits/rejected": -1.3353426456451416,
+      "logps/chosen": -267.3838806152344,
+      "logps/rejected": -314.8360290527344,
+      "loss": 0.4379,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.7916916608810425,
+      "rewards/margins": 3.1964616775512695,
+      "rewards/rejected": -3.9881534576416016,
+      "step": 1860
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3378196332174993e-05,
+      "logits/chosen": -1.4213106632232666,
+      "logits/rejected": -1.3859220743179321,
+      "logps/chosen": -195.9226837158203,
+      "logps/rejected": -256.7573547363281,
+      "loss": 0.3855,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.37132301926612854,
+      "rewards/margins": 2.312356472015381,
+      "rewards/rejected": -2.6836793422698975,
+      "step": 1865
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.324547198717972e-05,
+      "logits/chosen": -1.4690866470336914,
+      "logits/rejected": -1.4080109596252441,
+      "logps/chosen": -241.17703247070312,
+      "logps/rejected": -302.3997802734375,
+      "loss": 0.363,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.6016757488250732,
+      "rewards/margins": 2.820457696914673,
+      "rewards/rejected": -3.422133207321167,
+      "step": 1870
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.311279732043912e-05,
+      "logits/chosen": -1.4260601997375488,
+      "logits/rejected": -1.3536970615386963,
+      "logps/chosen": -200.6278533935547,
+      "logps/rejected": -253.57763671875,
+      "loss": 0.4365,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.11818097531795502,
+      "rewards/margins": 2.7506301403045654,
+      "rewards/rejected": -2.8688108921051025,
+      "step": 1875
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.2980176088545197e-05,
+      "logits/chosen": -1.4411252737045288,
+      "logits/rejected": -1.3734889030456543,
+      "logps/chosen": -226.0155029296875,
+      "logps/rejected": -277.5158996582031,
+      "loss": 0.3744,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.14310994744300842,
+      "rewards/margins": 3.070277452468872,
+      "rewards/rejected": -3.2133877277374268,
+      "step": 1880
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.284761204657696e-05,
+      "logits/chosen": -1.4405572414398193,
+      "logits/rejected": -1.3739019632339478,
+      "logps/chosen": -213.6031494140625,
+      "logps/rejected": -278.64849853515625,
+      "loss": 0.3599,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1836298406124115,
+      "rewards/margins": 3.2234256267547607,
+      "rewards/rejected": -3.407055377960205,
+      "step": 1885
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2715108947994152e-05,
+      "logits/chosen": -1.4480597972869873,
+      "logits/rejected": -1.3346760272979736,
+      "logps/chosen": -242.326904296875,
+      "logps/rejected": -276.35223388671875,
+      "loss": 0.3833,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.040028151124715805,
+      "rewards/margins": 2.910196542739868,
+      "rewards/rejected": -2.9502246379852295,
+      "step": 1890
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.258267054453091e-05,
+      "logits/chosen": -1.4914884567260742,
+      "logits/rejected": -1.3812105655670166,
+      "logps/chosen": -255.26089477539062,
+      "logps/rejected": -296.7649230957031,
+      "loss": 0.3651,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.10911808162927628,
+      "rewards/margins": 2.631997585296631,
+      "rewards/rejected": -2.7411160469055176,
+      "step": 1895
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2450300586089622e-05,
+      "logits/chosen": -1.5325438976287842,
+      "logits/rejected": -1.4326366186141968,
+      "logps/chosen": -233.3433837890625,
+      "logps/rejected": -272.3608703613281,
+      "loss": 0.378,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.07660797983407974,
+      "rewards/margins": 3.0263311862945557,
+      "rewards/rejected": -2.9497230052948,
+      "step": 1900
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2318002820634648e-05,
+      "logits/chosen": -1.5223352909088135,
+      "logits/rejected": -1.4486531019210815,
+      "logps/chosen": -251.2206268310547,
+      "logps/rejected": -311.48101806640625,
+      "loss": 0.3898,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.14458785951137543,
+      "rewards/margins": 2.646322250366211,
+      "rewards/rejected": -2.790910005569458,
+      "step": 1905
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.218578099408631e-05,
+      "logits/chosen": -1.4956655502319336,
+      "logits/rejected": -1.4093170166015625,
+      "logps/chosen": -220.2262420654297,
+      "logps/rejected": -277.7985534667969,
+      "loss": 0.3647,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.09166286140680313,
+      "rewards/margins": 2.5863606929779053,
+      "rewards/rejected": -2.67802357673645,
+      "step": 1910
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2053638850214704e-05,
+      "logits/chosen": -1.4817699193954468,
+      "logits/rejected": -1.3774340152740479,
+      "logps/chosen": -248.29226684570312,
+      "logps/rejected": -301.8695068359375,
+      "loss": 0.3206,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.18610629439353943,
+      "rewards/margins": 2.9152894020080566,
+      "rewards/rejected": -3.101395606994629,
+      "step": 1915
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1921580130533827e-05,
+      "logits/chosen": -1.4281337261199951,
+      "logits/rejected": -1.3430382013320923,
+      "logps/chosen": -248.2852020263672,
+      "logps/rejected": -295.68255615234375,
+      "loss": 0.417,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.4364057183265686,
+      "rewards/margins": 2.8607912063598633,
+      "rewards/rejected": -3.297196865081787,
+      "step": 1920
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.178960857419556e-05,
+      "logits/chosen": -1.4959535598754883,
+      "logits/rejected": -1.3279250860214233,
+      "logps/chosen": -235.4945831298828,
+      "logps/rejected": -272.96612548828125,
+      "loss": 0.3668,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.3301815986633301,
+      "rewards/margins": 3.2808384895324707,
+      "rewards/rejected": -3.6110198497772217,
+      "step": 1925
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.165772791788379e-05,
+      "logits/chosen": -1.3634750843048096,
+      "logits/rejected": -1.356945276260376,
+      "logps/chosen": -209.04238891601562,
+      "logps/rejected": -281.57305908203125,
+      "loss": 0.3819,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.26547056436538696,
+      "rewards/margins": 2.8412766456604004,
+      "rewards/rejected": -3.1067471504211426,
+      "step": 1930
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1525941895708663e-05,
+      "logits/chosen": -1.3758561611175537,
+      "logits/rejected": -1.3101266622543335,
+      "logps/chosen": -225.871826171875,
+      "logps/rejected": -287.9697265625,
+      "loss": 0.3345,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.41031938791275024,
+      "rewards/margins": 3.1809744834899902,
+      "rewards/rejected": -3.591294050216675,
+      "step": 1935
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1394254239100803e-05,
+      "logits/chosen": -1.4200907945632935,
+      "logits/rejected": -1.3337624073028564,
+      "logps/chosen": -238.2420196533203,
+      "logps/rejected": -286.61334228515625,
+      "loss": 0.3804,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5707160234451294,
+      "rewards/margins": 2.8512752056121826,
+      "rewards/rejected": -3.4219913482666016,
+      "step": 1940
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1262668676705695e-05,
+      "logits/chosen": -1.4637157917022705,
+      "logits/rejected": -1.3421003818511963,
+      "logps/chosen": -253.0985565185547,
+      "logps/rejected": -297.3470153808594,
+      "loss": 0.3698,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.4105305075645447,
+      "rewards/margins": 3.30517578125,
+      "rewards/rejected": -3.7157065868377686,
+      "step": 1945
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.113118893427809e-05,
+      "logits/chosen": -1.4735605716705322,
+      "logits/rejected": -1.3585256338119507,
+      "logps/chosen": -256.94464111328125,
+      "logps/rejected": -304.25311279296875,
+      "loss": 0.3457,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.34792160987854004,
+      "rewards/margins": 3.4550399780273438,
+      "rewards/rejected": -3.8029613494873047,
+      "step": 1950
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0999818734576517e-05,
+      "logits/chosen": -1.5473016500473022,
+      "logits/rejected": -1.4243013858795166,
+      "logps/chosen": -231.41488647460938,
+      "logps/rejected": -275.14227294921875,
+      "loss": 0.3553,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2229432314634323,
+      "rewards/margins": 3.035780191421509,
+      "rewards/rejected": -3.258723497390747,
+      "step": 1955
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0868561797257878e-05,
+      "logits/chosen": -1.3970296382904053,
+      "logits/rejected": -1.3482105731964111,
+      "logps/chosen": -229.5034942626953,
+      "logps/rejected": -291.2673645019531,
+      "loss": 0.351,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.32970067858695984,
+      "rewards/margins": 2.6836564540863037,
+      "rewards/rejected": -3.013357162475586,
+      "step": 1960
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0737421838772146e-05,
+      "logits/chosen": -1.3854676485061646,
+      "logits/rejected": -1.286738395690918,
+      "logps/chosen": -234.7396240234375,
+      "logps/rejected": -284.1507568359375,
+      "loss": 0.3544,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.48179665207862854,
+      "rewards/margins": 2.779966354370117,
+      "rewards/rejected": -3.261763095855713,
+      "step": 1965
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0606402572257084e-05,
+      "logits/chosen": -1.4137897491455078,
+      "logits/rejected": -1.313356637954712,
+      "logps/chosen": -248.8478546142578,
+      "logps/rejected": -297.3784484863281,
+      "loss": 0.402,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.7029843330383301,
+      "rewards/margins": 2.836820602416992,
+      "rewards/rejected": -3.5398049354553223,
+      "step": 1970
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.047550770743318e-05,
+      "logits/chosen": -1.4211134910583496,
+      "logits/rejected": -1.3550993204116821,
+      "logps/chosen": -244.33279418945312,
+      "logps/rejected": -308.0687561035156,
+      "loss": 0.3867,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.746593713760376,
+      "rewards/margins": 3.4636435508728027,
+      "rewards/rejected": -4.2102370262146,
+      "step": 1975
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.034474095049855e-05,
+      "logits/chosen": -1.4394162893295288,
+      "logits/rejected": -1.337714433670044,
+      "logps/chosen": -249.66024780273438,
+      "logps/rejected": -289.6098937988281,
+      "loss": 0.3449,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.6490699648857117,
+      "rewards/margins": 3.0966174602508545,
+      "rewards/rejected": -3.745687961578369,
+      "step": 1980
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.021410600402404e-05,
+      "logits/chosen": -1.3536403179168701,
+      "logits/rejected": -1.2263944149017334,
+      "logps/chosen": -229.769775390625,
+      "logps/rejected": -286.17205810546875,
+      "loss": 0.3394,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.8614643812179565,
+      "rewards/margins": 3.6408305168151855,
+      "rewards/rejected": -4.50229549407959,
+      "step": 1985
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.008360656684837e-05,
+      "logits/chosen": -1.3974249362945557,
+      "logits/rejected": -1.3158893585205078,
+      "logps/chosen": -243.1596221923828,
+      "logps/rejected": -301.7734375,
+      "loss": 0.3899,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.6392949819564819,
+      "rewards/margins": 3.155815362930298,
+      "rewards/rejected": -3.7951102256774902,
+      "step": 1990
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.995324633397338e-05,
+      "logits/chosen": -1.5212452411651611,
+      "logits/rejected": -1.4078960418701172,
+      "logps/chosen": -231.8392333984375,
+      "logps/rejected": -280.28375244140625,
+      "loss": 0.4023,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.47973960638046265,
+      "rewards/margins": 3.4426467418670654,
+      "rewards/rejected": -3.922386884689331,
+      "step": 1995
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9823028996459486e-05,
+      "logits/chosen": -1.3417774438858032,
+      "logits/rejected": -1.255110502243042,
+      "logps/chosen": -240.9922332763672,
+      "logps/rejected": -289.61871337890625,
+      "loss": 0.3945,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.3862563371658325,
+      "rewards/margins": 3.139626979827881,
+      "rewards/rejected": -3.525883436203003,
+      "step": 2000
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.969295824132107e-05,
+      "logits/chosen": -1.3991576433181763,
+      "logits/rejected": -1.2910772562026978,
+      "logps/chosen": -210.6538848876953,
+      "logps/rejected": -255.54403686523438,
+      "loss": 0.3442,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.38305678963661194,
+      "rewards/margins": 2.7472267150878906,
+      "rewards/rejected": -3.1302833557128906,
+      "step": 2005
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.956303775142217e-05,
+      "logits/chosen": -1.458589792251587,
+      "logits/rejected": -1.3729736804962158,
+      "logps/chosen": -236.9148406982422,
+      "logps/rejected": -294.9259338378906,
+      "loss": 0.3655,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.33419889211654663,
+      "rewards/margins": 3.009550094604492,
+      "rewards/rejected": -3.3437488079071045,
+      "step": 2010
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.943327120537215e-05,
+      "logits/chosen": -1.4556093215942383,
+      "logits/rejected": -1.4194831848144531,
+      "logps/chosen": -230.87850952148438,
+      "logps/rejected": -290.84881591796875,
+      "loss": 0.3859,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.5414482951164246,
+      "rewards/margins": 2.6852874755859375,
+      "rewards/rejected": -3.226736068725586,
+      "step": 2015
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9303662277421568e-05,
+      "logits/chosen": -1.408242106437683,
+      "logits/rejected": -1.3377116918563843,
+      "logps/chosen": -214.5132293701172,
+      "logps/rejected": -254.10140991210938,
+      "loss": 0.3929,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.29236260056495667,
+      "rewards/margins": 2.5391948223114014,
+      "rewards/rejected": -2.831557512283325,
+      "step": 2020
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9174214637358122e-05,
+      "logits/chosen": -1.4432224035263062,
+      "logits/rejected": -1.4354238510131836,
+      "logps/chosen": -205.9859619140625,
+      "logps/rejected": -270.5035705566406,
+      "loss": 0.4391,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.5980446338653564,
+      "rewards/margins": 2.451856851577759,
+      "rewards/rejected": -3.0499014854431152,
+      "step": 2025
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9044931950402774e-05,
+      "logits/chosen": -1.4746092557907104,
+      "logits/rejected": -1.400431513786316,
+      "logps/chosen": -238.9920196533203,
+      "logps/rejected": -290.6305236816406,
+      "loss": 0.3919,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.5437467098236084,
+      "rewards/margins": 2.831157684326172,
+      "rewards/rejected": -3.374904155731201,
+      "step": 2030
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8915817877105926e-05,
+      "logits/chosen": -1.523667335510254,
+      "logits/rejected": -1.418398380279541,
+      "logps/chosen": -237.66549682617188,
+      "logps/rejected": -272.54052734375,
+      "loss": 0.3978,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.42985543608665466,
+      "rewards/margins": 2.5595247745513916,
+      "rewards/rejected": -2.989380121231079,
+      "step": 2035
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.878687607324382e-05,
+      "logits/chosen": -1.4313665628433228,
+      "logits/rejected": -1.3328959941864014,
+      "logps/chosen": -254.4265594482422,
+      "logps/rejected": -286.2435607910156,
+      "loss": 0.3886,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.38601452112197876,
+      "rewards/margins": 2.6175618171691895,
+      "rewards/rejected": -3.0035765171051025,
+      "step": 2040
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.865811018971502e-05,
+      "logits/chosen": -1.5035260915756226,
+      "logits/rejected": -1.4406765699386597,
+      "logps/chosen": -235.6429443359375,
+      "logps/rejected": -284.11651611328125,
+      "loss": 0.3842,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5510352849960327,
+      "rewards/margins": 2.69769024848938,
+      "rewards/rejected": -3.248725175857544,
+      "step": 2045
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.852952387243698e-05,
+      "logits/chosen": -1.4961767196655273,
+      "logits/rejected": -1.431730031967163,
+      "logps/chosen": -216.1791229248047,
+      "logps/rejected": -258.23095703125,
+      "loss": 0.3778,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3444279730319977,
+      "rewards/margins": 2.5474395751953125,
+      "rewards/rejected": -2.8918673992156982,
+      "step": 2050
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.840112076224291e-05,
+      "logits/chosen": -1.382947564125061,
+      "logits/rejected": -1.3435488939285278,
+      "logps/chosen": -227.3728485107422,
+      "logps/rejected": -301.82159423828125,
+      "loss": 0.3834,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5828854441642761,
+      "rewards/margins": 3.2627499103546143,
+      "rewards/rejected": -3.845635175704956,
+      "step": 2055
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8272904494778602e-05,
+      "logits/chosen": -1.416355013847351,
+      "logits/rejected": -1.408719539642334,
+      "logps/chosen": -234.11172485351562,
+      "logps/rejected": -281.99505615234375,
+      "loss": 0.4241,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5792425870895386,
+      "rewards/margins": 2.3325040340423584,
+      "rewards/rejected": -2.9117465019226074,
+      "step": 2060
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.814487870039955e-05,
+      "logits/chosen": -1.419982671737671,
+      "logits/rejected": -1.3562277555465698,
+      "logps/chosen": -230.45144653320312,
+      "logps/rejected": -270.77203369140625,
+      "loss": 0.3769,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5390575528144836,
+      "rewards/margins": 2.4156157970428467,
+      "rewards/rejected": -2.9546732902526855,
+      "step": 2065
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8017047004068105e-05,
+      "logits/chosen": -1.4043166637420654,
+      "logits/rejected": -1.3527730703353882,
+      "logps/chosen": -235.4857177734375,
+      "logps/rejected": -319.1287841796875,
+      "loss": 0.3844,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5465725660324097,
+      "rewards/margins": 3.312981367111206,
+      "rewards/rejected": -3.859553575515747,
+      "step": 2070
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7889413025250897e-05,
+      "logits/chosen": -1.4501091241836548,
+      "logits/rejected": -1.3656995296478271,
+      "logps/chosen": -248.06228637695312,
+      "logps/rejected": -298.1331787109375,
+      "loss": 0.3948,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.6459044218063354,
+      "rewards/margins": 2.820582151412964,
+      "rewards/rejected": -3.466486692428589,
+      "step": 2075
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7761980377816287e-05,
+      "logits/chosen": -1.446747064590454,
+      "logits/rejected": -1.331923007965088,
+      "logps/chosen": -254.3787841796875,
+      "logps/rejected": -298.6913146972656,
+      "loss": 0.3788,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.58036208152771,
+      "rewards/margins": 3.1082186698913574,
+      "rewards/rejected": -3.688580274581909,
+      "step": 2080
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7634752669932115e-05,
+      "logits/chosen": -1.425175428390503,
+      "logits/rejected": -1.3348580598831177,
+      "logps/chosen": -231.65908813476562,
+      "logps/rejected": -274.4278869628906,
+      "loss": 0.4255,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.701018214225769,
+      "rewards/margins": 2.68603515625,
+      "rewards/rejected": -3.3870530128479004,
+      "step": 2085
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7507733503963457e-05,
+      "logits/chosen": -1.4499051570892334,
+      "logits/rejected": -1.3007951974868774,
+      "logps/chosen": -242.14804077148438,
+      "logps/rejected": -265.29937744140625,
+      "loss": 0.3743,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5920419096946716,
+      "rewards/margins": 2.760340690612793,
+      "rewards/rejected": -3.3523826599121094,
+      "step": 2090
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7380926476370702e-05,
+      "logits/chosen": -1.4862868785858154,
+      "logits/rejected": -1.3940832614898682,
+      "logps/chosen": -213.6471405029297,
+      "logps/rejected": -256.01214599609375,
+      "loss": 0.3692,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.5724049806594849,
+      "rewards/margins": 2.784367561340332,
+      "rewards/rejected": -3.3567726612091064,
+      "step": 2095
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.725433517760768e-05,
+      "logits/chosen": -1.4317893981933594,
+      "logits/rejected": -1.2953369617462158,
+      "logps/chosen": -274.6922607421875,
+      "logps/rejected": -308.44354248046875,
+      "loss": 0.3591,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.6146590113639832,
+      "rewards/margins": 3.500483751296997,
+      "rewards/rejected": -4.115141868591309,
+      "step": 2100
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7127963192019975e-05,
+      "logits/chosen": -1.3676774501800537,
+      "logits/rejected": -1.3125852346420288,
+      "logps/chosen": -214.765625,
+      "logps/rejected": -290.7088928222656,
+      "loss": 0.3849,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5840885043144226,
+      "rewards/margins": 3.0269782543182373,
+      "rewards/rejected": -3.6110668182373047,
+      "step": 2105
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7001814097743528e-05,
+      "logits/chosen": -1.4912971258163452,
+      "logits/rejected": -1.3937398195266724,
+      "logps/chosen": -243.7738037109375,
+      "logps/rejected": -289.00592041015625,
+      "loss": 0.3838,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.7238431572914124,
+      "rewards/margins": 2.812650203704834,
+      "rewards/rejected": -3.5364933013916016,
+      "step": 2110
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.6875891466603204e-05,
+      "logits/chosen": -1.4120972156524658,
+      "logits/rejected": -1.3030786514282227,
+      "logps/chosen": -245.34042358398438,
+      "logps/rejected": -291.8741149902344,
+      "loss": 0.3802,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5131228566169739,
+      "rewards/margins": 3.2208220958709717,
+      "rewards/rejected": -3.73394513130188,
+      "step": 2115
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.675019886401177e-05,
+      "logits/chosen": -1.441261649131775,
+      "logits/rejected": -1.3036904335021973,
+      "logps/chosen": -252.41976928710938,
+      "logps/rejected": -283.91204833984375,
+      "loss": 0.3985,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.39861178398132324,
+      "rewards/margins": 2.945107936859131,
+      "rewards/rejected": -3.343719959259033,
+      "step": 2120
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6624739848868854e-05,
+      "logits/chosen": -1.4387789964675903,
+      "logits/rejected": -1.3071900606155396,
+      "logps/chosen": -241.0995635986328,
+      "logps/rejected": -269.37548828125,
+      "loss": 0.366,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.20480592548847198,
+      "rewards/margins": 3.1126081943511963,
+      "rewards/rejected": -3.3174140453338623,
+      "step": 2125
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6499517973460245e-05,
+      "logits/chosen": -1.3849633932113647,
+      "logits/rejected": -1.394295334815979,
+      "logps/chosen": -244.7119903564453,
+      "logps/rejected": -331.77960205078125,
+      "loss": 0.3654,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.4295539855957031,
+      "rewards/margins": 3.0043957233428955,
+      "rewards/rejected": -3.4339497089385986,
+      "step": 2130
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6374536783357268e-05,
+      "logits/chosen": -1.3435999155044556,
+      "logits/rejected": -1.3118559122085571,
+      "logps/chosen": -228.97640991210938,
+      "logps/rejected": -318.62689208984375,
+      "loss": 0.3505,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2641984820365906,
+      "rewards/margins": 3.0577046871185303,
+      "rewards/rejected": -3.3219032287597656,
+      "step": 2135
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6249799817316415e-05,
+      "logits/chosen": -1.3465222120285034,
+      "logits/rejected": -1.2002493143081665,
+      "logps/chosen": -260.20477294921875,
+      "logps/rejected": -297.1739501953125,
+      "loss": 0.3054,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.2534390985965729,
+      "rewards/margins": 3.1337084770202637,
+      "rewards/rejected": -3.3871474266052246,
+      "step": 2140
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6125310607179133e-05,
+      "logits/chosen": -1.4271605014801025,
+      "logits/rejected": -1.4029309749603271,
+      "logps/chosen": -217.9933319091797,
+      "logps/rejected": -285.71282958984375,
+      "loss": 0.3688,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.2108803689479828,
+      "rewards/margins": 2.7702507972717285,
+      "rewards/rejected": -2.9811313152313232,
+      "step": 2145
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.6001072677771843e-05,
+      "logits/chosen": -1.434211015701294,
+      "logits/rejected": -1.3352999687194824,
+      "logps/chosen": -256.20953369140625,
+      "logps/rejected": -302.0452575683594,
+      "loss": 0.3685,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.3069499433040619,
+      "rewards/margins": 3.477191925048828,
+      "rewards/rejected": -3.784142017364502,
+      "step": 2150
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5877089546806125e-05,
+      "logits/chosen": -1.3840197324752808,
+      "logits/rejected": -1.2724934816360474,
+      "logps/chosen": -230.0450439453125,
+      "logps/rejected": -274.7979431152344,
+      "loss": 0.3782,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.3171940743923187,
+      "rewards/margins": 3.197904109954834,
+      "rewards/rejected": -3.5150985717773438,
+      "step": 2155
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5753364724779092e-05,
+      "logits/chosen": -1.4506622552871704,
+      "logits/rejected": -1.3692567348480225,
+      "logps/chosen": -231.21963500976562,
+      "logps/rejected": -269.173095703125,
+      "loss": 0.3128,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.2219165563583374,
+      "rewards/margins": 3.0431320667266846,
+      "rewards/rejected": -3.2650482654571533,
+      "step": 2160
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5629901714874056e-05,
+      "logits/chosen": -1.4132072925567627,
+      "logits/rejected": -1.301841378211975,
+      "logps/chosen": -215.6963348388672,
+      "logps/rejected": -276.54949951171875,
+      "loss": 0.4004,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.4771305024623871,
+      "rewards/margins": 2.786186933517456,
+      "rewards/rejected": -3.263317584991455,
+      "step": 2165
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5506704012861256e-05,
+      "logits/chosen": -1.4663952589035034,
+      "logits/rejected": -1.4031254053115845,
+      "logps/chosen": -215.5591278076172,
+      "logps/rejected": -278.62933349609375,
+      "loss": 0.4011,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.3410421907901764,
+      "rewards/margins": 2.8404176235198975,
+      "rewards/rejected": -3.181459903717041,
+      "step": 2170
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5383775106998976e-05,
+      "logits/chosen": -1.3863542079925537,
+      "logits/rejected": -1.353366494178772,
+      "logps/chosen": -219.19192504882812,
+      "logps/rejected": -289.38018798828125,
+      "loss": 0.4094,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5598834156990051,
+      "rewards/margins": 2.8867409229278564,
+      "rewards/rejected": -3.4466240406036377,
+      "step": 2175
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5261118477934645e-05,
+      "logits/chosen": -1.3812984228134155,
+      "logits/rejected": -1.2817041873931885,
+      "logps/chosen": -208.2887725830078,
+      "logps/rejected": -252.01492309570312,
+      "loss": 0.3853,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.492426335811615,
+      "rewards/margins": 2.9170594215393066,
+      "rewards/rejected": -3.4094855785369873,
+      "step": 2180
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5138737598606448e-05,
+      "logits/chosen": -1.4833437204360962,
+      "logits/rejected": -1.3331998586654663,
+      "logps/chosen": -275.10308837890625,
+      "logps/rejected": -283.90399169921875,
+      "loss": 0.4228,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.29744550585746765,
+      "rewards/margins": 2.7555899620056152,
+      "rewards/rejected": -3.0530357360839844,
+      "step": 2185
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5016635934144824e-05,
+      "logits/chosen": -1.4359524250030518,
+      "logits/rejected": -1.3323842287063599,
+      "logps/chosen": -227.65158081054688,
+      "logps/rejected": -264.79144287109375,
+      "loss": 0.3748,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.48231664299964905,
+      "rewards/margins": 2.5819931030273438,
+      "rewards/rejected": -3.06430983543396,
+      "step": 2190
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4894816941774508e-05,
+      "logits/chosen": -1.4607924222946167,
+      "logits/rejected": -1.3200831413269043,
+      "logps/chosen": -224.31387329101562,
+      "logps/rejected": -267.02325439453125,
+      "loss": 0.4454,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.3082335591316223,
+      "rewards/margins": 2.6731209754943848,
+      "rewards/rejected": -2.9813544750213623,
+      "step": 2195
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4773284070716503e-05,
+      "logits/chosen": -1.5401244163513184,
+      "logits/rejected": -1.4368436336517334,
+      "logps/chosen": -232.74081420898438,
+      "logps/rejected": -272.65423583984375,
+      "loss": 0.4142,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.3758106231689453,
+      "rewards/margins": 2.910980701446533,
+      "rewards/rejected": -3.2867913246154785,
+      "step": 2200
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4652040762090541e-05,
+      "logits/chosen": -1.4225276708602905,
+      "logits/rejected": -1.3527642488479614,
+      "logps/chosen": -230.27249145507812,
+      "logps/rejected": -273.2910461425781,
+      "loss": 0.3982,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.14205607771873474,
+      "rewards/margins": 2.690882682800293,
+      "rewards/rejected": -2.8329386711120605,
+      "step": 2205
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4531090448817558e-05,
+      "logits/chosen": -1.47641921043396,
+      "logits/rejected": -1.4135901927947998,
+      "logps/chosen": -233.48486328125,
+      "logps/rejected": -287.6616516113281,
+      "loss": 0.3689,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.20316210389137268,
+      "rewards/margins": 3.0185022354125977,
+      "rewards/rejected": -3.2216639518737793,
+      "step": 2210
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4410436555522522e-05,
+      "logits/chosen": -1.4257131814956665,
+      "logits/rejected": -1.275315761566162,
+      "logps/chosen": -252.2233123779297,
+      "logps/rejected": -284.6685791015625,
+      "loss": 0.2998,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.012933698482811451,
+      "rewards/margins": 3.136654853820801,
+      "rewards/rejected": -3.1495883464813232,
+      "step": 2215
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4290082498437515e-05,
+      "logits/chosen": -1.466830849647522,
+      "logits/rejected": -1.326336145401001,
+      "logps/chosen": -245.9455108642578,
+      "logps/rejected": -305.823486328125,
+      "loss": 0.3603,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.14640632271766663,
+      "rewards/margins": 3.019836902618408,
+      "rewards/rejected": -3.166242837905884,
+      "step": 2220
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4170031685304913e-05,
+      "logits/chosen": -1.408921241760254,
+      "logits/rejected": -1.3455395698547363,
+      "logps/chosen": -232.0191192626953,
+      "logps/rejected": -277.630859375,
+      "loss": 0.3774,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.4036007821559906,
+      "rewards/margins": 2.6838150024414062,
+      "rewards/rejected": -3.0874156951904297,
+      "step": 2225
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.405028751528099e-05,
+      "logits/chosen": -1.4571171998977661,
+      "logits/rejected": -1.3236163854599,
+      "logps/chosen": -253.90609741210938,
+      "logps/rejected": -288.3222961425781,
+      "loss": 0.3519,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.03257422894239426,
+      "rewards/margins": 3.569223403930664,
+      "rewards/rejected": -3.601797580718994,
+      "step": 2230
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3930853378839603e-05,
+      "logits/chosen": -1.4818923473358154,
+      "logits/rejected": -1.4186928272247314,
+      "logps/chosen": -258.0881652832031,
+      "logps/rejected": -309.25738525390625,
+      "loss": 0.3378,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2946815490722656,
+      "rewards/margins": 3.1481051445007324,
+      "rewards/rejected": -3.4427871704101562,
+      "step": 2235
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.381173265767623e-05,
+      "logits/chosen": -1.3591539859771729,
+      "logits/rejected": -1.2734744548797607,
+      "logps/chosen": -231.13619995117188,
+      "logps/rejected": -265.5144958496094,
+      "loss": 0.4036,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.3812045454978943,
+      "rewards/margins": 3.0088858604431152,
+      "rewards/rejected": -3.3900904655456543,
+      "step": 2240
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3692928724612203e-05,
+      "logits/chosen": -1.4619818925857544,
+      "logits/rejected": -1.4295190572738647,
+      "logps/chosen": -249.37570190429688,
+      "logps/rejected": -324.696533203125,
+      "loss": 0.3229,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.17562466859817505,
+      "rewards/margins": 3.0519189834594727,
+      "rewards/rejected": -3.227544069290161,
+      "step": 2245
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.357444494349924e-05,
+      "logits/chosen": -1.4158533811569214,
+      "logits/rejected": -1.3473224639892578,
+      "logps/chosen": -236.3145751953125,
+      "logps/rejected": -283.5877380371094,
+      "loss": 0.3509,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.2872604727745056,
+      "rewards/margins": 2.891709089279175,
+      "rewards/rejected": -3.178969383239746,
+      "step": 2250
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3456284669124158e-05,
+      "logits/chosen": -1.439143419265747,
+      "logits/rejected": -1.357162356376648,
+      "logps/chosen": -251.6182098388672,
+      "logps/rejected": -325.0864562988281,
+      "loss": 0.388,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.33754006028175354,
+      "rewards/margins": 3.1689419746398926,
+      "rewards/rejected": -3.5064823627471924,
+      "step": 2255
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3338451247113897e-05,
+      "logits/chosen": -1.3981112241744995,
+      "logits/rejected": -1.311858892440796,
+      "logps/chosen": -210.9602508544922,
+      "logps/rejected": -260.03875732421875,
+      "loss": 0.3967,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.4606190621852875,
+      "rewards/margins": 2.981788396835327,
+      "rewards/rejected": -3.4424071311950684,
+      "step": 2260
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3220948013840808e-05,
+      "logits/chosen": -1.3882707357406616,
+      "logits/rejected": -1.3152581453323364,
+      "logps/chosen": -204.07455444335938,
+      "logps/rejected": -260.4723205566406,
+      "loss": 0.3922,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.5220751762390137,
+      "rewards/margins": 2.884295701980591,
+      "rewards/rejected": -3.4063706398010254,
+      "step": 2265
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.310377829632818e-05,
+      "logits/chosen": -1.4722161293029785,
+      "logits/rejected": -1.382716178894043,
+      "logps/chosen": -255.38790893554688,
+      "logps/rejected": -315.30548095703125,
+      "loss": 0.3716,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.427694708108902,
+      "rewards/margins": 3.0956382751464844,
+      "rewards/rejected": -3.5233330726623535,
+      "step": 2270
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.2986945412156038e-05,
+      "logits/chosen": -1.4732109308242798,
+      "logits/rejected": -1.3891632556915283,
+      "logps/chosen": -267.89892578125,
+      "logps/rejected": -326.42291259765625,
+      "loss": 0.3559,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.5320366621017456,
+      "rewards/margins": 3.3837790489196777,
+      "rewards/rejected": -3.915815830230713,
+      "step": 2275
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.2870452669367155e-05,
+      "logits/chosen": -1.3830268383026123,
+      "logits/rejected": -1.2808607816696167,
+      "logps/chosen": -247.59646606445312,
+      "logps/rejected": -309.2234802246094,
+      "loss": 0.3803,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.5355652570724487,
+      "rewards/margins": 2.929332733154297,
+      "rewards/rejected": -3.4648983478546143,
+      "step": 2280
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2754303366373504e-05,
+      "logits/chosen": -1.5191096067428589,
+      "logits/rejected": -1.3283154964447021,
+      "logps/chosen": -246.34066772460938,
+      "logps/rejected": -265.673828125,
+      "loss": 0.3481,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.40870028734207153,
+      "rewards/margins": 3.1531982421875,
+      "rewards/rejected": -3.5618984699249268,
+      "step": 2285
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.263850079186274e-05,
+      "logits/chosen": -1.4764816761016846,
+      "logits/rejected": -1.430755853652954,
+      "logps/chosen": -212.9180145263672,
+      "logps/rejected": -274.1111145019531,
+      "loss": 0.3783,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.45547690987586975,
+      "rewards/margins": 2.7248294353485107,
+      "rewards/rejected": -3.1803061962127686,
+      "step": 2290
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2523048224705186e-05,
+      "logits/chosen": -1.4630491733551025,
+      "logits/rejected": -1.3331501483917236,
+      "logps/chosen": -270.1141052246094,
+      "logps/rejected": -308.35821533203125,
+      "loss": 0.3684,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.3026786148548126,
+      "rewards/margins": 3.3436999320983887,
+      "rewards/rejected": -3.646378755569458,
+      "step": 2295
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2407948933860892e-05,
+      "logits/chosen": -1.496614694595337,
+      "logits/rejected": -1.4215686321258545,
+      "logps/chosen": -235.4622039794922,
+      "logps/rejected": -290.6049499511719,
+      "loss": 0.4294,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.6504599452018738,
+      "rewards/margins": 2.79489803314209,
+      "rewards/rejected": -3.4453582763671875,
+      "step": 2300
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2293206178287184e-05,
+      "logits/chosen": -1.3421580791473389,
+      "logits/rejected": -1.217355489730835,
+      "logps/chosen": -213.7441864013672,
+      "logps/rejected": -239.513671875,
+      "loss": 0.3678,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.2297484427690506,
+      "rewards/margins": 2.904219150543213,
+      "rewards/rejected": -3.133967399597168,
+      "step": 2305
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2178823206846302e-05,
+      "logits/chosen": -1.4291033744812012,
+      "logits/rejected": -1.324205994606018,
+      "logps/chosen": -240.22793579101562,
+      "logps/rejected": -290.51068115234375,
+      "loss": 0.4002,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.3026602566242218,
+      "rewards/margins": 3.2725844383239746,
+      "rewards/rejected": -3.575244903564453,
+      "step": 2310
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.2064803258213476e-05,
+      "logits/chosen": -1.4654659032821655,
+      "logits/rejected": -1.4113140106201172,
+      "logps/chosen": -231.51815795898438,
+      "logps/rejected": -281.7957458496094,
+      "loss": 0.4204,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.29831621050834656,
+      "rewards/margins": 2.721684694290161,
+      "rewards/rejected": -3.02000093460083,
+      "step": 2315
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1951149560785167e-05,
+      "logits/chosen": -1.4634226560592651,
+      "logits/rejected": -1.3689024448394775,
+      "logps/chosen": -229.8677215576172,
+      "logps/rejected": -284.0724792480469,
+      "loss": 0.3761,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.2706076502799988,
+      "rewards/margins": 3.476905345916748,
+      "rewards/rejected": -3.7475128173828125,
+      "step": 2320
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.18378653325877e-05,
+      "logits/chosen": -1.4571824073791504,
+      "logits/rejected": -1.3364005088806152,
+      "logps/chosen": -261.83563232421875,
+      "logps/rejected": -289.01141357421875,
+      "loss": 0.3223,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2980685830116272,
+      "rewards/margins": 3.3560166358947754,
+      "rewards/rejected": -3.654085159301758,
+      "step": 2325
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1724953781186116e-05,
+      "logits/chosen": -1.459835410118103,
+      "logits/rejected": -1.3399070501327515,
+      "logps/chosen": -265.9317626953125,
+      "logps/rejected": -301.6705017089844,
+      "loss": 0.3793,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3601319193840027,
+      "rewards/margins": 2.9270853996276855,
+      "rewards/rejected": -3.287217378616333,
+      "step": 2330
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.16124181035934e-05,
+      "logits/chosen": -1.4324496984481812,
+      "logits/rejected": -1.3354243040084839,
+      "logps/chosen": -213.58486938476562,
+      "logps/rejected": -264.10089111328125,
+      "loss": 0.3614,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.3313239514827728,
+      "rewards/margins": 3.0802438259124756,
+      "rewards/rejected": -3.4115676879882812,
+      "step": 2335
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.15002614861799e-05,
+      "logits/chosen": -1.4642277956008911,
+      "logits/rejected": -1.3924884796142578,
+      "logps/chosen": -223.7937774658203,
+      "logps/rejected": -284.98785400390625,
+      "loss": 0.3665,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.4326489567756653,
+      "rewards/margins": 3.1457138061523438,
+      "rewards/rejected": -3.578362226486206,
+      "step": 2340
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.138848710458314e-05,
+      "logits/chosen": -1.5061982870101929,
+      "logits/rejected": -1.4258203506469727,
+      "logps/chosen": -210.1097869873047,
+      "logps/rejected": -278.6955261230469,
+      "loss": 0.3572,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.2931816875934601,
+      "rewards/margins": 3.354383945465088,
+      "rewards/rejected": -3.6475658416748047,
+      "step": 2345
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1277098123617922e-05,
+      "logits/chosen": -1.4186168909072876,
+      "logits/rejected": -1.2714909315109253,
+      "logps/chosen": -264.96453857421875,
+      "logps/rejected": -298.23052978515625,
+      "loss": 0.382,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5977450013160706,
+      "rewards/margins": 2.8557708263397217,
+      "rewards/rejected": -3.4535155296325684,
+      "step": 2350
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1166097697186654e-05,
+      "logits/chosen": -1.3840543031692505,
+      "logits/rejected": -1.3661469221115112,
+      "logps/chosen": -230.8970947265625,
+      "logps/rejected": -278.3932800292969,
+      "loss": 0.3896,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.44915327429771423,
+      "rewards/margins": 2.8046319484710693,
+      "rewards/rejected": -3.2537853717803955,
+      "step": 2355
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1055488968190145e-05,
+      "logits/chosen": -1.436781644821167,
+      "logits/rejected": -1.30691397190094,
+      "logps/chosen": -255.84805297851562,
+      "logps/rejected": -299.69842529296875,
+      "loss": 0.3509,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.25631824135780334,
+      "rewards/margins": 3.2734789848327637,
+      "rewards/rejected": -3.529797315597534,
+      "step": 2360
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.094527506843849e-05,
+      "logits/chosen": -1.5180524587631226,
+      "logits/rejected": -1.3637323379516602,
+      "logps/chosen": -235.41012573242188,
+      "logps/rejected": -272.5983581542969,
+      "loss": 0.3703,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.5200671553611755,
+      "rewards/margins": 2.5616183280944824,
+      "rewards/rejected": -3.0816853046417236,
+      "step": 2365
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.083545911856253e-05,
+      "logits/chosen": -1.410651683807373,
+      "logits/rejected": -1.3152965307235718,
+      "logps/chosen": -250.3278045654297,
+      "logps/rejected": -304.0352783203125,
+      "loss": 0.3922,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4448067545890808,
+      "rewards/margins": 2.8111159801483154,
+      "rewards/rejected": -3.255922794342041,
+      "step": 2370
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0726044227925381e-05,
+      "logits/chosen": -1.3654406070709229,
+      "logits/rejected": -1.3672550916671753,
+      "logps/chosen": -222.97714233398438,
+      "logps/rejected": -298.3196105957031,
+      "loss": 0.3789,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.3055039942264557,
+      "rewards/margins": 3.0886244773864746,
+      "rewards/rejected": -3.3941283226013184,
+      "step": 2375
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0617033494534486e-05,
+      "logits/chosen": -1.4127416610717773,
+      "logits/rejected": -1.365252137184143,
+      "logps/chosen": -248.2038116455078,
+      "logps/rejected": -304.67987060546875,
+      "loss": 0.399,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.549986720085144,
+      "rewards/margins": 2.9103312492370605,
+      "rewards/rejected": -3.460318088531494,
+      "step": 2380
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0508430004953821e-05,
+      "logits/chosen": -1.4619853496551514,
+      "logits/rejected": -1.3127577304840088,
+      "logps/chosen": -258.4478454589844,
+      "logps/rejected": -284.666748046875,
+      "loss": 0.3749,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4515061378479004,
+      "rewards/margins": 3.1675283908843994,
+      "rewards/rejected": -3.6190345287323,
+      "step": 2385
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0400236834216528e-05,
+      "logits/chosen": -1.4262062311172485,
+      "logits/rejected": -1.3755922317504883,
+      "logps/chosen": -217.9772186279297,
+      "logps/rejected": -275.2790222167969,
+      "loss": 0.3543,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.38236337900161743,
+      "rewards/margins": 2.9082367420196533,
+      "rewards/rejected": -3.290599822998047,
+      "step": 2390
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0292457045737895e-05,
+      "logits/chosen": -1.4641517400741577,
+      "logits/rejected": -1.3777649402618408,
+      "logps/chosen": -244.06765747070312,
+      "logps/rejected": -297.1828308105469,
+      "loss": 0.3964,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5129659175872803,
+      "rewards/margins": 2.698361873626709,
+      "rewards/rejected": -3.2113280296325684,
+      "step": 2395
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0185093691228534e-05,
+      "logits/chosen": -1.4218064546585083,
+      "logits/rejected": -1.3444823026657104,
+      "logps/chosen": -255.1289520263672,
+      "logps/rejected": -323.5635986328125,
+      "loss": 0.3172,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.28663378953933716,
+      "rewards/margins": 3.552114963531494,
+      "rewards/rejected": -3.8387484550476074,
+      "step": 2400
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0078149810608028e-05,
+      "logits/chosen": -1.3803118467330933,
+      "logits/rejected": -1.3066353797912598,
+      "logps/chosen": -256.8506164550781,
+      "logps/rejected": -305.9026794433594,
+      "loss": 0.4005,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.5916983485221863,
+      "rewards/margins": 3.0665206909179688,
+      "rewards/rejected": -3.6582188606262207,
+      "step": 2405
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 9.971628431918845e-06,
+      "logits/chosen": -1.4089895486831665,
+      "logits/rejected": -1.327468752861023,
+      "logps/chosen": -245.14462280273438,
+      "logps/rejected": -303.9306945800781,
+      "loss": 0.3771,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.32407158613204956,
+      "rewards/margins": 3.180250883102417,
+      "rewards/rejected": -3.5043225288391113,
+      "step": 2410
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.865532571240615e-06,
+      "logits/chosen": -1.3610029220581055,
+      "logits/rejected": -1.289146900177002,
+      "logps/chosen": -259.354248046875,
+      "logps/rejected": -309.7987060546875,
+      "loss": 0.326,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.3622656464576721,
+      "rewards/margins": 3.3705601692199707,
+      "rewards/rejected": -3.732825756072998,
+      "step": 2415
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.759865232604692e-06,
+      "logits/chosen": -1.433051347732544,
+      "logits/rejected": -1.3162428140640259,
+      "logps/chosen": -236.21365356445312,
+      "logps/rejected": -268.08502197265625,
+      "loss": 0.389,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.32003462314605713,
+      "rewards/margins": 3.065013885498047,
+      "rewards/rejected": -3.3850486278533936,
+      "step": 2420
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.654629407909163e-06,
+      "logits/chosen": -1.4370791912078857,
+      "logits/rejected": -1.3018367290496826,
+      "logps/chosen": -244.87490844726562,
+      "logps/rejected": -300.98974609375,
+      "loss": 0.3731,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.700079083442688,
+      "rewards/margins": 2.8450570106506348,
+      "rewards/rejected": -3.5451362133026123,
+      "step": 2425
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.549828076834033e-06,
+      "logits/chosen": -1.373586893081665,
+      "logits/rejected": -1.263668179512024,
+      "logps/chosen": -251.5384063720703,
+      "logps/rejected": -301.0669250488281,
+      "loss": 0.3579,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.4735511839389801,
+      "rewards/margins": 3.3861241340637207,
+      "rewards/rejected": -3.859675168991089,
+      "step": 2430
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.44546420675698e-06,
+      "logits/chosen": -1.3819353580474854,
+      "logits/rejected": -1.2966265678405762,
+      "logps/chosen": -228.5697021484375,
+      "logps/rejected": -270.13189697265625,
+      "loss": 0.3729,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.4449302554130554,
+      "rewards/margins": 2.756574869155884,
+      "rewards/rejected": -3.201505184173584,
+      "step": 2435
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.341540752669235e-06,
+      "logits/chosen": -1.4071505069732666,
+      "logits/rejected": -1.3509398698806763,
+      "logps/chosen": -209.3730926513672,
+      "logps/rejected": -246.5635223388672,
+      "loss": 0.3756,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.5815329551696777,
+      "rewards/margins": 2.4732906818389893,
+      "rewards/rejected": -3.054823398590088,
+      "step": 2440
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.238060657091988e-06,
+      "logits/chosen": -1.337214708328247,
+      "logits/rejected": -1.3006173372268677,
+      "logps/chosen": -230.82327270507812,
+      "logps/rejected": -289.1628112792969,
+      "loss": 0.3277,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.547378420829773,
+      "rewards/margins": 3.1115758419036865,
+      "rewards/rejected": -3.658954620361328,
+      "step": 2445
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.135026849992984e-06,
+      "logits/chosen": -1.4462355375289917,
+      "logits/rejected": -1.3543756008148193,
+      "logps/chosen": -229.1552276611328,
+      "logps/rejected": -260.8125915527344,
+      "loss": 0.3562,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.49785342812538147,
+      "rewards/margins": 2.897026538848877,
+      "rewards/rejected": -3.3948798179626465,
+      "step": 2450
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.032442248703666e-06,
+      "logits/chosen": -1.508615493774414,
+      "logits/rejected": -1.3623030185699463,
+      "logps/chosen": -238.5448760986328,
+      "logps/rejected": -265.12078857421875,
+      "loss": 0.3402,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.407143771648407,
+      "rewards/margins": 3.078805446624756,
+      "rewards/rejected": -3.485949754714966,
+      "step": 2455
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.930309757836517e-06,
+      "logits/chosen": -1.4162170886993408,
+      "logits/rejected": -1.3703190088272095,
+      "logps/chosen": -212.07846069335938,
+      "logps/rejected": -300.2138671875,
+      "loss": 0.3221,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.48511773347854614,
+      "rewards/margins": 3.757521867752075,
+      "rewards/rejected": -4.242639064788818,
+      "step": 2460
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.828632269202803e-06,
+      "logits/chosen": -1.5403014421463013,
+      "logits/rejected": -1.3916822671890259,
+      "logps/chosen": -260.9053649902344,
+      "logps/rejected": -289.03033447265625,
+      "loss": 0.3271,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.4829959273338318,
+      "rewards/margins": 2.945845365524292,
+      "rewards/rejected": -3.4288413524627686,
+      "step": 2465
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.727412661730724e-06,
+      "logits/chosen": -1.4161503314971924,
+      "logits/rejected": -1.3081133365631104,
+      "logps/chosen": -225.17373657226562,
+      "logps/rejected": -309.2181091308594,
+      "loss": 0.3697,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.393091082572937,
+      "rewards/margins": 3.462285280227661,
+      "rewards/rejected": -3.8553764820098877,
+      "step": 2470
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.626653801383885e-06,
+      "logits/chosen": -1.3759651184082031,
+      "logits/rejected": -1.3239902257919312,
+      "logps/chosen": -208.36666870117188,
+      "logps/rejected": -270.0406799316406,
+      "loss": 0.3714,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4686359763145447,
+      "rewards/margins": 3.1771905422210693,
+      "rewards/rejected": -3.645826816558838,
+      "step": 2475
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.526358541080173e-06,
+      "logits/chosen": -1.3998143672943115,
+      "logits/rejected": -1.286664366722107,
+      "logps/chosen": -270.0042419433594,
+      "logps/rejected": -293.59649658203125,
+      "loss": 0.376,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.6333631873130798,
+      "rewards/margins": 2.6090807914733887,
+      "rewards/rejected": -3.242443799972534,
+      "step": 2480
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.426529720610934e-06,
+      "logits/chosen": -1.4069288969039917,
+      "logits/rejected": -1.3146326541900635,
+      "logps/chosen": -269.93084716796875,
+      "logps/rejected": -314.04022216796875,
+      "loss": 0.375,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.5928846597671509,
+      "rewards/margins": 3.069899797439575,
+      "rewards/rejected": -3.6627845764160156,
+      "step": 2485
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.327170166560605e-06,
+      "logits/chosen": -1.5114113092422485,
+      "logits/rejected": -1.3770763874053955,
+      "logps/chosen": -245.75100708007812,
+      "logps/rejected": -278.27679443359375,
+      "loss": 0.4151,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.5059057474136353,
+      "rewards/margins": 2.6279234886169434,
+      "rewards/rejected": -3.1338295936584473,
+      "step": 2490
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.228282692226652e-06,
+      "logits/chosen": -1.4327267408370972,
+      "logits/rejected": -1.385161280632019,
+      "logps/chosen": -255.29782104492188,
+      "logps/rejected": -335.4405212402344,
+      "loss": 0.3429,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.5459723472595215,
+      "rewards/margins": 3.3763973712921143,
+      "rewards/rejected": -3.9223697185516357,
+      "step": 2495
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.129870097539951e-06,
+      "logits/chosen": -1.361555814743042,
+      "logits/rejected": -1.3569921255111694,
+      "logps/chosen": -235.18258666992188,
+      "logps/rejected": -307.37359619140625,
+      "loss": 0.3515,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.5130782127380371,
+      "rewards/margins": 3.2528796195983887,
+      "rewards/rejected": -3.7659575939178467,
+      "step": 2500
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.03193516898547e-06,
+      "logits/chosen": -1.401824712753296,
+      "logits/rejected": -1.3137303590774536,
+      "logps/chosen": -265.5319519042969,
+      "logps/rejected": -333.13201904296875,
+      "loss": 0.3844,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.23906204104423523,
+      "rewards/margins": 3.3118507862091064,
+      "rewards/rejected": -3.550912380218506,
+      "step": 2505
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.934480679523395e-06,
+      "logits/chosen": -1.4119746685028076,
+      "logits/rejected": -1.2916350364685059,
+      "logps/chosen": -229.8092041015625,
+      "logps/rejected": -273.4209899902344,
+      "loss": 0.3693,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.36957868933677673,
+      "rewards/margins": 3.204549789428711,
+      "rewards/rejected": -3.5741286277770996,
+      "step": 2510
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.837509388510611e-06,
+      "logits/chosen": -1.4083707332611084,
+      "logits/rejected": -1.2871843576431274,
+      "logps/chosen": -256.2283935546875,
+      "logps/rejected": -289.46746826171875,
+      "loss": 0.3276,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.42285409569740295,
+      "rewards/margins": 3.3278889656066895,
+      "rewards/rejected": -3.7507431507110596,
+      "step": 2515
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.741024041622557e-06,
+      "logits/chosen": -1.4926209449768066,
+      "logits/rejected": -1.3721725940704346,
+      "logps/chosen": -227.1091766357422,
+      "logps/rejected": -244.8163299560547,
+      "loss": 0.3926,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.2877279818058014,
+      "rewards/margins": 2.691335439682007,
+      "rewards/rejected": -2.9790635108947754,
+      "step": 2520
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.645027370775526e-06,
+      "logits/chosen": -1.4789535999298096,
+      "logits/rejected": -1.3919769525527954,
+      "logps/chosen": -239.02194213867188,
+      "logps/rejected": -276.45062255859375,
+      "loss": 0.4188,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.7315130233764648,
+      "rewards/margins": 2.4713168144226074,
+      "rewards/rejected": -3.2028298377990723,
+      "step": 2525
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.54952209404926e-06,
+      "logits/chosen": -1.4852955341339111,
+      "logits/rejected": -1.3242584466934204,
+      "logps/chosen": -252.9635467529297,
+      "logps/rejected": -278.1438293457031,
+      "loss": 0.3786,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.439079612493515,
+      "rewards/margins": 3.1143195629119873,
+      "rewards/rejected": -3.5533993244171143,
+      "step": 2530
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.454510915610019e-06,
+      "logits/chosen": -1.455822229385376,
+      "logits/rejected": -1.3178565502166748,
+      "logps/chosen": -249.8934326171875,
+      "logps/rejected": -284.191650390625,
+      "loss": 0.3985,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.43874090909957886,
+      "rewards/margins": 3.0221173763275146,
+      "rewards/rejected": -3.460858106613159,
+      "step": 2535
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.359996525634011e-06,
+      "logits/chosen": -1.4768798351287842,
+      "logits/rejected": -1.3438224792480469,
+      "logps/chosen": -249.227294921875,
+      "logps/rejected": -284.577392578125,
+      "loss": 0.4065,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.46533918380737305,
+      "rewards/margins": 2.8477797508239746,
+      "rewards/rejected": -3.3131186962127686,
+      "step": 2540
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.265981600231234e-06,
+      "logits/chosen": -1.5325887203216553,
+      "logits/rejected": -1.3816581964492798,
+      "logps/chosen": -246.61160278320312,
+      "logps/rejected": -293.80322265625,
+      "loss": 0.3348,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4386494755744934,
+      "rewards/margins": 3.20097279548645,
+      "rewards/rejected": -3.639622211456299,
+      "step": 2545
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.172468801369669e-06,
+      "logits/chosen": -1.5481603145599365,
+      "logits/rejected": -1.4134643077850342,
+      "logps/chosen": -220.052490234375,
+      "logps/rejected": -261.2530822753906,
+      "loss": 0.3884,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.42949026823043823,
+      "rewards/margins": 2.8239688873291016,
+      "rewards/rejected": -3.2534592151641846,
+      "step": 2550
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.07946077679994e-06,
+      "logits/chosen": -1.5553944110870361,
+      "logits/rejected": -1.4415475130081177,
+      "logps/chosen": -230.6360626220703,
+      "logps/rejected": -284.0323181152344,
+      "loss": 0.355,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.41859936714172363,
+      "rewards/margins": 2.953322410583496,
+      "rewards/rejected": -3.3719215393066406,
+      "step": 2555
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.986960159980327e-06,
+      "logits/chosen": -1.4129726886749268,
+      "logits/rejected": -1.3629024028778076,
+      "logps/chosen": -221.9413604736328,
+      "logps/rejected": -286.68896484375,
+      "loss": 0.4049,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.4513567388057709,
+      "rewards/margins": 3.1536812782287598,
+      "rewards/rejected": -3.6050381660461426,
+      "step": 2560
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.894969570002225e-06,
+      "logits/chosen": -1.4404191970825195,
+      "logits/rejected": -1.3535890579223633,
+      "logps/chosen": -240.3525848388672,
+      "logps/rejected": -292.9701843261719,
+      "loss": 0.4055,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.42974838614463806,
+      "rewards/margins": 2.7643489837646484,
+      "rewards/rejected": -3.1940975189208984,
+      "step": 2565
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.80349161151595e-06,
+      "logits/chosen": -1.4715224504470825,
+      "logits/rejected": -1.40964674949646,
+      "logps/chosen": -262.740234375,
+      "logps/rejected": -326.66021728515625,
+      "loss": 0.337,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.17057926952838898,
+      "rewards/margins": 3.535477876663208,
+      "rewards/rejected": -3.706057071685791,
+      "step": 2570
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.712528874657012e-06,
+      "logits/chosen": -1.4969851970672607,
+      "logits/rejected": -1.3666260242462158,
+      "logps/chosen": -256.3636169433594,
+      "logps/rejected": -294.6187744140625,
+      "loss": 0.4133,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.5324915647506714,
+      "rewards/margins": 2.963696002960205,
+      "rewards/rejected": -3.496187686920166,
+      "step": 2575
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.6220839349727945e-06,
+      "logits/chosen": -1.3161433935165405,
+      "logits/rejected": -1.2051467895507812,
+      "logps/chosen": -269.2625732421875,
+      "logps/rejected": -301.7975769042969,
+      "loss": 0.3369,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5251234769821167,
+      "rewards/margins": 3.5199074745178223,
+      "rewards/rejected": -4.0450310707092285,
+      "step": 2580
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.532159353349582e-06,
+      "logits/chosen": -1.4186230897903442,
+      "logits/rejected": -1.3696180582046509,
+      "logps/chosen": -225.25747680664062,
+      "logps/rejected": -318.4682922363281,
+      "loss": 0.3668,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3813226819038391,
+      "rewards/margins": 2.9579646587371826,
+      "rewards/rejected": -3.339287519454956,
+      "step": 2585
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.442757675940109e-06,
+      "logits/chosen": -1.3839209079742432,
+      "logits/rejected": -1.3220348358154297,
+      "logps/chosen": -251.20068359375,
+      "logps/rejected": -307.3045349121094,
+      "loss": 0.3602,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.4376640319824219,
+      "rewards/margins": 3.113976240158081,
+      "rewards/rejected": -3.551640272140503,
+      "step": 2590
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.353881434091405e-06,
+      "logits/chosen": -1.382564902305603,
+      "logits/rejected": -1.3075412511825562,
+      "logps/chosen": -229.39492797851562,
+      "logps/rejected": -264.2771911621094,
+      "loss": 0.3838,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.4839390814304352,
+      "rewards/margins": 2.7121593952178955,
+      "rewards/rejected": -3.196098804473877,
+      "step": 2595
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.265533144273175e-06,
+      "logits/chosen": -1.4169657230377197,
+      "logits/rejected": -1.3276934623718262,
+      "logps/chosen": -230.9569854736328,
+      "logps/rejected": -289.4246520996094,
+      "loss": 0.3691,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.4061087965965271,
+      "rewards/margins": 3.205444812774658,
+      "rewards/rejected": -3.611553192138672,
+      "step": 2600
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.177715308006505e-06,
+      "logits/chosen": -1.3769886493682861,
+      "logits/rejected": -1.3074685335159302,
+      "logps/chosen": -232.6650848388672,
+      "logps/rejected": -278.31683349609375,
+      "loss": 0.3099,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.3842293322086334,
+      "rewards/margins": 3.2725253105163574,
+      "rewards/rejected": -3.656754732131958,
+      "step": 2605
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.0904304117930825e-06,
+      "logits/chosen": -1.4004212617874146,
+      "logits/rejected": -1.2935984134674072,
+      "logps/chosen": -230.5190887451172,
+      "logps/rejected": -285.92791748046875,
+      "loss": 0.3489,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.3556326627731323,
+      "rewards/margins": 3.4840025901794434,
+      "rewards/rejected": -3.839635133743286,
+      "step": 2610
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.003680927044738e-06,
+      "logits/chosen": -1.403623342514038,
+      "logits/rejected": -1.350563645362854,
+      "logps/chosen": -250.65713500976562,
+      "logps/rejected": -304.454833984375,
+      "loss": 0.3803,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.49298906326293945,
+      "rewards/margins": 2.6846089363098145,
+      "rewards/rejected": -3.177597761154175,
+      "step": 2615
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.91746931001351e-06,
+      "logits/chosen": -1.3959264755249023,
+      "logits/rejected": -1.354936957359314,
+      "logps/chosen": -221.90097045898438,
+      "logps/rejected": -291.7824401855469,
+      "loss": 0.3779,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.4702683389186859,
+      "rewards/margins": 3.11871600151062,
+      "rewards/rejected": -3.588984251022339,
+      "step": 2620
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.831798001722058e-06,
+      "logits/chosen": -1.4298603534698486,
+      "logits/rejected": -1.303333044052124,
+      "logps/chosen": -237.63467407226562,
+      "logps/rejected": -282.2469177246094,
+      "loss": 0.3906,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.33407384157180786,
+      "rewards/margins": 3.3796210289001465,
+      "rewards/rejected": -3.7136950492858887,
+      "step": 2625
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.7466694278946046e-06,
+      "logits/chosen": -1.4614206552505493,
+      "logits/rejected": -1.3570278882980347,
+      "logps/chosen": -243.57266235351562,
+      "logps/rejected": -296.99383544921875,
+      "loss": 0.3934,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.53616863489151,
+      "rewards/margins": 2.9343109130859375,
+      "rewards/rejected": -3.4704794883728027,
+      "step": 2630
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.662085998888214e-06,
+      "logits/chosen": -1.428770661354065,
+      "logits/rejected": -1.3504103422164917,
+      "logps/chosen": -203.597900390625,
+      "logps/rejected": -243.631591796875,
+      "loss": 0.3839,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.4517253041267395,
+      "rewards/margins": 2.85153865814209,
+      "rewards/rejected": -3.3032639026641846,
+      "step": 2635
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.578050109624511e-06,
+      "logits/chosen": -1.4417340755462646,
+      "logits/rejected": -1.3601127862930298,
+      "logps/chosen": -213.7624969482422,
+      "logps/rejected": -274.27764892578125,
+      "loss": 0.3752,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5715819597244263,
+      "rewards/margins": 2.8983328342437744,
+      "rewards/rejected": -3.469914674758911,
+      "step": 2640
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.494564139521957e-06,
+      "logits/chosen": -1.4128557443618774,
+      "logits/rejected": -1.3376586437225342,
+      "logps/chosen": -198.64208984375,
+      "logps/rejected": -243.85733032226562,
+      "loss": 0.388,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4724113345146179,
+      "rewards/margins": 2.727562427520752,
+      "rewards/rejected": -3.1999735832214355,
+      "step": 2645
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.411630452428395e-06,
+      "logits/chosen": -1.4205583333969116,
+      "logits/rejected": -1.3981122970581055,
+      "logps/chosen": -249.38552856445312,
+      "logps/rejected": -318.7808837890625,
+      "loss": 0.398,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.6618324518203735,
+      "rewards/margins": 3.0907139778137207,
+      "rewards/rejected": -3.7525463104248047,
+      "step": 2650
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.329251396554186e-06,
+      "logits/chosen": -1.445494532585144,
+      "logits/rejected": -1.3906385898590088,
+      "logps/chosen": -210.5917510986328,
+      "logps/rejected": -269.33074951171875,
+      "loss": 0.4151,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.5111071467399597,
+      "rewards/margins": 2.8808178901672363,
+      "rewards/rejected": -3.391925096511841,
+      "step": 2655
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.247429304405663e-06,
+      "logits/chosen": -1.371177315711975,
+      "logits/rejected": -1.2566999197006226,
+      "logps/chosen": -267.86383056640625,
+      "logps/rejected": -303.1114807128906,
+      "loss": 0.327,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.42511287331581116,
+      "rewards/margins": 3.470594882965088,
+      "rewards/rejected": -3.895707607269287,
+      "step": 2660
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.166166492719124e-06,
+      "logits/chosen": -1.4190130233764648,
+      "logits/rejected": -1.3397761583328247,
+      "logps/chosen": -231.8660125732422,
+      "logps/rejected": -248.5600128173828,
+      "loss": 0.4795,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5424979329109192,
+      "rewards/margins": 2.146846294403076,
+      "rewards/rejected": -2.6893444061279297,
+      "step": 2665
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.08546526239522e-06,
+      "logits/chosen": -1.3313415050506592,
+      "logits/rejected": -1.2856453657150269,
+      "logps/chosen": -240.83554077148438,
+      "logps/rejected": -311.7351379394531,
+      "loss": 0.3312,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.3891278803348541,
+      "rewards/margins": 3.937739133834839,
+      "rewards/rejected": -4.326866149902344,
+      "step": 2670
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.005327898433826e-06,
+      "logits/chosen": -1.4672349691390991,
+      "logits/rejected": -1.4020793437957764,
+      "logps/chosen": -232.9001007080078,
+      "logps/rejected": -289.7752990722656,
+      "loss": 0.3673,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.4757001996040344,
+      "rewards/margins": 2.6827566623687744,
+      "rewards/rejected": -3.158457040786743,
+      "step": 2675
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.925756669869314e-06,
+      "logits/chosen": -1.439378023147583,
+      "logits/rejected": -1.3500677347183228,
+      "logps/chosen": -211.7573699951172,
+      "logps/rejected": -264.590087890625,
+      "loss": 0.403,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5147947072982788,
+      "rewards/margins": 2.793654203414917,
+      "rewards/rejected": -3.3084492683410645,
+      "step": 2680
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.846753829706321e-06,
+      "logits/chosen": -1.4199397563934326,
+      "logits/rejected": -1.3846681118011475,
+      "logps/chosen": -228.0806427001953,
+      "logps/rejected": -308.934814453125,
+      "loss": 0.368,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.5116747617721558,
+      "rewards/margins": 3.1513829231262207,
+      "rewards/rejected": -3.663057804107666,
+      "step": 2685
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.768321614855972e-06,
+      "logits/chosen": -1.3238885402679443,
+      "logits/rejected": -1.2452843189239502,
+      "logps/chosen": -251.7180938720703,
+      "logps/rejected": -303.04449462890625,
+      "loss": 0.3569,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.6033166646957397,
+      "rewards/margins": 3.017152786254883,
+      "rewards/rejected": -3.620469331741333,
+      "step": 2690
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.690462246072516e-06,
+      "logits/chosen": -1.3670374155044556,
+      "logits/rejected": -1.2038557529449463,
+      "logps/chosen": -240.1848602294922,
+      "logps/rejected": -265.8099060058594,
+      "loss": 0.3334,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3051196336746216,
+      "rewards/margins": 3.5024948120117188,
+      "rewards/rejected": -3.8076140880584717,
+      "step": 2695
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.6131779278904606e-06,
+      "logits/chosen": -1.3956595659255981,
+      "logits/rejected": -1.2981932163238525,
+      "logps/chosen": -211.70315551757812,
+      "logps/rejected": -269.17413330078125,
+      "loss": 0.35,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5337416529655457,
+      "rewards/margins": 3.1707746982574463,
+      "rewards/rejected": -3.704516649246216,
+      "step": 2700
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.536470848562143e-06,
+      "logits/chosen": -1.3783342838287354,
+      "logits/rejected": -1.293217420578003,
+      "logps/chosen": -238.942138671875,
+      "logps/rejected": -291.1051330566406,
+      "loss": 0.374,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.6116763949394226,
+      "rewards/margins": 2.8823437690734863,
+      "rewards/rejected": -3.4940199851989746,
+      "step": 2705
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.460343179995807e-06,
+      "logits/chosen": -1.5143723487854004,
+      "logits/rejected": -1.4304159879684448,
+      "logps/chosen": -204.8470916748047,
+      "logps/rejected": -232.28128051757812,
+      "loss": 0.3996,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.5758059024810791,
+      "rewards/margins": 2.4611544609069824,
+      "rewards/rejected": -3.0369603633880615,
+      "step": 2710
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.384797077694042e-06,
+      "logits/chosen": -1.4032760858535767,
+      "logits/rejected": -1.2930408716201782,
+      "logps/chosen": -235.9485626220703,
+      "logps/rejected": -260.80706787109375,
+      "loss": 0.3845,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4756258428096771,
+      "rewards/margins": 3.060300827026367,
+      "rewards/rejected": -3.5359268188476562,
+      "step": 2715
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.309834680692832e-06,
+      "logits/chosen": -1.4975941181182861,
+      "logits/rejected": -1.3984637260437012,
+      "logps/chosen": -238.25595092773438,
+      "logps/rejected": -294.8500061035156,
+      "loss": 0.3362,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.5681991577148438,
+      "rewards/margins": 3.3456757068634033,
+      "rewards/rejected": -3.9138755798339844,
+      "step": 2720
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.235458111500889e-06,
+      "logits/chosen": -1.347448706626892,
+      "logits/rejected": -1.2785007953643799,
+      "logps/chosen": -220.80105590820312,
+      "logps/rejected": -285.6271667480469,
+      "loss": 0.3648,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.49286168813705444,
+      "rewards/margins": 3.1875343322753906,
+      "rewards/rejected": -3.680396556854248,
+      "step": 2725
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.16166947603967e-06,
+      "logits/chosen": -1.5035779476165771,
+      "logits/rejected": -1.4337027072906494,
+      "logps/chosen": -248.08816528320312,
+      "logps/rejected": -293.638427734375,
+      "loss": 0.3728,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5657131671905518,
+      "rewards/margins": 3.0048439502716064,
+      "rewards/rejected": -3.570557117462158,
+      "step": 2730
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.088470863583655e-06,
+      "logits/chosen": -1.4668022394180298,
+      "logits/rejected": -1.3386309146881104,
+      "logps/chosen": -230.1422882080078,
+      "logps/rejected": -268.1689147949219,
+      "loss": 0.3495,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.4433426260948181,
+      "rewards/margins": 2.858656883239746,
+      "rewards/rejected": -3.30199933052063,
+      "step": 2735
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 4.015864346701251e-06,
+      "logits/chosen": -1.4615294933319092,
+      "logits/rejected": -1.3883612155914307,
+      "logps/chosen": -255.8359375,
+      "logps/rejected": -298.8800048828125,
+      "loss": 0.364,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.34084415435791016,
+      "rewards/margins": 3.069605588912964,
+      "rewards/rejected": -3.410449504852295,
+      "step": 2740
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.943851981196073e-06,
+      "logits/chosen": -1.4294850826263428,
+      "logits/rejected": -1.3411905765533447,
+      "logps/chosen": -243.9447784423828,
+      "logps/rejected": -310.84173583984375,
+      "loss": 0.343,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.6324904561042786,
+      "rewards/margins": 3.342142105102539,
+      "rewards/rejected": -3.9746322631835938,
+      "step": 2745
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.872435806048743e-06,
+      "logits/chosen": -1.4718683958053589,
+      "logits/rejected": -1.4558926820755005,
+      "logps/chosen": -223.1630859375,
+      "logps/rejected": -303.9229736328125,
+      "loss": 0.3797,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7409528493881226,
+      "rewards/margins": 2.7985827922821045,
+      "rewards/rejected": -3.5395359992980957,
+      "step": 2750
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.801617843359187e-06,
+      "logits/chosen": -1.3796002864837646,
+      "logits/rejected": -1.282780647277832,
+      "logps/chosen": -236.2412109375,
+      "logps/rejected": -280.6457214355469,
+      "loss": 0.3928,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.7115376591682434,
+      "rewards/margins": 2.9424045085906982,
+      "rewards/rejected": -3.653942108154297,
+      "step": 2755
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.731400098289331e-06,
+      "logits/chosen": -1.4562902450561523,
+      "logits/rejected": -1.3559983968734741,
+      "logps/chosen": -235.83462524414062,
+      "logps/rejected": -280.1587829589844,
+      "loss": 0.3715,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.6619512438774109,
+      "rewards/margins": 2.6791372299194336,
+      "rewards/rejected": -3.3410885334014893,
+      "step": 2760
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.661784559006362e-06,
+      "logits/chosen": -1.3674700260162354,
+      "logits/rejected": -1.295462727546692,
+      "logps/chosen": -223.27389526367188,
+      "logps/rejected": -271.55645751953125,
+      "loss": 0.4534,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.6604770421981812,
+      "rewards/margins": 2.8196301460266113,
+      "rewards/rejected": -3.480107069015503,
+      "step": 2765
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.592773196626417e-06,
+      "logits/chosen": -1.3982799053192139,
+      "logits/rejected": -1.3020669221878052,
+      "logps/chosen": -238.637939453125,
+      "logps/rejected": -287.42523193359375,
+      "loss": 0.382,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5564891695976257,
+      "rewards/margins": 3.2080490589141846,
+      "rewards/rejected": -3.764538526535034,
+      "step": 2770
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.524367965158798e-06,
+      "logits/chosen": -1.359069585800171,
+      "logits/rejected": -1.2711738348007202,
+      "logps/chosen": -229.8795166015625,
+      "logps/rejected": -286.6170654296875,
+      "loss": 0.3735,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.3981967866420746,
+      "rewards/margins": 3.4926648139953613,
+      "rewards/rejected": -3.8908615112304688,
+      "step": 2775
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.4565708014506066e-06,
+      "logits/chosen": -1.3851430416107178,
+      "logits/rejected": -1.2968069314956665,
+      "logps/chosen": -230.38229370117188,
+      "logps/rejected": -283.18670654296875,
+      "loss": 0.3322,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.47342461347579956,
+      "rewards/margins": 3.4595096111297607,
+      "rewards/rejected": -3.932934522628784,
+      "step": 2780
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.3893836251319422e-06,
+      "logits/chosen": -1.4177032709121704,
+      "logits/rejected": -1.3064008951187134,
+      "logps/chosen": -258.6094665527344,
+      "logps/rejected": -312.0318908691406,
+      "loss": 0.326,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5791738629341125,
+      "rewards/margins": 2.927536725997925,
+      "rewards/rejected": -3.5067107677459717,
+      "step": 2785
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.3228083385615004e-06,
+      "logits/chosen": -1.3896998167037964,
+      "logits/rejected": -1.2878631353378296,
+      "logps/chosen": -228.61428833007812,
+      "logps/rejected": -269.18634033203125,
+      "loss": 0.4131,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3808293342590332,
+      "rewards/margins": 3.2980189323425293,
+      "rewards/rejected": -3.6788482666015625,
+      "step": 2790
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.2568468267727775e-06,
+      "logits/chosen": -1.3433361053466797,
+      "logits/rejected": -1.2808220386505127,
+      "logps/chosen": -236.6678924560547,
+      "logps/rejected": -310.6268310546875,
+      "loss": 0.3305,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.4466523230075836,
+      "rewards/margins": 3.5550129413604736,
+      "rewards/rejected": -4.0016655921936035,
+      "step": 2795
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.1915009574206262e-06,
+      "logits/chosen": -1.4498833417892456,
+      "logits/rejected": -1.3102750778198242,
+      "logps/chosen": -275.7056579589844,
+      "logps/rejected": -311.9723815917969,
+      "loss": 0.4167,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.5615810751914978,
+      "rewards/margins": 2.9476184844970703,
+      "rewards/rejected": -3.509199619293213,
+      "step": 2800
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.126772580728432e-06,
+      "logits/chosen": -1.3818706274032593,
+      "logits/rejected": -1.3033558130264282,
+      "logps/chosen": -221.0478057861328,
+      "logps/rejected": -258.5665588378906,
+      "loss": 0.4022,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.3416077196598053,
+      "rewards/margins": 3.1456706523895264,
+      "rewards/rejected": -3.487278461456299,
+      "step": 2805
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.062663529435686e-06,
+      "logits/chosen": -1.4404270648956299,
+      "logits/rejected": -1.342151403427124,
+      "logps/chosen": -265.16644287109375,
+      "logps/rejected": -311.3492736816406,
+      "loss": 0.3744,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.4786798059940338,
+      "rewards/margins": 3.2908883094787598,
+      "rewards/rejected": -3.7695682048797607,
+      "step": 2810
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.9991756187461e-06,
+      "logits/chosen": -1.493826150894165,
+      "logits/rejected": -1.424478530883789,
+      "logps/chosen": -224.368408203125,
+      "logps/rejected": -282.0445556640625,
+      "loss": 0.3955,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5058669447898865,
+      "rewards/margins": 3.138854503631592,
+      "rewards/rejected": -3.644721508026123,
+      "step": 2815
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.9363106462762386e-06,
+      "logits/chosen": -1.5206154584884644,
+      "logits/rejected": -1.3797047138214111,
+      "logps/chosen": -255.96420288085938,
+      "logps/rejected": -288.0357666015625,
+      "loss": 0.4149,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.5714098215103149,
+      "rewards/margins": 3.050893783569336,
+      "rewards/rejected": -3.6223034858703613,
+      "step": 2820
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.87407039200458e-06,
+      "logits/chosen": -1.4571731090545654,
+      "logits/rejected": -1.4377485513687134,
+      "logps/chosen": -203.18972778320312,
+      "logps/rejected": -278.02984619140625,
+      "loss": 0.4169,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.32160016894340515,
+      "rewards/margins": 2.9917640686035156,
+      "rewards/rejected": -3.313364028930664,
+      "step": 2825
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.812456618221143e-06,
+      "logits/chosen": -1.4474319219589233,
+      "logits/rejected": -1.3620309829711914,
+      "logps/chosen": -237.1829376220703,
+      "logps/rejected": -291.8993225097656,
+      "loss": 0.3575,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.646533727645874,
+      "rewards/margins": 2.8670644760131836,
+      "rewards/rejected": -3.5135979652404785,
+      "step": 2830
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.7514710694775735e-06,
+      "logits/chosen": -1.3768714666366577,
+      "logits/rejected": -1.283097743988037,
+      "logps/chosen": -229.52603149414062,
+      "logps/rejected": -275.2613220214844,
+      "loss": 0.3919,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.4789879322052002,
+      "rewards/margins": 2.905561923980713,
+      "rewards/rejected": -3.384549617767334,
+      "step": 2835
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.691115472537778e-06,
+      "logits/chosen": -1.3752249479293823,
+      "logits/rejected": -1.2862586975097656,
+      "logps/chosen": -259.10321044921875,
+      "logps/rejected": -305.44573974609375,
+      "loss": 0.3621,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.5128841996192932,
+      "rewards/margins": 3.505547285079956,
+      "rewards/rejected": -4.018431186676025,
+      "step": 2840
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.631391536328992e-06,
+      "logits/chosen": -1.507230281829834,
+      "logits/rejected": -1.3867247104644775,
+      "logps/chosen": -262.347900390625,
+      "logps/rejected": -312.17578125,
+      "loss": 0.3645,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.6277137994766235,
+      "rewards/margins": 3.4604015350341797,
+      "rewards/rejected": -4.088115692138672,
+      "step": 2845
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5723009518934136e-06,
+      "logits/chosen": -1.4776110649108887,
+      "logits/rejected": -1.3628318309783936,
+      "logps/chosen": -243.9078369140625,
+      "logps/rejected": -274.2095031738281,
+      "loss": 0.4023,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5240500569343567,
+      "rewards/margins": 2.7778477668762207,
+      "rewards/rejected": -3.3018977642059326,
+      "step": 2850
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.513845392340322e-06,
+      "logits/chosen": -1.328366756439209,
+      "logits/rejected": -1.2408344745635986,
+      "logps/chosen": -244.29013061523438,
+      "logps/rejected": -292.7612609863281,
+      "loss": 0.3595,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.4736739993095398,
+      "rewards/margins": 3.4074509143829346,
+      "rewards/rejected": -3.8811252117156982,
+      "step": 2855
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.4560265127987147e-06,
+      "logits/chosen": -1.4597951173782349,
+      "logits/rejected": -1.3627371788024902,
+      "logps/chosen": -258.7566833496094,
+      "logps/rejected": -299.85479736328125,
+      "loss": 0.3925,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.48914337158203125,
+      "rewards/margins": 2.6332907676696777,
+      "rewards/rejected": -3.122434377670288,
+      "step": 2860
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.3988459503704154e-06,
+      "logits/chosen": -1.3771252632141113,
+      "logits/rejected": -1.3485709428787231,
+      "logps/chosen": -247.46517944335938,
+      "logps/rejected": -313.8783264160156,
+      "loss": 0.3822,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.637241542339325,
+      "rewards/margins": 2.9835617542266846,
+      "rewards/rejected": -3.620803117752075,
+      "step": 2865
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.3423053240837515e-06,
+      "logits/chosen": -1.4605720043182373,
+      "logits/rejected": -1.3656359910964966,
+      "logps/chosen": -230.97634887695312,
+      "logps/rejected": -291.53668212890625,
+      "loss": 0.3689,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.45648589730262756,
+      "rewards/margins": 3.1279473304748535,
+      "rewards/rejected": -3.584432601928711,
+      "step": 2870
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.2864062348476905e-06,
+      "logits/chosen": -1.4365028142929077,
+      "logits/rejected": -1.3548933267593384,
+      "logps/chosen": -243.0567169189453,
+      "logps/rejected": -294.0565490722656,
+      "loss": 0.3536,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3898884356021881,
+      "rewards/margins": 3.077261209487915,
+      "rewards/rejected": -3.4671497344970703,
+      "step": 2875
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.231150265406512e-06,
+      "logits/chosen": -1.4013705253601074,
+      "logits/rejected": -1.2795777320861816,
+      "logps/chosen": -262.1343078613281,
+      "logps/rejected": -303.52899169921875,
+      "loss": 0.3715,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.3743430972099304,
+      "rewards/margins": 3.6337814331054688,
+      "rewards/rejected": -4.008124351501465,
+      "step": 2880
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.176538980295023e-06,
+      "logits/chosen": -1.3362782001495361,
+      "logits/rejected": -1.2446343898773193,
+      "logps/chosen": -208.1090087890625,
+      "logps/rejected": -275.521240234375,
+      "loss": 0.3464,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.22351208329200745,
+      "rewards/margins": 3.3608238697052,
+      "rewards/rejected": -3.5843353271484375,
+      "step": 2885
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.122573925794219e-06,
+      "logits/chosen": -1.3743422031402588,
+      "logits/rejected": -1.3110918998718262,
+      "logps/chosen": -226.4436798095703,
+      "logps/rejected": -284.766845703125,
+      "loss": 0.4132,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.42287105321884155,
+      "rewards/margins": 3.1263039112091064,
+      "rewards/rejected": -3.5491747856140137,
+      "step": 2890
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0692566298875198e-06,
+      "logits/chosen": -1.3773993253707886,
+      "logits/rejected": -1.255491852760315,
+      "logps/chosen": -242.8449249267578,
+      "logps/rejected": -293.3464660644531,
+      "loss": 0.323,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6184684038162231,
+      "rewards/margins": 3.38958740234375,
+      "rewards/rejected": -4.008055686950684,
+      "step": 2895
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.016588602217512e-06,
+      "logits/chosen": -1.4417493343353271,
+      "logits/rejected": -1.3363769054412842,
+      "logps/chosen": -220.3513641357422,
+      "logps/rejected": -261.87921142578125,
+      "loss": 0.3987,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.4997718930244446,
+      "rewards/margins": 2.572354316711426,
+      "rewards/rejected": -3.0721261501312256,
+      "step": 2900
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.9645713340431997e-06,
+      "logits/chosen": -1.3745633363723755,
+      "logits/rejected": -1.3230297565460205,
+      "logps/chosen": -232.82363891601562,
+      "logps/rejected": -307.91510009765625,
+      "loss": 0.3933,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.6310940980911255,
+      "rewards/margins": 3.0805411338806152,
+      "rewards/rejected": -3.711635112762451,
+      "step": 2905
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.9132062981977783e-06,
+      "logits/chosen": -1.3700783252716064,
+      "logits/rejected": -1.2972946166992188,
+      "logps/chosen": -250.9999237060547,
+      "logps/rejected": -316.5565185546875,
+      "loss": 0.3469,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.36076945066452026,
+      "rewards/margins": 3.416379451751709,
+      "rewards/rejected": -3.777149200439453,
+      "step": 2910
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.8624949490469252e-06,
+      "logits/chosen": -1.3321553468704224,
+      "logits/rejected": -1.3144387006759644,
+      "logps/chosen": -226.41629028320312,
+      "logps/rejected": -283.4264221191406,
+      "loss": 0.3523,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.5463991165161133,
+      "rewards/margins": 2.817983388900757,
+      "rewards/rejected": -3.36438250541687,
+      "step": 2915
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.8124387224476347e-06,
+      "logits/chosen": -1.3554986715316772,
+      "logits/rejected": -1.248975396156311,
+      "logps/chosen": -236.5032196044922,
+      "logps/rejected": -301.41937255859375,
+      "loss": 0.3758,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5690854787826538,
+      "rewards/margins": 3.2041728496551514,
+      "rewards/rejected": -3.773258686065674,
+      "step": 2920
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.763039035707556e-06,
+      "logits/chosen": -1.445534586906433,
+      "logits/rejected": -1.3765289783477783,
+      "logps/chosen": -218.98098754882812,
+      "logps/rejected": -282.2618408203125,
+      "loss": 0.3595,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.42225924134254456,
+      "rewards/margins": 3.1719846725463867,
+      "rewards/rejected": -3.5942440032958984,
+      "step": 2925
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.714297287544872e-06,
+      "logits/chosen": -1.4084084033966064,
+      "logits/rejected": -1.3347828388214111,
+      "logps/chosen": -228.32858276367188,
+      "logps/rejected": -264.45947265625,
+      "loss": 0.3807,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.733644425868988,
+      "rewards/margins": 2.8378963470458984,
+      "rewards/rejected": -3.5715413093566895,
+      "step": 2930
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.6662148580486702e-06,
+      "logits/chosen": -1.4024819135665894,
+      "logits/rejected": -1.2530525922775269,
+      "logps/chosen": -274.0736999511719,
+      "logps/rejected": -300.11798095703125,
+      "loss": 0.4,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.44952473044395447,
+      "rewards/margins": 3.0304102897644043,
+      "rewards/rejected": -3.4799346923828125,
+      "step": 2935
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.6187931086398932e-06,
+      "logits/chosen": -1.4648611545562744,
+      "logits/rejected": -1.3795498609542847,
+      "logps/chosen": -244.27194213867188,
+      "logps/rejected": -287.3001403808594,
+      "loss": 0.3857,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.5816971659660339,
+      "rewards/margins": 2.8270397186279297,
+      "rewards/rejected": -3.4087371826171875,
+      "step": 2940
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.5720333820327782e-06,
+      "logits/chosen": -1.437745451927185,
+      "logits/rejected": -1.368643879890442,
+      "logps/chosen": -230.3911590576172,
+      "logps/rejected": -294.9569091796875,
+      "loss": 0.4097,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.6143894195556641,
+      "rewards/margins": 2.9358458518981934,
+      "rewards/rejected": -3.5502357482910156,
+      "step": 2945
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.525937002196845e-06,
+      "logits/chosen": -1.3672488927841187,
+      "logits/rejected": -1.3020068407058716,
+      "logps/chosen": -227.96572875976562,
+      "logps/rejected": -288.87579345703125,
+      "loss": 0.3754,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.48889145255088806,
+      "rewards/margins": 3.2010180950164795,
+      "rewards/rejected": -3.6899094581604004,
+      "step": 2950
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4805052743194048e-06,
+      "logits/chosen": -1.4185682535171509,
+      "logits/rejected": -1.3656527996063232,
+      "logps/chosen": -221.6512451171875,
+      "logps/rejected": -275.33544921875,
+      "loss": 0.4033,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.4141221046447754,
+      "rewards/margins": 2.6198203563690186,
+      "rewards/rejected": -3.033942461013794,
+      "step": 2955
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.435739484768603e-06,
+      "logits/chosen": -1.3836723566055298,
+      "logits/rejected": -1.3007224798202515,
+      "logps/chosen": -233.81982421875,
+      "logps/rejected": -294.37762451171875,
+      "loss": 0.3667,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.6608399152755737,
+      "rewards/margins": 3.0005805492401123,
+      "rewards/rejected": -3.6614208221435547,
+      "step": 2960
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.3916409010569926e-06,
+      "logits/chosen": -1.4528993368148804,
+      "logits/rejected": -1.3190717697143555,
+      "logps/chosen": -251.28640747070312,
+      "logps/rejected": -297.99755859375,
+      "loss": 0.4178,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.6956207156181335,
+      "rewards/margins": 3.153359889984131,
+      "rewards/rejected": -3.848980665206909,
+      "step": 2965
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.348210771805672e-06,
+      "logits/chosen": -1.4740675687789917,
+      "logits/rejected": -1.3801523447036743,
+      "logps/chosen": -266.302490234375,
+      "logps/rejected": -314.86102294921875,
+      "loss": 0.3896,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.6047788858413696,
+      "rewards/margins": 3.4077250957489014,
+      "rewards/rejected": -4.012503623962402,
+      "step": 2970
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.305450326708893e-06,
+      "logits/chosen": -1.5181069374084473,
+      "logits/rejected": -1.4141440391540527,
+      "logps/chosen": -232.6529998779297,
+      "logps/rejected": -269.83807373046875,
+      "loss": 0.3822,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.34738487005233765,
+      "rewards/margins": 2.8622207641601562,
+      "rewards/rejected": -3.2096054553985596,
+      "step": 2975
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2633607764992671e-06,
+      "logits/chosen": -1.4072999954223633,
+      "logits/rejected": -1.311030387878418,
+      "logps/chosen": -240.6744384765625,
+      "logps/rejected": -303.5523986816406,
+      "loss": 0.3021,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.32643240690231323,
+      "rewards/margins": 3.427372694015503,
+      "rewards/rejected": -3.753805160522461,
+      "step": 2980
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2219433129134733e-06,
+      "logits/chosen": -1.4545339345932007,
+      "logits/rejected": -1.372537612915039,
+      "logps/chosen": -249.2332000732422,
+      "logps/rejected": -285.46868896484375,
+      "loss": 0.3641,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5586282014846802,
+      "rewards/margins": 2.956866502761841,
+      "rewards/rejected": -3.5154948234558105,
+      "step": 2985
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1811991086585261e-06,
+      "logits/chosen": -1.421443223953247,
+      "logits/rejected": -1.3431203365325928,
+      "logps/chosen": -231.0830078125,
+      "logps/rejected": -286.212646484375,
+      "loss": 0.3906,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3323056101799011,
+      "rewards/margins": 3.385119915008545,
+      "rewards/rejected": -3.717425584793091,
+      "step": 2990
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1411293173785726e-06,
+      "logits/chosen": -1.5069319009780884,
+      "logits/rejected": -1.420163869857788,
+      "logps/chosen": -239.2240753173828,
+      "logps/rejected": -298.16607666015625,
+      "loss": 0.402,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.31917136907577515,
+      "rewards/margins": 3.190476894378662,
+      "rewards/rejected": -3.509648084640503,
+      "step": 2995
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1017350736221925e-06,
+      "logits/chosen": -1.418235421180725,
+      "logits/rejected": -1.3685309886932373,
+      "logps/chosen": -191.44699096679688,
+      "logps/rejected": -228.60757446289062,
+      "loss": 0.4359,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.48290008306503296,
+      "rewards/margins": 2.216280460357666,
+      "rewards/rejected": -2.6991806030273438,
+      "step": 3000
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0630174928103337e-06,
+      "logits/chosen": -1.4737344980239868,
+      "logits/rejected": -1.3487292528152466,
+      "logps/chosen": -258.76336669921875,
+      "logps/rejected": -320.27337646484375,
+      "loss": 0.4227,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.5768004059791565,
+      "rewards/margins": 3.205977201461792,
+      "rewards/rejected": -3.7827765941619873,
+      "step": 3005
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0249776712046744e-06,
+      "logits/chosen": -1.4237945079803467,
+      "logits/rejected": -1.2867224216461182,
+      "logps/chosen": -246.64108276367188,
+      "logps/rejected": -275.6163024902344,
+      "loss": 0.3447,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.47526636719703674,
+      "rewards/margins": 3.0651516914367676,
+      "rewards/rejected": -3.5404179096221924,
+      "step": 3010
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.876166858766244e-07,
+      "logits/chosen": -1.48415207862854,
+      "logits/rejected": -1.344327688217163,
+      "logps/chosen": -262.2154235839844,
+      "logps/rejected": -287.91632080078125,
+      "loss": 0.3705,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.5292236804962158,
+      "rewards/margins": 2.700580596923828,
+      "rewards/rejected": -3.2298038005828857,
+      "step": 3015
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.509355946767995e-07,
+      "logits/chosen": -1.4063690900802612,
+      "logits/rejected": -1.3571805953979492,
+      "logps/chosen": -265.2889404296875,
+      "logps/rejected": -301.1048889160156,
+      "loss": 0.4011,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.5893287658691406,
+      "rewards/margins": 3.2209103107452393,
+      "rewards/rejected": -3.810239315032959,
+      "step": 3020
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.149354362050805e-07,
+      "logits/chosen": -1.3574830293655396,
+      "logits/rejected": -1.2446348667144775,
+      "logps/chosen": -249.58804321289062,
+      "logps/rejected": -300.3077697753906,
+      "loss": 0.3128,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.2982921898365021,
+      "rewards/margins": 3.3077235221862793,
+      "rewards/rejected": -3.606015682220459,
+      "step": 3025
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.7961722978121e-07,
+      "logits/chosen": -1.4134116172790527,
+      "logits/rejected": -1.3037294149398804,
+      "logps/chosen": -263.79638671875,
+      "logps/rejected": -328.1442565917969,
+      "loss": 0.3121,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.4436197280883789,
+      "rewards/margins": 3.141730546951294,
+      "rewards/rejected": -3.5853500366210938,
+      "step": 3030
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.449819754159316e-07,
+      "logits/chosen": -1.401808738708496,
+      "logits/rejected": -1.296014666557312,
+      "logps/chosen": -275.23028564453125,
+      "logps/rejected": -319.0748291015625,
+      "loss": 0.3704,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7753769159317017,
+      "rewards/margins": 2.9746193885803223,
+      "rewards/rejected": -3.749995708465576,
+      "step": 3035
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.110306537826601e-07,
+      "logits/chosen": -1.4505449533462524,
+      "logits/rejected": -1.3536399602890015,
+      "logps/chosen": -254.4628448486328,
+      "logps/rejected": -301.4260559082031,
+      "loss": 0.3979,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.5519896745681763,
+      "rewards/margins": 2.8951056003570557,
+      "rewards/rejected": -3.4470953941345215,
+      "step": 3040
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.777642261897311e-07,
+      "logits/chosen": -1.3979469537734985,
+      "logits/rejected": -1.2560176849365234,
+      "logps/chosen": -241.8521270751953,
+      "logps/rejected": -274.6918640136719,
+      "loss": 0.3321,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.40183010697364807,
+      "rewards/margins": 3.511676788330078,
+      "rewards/rejected": -3.913506269454956,
+      "step": 3045
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.451836345531787e-07,
+      "logits/chosen": -1.4664833545684814,
+      "logits/rejected": -1.3185244798660278,
+      "logps/chosen": -237.0376739501953,
+      "logps/rejected": -264.0262145996094,
+      "loss": 0.358,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.3076043426990509,
+      "rewards/margins": 3.1689975261688232,
+      "rewards/rejected": -3.476602077484131,
+      "step": 3050
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.13289801370054e-07,
+      "logits/chosen": -1.3925743103027344,
+      "logits/rejected": -1.3529897928237915,
+      "logps/chosen": -203.59584045410156,
+      "logps/rejected": -267.44140625,
+      "loss": 0.363,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5152291059494019,
+      "rewards/margins": 3.0308661460876465,
+      "rewards/rejected": -3.546095371246338,
+      "step": 3055
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.820836296923316e-07,
+      "logits/chosen": -1.4813798666000366,
+      "logits/rejected": -1.4033396244049072,
+      "logps/chosen": -240.37704467773438,
+      "logps/rejected": -286.06890869140625,
+      "loss": 0.3794,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.5136979222297668,
+      "rewards/margins": 2.7911887168884277,
+      "rewards/rejected": -3.30488657951355,
+      "step": 3060
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.515660031013004e-07,
+      "logits/chosen": -1.5199018716812134,
+      "logits/rejected": -1.45121169090271,
+      "logps/chosen": -250.4469757080078,
+      "logps/rejected": -301.24481201171875,
+      "loss": 0.3259,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.4252336025238037,
+      "rewards/margins": 3.3114101886749268,
+      "rewards/rejected": -3.7366433143615723,
+      "step": 3065
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.217377856825885e-07,
+      "logits/chosen": -1.3836722373962402,
+      "logits/rejected": -1.2654017210006714,
+      "logps/chosen": -234.94223022460938,
+      "logps/rejected": -289.21533203125,
+      "loss": 0.3508,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.46201711893081665,
+      "rewards/margins": 3.144443988800049,
+      "rewards/rejected": -3.6064610481262207,
+      "step": 3070
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.925998220016659e-07,
+      "logits/chosen": -1.3546682596206665,
+      "logits/rejected": -1.281021237373352,
+      "logps/chosen": -223.7041473388672,
+      "logps/rejected": -256.8625183105469,
+      "loss": 0.4022,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.43491753935813904,
+      "rewards/margins": 2.5875766277313232,
+      "rewards/rejected": -3.0224945545196533,
+      "step": 3075
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.64152937079948e-07,
+      "logits/chosen": -1.498255968093872,
+      "logits/rejected": -1.425018310546875,
+      "logps/chosen": -248.11636352539062,
+      "logps/rejected": -293.47760009765625,
+      "loss": 0.4036,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.7960094213485718,
+      "rewards/margins": 2.6311516761779785,
+      "rewards/rejected": -3.4271609783172607,
+      "step": 3080
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.363979363714245e-07,
+      "logits/chosen": -1.3672258853912354,
+      "logits/rejected": -1.3166520595550537,
+      "logps/chosen": -257.9898681640625,
+      "logps/rejected": -312.44708251953125,
+      "loss": 0.3768,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6454734206199646,
+      "rewards/margins": 3.113892078399658,
+      "rewards/rejected": -3.7593655586242676,
+      "step": 3085
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.093356057398663e-07,
+      "logits/chosen": -1.509857416152954,
+      "logits/rejected": -1.3743062019348145,
+      "logps/chosen": -264.7552185058594,
+      "logps/rejected": -293.62603759765625,
+      "loss": 0.4094,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.4879273474216461,
+      "rewards/margins": 3.007258892059326,
+      "rewards/rejected": -3.4951863288879395,
+      "step": 3090
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.82966711436561e-07,
+      "logits/chosen": -1.3606380224227905,
+      "logits/rejected": -1.2969977855682373,
+      "logps/chosen": -246.0964813232422,
+      "logps/rejected": -316.4306945800781,
+      "loss": 0.3085,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.5718385577201843,
+      "rewards/margins": 3.2876274585723877,
+      "rewards/rejected": -3.8594658374786377,
+      "step": 3095
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.5729200007862683e-07,
+      "logits/chosen": -1.3616141080856323,
+      "logits/rejected": -1.3222945928573608,
+      "logps/chosen": -234.003662109375,
+      "logps/rejected": -307.3812561035156,
+      "loss": 0.3466,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5820930004119873,
+      "rewards/margins": 3.154688835144043,
+      "rewards/rejected": -3.7367820739746094,
+      "step": 3100
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.323121986278683e-07,
+      "logits/chosen": -1.446487545967102,
+      "logits/rejected": -1.3170316219329834,
+      "logps/chosen": -249.99685668945312,
+      "logps/rejected": -275.61956787109375,
+      "loss": 0.3646,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.3564623296260834,
+      "rewards/margins": 3.4546267986297607,
+      "rewards/rejected": -3.811089038848877,
+      "step": 3105
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.0802801437019033e-07,
+      "logits/chosen": -1.475476861000061,
+      "logits/rejected": -1.3680169582366943,
+      "logps/chosen": -246.2252197265625,
+      "logps/rejected": -293.28692626953125,
+      "loss": 0.3239,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.6484922766685486,
+      "rewards/margins": 2.8538057804107666,
+      "rewards/rejected": -3.5022976398468018,
+      "step": 3110
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.8444013489558337e-07,
+      "logits/chosen": -1.3692257404327393,
+      "logits/rejected": -1.2947582006454468,
+      "logps/chosen": -247.55606079101562,
+      "logps/rejected": -299.1629943847656,
+      "loss": 0.3442,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.43855223059654236,
+      "rewards/margins": 3.4032680988311768,
+      "rewards/rejected": -3.841820478439331,
+      "step": 3115
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.6154922807863643e-07,
+      "logits/chosen": -1.4646461009979248,
+      "logits/rejected": -1.2760677337646484,
+      "logps/chosen": -279.49127197265625,
+      "logps/rejected": -303.4764709472656,
+      "loss": 0.3364,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.5851499438285828,
+      "rewards/margins": 3.501290798187256,
+      "rewards/rejected": -4.086440563201904,
+      "step": 3120
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.393559420596437e-07,
+      "logits/chosen": -1.4126708507537842,
+      "logits/rejected": -1.3040239810943604,
+      "logps/chosen": -244.0888214111328,
+      "logps/rejected": -290.2459411621094,
+      "loss": 0.3495,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.5026829838752747,
+      "rewards/margins": 3.261399507522583,
+      "rewards/rejected": -3.764082431793213,
+      "step": 3125
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.1786090522624156e-07,
+      "logits/chosen": -1.4109928607940674,
+      "logits/rejected": -1.358865737915039,
+      "logps/chosen": -219.6774444580078,
+      "logps/rejected": -284.9616394042969,
+      "loss": 0.3981,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.25217631459236145,
+      "rewards/margins": 2.7937846183776855,
+      "rewards/rejected": -3.0459611415863037,
+      "step": 3130
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.970647261956255e-07,
+      "logits/chosen": -1.4728444814682007,
+      "logits/rejected": -1.3215397596359253,
+      "logps/chosen": -228.3172149658203,
+      "logps/rejected": -245.55490112304688,
+      "loss": 0.3836,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.45830535888671875,
+      "rewards/margins": 2.752584218978882,
+      "rewards/rejected": -3.2108893394470215,
+      "step": 3135
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.769679937973085e-07,
+      "logits/chosen": -1.4454705715179443,
+      "logits/rejected": -1.3466382026672363,
+      "logps/chosen": -237.8345489501953,
+      "logps/rejected": -294.45111083984375,
+      "loss": 0.3784,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5216543078422546,
+      "rewards/margins": 2.708383321762085,
+      "rewards/rejected": -3.230037212371826,
+      "step": 3140
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.575712770564592e-07,
+      "logits/chosen": -1.518226981163025,
+      "logits/rejected": -1.4551050662994385,
+      "logps/chosen": -221.72201538085938,
+      "logps/rejected": -278.15484619140625,
+      "loss": 0.3569,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.3757239282131195,
+      "rewards/margins": 3.098101854324341,
+      "rewards/rejected": -3.473825454711914,
+      "step": 3145
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.3887512517777324e-07,
+      "logits/chosen": -1.3695826530456543,
+      "logits/rejected": -1.2446963787078857,
+      "logps/chosen": -251.01058959960938,
+      "logps/rejected": -304.1330871582031,
+      "loss": 0.3612,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.4472174644470215,
+      "rewards/margins": 3.318666934967041,
+      "rewards/rejected": -3.7658848762512207,
+      "step": 3150
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2088006752994384e-07,
+      "logits/chosen": -1.4280850887298584,
+      "logits/rejected": -1.3461120128631592,
+      "logps/chosen": -250.29443359375,
+      "logps/rejected": -296.2013244628906,
+      "loss": 0.3697,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.6222056150436401,
+      "rewards/margins": 2.967963695526123,
+      "rewards/rejected": -3.5901694297790527,
+      "step": 3155
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0358661363065746e-07,
+      "logits/chosen": -1.5051259994506836,
+      "logits/rejected": -1.4378012418746948,
+      "logps/chosen": -215.5379180908203,
+      "logps/rejected": -270.934814453125,
+      "loss": 0.4377,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.33570045232772827,
+      "rewards/margins": 2.7214572429656982,
+      "rewards/rejected": -3.0571579933166504,
+      "step": 3160
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8699525313217447e-07,
+      "logits/chosen": -1.451647162437439,
+      "logits/rejected": -1.3913623094558716,
+      "logps/chosen": -228.0519561767578,
+      "logps/rejected": -294.03021240234375,
+      "loss": 0.3211,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.4577842652797699,
+      "rewards/margins": 3.234616756439209,
+      "rewards/rejected": -3.6924006938934326,
+      "step": 3165
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7110645580746264e-07,
+      "logits/chosen": -1.564744234085083,
+      "logits/rejected": -1.4929113388061523,
+      "logps/chosen": -236.1385498046875,
+      "logps/rejected": -284.2821044921875,
+      "loss": 0.3587,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.3766639530658722,
+      "rewards/margins": 3.0007431507110596,
+      "rewards/rejected": -3.3774070739746094,
+      "step": 3170
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.559206715368966e-07,
+      "logits/chosen": -1.4131274223327637,
+      "logits/rejected": -1.3240829706192017,
+      "logps/chosen": -235.37606811523438,
+      "logps/rejected": -287.25518798828125,
+      "loss": 0.3694,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.46932369470596313,
+      "rewards/margins": 3.396801710128784,
+      "rewards/rejected": -3.8661255836486816,
+      "step": 3175
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4143833029552355e-07,
+      "logits/chosen": -1.378590703010559,
+      "logits/rejected": -1.3106247186660767,
+      "logps/chosen": -237.6977996826172,
+      "logps/rejected": -297.2106018066406,
+      "loss": 0.3736,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.6389200091362,
+      "rewards/margins": 2.9030025005340576,
+      "rewards/rejected": -3.5419223308563232,
+      "step": 3180
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.276598421408759e-07,
+      "logits/chosen": -1.4562031030654907,
+      "logits/rejected": -1.3900493383407593,
+      "logps/chosen": -219.63406372070312,
+      "logps/rejected": -253.91976928710938,
+      "loss": 0.392,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.5725888013839722,
+      "rewards/margins": 2.4471755027770996,
+      "rewards/rejected": -3.0197644233703613,
+      "step": 3185
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.1458559720137762e-07,
+      "logits/chosen": -1.4543843269348145,
+      "logits/rejected": -1.3984777927398682,
+      "logps/chosen": -240.25765991210938,
+      "logps/rejected": -306.05010986328125,
+      "loss": 0.4252,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.5501489639282227,
+      "rewards/margins": 3.2002170085906982,
+      "rewards/rejected": -3.7503662109375,
+      "step": 3190
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.0221596566528657e-07,
+      "logits/chosen": -1.426992654800415,
+      "logits/rejected": -1.35605788230896,
+      "logps/chosen": -263.15740966796875,
+      "logps/rejected": -306.79266357421875,
+      "loss": 0.3263,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5317724347114563,
+      "rewards/margins": 3.079737901687622,
+      "rewards/rejected": -3.6115100383758545,
+      "step": 3195
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.055129777021665e-08,
+      "logits/chosen": -1.473975658416748,
+      "logits/rejected": -1.3327196836471558,
+      "logps/chosen": -261.496337890625,
+      "logps/rejected": -293.1902160644531,
+      "loss": 0.3396,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.5715179443359375,
+      "rewards/margins": 3.3157131671905518,
+      "rewards/rejected": -3.8872311115264893,
+      "step": 3200
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.959192379322077e-08,
+      "logits/chosen": -1.4569588899612427,
+      "logits/rejected": -1.3854528665542603,
+      "logps/chosen": -247.93746948242188,
+      "logps/rejected": -321.257080078125,
+      "loss": 0.3443,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.5199737548828125,
+      "rewards/margins": 3.264164686203003,
+      "rewards/rejected": -3.7841384410858154,
+      "step": 3205
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.933815404144561e-08,
+      "logits/chosen": -1.5174143314361572,
+      "logits/rejected": -1.4582128524780273,
+      "logps/chosen": -221.869384765625,
+      "logps/rejected": -289.20001220703125,
+      "loss": 0.3538,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.4151820242404938,
+      "rewards/margins": 2.987104892730713,
+      "rewards/rejected": -3.402287006378174,
+      "step": 3210
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 5.979027884332744e-08,
+      "logits/chosen": -1.4034459590911865,
+      "logits/rejected": -1.2570650577545166,
+      "logps/chosen": -261.0184631347656,
+      "logps/rejected": -288.6986999511719,
+      "loss": 0.3289,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.40868058800697327,
+      "rewards/margins": 3.1276040077209473,
+      "rewards/rejected": -3.5362846851348877,
+      "step": 3215
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 5.094856854039043e-08,
+      "logits/chosen": -1.525322675704956,
+      "logits/rejected": -1.3608064651489258,
+      "logps/chosen": -253.0553436279297,
+      "logps/rejected": -286.7688903808594,
+      "loss": 0.3748,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.5386615991592407,
+      "rewards/margins": 3.1457836627960205,
+      "rewards/rejected": -3.6844451427459717,
+      "step": 3220
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.281327347958608e-08,
+      "logits/chosen": -1.3911330699920654,
+      "logits/rejected": -1.3263561725616455,
+      "logps/chosen": -251.658447265625,
+      "logps/rejected": -292.4714660644531,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.48612624406814575,
+      "rewards/margins": 3.019296884536743,
+      "rewards/rejected": -3.505422592163086,
+      "step": 3225
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.5384624006201686e-08,
+      "logits/chosen": -1.4651706218719482,
+      "logits/rejected": -1.3821120262145996,
+      "logps/chosen": -231.82302856445312,
+      "logps/rejected": -290.69891357421875,
+      "loss": 0.365,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.6108437776565552,
+      "rewards/margins": 3.042966365814209,
+      "rewards/rejected": -3.6538097858428955,
+      "step": 3230
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.866283045734053e-08,
+      "logits/chosen": -1.4114316701889038,
+      "logits/rejected": -1.3266972303390503,
+      "logps/chosen": -236.2187957763672,
+      "logps/rejected": -279.8185119628906,
+      "loss": 0.4135,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.6854463815689087,
+      "rewards/margins": 2.7222111225128174,
+      "rewards/rejected": -3.4076576232910156,
+      "step": 3235
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.264808315596556e-08,
+      "logits/chosen": -1.443969964981079,
+      "logits/rejected": -1.3723801374435425,
+      "logps/chosen": -222.7816925048828,
+      "logps/rejected": -301.02911376953125,
+      "loss": 0.3736,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5945747494697571,
+      "rewards/margins": 3.3723983764648438,
+      "rewards/rejected": -3.966973066329956,
+      "step": 3240
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.73405524055148e-08,
+      "logits/chosen": -1.4318532943725586,
+      "logits/rejected": -1.289945125579834,
+      "logps/chosen": -234.6119384765625,
+      "logps/rejected": -275.46038818359375,
+      "loss": 0.3873,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.6027265191078186,
+      "rewards/margins": 2.830317258834839,
+      "rewards/rejected": -3.4330437183380127,
+      "step": 3245
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.2740388485071863e-08,
+      "logits/chosen": -1.328491449356079,
+      "logits/rejected": -1.2337000370025635,
+      "logps/chosen": -250.76171875,
+      "logps/rejected": -305.5037841796875,
+      "loss": 0.3748,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.47596779465675354,
+      "rewards/margins": 3.1194262504577637,
+      "rewards/rejected": -3.595393657684326,
+      "step": 3250
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.847721645116603e-09,
+      "logits/chosen": -1.3881988525390625,
+      "logits/rejected": -1.2779829502105713,
+      "logps/chosen": -247.0287322998047,
+      "logps/rejected": -300.8529052734375,
+      "loss": 0.3475,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.5738905668258667,
+      "rewards/margins": 3.4033915996551514,
+      "rewards/rejected": -3.9772822856903076,
+      "step": 3255
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.662662103833594e-09,
+      "logits/chosen": -1.4306684732437134,
+      "logits/rejected": -1.3507243394851685,
+      "logps/chosen": -231.3989715576172,
+      "logps/rejected": -270.3944396972656,
+      "loss": 0.3358,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.49925675988197327,
+      "rewards/margins": 3.0653929710388184,
+      "rewards/rejected": -3.5646495819091797,
+      "step": 3260
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.1853000439951987e-09,
+      "logits/chosen": -1.4770541191101074,
+      "logits/rejected": -1.361383318901062,
+      "logps/chosen": -235.0412139892578,
+      "logps/rejected": -268.224609375,
+      "loss": 0.3657,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5067777633666992,
+      "rewards/margins": 2.797917127609253,
+      "rewards/rejected": -3.304694652557373,
+      "step": 3265
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.4157056104052713e-09,
+      "logits/chosen": -1.4269344806671143,
+      "logits/rejected": -1.369800329208374,
+      "logps/chosen": -228.0611114501953,
+      "logps/rejected": -286.09051513671875,
+      "loss": 0.3807,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.6630217432975769,
+      "rewards/margins": 3.0376479625701904,
+      "rewards/rejected": -3.700669765472412,
+      "step": 3270
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.5392890791463574e-10,
+      "logits/chosen": -1.4762697219848633,
+      "logits/rejected": -1.370078444480896,
+      "logps/chosen": -224.59619140625,
+      "logps/rejected": -275.50189208984375,
+      "loss": 0.3731,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.5560105443000793,
+      "rewards/margins": 3.0750603675842285,
+      "rewards/rejected": -3.631071090698242,
+      "step": 3275
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 0.0,
+      "logits/chosen": -1.5287964344024658,
+      "logits/rejected": -1.4150562286376953,
+      "logps/chosen": -244.9498291015625,
+      "logps/rejected": -282.9684753417969,
+      "loss": 0.3593,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5622932314872742,
+      "rewards/margins": 2.9687438011169434,
+      "rewards/rejected": -3.531036853790283,
+      "step": 3280
+    },
+    {
+      "epoch": 1.0,
+      "eval_logits/chosen": -0.7773212790489197,
+      "eval_logits/rejected": -0.7749085426330566,
+      "eval_logps/chosen": -275.3572082519531,
+      "eval_logps/rejected": -324.0383605957031,
+      "eval_loss": 0.12900209426879883,
+      "eval_rewards/accuracies": 0.8596742749214172,
+      "eval_rewards/chosen": -0.17991267144680023,
+      "eval_rewards/margins": 5.889674186706543,
+      "eval_rewards/rejected": -6.069586277008057,
+      "eval_runtime": 76185.8679,
+      "eval_samples_per_second": 2.598,
+      "eval_steps_per_second": 1.299,
+      "step": 3280
+    },
+    {
+      "epoch": 1.0,
+      "step": 3280,
+      "total_flos": 0.0,
+      "train_loss": 0.3931771684165408,
+      "train_runtime": 248473.0607,
+      "train_samples_per_second": 0.845,
+      "train_steps_per_second": 0.013
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 3280,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}