diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,4871 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 100,
+  "global_step": 2902,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0006891798759476223,
+      "grad_norm": 1.1716080904006958,
+      "learning_rate": 1.718213058419244e-10,
+      "logits/chosen": -3.184086799621582,
+      "logits/rejected": -3.1319174766540527,
+      "logps/chosen": -49.95408630371094,
+      "logps/rejected": -44.33523178100586,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.006891798759476223,
+      "grad_norm": 1.0663460493087769,
+      "learning_rate": 1.718213058419244e-09,
+      "logits/chosen": -3.080113172531128,
+      "logits/rejected": -3.0596792697906494,
+      "logps/chosen": -54.03813171386719,
+      "logps/rejected": -53.65137481689453,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4565972089767456,
+      "rewards/chosen": 8.68273782543838e-05,
+      "rewards/margins": -1.9125265680486336e-05,
+      "rewards/rejected": 0.00010595263302093372,
+      "step": 10
+    },
+    {
+      "epoch": 0.013783597518952447,
+      "grad_norm": 1.1690140962600708,
+      "learning_rate": 3.436426116838488e-09,
+      "logits/chosen": -3.1165332794189453,
+      "logits/rejected": -3.0916168689727783,
+      "logps/chosen": -55.888938903808594,
+      "logps/rejected": -53.246864318847656,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.503125011920929,
+      "rewards/chosen": 3.240557634853758e-05,
+      "rewards/margins": -3.6290578009356977e-06,
+      "rewards/rejected": 3.603463846957311e-05,
+      "step": 20
+    },
+    {
+      "epoch": 0.02067539627842867,
+      "grad_norm": 1.2955037355422974,
+      "learning_rate": 5.154639175257731e-09,
+      "logits/chosen": -3.0878665447235107,
+      "logits/rejected": -3.058804988861084,
+      "logps/chosen": -54.54620361328125,
+      "logps/rejected": -52.591636657714844,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": -2.4173205019906163e-05,
+      "rewards/margins": -4.490778155741282e-05,
+      "rewards/rejected": 2.0734580175485462e-05,
+      "step": 30
+    },
+    {
+      "epoch": 0.027567195037904894,
+      "grad_norm": 1.1852333545684814,
+      "learning_rate": 6.872852233676976e-09,
+      "logits/chosen": -3.0849013328552246,
+      "logits/rejected": -3.0671732425689697,
+      "logps/chosen": -53.879005432128906,
+      "logps/rejected": -53.66566848754883,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.528124988079071,
+      "rewards/chosen": -5.587830673903227e-05,
+      "rewards/margins": 4.417077434482053e-05,
+      "rewards/rejected": -0.00010004905925597996,
+      "step": 40
+    },
+    {
+      "epoch": 0.03445899379738112,
+      "grad_norm": 1.2431070804595947,
+      "learning_rate": 8.59106529209622e-09,
+      "logits/chosen": -3.0804286003112793,
+      "logits/rejected": -3.0561296939849854,
+      "logps/chosen": -56.24019241333008,
+      "logps/rejected": -53.092872619628906,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5218750238418579,
+      "rewards/chosen": 8.934068318922073e-05,
+      "rewards/margins": 0.00011236695718253031,
+      "rewards/rejected": -2.302624488947913e-05,
+      "step": 50
+    },
+    {
+      "epoch": 0.04135079255685734,
+      "grad_norm": 1.1313049793243408,
+      "learning_rate": 1.0309278350515463e-08,
+      "logits/chosen": -3.0351052284240723,
+      "logits/rejected": -3.0099387168884277,
+      "logps/chosen": -52.579429626464844,
+      "logps/rejected": -52.6761589050293,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.48906248807907104,
+      "rewards/chosen": 2.3904693080112338e-05,
+      "rewards/margins": 2.025809772021603e-05,
+      "rewards/rejected": 3.6465789889916778e-06,
+      "step": 60
+    },
+    {
+      "epoch": 0.048242591316333565,
+      "grad_norm": 1.2357141971588135,
+      "learning_rate": 1.2027491408934707e-08,
+      "logits/chosen": -3.092390537261963,
+      "logits/rejected": -3.0711493492126465,
+      "logps/chosen": -54.469940185546875,
+      "logps/rejected": -53.86017990112305,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 5.560473709920188e-06,
+      "rewards/margins": 2.101451354974415e-05,
+      "rewards/rejected": -1.5454041204066016e-05,
+      "step": 70
+    },
+    {
+      "epoch": 0.05513439007580979,
+      "grad_norm": 1.1142845153808594,
+      "learning_rate": 1.3745704467353952e-08,
+      "logits/chosen": -3.0345962047576904,
+      "logits/rejected": -3.0208940505981445,
+      "logps/chosen": -54.06622314453125,
+      "logps/rejected": -52.69053268432617,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.48906248807907104,
+      "rewards/chosen": -4.671530405175872e-05,
+      "rewards/margins": -4.488803824642673e-05,
+      "rewards/rejected": -1.827271603360714e-06,
+      "step": 80
+    },
+    {
+      "epoch": 0.06202618883528601,
+      "grad_norm": 1.1985735893249512,
+      "learning_rate": 1.5463917525773195e-08,
+      "logits/chosen": -3.048698663711548,
+      "logits/rejected": -3.0217783451080322,
+      "logps/chosen": -54.59540939331055,
+      "logps/rejected": -52.060035705566406,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4906249940395355,
+      "rewards/chosen": -6.244768155738711e-05,
+      "rewards/margins": -3.21494007948786e-06,
+      "rewards/rejected": -5.923274511587806e-05,
+      "step": 90
+    },
+    {
+      "epoch": 0.06891798759476224,
+      "grad_norm": 1.3350454568862915,
+      "learning_rate": 1.718213058419244e-08,
+      "logits/chosen": -3.119621753692627,
+      "logits/rejected": -3.095787763595581,
+      "logps/chosen": -53.65461349487305,
+      "logps/rejected": -52.88787841796875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": -6.203976226970553e-05,
+      "rewards/margins": 5.055965812061913e-05,
+      "rewards/rejected": -0.00011259941675234586,
+      "step": 100
+    },
+    {
+      "epoch": 0.06891798759476224,
+      "eval_logits/chosen": -3.163339376449585,
+      "eval_logits/rejected": -3.157687187194824,
+      "eval_logps/chosen": -58.7006721496582,
+      "eval_logps/rejected": -63.17026138305664,
+      "eval_loss": 0.693140983581543,
+      "eval_rewards/accuracies": 0.5023234486579895,
+      "eval_rewards/chosen": 0.00011220378655707464,
+      "eval_rewards/margins": 1.3582017345470376e-05,
+      "eval_rewards/rejected": 9.862175647867844e-05,
+      "eval_runtime": 383.3503,
+      "eval_samples_per_second": 11.227,
+      "eval_steps_per_second": 1.403,
+      "step": 100
+    },
+    {
+      "epoch": 0.07580978635423846,
+      "grad_norm": 1.2324384450912476,
+      "learning_rate": 1.8900343642611684e-08,
+      "logits/chosen": -3.0891432762145996,
+      "logits/rejected": -3.0738348960876465,
+      "logps/chosen": -53.08173751831055,
+      "logps/rejected": -54.20978546142578,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5093749761581421,
+      "rewards/chosen": -4.9080466851592064e-05,
+      "rewards/margins": -7.88484321674332e-05,
+      "rewards/rejected": 2.976796167786233e-05,
+      "step": 110
+    },
+    {
+      "epoch": 0.08270158511371468,
+      "grad_norm": 1.2855055332183838,
+      "learning_rate": 2.0618556701030925e-08,
+      "logits/chosen": -3.043365478515625,
+      "logits/rejected": -3.0211169719696045,
+      "logps/chosen": -54.957427978515625,
+      "logps/rejected": -54.4825439453125,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.47187501192092896,
+      "rewards/chosen": -7.188355084508657e-05,
+      "rewards/margins": -0.00011629929940681905,
+      "rewards/rejected": 4.441575947566889e-05,
+      "step": 120
+    },
+    {
+      "epoch": 0.08959338387319091,
+      "grad_norm": 1.1282892227172852,
+      "learning_rate": 2.2336769759450173e-08,
+      "logits/chosen": -3.0101354122161865,
+      "logits/rejected": -2.9788012504577637,
+      "logps/chosen": -57.5596923828125,
+      "logps/rejected": -51.651153564453125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 2.1165338694117963e-05,
+      "rewards/margins": 0.00015237969637382776,
+      "rewards/rejected": -0.00013121434312779456,
+      "step": 130
+    },
+    {
+      "epoch": 0.09648518263266713,
+      "grad_norm": 1.1657721996307373,
+      "learning_rate": 2.4054982817869415e-08,
+      "logits/chosen": -3.067199468612671,
+      "logits/rejected": -3.046125888824463,
+      "logps/chosen": -53.55717849731445,
+      "logps/rejected": -52.773223876953125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": 1.3996473171573598e-05,
+      "rewards/margins": 0.00015452780644409359,
+      "rewards/rejected": -0.00014053132326807827,
+      "step": 140
+    },
+    {
+      "epoch": 0.10337698139214335,
+      "grad_norm": 1.2658566236495972,
+      "learning_rate": 2.5773195876288656e-08,
+      "logits/chosen": -3.04317569732666,
+      "logits/rejected": -3.0280072689056396,
+      "logps/chosen": -52.809234619140625,
+      "logps/rejected": -54.64301300048828,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.503125011920929,
+      "rewards/chosen": -1.0054915037471801e-05,
+      "rewards/margins": 0.0001238631666637957,
+      "rewards/rejected": -0.00013391808897722512,
+      "step": 150
+    },
+    {
+      "epoch": 0.11026878015161957,
+      "grad_norm": 1.2126415967941284,
+      "learning_rate": 2.7491408934707904e-08,
+      "logits/chosen": -3.09346342086792,
+      "logits/rejected": -3.07668399810791,
+      "logps/chosen": -53.59107208251953,
+      "logps/rejected": -52.9258918762207,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4703125059604645,
+      "rewards/chosen": -9.462583875574637e-06,
+      "rewards/margins": -2.1159441530471668e-05,
+      "rewards/rejected": 1.1696849469444714e-05,
+      "step": 160
+    },
+    {
+      "epoch": 0.1171605789110958,
+      "grad_norm": 1.1890392303466797,
+      "learning_rate": 2.9209621993127148e-08,
+      "logits/chosen": -3.0306668281555176,
+      "logits/rejected": -3.0220158100128174,
+      "logps/chosen": -53.26588821411133,
+      "logps/rejected": -53.87241744995117,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -5.447790681500919e-05,
+      "rewards/margins": -7.718646884313785e-06,
+      "rewards/rejected": -4.675926174968481e-05,
+      "step": 170
+    },
+    {
+      "epoch": 0.12405237767057202,
+      "grad_norm": 1.15412175655365,
+      "learning_rate": 3.092783505154639e-08,
+      "logits/chosen": -3.0752334594726562,
+      "logits/rejected": -3.0524303913116455,
+      "logps/chosen": -55.69530487060547,
+      "logps/rejected": -53.15666961669922,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.504687488079071,
+      "rewards/chosen": -1.1674828783725388e-05,
+      "rewards/margins": 9.358397619507741e-06,
+      "rewards/rejected": -2.1033218217780814e-05,
+      "step": 180
+    },
+    {
+      "epoch": 0.13094417643004824,
+      "grad_norm": 1.1720036268234253,
+      "learning_rate": 3.264604810996564e-08,
+      "logits/chosen": -3.1030337810516357,
+      "logits/rejected": -3.0736050605773926,
+      "logps/chosen": -55.423614501953125,
+      "logps/rejected": -52.4505500793457,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": 9.101578143599909e-06,
+      "rewards/margins": 0.0001561685057822615,
+      "rewards/rejected": -0.00014706689398735762,
+      "step": 190
+    },
+    {
+      "epoch": 0.13783597518952448,
+      "grad_norm": 1.2227604389190674,
+      "learning_rate": 3.436426116838488e-08,
+      "logits/chosen": -3.0704421997070312,
+      "logits/rejected": -3.041954278945923,
+      "logps/chosen": -53.747833251953125,
+      "logps/rejected": -52.85246658325195,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": -6.959711026865989e-05,
+      "rewards/margins": 9.592306014383212e-05,
+      "rewards/rejected": -0.0001655201631365344,
+      "step": 200
+    },
+    {
+      "epoch": 0.13783597518952448,
+      "eval_logits/chosen": -3.1631689071655273,
+      "eval_logits/rejected": -3.157501220703125,
+      "eval_logps/chosen": -58.700950622558594,
+      "eval_logps/rejected": -63.162139892578125,
+      "eval_loss": 0.6931830048561096,
+      "eval_rewards/accuracies": 0.48745352029800415,
+      "eval_rewards/chosen": 0.00010945786925731227,
+      "eval_rewards/margins": -7.042505603749305e-05,
+      "eval_rewards/rejected": 0.00017988293257076293,
+      "eval_runtime": 383.3981,
+      "eval_samples_per_second": 11.226,
+      "eval_steps_per_second": 1.403,
+      "step": 200
+    },
+    {
+      "epoch": 0.1447277739490007,
+      "grad_norm": 1.1424545049667358,
+      "learning_rate": 3.608247422680412e-08,
+      "logits/chosen": -3.08945631980896,
+      "logits/rejected": -3.0655088424682617,
+      "logps/chosen": -54.22871780395508,
+      "logps/rejected": -52.478431701660156,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.00013771439262200147,
+      "rewards/margins": 0.00012528176011983305,
+      "rewards/rejected": -0.00026299612363800406,
+      "step": 210
+    },
+    {
+      "epoch": 0.15161957270847692,
+      "grad_norm": 1.1047999858856201,
+      "learning_rate": 3.780068728522337e-08,
+      "logits/chosen": -3.0537705421447754,
+      "logits/rejected": -3.039431571960449,
+      "logps/chosen": -51.688323974609375,
+      "logps/rejected": -53.095741271972656,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.528124988079071,
+      "rewards/chosen": -0.00010369622759753838,
+      "rewards/margins": 0.00015849454212002456,
+      "rewards/rejected": -0.00026219076244160533,
+      "step": 220
+    },
+    {
+      "epoch": 0.15851137146795313,
+      "grad_norm": 1.2490479946136475,
+      "learning_rate": 3.951890034364261e-08,
+      "logits/chosen": -3.071945905685425,
+      "logits/rejected": -3.0471181869506836,
+      "logps/chosen": -54.49678421020508,
+      "logps/rejected": -52.037872314453125,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -3.5270270018372685e-05,
+      "rewards/margins": 0.0002025824796874076,
+      "rewards/rejected": -0.0002378527569817379,
+      "step": 230
+    },
+    {
+      "epoch": 0.16540317022742937,
+      "grad_norm": 1.141684889793396,
+      "learning_rate": 4.123711340206185e-08,
+      "logits/chosen": -3.028677225112915,
+      "logits/rejected": -3.0117344856262207,
+      "logps/chosen": -54.690513610839844,
+      "logps/rejected": -55.188621520996094,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5296875238418579,
+      "rewards/chosen": -0.00015048097702674568,
+      "rewards/margins": 0.00021787775040138513,
+      "rewards/rejected": -0.00036835874198004603,
+      "step": 240
+    },
+    {
+      "epoch": 0.17229496898690558,
+      "grad_norm": 1.171937108039856,
+      "learning_rate": 4.295532646048109e-08,
+      "logits/chosen": -3.06539249420166,
+      "logits/rejected": -3.0387420654296875,
+      "logps/chosen": -57.0573616027832,
+      "logps/rejected": -52.94896697998047,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5296875238418579,
+      "rewards/chosen": -0.0001992958423215896,
+      "rewards/margins": 0.00017509344615973532,
+      "rewards/rejected": -0.0003743892884813249,
+      "step": 250
+    },
+    {
+      "epoch": 0.17918676774638181,
+      "grad_norm": 1.1496978998184204,
+      "learning_rate": 4.4673539518900346e-08,
+      "logits/chosen": -3.0649943351745605,
+      "logits/rejected": -3.0493435859680176,
+      "logps/chosen": -54.52451705932617,
+      "logps/rejected": -54.94301223754883,
+      "loss": 0.693,
+      "rewards/accuracies": 0.535937488079071,
+      "rewards/chosen": -0.00030117519781924784,
+      "rewards/margins": 0.00021073469542898238,
+      "rewards/rejected": -0.0005119099514558911,
+      "step": 260
+    },
+    {
+      "epoch": 0.18607856650585802,
+      "grad_norm": 1.1325643062591553,
+      "learning_rate": 4.639175257731959e-08,
+      "logits/chosen": -3.066349506378174,
+      "logits/rejected": -3.0383307933807373,
+      "logps/chosen": -56.371307373046875,
+      "logps/rejected": -52.432106018066406,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.00030968443024903536,
+      "rewards/margins": 0.0001904324017232284,
+      "rewards/rejected": -0.000500116846524179,
+      "step": 270
+    },
+    {
+      "epoch": 0.19297036526533426,
+      "grad_norm": 1.2462892532348633,
+      "learning_rate": 4.810996563573883e-08,
+      "logits/chosen": -3.0566208362579346,
+      "logits/rejected": -3.051412582397461,
+      "logps/chosen": -53.14699172973633,
+      "logps/rejected": -54.41425323486328,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.49531251192092896,
+      "rewards/chosen": -0.00045495276572182775,
+      "rewards/margins": 9.360066178487614e-05,
+      "rewards/rejected": -0.0005485534202307463,
+      "step": 280
+    },
+    {
+      "epoch": 0.19986216402481047,
+      "grad_norm": 1.1743725538253784,
+      "learning_rate": 4.982817869415808e-08,
+      "logits/chosen": -3.0853469371795654,
+      "logits/rejected": -3.063814640045166,
+      "logps/chosen": -54.09833908081055,
+      "logps/rejected": -54.12751007080078,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0002439660020172596,
+      "rewards/margins": 0.000504250347148627,
+      "rewards/rejected": -0.0007482162909582257,
+      "step": 290
+    },
+    {
+      "epoch": 0.2067539627842867,
+      "grad_norm": 1.1194610595703125,
+      "learning_rate": 4.999853419300577e-08,
+      "logits/chosen": -3.012183666229248,
+      "logits/rejected": -2.9885506629943848,
+      "logps/chosen": -54.22556686401367,
+      "logps/rejected": -51.91581344604492,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5546875,
+      "rewards/chosen": -0.000393096124753356,
+      "rewards/margins": 0.0003973825369030237,
+      "rewards/rejected": -0.0007904786616563797,
+      "step": 300
+    },
+    {
+      "epoch": 0.2067539627842867,
+      "eval_logits/chosen": -3.1624693870544434,
+      "eval_logits/rejected": -3.156888484954834,
+      "eval_logps/chosen": -58.67123794555664,
+      "eval_logps/rejected": -63.15048599243164,
+      "eval_loss": 0.6930928826332092,
+      "eval_rewards/accuracies": 0.5148698687553406,
+      "eval_rewards/chosen": 0.00040659555816091597,
+      "eval_rewards/margins": 0.00011023049592040479,
+      "eval_rewards/rejected": 0.0002963650331366807,
+      "eval_runtime": 383.575,
+      "eval_samples_per_second": 11.221,
+      "eval_steps_per_second": 1.403,
+      "step": 300
+    },
+    {
+      "epoch": 0.2136457615437629,
+      "grad_norm": 1.1925629377365112,
+      "learning_rate": 4.9993467426542045e-08,
+      "logits/chosen": -3.086402416229248,
+      "logits/rejected": -3.0562937259674072,
+      "logps/chosen": -53.876312255859375,
+      "logps/rejected": -52.675437927246094,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5796874761581421,
+      "rewards/chosen": -0.0003693565959110856,
+      "rewards/margins": 0.0005247757071629167,
+      "rewards/rejected": -0.0008941322448663414,
+      "step": 310
+    },
+    {
+      "epoch": 0.22053756030323915,
+      "grad_norm": 1.154595136642456,
+      "learning_rate": 4.998478233757101e-08,
+      "logits/chosen": -3.0752129554748535,
+      "logits/rejected": -3.0584304332733154,
+      "logps/chosen": -52.4905891418457,
+      "logps/rejected": -54.12751388549805,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5718749761581421,
+      "rewards/chosen": -0.0005890514003112912,
+      "rewards/margins": 0.0004612796474248171,
+      "rewards/rejected": -0.0010503310477361083,
+      "step": 320
+    },
+    {
+      "epoch": 0.22742935906271536,
+      "grad_norm": 1.143236517906189,
+      "learning_rate": 4.9972480183439325e-08,
+      "logits/chosen": -3.075157642364502,
+      "logits/rejected": -3.0487570762634277,
+      "logps/chosen": -53.44994354248047,
+      "logps/rejected": -51.2059326171875,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.0005590206128545105,
+      "rewards/margins": 0.0007544254185631871,
+      "rewards/rejected": -0.0013134460896253586,
+      "step": 330
+    },
+    {
+      "epoch": 0.2343211578221916,
+      "grad_norm": 1.3542113304138184,
+      "learning_rate": 4.995656274513881e-08,
+      "logits/chosen": -3.0580501556396484,
+      "logits/rejected": -3.035737991333008,
+      "logps/chosen": -54.966087341308594,
+      "logps/rejected": -53.1796760559082,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5609375238418579,
+      "rewards/chosen": -0.0005089120240882039,
+      "rewards/margins": 0.0006240031216293573,
+      "rewards/rejected": -0.001132915262132883,
+      "step": 340
+    },
+    {
+      "epoch": 0.2412129565816678,
+      "grad_norm": 1.2170838117599487,
+      "learning_rate": 4.993703232704862e-08,
+      "logits/chosen": -3.0822110176086426,
+      "logits/rejected": -3.059418201446533,
+      "logps/chosen": -54.97810745239258,
+      "logps/rejected": -52.979820251464844,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.604687511920929,
+      "rewards/chosen": -0.0005054243374615908,
+      "rewards/margins": 0.0008770185522735119,
+      "rewards/rejected": -0.0013824428897351027,
+      "step": 350
+    },
+    {
+      "epoch": 0.24810475534114404,
+      "grad_norm": 1.107391595840454,
+      "learning_rate": 4.991389175660163e-08,
+      "logits/chosen": -3.0396039485931396,
+      "logits/rejected": -3.0273656845092773,
+      "logps/chosen": -52.375274658203125,
+      "logps/rejected": -53.336265563964844,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.573437511920929,
+      "rewards/chosen": -0.0008153729140758514,
+      "rewards/margins": 0.0007024986553005874,
+      "rewards/rejected": -0.001517871511168778,
+      "step": 360
+    },
+    {
+      "epoch": 0.2549965541006203,
+      "grad_norm": 1.1218314170837402,
+      "learning_rate": 4.98871443838751e-08,
+      "logits/chosen": -3.114689350128174,
+      "logits/rejected": -3.0790865421295166,
+      "logps/chosen": -53.952476501464844,
+      "logps/rejected": -52.38344192504883,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.6171875,
+      "rewards/chosen": -0.0007519819191657007,
+      "rewards/margins": 0.001077468739822507,
+      "rewards/rejected": -0.0018294507171958685,
+      "step": 370
+    },
+    {
+      "epoch": 0.2618883528600965,
+      "grad_norm": 1.1392273902893066,
+      "learning_rate": 4.985679408110568e-08,
+      "logits/chosen": -3.0398175716400146,
+      "logits/rejected": -3.0220084190368652,
+      "logps/chosen": -54.647239685058594,
+      "logps/rejected": -52.84843826293945,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.0008999688434414566,
+      "rewards/margins": 0.0009324215352535248,
+      "rewards/rejected": -0.001832390553317964,
+      "step": 380
+    },
+    {
+      "epoch": 0.2687801516195727,
+      "grad_norm": 1.1045254468917847,
+      "learning_rate": 4.9822845242128844e-08,
+      "logits/chosen": -3.0233044624328613,
+      "logits/rejected": -3.001706600189209,
+      "logps/chosen": -53.072547912597656,
+      "logps/rejected": -50.9716796875,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.589062511920929,
+      "rewards/chosen": -0.0011974747758358717,
+      "rewards/margins": 0.001123163616284728,
+      "rewards/rejected": -0.0023206386249512434,
+      "step": 390
+    },
+    {
+      "epoch": 0.27567195037904896,
+      "grad_norm": 1.1129488945007324,
+      "learning_rate": 4.9785302781742763e-08,
+      "logits/chosen": -3.050330400466919,
+      "logits/rejected": -3.035008192062378,
+      "logps/chosen": -52.901397705078125,
+      "logps/rejected": -54.134605407714844,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.582812488079071,
+      "rewards/chosen": -0.0011613852111622691,
+      "rewards/margins": 0.0009022338199429214,
+      "rewards/rejected": -0.0020636192057281733,
+      "step": 400
+    },
+    {
+      "epoch": 0.27567195037904896,
+      "eval_logits/chosen": -3.1611053943634033,
+      "eval_logits/rejected": -3.1554572582244873,
+      "eval_logps/chosen": -58.63969039916992,
+      "eval_logps/rejected": -63.135032653808594,
+      "eval_loss": 0.6930131316184998,
+      "eval_rewards/accuracies": 0.5257899761199951,
+      "eval_rewards/chosen": 0.0007220551487989724,
+      "eval_rewards/margins": 0.0002711908018682152,
+      "eval_rewards/rejected": 0.00045086428872309625,
+      "eval_runtime": 383.1949,
+      "eval_samples_per_second": 11.232,
+      "eval_steps_per_second": 1.404,
+      "step": 400
+    },
+    {
+      "epoch": 0.28256374913852517,
+      "grad_norm": 1.2506204843521118,
+      "learning_rate": 4.974417213499681e-08,
+      "logits/chosen": -3.0777323246002197,
+      "logits/rejected": -3.049983501434326,
+      "logps/chosen": -55.058868408203125,
+      "logps/rejected": -53.96419143676758,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.0010929839918389916,
+      "rewards/margins": 0.0014660651795566082,
+      "rewards/rejected": -0.0025590492878109217,
+      "step": 410
+    },
+    {
+      "epoch": 0.2894555478980014,
+      "grad_norm": 1.237091302871704,
+      "learning_rate": 4.9699459256404706e-08,
+      "logits/chosen": -3.105699300765991,
+      "logits/rejected": -3.0748677253723145,
+      "logps/chosen": -55.66558837890625,
+      "logps/rejected": -53.8339729309082,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.6109374761581421,
+      "rewards/chosen": -0.000828454561997205,
+      "rewards/margins": 0.001723860390484333,
+      "rewards/rejected": -0.002552315127104521,
+      "step": 420
+    },
+    {
+      "epoch": 0.2963473466574776,
+      "grad_norm": 1.1707303524017334,
+      "learning_rate": 4.965117061908251e-08,
+      "logits/chosen": -3.056098461151123,
+      "logits/rejected": -3.035871982574463,
+      "logps/chosen": -55.13801193237305,
+      "logps/rejected": -53.53112030029297,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.0015201037749648094,
+      "rewards/margins": 0.0009773834608495235,
+      "rewards/rejected": -0.002497487235814333,
+      "step": 430
+    },
+    {
+      "epoch": 0.30323914541695385,
+      "grad_norm": 1.1965198516845703,
+      "learning_rate": 4.959931321381145e-08,
+      "logits/chosen": -3.082432508468628,
+      "logits/rejected": -3.063544750213623,
+      "logps/chosen": -54.456016540527344,
+      "logps/rejected": -54.16331100463867,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.0014482419937849045,
+      "rewards/margins": 0.0013292920775711536,
+      "rewards/rejected": -0.002777534071356058,
+      "step": 440
+    },
+    {
+      "epoch": 0.31013094417643006,
+      "grad_norm": 1.2304091453552246,
+      "learning_rate": 4.954389454802591e-08,
+      "logits/chosen": -3.1104228496551514,
+      "logits/rejected": -3.090036153793335,
+      "logps/chosen": -53.494163513183594,
+      "logps/rejected": -53.315879821777344,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.5921875238418579,
+      "rewards/chosen": -0.0016971270088106394,
+      "rewards/margins": 0.0016467798268422484,
+      "rewards/rejected": -0.003343907417729497,
+      "step": 450
+    },
+    {
+      "epoch": 0.31702274293590627,
+      "grad_norm": 1.1292587518692017,
+      "learning_rate": 4.948492264472656e-08,
+      "logits/chosen": -3.1166298389434814,
+      "logits/rejected": -3.094527006149292,
+      "logps/chosen": -55.6964111328125,
+      "logps/rejected": -53.82384490966797,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0016075067687779665,
+      "rewards/margins": 0.001251583336852491,
+      "rewards/rejected": -0.0028590902220457792,
+      "step": 460
+    },
+    {
+      "epoch": 0.3239145416953825,
+      "grad_norm": 1.197009563446045,
+      "learning_rate": 4.9422406041318844e-08,
+      "logits/chosen": -3.0635745525360107,
+      "logits/rejected": -3.038623094558716,
+      "logps/chosen": -54.91028594970703,
+      "logps/rejected": -53.81779861450195,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.0011848278809338808,
+      "rewards/margins": 0.002738000126555562,
+      "rewards/rejected": -0.0039228275418281555,
+      "step": 470
+    },
+    {
+      "epoch": 0.33080634045485874,
+      "grad_norm": 1.2151196002960205,
+      "learning_rate": 4.9356353788377026e-08,
+      "logits/chosen": -3.055495023727417,
+      "logits/rejected": -3.0299649238586426,
+      "logps/chosen": -55.23993682861328,
+      "logps/rejected": -53.810813903808594,
+      "loss": 0.692,
+      "rewards/accuracies": 0.598437488079071,
+      "rewards/chosen": -0.0015645608073100448,
+      "rewards/margins": 0.0022706836462020874,
+      "rewards/rejected": -0.003835244569927454,
+      "step": 480
+    },
+    {
+      "epoch": 0.33769813921433495,
+      "grad_norm": 1.1114208698272705,
+      "learning_rate": 4.9286775448333944e-08,
+      "logits/chosen": -3.0453591346740723,
+      "logits/rejected": -3.0262703895568848,
+      "logps/chosen": -53.14439010620117,
+      "logps/rejected": -53.70630645751953,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6078125238418579,
+      "rewards/chosen": -0.0021999510936439037,
+      "rewards/margins": 0.0019915387965738773,
+      "rewards/rejected": -0.0041914889588952065,
+      "step": 490
+    },
+    {
+      "epoch": 0.34458993797381116,
+      "grad_norm": 1.2440327405929565,
+      "learning_rate": 4.921368109409663e-08,
+      "logits/chosen": -3.0790770053863525,
+      "logits/rejected": -3.0631680488586426,
+      "logps/chosen": -53.35895538330078,
+      "logps/rejected": -53.36548614501953,
+      "loss": 0.692,
+      "rewards/accuracies": 0.614062488079071,
+      "rewards/chosen": -0.002265265677124262,
+      "rewards/margins": 0.002222201321274042,
+      "rewards/rejected": -0.004487467464059591,
+      "step": 500
+    },
+    {
+      "epoch": 0.34458993797381116,
+      "eval_logits/chosen": -3.1591975688934326,
+      "eval_logits/rejected": -3.153568983078003,
+      "eval_logps/chosen": -58.59514236450195,
+      "eval_logps/rejected": -63.1102180480957,
+      "eval_loss": 0.6929171681404114,
+      "eval_rewards/accuracies": 0.5246282815933228,
+      "eval_rewards/chosen": 0.0011674691922962666,
+      "eval_rewards/margins": 0.0004684112500399351,
+      "eval_rewards/rejected": 0.0006990578840486705,
+      "eval_runtime": 382.8893,
+      "eval_samples_per_second": 11.241,
+      "eval_steps_per_second": 1.405,
+      "step": 500
+    },
+    {
+      "epoch": 0.35148173673328736,
+      "grad_norm": 1.1804462671279907,
+      "learning_rate": 4.913708130758806e-08,
+      "logits/chosen": -3.0682575702667236,
+      "logits/rejected": -3.046999454498291,
+      "logps/chosen": -54.03418731689453,
+      "logps/rejected": -54.376319885253906,
+      "loss": 0.692,
+      "rewards/accuracies": 0.5953124761581421,
+      "rewards/chosen": -0.002343302359804511,
+      "rewards/margins": 0.002379921730607748,
+      "rewards/rejected": -0.0047232238575816154,
+      "step": 510
+    },
+    {
+      "epoch": 0.35837353549276363,
+      "grad_norm": 1.1343954801559448,
+      "learning_rate": 4.9056987178215176e-08,
+      "logits/chosen": -3.1094601154327393,
+      "logits/rejected": -3.0802154541015625,
+      "logps/chosen": -53.637245178222656,
+      "logps/rejected": -53.262474060058594,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0020912564359605312,
+      "rewards/margins": 0.002018420724198222,
+      "rewards/rejected": -0.004109677392989397,
+      "step": 520
+    },
+    {
+      "epoch": 0.36526533425223984,
+      "grad_norm": 1.1996898651123047,
+      "learning_rate": 4.8973410301263516e-08,
+      "logits/chosen": -3.051212787628174,
+      "logits/rejected": -3.0387063026428223,
+      "logps/chosen": -53.287681579589844,
+      "logps/rejected": -53.440711975097656,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.0023356422316282988,
+      "rewards/margins": 0.0020052504260092974,
+      "rewards/rejected": -0.004340892191976309,
+      "step": 530
+    },
+    {
+      "epoch": 0.37215713301171605,
+      "grad_norm": 1.164119839668274,
+      "learning_rate": 4.8886362776218506e-08,
+      "logits/chosen": -3.0033349990844727,
+      "logits/rejected": -2.9812140464782715,
+      "logps/chosen": -53.450355529785156,
+      "logps/rejected": -51.471229553222656,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0030757987406104803,
+      "rewards/margins": 0.0025067501701414585,
+      "rewards/rejected": -0.005582548677921295,
+      "step": 540
+    },
+    {
+      "epoch": 0.37904893177119225,
+      "grad_norm": 1.28213632106781,
+      "learning_rate": 4.879585720501382e-08,
+      "logits/chosen": -3.148085355758667,
+      "logits/rejected": -3.127159595489502,
+      "logps/chosen": -54.660545349121094,
+      "logps/rejected": -53.745887756347656,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.0028407000936567783,
+      "rewards/margins": 0.0024546708445996046,
+      "rewards/rejected": -0.005295370705425739,
+      "step": 550
+    },
+    {
+      "epoch": 0.3859407305306685,
+      "grad_norm": 1.3044832944869995,
+      "learning_rate": 4.870190669020703e-08,
+      "logits/chosen": -3.0593714714050293,
+      "logits/rejected": -3.036311388015747,
+      "logps/chosen": -55.014060974121094,
+      "logps/rejected": -53.53757858276367,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.002411695895716548,
+      "rewards/margins": 0.0032335221767425537,
+      "rewards/rejected": -0.005645217839628458,
+      "step": 560
+    },
+    {
+      "epoch": 0.3928325292901447,
+      "grad_norm": 1.1550047397613525,
+      "learning_rate": 4.860452483308266e-08,
+      "logits/chosen": -2.9982199668884277,
+      "logits/rejected": -2.972108840942383,
+      "logps/chosen": -56.20374298095703,
+      "logps/rejected": -55.09558868408203,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.5953124761581421,
+      "rewards/chosen": -0.0028813418466597795,
+      "rewards/margins": 0.0031816777773201466,
+      "rewards/rejected": -0.00606301985681057,
+      "step": 570
+    },
+    {
+      "epoch": 0.39972432804962094,
+      "grad_norm": 1.230724573135376,
+      "learning_rate": 4.8503725731683204e-08,
+      "logits/chosen": -3.0479977130889893,
+      "logits/rejected": -3.0179476737976074,
+      "logps/chosen": -54.623687744140625,
+      "logps/rejected": -53.172157287597656,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": -0.002868245355784893,
+      "rewards/margins": 0.003858409356325865,
+      "rewards/rejected": -0.006726655177772045,
+      "step": 580
+    },
+    {
+      "epoch": 0.4066161268090972,
+      "grad_norm": 1.1609071493148804,
+      "learning_rate": 4.839952397876808e-08,
+      "logits/chosen": -3.0574018955230713,
+      "logits/rejected": -3.039822816848755,
+      "logps/chosen": -54.512779235839844,
+      "logps/rejected": -54.206886291503906,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.629687488079071,
+      "rewards/chosen": -0.0032920341473072767,
+      "rewards/margins": 0.0035798237659037113,
+      "rewards/rejected": -0.006871857680380344,
+      "step": 590
+    },
+    {
+      "epoch": 0.4135079255685734,
+      "grad_norm": 1.0820258855819702,
+      "learning_rate": 4.829193465970105e-08,
+      "logits/chosen": -3.089672327041626,
+      "logits/rejected": -3.069746494293213,
+      "logps/chosen": -54.53960418701172,
+      "logps/rejected": -53.9844970703125,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.604687511920929,
+      "rewards/chosen": -0.003799352329224348,
+      "rewards/margins": 0.0033282779622823,
+      "rewards/rejected": -0.007127630058676004,
+      "step": 600
+    },
+    {
+      "epoch": 0.4135079255685734,
+      "eval_logits/chosen": -3.1564178466796875,
+      "eval_logits/rejected": -3.150780200958252,
+      "eval_logps/chosen": -58.54813766479492,
+      "eval_logps/rejected": -63.1104850769043,
+      "eval_loss": 0.6926856637001038,
+      "eval_rewards/accuracies": 0.5504181981086731,
+      "eval_rewards/chosen": 0.001637543668039143,
+      "eval_rewards/margins": 0.0009411590872332454,
+      "eval_rewards/rejected": 0.0006963845225982368,
+      "eval_runtime": 383.4087,
+      "eval_samples_per_second": 11.226,
+      "eval_steps_per_second": 1.403,
+      "step": 600
+    },
+    {
+      "epoch": 0.4203997243280496,
+      "grad_norm": 1.1835054159164429,
+      "learning_rate": 4.818097335026631e-08,
+      "logits/chosen": -3.101921319961548,
+      "logits/rejected": -3.0772037506103516,
+      "logps/chosen": -55.26588821411133,
+      "logps/rejected": -53.28364181518555,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.002828064141795039,
+      "rewards/margins": 0.00399785814806819,
+      "rewards/rejected": -0.006825921591371298,
+      "step": 610
+    },
+    {
+      "epoch": 0.4272915230875258,
+      "grad_norm": 1.203052043914795,
+      "learning_rate": 4.806665611441354e-08,
+      "logits/chosen": -3.077770233154297,
+      "logits/rejected": -3.0505123138427734,
+      "logps/chosen": -55.078880310058594,
+      "logps/rejected": -52.72577667236328,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.004233444109559059,
+      "rewards/margins": 0.0032121867407113314,
+      "rewards/rejected": -0.007445631083101034,
+      "step": 620
+    },
+    {
+      "epoch": 0.4341833218470021,
+      "grad_norm": 1.1228797435760498,
+      "learning_rate": 4.794899950193235e-08,
+      "logits/chosen": -3.0409035682678223,
+      "logits/rejected": -3.0232279300689697,
+      "logps/chosen": -53.423980712890625,
+      "logps/rejected": -52.98765182495117,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.5796874761581421,
+      "rewards/chosen": -0.004781276918947697,
+      "rewards/margins": 0.0022252718918025494,
+      "rewards/rejected": -0.007006548345088959,
+      "step": 630
+    },
+    {
+      "epoch": 0.4410751206064783,
+      "grad_norm": 1.262542486190796,
+      "learning_rate": 4.782802054605635e-08,
+      "logits/chosen": -3.0899507999420166,
+      "logits/rejected": -3.0717437267303467,
+      "logps/chosen": -55.078704833984375,
+      "logps/rejected": -54.794776916503906,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.0041534146293997765,
+      "rewards/margins": 0.0038144378922879696,
+      "rewards/rejected": -0.007967852056026459,
+      "step": 640
+    },
+    {
+      "epoch": 0.4479669193659545,
+      "grad_norm": 1.2199469804763794,
+      "learning_rate": 4.77037367609972e-08,
+      "logits/chosen": -3.0735621452331543,
+      "logits/rejected": -3.0427281856536865,
+      "logps/chosen": -56.89426803588867,
+      "logps/rejected": -53.209136962890625,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.6171875,
+      "rewards/chosen": -0.004497360438108444,
+      "rewards/margins": 0.004101374186575413,
+      "rewards/rejected": -0.008598734624683857,
+      "step": 650
+    },
+    {
+      "epoch": 0.4548587181254307,
+      "grad_norm": 1.1544371843338013,
+      "learning_rate": 4.7576166139409105e-08,
+      "logits/chosen": -3.042221784591675,
+      "logits/rejected": -3.0110714435577393,
+      "logps/chosen": -54.11481475830078,
+      "logps/rejected": -52.04207229614258,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6390625238418579,
+      "rewards/chosen": -0.004763273987919092,
+      "rewards/margins": 0.004999758210033178,
+      "rewards/rejected": -0.00976303219795227,
+      "step": 660
+    },
+    {
+      "epoch": 0.461750516884907,
+      "grad_norm": 1.281175136566162,
+      "learning_rate": 4.744532714978399e-08,
+      "logits/chosen": -3.0140280723571777,
+      "logits/rejected": -2.9848811626434326,
+      "logps/chosen": -56.1414680480957,
+      "logps/rejected": -54.0085334777832,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.6265624761581421,
+      "rewards/chosen": -0.0041292086243629456,
+      "rewards/margins": 0.004748177714645863,
+      "rewards/rejected": -0.008877387270331383,
+      "step": 670
+    },
+    {
+      "epoch": 0.4686423156443832,
+      "grad_norm": 1.1695414781570435,
+      "learning_rate": 4.7311238733777815e-08,
+      "logits/chosen": -3.046804428100586,
+      "logits/rejected": -3.0304887294769287,
+      "logps/chosen": -54.355079650878906,
+      "logps/rejected": -54.04961395263672,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.005136381834745407,
+      "rewards/margins": 0.00416863476857543,
+      "rewards/rejected": -0.009305017068982124,
+      "step": 680
+    },
+    {
+      "epoch": 0.4755341144038594,
+      "grad_norm": 1.1991028785705566,
+      "learning_rate": 4.717392030346835e-08,
+      "logits/chosen": -3.028083562850952,
+      "logits/rejected": -3.011951446533203,
+      "logps/chosen": -54.25959396362305,
+      "logps/rejected": -54.1555061340332,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.00518420897424221,
+      "rewards/margins": 0.004269171506166458,
+      "rewards/rejected": -0.009453380480408669,
+      "step": 690
+    },
+    {
+      "epoch": 0.4824259131633356,
+      "grad_norm": 1.2611873149871826,
+      "learning_rate": 4.70333917385449e-08,
+      "logits/chosen": -3.079685926437378,
+      "logits/rejected": -3.049795627593994,
+      "logps/chosen": -55.45751190185547,
+      "logps/rejected": -53.548301696777344,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.004976336378604174,
+      "rewards/margins": 0.003996217157691717,
+      "rewards/rejected": -0.00897255353629589,
+      "step": 700
+    },
+    {
+      "epoch": 0.4824259131633356,
+      "eval_logits/chosen": -3.1538004875183105,
+      "eval_logits/rejected": -3.1481423377990723,
+      "eval_logps/chosen": -58.522918701171875,
+      "eval_logps/rejected": -63.142425537109375,
+      "eval_loss": 0.6924082636833191,
+      "eval_rewards/accuracies": 0.5671468377113342,
+      "eval_rewards/chosen": 0.0018897424452006817,
+      "eval_rewards/margins": 0.0015127337537705898,
+      "eval_rewards/rejected": 0.0003770088078454137,
+      "eval_runtime": 383.1967,
+      "eval_samples_per_second": 11.232,
+      "eval_steps_per_second": 1.404,
+      "step": 700
+    },
+    {
+      "epoch": 0.48931771192281187,
+      "grad_norm": 1.1392662525177002,
+      "learning_rate": 4.688967338343029e-08,
+      "logits/chosen": -3.0261685848236084,
+      "logits/rejected": -3.0102686882019043,
+      "logps/chosen": -54.990821838378906,
+      "logps/rejected": -54.767127990722656,
+      "loss": 0.691,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.00540867168456316,
+      "rewards/margins": 0.004442816134542227,
+      "rewards/rejected": -0.0098514873534441,
+      "step": 710
+    },
+    {
+      "epoch": 0.4962095106822881,
+      "grad_norm": 1.1779069900512695,
+      "learning_rate": 4.6742786044335625e-08,
+      "logits/chosen": -3.0809476375579834,
+      "logits/rejected": -3.057307481765747,
+      "logps/chosen": -55.18914794921875,
+      "logps/rejected": -53.8927116394043,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.0052046263590455055,
+      "rewards/margins": 0.006748650223016739,
+      "rewards/rejected": -0.011953277513384819,
+      "step": 720
+    },
+    {
+      "epoch": 0.5031013094417643,
+      "grad_norm": 1.2199147939682007,
+      "learning_rate": 4.6592750986248085e-08,
+      "logits/chosen": -3.107689380645752,
+      "logits/rejected": -3.1000123023986816,
+      "logps/chosen": -54.34379959106445,
+      "logps/rejected": -54.853431701660156,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.005630369298160076,
+      "rewards/margins": 0.004252653103321791,
+      "rewards/rejected": -0.009883022867143154,
+      "step": 730
+    },
+    {
+      "epoch": 0.5099931082012406,
+      "grad_norm": 1.307981014251709,
+      "learning_rate": 4.6439589929852476e-08,
+      "logits/chosen": -3.0687716007232666,
+      "logits/rejected": -3.0409016609191895,
+      "logps/chosen": -53.86914825439453,
+      "logps/rejected": -53.336158752441406,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.005739855580031872,
+      "rewards/margins": 0.0054161581210792065,
+      "rewards/rejected": -0.011156014166772366,
+      "step": 740
+    },
+    {
+      "epoch": 0.5168849069607168,
+      "grad_norm": 1.1373140811920166,
+      "learning_rate": 4.6283325048386624e-08,
+      "logits/chosen": -3.0201470851898193,
+      "logits/rejected": -2.998100519180298,
+      "logps/chosen": -55.00568389892578,
+      "logps/rejected": -54.43558883666992,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0055595808662474155,
+      "rewards/margins": 0.005786740221083164,
+      "rewards/rejected": -0.011346321552991867,
+      "step": 750
+    },
+    {
+      "epoch": 0.523776705720193,
+      "grad_norm": 1.1454448699951172,
+      "learning_rate": 4.612397896443138e-08,
+      "logits/chosen": -3.118800163269043,
+      "logits/rejected": -3.0978825092315674,
+      "logps/chosen": -54.798065185546875,
+      "logps/rejected": -54.3465461730957,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.5796874761581421,
+      "rewards/chosen": -0.007258473429828882,
+      "rewards/margins": 0.004668924491852522,
+      "rewards/rejected": -0.011927397921681404,
+      "step": 760
+    },
+    {
+      "epoch": 0.5306685044796692,
+      "grad_norm": 1.1706945896148682,
+      "learning_rate": 4.5961574746635536e-08,
+      "logits/chosen": -3.012247323989868,
+      "logits/rejected": -2.993521213531494,
+      "logps/chosen": -55.298187255859375,
+      "logps/rejected": -55.779624938964844,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.582812488079071,
+      "rewards/chosen": -0.00789455696940422,
+      "rewards/margins": 0.0037407889030873775,
+      "rewards/rejected": -0.011635346338152885,
+      "step": 770
+    },
+    {
+      "epoch": 0.5375603032391454,
+      "grad_norm": 1.2820113897323608,
+      "learning_rate": 4.5796135906376144e-08,
+      "logits/chosen": -3.0310168266296387,
+      "logits/rejected": -3.015160083770752,
+      "logps/chosen": -54.29914474487305,
+      "logps/rejected": -55.2180290222168,
+      "loss": 0.691,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": -0.007602076046168804,
+      "rewards/margins": 0.004507972858846188,
+      "rewards/rejected": -0.012110048905014992,
+      "step": 780
+    },
+    {
+      "epoch": 0.5444521019986216,
+      "grad_norm": 1.1508716344833374,
+      "learning_rate": 4.5627686394354766e-08,
+      "logits/chosen": -3.0379862785339355,
+      "logits/rejected": -3.017380475997925,
+      "logps/chosen": -53.72552490234375,
+      "logps/rejected": -54.60520553588867,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.0065218256786465645,
+      "rewards/margins": 0.006237885914742947,
+      "rewards/rejected": -0.012759710662066936,
+      "step": 790
+    },
+    {
+      "epoch": 0.5513439007580979,
+      "grad_norm": 1.1988805532455444,
+      "learning_rate": 4.545625059713011e-08,
+      "logits/chosen": -3.0689666271209717,
+      "logits/rejected": -3.046346664428711,
+      "logps/chosen": -54.87028121948242,
+      "logps/rejected": -53.7490119934082,
+      "loss": 0.69,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.006772381253540516,
+      "rewards/margins": 0.006438801996409893,
+      "rewards/rejected": -0.013211183249950409,
+      "step": 800
+    },
+    {
+      "epoch": 0.5513439007580979,
+      "eval_logits/chosen": -3.1500000953674316,
+      "eval_logits/rejected": -3.1443684101104736,
+      "eval_logps/chosen": -58.52486038208008,
+      "eval_logps/rejected": -63.183868408203125,
+      "eval_loss": 0.69222092628479,
+      "eval_rewards/accuracies": 0.5759758353233337,
+      "eval_rewards/chosen": 0.001870311563834548,
+      "eval_rewards/margins": 0.0019077310571447015,
+      "eval_rewards/rejected": -3.741981345228851e-05,
+      "eval_runtime": 383.2221,
+      "eval_samples_per_second": 11.231,
+      "eval_steps_per_second": 1.404,
+      "step": 800
+    },
+    {
+      "epoch": 0.5582356995175741,
+      "grad_norm": 1.181986927986145,
+      "learning_rate": 4.528185333358756e-08,
+      "logits/chosen": -3.026899814605713,
+      "logits/rejected": -3.0093157291412354,
+      "logps/chosen": -54.46189498901367,
+      "logps/rejected": -54.8513298034668,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.00789581798017025,
+      "rewards/margins": 0.004861229099333286,
+      "rewards/rejected": -0.01275704801082611,
+      "step": 810
+    },
+    {
+      "epoch": 0.5651274982770503,
+      "grad_norm": 1.1672871112823486,
+      "learning_rate": 4.510451985134616e-08,
+      "logits/chosen": -3.0875649452209473,
+      "logits/rejected": -3.0743203163146973,
+      "logps/chosen": -53.040733337402344,
+      "logps/rejected": -55.541954040527344,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.00876162201166153,
+      "rewards/margins": 0.00582465436309576,
+      "rewards/rejected": -0.014586275443434715,
+      "step": 820
+    },
+    {
+      "epoch": 0.5720192970365265,
+      "grad_norm": 1.2362406253814697,
+      "learning_rate": 4.492427582310346e-08,
+      "logits/chosen": -3.0630054473876953,
+      "logits/rejected": -3.0335052013397217,
+      "logps/chosen": -54.42986297607422,
+      "logps/rejected": -53.1016845703125,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.6390625238418579,
+      "rewards/chosen": -0.007075751665979624,
+      "rewards/margins": 0.007580357138067484,
+      "rewards/rejected": -0.014656109735369682,
+      "step": 830
+    },
+    {
+      "epoch": 0.5789110957960028,
+      "grad_norm": 1.1612728834152222,
+      "learning_rate": 4.4741147342918894e-08,
+      "logits/chosen": -3.076169013977051,
+      "logits/rejected": -3.0513038635253906,
+      "logps/chosen": -55.8946533203125,
+      "logps/rejected": -55.85911178588867,
+      "loss": 0.689,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.0071454280987381935,
+      "rewards/margins": 0.008417905308306217,
+      "rewards/rejected": -0.01556333340704441,
+      "step": 840
+    },
+    {
+      "epoch": 0.585802894555479,
+      "grad_norm": 1.1926907300949097,
+      "learning_rate": 4.4555160922436074e-08,
+      "logits/chosen": -3.079662322998047,
+      "logits/rejected": -3.0524630546569824,
+      "logps/chosen": -54.04046630859375,
+      "logps/rejected": -53.262847900390625,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.6234375238418579,
+      "rewards/chosen": -0.007903190329670906,
+      "rewards/margins": 0.007286435458809137,
+      "rewards/rejected": -0.015189625322818756,
+      "step": 850
+    },
+    {
+      "epoch": 0.5926946933149552,
+      "grad_norm": 1.2318311929702759,
+      "learning_rate": 4.4366343487044754e-08,
+      "logits/chosen": -3.031019926071167,
+      "logits/rejected": -3.0084445476531982,
+      "logps/chosen": -52.4871711730957,
+      "logps/rejected": -53.59075927734375,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.009235886856913567,
+      "rewards/margins": 0.007172322832047939,
+      "rewards/rejected": -0.01640820875763893,
+      "step": 860
+    },
+    {
+      "epoch": 0.5995864920744314,
+      "grad_norm": 1.2643660306930542,
+      "learning_rate": 4.417472237198275e-08,
+      "logits/chosen": -3.122987985610962,
+      "logits/rejected": -3.097611665725708,
+      "logps/chosen": -56.34685134887695,
+      "logps/rejected": -55.146095275878906,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.620312511920929,
+      "rewards/chosen": -0.0072593227960169315,
+      "rewards/margins": 0.007286491803824902,
+      "rewards/rejected": -0.01454581506550312,
+      "step": 870
+    },
+    {
+      "epoch": 0.6064782908339077,
+      "grad_norm": 1.2349611520767212,
+      "learning_rate": 4.398032531837865e-08,
+      "logits/chosen": -3.000382423400879,
+      "logits/rejected": -2.979700803756714,
+      "logps/chosen": -54.820579528808594,
+      "logps/rejected": -54.77504348754883,
+      "loss": 0.69,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.010271805338561535,
+      "rewards/margins": 0.006492338143289089,
+      "rewards/rejected": -0.016764143481850624,
+      "step": 880
+    },
+    {
+      "epoch": 0.6133700895933839,
+      "grad_norm": 1.2228236198425293,
+      "learning_rate": 4.378318046923567e-08,
+      "logits/chosen": -3.046607494354248,
+      "logits/rejected": -3.0200607776641846,
+      "logps/chosen": -55.26753616333008,
+      "logps/rejected": -54.108428955078125,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.6078125238418579,
+      "rewards/chosen": -0.00934204924851656,
+      "rewards/margins": 0.007791099604219198,
+      "rewards/rejected": -0.01713315024971962,
+      "step": 890
+    },
+    {
+      "epoch": 0.6202618883528601,
+      "grad_norm": 1.186522126197815,
+      "learning_rate": 4.3583316365357413e-08,
+      "logits/chosen": -3.081699848175049,
+      "logits/rejected": -3.0569376945495605,
+      "logps/chosen": -56.97715377807617,
+      "logps/rejected": -55.800636291503906,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.614062488079071,
+      "rewards/chosen": -0.009422613307833672,
+      "rewards/margins": 0.007982470095157623,
+      "rewards/rejected": -0.017405081540346146,
+      "step": 900
+    },
+    {
+      "epoch": 0.6202618883528601,
+      "eval_logits/chosen": -3.145947217941284,
+      "eval_logits/rejected": -3.140315532684326,
+      "eval_logps/chosen": -58.54254913330078,
+      "eval_logps/rejected": -63.26302719116211,
+      "eval_loss": 0.6919277906417847,
+      "eval_rewards/accuracies": 0.5708643198013306,
+      "eval_rewards/chosen": 0.0016934837913140655,
+      "eval_rewards/margins": 0.00252249906770885,
+      "eval_rewards/rejected": -0.0008290152181871235,
+      "eval_runtime": 383.2519,
+      "eval_samples_per_second": 11.23,
+      "eval_steps_per_second": 1.404,
+      "step": 900
+    },
+    {
+      "epoch": 0.6271536871123363,
+      "grad_norm": 1.234681248664856,
+      "learning_rate": 4.3380761941215947e-08,
+      "logits/chosen": -3.046011447906494,
+      "logits/rejected": -3.0302977561950684,
+      "logps/chosen": -54.25246047973633,
+      "logps/rejected": -55.46947479248047,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.009562310762703419,
+      "rewards/margins": 0.007873213849961758,
+      "rewards/rejected": -0.017435524612665176,
+      "step": 910
+    },
+    {
+      "epoch": 0.6340454858718125,
+      "grad_norm": 1.141934871673584,
+      "learning_rate": 4.317554652076299e-08,
+      "logits/chosen": -3.054769992828369,
+      "logits/rejected": -3.0311903953552246,
+      "logps/chosen": -54.04453659057617,
+      "logps/rejected": -54.37770462036133,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.009277190081775188,
+      "rewards/margins": 0.0076979040168225765,
+      "rewards/rejected": -0.016975093632936478,
+      "step": 920
+    },
+    {
+      "epoch": 0.6409372846312887,
+      "grad_norm": 1.236680269241333,
+      "learning_rate": 4.2967699813184615e-08,
+      "logits/chosen": -3.0500195026397705,
+      "logits/rejected": -3.0328176021575928,
+      "logps/chosen": -54.70762252807617,
+      "logps/rejected": -57.55879592895508,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.008434431627392769,
+      "rewards/margins": 0.009904151782393456,
+      "rewards/rejected": -0.018338583409786224,
+      "step": 930
+    },
+    {
+      "epoch": 0.647829083390765,
+      "grad_norm": 1.2360023260116577,
+      "learning_rate": 4.275725190860027e-08,
+      "logits/chosen": -3.073611259460449,
+      "logits/rejected": -3.0537660121917725,
+      "logps/chosen": -55.351104736328125,
+      "logps/rejected": -55.8747673034668,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.010648580268025398,
+      "rewards/margins": 0.007464288733899593,
+      "rewards/rejected": -0.018112869933247566,
+      "step": 940
+    },
+    {
+      "epoch": 0.6547208821502413,
+      "grad_norm": 1.2623155117034912,
+      "learning_rate": 4.2544233273706585e-08,
+      "logits/chosen": -3.0598671436309814,
+      "logits/rejected": -3.0294106006622314,
+      "logps/chosen": -55.8059196472168,
+      "logps/rejected": -53.73136520385742,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.6234375238418579,
+      "rewards/chosen": -0.009804973378777504,
+      "rewards/margins": 0.007985373958945274,
+      "rewards/rejected": -0.01779034733772278,
+      "step": 950
+    },
+    {
+      "epoch": 0.6616126809097175,
+      "grad_norm": 1.2945950031280518,
+      "learning_rate": 4.232867474736669e-08,
+      "logits/chosen": -3.0672502517700195,
+      "logits/rejected": -3.0369277000427246,
+      "logps/chosen": -56.809417724609375,
+      "logps/rejected": -55.6953239440918,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.660937488079071,
+      "rewards/chosen": -0.007269621826708317,
+      "rewards/margins": 0.010824671015143394,
+      "rewards/rejected": -0.018094293773174286,
+      "step": 960
+    },
+    {
+      "epoch": 0.6685044796691937,
+      "grad_norm": 1.1434519290924072,
+      "learning_rate": 4.211060753614565e-08,
+      "logits/chosen": -3.1128265857696533,
+      "logits/rejected": -3.0972368717193604,
+      "logps/chosen": -56.41877365112305,
+      "logps/rejected": -55.6785774230957,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.011154340580105782,
+      "rewards/margins": 0.005970745347440243,
+      "rewards/rejected": -0.0171250868588686,
+      "step": 970
+    },
+    {
+      "epoch": 0.6753962784286699,
+      "grad_norm": 1.1750149726867676,
+      "learning_rate": 4.1890063209792674e-08,
+      "logits/chosen": -3.1158690452575684,
+      "logits/rejected": -3.079075336456299,
+      "logps/chosen": -57.21317672729492,
+      "logps/rejected": -53.88423538208008,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.6421874761581421,
+      "rewards/chosen": -0.008817395195364952,
+      "rewards/margins": 0.01136676874011755,
+      "rewards/rejected": -0.020184166729450226,
+      "step": 980
+    },
+    {
+      "epoch": 0.6822880771881461,
+      "grad_norm": 1.3042854070663452,
+      "learning_rate": 4.166707369667073e-08,
+      "logits/chosen": -3.045738458633423,
+      "logits/rejected": -3.0281968116760254,
+      "logps/chosen": -54.19719696044922,
+      "logps/rejected": -56.0754280090332,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.009636206552386284,
+      "rewards/margins": 0.009063459932804108,
+      "rewards/rejected": -0.01869966648519039,
+      "step": 990
+    },
+    {
+      "epoch": 0.6891798759476223,
+      "grad_norm": 1.22942054271698,
+      "learning_rate": 4.144167127913426e-08,
+      "logits/chosen": -3.075810194015503,
+      "logits/rejected": -3.052361249923706,
+      "logps/chosen": -55.58427810668945,
+      "logps/rejected": -55.51273727416992,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.011259499937295914,
+      "rewards/margins": 0.008291425183415413,
+      "rewards/rejected": -0.019550926983356476,
+      "step": 1000
+    },
+    {
+      "epoch": 0.6891798759476223,
+      "eval_logits/chosen": -3.1417765617370605,
+      "eval_logits/rejected": -3.136131525039673,
+      "eval_logps/chosen": -58.606346130371094,
+      "eval_logps/rejected": -63.375797271728516,
+      "eval_loss": 0.6916959881782532,
+      "eval_rewards/accuracies": 0.5724906921386719,
+      "eval_rewards/chosen": 0.001055453554727137,
+      "eval_rewards/margins": 0.0030122159514576197,
+      "eval_rewards/rejected": -0.0019567625131458044,
+      "eval_runtime": 382.8427,
+      "eval_samples_per_second": 11.242,
+      "eval_steps_per_second": 1.405,
+      "step": 1000
+    },
+    {
+      "epoch": 0.6960716747070985,
+      "grad_norm": 1.228550910949707,
+      "learning_rate": 4.1213888588855636e-08,
+      "logits/chosen": -3.0645551681518555,
+      "logits/rejected": -3.0503764152526855,
+      "logps/chosen": -54.256507873535156,
+      "logps/rejected": -55.95310592651367,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.6234375238418579,
+      "rewards/chosen": -0.011437224224209785,
+      "rewards/margins": 0.008133414201438427,
+      "rewards/rejected": -0.019570637494325638,
+      "step": 1010
+    },
+    {
+      "epoch": 0.7029634734665747,
+      "grad_norm": 1.290880560874939,
+      "learning_rate": 4.098375860210107e-08,
+      "logits/chosen": -3.0364532470703125,
+      "logits/rejected": -3.0164756774902344,
+      "logps/chosen": -54.49522018432617,
+      "logps/rejected": -55.22959518432617,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.598437488079071,
+      "rewards/chosen": -0.013310156762599945,
+      "rewards/margins": 0.0093264514580369,
+      "rewards/rejected": -0.02263660728931427,
+      "step": 1020
+    },
+    {
+      "epoch": 0.709855272226051,
+      "grad_norm": 1.234087586402893,
+      "learning_rate": 4.075131463495657e-08,
+      "logits/chosen": -3.0410397052764893,
+      "logits/rejected": -3.023860454559326,
+      "logps/chosen": -54.86391067504883,
+      "logps/rejected": -54.73369598388672,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.604687511920929,
+      "rewards/chosen": -0.012769045308232307,
+      "rewards/margins": 0.009414998814463615,
+      "rewards/rejected": -0.022184044122695923,
+      "step": 1030
+    },
+    {
+      "epoch": 0.7167470709855273,
+      "grad_norm": 1.1497515439987183,
+      "learning_rate": 4.051659033850477e-08,
+      "logits/chosen": -3.0711050033569336,
+      "logits/rejected": -3.0434327125549316,
+      "logps/chosen": -55.960113525390625,
+      "logps/rejected": -53.39757537841797,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": -0.01206748653203249,
+      "rewards/margins": 0.011478706263005733,
+      "rewards/rejected": -0.023546192795038223,
+      "step": 1040
+    },
+    {
+      "epoch": 0.7236388697450035,
+      "grad_norm": 1.241176724433899,
+      "learning_rate": 4.0279619693953283e-08,
+      "logits/chosen": -3.0579118728637695,
+      "logits/rejected": -3.044525623321533,
+      "logps/chosen": -54.537757873535156,
+      "logps/rejected": -55.7606201171875,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.5953124761581421,
+      "rewards/chosen": -0.01158697810024023,
+      "rewards/margins": 0.008560305461287498,
+      "rewards/rejected": -0.020147282630205154,
+      "step": 1050
+    },
+    {
+      "epoch": 0.7305306685044797,
+      "grad_norm": 1.287839651107788,
+      "learning_rate": 4.0040437007715124e-08,
+      "logits/chosen": -3.0260822772979736,
+      "logits/rejected": -3.0041518211364746,
+      "logps/chosen": -55.73114013671875,
+      "logps/rejected": -56.4024772644043,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.6234375238418579,
+      "rewards/chosen": -0.013397350907325745,
+      "rewards/margins": 0.01179309468716383,
+      "rewards/rejected": -0.02519044652581215,
+      "step": 1060
+    },
+    {
+      "epoch": 0.7374224672639559,
+      "grad_norm": 1.1840453147888184,
+      "learning_rate": 3.979907690644222e-08,
+      "logits/chosen": -3.005467653274536,
+      "logits/rejected": -2.9843525886535645,
+      "logps/chosen": -54.47725296020508,
+      "logps/rejected": -54.86272048950195,
+      "loss": 0.688,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.012515179812908173,
+      "rewards/margins": 0.010800262913107872,
+      "rewards/rejected": -0.023315440863370895,
+      "step": 1070
+    },
+    {
+      "epoch": 0.7443142660234321,
+      "grad_norm": 1.2041012048721313,
+      "learning_rate": 3.9555574332012454e-08,
+      "logits/chosen": -3.0442147254943848,
+      "logits/rejected": -3.0234692096710205,
+      "logps/chosen": -56.4234619140625,
+      "logps/rejected": -55.07111740112305,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.013009254820644855,
+      "rewards/margins": 0.01034192182123661,
+      "rewards/rejected": -0.02335117571055889,
+      "step": 1080
+    },
+    {
+      "epoch": 0.7512060647829083,
+      "grad_norm": 1.192734956741333,
+      "learning_rate": 3.930996453647113e-08,
+      "logits/chosen": -3.008514881134033,
+      "logits/rejected": -2.986760139465332,
+      "logps/chosen": -53.92486572265625,
+      "logps/rejected": -53.8699951171875,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.016327153891324997,
+      "rewards/margins": 0.009530487470328808,
+      "rewards/rejected": -0.02585764229297638,
+      "step": 1090
+    },
+    {
+      "epoch": 0.7580978635423845,
+      "grad_norm": 1.1945998668670654,
+      "learning_rate": 3.906228307692747e-08,
+      "logits/chosen": -3.050058126449585,
+      "logits/rejected": -3.0325589179992676,
+      "logps/chosen": -56.26338577270508,
+      "logps/rejected": -56.20615768432617,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.598437488079071,
+      "rewards/chosen": -0.014123663306236267,
+      "rewards/margins": 0.0081967543810606,
+      "rewards/rejected": -0.022320415824651718,
+      "step": 1100
+    },
+    {
+      "epoch": 0.7580978635423845,
+      "eval_logits/chosen": -3.136918067932129,
+      "eval_logits/rejected": -3.131256103515625,
+      "eval_logps/chosen": -58.693904876708984,
+      "eval_logps/rejected": -63.52504348754883,
+      "eval_loss": 0.6914047598838806,
+      "eval_rewards/accuracies": 0.5808550119400024,
+      "eval_rewards/chosen": 0.00017988457693718374,
+      "eval_rewards/margins": 0.003629034385085106,
+      "eval_rewards/rejected": -0.0034491494297981262,
+      "eval_runtime": 382.7678,
+      "eval_samples_per_second": 11.244,
+      "eval_steps_per_second": 1.406,
+      "step": 1100
+    },
+    {
+      "epoch": 0.7649896623018608,
+      "grad_norm": 1.2611422538757324,
+      "learning_rate": 3.8812565810407006e-08,
+      "logits/chosen": -3.0509583950042725,
+      "logits/rejected": -3.019794225692749,
+      "logps/chosen": -57.166297912597656,
+      "logps/rejected": -55.56831741333008,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.614062488079071,
+      "rewards/chosen": -0.014432080090045929,
+      "rewards/margins": 0.01164001040160656,
+      "rewards/rejected": -0.02607208862900734,
+      "step": 1110
+    },
+    {
+      "epoch": 0.771881461061337,
+      "grad_norm": 1.1777701377868652,
+      "learning_rate": 3.856084888866052e-08,
+      "logits/chosen": -3.0596282482147217,
+      "logits/rejected": -3.045269250869751,
+      "logps/chosen": -55.52899932861328,
+      "logps/rejected": -54.93024444580078,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.01683993637561798,
+      "rewards/margins": 0.009132475592195988,
+      "rewards/rejected": -0.025972411036491394,
+      "step": 1120
+    },
+    {
+      "epoch": 0.7787732598208132,
+      "grad_norm": 1.2296311855316162,
+      "learning_rate": 3.830716875293038e-08,
+      "logits/chosen": -3.0673999786376953,
+      "logits/rejected": -3.0444142818450928,
+      "logps/chosen": -54.970741271972656,
+      "logps/rejected": -54.68275833129883,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.015940912067890167,
+      "rewards/margins": 0.009639047086238861,
+      "rewards/rejected": -0.02557995915412903,
+      "step": 1130
+    },
+    {
+      "epoch": 0.7856650585802895,
+      "grad_norm": 1.1905580759048462,
+      "learning_rate": 3.805156212867483e-08,
+      "logits/chosen": -3.029092788696289,
+      "logits/rejected": -3.0086588859558105,
+      "logps/chosen": -56.229042053222656,
+      "logps/rejected": -55.34952926635742,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.6421874761581421,
+      "rewards/chosen": -0.01458609290421009,
+      "rewards/margins": 0.011665640398859978,
+      "rewards/rejected": -0.02625173330307007,
+      "step": 1140
+    },
+    {
+      "epoch": 0.7925568573397657,
+      "grad_norm": 1.1570724248886108,
+      "learning_rate": 3.779406602025128e-08,
+      "logits/chosen": -3.007833957672119,
+      "logits/rejected": -2.9827017784118652,
+      "logps/chosen": -55.04015350341797,
+      "logps/rejected": -55.21650314331055,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.01640317775309086,
+      "rewards/margins": 0.010581018403172493,
+      "rewards/rejected": -0.02698419615626335,
+      "step": 1150
+    },
+    {
+      "epoch": 0.7994486560992419,
+      "grad_norm": 1.211165189743042,
+      "learning_rate": 3.7534717705559146e-08,
+      "logits/chosen": -3.036921977996826,
+      "logits/rejected": -3.0160536766052246,
+      "logps/chosen": -56.46533203125,
+      "logps/rejected": -57.42781448364258,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.017415925860404968,
+      "rewards/margins": 0.01034791674464941,
+      "rewards/rejected": -0.027763843536376953,
+      "step": 1160
+    },
+    {
+      "epoch": 0.8063404548587181,
+      "grad_norm": 1.1748243570327759,
+      "learning_rate": 3.727355473064308e-08,
+      "logits/chosen": -3.05203914642334,
+      "logits/rejected": -3.024839162826538,
+      "logps/chosen": -54.88653564453125,
+      "logps/rejected": -54.187705993652344,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.6234375238418579,
+      "rewards/chosen": -0.015425342135131359,
+      "rewards/margins": 0.011839036829769611,
+      "rewards/rejected": -0.02726438082754612,
+      "step": 1170
+    },
+    {
+      "epoch": 0.8132322536181944,
+      "grad_norm": 1.2590429782867432,
+      "learning_rate": 3.701061490425745e-08,
+      "logits/chosen": -3.053898334503174,
+      "logits/rejected": -3.0290002822875977,
+      "logps/chosen": -57.20033645629883,
+      "logps/rejected": -56.57124710083008,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.604687511920929,
+      "rewards/chosen": -0.016385816037654877,
+      "rewards/margins": 0.012309985235333443,
+      "rewards/rejected": -0.02869580127298832,
+      "step": 1180
+    },
+    {
+      "epoch": 0.8201240523776706,
+      "grad_norm": 1.2485055923461914,
+      "learning_rate": 3.6745936292392666e-08,
+      "logits/chosen": -3.021477460861206,
+      "logits/rejected": -3.0019021034240723,
+      "logps/chosen": -55.60076141357422,
+      "logps/rejected": -55.449058532714844,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": -0.01587783917784691,
+      "rewards/margins": 0.010499360039830208,
+      "rewards/rejected": -0.026377201080322266,
+      "step": 1190
+    },
+    {
+      "epoch": 0.8270158511371468,
+      "grad_norm": 1.2800626754760742,
+      "learning_rate": 3.6479557212764414e-08,
+      "logits/chosen": -3.028402090072632,
+      "logits/rejected": -3.008002519607544,
+      "logps/chosen": -56.96452713012695,
+      "logps/rejected": -55.789756774902344,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.6171875,
+      "rewards/chosen": -0.01752752624452114,
+      "rewards/margins": 0.009723445400595665,
+      "rewards/rejected": -0.027250971645116806,
+      "step": 1200
+    },
+    {
+      "epoch": 0.8270158511371468,
+      "eval_logits/chosen": -3.133814811706543,
+      "eval_logits/rejected": -3.128159284591675,
+      "eval_logps/chosen": -58.78531265258789,
+      "eval_logps/rejected": -63.680179595947266,
+      "eval_loss": 0.6911039352416992,
+      "eval_rewards/accuracies": 0.5755111575126648,
+      "eval_rewards/chosen": -0.0007341906311921775,
+      "eval_rewards/margins": 0.004266415257006884,
+      "eval_rewards/rejected": -0.005000605713576078,
+      "eval_runtime": 383.3119,
+      "eval_samples_per_second": 11.228,
+      "eval_steps_per_second": 1.404,
+      "step": 1200
+    },
+    {
+      "epoch": 0.833907649896623,
+      "grad_norm": 1.3170323371887207,
+      "learning_rate": 3.621151622926631e-08,
+      "logits/chosen": -3.022981643676758,
+      "logits/rejected": -2.9983408451080322,
+      "logps/chosen": -56.321983337402344,
+      "logps/rejected": -55.284454345703125,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.017917579039931297,
+      "rewards/margins": 0.011845814064145088,
+      "rewards/rejected": -0.029763391241431236,
+      "step": 1210
+    },
+    {
+      "epoch": 0.8407994486560992,
+      "grad_norm": 1.170351505279541,
+      "learning_rate": 3.594185214638704e-08,
+      "logits/chosen": -3.066943407058716,
+      "logits/rejected": -3.0385823249816895,
+      "logps/chosen": -57.5960807800293,
+      "logps/rejected": -54.60730743408203,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.0178567823022604,
+      "rewards/margins": 0.012462841346859932,
+      "rewards/rejected": -0.03031962178647518,
+      "step": 1220
+    },
+    {
+      "epoch": 0.8476912474155754,
+      "grad_norm": 1.224771499633789,
+      "learning_rate": 3.567060400359253e-08,
+      "logits/chosen": -3.0506491661071777,
+      "logits/rejected": -3.0242903232574463,
+      "logps/chosen": -56.49556350708008,
+      "logps/rejected": -55.71235275268555,
+      "loss": 0.686,
+      "rewards/accuracies": 0.635937511920929,
+      "rewards/chosen": -0.017950473353266716,
+      "rewards/margins": 0.014979615807533264,
+      "rewards/rejected": -0.03293009102344513,
+      "step": 1230
+    },
+    {
+      "epoch": 0.8545830461750517,
+      "grad_norm": 1.2280082702636719,
+      "learning_rate": 3.5397811069674256e-08,
+      "logits/chosen": -3.037538528442383,
+      "logits/rejected": -3.023832082748413,
+      "logps/chosen": -56.15264129638672,
+      "logps/rejected": -58.523162841796875,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.02116158790886402,
+      "rewards/margins": 0.0096513070166111,
+      "rewards/rejected": -0.03081289492547512,
+      "step": 1240
+    },
+    {
+      "epoch": 0.8614748449345279,
+      "grad_norm": 1.3131028413772583,
+      "learning_rate": 3.512351283706419e-08,
+      "logits/chosen": -3.0145888328552246,
+      "logits/rejected": -3.0035436153411865,
+      "logps/chosen": -55.00154495239258,
+      "logps/rejected": -56.4818229675293,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.02046709507703781,
+      "rewards/margins": 0.008517416194081306,
+      "rewards/rejected": -0.02898450754582882,
+      "step": 1250
+    },
+    {
+      "epoch": 0.8683666436940042,
+      "grad_norm": 1.2719794511795044,
+      "learning_rate": 3.484774901611753e-08,
+      "logits/chosen": -3.037191390991211,
+      "logits/rejected": -3.011030673980713,
+      "logps/chosen": -56.1077880859375,
+      "logps/rejected": -55.119110107421875,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.020265808328986168,
+      "rewards/margins": 0.012135522440075874,
+      "rewards/rejected": -0.03240133076906204,
+      "step": 1260
+    },
+    {
+      "epoch": 0.8752584424534804,
+      "grad_norm": 1.223455786705017,
+      "learning_rate": 3.4570559529363756e-08,
+      "logits/chosen": -3.0510623455047607,
+      "logits/rejected": -3.0273430347442627,
+      "logps/chosen": -56.024391174316406,
+      "logps/rejected": -54.66645431518555,
+      "loss": 0.687,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.01942160725593567,
+      "rewards/margins": 0.012839846312999725,
+      "rewards/rejected": -0.032261453568935394,
+      "step": 1270
+    },
+    {
+      "epoch": 0.8821502412129566,
+      "grad_norm": 1.224640965461731,
+      "learning_rate": 3.429198450572702e-08,
+      "logits/chosen": -3.045257568359375,
+      "logits/rejected": -3.0113143920898438,
+      "logps/chosen": -57.24846267700195,
+      "logps/rejected": -55.42502975463867,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.635937511920929,
+      "rewards/chosen": -0.019725024700164795,
+      "rewards/margins": 0.014653083868324757,
+      "rewards/rejected": -0.03437810391187668,
+      "step": 1280
+    },
+    {
+      "epoch": 0.8890420399724328,
+      "grad_norm": 1.1708803176879883,
+      "learning_rate": 3.401206427471665e-08,
+      "logits/chosen": -3.083014965057373,
+      "logits/rejected": -3.053872585296631,
+      "logps/chosen": -55.94194412231445,
+      "logps/rejected": -54.92655563354492,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.6234375238418579,
+      "rewards/chosen": -0.019591879099607468,
+      "rewards/margins": 0.015089405700564384,
+      "rewards/rejected": -0.034681286662817,
+      "step": 1290
+    },
+    {
+      "epoch": 0.895933838731909,
+      "grad_norm": 1.2637726068496704,
+      "learning_rate": 3.3730839360588633e-08,
+      "logits/chosen": -3.0728800296783447,
+      "logits/rejected": -3.0561113357543945,
+      "logps/chosen": -55.9746208190918,
+      "logps/rejected": -56.79145431518555,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.6078125238418579,
+      "rewards/chosen": -0.020700206980109215,
+      "rewards/margins": 0.011441068723797798,
+      "rewards/rejected": -0.03214127570390701,
+      "step": 1300
+    },
+    {
+      "epoch": 0.895933838731909,
+      "eval_logits/chosen": -3.12795090675354,
+      "eval_logits/rejected": -3.1222612857818604,
+      "eval_logps/chosen": -58.95671081542969,
+      "eval_logps/rejected": -63.90719223022461,
+      "eval_loss": 0.690849244594574,
+      "eval_rewards/accuracies": 0.578066885471344,
+      "eval_rewards/chosen": -0.002448198851197958,
+      "eval_rewards/margins": 0.004822447896003723,
+      "eval_rewards/rejected": -0.007270646747201681,
+      "eval_runtime": 383.2075,
+      "eval_samples_per_second": 11.232,
+      "eval_steps_per_second": 1.404,
+      "step": 1300
+    },
+    {
+      "epoch": 0.9028256374913852,
+      "grad_norm": 1.2189267873764038,
+      "learning_rate": 3.344835047647892e-08,
+      "logits/chosen": -3.038391351699829,
+      "logits/rejected": -3.0231399536132812,
+      "logps/chosen": -55.052391052246094,
+      "logps/rejected": -57.159202575683594,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.01980864442884922,
+      "rewards/margins": 0.013179932720959187,
+      "rewards/rejected": -0.03298857808113098,
+      "step": 1310
+    },
+    {
+      "epoch": 0.9097174362508614,
+      "grad_norm": 1.2375820875167847,
+      "learning_rate": 3.316463851850925e-08,
+      "logits/chosen": -3.0543713569641113,
+      "logits/rejected": -3.0287716388702393,
+      "logps/chosen": -55.84870147705078,
+      "logps/rejected": -54.72203826904297,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.620312511920929,
+      "rewards/chosen": -0.024270061403512955,
+      "rewards/margins": 0.012434338219463825,
+      "rewards/rejected": -0.036704398691654205,
+      "step": 1320
+    },
+    {
+      "epoch": 0.9166092350103378,
+      "grad_norm": 1.2331100702285767,
+      "learning_rate": 3.287974455986671e-08,
+      "logits/chosen": -3.0482242107391357,
+      "logits/rejected": -3.0168027877807617,
+      "logps/chosen": -58.51416015625,
+      "logps/rejected": -55.834266662597656,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.6578124761581421,
+      "rewards/chosen": -0.018537839874625206,
+      "rewards/margins": 0.015199096873402596,
+      "rewards/rejected": -0.0337369367480278,
+      "step": 1330
+    },
+    {
+      "epoch": 0.923501033769814,
+      "grad_norm": 1.2201625108718872,
+      "learning_rate": 3.259370984485746e-08,
+      "logits/chosen": -3.0217771530151367,
+      "logits/rejected": -2.998465061187744,
+      "logps/chosen": -55.553428649902344,
+      "logps/rejected": -56.95924758911133,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.022853773087263107,
+      "rewards/margins": 0.013605187647044659,
+      "rewards/rejected": -0.03645896166563034,
+      "step": 1340
+    },
+    {
+      "epoch": 0.9303928325292902,
+      "grad_norm": 1.2265971899032593,
+      "learning_rate": 3.2306575782935806e-08,
+      "logits/chosen": -3.043489456176758,
+      "logits/rejected": -3.013939380645752,
+      "logps/chosen": -56.77729415893555,
+      "logps/rejected": -56.281822204589844,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.635937511920929,
+      "rewards/chosen": -0.020281706005334854,
+      "rewards/margins": 0.016187874600291252,
+      "rewards/rejected": -0.036469582468271255,
+      "step": 1350
+    },
+    {
+      "epoch": 0.9372846312887664,
+      "grad_norm": 1.245123267173767,
+      "learning_rate": 3.201838394270931e-08,
+      "logits/chosen": -3.064115524291992,
+      "logits/rejected": -3.0484519004821777,
+      "logps/chosen": -57.521820068359375,
+      "logps/rejected": -57.416893005371094,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.6109374761581421,
+      "rewards/chosen": -0.024825390428304672,
+      "rewards/margins": 0.012256421148777008,
+      "rewards/rejected": -0.03708181157708168,
+      "step": 1360
+    },
+    {
+      "epoch": 0.9441764300482426,
+      "grad_norm": 1.2887479066848755,
+      "learning_rate": 3.172917604592084e-08,
+      "logits/chosen": -3.0228118896484375,
+      "logits/rejected": -3.0045721530914307,
+      "logps/chosen": -55.98120880126953,
+      "logps/rejected": -55.73798751831055,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.598437488079071,
+      "rewards/chosen": -0.025501202791929245,
+      "rewards/margins": 0.012837904505431652,
+      "rewards/rejected": -0.03833910822868347,
+      "step": 1370
+    },
+    {
+      "epoch": 0.9510682288077188,
+      "grad_norm": 1.2927711009979248,
+      "learning_rate": 3.143899396140856e-08,
+      "logits/chosen": -3.03184175491333,
+      "logits/rejected": -3.010230779647827,
+      "logps/chosen": -56.847740173339844,
+      "logps/rejected": -56.31840133666992,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.022407762706279755,
+      "rewards/margins": 0.01289152167737484,
+      "rewards/rejected": -0.035299282521009445,
+      "step": 1380
+    },
+    {
+      "epoch": 0.957960027567195,
+      "grad_norm": 1.2551859617233276,
+      "learning_rate": 3.114787969904446e-08,
+      "logits/chosen": -3.001943826675415,
+      "logits/rejected": -2.9838249683380127,
+      "logps/chosen": -56.46649169921875,
+      "logps/rejected": -57.01629638671875,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.629687488079071,
+      "rewards/chosen": -0.022768724709749222,
+      "rewards/margins": 0.014040583744645119,
+      "rewards/rejected": -0.03680930659174919,
+      "step": 1390
+    },
+    {
+      "epoch": 0.9648518263266712,
+      "grad_norm": 1.1776050329208374,
+      "learning_rate": 3.085587540365262e-08,
+      "logits/chosen": -3.054063081741333,
+      "logits/rejected": -3.0365357398986816,
+      "logps/chosen": -55.7647819519043,
+      "logps/rejected": -59.496559143066406,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.6078125238418579,
+      "rewards/chosen": -0.024700012058019638,
+      "rewards/margins": 0.01230792049318552,
+      "rewards/rejected": -0.03700793534517288,
+      "step": 1400
+    },
+    {
+      "epoch": 0.9648518263266712,
+      "eval_logits/chosen": -3.1261656284332275,
+      "eval_logits/rejected": -3.1204779148101807,
+      "eval_logps/chosen": -59.10846710205078,
+      "eval_logps/rejected": -64.10256958007812,
+      "eval_loss": 0.6906503438949585,
+      "eval_rewards/accuracies": 0.5771375298500061,
+      "eval_rewards/chosen": -0.003965714480727911,
+      "eval_rewards/margins": 0.0052587250247597694,
+      "eval_rewards/rejected": -0.009224439039826393,
+      "eval_runtime": 383.1495,
+      "eval_samples_per_second": 11.233,
+      "eval_steps_per_second": 1.404,
+      "step": 1400
+    },
+    {
+      "epoch": 0.9717436250861475,
+      "grad_norm": 1.347545862197876,
+      "learning_rate": 3.056302334890786e-08,
+      "logits/chosen": -3.0551466941833496,
+      "logits/rejected": -3.0341668128967285,
+      "logps/chosen": -56.55133056640625,
+      "logps/rejected": -57.29961395263672,
+      "loss": 0.6861,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.02232871949672699,
+      "rewards/margins": 0.014838054776191711,
+      "rewards/rejected": -0.037166766822338104,
+      "step": 1410
+    },
+    {
+      "epoch": 0.9786354238456237,
+      "grad_norm": 1.2241698503494263,
+      "learning_rate": 3.02693659312157e-08,
+      "logits/chosen": -2.9941155910491943,
+      "logits/rejected": -2.9760937690734863,
+      "logps/chosen": -55.75859451293945,
+      "logps/rejected": -55.846839904785156,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.598437488079071,
+      "rewards/chosen": -0.02684764564037323,
+      "rewards/margins": 0.012402022257447243,
+      "rewards/rejected": -0.039249666035175323,
+      "step": 1420
+    },
+    {
+      "epoch": 0.9855272226051,
+      "grad_norm": 1.3626172542572021,
+      "learning_rate": 2.9974945663574684e-08,
+      "logits/chosen": -3.026280403137207,
+      "logits/rejected": -3.0005555152893066,
+      "logps/chosen": -57.986793518066406,
+      "logps/rejected": -56.02061080932617,
+      "loss": 0.684,
+      "rewards/accuracies": 0.645312488079071,
+      "rewards/chosen": -0.02259395457804203,
+      "rewards/margins": 0.019132796674966812,
+      "rewards/rejected": -0.041726745665073395,
+      "step": 1430
+    },
+    {
+      "epoch": 0.9924190213645762,
+      "grad_norm": 1.289384126663208,
+      "learning_rate": 2.967980516942168e-08,
+      "logits/chosen": -3.0657455921173096,
+      "logits/rejected": -3.040670394897461,
+      "logps/chosen": -58.47277069091797,
+      "logps/rejected": -57.19081497192383,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.6234375238418579,
+      "rewards/chosen": -0.024709826335310936,
+      "rewards/margins": 0.016781199723482132,
+      "rewards/rejected": -0.04149102419614792,
+      "step": 1440
+    },
+    {
+      "epoch": 0.9993108201240524,
+      "grad_norm": 1.288656234741211,
+      "learning_rate": 2.9383987176461268e-08,
+      "logits/chosen": -2.991293430328369,
+      "logits/rejected": -2.9657962322235107,
+      "logps/chosen": -54.06468963623047,
+      "logps/rejected": -57.25890350341797,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.024288879707455635,
+      "rewards/margins": 0.012961057014763355,
+      "rewards/rejected": -0.037249937653541565,
+      "step": 1450
+    },
+    {
+      "epoch": 1.0062026188835287,
+      "grad_norm": 1.3280855417251587,
+      "learning_rate": 2.9087534510480032e-08,
+      "logits/chosen": -3.045292377471924,
+      "logits/rejected": -3.0192904472351074,
+      "logps/chosen": -55.768096923828125,
+      "logps/rejected": -56.869842529296875,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.02522682026028633,
+      "rewards/margins": 0.017563799396157265,
+      "rewards/rejected": -0.04279061779379845,
+      "step": 1460
+    },
+    {
+      "epoch": 1.0130944176430048,
+      "grad_norm": 1.2552244663238525,
+      "learning_rate": 2.879049008914664e-08,
+      "logits/chosen": -3.015263080596924,
+      "logits/rejected": -2.999004602432251,
+      "logps/chosen": -55.86402130126953,
+      "logps/rejected": -58.1766471862793,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.645312488079071,
+      "rewards/chosen": -0.024612870067358017,
+      "rewards/margins": 0.01742735505104065,
+      "rewards/rejected": -0.04204022139310837,
+      "step": 1470
+    },
+    {
+      "epoch": 1.019986216402481,
+      "grad_norm": 1.2557737827301025,
+      "learning_rate": 2.8492896915798605e-08,
+      "logits/chosen": -3.021721601486206,
+      "logits/rejected": -3.0094776153564453,
+      "logps/chosen": -56.1937141418457,
+      "logps/rejected": -59.952537536621094,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.024868253618478775,
+      "rewards/margins": 0.014400708489120007,
+      "rewards/rejected": -0.039268963038921356,
+      "step": 1480
+    },
+    {
+      "epoch": 1.0268780151619572,
+      "grad_norm": 1.3632835149765015,
+      "learning_rate": 2.8194798073216665e-08,
+      "logits/chosen": -2.9897549152374268,
+      "logits/rejected": -2.9639222621917725,
+      "logps/chosen": -56.546180725097656,
+      "logps/rejected": -56.685096740722656,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.635937511920929,
+      "rewards/chosen": -0.024227097630500793,
+      "rewards/margins": 0.017609497532248497,
+      "rewards/rejected": -0.04183660075068474,
+      "step": 1490
+    },
+    {
+      "epoch": 1.0337698139214335,
+      "grad_norm": 1.359270691871643,
+      "learning_rate": 2.7896236717387662e-08,
+      "logits/chosen": -2.9973204135894775,
+      "logits/rejected": -2.979785680770874,
+      "logps/chosen": -55.88494110107422,
+      "logps/rejected": -56.941490173339844,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.614062488079071,
+      "rewards/chosen": -0.029738834127783775,
+      "rewards/margins": 0.012930555269122124,
+      "rewards/rejected": -0.0426693893969059,
+      "step": 1500
+    },
+    {
+      "epoch": 1.0337698139214335,
+      "eval_logits/chosen": -3.121001720428467,
+      "eval_logits/rejected": -3.1152734756469727,
+      "eval_logps/chosen": -59.26029968261719,
+      "eval_logps/rejected": -64.31062316894531,
+      "eval_loss": 0.6903930306434631,
+      "eval_rewards/accuracies": 0.5824813842773438,
+      "eval_rewards/chosen": -0.005484058987349272,
+      "eval_rewards/margins": 0.005820916499942541,
+      "eval_rewards/rejected": -0.011304975487291813,
+      "eval_runtime": 383.1818,
+      "eval_samples_per_second": 11.232,
+      "eval_steps_per_second": 1.404,
+      "step": 1500
+    },
+    {
+      "epoch": 1.0406616126809096,
+      "grad_norm": 1.1823673248291016,
+      "learning_rate": 2.7597256071256836e-08,
+      "logits/chosen": -3.0232627391815186,
+      "logits/rejected": -3.002992630004883,
+      "logps/chosen": -55.69109344482422,
+      "logps/rejected": -55.20794677734375,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.6078125238418579,
+      "rewards/chosen": -0.029489045962691307,
+      "rewards/margins": 0.014345052652060986,
+      "rewards/rejected": -0.04383409768342972,
+      "step": 1510
+    },
+    {
+      "epoch": 1.047553411440386,
+      "grad_norm": 1.2642569541931152,
+      "learning_rate": 2.7297899418470372e-08,
+      "logits/chosen": -2.9904372692108154,
+      "logits/rejected": -2.9669933319091797,
+      "logps/chosen": -59.17595672607422,
+      "logps/rejected": -57.17033767700195,
+      "loss": 0.685,
+      "rewards/accuracies": 0.651562511920929,
+      "rewards/chosen": -0.027122503146529198,
+      "rewards/margins": 0.017171606421470642,
+      "rewards/rejected": -0.04429411143064499,
+      "step": 1520
+    },
+    {
+      "epoch": 1.0544452101998623,
+      "grad_norm": 1.3126106262207031,
+      "learning_rate": 2.6998210097109213e-08,
+      "logits/chosen": -3.062737226486206,
+      "logits/rejected": -3.041637659072876,
+      "logps/chosen": -55.9976806640625,
+      "logps/rejected": -57.3626823425293,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.6546875238418579,
+      "rewards/chosen": -0.0244468804448843,
+      "rewards/margins": 0.018709514290094376,
+      "rewards/rejected": -0.043156400322914124,
+      "step": 1530
+    },
+    {
+      "epoch": 1.0613370089593384,
+      "grad_norm": 1.1926969289779663,
+      "learning_rate": 2.669823149341498e-08,
+      "logits/chosen": -3.0017895698547363,
+      "logits/rejected": -2.9859423637390137,
+      "logps/chosen": -55.688560485839844,
+      "logps/rejected": -56.54026412963867,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.620312511920929,
+      "rewards/chosen": -0.02824980393052101,
+      "rewards/margins": 0.01474563218653202,
+      "rewards/rejected": -0.04299543425440788,
+      "step": 1540
+    },
+    {
+      "epoch": 1.0682288077188147,
+      "grad_norm": 1.2355592250823975,
+      "learning_rate": 2.6398007035508906e-08,
+      "logits/chosen": -3.020545482635498,
+      "logits/rejected": -2.9991073608398438,
+      "logps/chosen": -60.19884490966797,
+      "logps/rejected": -58.834068298339844,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.02622285485267639,
+      "rewards/margins": 0.01788989268243313,
+      "rewards/rejected": -0.04411274939775467,
+      "step": 1550
+    },
+    {
+      "epoch": 1.0751206064782908,
+      "grad_norm": 1.2842044830322266,
+      "learning_rate": 2.609758018710473e-08,
+      "logits/chosen": -3.0513670444488525,
+      "logits/rejected": -3.0258631706237793,
+      "logps/chosen": -58.162193298339844,
+      "logps/rejected": -58.37177276611328,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.026162385940551758,
+      "rewards/margins": 0.01831069216132164,
+      "rewards/rejected": -0.0444730743765831,
+      "step": 1560
+    },
+    {
+      "epoch": 1.082012405237767,
+      "grad_norm": 1.2734873294830322,
+      "learning_rate": 2.5796994441216392e-08,
+      "logits/chosen": -3.025871753692627,
+      "logits/rejected": -3.008380174636841,
+      "logps/chosen": -57.0385856628418,
+      "logps/rejected": -57.39351272583008,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.027973037213087082,
+      "rewards/margins": 0.017069904133677483,
+      "rewards/rejected": -0.045042943209409714,
+      "step": 1570
+    },
+    {
+      "epoch": 1.0889042039972432,
+      "grad_norm": 1.2615596055984497,
+      "learning_rate": 2.5496293313861533e-08,
+      "logits/chosen": -3.0057692527770996,
+      "logits/rejected": -2.9775753021240234,
+      "logps/chosen": -55.155418395996094,
+      "logps/rejected": -56.9516716003418,
+      "loss": 0.685,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.02907036617398262,
+      "rewards/margins": 0.01720438338816166,
+      "rewards/rejected": -0.04627475142478943,
+      "step": 1580
+    },
+    {
+      "epoch": 1.0957960027567195,
+      "grad_norm": 1.260473370552063,
+      "learning_rate": 2.519552033776168e-08,
+      "logits/chosen": -2.992969512939453,
+      "logits/rejected": -2.9799506664276123,
+      "logps/chosen": -57.432411193847656,
+      "logps/rejected": -58.8470458984375,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.029672112315893173,
+      "rewards/margins": 0.015414416790008545,
+      "rewards/rejected": -0.04508653283119202,
+      "step": 1590
+    },
+    {
+      "epoch": 1.1026878015161956,
+      "grad_norm": 1.3105090856552124,
+      "learning_rate": 2.4894719056039933e-08,
+      "logits/chosen": -3.059690237045288,
+      "logits/rejected": -3.0416653156280518,
+      "logps/chosen": -57.756431579589844,
+      "logps/rejected": -58.930335998535156,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.02842426858842373,
+      "rewards/margins": 0.014618475921452045,
+      "rewards/rejected": -0.04304274171590805,
+      "step": 1600
+    },
+    {
+      "epoch": 1.1026878015161956,
+      "eval_logits/chosen": -3.117943286895752,
+      "eval_logits/rejected": -3.112224817276001,
+      "eval_logps/chosen": -59.45923614501953,
+      "eval_logps/rejected": -64.5576171875,
+      "eval_loss": 0.6901748180389404,
+      "eval_rewards/accuracies": 0.5887546539306641,
+      "eval_rewards/chosen": -0.007473426405340433,
+      "eval_rewards/margins": 0.00630148034542799,
+      "eval_rewards/rejected": -0.013774906285107136,
+      "eval_runtime": 383.3589,
+      "eval_samples_per_second": 11.227,
+      "eval_steps_per_second": 1.403,
+      "step": 1600
+    },
+    {
+      "epoch": 1.109579600275672,
+      "grad_norm": 1.2811577320098877,
+      "learning_rate": 2.459393301591723e-08,
+      "logits/chosen": -3.0472471714019775,
+      "logits/rejected": -3.0280234813690186,
+      "logps/chosen": -56.1804313659668,
+      "logps/rejected": -56.194740295410156,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.029790574684739113,
+      "rewards/margins": 0.014425704255700111,
+      "rewards/rejected": -0.044216278940439224,
+      "step": 1610
+    },
+    {
+      "epoch": 1.1164713990351482,
+      "grad_norm": 1.2719690799713135,
+      "learning_rate": 2.429320576240796e-08,
+      "logits/chosen": -2.983424186706543,
+      "logits/rejected": -2.960758686065674,
+      "logps/chosen": -57.0593147277832,
+      "logps/rejected": -57.68733596801758,
+      "loss": 0.684,
+      "rewards/accuracies": 0.6703125238418579,
+      "rewards/chosen": -0.0267618540674448,
+      "rewards/margins": 0.01918993890285492,
+      "rewards/rejected": -0.04595179110765457,
+      "step": 1620
+    },
+    {
+      "epoch": 1.1233631977946243,
+      "grad_norm": 1.274936556816101,
+      "learning_rate": 2.3992580832015937e-08,
+      "logits/chosen": -3.0748069286346436,
+      "logits/rejected": -3.0516257286071777,
+      "logps/chosen": -57.92633056640625,
+      "logps/rejected": -57.392669677734375,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.028706436976790428,
+      "rewards/margins": 0.016950782388448715,
+      "rewards/rejected": -0.04565722122788429,
+      "step": 1630
+    },
+    {
+      "epoch": 1.1302549965541007,
+      "grad_norm": 1.3350669145584106,
+      "learning_rate": 2.3692101746431582e-08,
+      "logits/chosen": -2.999372720718384,
+      "logits/rejected": -2.9686479568481445,
+      "logps/chosen": -57.523155212402344,
+      "logps/rejected": -57.58971405029297,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.645312488079071,
+      "rewards/chosen": -0.0288604237139225,
+      "rewards/margins": 0.018762212246656418,
+      "rewards/rejected": -0.04762263223528862,
+      "step": 1640
+    },
+    {
+      "epoch": 1.1371467953135768,
+      "grad_norm": 1.1888097524642944,
+      "learning_rate": 2.3391812006231252e-08,
+      "logits/chosen": -3.051567792892456,
+      "logits/rejected": -3.024486780166626,
+      "logps/chosen": -57.35553741455078,
+      "logps/rejected": -57.0074462890625,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.5921875238418579,
+      "rewards/chosen": -0.030893787741661072,
+      "rewards/margins": 0.01476077176630497,
+      "rewards/rejected": -0.04565456137061119,
+      "step": 1650
+    },
+    {
+      "epoch": 1.144038594073053,
+      "grad_norm": 1.1799283027648926,
+      "learning_rate": 2.3091755084579655e-08,
+      "logits/chosen": -3.032055377960205,
+      "logits/rejected": -3.004883289337158,
+      "logps/chosen": -56.0220832824707,
+      "logps/rejected": -55.85259246826172,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.033284805715084076,
+      "rewards/margins": 0.018482720479369164,
+      "rewards/rejected": -0.05176752805709839,
+      "step": 1660
+    },
+    {
+      "epoch": 1.1509303928325294,
+      "grad_norm": 1.3657642602920532,
+      "learning_rate": 2.2791974420936168e-08,
+      "logits/chosen": -3.0568909645080566,
+      "logits/rejected": -3.0396854877471924,
+      "logps/chosen": -55.1065673828125,
+      "logps/rejected": -58.755226135253906,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.6390625238418579,
+      "rewards/chosen": -0.03204982727766037,
+      "rewards/margins": 0.018315287306904793,
+      "rewards/rejected": -0.05036511272192001,
+      "step": 1670
+    },
+    {
+      "epoch": 1.1578221915920055,
+      "grad_norm": 1.238609790802002,
+      "learning_rate": 2.2492513414766092e-08,
+      "logits/chosen": -3.015735626220703,
+      "logits/rejected": -2.9980287551879883,
+      "logps/chosen": -58.169593811035156,
+      "logps/rejected": -59.08977127075195,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.6390625238418579,
+      "rewards/chosen": -0.027626004070043564,
+      "rewards/margins": 0.0211968831717968,
+      "rewards/rejected": -0.04882288724184036,
+      "step": 1680
+    },
+    {
+      "epoch": 1.1647139903514818,
+      "grad_norm": 1.2068655490875244,
+      "learning_rate": 2.2193415419257697e-08,
+      "logits/chosen": -3.023995876312256,
+      "logits/rejected": -3.0071871280670166,
+      "logps/chosen": -57.2905158996582,
+      "logps/rejected": -58.23944854736328,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.5953124761581421,
+      "rewards/chosen": -0.03379104658961296,
+      "rewards/margins": 0.014122662134468555,
+      "rewards/rejected": -0.04791371151804924,
+      "step": 1690
+    },
+    {
+      "epoch": 1.171605789110958,
+      "grad_norm": 1.2340092658996582,
+      "learning_rate": 2.1894723735045923e-08,
+      "logits/chosen": -3.015665054321289,
+      "logits/rejected": -2.995542526245117,
+      "logps/chosen": -56.78801727294922,
+      "logps/rejected": -58.15932083129883,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.034682370722293854,
+      "rewards/margins": 0.01648074761033058,
+      "rewards/rejected": -0.051163118332624435,
+      "step": 1700
+    },
+    {
+      "epoch": 1.171605789110958,
+      "eval_logits/chosen": -3.11429500579834,
+      "eval_logits/rejected": -3.1086244583129883,
+      "eval_logps/chosen": -59.66813278198242,
+      "eval_logps/rejected": -64.80902099609375,
+      "eval_loss": 0.6899911165237427,
+      "eval_rewards/accuracies": 0.586663544178009,
+      "eval_rewards/chosen": -0.009562356397509575,
+      "eval_rewards/margins": 0.006726610474288464,
+      "eval_rewards/rejected": -0.016288965940475464,
+      "eval_runtime": 383.0366,
+      "eval_samples_per_second": 11.237,
+      "eval_steps_per_second": 1.405,
+      "step": 1700
+    },
+    {
+      "epoch": 1.1784975878704342,
+      "grad_norm": 1.361463189125061,
+      "learning_rate": 2.159648160394373e-08,
+      "logits/chosen": -3.077711582183838,
+      "logits/rejected": -3.057525396347046,
+      "logps/chosen": -58.2719612121582,
+      "logps/rejected": -58.065284729003906,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.629687488079071,
+      "rewards/chosen": -0.028816580772399902,
+      "rewards/margins": 0.018897056579589844,
+      "rewards/rejected": -0.047713637351989746,
+      "step": 1710
+    },
+    {
+      "epoch": 1.1853893866299103,
+      "grad_norm": 1.315765619277954,
+      "learning_rate": 2.1298732202681956e-08,
+      "logits/chosen": -2.9981892108917236,
+      "logits/rejected": -2.9703431129455566,
+      "logps/chosen": -57.9752311706543,
+      "logps/rejected": -57.5850944519043,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.6234375238418579,
+      "rewards/chosen": -0.03318381309509277,
+      "rewards/margins": 0.02043316699564457,
+      "rewards/rejected": -0.053616978228092194,
+      "step": 1720
+    },
+    {
+      "epoch": 1.1922811853893867,
+      "grad_norm": 1.2448103427886963,
+      "learning_rate": 2.1001518636658567e-08,
+      "logits/chosen": -3.0659806728363037,
+      "logits/rejected": -3.0397450923919678,
+      "logps/chosen": -57.8054313659668,
+      "logps/rejected": -58.7362174987793,
+      "loss": 0.684,
+      "rewards/accuracies": 0.6390625238418579,
+      "rewards/chosen": -0.030917003750801086,
+      "rewards/margins": 0.019438743591308594,
+      "rewards/rejected": -0.05035575106739998,
+      "step": 1730
+    },
+    {
+      "epoch": 1.1991729841488628,
+      "grad_norm": 1.3165340423583984,
+      "learning_rate": 2.0704883933698286e-08,
+      "logits/chosen": -3.0220000743865967,
+      "logits/rejected": -2.988614559173584,
+      "logps/chosen": -59.221092224121094,
+      "logps/rejected": -56.499412536621094,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.03028092160820961,
+      "rewards/margins": 0.020808402448892593,
+      "rewards/rejected": -0.0510893277823925,
+      "step": 1740
+    },
+    {
+      "epoch": 1.206064782908339,
+      "grad_norm": 1.2597706317901611,
+      "learning_rate": 2.0408871037823392e-08,
+      "logits/chosen": -3.050657033920288,
+      "logits/rejected": -3.0249600410461426,
+      "logps/chosen": -58.73793411254883,
+      "logps/rejected": -58.411109924316406,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.032266996800899506,
+      "rewards/margins": 0.020968910306692123,
+      "rewards/rejected": -0.05323590710759163,
+      "step": 1750
+    },
+    {
+      "epoch": 1.2129565816678154,
+      "grad_norm": 1.3488494157791138,
+      "learning_rate": 2.0113522803036697e-08,
+      "logits/chosen": -3.0145840644836426,
+      "logits/rejected": -2.9919447898864746,
+      "logps/chosen": -57.56928253173828,
+      "logps/rejected": -59.618064880371094,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.614062488079071,
+      "rewards/chosen": -0.033350322395563126,
+      "rewards/margins": 0.020749535411596298,
+      "rewards/rejected": -0.054099857807159424,
+      "step": 1760
+    },
+    {
+      "epoch": 1.2198483804272915,
+      "grad_norm": 1.2959116697311401,
+      "learning_rate": 1.981888198711757e-08,
+      "logits/chosen": -3.0438103675842285,
+      "logits/rejected": -3.0201594829559326,
+      "logps/chosen": -56.78424072265625,
+      "logps/rejected": -58.80998992919922,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.03599938377737999,
+      "rewards/margins": 0.018340986222028732,
+      "rewards/rejected": -0.05434036999940872,
+      "step": 1770
+    },
+    {
+      "epoch": 1.2267401791867678,
+      "grad_norm": 1.3248341083526611,
+      "learning_rate": 1.952499124543181e-08,
+      "logits/chosen": -3.050183057785034,
+      "logits/rejected": -3.0215609073638916,
+      "logps/chosen": -59.94126510620117,
+      "logps/rejected": -58.557289123535156,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.614062488079071,
+      "rewards/chosen": -0.03428710997104645,
+      "rewards/margins": 0.019783692434430122,
+      "rewards/rejected": -0.05407080054283142,
+      "step": 1780
+    },
+    {
+      "epoch": 1.233631977946244,
+      "grad_norm": 1.2419155836105347,
+      "learning_rate": 1.923189312475642e-08,
+      "logits/chosen": -3.0126285552978516,
+      "logits/rejected": -2.989089250564575,
+      "logps/chosen": -57.96059036254883,
+      "logps/rejected": -58.968994140625,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.035335466265678406,
+      "rewards/margins": 0.01862800493836403,
+      "rewards/rejected": -0.05396346375346184,
+      "step": 1790
+    },
+    {
+      "epoch": 1.2405237767057202,
+      "grad_norm": 1.3300213813781738,
+      "learning_rate": 1.8939630057120098e-08,
+      "logits/chosen": -3.000619888305664,
+      "logits/rejected": -2.9799740314483643,
+      "logps/chosen": -58.02915573120117,
+      "logps/rejected": -60.03089141845703,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.614062488079071,
+      "rewards/chosen": -0.03870057314634323,
+      "rewards/margins": 0.01641600951552391,
+      "rewards/rejected": -0.05511658638715744,
+      "step": 1800
+    },
+    {
+      "epoch": 1.2405237767057202,
+      "eval_logits/chosen": -3.11264705657959,
+      "eval_logits/rejected": -3.1069631576538086,
+      "eval_logps/chosen": -59.91139221191406,
+      "eval_logps/rejected": -65.09736633300781,
+      "eval_loss": 0.6897966265678406,
+      "eval_rewards/accuracies": 0.582713782787323,
+      "eval_rewards/chosen": -0.01199500635266304,
+      "eval_rewards/margins": 0.007177378050982952,
+      "eval_rewards/rejected": -0.019172383472323418,
+      "eval_runtime": 383.0708,
+      "eval_samples_per_second": 11.236,
+      "eval_steps_per_second": 1.404,
+      "step": 1800
+    },
+    {
+      "epoch": 1.2474155754651963,
+      "grad_norm": 1.2643280029296875,
+      "learning_rate": 1.8648244353660288e-08,
+      "logits/chosen": -3.0149238109588623,
+      "logits/rejected": -2.9926235675811768,
+      "logps/chosen": -59.53852081298828,
+      "logps/rejected": -58.4305419921875,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.03731811046600342,
+      "rewards/margins": 0.015873271971940994,
+      "rewards/rejected": -0.05319138243794441,
+      "step": 1810
+    },
+    {
+      "epoch": 1.2543073742246726,
+      "grad_norm": 1.353582501411438,
+      "learning_rate": 1.835777819849779e-08,
+      "logits/chosen": -3.084817886352539,
+      "logits/rejected": -3.052018880844116,
+      "logps/chosen": -58.27227783203125,
+      "logps/rejected": -58.49510955810547,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.676562488079071,
+      "rewards/chosen": -0.034875739365816116,
+      "rewards/margins": 0.023144185543060303,
+      "rewards/rejected": -0.05801992490887642,
+      "step": 1820
+    },
+    {
+      "epoch": 1.2611991729841487,
+      "grad_norm": 1.3098019361495972,
+      "learning_rate": 1.806827364262974e-08,
+      "logits/chosen": -2.970393657684326,
+      "logits/rejected": -2.9504239559173584,
+      "logps/chosen": -57.546607971191406,
+      "logps/rejected": -58.0165901184082,
+      "loss": 0.687,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0393044538795948,
+      "rewards/margins": 0.013580908067524433,
+      "rewards/rejected": -0.05288536101579666,
+      "step": 1830
+    },
+    {
+      "epoch": 1.268090971743625,
+      "grad_norm": 1.2913509607315063,
+      "learning_rate": 1.7779772597841818e-08,
+      "logits/chosen": -3.0347402095794678,
+      "logits/rejected": -3.008613109588623,
+      "logps/chosen": -58.36812210083008,
+      "logps/rejected": -58.02600860595703,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.03452888876199722,
+      "rewards/margins": 0.022128187119960785,
+      "rewards/rejected": -0.05665707588195801,
+      "step": 1840
+    },
+    {
+      "epoch": 1.2749827705031014,
+      "grad_norm": 1.252109169960022,
+      "learning_rate": 1.749231683064069e-08,
+      "logits/chosen": -2.9613466262817383,
+      "logits/rejected": -2.9358131885528564,
+      "logps/chosen": -57.309776306152344,
+      "logps/rejected": -57.78460693359375,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.635937511920929,
+      "rewards/chosen": -0.03900003433227539,
+      "rewards/margins": 0.017551228404045105,
+      "rewards/rejected": -0.056551266461610794,
+      "step": 1850
+    },
+    {
+      "epoch": 1.2818745692625775,
+      "grad_norm": 1.3581938743591309,
+      "learning_rate": 1.7205947956207416e-08,
+      "logits/chosen": -2.9560749530792236,
+      "logits/rejected": -2.9271953105926514,
+      "logps/chosen": -58.55373001098633,
+      "logps/rejected": -58.74352264404297,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.6265624761581421,
+      "rewards/chosen": -0.03720385953783989,
+      "rewards/margins": 0.021904241293668747,
+      "rewards/rejected": -0.059108100831508636,
+      "step": 1860
+    },
+    {
+      "epoch": 1.2887663680220538,
+      "grad_norm": 1.377907395362854,
+      "learning_rate": 1.69207074323728e-08,
+      "logits/chosen": -3.007751703262329,
+      "logits/rejected": -2.990427255630493,
+      "logps/chosen": -57.58440017700195,
+      "logps/rejected": -57.02080154418945,
+      "loss": 0.685,
+      "rewards/accuracies": 0.598437488079071,
+      "rewards/chosen": -0.0374065637588501,
+      "rewards/margins": 0.01753416657447815,
+      "rewards/rejected": -0.05494073033332825,
+      "step": 1870
+    },
+    {
+      "epoch": 1.29565816678153,
+      "grad_norm": 1.3684296607971191,
+      "learning_rate": 1.6636636553615502e-08,
+      "logits/chosen": -2.969104290008545,
+      "logits/rejected": -2.9459190368652344,
+      "logps/chosen": -57.26690673828125,
+      "logps/rejected": -58.23255157470703,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.03711014613509178,
+      "rewards/margins": 0.019704418256878853,
+      "rewards/rejected": -0.056814562529325485,
+      "step": 1880
+    },
+    {
+      "epoch": 1.3025499655410062,
+      "grad_norm": 1.2850284576416016,
+      "learning_rate": 1.6353776445083815e-08,
+      "logits/chosen": -3.0240912437438965,
+      "logits/rejected": -3.0120110511779785,
+      "logps/chosen": -55.608795166015625,
+      "logps/rejected": -58.87140655517578,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.6234375238418579,
+      "rewards/chosen": -0.03577902913093567,
+      "rewards/margins": 0.019716601818799973,
+      "rewards/rejected": -0.05549562722444534,
+      "step": 1890
+    },
+    {
+      "epoch": 1.3094417643004825,
+      "grad_norm": 1.2016737461090088,
+      "learning_rate": 1.6072168056641944e-08,
+      "logits/chosen": -3.0512993335723877,
+      "logits/rejected": -3.0232746601104736,
+      "logps/chosen": -59.6502571105957,
+      "logps/rejected": -57.75080490112305,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.03732553869485855,
+      "rewards/margins": 0.02288132533431053,
+      "rewards/rejected": -0.06020686775445938,
+      "step": 1900
+    },
+    {
+      "epoch": 1.3094417643004825,
+      "eval_logits/chosen": -3.109053134918213,
+      "eval_logits/rejected": -3.1033873558044434,
+      "eval_logps/chosen": -60.10012435913086,
+      "eval_logps/rejected": -65.30886840820312,
+      "eval_loss": 0.6897038221359253,
+      "eval_rewards/accuracies": 0.5824813842773438,
+      "eval_rewards/chosen": -0.013882317580282688,
+      "eval_rewards/margins": 0.007405092474073172,
+      "eval_rewards/rejected": -0.02128741145133972,
+      "eval_runtime": 383.365,
+      "eval_samples_per_second": 11.227,
+      "eval_steps_per_second": 1.403,
+      "step": 1900
+    },
+    {
+      "epoch": 1.3163335630599586,
+      "grad_norm": 1.3704478740692139,
+      "learning_rate": 1.5791852156941672e-08,
+      "logits/chosen": -2.9737439155578613,
+      "logits/rejected": -2.9562785625457764,
+      "logps/chosen": -58.3253173828125,
+      "logps/rejected": -58.146751403808594,
+      "loss": 0.685,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.03781484067440033,
+      "rewards/margins": 0.017499810084700584,
+      "rewards/rejected": -0.05531464144587517,
+      "step": 1910
+    },
+    {
+      "epoch": 1.323225361819435,
+      "grad_norm": 1.304793119430542,
+      "learning_rate": 1.5512869327520234e-08,
+      "logits/chosen": -3.0345849990844727,
+      "logits/rejected": -3.006624937057495,
+      "logps/chosen": -59.780555725097656,
+      "logps/rejected": -59.3910026550293,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.6390625238418579,
+      "rewards/chosen": -0.038787275552749634,
+      "rewards/margins": 0.022125843912363052,
+      "rewards/rejected": -0.060913123190402985,
+      "step": 1920
+    },
+    {
+      "epoch": 1.330117160578911,
+      "grad_norm": 1.281746506690979,
+      "learning_rate": 1.52352599569253e-08,
+      "logits/chosen": -3.0547759532928467,
+      "logits/rejected": -3.0221850872039795,
+      "logps/chosen": -57.975791931152344,
+      "logps/rejected": -56.89446258544922,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.629687488079071,
+      "rewards/chosen": -0.03873300552368164,
+      "rewards/margins": 0.018973171710968018,
+      "rewards/rejected": -0.05770616978406906,
+      "step": 1930
+    },
+    {
+      "epoch": 1.3370089593383874,
+      "grad_norm": 1.3232277631759644,
+      "learning_rate": 1.4959064234867925e-08,
+      "logits/chosen": -2.9585065841674805,
+      "logits/rejected": -2.936213970184326,
+      "logps/chosen": -56.48392868041992,
+      "logps/rejected": -58.73712158203125,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.03964737430214882,
+      "rewards/margins": 0.022885087877511978,
+      "rewards/rejected": -0.0625324696302414,
+      "step": 1940
+    },
+    {
+      "epoch": 1.3439007580978635,
+      "grad_norm": 1.266557216644287,
+      "learning_rate": 1.4684322146404215e-08,
+      "logits/chosen": -3.035268783569336,
+      "logits/rejected": -3.01952862739563,
+      "logps/chosen": -56.704620361328125,
+      "logps/rejected": -59.3856086730957,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.04403103142976761,
+      "rewards/margins": 0.017256928607821465,
+      "rewards/rejected": -0.061287958174943924,
+      "step": 1950
+    },
+    {
+      "epoch": 1.3507925568573398,
+      "grad_norm": 1.2548446655273438,
+      "learning_rate": 1.4411073466146698e-08,
+      "logits/chosen": -3.0059549808502197,
+      "logits/rejected": -2.9832520484924316,
+      "logps/chosen": -58.505836486816406,
+      "logps/rejected": -60.24690628051758,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.629687488079071,
+      "rewards/chosen": -0.03818178176879883,
+      "rewards/margins": 0.01996403932571411,
+      "rewards/rejected": -0.058145828545093536,
+      "step": 1960
+    },
+    {
+      "epoch": 1.3576843556168159,
+      "grad_norm": 1.3594894409179688,
+      "learning_rate": 1.413935775250609e-08,
+      "logits/chosen": -2.9915037155151367,
+      "logits/rejected": -2.9657304286956787,
+      "logps/chosen": -58.22015380859375,
+      "logps/rejected": -58.225196838378906,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.03904888778924942,
+      "rewards/margins": 0.023439262062311172,
+      "rewards/rejected": -0.06248814985156059,
+      "step": 1970
+    },
+    {
+      "epoch": 1.3645761543762922,
+      "grad_norm": 1.305829644203186,
+      "learning_rate": 1.3869214341964411e-08,
+      "logits/chosen": -2.9901621341705322,
+      "logits/rejected": -2.972623825073242,
+      "logps/chosen": -58.67400360107422,
+      "logps/rejected": -59.30767059326172,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0412491112947464,
+      "rewards/margins": 0.01679963245987892,
+      "rewards/rejected": -0.058048736304044724,
+      "step": 1980
+    },
+    {
+      "epoch": 1.3714679531357685,
+      "grad_norm": 1.2267251014709473,
+      "learning_rate": 1.3600682343380247e-08,
+      "logits/chosen": -2.953930616378784,
+      "logits/rejected": -2.9335622787475586,
+      "logps/chosen": -58.66155242919922,
+      "logps/rejected": -59.33677291870117,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.039415620267391205,
+      "rewards/margins": 0.020520631223917007,
+      "rewards/rejected": -0.05993625521659851,
+      "step": 1990
+    },
+    {
+      "epoch": 1.3783597518952446,
+      "grad_norm": 1.2556020021438599,
+      "learning_rate": 1.3333800632326858e-08,
+      "logits/chosen": -3.0334630012512207,
+      "logits/rejected": -3.0166397094726562,
+      "logps/chosen": -58.55223846435547,
+      "logps/rejected": -59.941978454589844,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.0418681763112545,
+      "rewards/margins": 0.017354335635900497,
+      "rewards/rejected": -0.0592225082218647,
+      "step": 2000
+    },
+    {
+      "epoch": 1.3783597518952446,
+      "eval_logits/chosen": -3.1071271896362305,
+      "eval_logits/rejected": -3.101414680480957,
+      "eval_logps/chosen": -60.261566162109375,
+      "eval_logps/rejected": -65.51657104492188,
+      "eval_loss": 0.6894936561584473,
+      "eval_rewards/accuracies": 0.5906133651733398,
+      "eval_rewards/chosen": -0.015496725216507912,
+      "eval_rewards/margins": 0.007867763750255108,
+      "eval_rewards/rejected": -0.023364488035440445,
+      "eval_runtime": 383.0695,
+      "eval_samples_per_second": 11.236,
+      "eval_steps_per_second": 1.404,
+      "step": 2000
+    },
+    {
+      "epoch": 1.385251550654721,
+      "grad_norm": 1.4159228801727295,
+      "learning_rate": 1.3068607845464202e-08,
+      "logits/chosen": -2.9797048568725586,
+      "logits/rejected": -2.952303171157837,
+      "logps/chosen": -59.8831901550293,
+      "logps/rejected": -59.22021484375,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -0.041412778198719025,
+      "rewards/margins": 0.0192607082426548,
+      "rewards/rejected": -0.06067349389195442,
+      "step": 2010
+    },
+    {
+      "epoch": 1.392143349414197,
+      "grad_norm": 1.3155369758605957,
+      "learning_rate": 1.2805142374945437e-08,
+      "logits/chosen": -3.0014488697052,
+      "logits/rejected": -2.978201389312744,
+      "logps/chosen": -57.95537185668945,
+      "logps/rejected": -59.4213752746582,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.03732657432556152,
+      "rewards/margins": 0.022235842421650887,
+      "rewards/rejected": -0.05956241488456726,
+      "step": 2020
+    },
+    {
+      "epoch": 1.3990351481736734,
+      "grad_norm": 1.2982782125473022,
+      "learning_rate": 1.254344236285888e-08,
+      "logits/chosen": -2.984819173812866,
+      "logits/rejected": -2.9616565704345703,
+      "logps/chosen": -58.2531623840332,
+      "logps/rejected": -59.219261169433594,
+      "loss": 0.684,
+      "rewards/accuracies": 0.629687488079071,
+      "rewards/chosen": -0.039121102541685104,
+      "rewards/margins": 0.019388314336538315,
+      "rewards/rejected": -0.05850941687822342,
+      "step": 2030
+    },
+    {
+      "epoch": 1.4059269469331497,
+      "grad_norm": 1.328587532043457,
+      "learning_rate": 1.2283545695706135e-08,
+      "logits/chosen": -2.9852476119995117,
+      "logits/rejected": -2.9641222953796387,
+      "logps/chosen": -58.166831970214844,
+      "logps/rejected": -58.347557067871094,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.042398639023303986,
+      "rewards/margins": 0.019147690385580063,
+      "rewards/rejected": -0.06154633313417435,
+      "step": 2040
+    },
+    {
+      "epoch": 1.4128187456926258,
+      "grad_norm": 1.2532403469085693,
+      "learning_rate": 1.2025489998917254e-08,
+      "logits/chosen": -3.011920690536499,
+      "logits/rejected": -2.985671043395996,
+      "logps/chosen": -61.16510009765625,
+      "logps/rejected": -59.185394287109375,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.03920884430408478,
+      "rewards/margins": 0.019202571362257004,
+      "rewards/rejected": -0.058411408215761185,
+      "step": 2050
+    },
+    {
+      "epoch": 1.4197105444521019,
+      "grad_norm": 1.379606008529663,
+      "learning_rate": 1.1769312631403659e-08,
+      "logits/chosen": -2.9937710762023926,
+      "logits/rejected": -2.9706432819366455,
+      "logps/chosen": -59.001708984375,
+      "logps/rejected": -58.9688720703125,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0409664623439312,
+      "rewards/margins": 0.017764370888471603,
+      "rewards/rejected": -0.0587308332324028,
+      "step": 2060
+    },
+    {
+      "epoch": 1.4266023432115782,
+      "grad_norm": 1.3206267356872559,
+      "learning_rate": 1.1515050680149687e-08,
+      "logits/chosen": -3.0447440147399902,
+      "logits/rejected": -3.0277929306030273,
+      "logps/chosen": -59.411956787109375,
+      "logps/rejected": -60.502716064453125,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.04452786594629288,
+      "rewards/margins": 0.01587734930217266,
+      "rewards/rejected": -0.06040521338582039,
+      "step": 2070
+    },
+    {
+      "epoch": 1.4334941419710545,
+      "grad_norm": 1.2896159887313843,
+      "learning_rate": 1.1262740954843456e-08,
+      "logits/chosen": -3.0021471977233887,
+      "logits/rejected": -2.971998691558838,
+      "logps/chosen": -60.27617645263672,
+      "logps/rejected": -59.302833557128906,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.03701635077595711,
+      "rewards/margins": 0.021184273064136505,
+      "rewards/rejected": -0.058200620114803314,
+      "step": 2080
+    },
+    {
+      "epoch": 1.4403859407305306,
+      "grad_norm": 1.326338768005371,
+      "learning_rate": 1.1012419982547905e-08,
+      "logits/chosen": -2.961845636367798,
+      "logits/rejected": -2.941849946975708,
+      "logps/chosen": -57.25081253051758,
+      "logps/rejected": -59.027015686035156,
+      "loss": 0.684,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.04554927721619606,
+      "rewards/margins": 0.019669197499752045,
+      "rewards/rejected": -0.0652184709906578,
+      "step": 2090
+    },
+    {
+      "epoch": 1.447277739490007,
+      "grad_norm": 1.316919207572937,
+      "learning_rate": 1.0764124002412758e-08,
+      "logits/chosen": -3.0356943607330322,
+      "logits/rejected": -3.012575626373291,
+      "logps/chosen": -56.51226806640625,
+      "logps/rejected": -59.925514221191406,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.04187322035431862,
+      "rewards/margins": 0.02075764164328575,
+      "rewards/rejected": -0.06263085454702377,
+      "step": 2100
+    },
+    {
+      "epoch": 1.447277739490007,
+      "eval_logits/chosen": -3.1055357456207275,
+      "eval_logits/rejected": -3.099832534790039,
+      "eval_logps/chosen": -60.38422775268555,
+      "eval_logps/rejected": -65.65011596679688,
+      "eval_loss": 0.6894546151161194,
+      "eval_rewards/accuracies": 0.5861988663673401,
+      "eval_rewards/chosen": -0.01672333851456642,
+      "eval_rewards/margins": 0.00797655712813139,
+      "eval_rewards/rejected": -0.024699894711375237,
+      "eval_runtime": 383.1595,
+      "eval_samples_per_second": 11.233,
+      "eval_steps_per_second": 1.404,
+      "step": 2100
+    },
+    {
+      "epoch": 1.454169538249483,
+      "grad_norm": 1.3180962800979614,
+      "learning_rate": 1.0517888960428139e-08,
+      "logits/chosen": -2.964921236038208,
+      "logits/rejected": -2.94468355178833,
+      "logps/chosen": -58.661651611328125,
+      "logps/rejected": -59.41533660888672,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.651562511920929,
+      "rewards/chosen": -0.042625896632671356,
+      "rewards/margins": 0.023367973044514656,
+      "rewards/rejected": -0.06599386781454086,
+      "step": 2110
+    },
+    {
+      "epoch": 1.4610613370089593,
+      "grad_norm": 1.2736924886703491,
+      "learning_rate": 1.0273750504220666e-08,
+      "logits/chosen": -2.982936382293701,
+      "logits/rejected": -2.9598629474639893,
+      "logps/chosen": -56.829505920410156,
+      "logps/rejected": -59.5037956237793,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.629687488079071,
+      "rewards/chosen": -0.04348963871598244,
+      "rewards/margins": 0.023124249652028084,
+      "rewards/rejected": -0.06661389768123627,
+      "step": 2120
+    },
+    {
+      "epoch": 1.4679531357684357,
+      "grad_norm": 1.305767297744751,
+      "learning_rate": 1.003174397789269e-08,
+      "logits/chosen": -2.981260299682617,
+      "logits/rejected": -2.9577889442443848,
+      "logps/chosen": -59.381996154785156,
+      "logps/rejected": -59.3449821472168,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.04265505075454712,
+      "rewards/margins": 0.023621436208486557,
+      "rewards/rejected": -0.06627649068832397,
+      "step": 2130
+    },
+    {
+      "epoch": 1.4748449345279118,
+      "grad_norm": 1.2957626581192017,
+      "learning_rate": 9.791904416905526e-09,
+      "logits/chosen": -3.0431559085845947,
+      "logits/rejected": -3.0270159244537354,
+      "logps/chosen": -58.642250061035156,
+      "logps/rejected": -59.5418586730957,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.044856615364551544,
+      "rewards/margins": 0.016749290749430656,
+      "rewards/rejected": -0.06160591170191765,
+      "step": 2140
+    },
+    {
+      "epoch": 1.481736733287388,
+      "grad_norm": 1.3108528852462769,
+      "learning_rate": 9.554266543007328e-09,
+      "logits/chosen": -3.007779598236084,
+      "logits/rejected": -2.9805774688720703,
+      "logps/chosen": -58.5262565612793,
+      "logps/rejected": -59.313941955566406,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.6421874761581421,
+      "rewards/chosen": -0.0379471480846405,
+      "rewards/margins": 0.02591646835207939,
+      "rewards/rejected": -0.0638636127114296,
+      "step": 2150
+    },
+    {
+      "epoch": 1.4886285320468642,
+      "grad_norm": 1.2914735078811646,
+      "learning_rate": 9.318864759206429e-09,
+      "logits/chosen": -2.9647016525268555,
+      "logits/rejected": -2.9434256553649902,
+      "logps/chosen": -58.31499481201172,
+      "logps/rejected": -57.80283737182617,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.6078125238418579,
+      "rewards/chosen": -0.04223569482564926,
+      "rewards/margins": 0.0191799309104681,
+      "rewards/rejected": -0.061415620148181915,
+      "step": 2160
+    },
+    {
+      "epoch": 1.4955203308063405,
+      "grad_norm": 1.3237831592559814,
+      "learning_rate": 9.085733144790744e-09,
+      "logits/chosen": -3.0011582374572754,
+      "logits/rejected": -2.9859848022460938,
+      "logps/chosen": -57.24296188354492,
+      "logps/rejected": -59.595008850097656,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04495619237422943,
+      "rewards/margins": 0.022577274590730667,
+      "rewards/rejected": -0.0675334706902504,
+      "step": 2170
+    },
+    {
+      "epoch": 1.5024121295658168,
+      "grad_norm": 1.409790277481079,
+      "learning_rate": 8.854905450394113e-09,
+      "logits/chosen": -3.0098440647125244,
+      "logits/rejected": -2.983942747116089,
+      "logps/chosen": -59.406158447265625,
+      "logps/rejected": -58.25774002075195,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.04101915657520294,
+      "rewards/margins": 0.020175766199827194,
+      "rewards/rejected": -0.06119491904973984,
+      "step": 2180
+    },
+    {
+      "epoch": 1.509303928325293,
+      "grad_norm": 1.3202848434448242,
+      "learning_rate": 8.626415093110202e-09,
+      "logits/chosen": -2.9948554039001465,
+      "logits/rejected": -2.975142240524292,
+      "logps/chosen": -56.601722717285156,
+      "logps/rejected": -59.769569396972656,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04384131729602814,
+      "rewards/margins": 0.020800283178687096,
+      "rewards/rejected": -0.06464160233736038,
+      "step": 2190
+    },
+    {
+      "epoch": 1.516195727084769,
+      "grad_norm": 1.366294503211975,
+      "learning_rate": 8.40029515165467e-09,
+      "logits/chosen": -3.006235361099243,
+      "logits/rejected": -2.983431577682495,
+      "logps/chosen": -57.9134635925293,
+      "logps/rejected": -58.527076721191406,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.04390479251742363,
+      "rewards/margins": 0.022031091153621674,
+      "rewards/rejected": -0.065935879945755,
+      "step": 2200
+    },
+    {
+      "epoch": 1.516195727084769,
+      "eval_logits/chosen": -3.104142904281616,
+      "eval_logits/rejected": -3.098437547683716,
+      "eval_logps/chosen": -60.504878997802734,
+      "eval_logps/rejected": -65.79142761230469,
+      "eval_loss": 0.6893645524978638,
+      "eval_rewards/accuracies": 0.5873606204986572,
+      "eval_rewards/chosen": -0.01792982593178749,
+      "eval_rewards/margins": 0.008183243684470654,
+      "eval_rewards/rejected": -0.02611307054758072,
+      "eval_runtime": 382.8386,
+      "eval_samples_per_second": 11.242,
+      "eval_steps_per_second": 1.405,
+      "step": 2200
+    },
+    {
+      "epoch": 1.5230875258442453,
+      "grad_norm": 1.290756106376648,
+      "learning_rate": 8.176578361576358e-09,
+      "logits/chosen": -2.9781322479248047,
+      "logits/rejected": -2.959134340286255,
+      "logps/chosen": -56.6655387878418,
+      "logps/rejected": -58.62583541870117,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.04583312198519707,
+      "rewards/margins": 0.01810682937502861,
+      "rewards/rejected": -0.06393995136022568,
+      "step": 2210
+    },
+    {
+      "epoch": 1.5299793246037217,
+      "grad_norm": 1.2776232957839966,
+      "learning_rate": 7.955297110518117e-09,
+      "logits/chosen": -3.0543761253356934,
+      "logits/rejected": -3.028890371322632,
+      "logps/chosen": -58.1224365234375,
+      "logps/rejected": -59.595802307128906,
+      "loss": 0.684,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.04209282249212265,
+      "rewards/margins": 0.019542943686246872,
+      "rewards/rejected": -0.06163576990365982,
+      "step": 2220
+    },
+    {
+      "epoch": 1.5368711233631978,
+      "grad_norm": 1.349440574645996,
+      "learning_rate": 7.73648343352806e-09,
+      "logits/chosen": -3.0225765705108643,
+      "logits/rejected": -2.996718168258667,
+      "logps/chosen": -59.868263244628906,
+      "logps/rejected": -58.85590744018555,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.043428223580121994,
+      "rewards/margins": 0.022792860865592957,
+      "rewards/rejected": -0.06622108817100525,
+      "step": 2230
+    },
+    {
+      "epoch": 1.5437629221226739,
+      "grad_norm": 1.3181602954864502,
+      "learning_rate": 7.520169008421775e-09,
+      "logits/chosen": -2.999849796295166,
+      "logits/rejected": -2.9812533855438232,
+      "logps/chosen": -59.130516052246094,
+      "logps/rejected": -60.17681884765625,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04423438385128975,
+      "rewards/margins": 0.019376900047063828,
+      "rewards/rejected": -0.06361128389835358,
+      "step": 2240
+    },
+    {
+      "epoch": 1.5506547208821502,
+      "grad_norm": 1.389096975326538,
+      "learning_rate": 7.3063851511963535e-09,
+      "logits/chosen": -3.01939058303833,
+      "logits/rejected": -2.992645740509033,
+      "logps/chosen": -59.214324951171875,
+      "logps/rejected": -59.09142303466797,
+      "loss": 0.6823,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.04182130843400955,
+      "rewards/margins": 0.023091908544301987,
+      "rewards/rejected": -0.06491322070360184,
+      "step": 2250
+    },
+    {
+      "epoch": 1.5575465196416265,
+      "grad_norm": 1.2921773195266724,
+      "learning_rate": 7.095162811496716e-09,
+      "logits/chosen": -2.9625678062438965,
+      "logits/rejected": -2.947840452194214,
+      "logps/chosen": -57.61260986328125,
+      "logps/rejected": -58.932640075683594,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.046415556222200394,
+      "rewards/margins": 0.01893490180373192,
+      "rewards/rejected": -0.06535045802593231,
+      "step": 2260
+    },
+    {
+      "epoch": 1.5644383184011028,
+      "grad_norm": 1.2538701295852661,
+      "learning_rate": 6.886532568135017e-09,
+      "logits/chosen": -2.9978413581848145,
+      "logits/rejected": -2.98121976852417,
+      "logps/chosen": -58.2430305480957,
+      "logps/rejected": -60.4543571472168,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.044888339936733246,
+      "rewards/margins": 0.019984986633062363,
+      "rewards/rejected": -0.06487332284450531,
+      "step": 2270
+    },
+    {
+      "epoch": 1.571330117160579,
+      "grad_norm": 1.3960515260696411,
+      "learning_rate": 6.680524624663763e-09,
+      "logits/chosen": -3.0089173316955566,
+      "logits/rejected": -2.977341413497925,
+      "logps/chosen": -60.883209228515625,
+      "logps/rejected": -59.40845489501953,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.6234375238418579,
+      "rewards/chosen": -0.03985728323459625,
+      "rewards/margins": 0.025174889713525772,
+      "rewards/rejected": -0.06503216922283173,
+      "step": 2280
+    },
+    {
+      "epoch": 1.578221915920055,
+      "grad_norm": 1.1940710544586182,
+      "learning_rate": 6.477168805003166e-09,
+      "logits/chosen": -3.00933575630188,
+      "logits/rejected": -2.982250690460205,
+      "logps/chosen": -59.1096076965332,
+      "logps/rejected": -59.2675895690918,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.6390625238418579,
+      "rewards/chosen": -0.04165520519018173,
+      "rewards/margins": 0.023930717259645462,
+      "rewards/rejected": -0.06558592617511749,
+      "step": 2290
+    },
+    {
+      "epoch": 1.5851137146795313,
+      "grad_norm": 1.3209586143493652,
+      "learning_rate": 6.276494549123546e-09,
+      "logits/chosen": -3.050356388092041,
+      "logits/rejected": -3.02972412109375,
+      "logps/chosen": -58.97772979736328,
+      "logps/rejected": -59.48607635498047,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.635937511920929,
+      "rewards/chosen": -0.04832325503230095,
+      "rewards/margins": 0.021100293844938278,
+      "rewards/rejected": -0.06942354887723923,
+      "step": 2300
+    },
+    {
+      "epoch": 1.5851137146795313,
+      "eval_logits/chosen": -3.1029651165008545,
+      "eval_logits/rejected": -3.097285509109497,
+      "eval_logps/chosen": -60.59327697753906,
+      "eval_logps/rejected": -65.90728759765625,
+      "eval_loss": 0.6892400979995728,
+      "eval_rewards/accuracies": 0.5901486873626709,
+      "eval_rewards/chosen": -0.018813807517290115,
+      "eval_rewards/margins": 0.008457801304757595,
+      "eval_rewards/rejected": -0.027271609753370285,
+      "eval_runtime": 383.2974,
+      "eval_samples_per_second": 11.229,
+      "eval_steps_per_second": 1.404,
+      "step": 2300
+    },
+    {
+      "epoch": 1.5920055134390076,
+      "grad_norm": 1.2838672399520874,
+      "learning_rate": 6.078530908783283e-09,
+      "logits/chosen": -2.946258068084717,
+      "logits/rejected": -2.9289188385009766,
+      "logps/chosen": -57.40240478515625,
+      "logps/rejected": -58.80238723754883,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.04573064297437668,
+      "rewards/margins": 0.020323526114225388,
+      "rewards/rejected": -0.06605416536331177,
+      "step": 2310
+    },
+    {
+      "epoch": 1.598897312198484,
+      "grad_norm": 1.3529164791107178,
+      "learning_rate": 5.883306543322963e-09,
+      "logits/chosen": -3.0067434310913086,
+      "logits/rejected": -2.983191967010498,
+      "logps/chosen": -57.39630126953125,
+      "logps/rejected": -58.9193000793457,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.045682333409786224,
+      "rewards/margins": 0.020494289696216583,
+      "rewards/rejected": -0.06617662310600281,
+      "step": 2320
+    },
+    {
+      "epoch": 1.60578911095796,
+      "grad_norm": 1.3721948862075806,
+      "learning_rate": 5.690849715516346e-09,
+      "logits/chosen": -2.9921982288360596,
+      "logits/rejected": -2.972947597503662,
+      "logps/chosen": -58.18434524536133,
+      "logps/rejected": -59.79640579223633,
+      "loss": 0.684,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.049553245306015015,
+      "rewards/margins": 0.019603563472628593,
+      "rewards/rejected": -0.06915681809186935,
+      "step": 2330
+    },
+    {
+      "epoch": 1.6126809097174362,
+      "grad_norm": 1.326244831085205,
+      "learning_rate": 5.50118828747877e-09,
+      "logits/chosen": -3.013467311859131,
+      "logits/rejected": -2.985992908477783,
+      "logps/chosen": -59.114105224609375,
+      "logps/rejected": -59.962989807128906,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.04709188640117645,
+      "rewards/margins": 0.024677757173776627,
+      "rewards/rejected": -0.07176963984966278,
+      "step": 2340
+    },
+    {
+      "epoch": 1.6195727084769125,
+      "grad_norm": 1.3307464122772217,
+      "learning_rate": 5.314349716633484e-09,
+      "logits/chosen": -2.999783515930176,
+      "logits/rejected": -2.978919744491577,
+      "logps/chosen": -58.51006317138672,
+      "logps/rejected": -59.8082389831543,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.04656077176332474,
+      "rewards/margins": 0.015919920057058334,
+      "rewards/rejected": -0.06248069554567337,
+      "step": 2350
+    },
+    {
+      "epoch": 1.6264645072363888,
+      "grad_norm": 1.3505630493164062,
+      "learning_rate": 5.130361051736656e-09,
+      "logits/chosen": -2.992077589035034,
+      "logits/rejected": -2.9786789417266846,
+      "logps/chosen": -57.856048583984375,
+      "logps/rejected": -58.31081008911133,
+      "loss": 0.685,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.05111172795295715,
+      "rewards/margins": 0.01770883984863758,
+      "rewards/rejected": -0.06882055848836899,
+      "step": 2360
+    },
+    {
+      "epoch": 1.633356305995865,
+      "grad_norm": 1.3144451379776,
+      "learning_rate": 4.9492489289614884e-09,
+      "logits/chosen": -2.9724109172821045,
+      "logits/rejected": -2.9529943466186523,
+      "logps/chosen": -58.423919677734375,
+      "logps/rejected": -59.29913330078125,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.053545523434877396,
+      "rewards/margins": 0.019974233582615852,
+      "rewards/rejected": -0.0735197439789772,
+      "step": 2370
+    },
+    {
+      "epoch": 1.640248104755341,
+      "grad_norm": 1.3916033506393433,
+      "learning_rate": 4.771039568042076e-09,
+      "logits/chosen": -3.004544734954834,
+      "logits/rejected": -2.988704204559326,
+      "logps/chosen": -57.34346389770508,
+      "logps/rejected": -61.55018997192383,
+      "loss": 0.683,
+      "rewards/accuracies": 0.635937511920929,
+      "rewards/chosen": -0.04598530754446983,
+      "rewards/margins": 0.021662291139364243,
+      "rewards/rejected": -0.06764759868383408,
+      "step": 2380
+    },
+    {
+      "epoch": 1.6471399035148173,
+      "grad_norm": 1.4496431350708008,
+      "learning_rate": 4.595758768477576e-09,
+      "logits/chosen": -3.0240254402160645,
+      "logits/rejected": -3.011583089828491,
+      "logps/chosen": -58.51326370239258,
+      "logps/rejected": -60.47749710083008,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.5921875238418579,
+      "rewards/chosen": -0.05048090219497681,
+      "rewards/margins": 0.01893479749560356,
+      "rewards/rejected": -0.06941570341587067,
+      "step": 2390
+    },
+    {
+      "epoch": 1.6540317022742936,
+      "grad_norm": 1.3277703523635864,
+      "learning_rate": 4.423431905797162e-09,
+      "logits/chosen": -3.039842128753662,
+      "logits/rejected": -3.0183472633361816,
+      "logps/chosen": -58.69083786010742,
+      "logps/rejected": -60.8518180847168,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.6109374761581421,
+      "rewards/chosen": -0.04716577008366585,
+      "rewards/margins": 0.020657068118453026,
+      "rewards/rejected": -0.06782282888889313,
+      "step": 2400
+    },
+    {
+      "epoch": 1.6540317022742936,
+      "eval_logits/chosen": -3.1017863750457764,
+      "eval_logits/rejected": -3.09609055519104,
+      "eval_logps/chosen": -60.646873474121094,
+      "eval_logps/rejected": -65.97390747070312,
+      "eval_loss": 0.6891800761222839,
+      "eval_rewards/accuracies": 0.5861988663673401,
+      "eval_rewards/chosen": -0.019349750131368637,
+      "eval_rewards/margins": 0.008588053286075592,
+      "eval_rewards/rejected": -0.027937807142734528,
+      "eval_runtime": 383.0908,
+      "eval_samples_per_second": 11.235,
+      "eval_steps_per_second": 1.404,
+      "step": 2400
+    },
+    {
+      "epoch": 1.66092350103377,
+      "grad_norm": 1.3616927862167358,
+      "learning_rate": 4.254083927886443e-09,
+      "logits/chosen": -3.052434206008911,
+      "logits/rejected": -3.0306789875030518,
+      "logps/chosen": -60.23524856567383,
+      "logps/rejected": -59.88490676879883,
+      "loss": 0.685,
+      "rewards/accuracies": 0.598437488079071,
+      "rewards/chosen": -0.04647786170244217,
+      "rewards/margins": 0.017560753971338272,
+      "rewards/rejected": -0.06403861939907074,
+      "step": 2410
+    },
+    {
+      "epoch": 1.667815299793246,
+      "grad_norm": 1.3540840148925781,
+      "learning_rate": 4.0877393513756795e-09,
+      "logits/chosen": -3.0015170574188232,
+      "logits/rejected": -2.9834518432617188,
+      "logps/chosen": -58.74982452392578,
+      "logps/rejected": -59.712005615234375,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.045878536999225616,
+      "rewards/margins": 0.018289810046553612,
+      "rewards/rejected": -0.06416834890842438,
+      "step": 2420
+    },
+    {
+      "epoch": 1.6747070985527222,
+      "grad_norm": 1.319036841392517,
+      "learning_rate": 3.924422258090529e-09,
+      "logits/chosen": -2.939756155014038,
+      "logits/rejected": -2.919666290283203,
+      "logps/chosen": -58.5392951965332,
+      "logps/rejected": -59.17338943481445,
+      "loss": 0.684,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.04658854380249977,
+      "rewards/margins": 0.019758421927690506,
+      "rewards/rejected": -0.06634696573019028,
+      "step": 2430
+    },
+    {
+      "epoch": 1.6815988973121985,
+      "grad_norm": 1.3268150091171265,
+      "learning_rate": 3.764156291565693e-09,
+      "logits/chosen": -3.0177316665649414,
+      "logits/rejected": -2.9926140308380127,
+      "logps/chosen": -58.570648193359375,
+      "logps/rejected": -58.78978729248047,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04440145939588547,
+      "rewards/margins": 0.022650301456451416,
+      "rewards/rejected": -0.06705176085233688,
+      "step": 2440
+    },
+    {
+      "epoch": 1.6884906960716748,
+      "grad_norm": 1.3211112022399902,
+      "learning_rate": 3.6069646536220357e-09,
+      "logits/chosen": -2.998032331466675,
+      "logits/rejected": -2.9721641540527344,
+      "logps/chosen": -60.38201904296875,
+      "logps/rejected": -60.990257263183594,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.682812511920929,
+      "rewards/chosen": -0.040809061378240585,
+      "rewards/margins": 0.026985710486769676,
+      "rewards/rejected": -0.06779477745294571,
+      "step": 2450
+    },
+    {
+      "epoch": 1.6953824948311509,
+      "grad_norm": 1.285194993019104,
+      "learning_rate": 3.4528701010076155e-09,
+      "logits/chosen": -3.003739833831787,
+      "logits/rejected": -2.9794375896453857,
+      "logps/chosen": -60.3626823425293,
+      "logps/rejected": -61.65105438232422,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.045110072940588,
+      "rewards/margins": 0.02330555021762848,
+      "rewards/rejected": -0.06841562688350677,
+      "step": 2460
+    },
+    {
+      "epoch": 1.7022742935906272,
+      "grad_norm": 1.2852113246917725,
+      "learning_rate": 3.3018949421032003e-09,
+      "logits/chosen": -3.0109405517578125,
+      "logits/rejected": -2.9975745677948,
+      "logps/chosen": -58.778053283691406,
+      "logps/rejected": -59.56703567504883,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.5703125,
+      "rewards/chosen": -0.04846884682774544,
+      "rewards/margins": 0.01748683862388134,
+      "rewards/rejected": -0.06595568358898163,
+      "step": 2470
+    },
+    {
+      "epoch": 1.7091660923501033,
+      "grad_norm": 1.258186936378479,
+      "learning_rate": 3.154061033692651e-09,
+      "logits/chosen": -3.0072379112243652,
+      "logits/rejected": -2.979935884475708,
+      "logps/chosen": -59.062705993652344,
+      "logps/rejected": -58.03764724731445,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.04467375949025154,
+      "rewards/margins": 0.026565441861748695,
+      "rewards/rejected": -0.07123919576406479,
+      "step": 2480
+    },
+    {
+      "epoch": 1.7160578911095796,
+      "grad_norm": 1.3505935668945312,
+      "learning_rate": 3.0093897777987098e-09,
+      "logits/chosen": -3.0517494678497314,
+      "logits/rejected": -3.0366005897521973,
+      "logps/chosen": -58.27477264404297,
+      "logps/rejected": -61.6846923828125,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.6109374761581421,
+      "rewards/chosen": -0.04682334139943123,
+      "rewards/margins": 0.016174782067537308,
+      "rewards/rejected": -0.06299812346696854,
+      "step": 2490
+    },
+    {
+      "epoch": 1.722949689869056,
+      "grad_norm": 1.3154429197311401,
+      "learning_rate": 2.8679021185845975e-09,
+      "logits/chosen": -3.023200273513794,
+      "logits/rejected": -2.997267246246338,
+      "logps/chosen": -58.07569122314453,
+      "logps/rejected": -59.87085723876953,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.620312511920929,
+      "rewards/chosen": -0.046663668006658554,
+      "rewards/margins": 0.022471796721220016,
+      "rewards/rejected": -0.06913547217845917,
+      "step": 2500
+    },
+    {
+      "epoch": 1.722949689869056,
+      "eval_logits/chosen": -3.1012966632843018,
+      "eval_logits/rejected": -3.095568895339966,
+      "eval_logps/chosen": -60.681880950927734,
+      "eval_logps/rejected": -66.00990295410156,
+      "eval_loss": 0.689177393913269,
+      "eval_rewards/accuracies": 0.5850371718406677,
+      "eval_rewards/chosen": -0.019699882715940475,
+      "eval_rewards/margins": 0.008597951382398605,
+      "eval_rewards/rejected": -0.02829783223569393,
+      "eval_runtime": 383.2161,
+      "eval_samples_per_second": 11.231,
+      "eval_steps_per_second": 1.404,
+      "step": 2500
+    },
+    {
+      "epoch": 1.729841488628532,
+      "grad_norm": 1.3162225484848022,
+      "learning_rate": 2.7296185393219316e-09,
+      "logits/chosen": -3.0459542274475098,
+      "logits/rejected": -3.0207812786102295,
+      "logps/chosen": -59.15156173706055,
+      "logps/rejected": -59.254676818847656,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.045805253088474274,
+      "rewards/margins": 0.020932307466864586,
+      "rewards/rejected": -0.06673755496740341,
+      "step": 2510
+    },
+    {
+      "epoch": 1.7367332873880081,
+      "grad_norm": 1.328461766242981,
+      "learning_rate": 2.5945590594253305e-09,
+      "logits/chosen": -2.9799602031707764,
+      "logits/rejected": -2.9713258743286133,
+      "logps/chosen": -58.0362548828125,
+      "logps/rejected": -60.381080627441406,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": -0.0511082224547863,
+      "rewards/margins": 0.012231842614710331,
+      "rewards/rejected": -0.06334006786346436,
+      "step": 2520
+    },
+    {
+      "epoch": 1.7436250861474845,
+      "grad_norm": 1.2932238578796387,
+      "learning_rate": 2.4627432315541986e-09,
+      "logits/chosen": -3.055954694747925,
+      "logits/rejected": -3.0452940464019775,
+      "logps/chosen": -58.16063690185547,
+      "logps/rejected": -61.20969772338867,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.6171875,
+      "rewards/chosen": -0.04746638238430023,
+      "rewards/margins": 0.02094622328877449,
+      "rewards/rejected": -0.06841260939836502,
+      "step": 2530
+    },
+    {
+      "epoch": 1.7505168849069608,
+      "grad_norm": 1.3644834756851196,
+      "learning_rate": 2.3341901387820717e-09,
+      "logits/chosen": -3.0201711654663086,
+      "logits/rejected": -2.995832681655884,
+      "logps/chosen": -59.7026481628418,
+      "logps/rejected": -60.433990478515625,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.04398275539278984,
+      "rewards/margins": 0.02229396626353264,
+      "rewards/rejected": -0.06627672165632248,
+      "step": 2540
+    },
+    {
+      "epoch": 1.757408683666437,
+      "grad_norm": 1.3037844896316528,
+      "learning_rate": 2.2089183918339445e-09,
+      "logits/chosen": -2.996652126312256,
+      "logits/rejected": -2.976022243499756,
+      "logps/chosen": -57.28471755981445,
+      "logps/rejected": -59.08478546142578,
+      "loss": 0.684,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.04602036252617836,
+      "rewards/margins": 0.019708681851625443,
+      "rewards/rejected": -0.0657290443778038,
+      "step": 2550
+    },
+    {
+      "epoch": 1.7643004824259132,
+      "grad_norm": 1.3543256521224976,
+      "learning_rate": 2.086946126391981e-09,
+      "logits/chosen": -2.9888083934783936,
+      "logits/rejected": -2.9732577800750732,
+      "logps/chosen": -56.89265823364258,
+      "logps/rejected": -60.136573791503906,
+      "loss": 0.6841,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.05068554729223251,
+      "rewards/margins": 0.019495617598295212,
+      "rewards/rejected": -0.07018117606639862,
+      "step": 2560
+    },
+    {
+      "epoch": 1.7711922811853893,
+      "grad_norm": 1.3135391473770142,
+      "learning_rate": 1.9682910004700155e-09,
+      "logits/chosen": -3.000701904296875,
+      "logits/rejected": -2.9841086864471436,
+      "logps/chosen": -59.79181671142578,
+      "logps/rejected": -60.78386306762695,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.048417720943689346,
+      "rewards/margins": 0.02133244276046753,
+      "rewards/rejected": -0.06975016742944717,
+      "step": 2570
+    },
+    {
+      "epoch": 1.7780840799448656,
+      "grad_norm": 1.2864971160888672,
+      "learning_rate": 1.852970191857159e-09,
+      "logits/chosen": -2.9674811363220215,
+      "logits/rejected": -2.94804048538208,
+      "logps/chosen": -59.39619064331055,
+      "logps/rejected": -60.783851623535156,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.04619085043668747,
+      "rewards/margins": 0.02428482472896576,
+      "rewards/rejected": -0.07047567516565323,
+      "step": 2580
+    },
+    {
+      "epoch": 1.784975878704342,
+      "grad_norm": 1.2950899600982666,
+      "learning_rate": 1.741000395630976e-09,
+      "logits/chosen": -3.034547805786133,
+      "logits/rejected": -3.0095696449279785,
+      "logps/chosen": -58.98634719848633,
+      "logps/rejected": -60.004661560058594,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.044211823493242264,
+      "rewards/margins": 0.02347356267273426,
+      "rewards/rejected": -0.06768538057804108,
+      "step": 2590
+    },
+    {
+      "epoch": 1.791867677463818,
+      "grad_norm": 1.3900827169418335,
+      "learning_rate": 1.6323978217405277e-09,
+      "logits/chosen": -2.962319850921631,
+      "logits/rejected": -2.9394538402557373,
+      "logps/chosen": -57.80451202392578,
+      "logps/rejected": -60.771339416503906,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.6390625238418579,
+      "rewards/chosen": -0.0485307052731514,
+      "rewards/margins": 0.022718578577041626,
+      "rewards/rejected": -0.07124929130077362,
+      "step": 2600
+    },
+    {
+      "epoch": 1.791867677463818,
+      "eval_logits/chosen": -3.1006553173065186,
+      "eval_logits/rejected": -3.094916820526123,
+      "eval_logps/chosen": -60.688175201416016,
+      "eval_logps/rejected": -66.03443145751953,
+      "eval_loss": 0.6890937089920044,
+      "eval_rewards/accuracies": 0.5889869928359985,
+      "eval_rewards/chosen": -0.019762787967920303,
+      "eval_rewards/margins": 0.008780322037637234,
+      "eval_rewards/rejected": -0.028543109074234962,
+      "eval_runtime": 383.3128,
+      "eval_samples_per_second": 11.228,
+      "eval_steps_per_second": 1.404,
+      "step": 2600
+    },
+    {
+      "epoch": 1.7987594762232941,
+      "grad_norm": 1.325190544128418,
+      "learning_rate": 1.5271781926596449e-09,
+      "logits/chosen": -3.0393474102020264,
+      "logits/rejected": -3.0156404972076416,
+      "logps/chosen": -60.42161178588867,
+      "logps/rejected": -61.184486389160156,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.04576939716935158,
+      "rewards/margins": 0.022201048210263252,
+      "rewards/rejected": -0.06797045469284058,
+      "step": 2610
+    },
+    {
+      "epoch": 1.8056512749827704,
+      "grad_norm": 1.3093925714492798,
+      "learning_rate": 1.4253567411107643e-09,
+      "logits/chosen": -2.989856243133545,
+      "logits/rejected": -2.9666576385498047,
+      "logps/chosen": -58.600990295410156,
+      "logps/rejected": -60.18854904174805,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.0458240807056427,
+      "rewards/margins": 0.021440699696540833,
+      "rewards/rejected": -0.06726478040218353,
+      "step": 2620
+    },
+    {
+      "epoch": 1.8125430737422468,
+      "grad_norm": 1.2918739318847656,
+      "learning_rate": 1.326948207859685e-09,
+      "logits/chosen": -3.0238237380981445,
+      "logits/rejected": -3.0077781677246094,
+      "logps/chosen": -57.79582595825195,
+      "logps/rejected": -60.6348762512207,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.046499475836753845,
+      "rewards/margins": 0.021402059122920036,
+      "rewards/rejected": -0.06790152937173843,
+      "step": 2630
+    },
+    {
+      "epoch": 1.819434872501723,
+      "grad_norm": 1.349001407623291,
+      "learning_rate": 1.2319668395815358e-09,
+      "logits/chosen": -3.0028393268585205,
+      "logits/rejected": -2.9857001304626465,
+      "logps/chosen": -58.69614791870117,
+      "logps/rejected": -59.920021057128906,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.604687511920929,
+      "rewards/chosen": -0.04932459071278572,
+      "rewards/margins": 0.0200694240629673,
+      "rewards/rejected": -0.06939400732517242,
+      "step": 2640
+    },
+    {
+      "epoch": 1.8263266712611992,
+      "grad_norm": 1.2818963527679443,
+      "learning_rate": 1.1404263867982738e-09,
+      "logits/chosen": -3.0455758571624756,
+      "logits/rejected": -3.0237960815429688,
+      "logps/chosen": -59.25251388549805,
+      "logps/rejected": -60.45496368408203,
+      "loss": 0.683,
+      "rewards/accuracies": 0.629687488079071,
+      "rewards/chosen": -0.04728539660573006,
+      "rewards/margins": 0.021762443706393242,
+      "rewards/rejected": -0.06904784590005875,
+      "step": 2650
+    },
+    {
+      "epoch": 1.8332184700206753,
+      "grad_norm": 1.2889119386672974,
+      "learning_rate": 1.0523401018880134e-09,
+      "logits/chosen": -2.983532428741455,
+      "logits/rejected": -2.9646944999694824,
+      "logps/chosen": -58.277976989746094,
+      "logps/rejected": -59.491722106933594,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.047298818826675415,
+      "rewards/margins": 0.020261693745851517,
+      "rewards/rejected": -0.06756050884723663,
+      "step": 2660
+    },
+    {
+      "epoch": 1.8401102687801516,
+      "grad_norm": 1.3029212951660156,
+      "learning_rate": 9.677207371664608e-10,
+      "logits/chosen": -3.0146260261535645,
+      "logits/rejected": -2.9899539947509766,
+      "logps/chosen": -59.18970489501953,
+      "logps/rejected": -60.14207077026367,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.05002979189157486,
+      "rewards/margins": 0.02303471975028515,
+      "rewards/rejected": -0.07306452095508575,
+      "step": 2670
+    },
+    {
+      "epoch": 1.847002067539628,
+      "grad_norm": 1.260703444480896,
+      "learning_rate": 8.865805430407575e-10,
+      "logits/chosen": -3.0160889625549316,
+      "logits/rejected": -2.9888625144958496,
+      "logps/chosen": -58.788368225097656,
+      "logps/rejected": -59.13869094848633,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.048010729253292084,
+      "rewards/margins": 0.0222895760089159,
+      "rewards/rejected": -0.07030030339956284,
+      "step": 2680
+    },
+    {
+      "epoch": 1.853893866299104,
+      "grad_norm": 1.3435821533203125,
+      "learning_rate": 8.089312662359904e-10,
+      "logits/chosen": -3.000649929046631,
+      "logits/rejected": -2.9763736724853516,
+      "logps/chosen": -58.345603942871094,
+      "logps/rejected": -59.42350387573242,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04748475179076195,
+      "rewards/margins": 0.02055184543132782,
+      "rewards/rejected": -0.06803660839796066,
+      "step": 2690
+    },
+    {
+      "epoch": 1.8607856650585803,
+      "grad_norm": 1.3498848676681519,
+      "learning_rate": 7.34784148094586e-10,
+      "logits/chosen": -3.062194347381592,
+      "logits/rejected": -3.040365219116211,
+      "logps/chosen": -58.02552032470703,
+      "logps/rejected": -60.96380615234375,
+      "loss": 0.6823,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.04390079155564308,
+      "rewards/margins": 0.023236598819494247,
+      "rewards/rejected": -0.06713739782571793,
+      "step": 2700
+    },
+    {
+      "epoch": 1.8607856650585803,
+      "eval_logits/chosen": -3.1006381511688232,
+      "eval_logits/rejected": -3.0949079990386963,
+      "eval_logps/chosen": -60.71648025512695,
+      "eval_logps/rejected": -66.05257415771484,
+      "eval_loss": 0.6891458034515381,
+      "eval_rewards/accuracies": 0.5889869928359985,
+      "eval_rewards/chosen": -0.020045887678861618,
+      "eval_rewards/margins": 0.008678610436618328,
+      "eval_rewards/rejected": -0.02872449718415737,
+      "eval_runtime": 383.351,
+      "eval_samples_per_second": 11.227,
+      "eval_steps_per_second": 1.403,
+      "step": 2700
+    },
+    {
+      "epoch": 1.8676774638180564,
+      "grad_norm": 1.2470507621765137,
+      "learning_rate": 6.641499229489145e-10,
+      "logits/chosen": -3.003091335296631,
+      "logits/rejected": -2.9715797901153564,
+      "logps/chosen": -58.2459831237793,
+      "logps/rejected": -58.3682975769043,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.6390625238418579,
+      "rewards/chosen": -0.04425545781850815,
+      "rewards/margins": 0.02511006034910679,
+      "rewards/rejected": -0.06936550885438919,
+      "step": 2710
+    },
+    {
+      "epoch": 1.8745692625775328,
+      "grad_norm": 1.3162429332733154,
+      "learning_rate": 5.970388165672691e-10,
+      "logits/chosen": -2.99006724357605,
+      "logits/rejected": -2.971386671066284,
+      "logps/chosen": -57.105255126953125,
+      "logps/rejected": -60.72968673706055,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.6171875,
+      "rewards/chosen": -0.0457664355635643,
+      "rewards/margins": 0.02246815897524357,
+      "rewards/rejected": -0.06823460012674332,
+      "step": 2720
+    },
+    {
+      "epoch": 1.881461061337009,
+      "grad_norm": 1.4187453985214233,
+      "learning_rate": 5.334605446734585e-10,
+      "logits/chosen": -3.0353336334228516,
+      "logits/rejected": -3.007887363433838,
+      "logps/chosen": -59.300804138183594,
+      "logps/rejected": -59.99883270263672,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.6421874761581421,
+      "rewards/chosen": -0.043865978717803955,
+      "rewards/margins": 0.02479901909828186,
+      "rewards/rejected": -0.06866499781608582,
+      "step": 2730
+    },
+    {
+      "epoch": 1.8883528600964852,
+      "grad_norm": 1.3848966360092163,
+      "learning_rate": 4.734243115402825e-10,
+      "logits/chosen": -2.9592947959899902,
+      "logits/rejected": -2.9372572898864746,
+      "logps/chosen": -59.5694465637207,
+      "logps/rejected": -60.315895080566406,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.047974247485399246,
+      "rewards/margins": 0.02036571130156517,
+      "rewards/rejected": -0.06833995878696442,
+      "step": 2740
+    },
+    {
+      "epoch": 1.8952446588559613,
+      "grad_norm": 1.4468791484832764,
+      "learning_rate": 4.169388086569886e-10,
+      "logits/chosen": -3.0385963916778564,
+      "logits/rejected": -3.0236904621124268,
+      "logps/chosen": -58.946388244628906,
+      "logps/rejected": -61.70532989501953,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.604687511920929,
+      "rewards/chosen": -0.04743208736181259,
+      "rewards/margins": 0.020196830853819847,
+      "rewards/rejected": -0.06762892007827759,
+      "step": 2750
+    },
+    {
+      "epoch": 1.9021364576154376,
+      "grad_norm": 1.3899776935577393,
+      "learning_rate": 3.640122134710294e-10,
+      "logits/chosen": -3.06270170211792,
+      "logits/rejected": -3.0440831184387207,
+      "logps/chosen": -59.356605529785156,
+      "logps/rejected": -60.465003967285156,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.0449027419090271,
+      "rewards/margins": 0.023394212126731873,
+      "rewards/rejected": -0.06829695403575897,
+      "step": 2760
+    },
+    {
+      "epoch": 1.909028256374914,
+      "grad_norm": 1.328192114830017,
+      "learning_rate": 3.1465218820418415e-10,
+      "logits/chosen": -3.037365436553955,
+      "logits/rejected": -3.0028114318847656,
+      "logps/chosen": -58.11069869995117,
+      "logps/rejected": -59.29419708251953,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.667187511920929,
+      "rewards/chosen": -0.04381219670176506,
+      "rewards/margins": 0.026473551988601685,
+      "rewards/rejected": -0.07028575241565704,
+      "step": 2770
+    },
+    {
+      "epoch": 1.9159200551343902,
+      "grad_norm": 1.2775218486785889,
+      "learning_rate": 2.688658787433157e-10,
+      "logits/chosen": -3.022888660430908,
+      "logits/rejected": -3.000300884246826,
+      "logps/chosen": -60.5079460144043,
+      "logps/rejected": -60.63434600830078,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.04978417605161667,
+      "rewards/margins": 0.021695107221603394,
+      "rewards/rejected": -0.07147928327322006,
+      "step": 2780
+    },
+    {
+      "epoch": 1.9228118538938663,
+      "grad_norm": 1.2735910415649414,
+      "learning_rate": 2.266599136058367e-10,
+      "logits/chosen": -3.0028035640716553,
+      "logits/rejected": -2.9840023517608643,
+      "logps/chosen": -59.78889083862305,
+      "logps/rejected": -59.44769287109375,
+      "loss": 0.6861,
+      "rewards/accuracies": 0.589062511920929,
+      "rewards/chosen": -0.04813474044203758,
+      "rewards/margins": 0.015620408579707146,
+      "rewards/rejected": -0.06375513970851898,
+      "step": 2790
+    },
+    {
+      "epoch": 1.9297036526533424,
+      "grad_norm": 1.376592755317688,
+      "learning_rate": 1.8804040298009693e-10,
+      "logits/chosen": -3.0288257598876953,
+      "logits/rejected": -3.0026650428771973,
+      "logps/chosen": -59.218475341796875,
+      "logps/rejected": -58.154075622558594,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": -0.04387739300727844,
+      "rewards/margins": 0.02458575740456581,
+      "rewards/rejected": -0.06846315413713455,
+      "step": 2800
+    },
+    {
+      "epoch": 1.9297036526533424,
+      "eval_logits/chosen": -3.1007766723632812,
+      "eval_logits/rejected": -3.095076084136963,
+      "eval_logps/chosen": -60.726348876953125,
+      "eval_logps/rejected": -66.07279968261719,
+      "eval_loss": 0.6890966892242432,
+      "eval_rewards/accuracies": 0.5841078162193298,
+      "eval_rewards/chosen": -0.020144494250416756,
+      "eval_rewards/margins": 0.008782317861914635,
+      "eval_rewards/rejected": -0.02892681024968624,
+      "eval_runtime": 383.6267,
+      "eval_samples_per_second": 11.219,
+      "eval_steps_per_second": 1.402,
+      "step": 2800
+    },
+    {
+      "epoch": 1.9365954514128187,
+      "grad_norm": 1.2682085037231445,
+      "learning_rate": 1.5301293784081847e-10,
+      "logits/chosen": -2.9736599922180176,
+      "logits/rejected": -2.9589531421661377,
+      "logps/chosen": -58.26537322998047,
+      "logps/rejected": -60.63109588623047,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.0504550039768219,
+      "rewards/margins": 0.019359614700078964,
+      "rewards/rejected": -0.06981462240219116,
+      "step": 2810
+    },
+    {
+      "epoch": 1.943487250172295,
+      "grad_norm": 1.3405542373657227,
+      "learning_rate": 1.2158258913967102e-10,
+      "logits/chosen": -3.0063540935516357,
+      "logits/rejected": -2.9755642414093018,
+      "logps/chosen": -60.50700759887695,
+      "logps/rejected": -58.998687744140625,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.04506916552782059,
+      "rewards/margins": 0.023324180394411087,
+      "rewards/rejected": -0.06839334219694138,
+      "step": 2820
+    },
+    {
+      "epoch": 1.9503790489317712,
+      "grad_norm": 1.3689327239990234,
+      "learning_rate": 9.37539070711646e-11,
+      "logits/chosen": -3.0321671962738037,
+      "logits/rejected": -3.012648820877075,
+      "logps/chosen": -60.28644943237305,
+      "logps/rejected": -60.640167236328125,
+      "loss": 0.681,
+      "rewards/accuracies": 0.6109374761581421,
+      "rewards/chosen": -0.042195506393909454,
+      "rewards/margins": 0.02599485218524933,
+      "rewards/rejected": -0.06819035857915878,
+      "step": 2830
+    },
+    {
+      "epoch": 1.9572708476912473,
+      "grad_norm": 1.3046759366989136,
+      "learning_rate": 6.953092041389607e-11,
+      "logits/chosen": -3.014383554458618,
+      "logits/rejected": -2.9899418354034424,
+      "logps/chosen": -59.1005859375,
+      "logps/rejected": -59.03815460205078,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.621874988079071,
+      "rewards/chosen": -0.04825712740421295,
+      "rewards/margins": 0.023372991010546684,
+      "rewards/rejected": -0.07163011282682419,
+      "step": 2840
+    },
+    {
+      "epoch": 1.9641626464507236,
+      "grad_norm": 1.3823450803756714,
+      "learning_rate": 4.891713594731006e-11,
+      "logits/chosen": -3.0164265632629395,
+      "logits/rejected": -2.993161678314209,
+      "logps/chosen": -58.6377067565918,
+      "logps/rejected": -59.821807861328125,
+      "loss": 0.6829,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04551283270120621,
+      "rewards/margins": 0.021898990496993065,
+      "rewards/rejected": -0.06741182506084442,
+      "step": 2850
+    },
+    {
+      "epoch": 1.9710544452102,
+      "grad_norm": 1.2972795963287354,
+      "learning_rate": 3.191553794401336e-11,
+      "logits/chosen": -3.0041041374206543,
+      "logits/rejected": -2.9793601036071777,
+      "logps/chosen": -58.8316535949707,
+      "logps/rejected": -59.15874481201172,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.5921875238418579,
+      "rewards/chosen": -0.04608858376741409,
+      "rewards/margins": 0.018982943147420883,
+      "rewards/rejected": -0.06507153064012527,
+      "step": 2860
+    },
+    {
+      "epoch": 1.9779462439696762,
+      "grad_norm": 1.4782917499542236,
+      "learning_rate": 1.8528587737753898e-11,
+      "logits/chosen": -3.00868558883667,
+      "logits/rejected": -2.981982707977295,
+      "logps/chosen": -59.979164123535156,
+      "logps/rejected": -59.065032958984375,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04683419317007065,
+      "rewards/margins": 0.025204036384820938,
+      "rewards/rejected": -0.07203822582960129,
+      "step": 2870
+    },
+    {
+      "epoch": 1.9848380427291523,
+      "grad_norm": 1.387197494506836,
+      "learning_rate": 8.758223367075212e-12,
+      "logits/chosen": -3.0078389644622803,
+      "logits/rejected": -2.978468656539917,
+      "logps/chosen": -59.639747619628906,
+      "logps/rejected": -58.05632781982422,
+      "loss": 0.683,
+      "rewards/accuracies": 0.6265624761581421,
+      "rewards/chosen": -0.04575073719024658,
+      "rewards/margins": 0.021932676434516907,
+      "rewards/rejected": -0.06768341362476349,
+      "step": 2880
+    },
+    {
+      "epoch": 1.9917298414886284,
+      "grad_norm": 1.3310401439666748,
+      "learning_rate": 2.605859294749213e-12,
+      "logits/chosen": -3.018655300140381,
+      "logits/rejected": -2.994286298751831,
+      "logps/chosen": -57.234657287597656,
+      "logps/rejected": -59.07853317260742,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.046042539179325104,
+      "rewards/margins": 0.022625811398029327,
+      "rewards/rejected": -0.06866835057735443,
+      "step": 2890
+    },
+    {
+      "epoch": 1.9986216402481047,
+      "grad_norm": 1.3212953805923462,
+      "learning_rate": 7.2386203012198e-14,
+      "logits/chosen": -3.0002474784851074,
+      "logits/rejected": -2.9772603511810303,
+      "logps/chosen": -59.88157272338867,
+      "logps/rejected": -60.886680603027344,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.614062488079071,
+      "rewards/chosen": -0.04801579564809799,
+      "rewards/margins": 0.0204045120626688,
+      "rewards/rejected": -0.06842031329870224,
+      "step": 2900
+    },
+    {
+      "epoch": 1.9986216402481047,
+      "eval_logits/chosen": -3.100578546524048,
+      "eval_logits/rejected": -3.094856023788452,
+      "eval_logps/chosen": -60.72254180908203,
+      "eval_logps/rejected": -66.06378173828125,
+      "eval_loss": 0.6891194581985474,
+      "eval_rewards/accuracies": 0.5910780429840088,
+      "eval_rewards/chosen": -0.020106395706534386,
+      "eval_rewards/margins": 0.008730227127671242,
+      "eval_rewards/rejected": -0.028836622834205627,
+      "eval_runtime": 383.6501,
+      "eval_samples_per_second": 11.219,
+      "eval_steps_per_second": 1.402,
+      "step": 2900
+    },
+    {
+      "epoch": 2.0,
+      "step": 2902,
+      "total_flos": 0.0,
+      "train_loss": 0.6870454205553093,
+      "train_runtime": 56536.4846,
+      "train_samples_per_second": 3.285,
+      "train_steps_per_second": 0.051
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2902,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}