diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -10,18170 +10,17036 @@
   "log_history": [
     {
       "epoch": 0.002676032781401572,
-      "grad_norm": 2.930448424601805,
+      "grad_norm": 1.4679550260452852,
       "learning_rate": 8.9126559714795e-09,
-      "logits/chosen": -0.05658132582902908,
-      "logits/rejected": 0.15209060907363892,
-      "logps/chosen": -1.7160199880599976,
-      "logps/rejected": -1.889544129371643,
-      "loss": 1.0121,
+      "logits/chosen": -0.056738246232271194,
+      "logits/rejected": 0.1520412713289261,
+      "logps/chosen": -1.7161811590194702,
+      "logps/rejected": -1.8896806240081787,
+      "loss": 0.6829,
       "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -1.7160199880599976,
-      "rewards/margins": 0.17352427542209625,
-      "rewards/rejected": -1.889544129371643,
-      "semantic_entropy": 0.6583887338638306,
+      "rewards/chosen": -1.7161811590194702,
+      "rewards/margins": 0.17349949479103088,
+      "rewards/rejected": -1.8896806240081787,
       "step": 5
     },
     {
       "epoch": 0.005352065562803144,
-      "grad_norm": 2.171475278087084,
+      "grad_norm": 1.360081716750971,
       "learning_rate": 1.7825311942959e-08,
-      "logits/chosen": 0.01099350955337286,
-      "logits/rejected": 0.13102369010448456,
-      "logps/chosen": -1.802465796470642,
-      "logps/rejected": -1.8456957340240479,
-      "loss": 1.0085,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -1.802465796470642,
-      "rewards/margins": 0.043229907751083374,
-      "rewards/rejected": -1.8456957340240479,
-      "semantic_entropy": 0.6393550634384155,
+      "logits/chosen": -0.003225961234420538,
+      "logits/rejected": 0.11442573368549347,
+      "logps/chosen": -1.8023093938827515,
+      "logps/rejected": -1.8466150760650635,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.8023093938827515,
+      "rewards/margins": 0.044305700808763504,
+      "rewards/rejected": -1.8466150760650635,
       "step": 10
     },
     {
       "epoch": 0.008028098344204716,
-      "grad_norm": 2.7133554826243436,
+      "grad_norm": 1.1020534724830071,
       "learning_rate": 2.67379679144385e-08,
-      "logits/chosen": -0.03254646062850952,
-      "logits/rejected": 0.06340529769659042,
-      "logps/chosen": -1.634946584701538,
-      "logps/rejected": -1.7654039859771729,
-      "loss": 1.0399,
+      "logits/chosen": -0.04021782428026199,
+      "logits/rejected": 0.05551953241229057,
+      "logps/chosen": -1.6355050802230835,
+      "logps/rejected": -1.7651970386505127,
+      "loss": 0.6933,
       "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -1.634946584701538,
-      "rewards/margins": 0.13045744597911835,
-      "rewards/rejected": -1.7654039859771729,
-      "semantic_entropy": 0.693306565284729,
+      "rewards/chosen": -1.6355050802230835,
+      "rewards/margins": 0.12969176471233368,
+      "rewards/rejected": -1.7651970386505127,
       "step": 15
     },
     {
       "epoch": 0.010704131125606288,
-      "grad_norm": 3.1907759207915536,
+      "grad_norm": 1.2877468405568107,
       "learning_rate": 3.5650623885918e-08,
-      "logits/chosen": -0.05287078768014908,
-      "logits/rejected": 0.02737407758831978,
-      "logps/chosen": -1.7258117198944092,
-      "logps/rejected": -1.8055559396743774,
-      "loss": 1.0315,
+      "logits/chosen": -0.03865063562989235,
+      "logits/rejected": 0.0461001992225647,
+      "logps/chosen": -1.726712942123413,
+      "logps/rejected": -1.8083343505859375,
+      "loss": 0.6972,
       "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -1.7258117198944092,
-      "rewards/margins": 0.07974427938461304,
-      "rewards/rejected": -1.8055559396743774,
-      "semantic_entropy": 0.6684954166412354,
+      "rewards/chosen": -1.726712942123413,
+      "rewards/margins": 0.08162130415439606,
+      "rewards/rejected": -1.8083343505859375,
       "step": 20
     },
     {
       "epoch": 0.013380163907007862,
-      "grad_norm": 3.756329890323183,
+      "grad_norm": 1.2051880916329103,
       "learning_rate": 4.45632798573975e-08,
-      "logits/chosen": -0.0586787648499012,
-      "logits/rejected": 0.02491995319724083,
-      "logps/chosen": -1.8697763681411743,
-      "logps/rejected": -1.7798073291778564,
-      "loss": 1.0477,
-      "rewards/accuracies": 0.3812499940395355,
-      "rewards/chosen": -1.8697763681411743,
-      "rewards/margins": -0.08996908366680145,
-      "rewards/rejected": -1.7798073291778564,
-      "semantic_entropy": 0.6430714726448059,
+      "logits/chosen": -0.04329826310276985,
+      "logits/rejected": 0.04113423079252243,
+      "logps/chosen": -1.8691736459732056,
+      "logps/rejected": -1.7786750793457031,
+      "loss": 0.7263,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -1.8691736459732056,
+      "rewards/margins": -0.0904988944530487,
+      "rewards/rejected": -1.7786750793457031,
       "step": 25
     },
     {
       "epoch": 0.016056196688409432,
-      "grad_norm": 3.140323264852576,
+      "grad_norm": 1.086893832798066,
       "learning_rate": 5.3475935828877e-08,
-      "logits/chosen": -0.0959538072347641,
-      "logits/rejected": -0.007798808626830578,
-      "logps/chosen": -1.9086625576019287,
-      "logps/rejected": -1.8328447341918945,
-      "loss": 0.9997,
-      "rewards/accuracies": 0.4437499940395355,
-      "rewards/chosen": -1.9086625576019287,
-      "rewards/margins": -0.07581742107868195,
-      "rewards/rejected": -1.8328447341918945,
-      "semantic_entropy": 0.6177659630775452,
+      "logits/chosen": -0.08495397865772247,
+      "logits/rejected": 0.006166330073028803,
+      "logps/chosen": -1.9090423583984375,
+      "logps/rejected": -1.8328630924224854,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -1.9090423583984375,
+      "rewards/margins": -0.0761791244149208,
+      "rewards/rejected": -1.8328630924224854,
       "step": 30
     },
     {
       "epoch": 0.018732229469811006,
-      "grad_norm": 3.690486764810422,
+      "grad_norm": 1.22423395123332,
       "learning_rate": 6.23885918003565e-08,
-      "logits/chosen": -0.0320345014333725,
-      "logits/rejected": 0.1324472427368164,
-      "logps/chosen": -1.8473193645477295,
-      "logps/rejected": -1.9983642101287842,
-      "loss": 1.0256,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -1.8473193645477295,
-      "rewards/margins": 0.15104477107524872,
-      "rewards/rejected": -1.9983642101287842,
-      "semantic_entropy": 0.634685754776001,
+      "logits/chosen": -0.07529322057962418,
+      "logits/rejected": 0.08046910911798477,
+      "logps/chosen": -1.8457038402557373,
+      "logps/rejected": -1.995661735534668,
+      "loss": 0.7084,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.8457038402557373,
+      "rewards/margins": 0.14995768666267395,
+      "rewards/rejected": -1.995661735534668,
       "step": 35
     },
     {
       "epoch": 0.021408262251212576,
-      "grad_norm": 2.92016477029076,
+      "grad_norm": 1.1870468271336057,
       "learning_rate": 7.1301247771836e-08,
-      "logits/chosen": 0.03928864747285843,
-      "logits/rejected": 0.21504434943199158,
-      "logps/chosen": -1.884832739830017,
-      "logps/rejected": -1.7459083795547485,
-      "loss": 1.0276,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -1.884832739830017,
-      "rewards/margins": -0.13892444968223572,
-      "rewards/rejected": -1.7459083795547485,
-      "semantic_entropy": 0.6423062682151794,
+      "logits/chosen": 0.05610765889286995,
+      "logits/rejected": 0.23188301920890808,
+      "logps/chosen": -1.8830211162567139,
+      "logps/rejected": -1.7434899806976318,
+      "loss": 0.7065,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -1.8830211162567139,
+      "rewards/margins": -0.139531210064888,
+      "rewards/rejected": -1.7434899806976318,
       "step": 40
     },
     {
       "epoch": 0.02408429503261415,
-      "grad_norm": 3.255241370747709,
+      "grad_norm": 1.1335638216190422,
       "learning_rate": 8.021390374331551e-08,
-      "logits/chosen": 0.022160563617944717,
-      "logits/rejected": 0.2186368703842163,
-      "logps/chosen": -1.8431684970855713,
-      "logps/rejected": -1.877727746963501,
-      "loss": 1.0203,
+      "logits/chosen": 0.053416311740875244,
+      "logits/rejected": 0.25668779015541077,
+      "logps/chosen": -1.838446021080017,
+      "logps/rejected": -1.8739427328109741,
+      "loss": 0.6962,
       "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -1.8431684970855713,
-      "rewards/margins": 0.03455904498696327,
-      "rewards/rejected": -1.877727746963501,
-      "semantic_entropy": 0.6480525732040405,
+      "rewards/chosen": -1.838446021080017,
+      "rewards/margins": 0.03549668937921524,
+      "rewards/rejected": -1.8739427328109741,
       "step": 45
     },
     {
       "epoch": 0.026760327814015723,
-      "grad_norm": 4.001600949574766,
+      "grad_norm": 1.0554583696577178,
       "learning_rate": 8.9126559714795e-08,
-      "logits/chosen": -0.04236506298184395,
-      "logits/rejected": 0.10813881456851959,
-      "logps/chosen": -1.9069163799285889,
-      "logps/rejected": -1.7855520248413086,
-      "loss": 1.0155,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -1.9069163799285889,
-      "rewards/margins": -0.12136442959308624,
-      "rewards/rejected": -1.7855520248413086,
-      "semantic_entropy": 0.6318446397781372,
+      "logits/chosen": -0.0318031869828701,
+      "logits/rejected": 0.11948704719543457,
+      "logps/chosen": -1.89895498752594,
+      "logps/rejected": -1.7795015573501587,
+      "loss": 0.6995,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.89895498752594,
+      "rewards/margins": -0.11945325136184692,
+      "rewards/rejected": -1.7795015573501587,
       "step": 50
     },
     {
       "epoch": 0.029436360595417294,
-      "grad_norm": 3.2395604459059726,
+      "grad_norm": 1.1321919613959832,
       "learning_rate": 9.80392156862745e-08,
-      "logits/chosen": -0.1024969071149826,
-      "logits/rejected": 0.11822938919067383,
-      "logps/chosen": -1.846925973892212,
-      "logps/rejected": -1.8805322647094727,
-      "loss": 1.0113,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -1.846925973892212,
-      "rewards/margins": 0.03360645845532417,
-      "rewards/rejected": -1.8805322647094727,
-      "semantic_entropy": 0.6411503553390503,
+      "logits/chosen": -0.09762812405824661,
+      "logits/rejected": 0.12537768483161926,
+      "logps/chosen": -1.8368524312973022,
+      "logps/rejected": -1.870375633239746,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.8368524312973022,
+      "rewards/margins": 0.0335230827331543,
+      "rewards/rejected": -1.870375633239746,
       "step": 55
     },
     {
       "epoch": 0.032112393376818864,
-      "grad_norm": 3.327317251792206,
+      "grad_norm": 1.149113713239392,
       "learning_rate": 1.06951871657754e-07,
-      "logits/chosen": -0.08264996111392975,
-      "logits/rejected": 0.10669752210378647,
-      "logps/chosen": -1.8061221837997437,
-      "logps/rejected": -1.9114030599594116,
-      "loss": 1.0043,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.8061221837997437,
-      "rewards/margins": 0.1052810326218605,
-      "rewards/rejected": -1.9114030599594116,
-      "semantic_entropy": 0.6321321129798889,
+      "logits/chosen": -0.08927972614765167,
+      "logits/rejected": 0.09905913472175598,
+      "logps/chosen": -1.7932891845703125,
+      "logps/rejected": -1.900164246559143,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.7932891845703125,
+      "rewards/margins": 0.10687510669231415,
+      "rewards/rejected": -1.900164246559143,
       "step": 60
     },
     {
       "epoch": 0.03478842615822044,
-      "grad_norm": 2.744428991049788,
+      "grad_norm": 1.1301067879613322,
       "learning_rate": 1.158645276292335e-07,
-      "logits/chosen": -0.032412897795438766,
-      "logits/rejected": 0.1137821227312088,
-      "logps/chosen": -1.6550521850585938,
-      "logps/rejected": -1.7884334325790405,
-      "loss": 1.034,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.6550521850585938,
-      "rewards/margins": 0.13338114321231842,
-      "rewards/rejected": -1.7884334325790405,
-      "semantic_entropy": 0.6913561224937439,
+      "logits/chosen": -0.035531699657440186,
+      "logits/rejected": 0.10954837501049042,
+      "logps/chosen": -1.6431306600570679,
+      "logps/rejected": -1.7746940851211548,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.6431306600570679,
+      "rewards/margins": 0.1315634697675705,
+      "rewards/rejected": -1.7746940851211548,
       "step": 65
     },
     {
       "epoch": 0.03746445893962201,
-      "grad_norm": 4.062748882737165,
+      "grad_norm": 1.5621953413947673,
       "learning_rate": 1.24777183600713e-07,
-      "logits/chosen": -0.061274684965610504,
-      "logits/rejected": 0.09166450798511505,
-      "logps/chosen": -1.7896575927734375,
-      "logps/rejected": -1.8396289348602295,
-      "loss": 1.0265,
-      "rewards/accuracies": 0.41874998807907104,
-      "rewards/chosen": -1.7896575927734375,
-      "rewards/margins": 0.04997147619724274,
-      "rewards/rejected": -1.8396289348602295,
-      "semantic_entropy": 0.6477381587028503,
+      "logits/chosen": -0.07948037981987,
+      "logits/rejected": 0.07091851532459259,
+      "logps/chosen": -1.7736015319824219,
+      "logps/rejected": -1.8201287984848022,
+      "loss": 0.7029,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": -1.7736015319824219,
+      "rewards/margins": 0.04652729630470276,
+      "rewards/rejected": -1.8201287984848022,
       "step": 70
     },
     {
       "epoch": 0.04014049172102358,
-      "grad_norm": 2.6701234707046755,
+      "grad_norm": 1.13302212812871,
       "learning_rate": 1.3368983957219251e-07,
-      "logits/chosen": -0.059405725449323654,
-      "logits/rejected": 0.12231621891260147,
-      "logps/chosen": -1.824541687965393,
-      "logps/rejected": -2.0940768718719482,
-      "loss": 1.005,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.824541687965393,
-      "rewards/margins": 0.2695353329181671,
-      "rewards/rejected": -2.0940768718719482,
-      "semantic_entropy": 0.6212378144264221,
+      "logits/chosen": -0.05116662383079529,
+      "logits/rejected": 0.12969276309013367,
+      "logps/chosen": -1.787790060043335,
+      "logps/rejected": -2.050652503967285,
+      "loss": 0.6942,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.787790060043335,
+      "rewards/margins": 0.2628622055053711,
+      "rewards/rejected": -2.050652503967285,
       "step": 75
     },
     {
       "epoch": 0.04281652450242515,
-      "grad_norm": 2.413127697723551,
+      "grad_norm": 1.3780248016494938,
       "learning_rate": 1.42602495543672e-07,
-      "logits/chosen": 0.004832886159420013,
-      "logits/rejected": 0.10948418080806732,
-      "logps/chosen": -1.7755324840545654,
-      "logps/rejected": -1.806046724319458,
-      "loss": 1.0275,
+      "logits/chosen": 0.013949116691946983,
+      "logits/rejected": 0.11857213079929352,
+      "logps/chosen": -1.7312885522842407,
+      "logps/rejected": -1.7630586624145508,
+      "loss": 0.7003,
       "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -1.7755324840545654,
-      "rewards/margins": 0.030514365062117577,
-      "rewards/rejected": -1.806046724319458,
-      "semantic_entropy": 0.6542164087295532,
+      "rewards/chosen": -1.7312885522842407,
+      "rewards/margins": 0.031769994646310806,
+      "rewards/rejected": -1.7630586624145508,
       "step": 80
     },
     {
       "epoch": 0.04549255728382673,
-      "grad_norm": 2.4502740569586896,
+      "grad_norm": 1.2138956616994308,
       "learning_rate": 1.5151515151515152e-07,
-      "logits/chosen": -0.14950814843177795,
-      "logits/rejected": 0.09958289563655853,
-      "logps/chosen": -1.8580639362335205,
-      "logps/rejected": -2.0500550270080566,
-      "loss": 1.0132,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -1.8580639362335205,
-      "rewards/margins": 0.19199132919311523,
-      "rewards/rejected": -2.0500550270080566,
-      "semantic_entropy": 0.6325597167015076,
+      "logits/chosen": -0.15687067806720734,
+      "logits/rejected": 0.08736223727464676,
+      "logps/chosen": -1.8056681156158447,
+      "logps/rejected": -1.9863067865371704,
+      "loss": 0.6972,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.8056681156158447,
+      "rewards/margins": 0.18063868582248688,
+      "rewards/rejected": -1.9863067865371704,
       "step": 85
     },
     {
       "epoch": 0.0481685900652283,
-      "grad_norm": 2.758199310279624,
+      "grad_norm": 1.0812106210547976,
       "learning_rate": 1.6042780748663102e-07,
-      "logits/chosen": 0.08901110291481018,
-      "logits/rejected": 0.05140423774719238,
-      "logps/chosen": -1.833431601524353,
-      "logps/rejected": -1.8403393030166626,
-      "loss": 1.0264,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -1.833431601524353,
-      "rewards/margins": 0.0069076926447451115,
-      "rewards/rejected": -1.8403393030166626,
-      "semantic_entropy": 0.6516891717910767,
+      "logits/chosen": 0.09858322143554688,
+      "logits/rejected": 0.058293379843235016,
+      "logps/chosen": -1.7690833806991577,
+      "logps/rejected": -1.791569471359253,
+      "loss": 0.7002,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.7690833806991577,
+      "rewards/margins": 0.022486215457320213,
+      "rewards/rejected": -1.791569471359253,
       "step": 90
     },
     {
       "epoch": 0.05084462284662987,
-      "grad_norm": 2.1352231781819357,
+      "grad_norm": 1.3289444963053783,
       "learning_rate": 1.693404634581105e-07,
-      "logits/chosen": -0.07817761600017548,
-      "logits/rejected": 0.07053651660680771,
-      "logps/chosen": -1.9201338291168213,
-      "logps/rejected": -1.999481201171875,
-      "loss": 1.0067,
+      "logits/chosen": -0.09153784811496735,
+      "logits/rejected": 0.05558202788233757,
+      "logps/chosen": -1.8358242511749268,
+      "logps/rejected": -1.9343065023422241,
+      "loss": 0.6963,
       "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -1.9201338291168213,
-      "rewards/margins": 0.07934737950563431,
-      "rewards/rejected": -1.999481201171875,
-      "semantic_entropy": 0.6206165552139282,
+      "rewards/chosen": -1.8358242511749268,
+      "rewards/margins": 0.09848236292600632,
+      "rewards/rejected": -1.9343065023422241,
       "step": 95
     },
     {
       "epoch": 0.05352065562803145,
-      "grad_norm": 2.1621041677969197,
+      "grad_norm": 1.0474864575442813,
       "learning_rate": 1.7825311942959e-07,
-      "logits/chosen": -0.02526218257844448,
-      "logits/rejected": 0.03817426413297653,
-      "logps/chosen": -1.7987804412841797,
-      "logps/rejected": -1.9119453430175781,
-      "loss": 1.0147,
+      "logits/chosen": -0.03250938281416893,
+      "logits/rejected": 0.03078448213636875,
+      "logps/chosen": -1.7165687084197998,
+      "logps/rejected": -1.824202299118042,
+      "loss": 0.6913,
       "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -1.7987804412841797,
-      "rewards/margins": 0.11316481977701187,
-      "rewards/rejected": -1.9119453430175781,
-      "semantic_entropy": 0.6462353467941284,
+      "rewards/chosen": -1.7165687084197998,
+      "rewards/margins": 0.10763350874185562,
+      "rewards/rejected": -1.824202299118042,
       "step": 100
     },
     {
       "epoch": 0.05619668840943302,
-      "grad_norm": 1.9433568654301852,
+      "grad_norm": 1.2531311290298135,
       "learning_rate": 1.8716577540106952e-07,
-      "logits/chosen": 0.057496070861816406,
-      "logits/rejected": 0.08447133004665375,
-      "logps/chosen": -1.7846513986587524,
-      "logps/rejected": -1.946898102760315,
-      "loss": 1.0102,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -1.7846513986587524,
-      "rewards/margins": 0.16224689781665802,
-      "rewards/rejected": -1.946898102760315,
-      "semantic_entropy": 0.6508017778396606,
+      "logits/chosen": 0.041659098118543625,
+      "logits/rejected": 0.06977207958698273,
+      "logps/chosen": -1.6725887060165405,
+      "logps/rejected": -1.8384208679199219,
+      "loss": 0.6829,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.6725887060165405,
+      "rewards/margins": 0.16583217680454254,
+      "rewards/rejected": -1.8384208679199219,
       "step": 105
     },
     {
       "epoch": 0.05887272119083459,
-      "grad_norm": 2.2147958280388162,
+      "grad_norm": 1.0925707633709578,
       "learning_rate": 1.96078431372549e-07,
-      "logits/chosen": 0.018007440492510796,
-      "logits/rejected": 0.11341071128845215,
-      "logps/chosen": -1.8752214908599854,
-      "logps/rejected": -1.9351987838745117,
-      "loss": 1.0174,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -1.8752214908599854,
-      "rewards/margins": 0.05997718498110771,
-      "rewards/rejected": -1.9351987838745117,
-      "semantic_entropy": 0.6334887742996216,
+      "logits/chosen": 0.008336322382092476,
+      "logits/rejected": 0.10319779068231583,
+      "logps/chosen": -1.722665786743164,
+      "logps/rejected": -1.7800474166870117,
+      "loss": 0.7006,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.722665786743164,
+      "rewards/margins": 0.057381521910429,
+      "rewards/rejected": -1.7800474166870117,
       "step": 110
     },
     {
       "epoch": 0.06154875397223616,
-      "grad_norm": 2.9825233697805955,
+      "grad_norm": 1.6524246779107246,
       "learning_rate": 2.049910873440285e-07,
-      "logits/chosen": 0.05384382605552673,
-      "logits/rejected": 0.2680582106113434,
-      "logps/chosen": -1.8651673793792725,
-      "logps/rejected": -2.2020838260650635,
-      "loss": 0.9855,
+      "logits/chosen": 0.048177897930145264,
+      "logits/rejected": 0.2588050365447998,
+      "logps/chosen": -1.7102701663970947,
+      "logps/rejected": -2.005235195159912,
+      "loss": 0.6837,
       "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.8651673793792725,
-      "rewards/margins": 0.3369162678718567,
-      "rewards/rejected": -2.2020838260650635,
-      "semantic_entropy": 0.6000307202339172,
+      "rewards/chosen": -1.7102701663970947,
+      "rewards/margins": 0.2949647307395935,
+      "rewards/rejected": -2.005235195159912,
       "step": 115
     },
     {
       "epoch": 0.06422478675363773,
-      "grad_norm": 1.9505912039931628,
+      "grad_norm": 1.0597266151382978,
       "learning_rate": 2.13903743315508e-07,
-      "logits/chosen": -0.08002709597349167,
-      "logits/rejected": 0.09454605728387833,
-      "logps/chosen": -2.015916585922241,
-      "logps/rejected": -2.1636242866516113,
-      "loss": 0.9842,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -2.015916585922241,
-      "rewards/margins": 0.14770765602588654,
-      "rewards/rejected": -2.1636242866516113,
-      "semantic_entropy": 0.602546215057373,
+      "logits/chosen": -0.07734085619449615,
+      "logits/rejected": 0.0967831015586853,
+      "logps/chosen": -1.8145679235458374,
+      "logps/rejected": -1.9412740468978882,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.8145679235458374,
+      "rewards/margins": 0.12670619785785675,
+      "rewards/rejected": -1.9412740468978882,
       "step": 120
     },
     {
       "epoch": 0.0669008195350393,
-      "grad_norm": 2.4378570169024125,
+      "grad_norm": 1.2207525811649398,
       "learning_rate": 2.2281639928698751e-07,
-      "logits/chosen": -0.06087902933359146,
-      "logits/rejected": 0.07570815086364746,
-      "logps/chosen": -1.9079694747924805,
-      "logps/rejected": -1.8186849355697632,
-      "loss": 1.0258,
+      "logits/chosen": -0.06347334384918213,
+      "logits/rejected": 0.06946655362844467,
+      "logps/chosen": -1.7345211505889893,
+      "logps/rejected": -1.6708170175552368,
+      "loss": 0.708,
       "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.9079694747924805,
-      "rewards/margins": -0.08928458392620087,
-      "rewards/rejected": -1.8186849355697632,
-      "semantic_entropy": 0.637684166431427,
+      "rewards/chosen": -1.7345211505889893,
+      "rewards/margins": -0.0637039765715599,
+      "rewards/rejected": -1.6708170175552368,
       "step": 125
     },
     {
       "epoch": 0.06957685231644088,
-      "grad_norm": 2.197936029029749,
+      "grad_norm": 1.7868467219071085,
       "learning_rate": 2.31729055258467e-07,
-      "logits/chosen": 0.05631799250841141,
-      "logits/rejected": 0.19545042514801025,
-      "logps/chosen": -1.9789546728134155,
-      "logps/rejected": -2.1014046669006348,
-      "loss": 0.9897,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -1.9789546728134155,
-      "rewards/margins": 0.12244995683431625,
-      "rewards/rejected": -2.1014046669006348,
-      "semantic_entropy": 0.5949744582176208,
+      "logits/chosen": 0.047285933047533035,
+      "logits/rejected": 0.18318286538124084,
+      "logps/chosen": -1.7868776321411133,
+      "logps/rejected": -1.9025436639785767,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.7868776321411133,
+      "rewards/margins": 0.1156659945845604,
+      "rewards/rejected": -1.9025436639785767,
       "step": 130
     },
     {
       "epoch": 0.07225288509784245,
-      "grad_norm": 2.032783408722629,
+      "grad_norm": 1.0574437317309833,
       "learning_rate": 2.406417112299465e-07,
-      "logits/chosen": -0.03284865617752075,
-      "logits/rejected": 0.08819415420293808,
-      "logps/chosen": -2.0662684440612793,
-      "logps/rejected": -2.033278703689575,
-      "loss": 0.995,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -2.0662684440612793,
-      "rewards/margins": -0.03298996761441231,
-      "rewards/rejected": -2.033278703689575,
-      "semantic_entropy": 0.5904109477996826,
+      "logits/chosen": -0.02000136487185955,
+      "logits/rejected": 0.10176044702529907,
+      "logps/chosen": -1.8516027927398682,
+      "logps/rejected": -1.8486722707748413,
+      "loss": 0.7008,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.8516027927398682,
+      "rewards/margins": -0.0029304996132850647,
+      "rewards/rejected": -1.8486722707748413,
       "step": 135
     },
     {
       "epoch": 0.07492891787924402,
-      "grad_norm": 2.6671873642238926,
+      "grad_norm": 1.55667604518512,
       "learning_rate": 2.49554367201426e-07,
-      "logits/chosen": -0.017542561516165733,
-      "logits/rejected": 0.15911996364593506,
-      "logps/chosen": -2.0408124923706055,
-      "logps/rejected": -2.313681125640869,
-      "loss": 0.9716,
+      "logits/chosen": -0.029603471979498863,
+      "logits/rejected": 0.138214111328125,
+      "logps/chosen": -1.8117822408676147,
+      "logps/rejected": -2.000080108642578,
+      "loss": 0.6906,
       "rewards/accuracies": 0.53125,
-      "rewards/chosen": -2.0408124923706055,
-      "rewards/margins": 0.2728685736656189,
-      "rewards/rejected": -2.313681125640869,
-      "semantic_entropy": 0.5628100633621216,
+      "rewards/chosen": -1.8117822408676147,
+      "rewards/margins": 0.18829792737960815,
+      "rewards/rejected": -2.000080108642578,
       "step": 140
     },
     {
       "epoch": 0.0776049506606456,
-      "grad_norm": 1.7371072314652518,
+      "grad_norm": 1.2093578091845676,
       "learning_rate": 2.5846702317290554e-07,
-      "logits/chosen": -0.001353137195110321,
-      "logits/rejected": 0.15915007889270782,
-      "logps/chosen": -2.0290820598602295,
-      "logps/rejected": -2.180528402328491,
-      "loss": 0.9847,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -2.0290820598602295,
-      "rewards/margins": 0.15144631266593933,
-      "rewards/rejected": -2.180528402328491,
-      "semantic_entropy": 0.5903438925743103,
+      "logits/chosen": -0.010156979784369469,
+      "logits/rejected": 0.1453566551208496,
+      "logps/chosen": -1.7418346405029297,
+      "logps/rejected": -1.8639001846313477,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.7418346405029297,
+      "rewards/margins": 0.12206555902957916,
+      "rewards/rejected": -1.8639001846313477,
       "step": 145
     },
     {
       "epoch": 0.08028098344204716,
-      "grad_norm": 1.7435071095100478,
+      "grad_norm": 1.4299081591739666,
       "learning_rate": 2.6737967914438503e-07,
-      "logits/chosen": -0.04138721898198128,
-      "logits/rejected": 0.1333373486995697,
-      "logps/chosen": -2.02736496925354,
-      "logps/rejected": -2.030186176300049,
-      "loss": 1.0075,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -2.02736496925354,
-      "rewards/margins": 0.002821344183757901,
-      "rewards/rejected": -2.030186176300049,
-      "semantic_entropy": 0.6304464340209961,
+      "logits/chosen": -0.027264539152383804,
+      "logits/rejected": 0.14219091832637787,
+      "logps/chosen": -1.6984117031097412,
+      "logps/rejected": -1.693185567855835,
+      "loss": 0.6939,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.6984117031097412,
+      "rewards/margins": -0.005226260516792536,
+      "rewards/rejected": -1.693185567855835,
       "step": 150
     },
     {
       "epoch": 0.08295701622344874,
-      "grad_norm": 1.9328024034887885,
+      "grad_norm": 1.454815905431826,
       "learning_rate": 2.762923351158645e-07,
-      "logits/chosen": -0.04909590631723404,
-      "logits/rejected": -0.0030919373966753483,
-      "logps/chosen": -2.1588127613067627,
-      "logps/rejected": -2.1994755268096924,
-      "loss": 0.9896,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -2.1588127613067627,
-      "rewards/margins": 0.04066266492009163,
-      "rewards/rejected": -2.1994755268096924,
-      "semantic_entropy": 0.5864036679267883,
+      "logits/chosen": -0.04551117494702339,
+      "logits/rejected": 0.0031908005475997925,
+      "logps/chosen": -1.7394298315048218,
+      "logps/rejected": -1.8158365488052368,
+      "loss": 0.6939,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.7394298315048218,
+      "rewards/margins": 0.07640668004751205,
+      "rewards/rejected": -1.8158365488052368,
       "step": 155
     },
     {
       "epoch": 0.0856330490048503,
-      "grad_norm": 2.1134867518052562,
+      "grad_norm": 1.2199813409864593,
       "learning_rate": 2.85204991087344e-07,
-      "logits/chosen": -0.12479829788208008,
-      "logits/rejected": 0.024095604196190834,
-      "logps/chosen": -2.371675491333008,
-      "logps/rejected": -2.3460021018981934,
-      "loss": 0.9748,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -2.371675491333008,
-      "rewards/margins": -0.025673795491456985,
-      "rewards/rejected": -2.3460021018981934,
-      "semantic_entropy": 0.5358477830886841,
+      "logits/chosen": -0.14175042510032654,
+      "logits/rejected": -0.00038212575600482523,
+      "logps/chosen": -1.8869050741195679,
+      "logps/rejected": -1.8587558269500732,
+      "loss": 0.7102,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.8869050741195679,
+      "rewards/margins": -0.02814950980246067,
+      "rewards/rejected": -1.8587558269500732,
       "step": 160
     },
     {
       "epoch": 0.08830908178625188,
-      "grad_norm": 1.8356875110494215,
+      "grad_norm": 1.2605017499754132,
       "learning_rate": 2.941176470588235e-07,
-      "logits/chosen": -0.026149773970246315,
-      "logits/rejected": 0.16260910034179688,
-      "logps/chosen": -2.0882153511047363,
-      "logps/rejected": -2.4208343029022217,
-      "loss": 0.9796,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -2.0882153511047363,
-      "rewards/margins": 0.3326188921928406,
-      "rewards/rejected": -2.4208343029022217,
-      "semantic_entropy": 0.5765129327774048,
+      "logits/chosen": -0.02943740412592888,
+      "logits/rejected": 0.14758142828941345,
+      "logps/chosen": -1.689809799194336,
+      "logps/rejected": -1.883737325668335,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.689809799194336,
+      "rewards/margins": 0.1939275860786438,
+      "rewards/rejected": -1.883737325668335,
       "step": 165
     },
     {
       "epoch": 0.09098511456765346,
-      "grad_norm": 1.848284435517593,
+      "grad_norm": 1.2772833855323904,
       "learning_rate": 3.0303030303030305e-07,
-      "logits/chosen": -0.05960767716169357,
-      "logits/rejected": -0.005197498947381973,
-      "logps/chosen": -2.387760639190674,
-      "logps/rejected": -2.352987289428711,
-      "loss": 0.9696,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -2.387760639190674,
-      "rewards/margins": -0.03477310389280319,
-      "rewards/rejected": -2.352987289428711,
-      "semantic_entropy": 0.5449532866477966,
+      "logits/chosen": -0.07181887328624725,
+      "logits/rejected": -0.01916142739355564,
+      "logps/chosen": -1.8710161447525024,
+      "logps/rejected": -1.8953132629394531,
+      "loss": 0.6985,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.8710161447525024,
+      "rewards/margins": 0.024297043681144714,
+      "rewards/rejected": -1.8953132629394531,
       "step": 170
     },
     {
       "epoch": 0.09366114734905502,
-      "grad_norm": 1.9387824336292638,
+      "grad_norm": 1.2022506651642322,
       "learning_rate": 3.1194295900178254e-07,
-      "logits/chosen": 0.05171019956469536,
-      "logits/rejected": 0.04930907487869263,
-      "logps/chosen": -2.2678236961364746,
-      "logps/rejected": -2.327683448791504,
-      "loss": 0.9899,
-      "rewards/accuracies": 0.46875,
-      "rewards/chosen": -2.2678236961364746,
-      "rewards/margins": 0.059859711676836014,
-      "rewards/rejected": -2.327683448791504,
-      "semantic_entropy": 0.5564080476760864,
+      "logits/chosen": 0.08276864141225815,
+      "logits/rejected": 0.08133365213871002,
+      "logps/chosen": -1.7389723062515259,
+      "logps/rejected": -1.8196957111358643,
+      "loss": 0.7091,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.7389723062515259,
+      "rewards/margins": 0.08072330802679062,
+      "rewards/rejected": -1.8196957111358643,
       "step": 175
     },
     {
       "epoch": 0.0963371801304566,
-      "grad_norm": 1.4929818472340384,
+      "grad_norm": 1.145333564131534,
       "learning_rate": 3.2085561497326203e-07,
-      "logits/chosen": 0.029514130204916,
-      "logits/rejected": 0.033892035484313965,
-      "logps/chosen": -2.38169002532959,
-      "logps/rejected": -2.313992500305176,
-      "loss": 0.9687,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -2.38169002532959,
-      "rewards/margins": -0.06769762188196182,
-      "rewards/rejected": -2.313992500305176,
-      "semantic_entropy": 0.5526784658432007,
+      "logits/chosen": 0.046460725367069244,
+      "logits/rejected": 0.04928620904684067,
+      "logps/chosen": -1.7828443050384521,
+      "logps/rejected": -1.871498703956604,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.7828443050384521,
+      "rewards/margins": 0.08865445107221603,
+      "rewards/rejected": -1.871498703956604,
       "step": 180
     },
     {
       "epoch": 0.09901321291185818,
-      "grad_norm": 2.000432126573273,
+      "grad_norm": 1.556205489207771,
       "learning_rate": 3.297682709447415e-07,
-      "logits/chosen": -0.12632909417152405,
-      "logits/rejected": -0.030294334515929222,
-      "logps/chosen": -2.315847873687744,
-      "logps/rejected": -2.402444839477539,
-      "loss": 0.9894,
+      "logits/chosen": -0.1105707660317421,
+      "logits/rejected": -0.019754672423005104,
+      "logps/chosen": -1.7332690954208374,
+      "logps/rejected": -1.7963947057724,
+      "loss": 0.7034,
       "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -2.315847873687744,
-      "rewards/margins": 0.08659707754850388,
-      "rewards/rejected": -2.402444839477539,
-      "semantic_entropy": 0.5762835144996643,
+      "rewards/chosen": -1.7332690954208374,
+      "rewards/margins": 0.06312567740678787,
+      "rewards/rejected": -1.7963947057724,
       "step": 185
     },
     {
       "epoch": 0.10168924569325974,
-      "grad_norm": 3.647045810045853,
+      "grad_norm": 2.092232288493559,
       "learning_rate": 3.38680926916221e-07,
-      "logits/chosen": -0.06854553520679474,
-      "logits/rejected": 0.06519101560115814,
-      "logps/chosen": -2.957479238510132,
-      "logps/rejected": -2.758251667022705,
-      "loss": 0.9394,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -2.957479238510132,
-      "rewards/margins": -0.1992277055978775,
-      "rewards/rejected": -2.758251667022705,
-      "semantic_entropy": 0.4874970018863678,
+      "logits/chosen": -0.014930409379303455,
+      "logits/rejected": 0.11289344727993011,
+      "logps/chosen": -1.9244152307510376,
+      "logps/rejected": -1.9324283599853516,
+      "loss": 0.7007,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -1.9244152307510376,
+      "rewards/margins": 0.008013313636183739,
+      "rewards/rejected": -1.9324283599853516,
       "step": 190
     },
     {
       "epoch": 0.10436527847466132,
-      "grad_norm": 3.2099582882536097,
+      "grad_norm": 1.1452321555176908,
       "learning_rate": 3.475935828877005e-07,
-      "logits/chosen": 0.04098251461982727,
-      "logits/rejected": 0.21246996521949768,
-      "logps/chosen": -2.2964203357696533,
-      "logps/rejected": -2.3343770503997803,
-      "loss": 0.9712,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -2.2964203357696533,
-      "rewards/margins": 0.03795675188302994,
-      "rewards/rejected": -2.3343770503997803,
-      "semantic_entropy": 0.5516759157180786,
+      "logits/chosen": 0.05341753363609314,
+      "logits/rejected": 0.21590080857276917,
+      "logps/chosen": -1.6111427545547485,
+      "logps/rejected": -1.7328262329101562,
+      "loss": 0.6935,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.6111427545547485,
+      "rewards/margins": 0.12168345600366592,
+      "rewards/rejected": -1.7328262329101562,
       "step": 195
     },
     {
       "epoch": 0.1070413112560629,
-      "grad_norm": 2.342954741982292,
+      "grad_norm": 1.1597703022470536,
       "learning_rate": 3.5650623885918e-07,
-      "logits/chosen": -0.061466217041015625,
-      "logits/rejected": 0.09133400768041611,
-      "logps/chosen": -3.0099568367004395,
-      "logps/rejected": -2.6031436920166016,
-      "loss": 0.9477,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -3.0099568367004395,
-      "rewards/margins": -0.4068129062652588,
-      "rewards/rejected": -2.6031436920166016,
-      "semantic_entropy": 0.4999690055847168,
+      "logits/chosen": -0.052175890654325485,
+      "logits/rejected": 0.09055675566196442,
+      "logps/chosen": -1.834405541419983,
+      "logps/rejected": -1.7582638263702393,
+      "loss": 0.7,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.834405541419983,
+      "rewards/margins": -0.07614155858755112,
+      "rewards/rejected": -1.7582638263702393,
       "step": 200
     },
     {
       "epoch": 0.10971734403746446,
-      "grad_norm": 2.206781770562224,
+      "grad_norm": 1.3065559049065374,
       "learning_rate": 3.654188948306595e-07,
-      "logits/chosen": -0.02822372317314148,
-      "logits/rejected": 0.14092370867729187,
-      "logps/chosen": -3.579482316970825,
-      "logps/rejected": -3.0718302726745605,
-      "loss": 0.9242,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -3.579482316970825,
-      "rewards/margins": -0.5076526999473572,
-      "rewards/rejected": -3.0718302726745605,
-      "semantic_entropy": 0.4719177186489105,
+      "logits/chosen": -0.01260521449148655,
+      "logits/rejected": 0.1416720598936081,
+      "logps/chosen": -1.878777265548706,
+      "logps/rejected": -1.7897173166275024,
+      "loss": 0.6976,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.878777265548706,
+      "rewards/margins": -0.08905981481075287,
+      "rewards/rejected": -1.7897173166275024,
       "step": 205
     },
     {
       "epoch": 0.11239337681886603,
-      "grad_norm": 1.9536371898158273,
+      "grad_norm": 1.1121907087083491,
       "learning_rate": 3.7433155080213904e-07,
-      "logits/chosen": -0.14045406877994537,
-      "logits/rejected": 0.07738684117794037,
-      "logps/chosen": -3.3335089683532715,
-      "logps/rejected": -3.8232810497283936,
-      "loss": 0.9007,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -3.3335089683532715,
-      "rewards/margins": 0.48977145552635193,
-      "rewards/rejected": -3.8232810497283936,
-      "semantic_entropy": 0.4463028013706207,
+      "logits/chosen": -0.12877874076366425,
+      "logits/rejected": 0.07078342139720917,
+      "logps/chosen": -1.8128242492675781,
+      "logps/rejected": -2.02532958984375,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.8128242492675781,
+      "rewards/margins": 0.21250522136688232,
+      "rewards/rejected": -2.02532958984375,
       "step": 210
     },
     {
       "epoch": 0.1150694096002676,
-      "grad_norm": 1.9027023859188623,
+      "grad_norm": 1.0341378504812355,
       "learning_rate": 3.8324420677361853e-07,
-      "logits/chosen": -0.17621003091335297,
-      "logits/rejected": 0.0929396003484726,
-      "logps/chosen": -3.0562806129455566,
-      "logps/rejected": -3.216024398803711,
-      "loss": 0.9157,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -3.0562806129455566,
-      "rewards/margins": 0.1597435176372528,
-      "rewards/rejected": -3.216024398803711,
-      "semantic_entropy": 0.46098580956459045,
+      "logits/chosen": -0.1585201919078827,
+      "logits/rejected": 0.09438496083021164,
+      "logps/chosen": -1.715761423110962,
+      "logps/rejected": -1.7956126928329468,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.715761423110962,
+      "rewards/margins": 0.0798514112830162,
+      "rewards/rejected": -1.7956126928329468,
       "step": 215
     },
     {
       "epoch": 0.11774544238166917,
-      "grad_norm": 1.8361832882796905,
+      "grad_norm": 1.1248004065302584,
       "learning_rate": 3.92156862745098e-07,
-      "logits/chosen": 0.05754054710268974,
-      "logits/rejected": 0.16974332928657532,
-      "logps/chosen": -3.714381694793701,
-      "logps/rejected": -4.291443824768066,
-      "loss": 0.8891,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -3.714381694793701,
-      "rewards/margins": 0.5770620703697205,
-      "rewards/rejected": -4.291443824768066,
-      "semantic_entropy": 0.40960121154785156,
+      "logits/chosen": 0.08871813863515854,
+      "logits/rejected": 0.19400301575660706,
+      "logps/chosen": -1.752228021621704,
+      "logps/rejected": -1.9849464893341064,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.752228021621704,
+      "rewards/margins": 0.2327185869216919,
+      "rewards/rejected": -1.9849464893341064,
       "step": 220
     },
     {
       "epoch": 0.12042147516307075,
-      "grad_norm": 2.285658486805942,
+      "grad_norm": 1.2919471977237502,
       "learning_rate": 4.010695187165775e-07,
-      "logits/chosen": -0.11826293170452118,
-      "logits/rejected": 0.06798942387104034,
-      "logps/chosen": -3.4444961547851562,
-      "logps/rejected": -3.5065066814422607,
-      "loss": 0.8963,
+      "logits/chosen": -0.07584510743618011,
+      "logits/rejected": 0.09964814782142639,
+      "logps/chosen": -1.6696830987930298,
+      "logps/rejected": -1.793252944946289,
+      "loss": 0.6837,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -3.4444961547851562,
-      "rewards/margins": 0.062010981142520905,
-      "rewards/rejected": -3.5065066814422607,
-      "semantic_entropy": 0.4190169870853424,
+      "rewards/chosen": -1.6696830987930298,
+      "rewards/margins": 0.12356986850500107,
+      "rewards/rejected": -1.793252944946289,
       "step": 225
     },
     {
       "epoch": 0.12309750794447231,
-      "grad_norm": 2.861208367496247,
+      "grad_norm": 1.2939926446713124,
       "learning_rate": 4.09982174688057e-07,
-      "logits/chosen": 0.020364487543702126,
-      "logits/rejected": 0.11460238695144653,
-      "logps/chosen": -4.294821739196777,
-      "logps/rejected": -4.23034143447876,
-      "loss": 0.8802,
+      "logits/chosen": 0.022558122873306274,
+      "logits/rejected": 0.1035970002412796,
+      "logps/chosen": -1.7908645868301392,
+      "logps/rejected": -1.9070565700531006,
+      "loss": 0.6899,
       "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -4.294821739196777,
-      "rewards/margins": -0.06447985023260117,
-      "rewards/rejected": -4.23034143447876,
-      "semantic_entropy": 0.3771011233329773,
+      "rewards/chosen": -1.7908645868301392,
+      "rewards/margins": 0.11619196832180023,
+      "rewards/rejected": -1.9070565700531006,
       "step": 230
     },
     {
       "epoch": 0.1257735407258739,
-      "grad_norm": 2.25490626392827,
+      "grad_norm": 1.189895693940023,
       "learning_rate": 4.188948306595365e-07,
-      "logits/chosen": 0.04717455059289932,
-      "logits/rejected": 0.21829184889793396,
-      "logps/chosen": -4.091797828674316,
-      "logps/rejected": -4.45542573928833,
-      "loss": 0.863,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -4.091797828674316,
-      "rewards/margins": 0.36362800002098083,
-      "rewards/rejected": -4.45542573928833,
-      "semantic_entropy": 0.37177348136901855,
+      "logits/chosen": 0.04974474385380745,
+      "logits/rejected": 0.2011227309703827,
+      "logps/chosen": -1.6742194890975952,
+      "logps/rejected": -1.8701362609863281,
+      "loss": 0.6735,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.6742194890975952,
+      "rewards/margins": 0.19591686129570007,
+      "rewards/rejected": -1.8701362609863281,
       "step": 235
     },
     {
       "epoch": 0.12844957350727546,
-      "grad_norm": 2.038862467341056,
+      "grad_norm": 1.27568186232975,
       "learning_rate": 4.27807486631016e-07,
-      "logits/chosen": 0.033001385629177094,
-      "logits/rejected": 0.17964918911457062,
-      "logps/chosen": -4.601205348968506,
-      "logps/rejected": -4.567494869232178,
-      "loss": 0.8617,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -4.601205348968506,
-      "rewards/margins": -0.03371014446020126,
-      "rewards/rejected": -4.567494869232178,
-      "semantic_entropy": 0.343085378408432,
+      "logits/chosen": 0.020980415865778923,
+      "logits/rejected": 0.15631407499313354,
+      "logps/chosen": -1.7081215381622314,
+      "logps/rejected": -1.8699896335601807,
+      "loss": 0.6949,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.7081215381622314,
+      "rewards/margins": 0.16186803579330444,
+      "rewards/rejected": -1.8699896335601807,
       "step": 240
     },
     {
       "epoch": 0.13112560628867703,
-      "grad_norm": 1.5335908988324936,
+      "grad_norm": 1.29646845790665,
       "learning_rate": 4.3672014260249554e-07,
-      "logits/chosen": 0.08814103901386261,
-      "logits/rejected": 0.23658275604248047,
-      "logps/chosen": -4.161761283874512,
-      "logps/rejected": -5.193979740142822,
-      "loss": 0.8613,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -4.161761283874512,
-      "rewards/margins": 1.0322182178497314,
-      "rewards/rejected": -5.193979740142822,
-      "semantic_entropy": 0.3477904200553894,
+      "logits/chosen": 0.05687965080142021,
+      "logits/rejected": 0.1840699166059494,
+      "logps/chosen": -1.7213294506072998,
+      "logps/rejected": -1.86166512966156,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.7213294506072998,
+      "rewards/margins": 0.1403358280658722,
+      "rewards/rejected": -1.86166512966156,
       "step": 245
     },
     {
       "epoch": 0.1338016390700786,
-      "grad_norm": 1.3637568549056873,
+      "grad_norm": 1.2864264065672446,
       "learning_rate": 4.4563279857397503e-07,
-      "logits/chosen": 0.03782828524708748,
-      "logits/rejected": 0.23347607254981995,
-      "logps/chosen": -6.4975409507751465,
-      "logps/rejected": -6.319209575653076,
-      "loss": 0.8508,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -6.4975409507751465,
-      "rewards/margins": -0.1783314198255539,
-      "rewards/rejected": -6.319209575653076,
-      "semantic_entropy": 0.32665157318115234,
+      "logits/chosen": -0.028925329446792603,
+      "logits/rejected": 0.1428970992565155,
+      "logps/chosen": -1.8295762538909912,
+      "logps/rejected": -1.8573747873306274,
+      "loss": 0.697,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.8295762538909912,
+      "rewards/margins": 0.027798574417829514,
+      "rewards/rejected": -1.8573747873306274,
       "step": 250
     },
     {
       "epoch": 0.1364776718514802,
-      "grad_norm": 1.1763340019359527,
+      "grad_norm": 1.3759152624449766,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": 0.061561692506074905,
-      "logits/rejected": 0.2616002559661865,
-      "logps/chosen": -4.340880393981934,
-      "logps/rejected": -5.2646636962890625,
-      "loss": 0.8575,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -4.340880393981934,
-      "rewards/margins": 0.9237837791442871,
-      "rewards/rejected": -5.2646636962890625,
-      "semantic_entropy": 0.34385281801223755,
+      "logits/chosen": 0.002454365137964487,
+      "logits/rejected": 0.15873923897743225,
+      "logps/chosen": -1.5566253662109375,
+      "logps/rejected": -1.7483634948730469,
+      "loss": 0.6787,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.5566253662109375,
+      "rewards/margins": 0.19173792004585266,
+      "rewards/rejected": -1.7483634948730469,
       "step": 255
     },
     {
       "epoch": 0.13915370463288176,
-      "grad_norm": 1.4195645633126868,
+      "grad_norm": 1.3112288468294648,
       "learning_rate": 4.63458110516934e-07,
-      "logits/chosen": -0.13105076551437378,
-      "logits/rejected": 0.0006925135967321694,
-      "logps/chosen": -5.7697014808654785,
-      "logps/rejected": -4.83870792388916,
-      "loss": 0.8343,
+      "logits/chosen": -0.23012837767601013,
+      "logits/rejected": -0.12351049482822418,
+      "logps/chosen": -1.7591259479522705,
+      "logps/rejected": -1.8154079914093018,
+      "loss": 0.6831,
       "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -5.7697014808654785,
-      "rewards/margins": -0.9309929609298706,
-      "rewards/rejected": -4.83870792388916,
-      "semantic_entropy": 0.29603084921836853,
+      "rewards/chosen": -1.7591259479522705,
+      "rewards/margins": 0.05628199130296707,
+      "rewards/rejected": -1.8154079914093018,
       "step": 260
     },
     {
       "epoch": 0.1418297374142833,
-      "grad_norm": 1.462524087828739,
+      "grad_norm": 0.944708262887706,
       "learning_rate": 4.723707664884135e-07,
-      "logits/chosen": 0.055274952203035355,
-      "logits/rejected": 0.14958932995796204,
-      "logps/chosen": -6.936572074890137,
-      "logps/rejected": -5.408553600311279,
-      "loss": 0.827,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -6.936572074890137,
-      "rewards/margins": -1.5280190706253052,
-      "rewards/rejected": -5.408553600311279,
-      "semantic_entropy": 0.259237140417099,
+      "logits/chosen": -0.0648413747549057,
+      "logits/rejected": 0.01500119548290968,
+      "logps/chosen": -1.8386303186416626,
+      "logps/rejected": -1.8570829629898071,
+      "loss": 0.7013,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.8386303186416626,
+      "rewards/margins": 0.01845243014395237,
+      "rewards/rejected": -1.8570829629898071,
       "step": 265
     },
     {
       "epoch": 0.1445057701956849,
-      "grad_norm": 1.4360352919402117,
+      "grad_norm": 1.3974096751313816,
       "learning_rate": 4.81283422459893e-07,
-      "logits/chosen": 0.010314211249351501,
-      "logits/rejected": 0.19535258412361145,
-      "logps/chosen": -5.553572177886963,
-      "logps/rejected": -5.879666805267334,
-      "loss": 0.8289,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -5.553572177886963,
-      "rewards/margins": 0.3260951638221741,
-      "rewards/rejected": -5.879666805267334,
-      "semantic_entropy": 0.27346399426460266,
+      "logits/chosen": -0.0771542564034462,
+      "logits/rejected": 0.05933430790901184,
+      "logps/chosen": -1.6465028524398804,
+      "logps/rejected": -1.7822353839874268,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.6465028524398804,
+      "rewards/margins": 0.13573241233825684,
+      "rewards/rejected": -1.7822353839874268,
       "step": 270
     },
     {
       "epoch": 0.14718180297708647,
-      "grad_norm": 2.4387140987358853,
+      "grad_norm": 1.5124992015842769,
       "learning_rate": 4.901960784313725e-07,
-      "logits/chosen": 0.14650209248065948,
-      "logits/rejected": 0.2420198619365692,
-      "logps/chosen": -6.479383945465088,
-      "logps/rejected": -7.153507232666016,
-      "loss": 0.8353,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -6.479383945465088,
-      "rewards/margins": 0.6741241216659546,
-      "rewards/rejected": -7.153507232666016,
-      "semantic_entropy": 0.27780357003211975,
+      "logits/chosen": 0.007635164074599743,
+      "logits/rejected": 0.10988122224807739,
+      "logps/chosen": -1.6370174884796143,
+      "logps/rejected": -1.8470070362091064,
+      "loss": 0.7002,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.6370174884796143,
+      "rewards/margins": 0.2099895477294922,
+      "rewards/rejected": -1.8470070362091064,
       "step": 275
     },
     {
       "epoch": 0.14985783575848804,
-      "grad_norm": 3.7956134985906087,
+      "grad_norm": 1.2548255793990983,
       "learning_rate": 4.99108734402852e-07,
-      "logits/chosen": 0.036770034581422806,
-      "logits/rejected": 0.2553003430366516,
-      "logps/chosen": -7.673718452453613,
-      "logps/rejected": -7.357241153717041,
-      "loss": 0.805,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -7.673718452453613,
-      "rewards/margins": -0.3164777457714081,
-      "rewards/rejected": -7.357241153717041,
-      "semantic_entropy": 0.23823335766792297,
+      "logits/chosen": -0.1115298867225647,
+      "logits/rejected": 0.04553290084004402,
+      "logps/chosen": -1.7539443969726562,
+      "logps/rejected": -1.8172544240951538,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.7539443969726562,
+      "rewards/margins": 0.06331010162830353,
+      "rewards/rejected": -1.8172544240951538,
       "step": 280
     },
     {
       "epoch": 0.15253386853988962,
-      "grad_norm": 8.948656561947692,
+      "grad_norm": 1.496111082155346,
       "learning_rate": 5.080213903743315e-07,
-      "logits/chosen": 0.13513311743736267,
-      "logits/rejected": 0.32487455010414124,
-      "logps/chosen": -7.544847011566162,
-      "logps/rejected": -7.555774688720703,
-      "loss": 0.7811,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -7.544847011566162,
-      "rewards/margins": 0.010928010568022728,
-      "rewards/rejected": -7.555774688720703,
-      "semantic_entropy": 0.17771926522254944,
+      "logits/chosen": -0.0698985680937767,
+      "logits/rejected": 0.07616071403026581,
+      "logps/chosen": -1.6921838521957397,
+      "logps/rejected": -1.78534734249115,
+      "loss": 0.7012,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.6921838521957397,
+      "rewards/margins": 0.09316326677799225,
+      "rewards/rejected": -1.78534734249115,
       "step": 285
     },
     {
       "epoch": 0.1552099013212912,
-      "grad_norm": 1.9471413709731125,
+      "grad_norm": 1.315967936336575,
       "learning_rate": 5.169340463458111e-07,
-      "logits/chosen": 0.5130095481872559,
-      "logits/rejected": 0.8595854043960571,
-      "logps/chosen": -8.675241470336914,
-      "logps/rejected": -10.537249565124512,
-      "loss": 0.7182,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -8.675241470336914,
-      "rewards/margins": 1.8620080947875977,
-      "rewards/rejected": -10.537249565124512,
-      "semantic_entropy": 0.055742353200912476,
+      "logits/chosen": -0.13665227591991425,
+      "logits/rejected": 0.16268080472946167,
+      "logps/chosen": -1.6165683269500732,
+      "logps/rejected": -1.8295272588729858,
+      "loss": 0.6703,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.6165683269500732,
+      "rewards/margins": 0.2129589021205902,
+      "rewards/rejected": -1.8295272588729858,
       "step": 290
     },
     {
       "epoch": 0.15788593410269275,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9836657426389332,
       "learning_rate": 5.258467023172905e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 430.0484,
-      "rewards/accuracies": 0.32499998807907104,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.08361764252185822,
+      "logits/rejected": -0.0278538279235363,
+      "logps/chosen": -1.627699851989746,
+      "logps/rejected": -1.7145698070526123,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.627699851989746,
+      "rewards/margins": 0.08687015622854233,
+      "rewards/rejected": -1.7145698070526123,
       "step": 295
     },
     {
       "epoch": 0.16056196688409433,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.545549627439456,
       "learning_rate": 5.347593582887701e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0865083858370781,
+      "logits/rejected": 0.08300237357616425,
+      "logps/chosen": -1.6731727123260498,
+      "logps/rejected": -1.7616103887557983,
+      "loss": 0.7045,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.6731727123260498,
+      "rewards/margins": 0.08843781054019928,
+      "rewards/rejected": -1.7616103887557983,
       "step": 300
     },
     {
       "epoch": 0.1632379996654959,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.537115335620621,
       "learning_rate": 5.436720142602496e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.04730618745088577,
+      "logits/rejected": 0.024881727993488312,
+      "logps/chosen": -1.6922677755355835,
+      "logps/rejected": -1.6909148693084717,
+      "loss": 0.6982,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.6922677755355835,
+      "rewards/margins": -0.0013529777061194181,
+      "rewards/rejected": -1.6909148693084717,
       "step": 305
     },
     {
       "epoch": 0.16591403244689748,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4741852862760636,
       "learning_rate": 5.52584670231729e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23181238770484924,
+      "logits/rejected": -0.1436997950077057,
+      "logps/chosen": -1.6866481304168701,
+      "logps/rejected": -1.7866075038909912,
+      "loss": 0.6946,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.6866481304168701,
+      "rewards/margins": 0.0999593734741211,
+      "rewards/rejected": -1.7866075038909912,
       "step": 310
     },
     {
       "epoch": 0.16859006522829906,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.536444689072242,
       "learning_rate": 5.614973262032086e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.04109283536672592,
+      "logits/rejected": 0.11791141331195831,
+      "logps/chosen": -1.6222527027130127,
+      "logps/rejected": -1.8005802631378174,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.6222527027130127,
+      "rewards/margins": 0.17832748591899872,
+      "rewards/rejected": -1.8005802631378174,
       "step": 315
     },
     {
       "epoch": 0.1712660980097006,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.4443551624050617,
       "learning_rate": 5.70409982174688e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.08019106090068817,
+      "logits/rejected": 0.05177772045135498,
+      "logps/chosen": -1.516701340675354,
+      "logps/rejected": -1.5698729753494263,
+      "loss": 0.6948,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.516701340675354,
+      "rewards/margins": 0.05317187309265137,
+      "rewards/rejected": -1.5698729753494263,
       "step": 320
     },
     {
       "epoch": 0.17394213079110218,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0668696459141356,
       "learning_rate": 5.793226381461676e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15901736915111542,
+      "logits/rejected": -0.042800091207027435,
+      "logps/chosen": -1.5198718309402466,
+      "logps/rejected": -1.7961242198944092,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.5198718309402466,
+      "rewards/margins": 0.2762523591518402,
+      "rewards/rejected": -1.7961242198944092,
       "step": 325
     },
     {
       "epoch": 0.17661816357250376,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8024615212471526,
       "learning_rate": 5.88235294117647e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.07843993604183197,
+      "logits/rejected": 0.06718756258487701,
+      "logps/chosen": -1.4581224918365479,
+      "logps/rejected": -1.7021875381469727,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.4581224918365479,
+      "rewards/margins": 0.2440650761127472,
+      "rewards/rejected": -1.7021875381469727,
       "step": 330
     },
     {
       "epoch": 0.17929419635390534,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.299883450479667,
       "learning_rate": 5.971479500891266e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.02555288001894951,
+      "logits/rejected": 0.07491449266672134,
+      "logps/chosen": -1.4644086360931396,
+      "logps/rejected": -1.5261727571487427,
+      "loss": 0.6976,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.4644086360931396,
+      "rewards/margins": 0.06176407262682915,
+      "rewards/rejected": -1.5261727571487427,
       "step": 335
     },
     {
       "epoch": 0.18197022913530692,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.36431082871939,
       "learning_rate": 6.060606060606061e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.07830534130334854,
+      "logits/rejected": 0.062340427190065384,
+      "logps/chosen": -1.5719006061553955,
+      "logps/rejected": -1.660887360572815,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5719006061553955,
+      "rewards/margins": 0.08898665755987167,
+      "rewards/rejected": -1.660887360572815,
       "step": 340
     },
     {
       "epoch": 0.1846462619167085,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3907462400824833,
       "learning_rate": 6.149732620320855e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.030298754572868347,
+      "logits/rejected": 0.058118999004364014,
+      "logps/chosen": -1.5408066511154175,
+      "logps/rejected": -1.704358458518982,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.5408066511154175,
+      "rewards/margins": 0.1635517179965973,
+      "rewards/rejected": -1.704358458518982,
       "step": 345
     },
     {
       "epoch": 0.18732229469811004,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.117168529312069,
       "learning_rate": 6.238859180035651e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.04363744705915451,
+      "logits/rejected": 0.050693780183792114,
+      "logps/chosen": -1.519193410873413,
+      "logps/rejected": -1.6225532293319702,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.519193410873413,
+      "rewards/margins": 0.10335978120565414,
+      "rewards/rejected": -1.6225532293319702,
       "step": 350
     },
     {
       "epoch": 0.18999832747951162,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.497862102687126,
       "learning_rate": 6.327985739750445e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.11044670641422272,
+      "logits/rejected": 0.11048382520675659,
+      "logps/chosen": -1.6010332107543945,
+      "logps/rejected": -1.6560262441635132,
+      "loss": 0.6941,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.6010332107543945,
+      "rewards/margins": 0.05499298498034477,
+      "rewards/rejected": -1.6560262441635132,
       "step": 355
     },
     {
       "epoch": 0.1926743602609132,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.577392393722349,
       "learning_rate": 6.417112299465241e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.11986102908849716,
+      "logits/rejected": -0.04494617134332657,
+      "logps/chosen": -1.5258471965789795,
+      "logps/rejected": -1.6857599020004272,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5258471965789795,
+      "rewards/margins": 0.15991251170635223,
+      "rewards/rejected": -1.6857599020004272,
       "step": 360
     },
     {
       "epoch": 0.19535039304231477,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 12.596458344936327,
       "learning_rate": 6.506238859180035e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.04750268533825874,
+      "logits/rejected": 0.03312084451317787,
+      "logps/chosen": -1.442544937133789,
+      "logps/rejected": -1.5319559574127197,
+      "loss": 0.6937,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.442544937133789,
+      "rewards/margins": 0.08941099047660828,
+      "rewards/rejected": -1.5319559574127197,
       "step": 365
     },
     {
       "epoch": 0.19802642582371635,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.080892232870765,
       "learning_rate": 6.59536541889483e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0880110040307045,
+      "logits/rejected": 0.0006408154731616378,
+      "logps/chosen": -1.4212582111358643,
+      "logps/rejected": -1.5003842115402222,
+      "loss": 0.7052,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.4212582111358643,
+      "rewards/margins": 0.07912586629390717,
+      "rewards/rejected": -1.5003842115402222,
       "step": 370
     },
     {
       "epoch": 0.2007024586051179,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.624518938969881,
       "learning_rate": 6.684491978609626e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.10961814969778061,
+      "logits/rejected": 0.04463160037994385,
+      "logps/chosen": -1.4106605052947998,
+      "logps/rejected": -1.6069059371948242,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.4106605052947998,
+      "rewards/margins": 0.1962454617023468,
+      "rewards/rejected": -1.6069059371948242,
       "step": 375
     },
     {
       "epoch": 0.20337849138651948,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0357135058274825,
       "learning_rate": 6.77361853832442e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0923214703798294,
+      "logits/rejected": -0.007986297830939293,
+      "logps/chosen": -1.4541934728622437,
+      "logps/rejected": -1.6574127674102783,
+      "loss": 0.6676,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.4541934728622437,
+      "rewards/margins": 0.20321913063526154,
+      "rewards/rejected": -1.6574127674102783,
       "step": 380
     },
     {
       "epoch": 0.20605452416792105,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.341203657727559,
       "learning_rate": 6.862745098039216e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.05698399618268013,
+      "logits/rejected": 0.01997918076813221,
+      "logps/chosen": -1.5176222324371338,
+      "logps/rejected": -1.570155382156372,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.5176222324371338,
+      "rewards/margins": 0.052533108741045,
+      "rewards/rejected": -1.570155382156372,
       "step": 385
     },
     {
       "epoch": 0.20873055694932263,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5366930031259143,
       "learning_rate": 6.95187165775401e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.014153100550174713,
+      "logits/rejected": 0.1775176227092743,
+      "logps/chosen": -1.5498108863830566,
+      "logps/rejected": -1.6640427112579346,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.5498108863830566,
+      "rewards/margins": 0.11423186957836151,
+      "rewards/rejected": -1.6640427112579346,
       "step": 390
     },
     {
       "epoch": 0.2114065897307242,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3658771085166004,
       "learning_rate": 7.040998217468806e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09060199558734894,
+      "logits/rejected": 0.0706435814499855,
+      "logps/chosen": -1.5506666898727417,
+      "logps/rejected": -1.6366159915924072,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5506666898727417,
+      "rewards/margins": 0.08594916760921478,
+      "rewards/rejected": -1.6366159915924072,
       "step": 395
     },
     {
       "epoch": 0.2140826225121258,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6335415473293184,
       "learning_rate": 7.1301247771836e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.012495418079197407,
+      "logits/rejected": 0.10497476905584335,
+      "logps/chosen": -1.6102592945098877,
+      "logps/rejected": -1.7506154775619507,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.6102592945098877,
+      "rewards/margins": 0.14035633206367493,
+      "rewards/rejected": -1.7506154775619507,
       "step": 400
     },
     {
       "epoch": 0.2140826225121258,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 35.7987,
-      "eval_samples_per_second": 37.571,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.414,
+      "eval_logits/chosen": 0.19963997602462769,
+      "eval_logits/rejected": 0.2857072055339813,
+      "eval_logps/chosen": -1.6443676948547363,
+      "eval_logps/rejected": -1.823267936706543,
+      "eval_loss": 0.6872766613960266,
+      "eval_rewards/accuracies": 0.5474777221679688,
+      "eval_rewards/chosen": -1.6443676948547363,
+      "eval_rewards/margins": 0.17890027165412903,
+      "eval_rewards/rejected": -1.823267936706543,
+      "eval_runtime": 41.414,
+      "eval_samples_per_second": 32.477,
+      "eval_steps_per_second": 8.137,
       "step": 400
     },
     {
       "epoch": 0.21675865529352734,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5827000876498674,
       "learning_rate": 7.219251336898395e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.074187733232975,
+      "logits/rejected": 0.012620595283806324,
+      "logps/chosen": -1.6763885021209717,
+      "logps/rejected": -1.8131437301635742,
+      "loss": 0.7013,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.6763885021209717,
+      "rewards/margins": 0.13675537705421448,
+      "rewards/rejected": -1.8131437301635742,
       "step": 405
     },
     {
       "epoch": 0.2194346880749289,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2549273092737847,
       "learning_rate": 7.30837789661319e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.047247715294361115,
+      "logits/rejected": 0.07532905042171478,
+      "logps/chosen": -1.616379737854004,
+      "logps/rejected": -1.7932064533233643,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.616379737854004,
+      "rewards/margins": 0.17682664096355438,
+      "rewards/rejected": -1.7932064533233643,
       "step": 410
     },
     {
       "epoch": 0.2221107208563305,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.2591510084854571,
       "learning_rate": 7.397504456327985e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.07612961530685425,
+      "logits/rejected": -0.037372883409261703,
+      "logps/chosen": -1.6571595668792725,
+      "logps/rejected": -1.8091316223144531,
+      "loss": 0.689,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.6571595668792725,
+      "rewards/margins": 0.15197204053401947,
+      "rewards/rejected": -1.8091316223144531,
       "step": 415
     },
     {
       "epoch": 0.22478675363773207,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8982166867018317,
       "learning_rate": 7.486631016042781e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.08281673491001129,
+      "logits/rejected": 0.10642486810684204,
+      "logps/chosen": -1.5453522205352783,
+      "logps/rejected": -1.6890758275985718,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.5453522205352783,
+      "rewards/margins": 0.14372369647026062,
+      "rewards/rejected": -1.6890758275985718,
       "step": 420
     },
     {
       "epoch": 0.22746278641913364,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.384911322942381,
       "learning_rate": 7.575757575757575e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12255939096212387,
+      "logits/rejected": 0.0701722577214241,
+      "logps/chosen": -1.6026155948638916,
+      "logps/rejected": -1.8186794519424438,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6026155948638916,
+      "rewards/margins": 0.21606406569480896,
+      "rewards/rejected": -1.8186794519424438,
       "step": 425
     },
     {
       "epoch": 0.2301388192005352,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.424244223702352,
       "learning_rate": 7.664884135472371e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.16784095764160156,
+      "logits/rejected": 0.025425409898161888,
+      "logps/chosen": -1.5373917818069458,
+      "logps/rejected": -1.8187201023101807,
+      "loss": 0.682,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.5373917818069458,
+      "rewards/margins": 0.2813282608985901,
+      "rewards/rejected": -1.8187201023101807,
       "step": 430
     },
     {
       "epoch": 0.23281485198193677,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2405100335861117,
       "learning_rate": 7.754010695187165e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09217321872711182,
+      "logits/rejected": -0.00494597852230072,
+      "logps/chosen": -1.4401617050170898,
+      "logps/rejected": -1.570569396018982,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.4401617050170898,
+      "rewards/margins": 0.13040751218795776,
+      "rewards/rejected": -1.570569396018982,
       "step": 435
     },
     {
       "epoch": 0.23549088476333835,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.0599020438504265,
       "learning_rate": 7.84313725490196e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1174207329750061,
+      "logits/rejected": -0.030725499615073204,
+      "logps/chosen": -1.4568153619766235,
+      "logps/rejected": -1.6396468877792358,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.4568153619766235,
+      "rewards/margins": 0.18283167481422424,
+      "rewards/rejected": -1.6396468877792358,
       "step": 440
     },
     {
       "epoch": 0.23816691754473993,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3331163757303437,
       "learning_rate": 7.932263814616755e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15622654557228088,
+      "logits/rejected": -0.053152941167354584,
+      "logps/chosen": -1.492536187171936,
+      "logps/rejected": -1.7137813568115234,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.492536187171936,
+      "rewards/margins": 0.22124521434307098,
+      "rewards/rejected": -1.7137813568115234,
       "step": 445
     },
     {
       "epoch": 0.2408429503261415,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6087963958837954,
       "learning_rate": 8.02139037433155e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.07289566099643707,
+      "logits/rejected": 0.04717857763171196,
+      "logps/chosen": -1.5369737148284912,
+      "logps/rejected": -1.7029969692230225,
+      "loss": 0.6938,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5369737148284912,
+      "rewards/margins": 0.1660231649875641,
+      "rewards/rejected": -1.7029969692230225,
       "step": 450
     },
     {
       "epoch": 0.24351898310754308,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4499671267274943,
       "learning_rate": 8.110516934046346e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.07699526846408844,
+      "logits/rejected": 0.004707801155745983,
+      "logps/chosen": -1.5031605958938599,
+      "logps/rejected": -1.7715861797332764,
+      "loss": 0.671,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.5031605958938599,
+      "rewards/margins": 0.26842576265335083,
+      "rewards/rejected": -1.7715861797332764,
       "step": 455
     },
     {
       "epoch": 0.24619501588894463,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5957667665435737,
       "learning_rate": 8.19964349376114e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23035934567451477,
+      "logits/rejected": -0.1179380863904953,
+      "logps/chosen": -1.6268441677093506,
+      "logps/rejected": -1.7373063564300537,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.6268441677093506,
+      "rewards/margins": 0.11046233028173447,
+      "rewards/rejected": -1.7373063564300537,
       "step": 460
     },
     {
       "epoch": 0.2488710486703462,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.204935554486172,
       "learning_rate": 8.288770053475936e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.047541871666908264,
+      "logits/rejected": 0.06139270216226578,
+      "logps/chosen": -1.5868889093399048,
+      "logps/rejected": -1.7831588983535767,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.5868889093399048,
+      "rewards/margins": 0.19627000391483307,
+      "rewards/rejected": -1.7831588983535767,
       "step": 465
     },
     {
       "epoch": 0.2515470814517478,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6547895322645783,
       "learning_rate": 8.37789661319073e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.040872037410736084,
+      "logits/rejected": 0.0001850567787187174,
+      "logps/chosen": -1.5029776096343994,
+      "logps/rejected": -1.6946609020233154,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5029776096343994,
+      "rewards/margins": 0.19168336689472198,
+      "rewards/rejected": -1.6946609020233154,
       "step": 470
     },
     {
       "epoch": 0.25422311423314936,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6475098717364483,
       "learning_rate": 8.467023172905525e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.16523709893226624,
+      "logits/rejected": -0.038014061748981476,
+      "logps/chosen": -1.476473331451416,
+      "logps/rejected": -1.8386907577514648,
+      "loss": 0.668,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.476473331451416,
+      "rewards/margins": 0.362217515707016,
+      "rewards/rejected": -1.8386907577514648,
       "step": 475
     },
     {
       "epoch": 0.2568991470145509,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.444354618927683,
       "learning_rate": 8.55614973262032e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13515475392341614,
+      "logits/rejected": 0.053321726620197296,
+      "logps/chosen": -1.456409215927124,
+      "logps/rejected": -1.6048309803009033,
+      "loss": 0.7043,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.456409215927124,
+      "rewards/margins": 0.14842167496681213,
+      "rewards/rejected": -1.6048309803009033,
       "step": 480
     },
     {
       "epoch": 0.2595751797959525,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 16.97453179812442,
       "learning_rate": 8.645276292335115e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0826820656657219,
+      "logits/rejected": -0.043180257081985474,
+      "logps/chosen": -1.5883818864822388,
+      "logps/rejected": -1.7199512720108032,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5883818864822388,
+      "rewards/margins": 0.13156947493553162,
+      "rewards/rejected": -1.7199512720108032,
       "step": 485
     },
     {
       "epoch": 0.26225121257735406,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4692260594522466,
       "learning_rate": 8.734402852049911e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09728191792964935,
+      "logits/rejected": -0.0335582010447979,
+      "logps/chosen": -1.5551732778549194,
+      "logps/rejected": -1.6958357095718384,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.5551732778549194,
+      "rewards/margins": 0.14066243171691895,
+      "rewards/rejected": -1.6958357095718384,
       "step": 490
     },
     {
       "epoch": 0.26492724535875567,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.798898196075673,
       "learning_rate": 8.823529411764705e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.176707461476326,
+      "logits/rejected": -0.155235156416893,
+      "logps/chosen": -1.6456897258758545,
+      "logps/rejected": -1.7731891870498657,
+      "loss": 0.692,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.6456897258758545,
+      "rewards/margins": 0.12749946117401123,
+      "rewards/rejected": -1.7731891870498657,
       "step": 495
     },
     {
       "epoch": 0.2676032781401572,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3303167512888394,
       "learning_rate": 8.912655971479501e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1652212142944336,
+      "logits/rejected": -0.06968554854393005,
+      "logps/chosen": -1.6018190383911133,
+      "logps/rejected": -1.822584867477417,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.6018190383911133,
+      "rewards/margins": 0.2207658588886261,
+      "rewards/rejected": -1.822584867477417,
       "step": 500
     },
     {
       "epoch": 0.27027931092155877,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.558670276017014,
       "learning_rate": 9.001782531194295e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20792119204998016,
+      "logits/rejected": -0.07148849219083786,
+      "logps/chosen": -1.6898998022079468,
+      "logps/rejected": -1.7409954071044922,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.6898998022079468,
+      "rewards/margins": 0.05109572410583496,
+      "rewards/rejected": -1.7409954071044922,
       "step": 505
     },
     {
       "epoch": 0.2729553437029604,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9694431641717594,
       "learning_rate": 9.09090909090909e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.04535558074712753,
+      "logits/rejected": 0.013482170179486275,
+      "logps/chosen": -1.6403038501739502,
+      "logps/rejected": -1.8646135330200195,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.6403038501739502,
+      "rewards/margins": 0.22430963814258575,
+      "rewards/rejected": -1.8646135330200195,
       "step": 510
     },
     {
       "epoch": 0.2756313764843619,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6123356402854916,
       "learning_rate": 9.180035650623885e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.07683808356523514,
+      "logits/rejected": 0.015434248372912407,
+      "logps/chosen": -1.5535920858383179,
+      "logps/rejected": -1.731636643409729,
+      "loss": 0.6665,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.5535920858383179,
+      "rewards/margins": 0.17804455757141113,
+      "rewards/rejected": -1.731636643409729,
       "step": 515
     },
     {
       "epoch": 0.27830740926576353,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1072934351149026,
       "learning_rate": 9.26916221033868e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2109527587890625,
+      "logits/rejected": -0.08198980987071991,
+      "logps/chosen": -1.551720142364502,
+      "logps/rejected": -1.7458961009979248,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.551720142364502,
+      "rewards/margins": 0.19417604804039001,
+      "rewards/rejected": -1.7458961009979248,
       "step": 520
     },
     {
       "epoch": 0.2809834420471651,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.044610405719208,
       "learning_rate": 9.358288770053476e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.046631328761577606,
+      "logits/rejected": 0.01980605348944664,
+      "logps/chosen": -1.5132060050964355,
+      "logps/rejected": -1.812970519065857,
+      "loss": 0.6783,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.5132060050964355,
+      "rewards/margins": 0.2997646927833557,
+      "rewards/rejected": -1.812970519065857,
       "step": 525
     },
     {
       "epoch": 0.2836594748285666,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9357961791601648,
       "learning_rate": 9.44741532976827e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.05468446761369705,
+      "logits/rejected": 0.02262100577354431,
+      "logps/chosen": -1.4412858486175537,
+      "logps/rejected": -1.6414167881011963,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.4412858486175537,
+      "rewards/margins": 0.2001311033964157,
+      "rewards/rejected": -1.6414167881011963,
       "step": 530
     },
     {
       "epoch": 0.28633550760996823,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.774556836006282,
       "learning_rate": 9.536541889483066e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2134787142276764,
+      "logits/rejected": 0.03333660960197449,
+      "logps/chosen": -1.4606363773345947,
+      "logps/rejected": -1.6222416162490845,
+      "loss": 0.6733,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4606363773345947,
+      "rewards/margins": 0.1616051197052002,
+      "rewards/rejected": -1.6222416162490845,
       "step": 535
     },
     {
       "epoch": 0.2890115403913698,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.896879135505514,
       "learning_rate": 9.62566844919786e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.10530916601419449,
+      "logits/rejected": -0.03971962258219719,
+      "logps/chosen": -1.6086992025375366,
+      "logps/rejected": -1.7688732147216797,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.6086992025375366,
+      "rewards/margins": 0.16017408668994904,
+      "rewards/rejected": -1.7688732147216797,
       "step": 540
     },
     {
       "epoch": 0.2916875731727714,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.121131957132343,
       "learning_rate": 9.714795008912655e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21385395526885986,
+      "logits/rejected": -0.028972690925002098,
+      "logps/chosen": -1.49721360206604,
+      "logps/rejected": -1.7099815607070923,
+      "loss": 0.6657,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.49721360206604,
+      "rewards/margins": 0.21276798844337463,
+      "rewards/rejected": -1.7099815607070923,
       "step": 545
     },
     {
       "epoch": 0.29436360595417294,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.40109025226904,
       "learning_rate": 9.80392156862745e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0705844983458519,
+      "logits/rejected": -0.0048699527978897095,
+      "logps/chosen": -1.5063307285308838,
+      "logps/rejected": -1.708571195602417,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.5063307285308838,
+      "rewards/margins": 0.2022404670715332,
+      "rewards/rejected": -1.708571195602417,
       "step": 550
     },
     {
       "epoch": 0.2970396387355745,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.6849677283577265,
       "learning_rate": 9.893048128342244e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17945453524589539,
+      "logits/rejected": -0.06290026754140854,
+      "logps/chosen": -1.5804903507232666,
+      "logps/rejected": -1.7054977416992188,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.5804903507232666,
+      "rewards/margins": 0.12500743567943573,
+      "rewards/rejected": -1.7054977416992188,
       "step": 555
     },
     {
       "epoch": 0.2997156715169761,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.658049549262026,
       "learning_rate": 9.98217468805704e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.05600447207689285,
+      "logits/rejected": -0.0445651113986969,
+      "logps/chosen": -1.4208638668060303,
+      "logps/rejected": -1.6305341720581055,
+      "loss": 0.6636,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4208638668060303,
+      "rewards/margins": 0.2096702605485916,
+      "rewards/rejected": -1.6305341720581055,
       "step": 560
     },
     {
       "epoch": 0.30239170429837764,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8146972590213324,
       "learning_rate": 9.999984476788462e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.08100216090679169,
+      "logits/rejected": -0.03155604749917984,
+      "logps/chosen": -1.5233267545700073,
+      "logps/rejected": -1.7266219854354858,
+      "loss": 0.6956,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.5233267545700073,
+      "rewards/margins": 0.20329518616199493,
+      "rewards/rejected": -1.7266219854354858,
       "step": 565
     },
     {
       "epoch": 0.30506773707977924,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.467483101115334,
       "learning_rate": 9.999921413906797e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17826507985591888,
+      "logits/rejected": 0.017647134140133858,
+      "logps/chosen": -1.4769870042800903,
+      "logps/rejected": -1.6659257411956787,
+      "loss": 0.6795,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.4769870042800903,
+      "rewards/margins": 0.18893864750862122,
+      "rewards/rejected": -1.6659257411956787,
       "step": 570
     },
     {
       "epoch": 0.3077437698611808,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7668247973147397,
       "learning_rate": 9.999809841765644e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1355060636997223,
+      "logits/rejected": -0.07719534635543823,
+      "logps/chosen": -1.425758957862854,
+      "logps/rejected": -1.6124694347381592,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.425758957862854,
+      "rewards/margins": 0.1867104023694992,
+      "rewards/rejected": -1.6124694347381592,
       "step": 575
     },
     {
       "epoch": 0.3104198026425824,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3952391616358817,
       "learning_rate": 9.999649761447477e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.14339084923267365,
+      "logits/rejected": 0.006741710007190704,
+      "logps/chosen": -1.4567533731460571,
+      "logps/rejected": -1.6972824335098267,
+      "loss": 0.6744,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.4567533731460571,
+      "rewards/margins": 0.2405288964509964,
+      "rewards/rejected": -1.6972824335098267,
       "step": 580
     },
     {
       "epoch": 0.31309583542398395,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1698734322255335,
       "learning_rate": 9.999441174505398e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.18116280436515808,
+      "logits/rejected": -0.08167508989572525,
+      "logps/chosen": -1.6449463367462158,
+      "logps/rejected": -1.7567377090454102,
+      "loss": 0.699,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.6449463367462158,
+      "rewards/margins": 0.11179134994745255,
+      "rewards/rejected": -1.7567377090454102,
       "step": 585
     },
     {
       "epoch": 0.3157718682053855,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 16.883207575655465,
       "learning_rate": 9.999184082963116e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1395900547504425,
+      "logits/rejected": -0.01637064851820469,
+      "logps/chosen": -1.5722973346710205,
+      "logps/rejected": -1.6663665771484375,
+      "loss": 0.6942,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.5722973346710205,
+      "rewards/margins": 0.09406914561986923,
+      "rewards/rejected": -1.6663665771484375,
       "step": 590
     },
     {
       "epoch": 0.3184479009867871,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7669880787499452,
       "learning_rate": 9.998878489314937e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0933346226811409,
+      "logits/rejected": 0.029161576181650162,
+      "logps/chosen": -1.5123186111450195,
+      "logps/rejected": -1.7037715911865234,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.5123186111450195,
+      "rewards/margins": 0.19145306944847107,
+      "rewards/rejected": -1.7037715911865234,
       "step": 595
     },
     {
       "epoch": 0.32112393376818865,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.192414627744019,
       "learning_rate": 9.99852439652573e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17058098316192627,
+      "logits/rejected": -0.025467002764344215,
+      "logps/chosen": -1.536420464515686,
+      "logps/rejected": -1.6864429712295532,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.536420464515686,
+      "rewards/margins": 0.15002265572547913,
+      "rewards/rejected": -1.6864429712295532,
       "step": 600
     },
     {
       "epoch": 0.32379996654959026,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.58025006011481,
       "learning_rate": 9.998121808030904e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1666463017463684,
+      "logits/rejected": -0.08217499405145645,
+      "logps/chosen": -1.6666057109832764,
+      "logps/rejected": -1.8665059804916382,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6666057109832764,
+      "rewards/margins": 0.19990026950836182,
+      "rewards/rejected": -1.8665059804916382,
       "step": 605
     },
     {
       "epoch": 0.3264759993309918,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.297197914832356,
       "learning_rate": 9.997670727736379e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.04118679463863373,
+      "logits/rejected": 0.12790530920028687,
+      "logps/chosen": -1.6160228252410889,
+      "logps/rejected": -1.822951078414917,
+      "loss": 0.685,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.6160228252410889,
+      "rewards/margins": 0.20692801475524902,
+      "rewards/rejected": -1.822951078414917,
       "step": 610
     },
     {
       "epoch": 0.32915203211239336,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6580044773982507,
       "learning_rate": 9.99717116001853e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13626885414123535,
+      "logits/rejected": -0.03361672908067703,
+      "logps/chosen": -1.6063220500946045,
+      "logps/rejected": -1.8777974843978882,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.6063220500946045,
+      "rewards/margins": 0.27147549390792847,
+      "rewards/rejected": -1.8777974843978882,
       "step": 615
     },
     {
       "epoch": 0.33182806489379496,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0882143212085107,
       "learning_rate": 9.996623109724173e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.05810218304395676,
+      "logits/rejected": 0.015411163680255413,
+      "logps/chosen": -1.7232177257537842,
+      "logps/rejected": -1.8944199085235596,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.7232177257537842,
+      "rewards/margins": 0.1712021827697754,
+      "rewards/rejected": -1.8944199085235596,
       "step": 620
     },
     {
       "epoch": 0.3345040976751965,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3448447446826925,
       "learning_rate": 9.996026582170488e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.03570134937763214,
+      "logits/rejected": 0.08904433995485306,
+      "logps/chosen": -1.5831081867218018,
+      "logps/rejected": -1.8833885192871094,
+      "loss": 0.6659,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.5831081867218018,
+      "rewards/margins": 0.3002806305885315,
+      "rewards/rejected": -1.8833885192871094,
       "step": 625
     },
     {
       "epoch": 0.3371801304565981,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8769767730855682,
       "learning_rate": 9.995381583144996e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.11172594130039215,
+      "logits/rejected": 0.001776664750650525,
+      "logps/chosen": -1.628483533859253,
+      "logps/rejected": -1.9026548862457275,
+      "loss": 0.6737,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.628483533859253,
+      "rewards/margins": 0.27417120337486267,
+      "rewards/rejected": -1.9026548862457275,
       "step": 630
     },
     {
       "epoch": 0.33985616323799966,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6081869112068023,
       "learning_rate": 9.994688118905471e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.07560587674379349,
+      "logits/rejected": 0.17433953285217285,
+      "logps/chosen": -1.696703314781189,
+      "logps/rejected": -1.9176191091537476,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.696703314781189,
+      "rewards/margins": 0.22091583907604218,
+      "rewards/rejected": -1.9176191091537476,
       "step": 635
     },
     {
       "epoch": 0.3425321960194012,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.855069498771731,
       "learning_rate": 9.993946196179912e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.16614773869514465,
+      "logits/rejected": 0.03592498227953911,
+      "logps/chosen": -1.671987771987915,
+      "logps/rejected": -1.8881022930145264,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.671987771987915,
+      "rewards/margins": 0.2161145955324173,
+      "rewards/rejected": -1.8881022930145264,
       "step": 640
     },
     {
       "epoch": 0.3452082288008028,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.673098754984692,
       "learning_rate": 9.993155822166455e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15226007997989655,
+      "logits/rejected": -0.06680991500616074,
+      "logps/chosen": -1.5783536434173584,
+      "logps/rejected": -1.874826431274414,
+      "loss": 0.687,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.5783536434173584,
+      "rewards/margins": 0.2964726388454437,
+      "rewards/rejected": -1.874826431274414,
       "step": 645
     },
     {
       "epoch": 0.34788426158220437,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3116525254951634,
       "learning_rate": 9.992317004533313e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.08254249393939972,
+      "logits/rejected": 0.059993743896484375,
+      "logps/chosen": -1.7293264865875244,
+      "logps/rejected": -2.0323519706726074,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7293264865875244,
+      "rewards/margins": 0.30302566289901733,
+      "rewards/rejected": -2.0323519706726074,
       "step": 650
     },
     {
       "epoch": 0.350560294363606,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.537443925897934,
       "learning_rate": 9.991429751418696e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.03274206817150116,
+      "logits/rejected": -0.018868273124098778,
+      "logps/chosen": -1.6904783248901367,
+      "logps/rejected": -1.9651095867156982,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.6904783248901367,
+      "rewards/margins": 0.27463164925575256,
+      "rewards/rejected": -1.9651095867156982,
       "step": 655
     },
     {
       "epoch": 0.3532363271450075,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.882105990698898,
       "learning_rate": 9.99049407143074e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.06752294301986694,
+      "logits/rejected": 0.058677542954683304,
+      "logps/chosen": -1.6685256958007812,
+      "logps/rejected": -1.8351339101791382,
+      "loss": 0.6767,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.6685256958007812,
+      "rewards/margins": 0.1666082888841629,
+      "rewards/rejected": -1.8351339101791382,
       "step": 660
     },
     {
       "epoch": 0.35591235992640907,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.980155254517534,
       "learning_rate": 9.989509973647416e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.07777120172977448,
+      "logits/rejected": 0.06430487334728241,
+      "logps/chosen": -1.5754632949829102,
+      "logps/rejected": -1.8427619934082031,
+      "loss": 0.672,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5754632949829102,
+      "rewards/margins": 0.26729848980903625,
+      "rewards/rejected": -1.8427619934082031,
       "step": 665
     },
     {
       "epoch": 0.3585883927078107,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.415682230027615,
       "learning_rate": 9.988477467616445e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.08580729365348816,
+      "logits/rejected": 0.11740393936634064,
+      "logps/chosen": -1.5971882343292236,
+      "logps/rejected": -1.8057384490966797,
+      "loss": 0.6708,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.5971882343292236,
+      "rewards/margins": 0.20855006575584412,
+      "rewards/rejected": -1.8057384490966797,
       "step": 670
     },
     {
       "epoch": 0.3612644254892122,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.285021373445158,
       "learning_rate": 9.987396563355205e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.10953493416309357,
+      "logits/rejected": -0.03255758807063103,
+      "logps/chosen": -1.5874454975128174,
+      "logps/rejected": -1.9195743799209595,
+      "loss": 0.667,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5874454975128174,
+      "rewards/margins": 0.3321287930011749,
+      "rewards/rejected": -1.9195743799209595,
       "step": 675
     },
     {
       "epoch": 0.36394045827061383,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.199749204977624,
       "learning_rate": 9.986267271350631e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.005286151077598333,
+      "logits/rejected": 0.16616101562976837,
+      "logps/chosen": -1.6424648761749268,
+      "logps/rejected": -1.868404746055603,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.6424648761749268,
+      "rewards/margins": 0.22593989968299866,
+      "rewards/rejected": -1.868404746055603,
       "step": 680
     },
     {
       "epoch": 0.3666164910520154,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2082556381479916,
       "learning_rate": 9.985089602559123e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.01213379018008709,
+      "logits/rejected": 0.14912262558937073,
+      "logps/chosen": -1.6230571269989014,
+      "logps/rejected": -1.8262178897857666,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.6230571269989014,
+      "rewards/margins": 0.2031606137752533,
+      "rewards/rejected": -1.8262178897857666,
       "step": 685
     },
     {
       "epoch": 0.369292523833417,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.202302319345987,
       "learning_rate": 9.983863568406428e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0033202157355844975,
+      "logits/rejected": 0.033788081258535385,
+      "logps/chosen": -1.6168022155761719,
+      "logps/rejected": -1.8522506952285767,
+      "loss": 0.66,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.6168022155761719,
+      "rewards/margins": 0.23544839024543762,
+      "rewards/rejected": -1.8522506952285767,
       "step": 690
     },
     {
       "epoch": 0.37196855661481854,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.373425131704945,
       "learning_rate": 9.982589180787532e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0433073565363884,
+      "logits/rejected": 0.0469246432185173,
+      "logps/chosen": -1.5217663049697876,
+      "logps/rejected": -1.802795648574829,
+      "loss": 0.6573,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5217663049697876,
+      "rewards/margins": 0.2810293138027191,
+      "rewards/rejected": -1.802795648574829,
       "step": 695
     },
     {
       "epoch": 0.3746445893962201,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.122643761046947,
       "learning_rate": 9.981266452066553e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17252962291240692,
+      "logits/rejected": -0.03989897295832634,
+      "logps/chosen": -1.7075554132461548,
+      "logps/rejected": -1.9116283655166626,
+      "loss": 0.679,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.7075554132461548,
+      "rewards/margins": 0.20407266914844513,
+      "rewards/rejected": -1.9116283655166626,
       "step": 700
     },
     {
       "epoch": 0.3773206221776217,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1055530909578732,
       "learning_rate": 9.979895395076608e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1264806091785431,
+      "logits/rejected": 0.048874445259571075,
+      "logps/chosen": -1.670810341835022,
+      "logps/rejected": -2.0343475341796875,
+      "loss": 0.6684,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.670810341835022,
+      "rewards/margins": 0.3635370135307312,
+      "rewards/rejected": -2.0343475341796875,
       "step": 705
     },
     {
       "epoch": 0.37999665495902324,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.569240201371488,
       "learning_rate": 9.9784760231197e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.025698263198137283,
+      "logits/rejected": 0.07422598451375961,
+      "logps/chosen": -1.6389007568359375,
+      "logps/rejected": -1.9179233312606812,
+      "loss": 0.67,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6389007568359375,
+      "rewards/margins": 0.27902257442474365,
+      "rewards/rejected": -1.9179233312606812,
       "step": 710
     },
     {
       "epoch": 0.38267268774042484,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.647535363852585,
       "learning_rate": 9.97700834996658e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09092571586370468,
+      "logits/rejected": 0.07322002947330475,
+      "logps/chosen": -1.7608976364135742,
+      "logps/rejected": -1.9912211894989014,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7608976364135742,
+      "rewards/margins": 0.23032355308532715,
+      "rewards/rejected": -1.9912211894989014,
       "step": 715
     },
     {
       "epoch": 0.3853487205218264,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0791184677280823,
       "learning_rate": 9.97549238985662e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.022224286571145058,
+      "logits/rejected": 0.16713587939739227,
+      "logps/chosen": -1.802594542503357,
+      "logps/rejected": -2.0594277381896973,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.802594542503357,
+      "rewards/margins": 0.25683319568634033,
+      "rewards/rejected": -2.0594277381896973,
       "step": 720
     },
     {
       "epoch": 0.38802475330322794,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.2800816020745405,
       "learning_rate": 9.973928157497674e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12780146300792694,
+      "logits/rejected": 0.015031392686069012,
+      "logps/chosen": -1.5907856225967407,
+      "logps/rejected": -1.9894018173217773,
+      "loss": 0.6623,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5907856225967407,
+      "rewards/margins": 0.39861640334129333,
+      "rewards/rejected": -1.9894018173217773,
       "step": 725
     },
     {
       "epoch": 0.39070078608462955,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.901977269612578,
       "learning_rate": 9.972315668065927e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.18473657965660095,
+      "logits/rejected": -0.03216997906565666,
+      "logps/chosen": -1.726334571838379,
+      "logps/rejected": -1.970254898071289,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.726334571838379,
+      "rewards/margins": 0.24392008781433105,
+      "rewards/rejected": -1.970254898071289,
       "step": 730
     },
     {
       "epoch": 0.3933768188660311,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9811732346532116,
       "learning_rate": 9.97065493720576e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12168679386377335,
+      "logits/rejected": -0.020994920283555984,
+      "logps/chosen": -1.7253637313842773,
+      "logps/rejected": -1.956683874130249,
+      "loss": 0.6721,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7253637313842773,
+      "rewards/margins": 0.23132026195526123,
+      "rewards/rejected": -1.956683874130249,
       "step": 735
     },
     {
       "epoch": 0.3960528516474327,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.337350281140361,
       "learning_rate": 9.968945981029594e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1199377179145813,
+      "logits/rejected": 0.05634986609220505,
+      "logps/chosen": -1.7963206768035889,
+      "logps/rejected": -2.0022380352020264,
+      "loss": 0.6684,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.7963206768035889,
+      "rewards/margins": 0.20591731369495392,
+      "rewards/rejected": -2.0022380352020264,
       "step": 740
     },
     {
       "epoch": 0.39872888442883425,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8750843989821635,
       "learning_rate": 9.967188816117726e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.003614559769630432,
+      "logits/rejected": 0.07337919622659683,
+      "logps/chosen": -1.7780086994171143,
+      "logps/rejected": -2.1441969871520996,
+      "loss": 0.6712,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7780086994171143,
+      "rewards/margins": 0.3661887049674988,
+      "rewards/rejected": -2.1441969871520996,
       "step": 745
     },
     {
       "epoch": 0.4014049172102358,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.285779092644056,
       "learning_rate": 9.965383459518179e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.08700697124004364,
+      "logits/rejected": 0.07441326975822449,
+      "logps/chosen": -1.7184171676635742,
+      "logps/rejected": -2.0446624755859375,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7184171676635742,
+      "rewards/margins": 0.32624536752700806,
+      "rewards/rejected": -2.0446624755859375,
       "step": 750
     },
     {
       "epoch": 0.4040809499916374,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.538247246481265,
       "learning_rate": 9.963529928746533e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.02840186282992363,
+      "logits/rejected": 0.10040257126092911,
+      "logps/chosen": -1.7163078784942627,
+      "logps/rejected": -2.0019516944885254,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7163078784942627,
+      "rewards/margins": 0.28564396500587463,
+      "rewards/rejected": -2.0019516944885254,
       "step": 755
     },
     {
       "epoch": 0.40675698277303896,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7645034480746618,
       "learning_rate": 9.961628241785746e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13142284750938416,
+      "logits/rejected": -0.061367761343717575,
+      "logps/chosen": -1.736936330795288,
+      "logps/rejected": -2.0846924781799316,
+      "loss": 0.678,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.736936330795288,
+      "rewards/margins": 0.34775614738464355,
+      "rewards/rejected": -2.0846924781799316,
       "step": 760
     },
     {
       "epoch": 0.40943301555444056,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.95251391570192,
       "learning_rate": 9.959678417085998e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09793178737163544,
+      "logits/rejected": -0.007523351814597845,
+      "logps/chosen": -1.722394585609436,
+      "logps/rejected": -1.9656988382339478,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.722394585609436,
+      "rewards/margins": 0.24330420792102814,
+      "rewards/rejected": -1.9656988382339478,
       "step": 765
     },
     {
       "epoch": 0.4121090483358421,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.880154060422788,
       "learning_rate": 9.957680473564493e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.012364056892693043,
+      "logits/rejected": 0.13697725534439087,
+      "logps/chosen": -1.7034928798675537,
+      "logps/rejected": -2.198805332183838,
+      "loss": 0.6459,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.7034928798675537,
+      "rewards/margins": 0.49531251192092896,
+      "rewards/rejected": -2.198805332183838,
       "step": 770
     },
     {
       "epoch": 0.41478508111724366,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.15756728634906,
       "learning_rate": 9.95563443060529e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1336345374584198,
+      "logits/rejected": 0.03753276914358139,
+      "logps/chosen": -1.7057218551635742,
+      "logps/rejected": -2.017930030822754,
+      "loss": 0.677,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7057218551635742,
+      "rewards/margins": 0.31220826506614685,
+      "rewards/rejected": -2.017930030822754,
       "step": 775
     },
     {
       "epoch": 0.41746111389864526,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.858808018414662,
       "learning_rate": 9.95354030805911e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20768341422080994,
+      "logits/rejected": -0.06138274818658829,
+      "logps/chosen": -1.6338584423065186,
+      "logps/rejected": -1.955957055091858,
+      "loss": 0.672,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.6338584423065186,
+      "rewards/margins": 0.32209888100624084,
+      "rewards/rejected": -1.955957055091858,
       "step": 780
     },
     {
       "epoch": 0.4201371466800468,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.97666322951967,
       "learning_rate": 9.951398126243133e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.04690421372652054,
+      "logits/rejected": 0.08271575719118118,
+      "logps/chosen": -1.5999151468276978,
+      "logps/rejected": -1.9750950336456299,
+      "loss": 0.6542,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.5999151468276978,
+      "rewards/margins": 0.3751799762248993,
+      "rewards/rejected": -1.9750950336456299,
       "step": 785
     },
     {
       "epoch": 0.4228131794614484,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.58349830519941,
       "learning_rate": 9.94920790594082e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09822144359350204,
+      "logits/rejected": 0.031502459198236465,
+      "logps/chosen": -1.6053279638290405,
+      "logps/rejected": -1.9106804132461548,
+      "loss": 0.6715,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6053279638290405,
+      "rewards/margins": 0.3053523898124695,
+      "rewards/rejected": -1.9106804132461548,
       "step": 790
     },
     {
       "epoch": 0.42548921224284997,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1577588270402823,
       "learning_rate": 9.946969668401696e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12457843869924545,
+      "logits/rejected": 0.060071446001529694,
+      "logps/chosen": -1.5792407989501953,
+      "logps/rejected": -1.9661684036254883,
+      "loss": 0.6605,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.5792407989501953,
+      "rewards/margins": 0.3869275450706482,
+      "rewards/rejected": -1.9661684036254883,
       "step": 795
     },
     {
       "epoch": 0.4281652450242516,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.953867863137142,
       "learning_rate": 9.944683435341155e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0735955610871315,
+      "logits/rejected": 0.00900576263666153,
+      "logps/chosen": -1.5782428979873657,
+      "logps/rejected": -1.853635549545288,
+      "loss": 0.6757,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.5782428979873657,
+      "rewards/margins": 0.27539271116256714,
+      "rewards/rejected": -1.853635549545288,
       "step": 800
     },
     {
       "epoch": 0.4281652450242516,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.4429,
-      "eval_samples_per_second": 39.05,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.784,
+      "eval_logits/chosen": 0.2730039954185486,
+      "eval_logits/rejected": 0.36653444170951843,
+      "eval_logps/chosen": -1.634755253791809,
+      "eval_logps/rejected": -1.98146653175354,
+      "eval_loss": 0.6641331911087036,
+      "eval_rewards/accuracies": 0.6238872408866882,
+      "eval_rewards/chosen": -1.634755253791809,
+      "eval_rewards/margins": 0.34671124815940857,
+      "eval_rewards/rejected": -1.98146653175354,
+      "eval_runtime": 40.5931,
+      "eval_samples_per_second": 33.134,
+      "eval_steps_per_second": 8.302,
       "step": 800
     },
     {
       "epoch": 0.4308412778056531,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.3700620844155365,
       "learning_rate": 9.942349228940236e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1348133236169815,
+      "logits/rejected": 0.02547209896147251,
+      "logps/chosen": -1.628843903541565,
+      "logps/rejected": -2.048295259475708,
+      "loss": 0.6668,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.628843903541565,
+      "rewards/margins": 0.4194514751434326,
+      "rewards/rejected": -2.048295259475708,
       "step": 805
     },
     {
       "epoch": 0.43351731058705467,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0779121064766035,
       "learning_rate": 9.939967071845424e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0369248203933239,
+      "logits/rejected": 0.03833397105336189,
+      "logps/chosen": -1.6127468347549438,
+      "logps/rejected": -1.902665138244629,
+      "loss": 0.6628,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.6127468347549438,
+      "rewards/margins": 0.28991830348968506,
+      "rewards/rejected": -1.902665138244629,
       "step": 810
     },
     {
       "epoch": 0.4361933433684563,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.001430834669253,
       "learning_rate": 9.937536987168413e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.05105900019407272,
+      "logits/rejected": 0.06912391632795334,
+      "logps/chosen": -1.5644266605377197,
+      "logps/rejected": -2.031595230102539,
+      "loss": 0.6408,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5644266605377197,
+      "rewards/margins": 0.46716856956481934,
+      "rewards/rejected": -2.031595230102539,
       "step": 815
     },
     {
       "epoch": 0.4388693761498578,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9953704630859876,
       "learning_rate": 9.935058998485896e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.01672717183828354,
+      "logits/rejected": 0.03726360574364662,
+      "logps/chosen": -1.6919682025909424,
+      "logps/rejected": -2.004424810409546,
+      "loss": 0.678,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.6919682025909424,
+      "rewards/margins": 0.3124566972255707,
+      "rewards/rejected": -2.004424810409546,
       "step": 820
     },
     {
       "epoch": 0.44154540893125943,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.216243451088041,
       "learning_rate": 9.932533129839333e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.07403428852558136,
+      "logits/rejected": 0.04845621436834335,
+      "logps/chosen": -1.5828664302825928,
+      "logps/rejected": -1.9342164993286133,
+      "loss": 0.659,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5828664302825928,
+      "rewards/margins": 0.3513498902320862,
+      "rewards/rejected": -1.9342164993286133,
       "step": 825
     },
     {
       "epoch": 0.444221441712661,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2814155875708253,
       "learning_rate": 9.929959405734711e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.001561010256409645,
+      "logits/rejected": 0.1644328385591507,
+      "logps/chosen": -1.6538890600204468,
+      "logps/rejected": -1.9058916568756104,
+      "loss": 0.6722,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.6538890600204468,
+      "rewards/margins": 0.25200265645980835,
+      "rewards/rejected": -1.9058916568756104,
       "step": 830
     },
     {
       "epoch": 0.44689747449406253,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.340496516478616,
       "learning_rate": 9.927337851142314e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.037868354469537735,
+      "logits/rejected": 0.0829586535692215,
+      "logps/chosen": -1.5983383655548096,
+      "logps/rejected": -1.869556188583374,
+      "loss": 0.6685,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.5983383655548096,
+      "rewards/margins": 0.2712177634239197,
+      "rewards/rejected": -1.869556188583374,
       "step": 835
     },
     {
       "epoch": 0.44957350727546413,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.629414451079738,
       "learning_rate": 9.924668491496474e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.08104674518108368,
+      "logits/rejected": 0.07732544839382172,
+      "logps/chosen": -1.6486152410507202,
+      "logps/rejected": -1.9830068349838257,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.6486152410507202,
+      "rewards/margins": 0.33439162373542786,
+      "rewards/rejected": -1.9830068349838257,
       "step": 840
     },
     {
       "epoch": 0.4522495400568657,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6849993756390726,
       "learning_rate": 9.92195135269533e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0002634204865898937,
+      "logits/rejected": 0.06444650888442993,
+      "logps/chosen": -1.6533575057983398,
+      "logps/rejected": -1.869270920753479,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.6533575057983398,
+      "rewards/margins": 0.21591326594352722,
+      "rewards/rejected": -1.869270920753479,
       "step": 845
     },
     {
       "epoch": 0.4549255728382673,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.387502136580706,
       "learning_rate": 9.919186461100574e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.03809467703104019,
+      "logits/rejected": 0.034446053206920624,
+      "logps/chosen": -1.6300861835479736,
+      "logps/rejected": -1.9234521389007568,
+      "loss": 0.6655,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.6300861835479736,
+      "rewards/margins": 0.29336604475975037,
+      "rewards/rejected": -1.9234521389007568,
       "step": 850
     },
     {
       "epoch": 0.45760160561966884,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8551394180936454,
       "learning_rate": 9.9163738435372e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.10677790641784668,
+      "logits/rejected": 0.022732479497790337,
+      "logps/chosen": -1.6749985218048096,
+      "logps/rejected": -2.033318042755127,
+      "loss": 0.695,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.6749985218048096,
+      "rewards/margins": 0.35831961035728455,
+      "rewards/rejected": -2.033318042755127,
       "step": 855
     },
     {
       "epoch": 0.4602776384010704,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.443578628839806,
       "learning_rate": 9.913513527293234e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15035206079483032,
+      "logits/rejected": 0.002962495433166623,
+      "logps/chosen": -1.7286573648452759,
+      "logps/rejected": -2.15730357170105,
+      "loss": 0.6658,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7286573648452759,
+      "rewards/margins": 0.4286460280418396,
+      "rewards/rejected": -2.15730357170105,
       "step": 860
     },
     {
       "epoch": 0.462953671182472,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.492922943890423,
       "learning_rate": 9.910605540119474e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.049700915813446045,
+      "logits/rejected": 0.042247481644153595,
+      "logps/chosen": -1.6477056741714478,
+      "logps/rejected": -2.1029722690582275,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.6477056741714478,
+      "rewards/margins": 0.455266535282135,
+      "rewards/rejected": -2.1029722690582275,
       "step": 865
     },
     {
       "epoch": 0.46562970396387354,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.53328424800888,
       "learning_rate": 9.907649910229227e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.159202441573143,
+      "logits/rejected": 0.08217189460992813,
+      "logps/chosen": -1.6673681735992432,
+      "logps/rejected": -2.0735726356506348,
+      "loss": 0.6565,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.6673681735992432,
+      "rewards/margins": 0.4062044620513916,
+      "rewards/rejected": -2.0735726356506348,
       "step": 870
     },
     {
       "epoch": 0.46830573674527515,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5317055911373934,
       "learning_rate": 9.90464666629803e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.06338021904230118,
+      "logits/rejected": 0.006542509887367487,
+      "logps/chosen": -1.7688989639282227,
+      "logps/rejected": -2.063070297241211,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.7688989639282227,
+      "rewards/margins": 0.29417139291763306,
+      "rewards/rejected": -2.063070297241211,
       "step": 875
     },
     {
       "epoch": 0.4709817695266767,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.960109770817398,
       "learning_rate": 9.901595837463363e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.061289966106414795,
+      "logits/rejected": 0.10721304267644882,
+      "logps/chosen": -1.7844291925430298,
+      "logps/rejected": -2.1540517807006836,
+      "loss": 0.6653,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7844291925430298,
+      "rewards/margins": 0.3696226477622986,
+      "rewards/rejected": -2.1540517807006836,
       "step": 880
     },
     {
       "epoch": 0.47365780230807825,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.260740117512628,
       "learning_rate": 9.898497453324384e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13106122612953186,
+      "logits/rejected": -0.05035706236958504,
+      "logps/chosen": -1.7084850072860718,
+      "logps/rejected": -2.129013776779175,
+      "loss": 0.6563,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7084850072860718,
+      "rewards/margins": 0.42052897810935974,
+      "rewards/rejected": -2.129013776779175,
       "step": 885
     },
     {
       "epoch": 0.47633383508947985,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.09349239423229,
       "learning_rate": 9.895351543941628e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2018505334854126,
+      "logits/rejected": -0.07470308989286423,
+      "logps/chosen": -1.7016435861587524,
+      "logps/rejected": -2.028141736984253,
+      "loss": 0.6576,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7016435861587524,
+      "rewards/margins": 0.32649821043014526,
+      "rewards/rejected": -2.028141736984253,
       "step": 890
     },
     {
       "epoch": 0.4790098678708814,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.1405042078373295,
       "learning_rate": 9.892158139836724e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.029820015653967857,
+      "logits/rejected": 0.07523909211158752,
+      "logps/chosen": -1.605481505393982,
+      "logps/rejected": -1.8919223546981812,
+      "loss": 0.6664,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.605481505393982,
+      "rewards/margins": 0.28644081950187683,
+      "rewards/rejected": -1.8919223546981812,
       "step": 895
     },
     {
       "epoch": 0.481685900652283,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.919868577419085,
       "learning_rate": 9.88891727199209e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13492530584335327,
+      "logits/rejected": -0.06405442208051682,
+      "logps/chosen": -1.5982228517532349,
+      "logps/rejected": -2.0329458713531494,
+      "loss": 0.6683,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5982228517532349,
+      "rewards/margins": 0.4347231388092041,
+      "rewards/rejected": -2.0329458713531494,
       "step": 900
     },
     {
       "epoch": 0.48436193343368455,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9893130075973144,
       "learning_rate": 9.885628971850641e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.03922805190086365,
+      "logits/rejected": 0.13861069083213806,
+      "logps/chosen": -1.6975027322769165,
+      "logps/rejected": -2.1410207748413086,
+      "loss": 0.6659,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.6975027322769165,
+      "rewards/margins": 0.44351816177368164,
+      "rewards/rejected": -2.1410207748413086,
       "step": 905
     },
     {
       "epoch": 0.48703796621508616,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7963707273336378,
       "learning_rate": 9.882293271315481e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.049951545894145966,
+      "logits/rejected": 0.04248204082250595,
+      "logps/chosen": -1.7056198120117188,
+      "logps/rejected": -2.0142719745635986,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.7056198120117188,
+      "rewards/margins": 0.30865222215652466,
+      "rewards/rejected": -2.0142719745635986,
       "step": 910
     },
     {
       "epoch": 0.4897139989964877,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2220449292822724,
       "learning_rate": 9.878910202749589e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.044047705829143524,
+      "logits/rejected": 0.1387040913105011,
+      "logps/chosen": -1.6319265365600586,
+      "logps/rejected": -2.0186896324157715,
+      "loss": 0.6631,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6319265365600586,
+      "rewards/margins": 0.3867631256580353,
+      "rewards/rejected": -2.0186896324157715,
       "step": 915
     },
     {
       "epoch": 0.49239003177788926,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.462452795813236,
       "learning_rate": 9.875479798975512e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.05937979742884636,
+      "logits/rejected": 0.19597770273685455,
+      "logps/chosen": -1.5628483295440674,
+      "logps/rejected": -1.983176827430725,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5628483295440674,
+      "rewards/margins": 0.4203285574913025,
+      "rewards/rejected": -1.983176827430725,
       "step": 920
     },
     {
       "epoch": 0.49506606455929086,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2574143676536385,
       "learning_rate": 9.87200209327504e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.07709117233753204,
+      "logits/rejected": 0.074994757771492,
+      "logps/chosen": -1.7194395065307617,
+      "logps/rejected": -2.016273260116577,
+      "loss": 0.6646,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.7194395065307617,
+      "rewards/margins": 0.29683375358581543,
+      "rewards/rejected": -2.016273260116577,
       "step": 925
     },
     {
       "epoch": 0.4977420973406924,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.42452897742449,
       "learning_rate": 9.868477119388894e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.10830855369567871,
+      "logits/rejected": 0.0028678863309323788,
+      "logps/chosen": -1.6048414707183838,
+      "logps/rejected": -2.0778844356536865,
+      "loss": 0.6722,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.6048414707183838,
+      "rewards/margins": 0.4730430245399475,
+      "rewards/rejected": -2.0778844356536865,
       "step": 930
     },
     {
       "epoch": 0.500418130122094,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.387372467475317,
       "learning_rate": 9.864904911516383e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -3.785267472267151e-05,
+      "logits/rejected": 0.04062112420797348,
+      "logps/chosen": -1.6054496765136719,
+      "logps/rejected": -1.957782506942749,
+      "loss": 0.6629,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.6054496765136719,
+      "rewards/margins": 0.3523326814174652,
+      "rewards/rejected": -1.957782506942749,
       "step": 935
     },
     {
       "epoch": 0.5030941629034956,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.18157640641936,
       "learning_rate": 9.861285504315084e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0510580949485302,
+      "logits/rejected": 0.046931274235248566,
+      "logps/chosen": -1.6503483057022095,
+      "logps/rejected": -1.9631057977676392,
+      "loss": 0.6596,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6503483057022095,
+      "rewards/margins": 0.3127575218677521,
+      "rewards/rejected": -1.9631057977676392,
       "step": 940
     },
     {
       "epoch": 0.5057701956848971,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7679236425916933,
       "learning_rate": 9.857618932900502e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.08321430534124374,
+      "logits/rejected": 0.039458055049180984,
+      "logps/chosen": -1.6388019323349,
+      "logps/rejected": -2.075631856918335,
+      "loss": 0.6526,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6388019323349,
+      "rewards/margins": 0.4368300437927246,
+      "rewards/rejected": -2.075631856918335,
       "step": 945
     },
     {
       "epoch": 0.5084462284662987,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8699511149048744,
       "learning_rate": 9.853905232845727e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.08022579550743103,
+      "logits/rejected": 0.07565019279718399,
+      "logps/chosen": -1.7417291402816772,
+      "logps/rejected": -1.995989203453064,
+      "loss": 0.679,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.7417291402816772,
+      "rewards/margins": 0.25426021218299866,
+      "rewards/rejected": -1.995989203453064,
       "step": 950
     },
     {
       "epoch": 0.5111222612477003,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7608898091492513,
       "learning_rate": 9.850144440181095e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.05249304696917534,
+      "logits/rejected": 0.16120192408561707,
+      "logps/chosen": -1.7640339136123657,
+      "logps/rejected": -2.0768134593963623,
+      "loss": 0.6763,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.7640339136123657,
+      "rewards/margins": 0.3127795159816742,
+      "rewards/rejected": -2.0768134593963623,
       "step": 955
     },
     {
       "epoch": 0.5137982940291018,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0214115083058624,
       "learning_rate": 9.846336591393832e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09741229563951492,
+      "logits/rejected": 0.03611370548605919,
+      "logps/chosen": -1.7557131052017212,
+      "logps/rejected": -2.04835844039917,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.7557131052017212,
+      "rewards/margins": 0.29264554381370544,
+      "rewards/rejected": -2.04835844039917,
       "step": 960
     },
     {
       "epoch": 0.5164743268105034,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8540054227346134,
       "learning_rate": 9.842481723427704e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.020888954401016235,
+      "logits/rejected": -0.01967158541083336,
+      "logps/chosen": -1.838861107826233,
+      "logps/rejected": -2.2295351028442383,
+      "loss": 0.6678,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.838861107826233,
+      "rewards/margins": 0.39067405462265015,
+      "rewards/rejected": -2.2295351028442383,
       "step": 965
     },
     {
       "epoch": 0.519150359591905,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.1106180744151155,
       "learning_rate": 9.838579873682658e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.019740425050258636,
+      "logits/rejected": -0.01291103195399046,
+      "logps/chosen": -1.682967185974121,
+      "logps/rejected": -2.0083885192871094,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.682967185974121,
+      "rewards/margins": 0.3254212737083435,
+      "rewards/rejected": -2.0083885192871094,
       "step": 970
     },
     {
       "epoch": 0.5218263923733065,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8390541541916978,
       "learning_rate": 9.834631080014457e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1357208490371704,
+      "logits/rejected": 0.039034873247146606,
+      "logps/chosen": -1.7357584238052368,
+      "logps/rejected": -2.0989975929260254,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7357584238052368,
+      "rewards/margins": 0.36323946714401245,
+      "rewards/rejected": -2.0989975929260254,
       "step": 975
     },
     {
       "epoch": 0.5245024251547081,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 11.141032180658433,
       "learning_rate": 9.830635380734312e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.14133158326148987,
+      "logits/rejected": 0.04091829061508179,
+      "logps/chosen": -1.82406485080719,
+      "logps/rejected": -2.1590054035186768,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.82406485080719,
+      "rewards/margins": 0.334940642118454,
+      "rewards/rejected": -2.1590054035186768,
       "step": 980
     },
     {
       "epoch": 0.5271784579361097,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.077557252045336,
       "learning_rate": 9.826592814608517e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0542990043759346,
+      "logits/rejected": 0.12407088279724121,
+      "logps/chosen": -1.7773380279541016,
+      "logps/rejected": -2.110229015350342,
+      "loss": 0.6667,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7773380279541016,
+      "rewards/margins": 0.3328908383846283,
+      "rewards/rejected": -2.110229015350342,
       "step": 985
     },
     {
       "epoch": 0.5298544907175113,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.80000685499654,
       "learning_rate": 9.822503420858067e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.008070261217653751,
+      "logits/rejected": 0.04234262928366661,
+      "logps/chosen": -1.5474274158477783,
+      "logps/rejected": -1.9867852926254272,
+      "loss": 0.6542,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5474274158477783,
+      "rewards/margins": 0.43935805559158325,
+      "rewards/rejected": -1.9867852926254272,
       "step": 990
     },
     {
       "epoch": 0.5325305234989128,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8871184571425457,
       "learning_rate": 9.818367239158277e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.008003910072147846,
+      "logits/rejected": 0.07962622493505478,
+      "logps/chosen": -1.668299674987793,
+      "logps/rejected": -1.9414989948272705,
+      "loss": 0.6861,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.668299674987793,
+      "rewards/margins": 0.27319931983947754,
+      "rewards/rejected": -1.9414989948272705,
       "step": 995
     },
     {
       "epoch": 0.5352065562803144,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3516108017173676,
       "learning_rate": 9.8141843096384e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.0025300756096839905,
+      "logits/rejected": 0.11559418588876724,
+      "logps/chosen": -1.720831274986267,
+      "logps/rejected": -2.140369176864624,
+      "loss": 0.6526,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.720831274986267,
+      "rewards/margins": 0.4195381999015808,
+      "rewards/rejected": -2.140369176864624,
       "step": 1000
     },
     {
       "epoch": 0.537882589061716,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 12.306085008270562,
       "learning_rate": 9.809954672881237e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.009598026983439922,
+      "logits/rejected": 0.13951411843299866,
+      "logps/chosen": -1.802060842514038,
+      "logps/rejected": -2.161684036254883,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.802060842514038,
+      "rewards/margins": 0.3596231937408447,
+      "rewards/rejected": -2.161684036254883,
       "step": 1005
     },
     {
       "epoch": 0.5405586218431175,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8802506398064536,
       "learning_rate": 9.80567836992274e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.025261417031288147,
+      "logits/rejected": 0.1534184068441391,
+      "logps/chosen": -1.6146175861358643,
+      "logps/rejected": -2.123509407043457,
+      "loss": 0.6524,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6146175861358643,
+      "rewards/margins": 0.5088918805122375,
+      "rewards/rejected": -2.123509407043457,
       "step": 1010
     },
     {
       "epoch": 0.5432346546245191,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.503605806851633,
       "learning_rate": 9.801355442251625e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.048169076442718506,
+      "logits/rejected": 0.11361920833587646,
+      "logps/chosen": -1.652345061302185,
+      "logps/rejected": -2.0673012733459473,
+      "loss": 0.6523,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.652345061302185,
+      "rewards/margins": 0.414956271648407,
+      "rewards/rejected": -2.0673012733459473,
       "step": 1015
     },
     {
       "epoch": 0.5459106874059207,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8920624466714324,
       "learning_rate": 9.796985931808949e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.005441828165203333,
+      "logits/rejected": 0.12581264972686768,
+      "logps/chosen": -1.703714370727539,
+      "logps/rejected": -2.1398041248321533,
+      "loss": 0.6557,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.703714370727539,
+      "rewards/margins": 0.43608999252319336,
+      "rewards/rejected": -2.1398041248321533,
       "step": 1020
     },
     {
       "epoch": 0.5485867201873222,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7216479779328018,
       "learning_rate": 9.792569880987724e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.094251349568367,
+      "logits/rejected": 0.024543244391679764,
+      "logps/chosen": -1.6764507293701172,
+      "logps/rejected": -2.2026238441467285,
+      "loss": 0.6486,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6764507293701172,
+      "rewards/margins": 0.5261729955673218,
+      "rewards/rejected": -2.2026238441467285,
       "step": 1025
     },
     {
       "epoch": 0.5512627529687238,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3909593706914705,
       "learning_rate": 9.788107332632493e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.024389835074543953,
+      "logits/rejected": 0.060034800320863724,
+      "logps/chosen": -1.7570539712905884,
+      "logps/rejected": -2.1015496253967285,
+      "loss": 0.668,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.7570539712905884,
+      "rewards/margins": 0.3444955050945282,
+      "rewards/rejected": -2.1015496253967285,
       "step": 1030
     },
     {
       "epoch": 0.5539387857501255,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.389248411944838,
       "learning_rate": 9.783598330038924e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.06158335134387016,
+      "logits/rejected": 0.0465625524520874,
+      "logps/chosen": -1.8137286901474,
+      "logps/rejected": -2.08414888381958,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.8137286901474,
+      "rewards/margins": 0.27042001485824585,
+      "rewards/rejected": -2.08414888381958,
       "step": 1035
     },
     {
       "epoch": 0.5566148185315271,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8852175571423089,
       "learning_rate": 9.779042916953376e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.019505204632878304,
+      "logits/rejected": 0.134805828332901,
+      "logps/chosen": -1.6921045780181885,
+      "logps/rejected": -2.1497609615325928,
+      "loss": 0.6577,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6921045780181885,
+      "rewards/margins": 0.45765653252601624,
+      "rewards/rejected": -2.1497609615325928,
       "step": 1040
     },
     {
       "epoch": 0.5592908513129285,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.711379668853581,
       "learning_rate": 9.774441137572487e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.10618245601654053,
+      "logits/rejected": 0.03390948846936226,
+      "logps/chosen": -1.6680119037628174,
+      "logps/rejected": -2.168739080429077,
+      "loss": 0.6735,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.6680119037628174,
+      "rewards/margins": 0.5007274746894836,
+      "rewards/rejected": -2.168739080429077,
       "step": 1045
     },
     {
       "epoch": 0.5619668840943302,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 14.311227259240903,
       "learning_rate": 9.76979303654274e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.08170074224472046,
+      "logits/rejected": 0.020473670214414597,
+      "logps/chosen": -1.7786178588867188,
+      "logps/rejected": -2.2663474082946777,
+      "loss": 0.6697,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.7786178588867188,
+      "rewards/margins": 0.487729549407959,
+      "rewards/rejected": -2.2663474082946777,
       "step": 1050
     },
     {
       "epoch": 0.5646429168757318,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 11.04246327784919,
       "learning_rate": 9.765098658960035e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.02644316479563713,
+      "logits/rejected": 0.043627046048641205,
+      "logps/chosen": -1.7438608407974243,
+      "logps/rejected": -2.206631660461426,
+      "loss": 0.6576,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7438608407974243,
+      "rewards/margins": 0.4627709984779358,
+      "rewards/rejected": -2.206631660461426,
       "step": 1055
     },
     {
       "epoch": 0.5673189496571333,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1916862227937344,
       "learning_rate": 9.76035805036924e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.02136054076254368,
+      "logits/rejected": 0.1972132921218872,
+      "logps/chosen": -1.854236364364624,
+      "logps/rejected": -2.2289860248565674,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.854236364364624,
+      "rewards/margins": 0.37474963068962097,
+      "rewards/rejected": -2.2289860248565674,
       "step": 1060
     },
     {
       "epoch": 0.5699949824385349,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8683103639223546,
       "learning_rate": 9.755571256763764e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": 0.014114683493971825,
+      "logits/rejected": 0.1376493275165558,
+      "logps/chosen": -1.717812180519104,
+      "logps/rejected": -2.250951051712036,
+      "loss": 0.6437,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.717812180519104,
+      "rewards/margins": 0.5331388711929321,
+      "rewards/rejected": -2.250951051712036,
       "step": 1065
     },
     {
       "epoch": 0.5726710152199365,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.377918520560742,
       "learning_rate": 9.750738324585097e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.11561836302280426,
+      "logits/rejected": 0.11747980117797852,
+      "logps/chosen": -1.7580969333648682,
+      "logps/rejected": -2.2344307899475098,
+      "loss": 0.6567,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7580969333648682,
+      "rewards/margins": 0.4763340353965759,
+      "rewards/rejected": -2.2344307899475098,
       "step": 1070
     },
     {
       "epoch": 0.5753470480013381,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2746652264331044,
       "learning_rate": 9.74585930072237e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.03118690848350525,
+      "logits/rejected": 0.09015442430973053,
+      "logps/chosen": -1.7069345712661743,
+      "logps/rejected": -2.2165799140930176,
+      "loss": 0.659,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7069345712661743,
+      "rewards/margins": 0.5096453428268433,
+      "rewards/rejected": -2.2165799140930176,
       "step": 1075
     },
     {
       "epoch": 0.5780230807827396,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8733217880073845,
       "learning_rate": 9.740934232511892e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09890493750572205,
+      "logits/rejected": 0.010432164184749126,
+      "logps/chosen": -1.8495237827301025,
+      "logps/rejected": -2.199219226837158,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.8495237827301025,
+      "rewards/margins": 0.3496953845024109,
+      "rewards/rejected": -2.199219226837158,
       "step": 1080
     },
     {
       "epoch": 0.5806991135641412,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.331743438077445,
       "learning_rate": 9.735963167736698e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.05797378346323967,
+      "logits/rejected": 0.10634851455688477,
+      "logps/chosen": -1.8227405548095703,
+      "logps/rejected": -2.0645289421081543,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.8227405548095703,
+      "rewards/margins": 0.24178798496723175,
+      "rewards/rejected": -2.0645289421081543,
       "step": 1085
     },
     {
       "epoch": 0.5833751463455428,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6010831517529542,
       "learning_rate": 9.730946154626078e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0640147253870964,
+      "logits/rejected": 0.03476481884717941,
+      "logps/chosen": -1.7731695175170898,
+      "logps/rejected": -2.0229616165161133,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.7731695175170898,
+      "rewards/margins": 0.24979212880134583,
+      "rewards/rejected": -2.0229616165161133,
       "step": 1090
     },
     {
       "epoch": 0.5860511791269443,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.445150284187816,
       "learning_rate": 9.725883241855117e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.18426083028316498,
+      "logits/rejected": -0.04870029166340828,
+      "logps/chosen": -1.762274980545044,
+      "logps/rejected": -2.2347359657287598,
+      "loss": 0.6725,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.762274980545044,
+      "rewards/margins": 0.47246068716049194,
+      "rewards/rejected": -2.2347359657287598,
       "step": 1095
     },
     {
       "epoch": 0.5887272119083459,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.221072726600048,
       "learning_rate": 9.720774478544218e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.07620300352573395,
+      "logits/rejected": 0.03012581542134285,
+      "logps/chosen": -1.7578318119049072,
+      "logps/rejected": -2.3410239219665527,
+      "loss": 0.6543,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7578318119049072,
+      "rewards/margins": 0.5831922888755798,
+      "rewards/rejected": -2.3410239219665527,
       "step": 1100
     },
     {
       "epoch": 0.5914032446897475,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1893924394091604,
       "learning_rate": 9.715619914258624e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1333165168762207,
+      "logits/rejected": -0.051289886236190796,
+      "logps/chosen": -1.8099966049194336,
+      "logps/rejected": -2.1711981296539307,
+      "loss": 0.6704,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.8099966049194336,
+      "rewards/margins": 0.3612014353275299,
+      "rewards/rejected": -2.1711981296539307,
       "step": 1105
     },
     {
       "epoch": 0.594079277471149,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.306455969287518,
       "learning_rate": 9.710419599007937e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12721973657608032,
+      "logits/rejected": -0.015675177797675133,
+      "logps/chosen": -1.8089733123779297,
+      "logps/rejected": -2.118504047393799,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.8089733123779297,
+      "rewards/margins": 0.30953049659729004,
+      "rewards/rejected": -2.118504047393799,
       "step": 1110
     },
     {
       "epoch": 0.5967553102525506,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 12.748790422506966,
       "learning_rate": 9.705173583245643e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.016397854313254356,
+      "logits/rejected": 0.10759837925434113,
+      "logps/chosen": -1.6831222772598267,
+      "logps/rejected": -2.233370304107666,
+      "loss": 0.6634,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6831222772598267,
+      "rewards/margins": 0.5502480268478394,
+      "rewards/rejected": -2.233370304107666,
       "step": 1115
     },
     {
       "epoch": 0.5994313430339522,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3005528277791463,
       "learning_rate": 9.699881917868609e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2024873048067093,
+      "logits/rejected": -0.10017645359039307,
+      "logps/chosen": -1.6752970218658447,
+      "logps/rejected": -2.1278557777404785,
+      "loss": 0.6501,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.6752970218658447,
+      "rewards/margins": 0.452558696269989,
+      "rewards/rejected": -2.1278557777404785,
       "step": 1120
     },
     {
       "epoch": 0.6021073758153538,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.292969465206174,
       "learning_rate": 9.694544654216594e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1948508322238922,
+      "logits/rejected": -0.014919877052307129,
+      "logps/chosen": -1.793273687362671,
+      "logps/rejected": -2.3364644050598145,
+      "loss": 0.6538,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.793273687362671,
+      "rewards/margins": 0.5431903600692749,
+      "rewards/rejected": -2.3364644050598145,
       "step": 1125
     },
     {
       "epoch": 0.6047834085967553,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3753760107612982,
       "learning_rate": 9.689161844071755e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0226006917655468,
+      "logits/rejected": 0.04176792502403259,
+      "logps/chosen": -1.7340033054351807,
+      "logps/rejected": -2.132333517074585,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7340033054351807,
+      "rewards/margins": 0.3983303904533386,
+      "rewards/rejected": -2.132333517074585,
       "step": 1130
     },
     {
       "epoch": 0.6074594413781569,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.032099950540524,
       "learning_rate": 9.683733539658138e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0938732847571373,
+      "logits/rejected": 0.054357677698135376,
+      "logps/chosen": -1.7786476612091064,
+      "logps/rejected": -2.2297985553741455,
+      "loss": 0.6603,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7786476612091064,
+      "rewards/margins": 0.45115095376968384,
+      "rewards/rejected": -2.2297985553741455,
       "step": 1135
     },
     {
       "epoch": 0.6101354741595585,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.449823109605845,
       "learning_rate": 9.678259793641178e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.10175579786300659,
+      "logits/rejected": -0.06175720691680908,
+      "logps/chosen": -1.760642647743225,
+      "logps/rejected": -1.9984248876571655,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.760642647743225,
+      "rewards/margins": 0.23778240382671356,
+      "rewards/rejected": -1.9984248876571655,
       "step": 1140
     },
     {
       "epoch": 0.61281150694096,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.401217884234884,
       "learning_rate": 9.672740659127183e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24253353476524353,
+      "logits/rejected": -0.1331673115491867,
+      "logps/chosen": -1.7018253803253174,
+      "logps/rejected": -2.1770553588867188,
+      "loss": 0.668,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7018253803253174,
+      "rewards/margins": 0.4752301573753357,
+      "rewards/rejected": -2.1770553588867188,
       "step": 1145
     },
     {
       "epoch": 0.6154875397223616,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6859677796291717,
       "learning_rate": 9.667176189662818e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2138625681400299,
+      "logits/rejected": -0.0839182585477829,
+      "logps/chosen": -1.6342815160751343,
+      "logps/rejected": -2.1070826053619385,
+      "loss": 0.6581,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6342815160751343,
+      "rewards/margins": 0.4728010594844818,
+      "rewards/rejected": -2.1070826053619385,
       "step": 1150
     },
     {
       "epoch": 0.6181635725037632,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2890582235307257,
       "learning_rate": 9.661566439234592e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.11411279439926147,
+      "logits/rejected": -0.024660751223564148,
+      "logps/chosen": -1.7312877178192139,
+      "logps/rejected": -2.0400102138519287,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.7312877178192139,
+      "rewards/margins": 0.30872276425361633,
+      "rewards/rejected": -2.0400102138519287,
       "step": 1155
     },
     {
       "epoch": 0.6208396052851648,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.091529631871589,
       "learning_rate": 9.655911462268327e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0317564532160759,
+      "logits/rejected": 0.06099940463900566,
+      "logps/chosen": -1.655419945716858,
+      "logps/rejected": -2.151423931121826,
+      "loss": 0.6511,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.655419945716858,
+      "rewards/margins": 0.4960041642189026,
+      "rewards/rejected": -2.151423931121826,
       "step": 1160
     },
     {
       "epoch": 0.6235156380665663,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.627058565197511,
       "learning_rate": 9.650211313628636e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.10680337250232697,
+      "logits/rejected": -0.03673669695854187,
+      "logps/chosen": -1.657050371170044,
+      "logps/rejected": -2.0230600833892822,
+      "loss": 0.6574,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.657050371170044,
+      "rewards/margins": 0.3660094738006592,
+      "rewards/rejected": -2.0230600833892822,
       "step": 1165
     },
     {
       "epoch": 0.6261916708479679,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7795952575608585,
       "learning_rate": 9.644466048618386e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12370695918798447,
+      "logits/rejected": 0.017783131450414658,
+      "logps/chosen": -1.9217584133148193,
+      "logps/rejected": -2.285383701324463,
+      "loss": 0.6741,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.9217584133148193,
+      "rewards/margins": 0.36362525820732117,
+      "rewards/rejected": -2.285383701324463,
       "step": 1170
     },
     {
       "epoch": 0.6288677036293695,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.487656765023967,
       "learning_rate": 9.63867572297816e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12050715833902359,
+      "logits/rejected": 0.05226340889930725,
+      "logps/chosen": -1.6750705242156982,
+      "logps/rejected": -2.086409330368042,
+      "loss": 0.6592,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.6750705242156982,
+      "rewards/margins": 0.4113388657569885,
+      "rewards/rejected": -2.086409330368042,
       "step": 1175
     },
     {
       "epoch": 0.631543736410771,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9668735430342283,
       "learning_rate": 9.632840392885727e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.11214729398488998,
+      "logits/rejected": 0.022331157699227333,
+      "logps/chosen": -1.7930923700332642,
+      "logps/rejected": -2.1929445266723633,
+      "loss": 0.6648,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.7930923700332642,
+      "rewards/margins": 0.39985212683677673,
+      "rewards/rejected": -2.1929445266723633,
       "step": 1180
     },
     {
       "epoch": 0.6342197691921726,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3164612021678814,
       "learning_rate": 9.626960114955483e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.04952782392501831,
+      "logits/rejected": 0.08291064202785492,
+      "logps/chosen": -1.7197091579437256,
+      "logps/rejected": -2.2075252532958984,
+      "loss": 0.666,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7197091579437256,
+      "rewards/margins": 0.4878160059452057,
+      "rewards/rejected": -2.2075252532958984,
       "step": 1185
     },
     {
       "epoch": 0.6368958019735742,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.071565362423332,
       "learning_rate": 9.621034946237909e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15675176680088043,
+      "logits/rejected": -0.026541167870163918,
+      "logps/chosen": -1.7669098377227783,
+      "logps/rejected": -2.272287607192993,
+      "loss": 0.6481,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.7669098377227783,
+      "rewards/margins": 0.5053777098655701,
+      "rewards/rejected": -2.272287607192993,
       "step": 1190
     },
     {
       "epoch": 0.6395718347549757,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.468653500944063,
       "learning_rate": 9.615064944219021e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09730833768844604,
+      "logits/rejected": 0.007259604521095753,
+      "logps/chosen": -1.6379356384277344,
+      "logps/rejected": -2.1625778675079346,
+      "loss": 0.6525,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6379356384277344,
+      "rewards/margins": 0.5246421694755554,
+      "rewards/rejected": -2.1625778675079346,
       "step": 1195
     },
     {
       "epoch": 0.6422478675363773,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4749770740421173,
       "learning_rate": 9.609050166819803e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.14909805357456207,
+      "logits/rejected": -0.08091595768928528,
+      "logps/chosen": -1.7381891012191772,
+      "logps/rejected": -2.083422899246216,
+      "loss": 0.6569,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7381891012191772,
+      "rewards/margins": 0.3452337682247162,
+      "rewards/rejected": -2.083422899246216,
       "step": 1200
     },
     {
       "epoch": 0.6422478675363773,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.7921,
-      "eval_samples_per_second": 38.658,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.686,
+      "eval_logits/chosen": 0.17488756775856018,
+      "eval_logits/rejected": 0.26006409525871277,
+      "eval_logps/chosen": -1.705970287322998,
+      "eval_logps/rejected": -2.1643903255462646,
+      "eval_loss": 0.6602288484573364,
+      "eval_rewards/accuracies": 0.642433226108551,
+      "eval_rewards/chosen": -1.705970287322998,
+      "eval_rewards/margins": 0.4584200382232666,
+      "eval_rewards/rejected": -2.1643903255462646,
+      "eval_runtime": 40.6019,
+      "eval_samples_per_second": 33.127,
+      "eval_steps_per_second": 8.3,
       "step": 1200
     },
     {
       "epoch": 0.6449239003177789,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.916896033577594,
       "learning_rate": 9.602990672395653e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2532616853713989,
+      "logits/rejected": -0.09220051765441895,
+      "logps/chosen": -1.6368019580841064,
+      "logps/rejected": -2.082883834838867,
+      "loss": 0.6577,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6368019580841064,
+      "rewards/margins": 0.44608181715011597,
+      "rewards/rejected": -2.082883834838867,
       "step": 1205
     },
     {
       "epoch": 0.6475999330991805,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.017392242420325,
       "learning_rate": 9.59688651973581e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13524527847766876,
+      "logits/rejected": 0.03598679229617119,
+      "logps/chosen": -1.6347614526748657,
+      "logps/rejected": -2.040713310241699,
+      "loss": 0.6709,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.6347614526748657,
+      "rewards/margins": 0.40595197677612305,
+      "rewards/rejected": -2.040713310241699,
       "step": 1210
     },
     {
       "epoch": 0.650275965880582,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9184335427335126,
       "learning_rate": 9.590737768062792e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1865207999944687,
+      "logits/rejected": -0.07574017345905304,
+      "logps/chosen": -1.6992533206939697,
+      "logps/rejected": -2.023419141769409,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.6992533206939697,
+      "rewards/margins": 0.3241662085056305,
+      "rewards/rejected": -2.023419141769409,
       "step": 1215
     },
     {
       "epoch": 0.6529519986619836,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.889784855166525,
       "learning_rate": 9.584544477031816e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.016246292740106583,
+      "logits/rejected": 0.08368152379989624,
+      "logps/chosen": -1.6252052783966064,
+      "logps/rejected": -1.9583289623260498,
+      "loss": 0.6738,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.6252052783966064,
+      "rewards/margins": 0.33312374353408813,
+      "rewards/rejected": -1.9583289623260498,
       "step": 1220
     },
     {
       "epoch": 0.6556280314433852,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.426279933958593,
       "learning_rate": 9.578306706730215e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24190382659435272,
+      "logits/rejected": -0.039002932608127594,
+      "logps/chosen": -1.7298275232315063,
+      "logps/rejected": -2.094015598297119,
+      "loss": 0.6744,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.7298275232315063,
+      "rewards/margins": 0.3641875982284546,
+      "rewards/rejected": -2.094015598297119,
       "step": 1225
     },
     {
       "epoch": 0.6583040642247867,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.666481756651252,
       "learning_rate": 9.572024517676865e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12973980605602264,
+      "logits/rejected": -0.036396563053131104,
+      "logps/chosen": -1.733823537826538,
+      "logps/rejected": -2.0810062885284424,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.733823537826538,
+      "rewards/margins": 0.3471828103065491,
+      "rewards/rejected": -2.0810062885284424,
       "step": 1230
     },
     {
       "epoch": 0.6609800970061883,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6893696770600373,
       "learning_rate": 9.565697970821593e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09815509617328644,
+      "logits/rejected": 0.025978904217481613,
+      "logps/chosen": -1.800812005996704,
+      "logps/rejected": -2.161504030227661,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.800812005996704,
+      "rewards/margins": 0.3606918752193451,
+      "rewards/rejected": -2.161504030227661,
       "step": 1235
     },
     {
       "epoch": 0.6636561297875899,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9489568915557183,
       "learning_rate": 9.559327127544585e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24666114151477814,
+      "logits/rejected": -0.11542533338069916,
+      "logps/chosen": -1.9023488759994507,
+      "logps/rejected": -2.3293795585632324,
+      "loss": 0.6662,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.9023488759994507,
+      "rewards/margins": 0.4270308017730713,
+      "rewards/rejected": -2.3293795585632324,
       "step": 1240
     },
     {
       "epoch": 0.6663321625689914,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3988155492099814,
       "learning_rate": 9.552912049655789e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15687593817710876,
+      "logits/rejected": 0.021413544192910194,
+      "logps/chosen": -1.8875694274902344,
+      "logps/rejected": -2.237082004547119,
+      "loss": 0.675,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.8875694274902344,
+      "rewards/margins": 0.3495125472545624,
+      "rewards/rejected": -2.237082004547119,
       "step": 1245
     },
     {
       "epoch": 0.669008195350393,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.549124955524258,
       "learning_rate": 9.546452799394315e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.14900559186935425,
+      "logits/rejected": 0.05005853250622749,
+      "logps/chosen": -1.9593194723129272,
+      "logps/rejected": -2.337949275970459,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.9593194723129272,
+      "rewards/margins": 0.3786296844482422,
+      "rewards/rejected": -2.337949275970459,
       "step": 1250
     },
     {
       "epoch": 0.6716842281317946,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.254623439579811,
       "learning_rate": 9.539949439427846e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17451117932796478,
+      "logits/rejected": -0.054794441908597946,
+      "logps/chosen": -1.8193778991699219,
+      "logps/rejected": -2.243708848953247,
+      "loss": 0.6692,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.8193778991699219,
+      "rewards/margins": 0.42433086037635803,
+      "rewards/rejected": -2.243708848953247,
       "step": 1255
     },
     {
       "epoch": 0.6743602609131962,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5844359519923557,
       "learning_rate": 9.533402032852002e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1947668194770813,
+      "logits/rejected": -0.06924370676279068,
+      "logps/chosen": -1.7528793811798096,
+      "logps/rejected": -2.316730499267578,
+      "loss": 0.6724,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7528793811798096,
+      "rewards/margins": 0.5638511776924133,
+      "rewards/rejected": -2.316730499267578,
       "step": 1260
     },
     {
       "epoch": 0.6770362936945977,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0056927506122486,
       "learning_rate": 9.526810643189754e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.11063408851623535,
+      "logits/rejected": 0.018481066450476646,
+      "logps/chosen": -1.7270561456680298,
+      "logps/rejected": -2.2389233112335205,
+      "loss": 0.6634,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7270561456680298,
+      "rewards/margins": 0.5118670463562012,
+      "rewards/rejected": -2.2389233112335205,
       "step": 1265
     },
     {
       "epoch": 0.6797123264759993,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.482738498093495,
       "learning_rate": 9.52017533439079e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17356248199939728,
+      "logits/rejected": -0.07853259146213531,
+      "logps/chosen": -1.719102144241333,
+      "logps/rejected": -2.198641300201416,
+      "loss": 0.6573,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.719102144241333,
+      "rewards/margins": 0.47953909635543823,
+      "rewards/rejected": -2.198641300201416,
       "step": 1270
     },
     {
       "epoch": 0.6823883592574009,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.782664247877279,
       "learning_rate": 9.513496170830909e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1619473099708557,
+      "logits/rejected": -0.055688511580228806,
+      "logps/chosen": -1.7614761590957642,
+      "logps/rejected": -2.1841397285461426,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7614761590957642,
+      "rewards/margins": 0.4226636290550232,
+      "rewards/rejected": -2.1841397285461426,
       "step": 1275
     },
     {
       "epoch": 0.6850643920388024,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.246881021369294,
       "learning_rate": 9.506773217311382e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17021796107292175,
+      "logits/rejected": -0.027279064059257507,
+      "logps/chosen": -1.7847516536712646,
+      "logps/rejected": -2.1254241466522217,
+      "loss": 0.6753,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7847516536712646,
+      "rewards/margins": 0.34067273139953613,
+      "rewards/rejected": -2.1254241466522217,
       "step": 1280
     },
     {
       "epoch": 0.687740424820204,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.056201482521696,
       "learning_rate": 9.500006539058334e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12388406693935394,
+      "logits/rejected": -0.00015998184971977025,
+      "logps/chosen": -1.6517744064331055,
+      "logps/rejected": -1.9869533777236938,
+      "loss": 0.6545,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.6517744064331055,
+      "rewards/margins": 0.3351789116859436,
+      "rewards/rejected": -1.9869533777236938,
       "step": 1285
     },
     {
       "epoch": 0.6904164576016056,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4097775436329707,
       "learning_rate": 9.493196201722109e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24897980690002441,
+      "logits/rejected": -0.10843881219625473,
+      "logps/chosen": -1.7364314794540405,
+      "logps/rejected": -2.017374038696289,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.7364314794540405,
+      "rewards/margins": 0.2809421122074127,
+      "rewards/rejected": -2.017374038696289,
       "step": 1290
     },
     {
       "epoch": 0.6930924903830072,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9907636947406053,
       "learning_rate": 9.486342271376628e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15651632845401764,
+      "logits/rejected": -0.14402654767036438,
+      "logps/chosen": -1.7462513446807861,
+      "logps/rejected": -2.2301764488220215,
+      "loss": 0.6552,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7462513446807861,
+      "rewards/margins": 0.4839250445365906,
+      "rewards/rejected": -2.2301764488220215,
       "step": 1295
     },
     {
       "epoch": 0.6957685231644087,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.77199928635151,
       "learning_rate": 9.479444814518755e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1784646064043045,
+      "logits/rejected": 0.04614611342549324,
+      "logps/chosen": -1.65802001953125,
+      "logps/rejected": -2.1850409507751465,
+      "loss": 0.6585,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.65802001953125,
+      "rewards/margins": 0.5270209908485413,
+      "rewards/rejected": -2.1850409507751465,
       "step": 1300
     },
     {
       "epoch": 0.6984445559458103,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2991881421718525,
       "learning_rate": 9.472503898067645e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.06048533320426941,
+      "logits/rejected": -0.014150974340736866,
+      "logps/chosen": -1.7293155193328857,
+      "logps/rejected": -2.081012487411499,
+      "loss": 0.6662,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7293155193328857,
+      "rewards/margins": 0.35169678926467896,
+      "rewards/rejected": -2.081012487411499,
       "step": 1305
     },
     {
       "epoch": 0.701120588727212,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.121033881181233,
       "learning_rate": 9.465519589364099e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0621965117752552,
+      "logits/rejected": 0.01737886294722557,
+      "logps/chosen": -1.6356929540634155,
+      "logps/rejected": -2.052541971206665,
+      "loss": 0.6647,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6356929540634155,
+      "rewards/margins": 0.4168490767478943,
+      "rewards/rejected": -2.052541971206665,
       "step": 1310
     },
     {
       "epoch": 0.7037966215086134,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.025114236737202,
       "learning_rate": 9.458491956169914e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12872114777565002,
+      "logits/rejected": 0.03127479553222656,
+      "logps/chosen": -1.6047554016113281,
+      "logps/rejected": -2.1051621437072754,
+      "loss": 0.6412,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6047554016113281,
+      "rewards/margins": 0.500406801700592,
+      "rewards/rejected": -2.1051621437072754,
       "step": 1315
     },
     {
       "epoch": 0.706472654290015,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3063799247645327,
       "learning_rate": 9.451421066667215e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2314561903476715,
+      "logits/rejected": -0.053547583520412445,
+      "logps/chosen": -1.6337976455688477,
+      "logps/rejected": -2.0494258403778076,
+      "loss": 0.6591,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.6337976455688477,
+      "rewards/margins": 0.41562825441360474,
+      "rewards/rejected": -2.0494258403778076,
       "step": 1320
     },
     {
       "epoch": 0.7091486870714167,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 12.65428942185779,
       "learning_rate": 9.444306989457805e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.10532049834728241,
+      "logits/rejected": -0.0100431302562356,
+      "logps/chosen": -1.6710853576660156,
+      "logps/rejected": -2.082911491394043,
+      "loss": 0.6765,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.6710853576660156,
+      "rewards/margins": 0.41182613372802734,
+      "rewards/rejected": -2.082911491394043,
       "step": 1325
     },
     {
       "epoch": 0.7118247198528181,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3663016840086994,
       "learning_rate": 9.437149793562489e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13778424263000488,
+      "logits/rejected": -0.0301259346306324,
+      "logps/chosen": -1.7002109289169312,
+      "logps/rejected": -2.0097248554229736,
+      "loss": 0.6634,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7002109289169312,
+      "rewards/margins": 0.30951395630836487,
+      "rewards/rejected": -2.0097248554229736,
       "step": 1330
     },
     {
       "epoch": 0.7145007526342197,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.887185543499853,
       "learning_rate": 9.429949548420417e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.10622577369213104,
+      "logits/rejected": -0.035293955355882645,
+      "logps/chosen": -1.7940342426300049,
+      "logps/rejected": -2.1592941284179688,
+      "loss": 0.6647,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7940342426300049,
+      "rewards/margins": 0.36525991559028625,
+      "rewards/rejected": -2.1592941284179688,
       "step": 1335
     },
     {
       "epoch": 0.7171767854156214,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1019016262857404,
       "learning_rate": 9.422706323888396e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1157793253660202,
+      "logits/rejected": -0.0802188441157341,
+      "logps/chosen": -1.7065680027008057,
+      "logps/rejected": -2.0638415813446045,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7065680027008057,
+      "rewards/margins": 0.357273668050766,
+      "rewards/rejected": -2.0638415813446045,
       "step": 1340
     },
     {
       "epoch": 0.719852818197023,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.842105082202058,
       "learning_rate": 9.415420190240225e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.06623774766921997,
+      "logits/rejected": 0.11135341972112656,
+      "logps/chosen": -1.6813198328018188,
+      "logps/rejected": -2.2106850147247314,
+      "loss": 0.6601,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.6813198328018188,
+      "rewards/margins": 0.5293651819229126,
+      "rewards/rejected": -2.2106850147247314,
       "step": 1345
     },
     {
       "epoch": 0.7225288509784245,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0109858294449983,
       "learning_rate": 9.408091218166002e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.06685715168714523,
+      "logits/rejected": -0.011635723523795605,
+      "logps/chosen": -1.697882056236267,
+      "logps/rejected": -1.9295867681503296,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.697882056236267,
+      "rewards/margins": 0.2317044734954834,
+      "rewards/rejected": -1.9295867681503296,
       "step": 1350
     },
     {
       "epoch": 0.7252048837598261,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.300332569159847,
       "learning_rate": 9.400719478771449e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.11403868347406387,
+      "logits/rejected": 0.1429174393415451,
+      "logps/chosen": -1.8290170431137085,
+      "logps/rejected": -2.236776828765869,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.8290170431137085,
+      "rewards/margins": 0.4077596664428711,
+      "rewards/rejected": -2.236776828765869,
       "step": 1355
     },
     {
       "epoch": 0.7278809165412277,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.79154959142278,
       "learning_rate": 9.393305043577209e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2181086540222168,
+      "logits/rejected": -0.08775530755519867,
+      "logps/chosen": -1.8719406127929688,
+      "logps/rejected": -2.3547422885894775,
+      "loss": 0.662,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.8719406127929688,
+      "rewards/margins": 0.4828016757965088,
+      "rewards/rejected": -2.3547422885894775,
       "step": 1360
     },
     {
       "epoch": 0.7305569493226292,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0063042969187435,
       "learning_rate": 9.38584798451817e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12358502298593521,
+      "logits/rejected": -0.001854392932727933,
+      "logps/chosen": -1.7356081008911133,
+      "logps/rejected": -2.143638849258423,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7356081008911133,
+      "rewards/margins": 0.40803074836730957,
+      "rewards/rejected": -2.143638849258423,
       "step": 1365
     },
     {
       "epoch": 0.7332329821040308,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9996821882110303,
       "learning_rate": 9.37834837394275e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12289956957101822,
+      "logits/rejected": -0.02455131709575653,
+      "logps/chosen": -1.8240686655044556,
+      "logps/rejected": -2.42940616607666,
+      "loss": 0.6525,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.8240686655044556,
+      "rewards/margins": 0.6053375601768494,
+      "rewards/rejected": -2.42940616607666,
       "step": 1370
     },
     {
       "epoch": 0.7359090148854324,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9043312886264583,
       "learning_rate": 9.370806284612203e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15965647995471954,
+      "logits/rejected": -0.025019938126206398,
+      "logps/chosen": -1.7329797744750977,
+      "logps/rejected": -2.3140969276428223,
+      "loss": 0.6489,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7329797744750977,
+      "rewards/margins": 0.5811169147491455,
+      "rewards/rejected": -2.3140969276428223,
       "step": 1375
     },
     {
       "epoch": 0.738585047666834,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.660211093350471,
       "learning_rate": 9.363221789699912e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19298836588859558,
+      "logits/rejected": -0.08077849447727203,
+      "logps/chosen": -1.7543102502822876,
+      "logps/rejected": -2.081346035003662,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.7543102502822876,
+      "rewards/margins": 0.3270358443260193,
+      "rewards/rejected": -2.081346035003662,
       "step": 1380
     },
     {
       "epoch": 0.7412610804482355,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.016073854888811,
       "learning_rate": 9.355594962790682e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19028612971305847,
+      "logits/rejected": -0.07924105226993561,
+      "logps/chosen": -1.6031452417373657,
+      "logps/rejected": -2.058067798614502,
+      "loss": 0.6779,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6031452417373657,
+      "rewards/margins": 0.4549228250980377,
+      "rewards/rejected": -2.058067798614502,
       "step": 1385
     },
     {
       "epoch": 0.7439371132296371,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.337801309136303,
       "learning_rate": 9.34792587788002e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12197472900152206,
+      "logits/rejected": -0.01977367140352726,
+      "logps/chosen": -1.7272751331329346,
+      "logps/rejected": -2.1413514614105225,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7272751331329346,
+      "rewards/margins": 0.41407614946365356,
+      "rewards/rejected": -2.1413514614105225,
       "step": 1390
     },
     {
       "epoch": 0.7466131460110387,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8847854318044535,
       "learning_rate": 9.34021460937342e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.07927948981523514,
+      "logits/rejected": 0.004415563307702541,
+      "logps/chosen": -1.6725761890411377,
+      "logps/rejected": -1.970023512840271,
+      "loss": 0.6713,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.6725761890411377,
+      "rewards/margins": 0.29744744300842285,
+      "rewards/rejected": -1.970023512840271,
       "step": 1395
     },
     {
       "epoch": 0.7492891787924402,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.341791341212044,
       "learning_rate": 9.332461232085646e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2938159108161926,
+      "logits/rejected": -0.17975321412086487,
+      "logps/chosen": -1.7938270568847656,
+      "logps/rejected": -2.1823954582214355,
+      "loss": 0.6674,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.7938270568847656,
+      "rewards/margins": 0.3885686993598938,
+      "rewards/rejected": -2.1823954582214355,
       "step": 1400
     },
     {
       "epoch": 0.7519652115738418,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.572494555499592,
       "learning_rate": 9.324665821239998e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1791909784078598,
+      "logits/rejected": -0.015470030717551708,
+      "logps/chosen": -1.5862165689468384,
+      "logps/rejected": -2.154858112335205,
+      "loss": 0.6646,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.5862165689468384,
+      "rewards/margins": 0.5686416625976562,
+      "rewards/rejected": -2.154858112335205,
       "step": 1405
     },
     {
       "epoch": 0.7546412443552434,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9670598377832444,
       "learning_rate": 9.316828452467583e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21215656399726868,
+      "logits/rejected": -0.05234779044985771,
+      "logps/chosen": -1.7728582620620728,
+      "logps/rejected": -2.2807040214538574,
+      "loss": 0.6562,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.7728582620620728,
+      "rewards/margins": 0.5078460574150085,
+      "rewards/rejected": -2.2807040214538574,
       "step": 1410
     },
     {
       "epoch": 0.7573172771366449,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.382371873581466,
       "learning_rate": 9.30894920180659e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13545528054237366,
+      "logits/rejected": -0.0052247922867536545,
+      "logps/chosen": -1.757328987121582,
+      "logps/rejected": -2.0225307941436768,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.757328987121582,
+      "rewards/margins": 0.2652018070220947,
+      "rewards/rejected": -2.0225307941436768,
       "step": 1415
     },
     {
       "epoch": 0.7599933099180465,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1218034161703456,
       "learning_rate": 9.301028145701543e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1445949524641037,
+      "logits/rejected": -0.03361133486032486,
+      "logps/chosen": -1.699582815170288,
+      "logps/rejected": -2.3167619705200195,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.699582815170288,
+      "rewards/margins": 0.6171795129776001,
+      "rewards/rejected": -2.3167619705200195,
       "step": 1420
     },
     {
       "epoch": 0.7626693426994481,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.447112484580126,
       "learning_rate": 9.293065361002563e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.10684005916118622,
+      "logits/rejected": -0.04240207001566887,
+      "logps/chosen": -1.7155735492706299,
+      "logps/rejected": -2.266064167022705,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7155735492706299,
+      "rewards/margins": 0.5504907369613647,
+      "rewards/rejected": -2.266064167022705,
       "step": 1425
     },
     {
       "epoch": 0.7653453754808497,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.364597013363824,
       "learning_rate": 9.285060924964622e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21582290530204773,
+      "logits/rejected": -0.09401645511388779,
+      "logps/chosen": -1.7733345031738281,
+      "logps/rejected": -2.1813764572143555,
+      "loss": 0.664,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.7733345031738281,
+      "rewards/margins": 0.4080421030521393,
+      "rewards/rejected": -2.1813764572143555,
       "step": 1430
     },
     {
       "epoch": 0.7680214082622512,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.98410973501143,
       "learning_rate": 9.277014915246792e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.11604003608226776,
+      "logits/rejected": -0.06317834556102753,
+      "logps/chosen": -1.652482032775879,
+      "logps/rejected": -2.2593393325805664,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.652482032775879,
+      "rewards/margins": 0.6068572402000427,
+      "rewards/rejected": -2.2593393325805664,
       "step": 1435
     },
     {
       "epoch": 0.7706974410436528,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.771024919379363,
       "learning_rate": 9.268927409911498e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19965513050556183,
+      "logits/rejected": -0.10314589738845825,
+      "logps/chosen": -1.6828935146331787,
+      "logps/rejected": -2.0405712127685547,
+      "loss": 0.6677,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.6828935146331787,
+      "rewards/margins": 0.3576778173446655,
+      "rewards/rejected": -2.0405712127685547,
       "step": 1440
     },
     {
       "epoch": 0.7733734738250544,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.042934098270068,
       "learning_rate": 9.260798487423749e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2385113686323166,
+      "logits/rejected": -0.057984281331300735,
+      "logps/chosen": -1.7373443841934204,
+      "logps/rejected": -2.082923412322998,
+      "loss": 0.6621,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7373443841934204,
+      "rewards/margins": 0.3455789089202881,
+      "rewards/rejected": -2.082923412322998,
       "step": 1445
     },
     {
       "epoch": 0.7760495066064559,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 12.101644983437081,
       "learning_rate": 9.252628226650389e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12450124323368073,
+      "logits/rejected": -0.04279579967260361,
+      "logps/chosen": -1.7761399745941162,
+      "logps/rejected": -2.0677731037139893,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7761399745941162,
+      "rewards/margins": 0.2916330397129059,
+      "rewards/rejected": -2.0677731037139893,
       "step": 1450
     },
     {
       "epoch": 0.7787255393878575,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8233699780653803,
       "learning_rate": 9.244416706859321e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.18640455603599548,
+      "logits/rejected": -0.05290110036730766,
+      "logps/chosen": -1.6450319290161133,
+      "logps/rejected": -2.1304328441619873,
+      "loss": 0.6616,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6450319290161133,
+      "rewards/margins": 0.48540106415748596,
+      "rewards/rejected": -2.1304328441619873,
       "step": 1455
     },
     {
       "epoch": 0.7814015721692591,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.846177703822563,
       "learning_rate": 9.23616400771875e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1559382677078247,
+      "logits/rejected": -0.022602161392569542,
+      "logps/chosen": -1.6522232294082642,
+      "logps/rejected": -2.2092044353485107,
+      "loss": 0.6497,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6522232294082642,
+      "rewards/margins": 0.5569813847541809,
+      "rewards/rejected": -2.2092044353485107,
       "step": 1460
     },
     {
       "epoch": 0.7840776049506607,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8062725192166929,
       "learning_rate": 9.227870209296395e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15481138229370117,
+      "logits/rejected": -0.06418724358081818,
+      "logps/chosen": -1.8068405389785767,
+      "logps/rejected": -2.172001600265503,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.8068405389785767,
+      "rewards/margins": 0.36516129970550537,
+      "rewards/rejected": -2.172001600265503,
       "step": 1465
     },
     {
       "epoch": 0.7867536377320622,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.710445882963896,
       "learning_rate": 9.219535392058728e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23232483863830566,
+      "logits/rejected": -0.19958427548408508,
+      "logps/chosen": -1.746352195739746,
+      "logps/rejected": -2.2105603218078613,
+      "loss": 0.6668,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.746352195739746,
+      "rewards/margins": 0.46420836448669434,
+      "rewards/rejected": -2.2105603218078613,
       "step": 1470
     },
     {
       "epoch": 0.7894296705134638,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7917505544191816,
       "learning_rate": 9.211159636870181e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22857098281383514,
+      "logits/rejected": -0.0698053389787674,
+      "logps/chosen": -1.7330665588378906,
+      "logps/rejected": -2.2428784370422363,
+      "loss": 0.6624,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7330665588378906,
+      "rewards/margins": 0.5098117589950562,
+      "rewards/rejected": -2.2428784370422363,
       "step": 1475
     },
     {
       "epoch": 0.7921057032948654,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.368093475696553,
       "learning_rate": 9.202743024992367e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12836043536663055,
+      "logits/rejected": -0.029153283685445786,
+      "logps/chosen": -1.6937240362167358,
+      "logps/rejected": -2.2347898483276367,
+      "loss": 0.6613,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.6937240362167358,
+      "rewards/margins": 0.5410657525062561,
+      "rewards/rejected": -2.2347898483276367,
       "step": 1480
     },
     {
       "epoch": 0.7947817360762669,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.881615396429785,
       "learning_rate": 9.194285638083293e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12081925570964813,
+      "logits/rejected": 0.03210361674427986,
+      "logps/chosen": -1.7830228805541992,
+      "logps/rejected": -2.3935394287109375,
+      "loss": 0.6503,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7830228805541992,
+      "rewards/margins": 0.610516369342804,
+      "rewards/rejected": -2.3935394287109375,
       "step": 1485
     },
     {
       "epoch": 0.7974577688576685,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.294513969833023,
       "learning_rate": 9.185787558196562e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2203497439622879,
+      "logits/rejected": -0.11334848403930664,
+      "logps/chosen": -1.743948221206665,
+      "logps/rejected": -2.175447940826416,
+      "loss": 0.6714,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.743948221206665,
+      "rewards/margins": 0.431499719619751,
+      "rewards/rejected": -2.175447940826416,
       "step": 1490
     },
     {
       "epoch": 0.8001338016390701,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.216289229117883,
       "learning_rate": 9.177248867780583e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.18983697891235352,
+      "logits/rejected": -0.08680440485477448,
+      "logps/chosen": -1.9608757495880127,
+      "logps/rejected": -2.247036933898926,
+      "loss": 0.6742,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.9608757495880127,
+      "rewards/margins": 0.2861611247062683,
+      "rewards/rejected": -2.247036933898926,
       "step": 1495
     },
     {
       "epoch": 0.8028098344204716,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.670837064988489,
       "learning_rate": 9.168669649677769e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19543516635894775,
+      "logits/rejected": -0.09639539569616318,
+      "logps/chosen": -1.828667402267456,
+      "logps/rejected": -2.2906384468078613,
+      "loss": 0.6679,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.828667402267456,
+      "rewards/margins": 0.4619710445404053,
+      "rewards/rejected": -2.2906384468078613,
       "step": 1500
     },
     {
       "epoch": 0.8054858672018732,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7415965860572045,
       "learning_rate": 9.16004998712373e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13242021203041077,
+      "logits/rejected": -0.07562603056430817,
+      "logps/chosen": -1.8647769689559937,
+      "logps/rejected": -2.434131145477295,
+      "loss": 0.6555,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8647769689559937,
+      "rewards/margins": 0.5693541169166565,
+      "rewards/rejected": -2.434131145477295,
       "step": 1505
     },
     {
       "epoch": 0.8081618999832748,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2313995964366566,
       "learning_rate": 9.151389963746472e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.193832129240036,
+      "logits/rejected": 0.06302468478679657,
+      "logps/chosen": -1.9284769296646118,
+      "logps/rejected": -2.597747802734375,
+      "loss": 0.6624,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.9284769296646118,
+      "rewards/margins": 0.6692708730697632,
+      "rewards/rejected": -2.597747802734375,
       "step": 1510
     },
     {
       "epoch": 0.8108379327646764,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.39671111169546,
       "learning_rate": 9.142689663565577e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12289537489414215,
+      "logits/rejected": -0.06232582777738571,
+      "logps/chosen": -1.804357886314392,
+      "logps/rejected": -2.357199192047119,
+      "loss": 0.6589,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.804357886314392,
+      "rewards/margins": 0.5528410077095032,
+      "rewards/rejected": -2.357199192047119,
       "step": 1515
     },
     {
       "epoch": 0.8135139655460779,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.326091278931021,
       "learning_rate": 9.133949170991397e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15848007798194885,
+      "logits/rejected": -0.0786110982298851,
+      "logps/chosen": -1.8076632022857666,
+      "logps/rejected": -2.2849860191345215,
+      "loss": 0.6608,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.8076632022857666,
+      "rewards/margins": 0.47732311487197876,
+      "rewards/rejected": -2.2849860191345215,
       "step": 1520
     },
     {
       "epoch": 0.8161899983274795,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3258388930653697,
       "learning_rate": 9.125168570824231e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1813269853591919,
+      "logits/rejected": -0.025487791746854782,
+      "logps/chosen": -1.7776210308074951,
+      "logps/rejected": -2.15429949760437,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7776210308074951,
+      "rewards/margins": 0.37667855620384216,
+      "rewards/rejected": -2.15429949760437,
       "step": 1525
     },
     {
       "epoch": 0.8188660311088811,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9440018585440715,
       "learning_rate": 9.116347948253496e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19304604828357697,
+      "logits/rejected": -0.09000559896230698,
+      "logps/chosen": -1.7671136856079102,
+      "logps/rejected": -2.147706985473633,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7671136856079102,
+      "rewards/margins": 0.3805930018424988,
+      "rewards/rejected": -2.147706985473633,
       "step": 1530
     },
     {
       "epoch": 0.8215420638902826,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.118041898189665,
       "learning_rate": 9.107487388856916e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21291561424732208,
+      "logits/rejected": -0.05352047085762024,
+      "logps/chosen": -1.6584066152572632,
+      "logps/rejected": -2.14587664604187,
+      "loss": 0.6494,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6584066152572632,
+      "rewards/margins": 0.48747020959854126,
+      "rewards/rejected": -2.14587664604187,
       "step": 1535
     },
     {
       "epoch": 0.8242180966716842,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2528575704637404,
       "learning_rate": 9.098586978599673e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15449278056621552,
+      "logits/rejected": -0.011315232142806053,
+      "logps/chosen": -1.6943954229354858,
+      "logps/rejected": -2.3861966133117676,
+      "loss": 0.6479,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6943954229354858,
+      "rewards/margins": 0.6918010115623474,
+      "rewards/rejected": -2.3861966133117676,
       "step": 1540
     },
     {
       "epoch": 0.8268941294530858,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.629434314441243,
       "learning_rate": 9.089646803833588e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.14946050941944122,
+      "logits/rejected": -0.00869622640311718,
+      "logps/chosen": -1.6699333190917969,
+      "logps/rejected": -2.0994648933410645,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6699333190917969,
+      "rewards/margins": 0.42953166365623474,
+      "rewards/rejected": -2.0994648933410645,
       "step": 1545
     },
     {
       "epoch": 0.8295701622344873,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.250691267041783,
       "learning_rate": 9.080666951296276e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2801513969898224,
+      "logits/rejected": -0.0465787835419178,
+      "logps/chosen": -1.7065759897232056,
+      "logps/rejected": -2.2669596672058105,
+      "loss": 0.6646,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7065759897232056,
+      "rewards/margins": 0.5603836178779602,
+      "rewards/rejected": -2.2669596672058105,
       "step": 1550
     },
     {
       "epoch": 0.8322461950158889,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.562284395752165,
       "learning_rate": 9.071647508110305e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23921728134155273,
+      "logits/rejected": -0.019418274983763695,
+      "logps/chosen": -1.7232980728149414,
+      "logps/rejected": -2.306126117706299,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7232980728149414,
+      "rewards/margins": 0.5828280448913574,
+      "rewards/rejected": -2.306126117706299,
       "step": 1555
     },
     {
       "epoch": 0.8349222277972905,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3050259951492222,
       "learning_rate": 9.062588561782354e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.110140860080719,
+      "logits/rejected": -0.041213177144527435,
+      "logps/chosen": -1.7901302576065063,
+      "logps/rejected": -2.1625008583068848,
+      "loss": 0.6524,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7901302576065063,
+      "rewards/margins": 0.37237095832824707,
+      "rewards/rejected": -2.1625008583068848,
       "step": 1560
     },
     {
       "epoch": 0.8375982605786921,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6386168340624407,
       "learning_rate": 9.053490200202358e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.11531031131744385,
+      "logits/rejected": -0.02741876244544983,
+      "logps/chosen": -1.762438416481018,
+      "logps/rejected": -2.131227731704712,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.762438416481018,
+      "rewards/margins": 0.3687894642353058,
+      "rewards/rejected": -2.131227731704712,
       "step": 1565
     },
     {
       "epoch": 0.8402742933600936,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.413829151484131,
       "learning_rate": 9.044352511642661e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09510122984647751,
+      "logits/rejected": -0.07887458801269531,
+      "logps/chosen": -1.7040588855743408,
+      "logps/rejected": -1.9877254962921143,
+      "loss": 0.6706,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7040588855743408,
+      "rewards/margins": 0.28366661071777344,
+      "rewards/rejected": -1.9877254962921143,
       "step": 1570
     },
     {
       "epoch": 0.8429503261414952,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.629263218698866,
       "learning_rate": 9.03517558475716e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12605051696300507,
+      "logits/rejected": -0.03476902097463608,
+      "logps/chosen": -1.6712207794189453,
+      "logps/rejected": -2.0244712829589844,
+      "loss": 0.6757,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.6712207794189453,
+      "rewards/margins": 0.35325053334236145,
+      "rewards/rejected": -2.0244712829589844,
       "step": 1575
     },
     {
       "epoch": 0.8456263589228968,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5917490619583,
       "learning_rate": 9.025959508580436e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.10047215223312378,
+      "logits/rejected": 0.12288226932287216,
+      "logps/chosen": -1.7938168048858643,
+      "logps/rejected": -2.256577968597412,
+      "loss": 0.6579,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7938168048858643,
+      "rewards/margins": 0.462761253118515,
+      "rewards/rejected": -2.256577968597412,
       "step": 1580
     },
     {
       "epoch": 0.8483023917042983,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4120035878832002,
       "learning_rate": 9.016704372526905e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15340359508991241,
+      "logits/rejected": -0.010615167208015919,
+      "logps/chosen": -1.6097633838653564,
+      "logps/rejected": -2.20293927192688,
+      "loss": 0.6503,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6097633838653564,
+      "rewards/margins": 0.5931758880615234,
+      "rewards/rejected": -2.20293927192688,
       "step": 1585
     },
     {
       "epoch": 0.8509784244856999,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.996266955619694,
       "learning_rate": 9.007410266389934e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21473315358161926,
+      "logits/rejected": -0.1343342512845993,
+      "logps/chosen": -1.6755809783935547,
+      "logps/rejected": -2.0811266899108887,
+      "loss": 0.6586,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6755809783935547,
+      "rewards/margins": 0.4055458605289459,
+      "rewards/rejected": -2.0811266899108887,
       "step": 1590
     },
     {
       "epoch": 0.8536544572671015,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.0822280911075675,
       "learning_rate": 8.998077280340981e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1391085684299469,
+      "logits/rejected": -0.061352647840976715,
+      "logps/chosen": -1.9422428607940674,
+      "logps/rejected": -2.2479662895202637,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.9422428607940674,
+      "rewards/margins": 0.30572310090065,
+      "rewards/rejected": -2.2479662895202637,
       "step": 1595
     },
     {
       "epoch": 0.8563304900485031,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.997065991905919,
       "learning_rate": 8.988705504928722e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24875064194202423,
+      "logits/rejected": -0.07590314000844955,
+      "logps/chosen": -1.8205833435058594,
+      "logps/rejected": -2.4661574363708496,
+      "loss": 0.6562,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.8205833435058594,
+      "rewards/margins": 0.6455739140510559,
+      "rewards/rejected": -2.4661574363708496,
       "step": 1600
     },
     {
       "epoch": 0.8563304900485031,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.4651,
-      "eval_samples_per_second": 39.025,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.778,
+      "eval_logits/chosen": 0.09436248242855072,
+      "eval_logits/rejected": 0.17959809303283691,
+      "eval_logps/chosen": -1.836761236190796,
+      "eval_logps/rejected": -2.383552074432373,
+      "eval_loss": 0.6583895683288574,
+      "eval_rewards/accuracies": 0.6513352990150452,
+      "eval_rewards/chosen": -1.836761236190796,
+      "eval_rewards/margins": 0.5467912554740906,
+      "eval_rewards/rejected": -2.383552074432373,
+      "eval_runtime": 40.4652,
+      "eval_samples_per_second": 33.238,
+      "eval_steps_per_second": 8.328,
       "step": 1600
     },
     {
       "epoch": 0.8590065228299046,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.159948092625599,
       "learning_rate": 8.979295031078157e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23401093482971191,
+      "logits/rejected": -0.033759430050849915,
+      "logps/chosen": -1.8151342868804932,
+      "logps/rejected": -2.403872489929199,
+      "loss": 0.6544,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.8151342868804932,
+      "rewards/margins": 0.588738203048706,
+      "rewards/rejected": -2.403872489929199,
       "step": 1605
     },
     {
       "epoch": 0.8616825556113062,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7415338707517383,
       "learning_rate": 8.969845950089751e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.252411812543869,
+      "logits/rejected": -0.10224980115890503,
+      "logps/chosen": -1.7924762964248657,
+      "logps/rejected": -2.5826337337493896,
+      "loss": 0.6603,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7924762964248657,
+      "rewards/margins": 0.7901574373245239,
+      "rewards/rejected": -2.5826337337493896,
       "step": 1610
     },
     {
       "epoch": 0.8643585883927078,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.433788704066424,
       "learning_rate": 8.960358353638526e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19758816063404083,
+      "logits/rejected": -0.10357751697301865,
+      "logps/chosen": -1.9330323934555054,
+      "logps/rejected": -2.5681416988372803,
+      "loss": 0.6558,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.9330323934555054,
+      "rewards/margins": 0.6351091265678406,
+      "rewards/rejected": -2.5681416988372803,
       "step": 1615
     },
     {
       "epoch": 0.8670346211741093,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.534435941171359,
       "learning_rate": 8.950832333773184e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.14965714514255524,
+      "logits/rejected": -0.012872450053691864,
+      "logps/chosen": -1.7339420318603516,
+      "logps/rejected": -2.2898683547973633,
+      "loss": 0.6579,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7339420318603516,
+      "rewards/margins": 0.5559263229370117,
+      "rewards/rejected": -2.2898683547973633,
       "step": 1620
     },
     {
       "epoch": 0.869710653955511,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.312433184579841,
       "learning_rate": 8.941267982915213e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09175966680049896,
+      "logits/rejected": -0.0425976924598217,
+      "logps/chosen": -1.940840721130371,
+      "logps/rejected": -2.190351963043213,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.940840721130371,
+      "rewards/margins": 0.24951119720935822,
+      "rewards/rejected": -2.190351963043213,
       "step": 1625
     },
     {
       "epoch": 0.8723866867369126,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.839868554783555,
       "learning_rate": 8.931665393857983e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13728447258472443,
+      "logits/rejected": -0.009815283119678497,
+      "logps/chosen": -1.78765869140625,
+      "logps/rejected": -2.3569040298461914,
+      "loss": 0.659,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.78765869140625,
+      "rewards/margins": 0.5692452788352966,
+      "rewards/rejected": -2.3569040298461914,
       "step": 1630
     },
     {
       "epoch": 0.875062719518314,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.400717544352508,
       "learning_rate": 8.922024659765861e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21612973511219025,
+      "logits/rejected": -0.10596023499965668,
+      "logps/chosen": -1.651301383972168,
+      "logps/rejected": -2.2570128440856934,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.651301383972168,
+      "rewards/margins": 0.6057115793228149,
+      "rewards/rejected": -2.2570128440856934,
       "step": 1635
     },
     {
       "epoch": 0.8777387522997157,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0040755600094675,
       "learning_rate": 8.912345874173288e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20477314293384552,
+      "logits/rejected": -0.10000059753656387,
+      "logps/chosen": -1.6596043109893799,
+      "logps/rejected": -2.3223023414611816,
+      "loss": 0.6531,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6596043109893799,
+      "rewards/margins": 0.662697970867157,
+      "rewards/rejected": -2.3223023414611816,
       "step": 1640
     },
     {
       "epoch": 0.8804147850811173,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.58947698766755,
       "learning_rate": 8.902629130983885e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.14497405290603638,
+      "logits/rejected": -0.09471423923969269,
+      "logps/chosen": -1.7040191888809204,
+      "logps/rejected": -2.0789732933044434,
+      "loss": 0.6709,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7040191888809204,
+      "rewards/margins": 0.3749541938304901,
+      "rewards/rejected": -2.0789732933044434,
       "step": 1645
     },
     {
       "epoch": 0.8830908178625189,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.055084237376809,
       "learning_rate": 8.892874524469537e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.08851578086614609,
+      "logits/rejected": -0.025071466341614723,
+      "logps/chosen": -1.6397254467010498,
+      "logps/rejected": -2.0935370922088623,
+      "loss": 0.6525,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.6397254467010498,
+      "rewards/margins": 0.4538115859031677,
+      "rewards/rejected": -2.0935370922088623,
       "step": 1650
     },
     {
       "epoch": 0.8857668506439204,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.671015570141012,
       "learning_rate": 8.883082149269478e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20095396041870117,
+      "logits/rejected": -0.09608559310436249,
+      "logps/chosen": -1.693265676498413,
+      "logps/rejected": -2.148573637008667,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.693265676498413,
+      "rewards/margins": 0.45530787110328674,
+      "rewards/rejected": -2.148573637008667,
       "step": 1655
     },
     {
       "epoch": 0.888442883425322,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.776456124066758,
       "learning_rate": 8.873252100389377e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12443655729293823,
+      "logits/rejected": -0.10364575684070587,
+      "logps/chosen": -1.5704820156097412,
+      "logps/rejected": -2.023597478866577,
+      "loss": 0.6616,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.5704820156097412,
+      "rewards/margins": 0.4531156122684479,
+      "rewards/rejected": -2.023597478866577,
       "step": 1660
     },
     {
       "epoch": 0.8911189162067236,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8872874976440377,
       "learning_rate": 8.863384473200411e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.14885106682777405,
+      "logits/rejected": -0.08296102285385132,
+      "logps/chosen": -1.7899055480957031,
+      "logps/rejected": -2.090120315551758,
+      "loss": 0.6736,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7899055480957031,
+      "rewards/margins": 0.30021506547927856,
+      "rewards/rejected": -2.090120315551758,
       "step": 1665
     },
     {
       "epoch": 0.8937949489881251,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2612076830775267,
       "learning_rate": 8.853479363438342e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1208019033074379,
+      "logits/rejected": 0.028531115502119064,
+      "logps/chosen": -1.8726457357406616,
+      "logps/rejected": -2.227069616317749,
+      "loss": 0.6962,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.8726457357406616,
+      "rewards/margins": 0.354423463344574,
+      "rewards/rejected": -2.227069616317749,
       "step": 1670
     },
     {
       "epoch": 0.8964709817695267,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7701987703569677,
       "learning_rate": 8.843536867202588e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.134661465883255,
+      "logits/rejected": 0.06566232442855835,
+      "logps/chosen": -1.8035697937011719,
+      "logps/rejected": -2.3893465995788574,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.8035697937011719,
+      "rewards/margins": 0.5857768058776855,
+      "rewards/rejected": -2.3893465995788574,
       "step": 1675
     },
     {
       "epoch": 0.8991470145509283,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0405893059828975,
       "learning_rate": 8.833557080955292e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2393105924129486,
+      "logits/rejected": -0.13887354731559753,
+      "logps/chosen": -1.7908332347869873,
+      "logps/rejected": -2.2589499950408936,
+      "loss": 0.665,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7908332347869873,
+      "rewards/margins": 0.4681168496608734,
+      "rewards/rejected": -2.2589499950408936,
       "step": 1680
     },
     {
       "epoch": 0.9018230473323299,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.815891762205357,
       "learning_rate": 8.823540101520381e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20145049691200256,
+      "logits/rejected": 0.011666422709822655,
+      "logps/chosen": -1.784388780593872,
+      "logps/rejected": -2.3641066551208496,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.784388780593872,
+      "rewards/margins": 0.5797180533409119,
+      "rewards/rejected": -2.3641066551208496,
       "step": 1685
     },
     {
       "epoch": 0.9044990801137314,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1685332819293417,
       "learning_rate": 8.813486026082637e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20556998252868652,
+      "logits/rejected": -0.03496624901890755,
+      "logps/chosen": -1.666202187538147,
+      "logps/rejected": -2.3008179664611816,
+      "loss": 0.6507,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.666202187538147,
+      "rewards/margins": 0.6346157193183899,
+      "rewards/rejected": -2.3008179664611816,
       "step": 1690
     },
     {
       "epoch": 0.907175112895133,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.124965020861085,
       "learning_rate": 8.803394952186742e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3144170641899109,
+      "logits/rejected": -0.17547264695167542,
+      "logps/chosen": -1.7483456134796143,
+      "logps/rejected": -2.3420894145965576,
+      "loss": 0.6579,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7483456134796143,
+      "rewards/margins": 0.5937438011169434,
+      "rewards/rejected": -2.3420894145965576,
       "step": 1695
     },
     {
       "epoch": 0.9098511456765346,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.079951814521911,
       "learning_rate": 8.793266977736342e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.14634238183498383,
+      "logits/rejected": -0.17982931435108185,
+      "logps/chosen": -1.772157907485962,
+      "logps/rejected": -2.0022144317626953,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.772157907485962,
+      "rewards/margins": 0.230056494474411,
+      "rewards/rejected": -2.0022144317626953,
       "step": 1700
     },
     {
       "epoch": 0.9125271784579361,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.41409205986597,
       "learning_rate": 8.783102200993085e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.14899399876594543,
+      "logits/rejected": -0.020769309252500534,
+      "logps/chosen": -1.6814422607421875,
+      "logps/rejected": -2.1317801475524902,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6814422607421875,
+      "rewards/margins": 0.4503380358219147,
+      "rewards/rejected": -2.1317801475524902,
       "step": 1705
     },
     {
       "epoch": 0.9152032112393377,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9092401900846077,
       "learning_rate": 8.772900720575683e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15680351853370667,
+      "logits/rejected": -0.08702749013900757,
+      "logps/chosen": -1.7041664123535156,
+      "logps/rejected": -2.104196071624756,
+      "loss": 0.6701,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7041664123535156,
+      "rewards/margins": 0.400029718875885,
+      "rewards/rejected": -2.104196071624756,
       "step": 1710
     },
     {
       "epoch": 0.9178792440207393,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2084516614302396,
       "learning_rate": 8.762662635458944e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1790921986103058,
+      "logits/rejected": 0.015415546484291553,
+      "logps/chosen": -1.8289378881454468,
+      "logps/rejected": -2.2158546447753906,
+      "loss": 0.6783,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.8289378881454468,
+      "rewards/margins": 0.3869166672229767,
+      "rewards/rejected": -2.2158546447753906,
       "step": 1715
     },
     {
       "epoch": 0.9205552768021408,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.544679090697845,
       "learning_rate": 8.752388044972811e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17626814544200897,
+      "logits/rejected": -0.10983548313379288,
+      "logps/chosen": -1.6182823181152344,
+      "logps/rejected": -2.141753673553467,
+      "loss": 0.6553,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6182823181152344,
+      "rewards/margins": 0.5234711170196533,
+      "rewards/rejected": -2.141753673553467,
       "step": 1720
     },
     {
       "epoch": 0.9232313095835424,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2459273075422734,
       "learning_rate": 8.74207704880141e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13397088646888733,
+      "logits/rejected": -0.03532525897026062,
+      "logps/chosen": -1.7240616083145142,
+      "logps/rejected": -2.3626952171325684,
+      "loss": 0.6539,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7240616083145142,
+      "rewards/margins": 0.6386336088180542,
+      "rewards/rejected": -2.3626952171325684,
       "step": 1725
     },
     {
       "epoch": 0.925907342364944,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.121365442521739,
       "learning_rate": 8.731729746982068e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09820817410945892,
+      "logits/rejected": -0.03878841921687126,
+      "logps/chosen": -1.685036301612854,
+      "logps/rejected": -2.0501296520233154,
+      "loss": 0.6695,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.685036301612854,
+      "rewards/margins": 0.3650934398174286,
+      "rewards/rejected": -2.0501296520233154,
       "step": 1730
     },
     {
       "epoch": 0.9285833751463456,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.430357055601366,
       "learning_rate": 8.721346239904355e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24304267764091492,
+      "logits/rejected": -0.08955205231904984,
+      "logps/chosen": -1.7179458141326904,
+      "logps/rejected": -2.4652867317199707,
+      "loss": 0.652,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7179458141326904,
+      "rewards/margins": 0.7473407983779907,
+      "rewards/rejected": -2.4652867317199707,
       "step": 1735
     },
     {
       "epoch": 0.9312594079277471,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8524311495718684,
       "learning_rate": 8.710926628309101e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1987670361995697,
+      "logits/rejected": -0.06383465230464935,
+      "logps/chosen": -1.7124691009521484,
+      "logps/rejected": -2.297957181930542,
+      "loss": 0.6567,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7124691009521484,
+      "rewards/margins": 0.5854877829551697,
+      "rewards/rejected": -2.297957181930542,
       "step": 1740
     },
     {
       "epoch": 0.9339354407091487,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7424373531776203,
       "learning_rate": 8.700471013287424e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13860243558883667,
+      "logits/rejected": -0.10608258098363876,
+      "logps/chosen": -1.689626693725586,
+      "logps/rejected": -2.1752796173095703,
+      "loss": 0.664,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.689626693725586,
+      "rewards/margins": 0.48565322160720825,
+      "rewards/rejected": -2.1752796173095703,
       "step": 1745
     },
     {
       "epoch": 0.9366114734905503,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9563266035447913,
       "learning_rate": 8.689979496279746e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.18505673110485077,
+      "logits/rejected": -0.12862268090248108,
+      "logps/chosen": -1.7912029027938843,
+      "logps/rejected": -2.2296857833862305,
+      "loss": 0.6685,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.7912029027938843,
+      "rewards/margins": 0.4384829103946686,
+      "rewards/rejected": -2.2296857833862305,
       "step": 1750
     },
     {
       "epoch": 0.9392875062719518,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1048687309280445,
       "learning_rate": 8.679452179074811e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2021680772304535,
+      "logits/rejected": -0.11145065724849701,
+      "logps/chosen": -1.6455810070037842,
+      "logps/rejected": -2.0956625938415527,
+      "loss": 0.6609,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6455810070037842,
+      "rewards/margins": 0.4500814974308014,
+      "rewards/rejected": -2.0956625938415527,
       "step": 1755
     },
     {
       "epoch": 0.9419635390533534,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.404966088483113,
       "learning_rate": 8.668889163808698e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1958305686712265,
+      "logits/rejected": -0.08931040018796921,
+      "logps/chosen": -1.6436002254486084,
+      "logps/rejected": -2.114105224609375,
+      "loss": 0.6579,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6436002254486084,
+      "rewards/margins": 0.47050514817237854,
+      "rewards/rejected": -2.114105224609375,
       "step": 1760
     },
     {
       "epoch": 0.944639571834755,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3825033347904156,
       "learning_rate": 8.658290552963827e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12904247641563416,
+      "logits/rejected": -0.11124918609857559,
+      "logps/chosen": -1.7082545757293701,
+      "logps/rejected": -2.173949956893921,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7082545757293701,
+      "rewards/margins": 0.4656955599784851,
+      "rewards/rejected": -2.173949956893921,
       "step": 1765
     },
     {
       "epoch": 0.9473156046161565,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1162083860595073,
       "learning_rate": 8.647656449367966e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17378933727741241,
+      "logits/rejected": -0.03233933821320534,
+      "logps/chosen": -1.771950364112854,
+      "logps/rejected": -2.133030414581299,
+      "loss": 0.6583,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.771950364112854,
+      "rewards/margins": 0.3610798716545105,
+      "rewards/rejected": -2.133030414581299,
       "step": 1770
     },
     {
       "epoch": 0.9499916373975581,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.1631138785848245,
       "learning_rate": 8.636986956193235e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20405487716197968,
+      "logits/rejected": -0.12140259891748428,
+      "logps/chosen": -1.6616615056991577,
+      "logps/rejected": -2.1474201679229736,
+      "loss": 0.6585,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.6616615056991577,
+      "rewards/margins": 0.4857584536075592,
+      "rewards/rejected": -2.1474201679229736,
       "step": 1775
     },
     {
       "epoch": 0.9526676701789597,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3717317952774706,
       "learning_rate": 8.626282176955104e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.204624742269516,
+      "logits/rejected": -0.0911920890212059,
+      "logps/chosen": -1.6643329858779907,
+      "logps/rejected": -2.206806182861328,
+      "loss": 0.6613,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.6643329858779907,
+      "rewards/margins": 0.5424734354019165,
+      "rewards/rejected": -2.206806182861328,
       "step": 1780
     },
     {
       "epoch": 0.9553437029603613,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5828748351768076,
       "learning_rate": 8.615542215511389e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.09748892486095428,
+      "logits/rejected": -0.029999341815710068,
+      "logps/chosen": -1.643737554550171,
+      "logps/rejected": -1.9389588832855225,
+      "loss": 0.6739,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.643737554550171,
+      "rewards/margins": 0.29522156715393066,
+      "rewards/rejected": -1.9389588832855225,
       "step": 1785
     },
     {
       "epoch": 0.9580197357417628,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.702377696032862,
       "learning_rate": 8.604767176061241e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1290159523487091,
+      "logits/rejected": -0.08263057470321655,
+      "logps/chosen": -1.7570621967315674,
+      "logps/rejected": -2.1943087577819824,
+      "loss": 0.664,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7570621967315674,
+      "rewards/margins": 0.43724650144577026,
+      "rewards/rejected": -2.1943087577819824,
       "step": 1790
     },
     {
       "epoch": 0.9606957685231644,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7842879691107334,
       "learning_rate": 8.593957163144141e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21867045760154724,
+      "logits/rejected": -0.09143811464309692,
+      "logps/chosen": -1.6201364994049072,
+      "logps/rejected": -2.2283973693847656,
+      "loss": 0.6499,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6201364994049072,
+      "rewards/margins": 0.6082608103752136,
+      "rewards/rejected": -2.2283973693847656,
       "step": 1795
     },
     {
       "epoch": 0.963371801304566,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2548500962050237,
       "learning_rate": 8.58311228163888e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15841044485569,
+      "logits/rejected": -0.0861646756529808,
+      "logps/chosen": -1.6833816766738892,
+      "logps/rejected": -2.0910985469818115,
+      "loss": 0.6597,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.6833816766738892,
+      "rewards/margins": 0.4077169895172119,
+      "rewards/rejected": -2.0910985469818115,
       "step": 1800
     },
     {
       "epoch": 0.9660478340859675,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.040689484997148,
       "learning_rate": 8.57223263676255e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27500978112220764,
+      "logits/rejected": -0.15153616666793823,
+      "logps/chosen": -1.6174161434173584,
+      "logps/rejected": -2.2804646492004395,
+      "loss": 0.6506,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6174161434173584,
+      "rewards/margins": 0.6630482077598572,
+      "rewards/rejected": -2.2804646492004395,
       "step": 1805
     },
     {
       "epoch": 0.9687238668673691,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.477780604566808,
       "learning_rate": 8.561318334069511e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15117596089839935,
+      "logits/rejected": -0.010185310617089272,
+      "logps/chosen": -1.6634376049041748,
+      "logps/rejected": -2.1122171878814697,
+      "loss": 0.6682,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6634376049041748,
+      "rewards/margins": 0.448779433965683,
+      "rewards/rejected": -2.1122171878814697,
       "step": 1810
     },
     {
       "epoch": 0.9713998996487707,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7801504310514953,
       "learning_rate": 8.550369479450375e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17215795814990997,
+      "logits/rejected": -0.04347606375813484,
+      "logps/chosen": -1.7053887844085693,
+      "logps/rejected": -2.2592666149139404,
+      "loss": 0.6546,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7053887844085693,
+      "rewards/margins": 0.5538779497146606,
+      "rewards/rejected": -2.2592666149139404,
       "step": 1815
     },
     {
       "epoch": 0.9740759324301723,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9021866906608773,
       "learning_rate": 8.539386179130977e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.14532248675823212,
+      "logits/rejected": -0.07133295387029648,
+      "logps/chosen": -1.7870458364486694,
+      "logps/rejected": -2.184950351715088,
+      "loss": 0.6608,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.7870458364486694,
+      "rewards/margins": 0.39790406823158264,
+      "rewards/rejected": -2.184950351715088,
       "step": 1820
     },
     {
       "epoch": 0.9767519652115738,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0173048616718763,
       "learning_rate": 8.528368539671347e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20779776573181152,
+      "logits/rejected": -0.07001302391290665,
+      "logps/chosen": -1.7165762186050415,
+      "logps/rejected": -2.382704257965088,
+      "loss": 0.6604,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7165762186050415,
+      "rewards/margins": 0.6661278605461121,
+      "rewards/rejected": -2.382704257965088,
       "step": 1825
     },
     {
       "epoch": 0.9794279979929754,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.52857867527269,
       "learning_rate": 8.51731666796467e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.05026236176490784,
+      "logits/rejected": -0.01709885336458683,
+      "logps/chosen": -1.7864534854888916,
+      "logps/rejected": -2.1371140480041504,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7864534854888916,
+      "rewards/margins": 0.35066041350364685,
+      "rewards/rejected": -2.1371140480041504,
       "step": 1830
     },
     {
       "epoch": 0.982104030774377,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.467075560512466,
       "learning_rate": 8.506230671236254e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.18817004561424255,
+      "logits/rejected": -0.12967504560947418,
+      "logps/chosen": -1.764174461364746,
+      "logps/rejected": -2.11920166015625,
+      "loss": 0.6683,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.764174461364746,
+      "rewards/margins": 0.3550271987915039,
+      "rewards/rejected": -2.11920166015625,
       "step": 1835
     },
     {
       "epoch": 0.9847800635557785,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4530390135984543,
       "learning_rate": 8.495110657042488e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1285647600889206,
+      "logits/rejected": -0.02805677056312561,
+      "logps/chosen": -1.7589118480682373,
+      "logps/rejected": -2.3351047039031982,
+      "loss": 0.6578,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7589118480682373,
+      "rewards/margins": 0.5761929750442505,
+      "rewards/rejected": -2.3351047039031982,
       "step": 1840
     },
     {
       "epoch": 0.9874560963371801,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.684651992332132,
       "learning_rate": 8.483956733269799e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15700112283229828,
+      "logits/rejected": -0.06916292011737823,
+      "logps/chosen": -1.7566595077514648,
+      "logps/rejected": -2.2062172889709473,
+      "loss": 0.6641,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.7566595077514648,
+      "rewards/margins": 0.4495576322078705,
+      "rewards/rejected": -2.2062172889709473,
       "step": 1845
     },
     {
       "epoch": 0.9901321291185817,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.778796908197804,
       "learning_rate": 8.472769008133602e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2883799970149994,
+      "logits/rejected": -0.17105185985565186,
+      "logps/chosen": -1.7560796737670898,
+      "logps/rejected": -2.110762119293213,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7560796737670898,
+      "rewards/margins": 0.35468223690986633,
+      "rewards/rejected": -2.110762119293213,
       "step": 1850
     },
     {
       "epoch": 0.9928081618999832,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1611563952793675,
       "learning_rate": 8.461547590177259e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20120088756084442,
+      "logits/rejected": -0.10221078246831894,
+      "logps/chosen": -1.6823533773422241,
+      "logps/rejected": -2.1616406440734863,
+      "loss": 0.6642,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6823533773422241,
+      "rewards/margins": 0.47928720712661743,
+      "rewards/rejected": -2.1616406440734863,
       "step": 1855
     },
     {
       "epoch": 0.9954841946813848,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.271656254138914,
       "learning_rate": 8.450292588271014e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1693296730518341,
+      "logits/rejected": -0.08926688879728317,
+      "logps/chosen": -1.8450262546539307,
+      "logps/rejected": -2.251145124435425,
+      "loss": 0.6622,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.8450262546539307,
+      "rewards/margins": 0.4061189591884613,
+      "rewards/rejected": -2.251145124435425,
       "step": 1860
     },
     {
       "epoch": 0.9981602274627864,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.064497960539952,
       "learning_rate": 8.439004111610945e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.18421629071235657,
+      "logits/rejected": -0.11431281268596649,
+      "logps/chosen": -1.6581480503082275,
+      "logps/rejected": -2.204413414001465,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6581480503082275,
+      "rewards/margins": 0.5462655425071716,
+      "rewards/rejected": -2.204413414001465,
       "step": 1865
     },
     {
       "epoch": 1.000836260244188,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.26492806919797,
       "learning_rate": 8.427682269717901e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24960891902446747,
+      "logits/rejected": -0.10946635156869888,
+      "logps/chosen": -1.893812894821167,
+      "logps/rejected": -2.4895453453063965,
+      "loss": 0.672,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.893812894821167,
+      "rewards/margins": 0.5957326292991638,
+      "rewards/rejected": -2.4895453453063965,
       "step": 1870
     },
     {
       "epoch": 1.0035122930255895,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3035901512663326,
       "learning_rate": 8.416327172436446e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27780473232269287,
+      "logits/rejected": -0.1568005234003067,
+      "logps/chosen": -1.8726648092269897,
+      "logps/rejected": -2.216933488845825,
+      "loss": 0.672,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.8726648092269897,
+      "rewards/margins": 0.3442686200141907,
+      "rewards/rejected": -2.216933488845825,
       "step": 1875
     },
     {
       "epoch": 1.0061883258069912,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.149572757392751,
       "learning_rate": 8.404938929933778e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1709388941526413,
+      "logits/rejected": -0.010907994583249092,
+      "logps/chosen": -1.8402321338653564,
+      "logps/rejected": -2.551429033279419,
+      "loss": 0.6555,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8402321338653564,
+      "rewards/margins": 0.7111968994140625,
+      "rewards/rejected": -2.551429033279419,
       "step": 1880
     },
     {
       "epoch": 1.0088643585883927,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7668125551276908,
       "learning_rate": 8.39351765269868e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20978617668151855,
+      "logits/rejected": -0.13652516901493073,
+      "logps/chosen": -1.7452433109283447,
+      "logps/rejected": -2.2248101234436035,
+      "loss": 0.6527,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7452433109283447,
+      "rewards/margins": 0.4795669615268707,
+      "rewards/rejected": -2.2248101234436035,
       "step": 1885
     },
     {
       "epoch": 1.0115403913697942,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.885704959880113,
       "learning_rate": 8.382063451540431e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21603664755821228,
+      "logits/rejected": -0.02857707068324089,
+      "logps/chosen": -1.7436602115631104,
+      "logps/rejected": -2.3155527114868164,
+      "loss": 0.6717,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.7436602115631104,
+      "rewards/margins": 0.5718926191329956,
+      "rewards/rejected": -2.3155527114868164,
       "step": 1890
     },
     {
       "epoch": 1.014216424151196,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.345654626325416,
       "learning_rate": 8.370576437587742e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17020276188850403,
+      "logits/rejected": -0.12324122339487076,
+      "logps/chosen": -1.7798153162002563,
+      "logps/rejected": -2.204895496368408,
+      "loss": 0.6555,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7798153162002563,
+      "rewards/margins": 0.42508020997047424,
+      "rewards/rejected": -2.204895496368408,
       "step": 1895
     },
     {
       "epoch": 1.0168924569325974,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9085640451478105,
       "learning_rate": 8.359056722287674e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25659728050231934,
+      "logits/rejected": -0.02260657027363777,
+      "logps/chosen": -1.7561557292938232,
+      "logps/rejected": -2.2739100456237793,
+      "loss": 0.654,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.7561557292938232,
+      "rewards/margins": 0.5177541971206665,
+      "rewards/rejected": -2.2739100456237793,
       "step": 1900
     },
     {
       "epoch": 1.019568489713999,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4564266822263185,
       "learning_rate": 8.347504417404553e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1817479431629181,
+      "logits/rejected": -0.04545123130083084,
+      "logps/chosen": -1.7436368465423584,
+      "logps/rejected": -2.1051554679870605,
+      "loss": 0.6651,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7436368465423584,
+      "rewards/margins": 0.36151885986328125,
+      "rewards/rejected": -2.1051554679870605,
       "step": 1905
     },
     {
       "epoch": 1.0222445224954007,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5806481410537,
       "learning_rate": 8.335919635018893e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2726041078567505,
+      "logits/rejected": -0.15640738606452942,
+      "logps/chosen": -1.663525938987732,
+      "logps/rejected": -2.043221950531006,
+      "loss": 0.6663,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.663525938987732,
+      "rewards/margins": 0.3796958029270172,
+      "rewards/rejected": -2.043221950531006,
       "step": 1910
     },
     {
       "epoch": 1.0249205552768021,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0886941069508125,
       "learning_rate": 8.324302487526303e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1850888431072235,
+      "logits/rejected": -0.11027760803699493,
+      "logps/chosen": -1.6806888580322266,
+      "logps/rejected": -2.019749879837036,
+      "loss": 0.6577,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.6806888580322266,
+      "rewards/margins": 0.33906090259552,
+      "rewards/rejected": -2.019749879837036,
       "step": 1915
     },
     {
       "epoch": 1.0275965880582036,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3133605366999643,
       "learning_rate": 8.312653087636398e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22945375740528107,
+      "logits/rejected": -0.16329894959926605,
+      "logps/chosen": -1.5610296726226807,
+      "logps/rejected": -2.0574774742126465,
+      "loss": 0.6406,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5610296726226807,
+      "rewards/margins": 0.49644798040390015,
+      "rewards/rejected": -2.0574774742126465,
       "step": 1920
     },
     {
       "epoch": 1.0302726208396054,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.857607653404868,
       "learning_rate": 8.300971548371711e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30790385603904724,
+      "logits/rejected": -0.1197204738855362,
+      "logps/chosen": -1.7262077331542969,
+      "logps/rejected": -2.207197427749634,
+      "loss": 0.6555,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7262077331542969,
+      "rewards/margins": 0.4809897541999817,
+      "rewards/rejected": -2.207197427749634,
       "step": 1925
     },
     {
       "epoch": 1.0329486536210069,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.578884559357895,
       "learning_rate": 8.289257983066582e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23078329861164093,
+      "logits/rejected": -0.10627172142267227,
+      "logps/chosen": -1.618760347366333,
+      "logps/rejected": -2.188694477081299,
+      "loss": 0.6414,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.618760347366333,
+      "rewards/margins": 0.5699342489242554,
+      "rewards/rejected": -2.188694477081299,
       "step": 1930
     },
     {
       "epoch": 1.0356246864024083,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8386709643030343,
       "learning_rate": 8.277512505366077e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2667911946773529,
+      "logits/rejected": -0.0994887724518776,
+      "logps/chosen": -1.7833267450332642,
+      "logps/rejected": -2.3820574283599854,
+      "loss": 0.6601,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7833267450332642,
+      "rewards/margins": 0.598730742931366,
+      "rewards/rejected": -2.3820574283599854,
       "step": 1935
     },
     {
       "epoch": 1.03830071918381,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1518617641153908,
       "learning_rate": 8.265735229224868e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.18283165991306305,
+      "logits/rejected": -0.07191173732280731,
+      "logps/chosen": -1.707069993019104,
+      "logps/rejected": -2.355748176574707,
+      "loss": 0.6536,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.707069993019104,
+      "rewards/margins": 0.648678183555603,
+      "rewards/rejected": -2.355748176574707,
       "step": 1940
     },
     {
       "epoch": 1.0409767519652116,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.91714363440596,
       "learning_rate": 8.253926268906144e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27594807744026184,
+      "logits/rejected": -0.12020949274301529,
+      "logps/chosen": -1.7843881845474243,
+      "logps/rejected": -2.502321720123291,
+      "loss": 0.6491,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7843881845474243,
+      "rewards/margins": 0.7179337739944458,
+      "rewards/rejected": -2.502321720123291,
       "step": 1945
     },
     {
       "epoch": 1.043652784746613,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9698438326856478,
       "learning_rate": 8.242085738980487e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20048010349273682,
+      "logits/rejected": -0.0054269894026219845,
+      "logps/chosen": -1.7941757440567017,
+      "logps/rejected": -2.3263156414031982,
+      "loss": 0.6566,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7941757440567017,
+      "rewards/margins": 0.5321398377418518,
+      "rewards/rejected": -2.3263156414031982,
       "step": 1950
     },
     {
       "epoch": 1.0463288175280148,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.594633249657895,
       "learning_rate": 8.230213754324772e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1820477545261383,
+      "logits/rejected": -0.11782010644674301,
+      "logps/chosen": -1.6170260906219482,
+      "logps/rejected": -2.1097934246063232,
+      "loss": 0.6473,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.6170260906219482,
+      "rewards/margins": 0.4927673935890198,
+      "rewards/rejected": -2.1097934246063232,
       "step": 1955
     },
     {
       "epoch": 1.0490048503094163,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.521872994741012,
       "learning_rate": 8.218310430121045e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20924706757068634,
+      "logits/rejected": -0.17469140887260437,
+      "logps/chosen": -1.678802728652954,
+      "logps/rejected": -2.0900707244873047,
+      "loss": 0.6604,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.678802728652954,
+      "rewards/margins": 0.41126805543899536,
+      "rewards/rejected": -2.0900707244873047,
       "step": 1960
     },
     {
       "epoch": 1.051680883090818,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.308285545017951,
       "learning_rate": 8.20637588185541e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1450272500514984,
+      "logits/rejected": -0.08384187519550323,
+      "logps/chosen": -1.6316436529159546,
+      "logps/rejected": -2.3678953647613525,
+      "loss": 0.641,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6316436529159546,
+      "rewards/margins": 0.7362518906593323,
+      "rewards/rejected": -2.3678953647613525,
       "step": 1965
     },
     {
       "epoch": 1.0543569158722195,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.358158134676778,
       "learning_rate": 8.194410225316906e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19815480709075928,
+      "logits/rejected": -0.06416761875152588,
+      "logps/chosen": -1.5755919218063354,
+      "logps/rejected": -2.0860962867736816,
+      "loss": 0.6559,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.5755919218063354,
+      "rewards/margins": 0.5105042457580566,
+      "rewards/rejected": -2.0860962867736816,
       "step": 1970
     },
     {
       "epoch": 1.057032948653621,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3832507917130963,
       "learning_rate": 8.182413576596385e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1305771768093109,
+      "logits/rejected": -0.05065881460905075,
+      "logps/chosen": -1.593427062034607,
+      "logps/rejected": -2.0549588203430176,
+      "loss": 0.6477,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.593427062034607,
+      "rewards/margins": 0.4615318179130554,
+      "rewards/rejected": -2.0549588203430176,
       "step": 1975
     },
     {
       "epoch": 1.0597089814350227,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.028000735465466,
       "learning_rate": 8.170386052085389e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1088295429944992,
+      "logits/rejected": 0.004176872782409191,
+      "logps/chosen": -1.7239269018173218,
+      "logps/rejected": -2.2126667499542236,
+      "loss": 0.6565,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7239269018173218,
+      "rewards/margins": 0.4887397289276123,
+      "rewards/rejected": -2.2126667499542236,
       "step": 1980
     },
     {
       "epoch": 1.0623850142164242,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8286598452784566,
       "learning_rate": 8.158327768475008e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.16414202749729156,
+      "logits/rejected": -0.017862845212221146,
+      "logps/chosen": -1.8199892044067383,
+      "logps/rejected": -2.181795835494995,
+      "loss": 0.6643,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.8199892044067383,
+      "rewards/margins": 0.36180680990219116,
+      "rewards/rejected": -2.181795835494995,
       "step": 1985
     },
     {
       "epoch": 1.0650610469978257,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.046154294049123,
       "learning_rate": 8.146238842754767e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19878873229026794,
+      "logits/rejected": -0.10793648660182953,
+      "logps/chosen": -1.8073660135269165,
+      "logps/rejected": -2.225064992904663,
+      "loss": 0.6551,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.8073660135269165,
+      "rewards/margins": 0.41769909858703613,
+      "rewards/rejected": -2.225064992904663,
       "step": 1990
     },
     {
       "epoch": 1.0677370797792274,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.234720084387265,
       "learning_rate": 8.134119392211476e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.0991034135222435,
+      "logits/rejected": 0.04456945136189461,
+      "logps/chosen": -1.6650326251983643,
+      "logps/rejected": -2.376002788543701,
+      "loss": 0.634,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6650326251983643,
+      "rewards/margins": 0.7109704613685608,
+      "rewards/rejected": -2.376002788543701,
       "step": 1995
     },
     {
       "epoch": 1.0704131125606289,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 13.507719342396253,
       "learning_rate": 8.121969534428094e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21368630230426788,
+      "logits/rejected": -0.06342948973178864,
+      "logps/chosen": -1.8885704278945923,
+      "logps/rejected": -2.4035730361938477,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.8885704278945923,
+      "rewards/margins": 0.5150026082992554,
+      "rewards/rejected": -2.4035730361938477,
       "step": 2000
     },
     {
       "epoch": 1.0704131125606289,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.5887,
-      "eval_samples_per_second": 38.885,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.743,
+      "eval_logits/chosen": 0.12483588606119156,
+      "eval_logits/rejected": 0.21461640298366547,
+      "eval_logps/chosen": -1.7098091840744019,
+      "eval_logps/rejected": -2.2986159324645996,
+      "eval_loss": 0.6545049548149109,
+      "eval_rewards/accuracies": 0.6639465689659119,
+      "eval_rewards/chosen": -1.7098091840744019,
+      "eval_rewards/margins": 0.5888065099716187,
+      "eval_rewards/rejected": -2.2986159324645996,
+      "eval_runtime": 40.4443,
+      "eval_samples_per_second": 33.256,
+      "eval_steps_per_second": 8.332,
       "step": 2000
     },
     {
       "epoch": 1.0730891453420304,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4614746395934994,
       "learning_rate": 8.109789387282599e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17246174812316895,
+      "logits/rejected": -0.09183319658041,
+      "logps/chosen": -1.7540531158447266,
+      "logps/rejected": -2.233997106552124,
+      "loss": 0.6624,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7540531158447266,
+      "rewards/margins": 0.47994405031204224,
+      "rewards/rejected": -2.233997106552124,
       "step": 2005
     },
     {
       "epoch": 1.075765178123432,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.362830202020899,
       "learning_rate": 8.097579068946827e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1247321367263794,
+      "logits/rejected": -0.02580316737294197,
+      "logps/chosen": -1.6027886867523193,
+      "logps/rejected": -2.089876890182495,
+      "loss": 0.656,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6027886867523193,
+      "rewards/margins": 0.4870882034301758,
+      "rewards/rejected": -2.089876890182495,
       "step": 2010
     },
     {
       "epoch": 1.0784412109048336,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3270407378058553,
       "learning_rate": 8.085338697885344e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1918140947818756,
+      "logits/rejected": -0.05760322883725166,
+      "logps/chosen": -1.7067158222198486,
+      "logps/rejected": -2.26135516166687,
+      "loss": 0.666,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7067158222198486,
+      "rewards/margins": 0.5546393990516663,
+      "rewards/rejected": -2.26135516166687,
       "step": 2015
     },
     {
       "epoch": 1.081117243686235,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.1734869105926,
       "learning_rate": 8.073068392854282e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2682078778743744,
+      "logits/rejected": -0.09298717975616455,
+      "logps/chosen": -1.7584478855133057,
+      "logps/rejected": -2.380786657333374,
+      "loss": 0.6478,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7584478855133057,
+      "rewards/margins": 0.6223388314247131,
+      "rewards/rejected": -2.380786657333374,
       "step": 2020
     },
     {
       "epoch": 1.0837932764676368,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8936601580443635,
       "learning_rate": 8.060768272900193e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1553167998790741,
+      "logits/rejected": -0.04357858747243881,
+      "logps/chosen": -1.6950851678848267,
+      "logps/rejected": -2.3984155654907227,
+      "loss": 0.6373,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6950851678848267,
+      "rewards/margins": 0.7033304572105408,
+      "rewards/rejected": -2.3984155654907227,
       "step": 2025
     },
     {
       "epoch": 1.0864693092490383,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5968545386267725,
       "learning_rate": 8.0484384573589e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23250460624694824,
+      "logits/rejected": -0.2020358294248581,
+      "logps/chosen": -1.674464464187622,
+      "logps/rejected": -2.1252400875091553,
+      "loss": 0.649,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.674464464187622,
+      "rewards/margins": 0.45077571272850037,
+      "rewards/rejected": -2.1252400875091553,
       "step": 2030
     },
     {
       "epoch": 1.0891453420304398,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3324482689469574,
       "learning_rate": 8.03607906585432e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25278240442276,
+      "logits/rejected": -0.08986358344554901,
+      "logps/chosen": -1.7761884927749634,
+      "logps/rejected": -2.2815897464752197,
+      "loss": 0.6543,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7761884927749634,
+      "rewards/margins": 0.5054013133049011,
+      "rewards/rejected": -2.2815897464752197,
       "step": 2035
     },
     {
       "epoch": 1.0918213748118415,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3320730256170106,
       "learning_rate": 8.023690218297329e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3003383278846741,
+      "logits/rejected": -0.23502881824970245,
+      "logps/chosen": -1.7876596450805664,
+      "logps/rejected": -2.224050521850586,
+      "loss": 0.643,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7876596450805664,
+      "rewards/margins": 0.4363909363746643,
+      "rewards/rejected": -2.224050521850586,
       "step": 2040
     },
     {
       "epoch": 1.094497407593243,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.758787147725686,
       "learning_rate": 8.01127203488458e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.16633997857570648,
+      "logits/rejected": -0.13713163137435913,
+      "logps/chosen": -1.7426553964614868,
+      "logps/rejected": -2.2400920391082764,
+      "loss": 0.6545,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7426553964614868,
+      "rewards/margins": 0.4974365234375,
+      "rewards/rejected": -2.2400920391082764,
       "step": 2045
     },
     {
       "epoch": 1.0971734403746445,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 12.856936391767288,
       "learning_rate": 7.998824636097339e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2651461362838745,
+      "logits/rejected": -0.15107125043869019,
+      "logps/chosen": -1.8224207162857056,
+      "logps/rejected": -2.269073247909546,
+      "loss": 0.678,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.8224207162857056,
+      "rewards/margins": 0.44665274024009705,
+      "rewards/rejected": -2.269073247909546,
       "step": 2050
     },
     {
       "epoch": 1.0998494731560462,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.926640431745493,
       "learning_rate": 7.986348142700328e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.18771898746490479,
+      "logits/rejected": -0.0607999786734581,
+      "logps/chosen": -1.799108862876892,
+      "logps/rejected": -2.350358724594116,
+      "loss": 0.6438,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.799108862876892,
+      "rewards/margins": 0.5512499213218689,
+      "rewards/rejected": -2.350358724594116,
       "step": 2055
     },
     {
       "epoch": 1.1025255059374477,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.026414403985915,
       "learning_rate": 7.973842675740539e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1648760437965393,
+      "logits/rejected": -0.12342949211597443,
+      "logps/chosen": -1.804086446762085,
+      "logps/rejected": -2.480442762374878,
+      "loss": 0.649,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.804086446762085,
+      "rewards/margins": 0.676356315612793,
+      "rewards/rejected": -2.480442762374878,
       "step": 2060
     },
     {
       "epoch": 1.1052015387188494,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3009174175295635,
       "learning_rate": 7.961308356546066e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23486128449440002,
+      "logits/rejected": -0.10844055563211441,
+      "logps/chosen": -1.791778564453125,
+      "logps/rejected": -2.3047947883605957,
+      "loss": 0.6506,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.791778564453125,
+      "rewards/margins": 0.5130161643028259,
+      "rewards/rejected": -2.3047947883605957,
       "step": 2065
     },
     {
       "epoch": 1.107877571500251,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.565085959472698,
       "learning_rate": 7.948745306724931e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21824876964092255,
+      "logits/rejected": -0.07461760193109512,
+      "logps/chosen": -1.7341811656951904,
+      "logps/rejected": -2.439887046813965,
+      "loss": 0.634,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7341811656951904,
+      "rewards/margins": 0.7057058215141296,
+      "rewards/rejected": -2.439887046813965,
       "step": 2070
     },
     {
       "epoch": 1.1105536042816524,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.451791098537331,
       "learning_rate": 7.936153648163897e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2557971477508545,
+      "logits/rejected": -0.1556805968284607,
+      "logps/chosen": -1.913243055343628,
+      "logps/rejected": -2.5344691276550293,
+      "loss": 0.6529,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.913243055343628,
+      "rewards/margins": 0.6212257742881775,
+      "rewards/rejected": -2.5344691276550293,
       "step": 2075
     },
     {
       "epoch": 1.1132296370630541,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7070524903723676,
       "learning_rate": 7.92353350302729e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28152820467948914,
+      "logits/rejected": -0.12565365433692932,
+      "logps/chosen": -1.697198510169983,
+      "logps/rejected": -2.3664944171905518,
+      "loss": 0.6385,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.697198510169983,
+      "rewards/margins": 0.6692960262298584,
+      "rewards/rejected": -2.3664944171905518,
       "step": 2080
     },
     {
       "epoch": 1.1159056698444556,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.20894892688694,
       "learning_rate": 7.910884993755816e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2493903636932373,
+      "logits/rejected": -0.1442599594593048,
+      "logps/chosen": -1.7548630237579346,
+      "logps/rejected": -2.4789328575134277,
+      "loss": 0.6453,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7548630237579346,
+      "rewards/margins": 0.7240699529647827,
+      "rewards/rejected": -2.4789328575134277,
       "step": 2085
     },
     {
       "epoch": 1.118581702625857,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.648093277699584,
       "learning_rate": 7.898208243065367e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2797894775867462,
+      "logits/rejected": -0.26872801780700684,
+      "logps/chosen": -1.7379087209701538,
+      "logps/rejected": -2.087397575378418,
+      "loss": 0.6544,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7379087209701538,
+      "rewards/margins": 0.34948891401290894,
+      "rewards/rejected": -2.087397575378418,
       "step": 2090
     },
     {
       "epoch": 1.1212577354072588,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.879413346137865,
       "learning_rate": 7.88550337394583e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2493230551481247,
+      "logits/rejected": -0.11252955347299576,
+      "logps/chosen": -1.9056915044784546,
+      "logps/rejected": -2.3451037406921387,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9056915044784546,
+      "rewards/margins": 0.43941205739974976,
+      "rewards/rejected": -2.3451037406921387,
       "step": 2095
     },
     {
       "epoch": 1.1239337681886603,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.910540981347752,
       "learning_rate": 7.872770509659905e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15982334315776825,
+      "logits/rejected": -0.11562427133321762,
+      "logps/chosen": -1.8902509212493896,
+      "logps/rejected": -2.3074734210968018,
+      "loss": 0.6559,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.8902509212493896,
+      "rewards/margins": 0.41722220182418823,
+      "rewards/rejected": -2.3074734210968018,
       "step": 2100
     },
     {
       "epoch": 1.1266098009700618,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.791895739592682,
       "learning_rate": 7.860009773741896e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.144171804189682,
+      "logits/rejected": -0.028581952676177025,
+      "logps/chosen": -1.797886610031128,
+      "logps/rejected": -2.4341466426849365,
+      "loss": 0.6473,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.797886610031128,
+      "rewards/margins": 0.6362601518630981,
+      "rewards/rejected": -2.4341466426849365,
       "step": 2105
     },
     {
       "epoch": 1.1292858337514635,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.11464423222663,
       "learning_rate": 7.84722128999652e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2354166954755783,
+      "logits/rejected": -0.09281990677118301,
+      "logps/chosen": -1.7944738864898682,
+      "logps/rejected": -2.5797717571258545,
+      "loss": 0.6429,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7944738864898682,
+      "rewards/margins": 0.785298228263855,
+      "rewards/rejected": -2.5797717571258545,
       "step": 2110
     },
     {
       "epoch": 1.131961866532865,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.4601199986964835,
       "learning_rate": 7.834405182497699e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12684021890163422,
+      "logits/rejected": -0.07805754989385605,
+      "logps/chosen": -1.8538509607315063,
+      "logps/rejected": -2.3863778114318848,
+      "loss": 0.6376,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.8538509607315063,
+      "rewards/margins": 0.532526969909668,
+      "rewards/rejected": -2.3863778114318848,
       "step": 2115
     },
     {
       "epoch": 1.1346378993142665,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.316729632864037,
       "learning_rate": 7.821561575587368e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23475833237171173,
+      "logits/rejected": -0.19683800637722015,
+      "logps/chosen": -1.694981336593628,
+      "logps/rejected": -2.228874921798706,
+      "loss": 0.6461,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.694981336593628,
+      "rewards/margins": 0.5338935852050781,
+      "rewards/rejected": -2.228874921798706,
       "step": 2120
     },
     {
       "epoch": 1.1373139320956682,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.007053042841388,
       "learning_rate": 7.808690593874254e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21208183467388153,
+      "logits/rejected": -0.15762484073638916,
+      "logps/chosen": -1.6656156778335571,
+      "logps/rejected": -2.319190502166748,
+      "loss": 0.6308,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6656156778335571,
+      "rewards/margins": 0.6535750031471252,
+      "rewards/rejected": -2.319190502166748,
       "step": 2125
     },
     {
       "epoch": 1.1399899648770697,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.612084220219568,
       "learning_rate": 7.79579236223268e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1487949937582016,
+      "logits/rejected": 0.07376561313867569,
+      "logps/chosen": -1.6839650869369507,
+      "logps/rejected": -2.3944759368896484,
+      "loss": 0.6419,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6839650869369507,
+      "rewards/margins": 0.7105108499526978,
+      "rewards/rejected": -2.3944759368896484,
       "step": 2130
     },
     {
       "epoch": 1.1426659976584714,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.790330779041942,
       "learning_rate": 7.782867005801346e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.18245534598827362,
+      "logits/rejected": -0.02702302671968937,
+      "logps/chosen": -1.705817461013794,
+      "logps/rejected": -2.400905132293701,
+      "loss": 0.6449,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.705817461013794,
+      "rewards/margins": 0.695087730884552,
+      "rewards/rejected": -2.400905132293701,
       "step": 2135
     },
     {
       "epoch": 1.145342030439873,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5514274980966087,
       "learning_rate": 7.769914649982117e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21617178618907928,
+      "logits/rejected": -0.08361689001321793,
+      "logps/chosen": -1.739519715309143,
+      "logps/rejected": -2.3060176372528076,
+      "loss": 0.6601,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.739519715309143,
+      "rewards/margins": 0.5664979219436646,
+      "rewards/rejected": -2.3060176372528076,
       "step": 2140
     },
     {
       "epoch": 1.1480180632212744,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9993010540159313,
       "learning_rate": 7.756935420438803e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.202193021774292,
+      "logits/rejected": -0.11842985451221466,
+      "logps/chosen": -1.6297099590301514,
+      "logps/rejected": -2.2481462955474854,
+      "loss": 0.63,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6297099590301514,
+      "rewards/margins": 0.6184362173080444,
+      "rewards/rejected": -2.2481462955474854,
       "step": 2145
     },
     {
       "epoch": 1.1506940960026761,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8787568977669955,
       "learning_rate": 7.743929443095951e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24711282551288605,
+      "logits/rejected": -0.19685444235801697,
+      "logps/chosen": -1.7927589416503906,
+      "logps/rejected": -2.337613344192505,
+      "loss": 0.6598,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7927589416503906,
+      "rewards/margins": 0.5448542237281799,
+      "rewards/rejected": -2.337613344192505,
       "step": 2150
     },
     {
       "epoch": 1.1533701287840776,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9607563700853365,
       "learning_rate": 7.730896844137609e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17122125625610352,
+      "logits/rejected": -0.10763285309076309,
+      "logps/chosen": -1.8190664052963257,
+      "logps/rejected": -2.282247304916382,
+      "loss": 0.6462,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8190664052963257,
+      "rewards/margins": 0.4631810784339905,
+      "rewards/rejected": -2.282247304916382,
       "step": 2155
     },
     {
       "epoch": 1.1560461615654791,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.379796864152493,
       "learning_rate": 7.717837750006106e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26187825202941895,
+      "logits/rejected": -0.17092230916023254,
+      "logps/chosen": -1.694409966468811,
+      "logps/rejected": -2.3330605030059814,
+      "loss": 0.6249,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.694409966468811,
+      "rewards/margins": 0.6386507153511047,
+      "rewards/rejected": -2.3330605030059814,
       "step": 2160
     },
     {
       "epoch": 1.1587221943468808,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9409017630119845,
       "learning_rate": 7.704752287400832e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23209333419799805,
+      "logits/rejected": -0.05963393300771713,
+      "logps/chosen": -1.753718614578247,
+      "logps/rejected": -2.44122576713562,
+      "loss": 0.6528,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.753718614578247,
+      "rewards/margins": 0.6875072717666626,
+      "rewards/rejected": -2.44122576713562,
       "step": 2165
     },
     {
       "epoch": 1.1613982271282823,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0686094053534747,
       "learning_rate": 7.691640583277004e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22047261893749237,
+      "logits/rejected": -0.06881072372198105,
+      "logps/chosen": -1.6510124206542969,
+      "logps/rejected": -2.343684673309326,
+      "loss": 0.6569,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6510124206542969,
+      "rewards/margins": 0.6926722526550293,
+      "rewards/rejected": -2.343684673309326,
       "step": 2170
     },
     {
       "epoch": 1.1640742599096838,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3604095165528527,
       "learning_rate": 7.678502764844433e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24528780579566956,
+      "logits/rejected": -0.08623888343572617,
+      "logps/chosen": -1.7231464385986328,
+      "logps/rejected": -2.2840442657470703,
+      "loss": 0.6518,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7231464385986328,
+      "rewards/margins": 0.560897946357727,
+      "rewards/rejected": -2.2840442657470703,
       "step": 2175
     },
     {
       "epoch": 1.1667502926910855,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.224298971891653,
       "learning_rate": 7.665338959566288e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2664191722869873,
+      "logits/rejected": -0.1883905827999115,
+      "logps/chosen": -1.713862657546997,
+      "logps/rejected": -2.2637085914611816,
+      "loss": 0.6324,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.713862657546997,
+      "rewards/margins": 0.5498460531234741,
+      "rewards/rejected": -2.2637085914611816,
       "step": 2180
     },
     {
       "epoch": 1.169426325472487,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9226327795958595,
       "learning_rate": 7.652149295157868e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17648489773273468,
+      "logits/rejected": -0.0486362986266613,
+      "logps/chosen": -1.778388261795044,
+      "logps/rejected": -2.2680180072784424,
+      "loss": 0.655,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.778388261795044,
+      "rewards/margins": 0.4896295964717865,
+      "rewards/rejected": -2.2680180072784424,
       "step": 2185
     },
     {
       "epoch": 1.1721023582538885,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.961776631159982,
       "learning_rate": 7.638933899585354e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.10730533301830292,
+      "logits/rejected": -0.05697392299771309,
+      "logps/chosen": -1.7031389474868774,
+      "logps/rejected": -2.343703269958496,
+      "loss": 0.6347,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7031389474868774,
+      "rewards/margins": 0.6405643224716187,
+      "rewards/rejected": -2.343703269958496,
       "step": 2190
     },
     {
       "epoch": 1.1747783910352902,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7846652390001485,
       "learning_rate": 7.625692901064573e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17884695529937744,
+      "logits/rejected": -0.09309454262256622,
+      "logps/chosen": -1.755706548690796,
+      "logps/rejected": -2.3891284465789795,
+      "loss": 0.6468,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.755706548690796,
+      "rewards/margins": 0.6334219574928284,
+      "rewards/rejected": -2.3891284465789795,
       "step": 2195
     },
     {
       "epoch": 1.1774544238166917,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9560661998320668,
       "learning_rate": 7.61242642805975e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24442346394062042,
+      "logits/rejected": -0.25473296642303467,
+      "logps/chosen": -1.7628345489501953,
+      "logps/rejected": -2.2831826210021973,
+      "loss": 0.6465,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7628345489501953,
+      "rewards/margins": 0.520348310470581,
+      "rewards/rejected": -2.2831826210021973,
       "step": 2200
     },
     {
       "epoch": 1.1801304565980932,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.20666342874739,
       "learning_rate": 7.599134609282266e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2979923188686371,
+      "logits/rejected": -0.12901487946510315,
+      "logps/chosen": -1.723496437072754,
+      "logps/rejected": -2.3329784870147705,
+      "loss": 0.6535,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.723496437072754,
+      "rewards/margins": 0.6094823479652405,
+      "rewards/rejected": -2.3329784870147705,
       "step": 2205
     },
     {
       "epoch": 1.182806489379495,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.006446869340445,
       "learning_rate": 7.585817573689402e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.321370005607605,
+      "logits/rejected": -0.20671744644641876,
+      "logps/chosen": -1.5410319566726685,
+      "logps/rejected": -2.3826990127563477,
+      "loss": 0.6246,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5410319566726685,
+      "rewards/margins": 0.8416669964790344,
+      "rewards/rejected": -2.3826990127563477,
       "step": 2210
     },
     {
       "epoch": 1.1854825221608964,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.719092976751575,
       "learning_rate": 7.572475450483098e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2725794315338135,
+      "logits/rejected": -0.20501455664634705,
+      "logps/chosen": -1.894679307937622,
+      "logps/rejected": -2.5304512977600098,
+      "loss": 0.6546,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.894679307937622,
+      "rewards/margins": 0.6357722878456116,
+      "rewards/rejected": -2.5304512977600098,
       "step": 2215
     },
     {
       "epoch": 1.188158554942298,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.282486498688812,
       "learning_rate": 7.559108369108689e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33604344725608826,
+      "logits/rejected": -0.21061226725578308,
+      "logps/chosen": -1.6714082956314087,
+      "logps/rejected": -2.2317934036254883,
+      "loss": 0.6556,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6714082956314087,
+      "rewards/margins": 0.5603851079940796,
+      "rewards/rejected": -2.2317934036254883,
       "step": 2220
     },
     {
       "epoch": 1.1908345877236997,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5532936602858474,
       "learning_rate": 7.54571645925366e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3049870729446411,
+      "logits/rejected": -0.11253170669078827,
+      "logps/chosen": -1.6460866928100586,
+      "logps/rejected": -2.4816412925720215,
+      "loss": 0.6374,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6460866928100586,
+      "rewards/margins": 0.8355547189712524,
+      "rewards/rejected": -2.4816412925720215,
       "step": 2225
     },
     {
       "epoch": 1.1935106205051011,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 20.99984858346219,
       "learning_rate": 7.532299850846378e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29827529191970825,
+      "logits/rejected": -0.16084110736846924,
+      "logps/chosen": -1.689149260520935,
+      "logps/rejected": -2.6077189445495605,
+      "loss": 0.661,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.689149260520935,
+      "rewards/margins": 0.9185697436332703,
+      "rewards/rejected": -2.6077189445495605,
       "step": 2230
     },
     {
       "epoch": 1.1961866532865026,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.0256505697169755,
       "learning_rate": 7.518858674054838e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28648149967193604,
+      "logits/rejected": -0.11803291738033295,
+      "logps/chosen": -1.631546974182129,
+      "logps/rejected": -2.346304178237915,
+      "loss": 0.6328,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.631546974182129,
+      "rewards/margins": 0.7147571444511414,
+      "rewards/rejected": -2.346304178237915,
       "step": 2235
     },
     {
       "epoch": 1.1988626860679044,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.106899180660168,
       "learning_rate": 7.505393059285394e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27103087306022644,
+      "logits/rejected": -0.12051932513713837,
+      "logps/chosen": -1.6719377040863037,
+      "logps/rejected": -2.296389579772949,
+      "loss": 0.6333,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.6719377040863037,
+      "rewards/margins": 0.6244516372680664,
+      "rewards/rejected": -2.296389579772949,
       "step": 2240
     },
     {
       "epoch": 1.2015387188493059,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.221079479762464,
       "learning_rate": 7.491903137181501e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23066358268260956,
+      "logits/rejected": -0.18099571764469147,
+      "logps/chosen": -1.6226431131362915,
+      "logps/rejected": -2.1510086059570312,
+      "loss": 0.6476,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6226431131362915,
+      "rewards/margins": 0.528365433216095,
+      "rewards/rejected": -2.1510086059570312,
       "step": 2245
     },
     {
       "epoch": 1.2042147516307076,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.851408952245976,
       "learning_rate": 7.478389038622441e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.16515859961509705,
+      "logits/rejected": -0.13762590289115906,
+      "logps/chosen": -1.6238981485366821,
+      "logps/rejected": -2.2609801292419434,
+      "loss": 0.6359,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.6238981485366821,
+      "rewards/margins": 0.6370822191238403,
+      "rewards/rejected": -2.2609801292419434,
       "step": 2250
     },
     {
       "epoch": 1.206890784412109,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2682767739808414,
       "learning_rate": 7.46485089472206e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25246661901474,
+      "logits/rejected": -0.1497802734375,
+      "logps/chosen": -1.764852523803711,
+      "logps/rejected": -2.1862080097198486,
+      "loss": 0.6516,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.764852523803711,
+      "rewards/margins": 0.42135539650917053,
+      "rewards/rejected": -2.1862080097198486,
       "step": 2255
     },
     {
       "epoch": 1.2095668171935106,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.853398969715543,
       "learning_rate": 7.451288836827487e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20335189998149872,
+      "logits/rejected": -0.20493245124816895,
+      "logps/chosen": -1.629525899887085,
+      "logps/rejected": -2.021247386932373,
+      "loss": 0.6457,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.629525899887085,
+      "rewards/margins": 0.39172154664993286,
+      "rewards/rejected": -2.021247386932373,
       "step": 2260
     },
     {
       "epoch": 1.2122428499749123,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.031875546615918,
       "learning_rate": 7.437702996517869e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26303616166114807,
+      "logits/rejected": -0.17618155479431152,
+      "logps/chosen": -1.6762644052505493,
+      "logps/rejected": -2.197592258453369,
+      "loss": 0.6494,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6762644052505493,
+      "rewards/margins": 0.5213276743888855,
+      "rewards/rejected": -2.197592258453369,
       "step": 2265
     },
     {
       "epoch": 1.2149188827563138,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.747334732625012,
       "learning_rate": 7.424093505603087e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3505941331386566,
+      "logits/rejected": -0.19671902060508728,
+      "logps/chosen": -1.6550438404083252,
+      "logps/rejected": -2.3592238426208496,
+      "loss": 0.6297,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.6550438404083252,
+      "rewards/margins": 0.7041803598403931,
+      "rewards/rejected": -2.3592238426208496,
       "step": 2270
     },
     {
       "epoch": 1.2175949155377153,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.369863477158843,
       "learning_rate": 7.410460496122482e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24333210289478302,
+      "logits/rejected": -0.12489913403987885,
+      "logps/chosen": -1.6170415878295898,
+      "logps/rejected": -2.3260719776153564,
+      "loss": 0.6207,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.6170415878295898,
+      "rewards/margins": 0.7090303301811218,
+      "rewards/rejected": -2.3260719776153564,
       "step": 2275
     },
     {
       "epoch": 1.220270948319117,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.0900888552120875,
       "learning_rate": 7.396804100343572e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31125661730766296,
+      "logits/rejected": -0.16720952093601227,
+      "logps/chosen": -1.5053658485412598,
+      "logps/rejected": -2.1162755489349365,
+      "loss": 0.638,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5053658485412598,
+      "rewards/margins": 0.6109097599983215,
+      "rewards/rejected": -2.1162755489349365,
       "step": 2280
     },
     {
       "epoch": 1.2229469811005185,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4017647421253456,
       "learning_rate": 7.383124450760768e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23173418641090393,
+      "logits/rejected": -0.05484575778245926,
+      "logps/chosen": -1.721422791481018,
+      "logps/rejected": -2.3558170795440674,
+      "loss": 0.6441,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.721422791481018,
+      "rewards/margins": 0.6343944668769836,
+      "rewards/rejected": -2.3558170795440674,
       "step": 2285
     },
     {
       "epoch": 1.22562301388192,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 10.842044254784575,
       "learning_rate": 7.369421680094091e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3485881984233856,
+      "logits/rejected": -0.20485374331474304,
+      "logps/chosen": -1.58369779586792,
+      "logps/rejected": -2.1650702953338623,
+      "loss": 0.6714,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.58369779586792,
+      "rewards/margins": 0.5813725590705872,
+      "rewards/rejected": -2.1650702953338623,
       "step": 2290
     },
     {
       "epoch": 1.2282990466633217,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.6208229171431325,
       "learning_rate": 7.355695921287881e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2787173390388489,
+      "logits/rejected": -0.19939202070236206,
+      "logps/chosen": -1.707712173461914,
+      "logps/rejected": -2.265514850616455,
+      "loss": 0.6534,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.707712173461914,
+      "rewards/margins": 0.557802677154541,
+      "rewards/rejected": -2.265514850616455,
       "step": 2295
     },
     {
       "epoch": 1.2309750794447232,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.499169812046216,
       "learning_rate": 7.341947307509513e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24184389412403107,
+      "logits/rejected": -0.11580054461956024,
+      "logps/chosen": -1.7309820652008057,
+      "logps/rejected": -2.23922061920166,
+      "loss": 0.6646,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7309820652008057,
+      "rewards/margins": 0.5082387328147888,
+      "rewards/rejected": -2.23922061920166,
       "step": 2300
     },
     {
       "epoch": 1.233651112226125,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3147217103790565,
       "learning_rate": 7.328175972148094e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2898957431316376,
+      "logits/rejected": -0.1566167026758194,
+      "logps/chosen": -1.928009271621704,
+      "logps/rejected": -2.7197189331054688,
+      "loss": 0.6495,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.928009271621704,
+      "rewards/margins": 0.7917099595069885,
+      "rewards/rejected": -2.7197189331054688,
       "step": 2305
     },
     {
       "epoch": 1.2363271450075264,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 15.066933241629924,
       "learning_rate": 7.314382048813185e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21772077679634094,
+      "logits/rejected": 0.034819506108760834,
+      "logps/chosen": -1.750694990158081,
+      "logps/rejected": -2.5712838172912598,
+      "loss": 0.6586,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.750694990158081,
+      "rewards/margins": 0.8205884099006653,
+      "rewards/rejected": -2.5712838172912598,
       "step": 2310
     },
     {
       "epoch": 1.2390031777889279,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.566895692005913,
       "learning_rate": 7.300565671333486e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2326337844133377,
+      "logits/rejected": -0.05129672959446907,
+      "logps/chosen": -1.8421958684921265,
+      "logps/rejected": -2.523092031478882,
+      "loss": 0.6434,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.8421958684921265,
+      "rewards/margins": 0.6808963418006897,
+      "rewards/rejected": -2.523092031478882,
       "step": 2315
     },
     {
       "epoch": 1.2416792105703296,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9501135378056738,
       "learning_rate": 7.286726973755554e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13500699400901794,
+      "logits/rejected": -0.10361503064632416,
+      "logps/chosen": -1.802919626235962,
+      "logps/rejected": -2.3099308013916016,
+      "loss": 0.6632,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.802919626235962,
+      "rewards/margins": 0.5070111751556396,
+      "rewards/rejected": -2.3099308013916016,
       "step": 2320
     },
     {
       "epoch": 1.244355243351731,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9423023298071156,
       "learning_rate": 7.272866090342493e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.12883326411247253,
+      "logits/rejected": -0.05190613120794296,
+      "logps/chosen": -1.7764946222305298,
+      "logps/rejected": -2.480633497238159,
+      "loss": 0.6527,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7764946222305298,
+      "rewards/margins": 0.7041390538215637,
+      "rewards/rejected": -2.480633497238159,
       "step": 2325
     },
     {
       "epoch": 1.2470312761331326,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.591789381588857,
       "learning_rate": 7.258983155572656e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28567036986351013,
+      "logits/rejected": -0.17964895069599152,
+      "logps/chosen": -1.7798715829849243,
+      "logps/rejected": -2.3365001678466797,
+      "loss": 0.6524,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7798715829849243,
+      "rewards/margins": 0.556628406047821,
+      "rewards/rejected": -2.3365001678466797,
       "step": 2330
     },
     {
       "epoch": 1.2497073089145343,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.217697739553699,
       "learning_rate": 7.245078304138335e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1756407767534256,
+      "logits/rejected": -0.11080668866634369,
+      "logps/chosen": -1.833605170249939,
+      "logps/rejected": -2.480989694595337,
+      "loss": 0.6466,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.833605170249939,
+      "rewards/margins": 0.6473843455314636,
+      "rewards/rejected": -2.480989694595337,
       "step": 2335
     },
     {
       "epoch": 1.2523833416959358,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4698697224459916,
       "learning_rate": 7.231151670944462e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3368038833141327,
+      "logits/rejected": -0.16689829528331757,
+      "logps/chosen": -1.8229541778564453,
+      "logps/rejected": -2.4259257316589355,
+      "loss": 0.6607,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8229541778564453,
+      "rewards/margins": 0.6029714345932007,
+      "rewards/rejected": -2.4259257316589355,
       "step": 2340
     },
     {
       "epoch": 1.2550593744773373,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.6459537157453346,
       "learning_rate": 7.217203391107291e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27212661504745483,
+      "logits/rejected": -0.10502076148986816,
+      "logps/chosen": -1.7439134120941162,
+      "logps/rejected": -2.574840545654297,
+      "loss": 0.6361,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7439134120941162,
+      "rewards/margins": 0.8309271931648254,
+      "rewards/rejected": -2.574840545654297,
       "step": 2345
     },
     {
       "epoch": 1.257735407258739,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.13477837136375,
       "learning_rate": 7.203233599953096e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26379159092903137,
+      "logits/rejected": -0.1222984567284584,
+      "logps/chosen": -1.8122615814208984,
+      "logps/rejected": -2.344892978668213,
+      "loss": 0.6576,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.8122615814208984,
+      "rewards/margins": 0.532631516456604,
+      "rewards/rejected": -2.344892978668213,
       "step": 2350
     },
     {
       "epoch": 1.2604114400401405,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.506287477512618,
       "learning_rate": 7.189242433016852e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.202241450548172,
+      "logits/rejected": -0.06128978729248047,
+      "logps/chosen": -1.6434932947158813,
+      "logps/rejected": -2.535926103591919,
+      "loss": 0.6365,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6434932947158813,
+      "rewards/margins": 0.892432689666748,
+      "rewards/rejected": -2.535926103591919,
       "step": 2355
     },
     {
       "epoch": 1.263087472821542,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.442128027373577,
       "learning_rate": 7.17523002604092e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22096750140190125,
+      "logits/rejected": -0.08924611657857895,
+      "logps/chosen": -1.756000280380249,
+      "logps/rejected": -2.598994731903076,
+      "loss": 0.6353,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.756000280380249,
+      "rewards/margins": 0.842994213104248,
+      "rewards/rejected": -2.598994731903076,
       "step": 2360
     },
     {
       "epoch": 1.2657635056029437,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6078270729241515,
       "learning_rate": 7.161196514973734e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19537633657455444,
+      "logits/rejected": -0.06731315702199936,
+      "logps/chosen": -1.742647409439087,
+      "logps/rejected": -2.4969260692596436,
+      "loss": 0.6452,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.742647409439087,
+      "rewards/margins": 0.7542788982391357,
+      "rewards/rejected": -2.4969260692596436,
       "step": 2365
     },
     {
       "epoch": 1.2684395383843452,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9613826830065477,
       "learning_rate": 7.147142035968483e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17859426140785217,
+      "logits/rejected": -0.05371304601430893,
+      "logps/chosen": -1.7746015787124634,
+      "logps/rejected": -2.3535525798797607,
+      "loss": 0.6429,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7746015787124634,
+      "rewards/margins": 0.5789510011672974,
+      "rewards/rejected": -2.3535525798797607,
       "step": 2370
     },
     {
       "epoch": 1.2711155711657467,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0452657804866865,
       "learning_rate": 7.133066725381781e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.340522825717926,
+      "logits/rejected": -0.16956861317157745,
+      "logps/chosen": -1.6016219854354858,
+      "logps/rejected": -2.131476879119873,
+      "loss": 0.6559,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6016219854354858,
+      "rewards/margins": 0.5298551321029663,
+      "rewards/rejected": -2.131476879119873,
       "step": 2375
     },
     {
       "epoch": 1.2737916039471484,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.350082894384947,
       "learning_rate": 7.118970719772354e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2894574999809265,
+      "logits/rejected": -0.10081624984741211,
+      "logps/chosen": -1.8005459308624268,
+      "logps/rejected": -2.5298967361450195,
+      "loss": 0.6319,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.8005459308624268,
+      "rewards/margins": 0.7293506264686584,
+      "rewards/rejected": -2.5298967361450195,
       "step": 2380
     },
     {
       "epoch": 1.27646763672855,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.220587825857154,
       "learning_rate": 7.104854155899711e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.18031558394432068,
+      "logits/rejected": -0.07565481960773468,
+      "logps/chosen": -1.7997602224349976,
+      "logps/rejected": -2.4117515087127686,
+      "loss": 0.654,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.7997602224349976,
+      "rewards/margins": 0.6119914650917053,
+      "rewards/rejected": -2.4117515087127686,
       "step": 2385
     },
     {
       "epoch": 1.2791436695099514,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9313468190573047,
       "learning_rate": 7.090717170722817e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19649623334407806,
+      "logits/rejected": -0.12767013907432556,
+      "logps/chosen": -1.7550485134124756,
+      "logps/rejected": -2.481353282928467,
+      "loss": 0.6338,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7550485134124756,
+      "rewards/margins": 0.7263044714927673,
+      "rewards/rejected": -2.481353282928467,
       "step": 2390
     },
     {
       "epoch": 1.2818197022913531,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.468021410476975,
       "learning_rate": 7.076559901398762e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3531740605831146,
+      "logits/rejected": -0.23127298057079315,
+      "logps/chosen": -1.6182587146759033,
+      "logps/rejected": -2.2782835960388184,
+      "loss": 0.6536,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6182587146759033,
+      "rewards/margins": 0.6600251793861389,
+      "rewards/rejected": -2.2782835960388184,
       "step": 2395
     },
     {
       "epoch": 1.2844957350727546,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.813671453879912,
       "learning_rate": 7.062382485281436e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21747584640979767,
+      "logits/rejected": -0.10379443317651749,
+      "logps/chosen": -1.6565605401992798,
+      "logps/rejected": -2.202540636062622,
+      "loss": 0.6581,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.6565605401992798,
+      "rewards/margins": 0.5459800958633423,
+      "rewards/rejected": -2.202540636062622,
       "step": 2400
     },
     {
       "epoch": 1.2844957350727546,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.6413,
-      "eval_samples_per_second": 38.826,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.728,
+      "eval_logits/chosen": 0.06444454938173294,
+      "eval_logits/rejected": 0.15299274027347565,
+      "eval_logps/chosen": -1.7443679571151733,
+      "eval_logps/rejected": -2.3860929012298584,
+      "eval_loss": 0.6532934308052063,
+      "eval_rewards/accuracies": 0.6691394448280334,
+      "eval_rewards/chosen": -1.7443679571151733,
+      "eval_rewards/margins": 0.6417250633239746,
+      "eval_rewards/rejected": -2.3860929012298584,
+      "eval_runtime": 40.4208,
+      "eval_samples_per_second": 33.275,
+      "eval_steps_per_second": 8.337,
       "step": 2400
     },
     {
       "epoch": 1.287171767854156,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2930909091704157,
       "learning_rate": 7.048185059920193e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20422916114330292,
+      "logits/rejected": -0.0641867145895958,
+      "logps/chosen": -1.7080166339874268,
+      "logps/rejected": -2.477492094039917,
+      "loss": 0.6427,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7080166339874268,
+      "rewards/margins": 0.7694756388664246,
+      "rewards/rejected": -2.477492094039917,
       "step": 2405
     },
     {
       "epoch": 1.2898478006355578,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2778774804071125,
       "learning_rate": 7.033967763058516e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3471071422100067,
+      "logits/rejected": -0.15714314579963684,
+      "logps/chosen": -1.7580512762069702,
+      "logps/rejected": -2.132215976715088,
+      "loss": 0.6661,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7580512762069702,
+      "rewards/margins": 0.37416478991508484,
+      "rewards/rejected": -2.132215976715088,
       "step": 2410
     },
     {
       "epoch": 1.2925238334169593,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2564212805925123,
       "learning_rate": 7.019730732632681e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.18613675236701965,
+      "logits/rejected": -0.1197533831000328,
+      "logps/chosen": -1.6032402515411377,
+      "logps/rejected": -2.400292158126831,
+      "loss": 0.629,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6032402515411377,
+      "rewards/margins": 0.7970517873764038,
+      "rewards/rejected": -2.400292158126831,
       "step": 2415
     },
     {
       "epoch": 1.2951998661983608,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.072100366566522,
       "learning_rate": 7.005474106770418e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30652913451194763,
+      "logits/rejected": -0.18551388382911682,
+      "logps/chosen": -1.75790536403656,
+      "logps/rejected": -2.449946403503418,
+      "loss": 0.6292,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.75790536403656,
+      "rewards/margins": 0.6920409202575684,
+      "rewards/rejected": -2.449946403503418,
       "step": 2420
     },
     {
       "epoch": 1.2978758989797625,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.1438433049543,
       "learning_rate": 6.991198023789577e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20296776294708252,
+      "logits/rejected": -0.12820060551166534,
+      "logps/chosen": -1.5569179058074951,
+      "logps/rejected": -2.1158840656280518,
+      "loss": 0.6394,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.5569179058074951,
+      "rewards/margins": 0.5589659810066223,
+      "rewards/rejected": -2.1158840656280518,
       "step": 2425
     },
     {
       "epoch": 1.300551931761164,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.062615881415599,
       "learning_rate": 6.976902622196776e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1964765340089798,
+      "logits/rejected": -0.13950295746326447,
+      "logps/chosen": -1.8566795587539673,
+      "logps/rejected": -2.4052553176879883,
+      "loss": 0.6603,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.8566795587539673,
+      "rewards/margins": 0.5485759377479553,
+      "rewards/rejected": -2.4052553176879883,
       "step": 2430
     },
     {
       "epoch": 1.3032279645425655,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.953931878950312,
       "learning_rate": 6.962588040686064e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19939246773719788,
+      "logits/rejected": -0.07737383991479874,
+      "logps/chosen": -1.732039451599121,
+      "logps/rejected": -2.2010560035705566,
+      "loss": 0.6559,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.732039451599121,
+      "rewards/margins": 0.4690164625644684,
+      "rewards/rejected": -2.2010560035705566,
       "step": 2435
     },
     {
       "epoch": 1.3059039973239672,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6481843877111553,
       "learning_rate": 6.948254418137573e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31365376710891724,
+      "logits/rejected": -0.2086184024810791,
+      "logps/chosen": -1.7891178131103516,
+      "logps/rejected": -2.443114757537842,
+      "loss": 0.6565,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7891178131103516,
+      "rewards/margins": 0.6539971232414246,
+      "rewards/rejected": -2.443114757537842,
       "step": 2440
     },
     {
       "epoch": 1.3085800301053687,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3477607163207255,
       "learning_rate": 6.933901893616174e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27392154932022095,
+      "logits/rejected": -0.1394931972026825,
+      "logps/chosen": -1.851259469985962,
+      "logps/rejected": -2.3668293952941895,
+      "loss": 0.6543,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.851259469985962,
+      "rewards/margins": 0.5155699849128723,
+      "rewards/rejected": -2.3668293952941895,
       "step": 2445
     },
     {
       "epoch": 1.3112560628867704,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.615834162952317,
       "learning_rate": 6.919530606370121e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.252876877784729,
+      "logits/rejected": -0.11027371883392334,
+      "logps/chosen": -1.795531988143921,
+      "logps/rejected": -2.558276653289795,
+      "loss": 0.644,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.795531988143921,
+      "rewards/margins": 0.7627445459365845,
+      "rewards/rejected": -2.558276653289795,
       "step": 2450
     },
     {
       "epoch": 1.313932095668172,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.5160320451891758,
       "learning_rate": 6.905140695829706e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29107701778411865,
+      "logits/rejected": -0.06807577610015869,
+      "logps/chosen": -1.8059475421905518,
+      "logps/rejected": -2.6131606101989746,
+      "loss": 0.6411,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8059475421905518,
+      "rewards/margins": 0.8072134256362915,
+      "rewards/rejected": -2.6131606101989746,
       "step": 2455
     },
     {
       "epoch": 1.3166081284495736,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.154784546094847,
       "learning_rate": 6.890732301605904e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2287755012512207,
+      "logits/rejected": -0.1226111426949501,
+      "logps/chosen": -1.8170026540756226,
+      "logps/rejected": -2.309248685836792,
+      "loss": 0.6462,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.8170026540756226,
+      "rewards/margins": 0.492246150970459,
+      "rewards/rejected": -2.309248685836792,
       "step": 2460
     },
     {
       "epoch": 1.3192841612309751,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0284237255427175,
       "learning_rate": 6.876305563489021e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21846452355384827,
+      "logits/rejected": -0.14045235514640808,
+      "logps/chosen": -1.796626329421997,
+      "logps/rejected": -2.5444130897521973,
+      "loss": 0.6464,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.796626329421997,
+      "rewards/margins": 0.7477869391441345,
+      "rewards/rejected": -2.5444130897521973,
       "step": 2465
     },
     {
       "epoch": 1.3219601940123766,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.756351606073223,
       "learning_rate": 6.861860621447331e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3606422543525696,
+      "logits/rejected": -0.24150483310222626,
+      "logps/chosen": -1.7419849634170532,
+      "logps/rejected": -2.324956178665161,
+      "loss": 0.6528,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7419849634170532,
+      "rewards/margins": 0.5829712152481079,
+      "rewards/rejected": -2.324956178665161,
       "step": 2470
     },
     {
       "epoch": 1.3246362267937783,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8686696284502426,
       "learning_rate": 6.847397615625725e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23994198441505432,
+      "logits/rejected": -0.17828592658042908,
+      "logps/chosen": -1.7745647430419922,
+      "logps/rejected": -2.4297540187835693,
+      "loss": 0.645,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7745647430419922,
+      "rewards/margins": 0.6551893353462219,
+      "rewards/rejected": -2.4297540187835693,
       "step": 2475
     },
     {
       "epoch": 1.3273122595751798,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.348271317768281,
       "learning_rate": 6.83291668634435e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3656330406665802,
+      "logits/rejected": -0.2055610716342926,
+      "logps/chosen": -1.7889690399169922,
+      "logps/rejected": -2.559612512588501,
+      "loss": 0.631,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7889690399169922,
+      "rewards/margins": 0.7706435918807983,
+      "rewards/rejected": -2.559612512588501,
       "step": 2480
     },
     {
       "epoch": 1.3299882923565813,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4457932541465297,
       "learning_rate": 6.818417974097246e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17108380794525146,
+      "logits/rejected": -0.009906845167279243,
+      "logps/chosen": -1.746321678161621,
+      "logps/rejected": -2.568065643310547,
+      "loss": 0.6527,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.746321678161621,
+      "rewards/margins": 0.8217439651489258,
+      "rewards/rejected": -2.568065643310547,
       "step": 2485
     },
     {
       "epoch": 1.332664325137983,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.286487296499313,
       "learning_rate": 6.803901619550981e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2997012734413147,
+      "logits/rejected": -0.2451844960451126,
+      "logps/chosen": -1.8361232280731201,
+      "logps/rejected": -2.42592716217041,
+      "loss": 0.6444,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8361232280731201,
+      "rewards/margins": 0.5898037552833557,
+      "rewards/rejected": -2.42592716217041,
       "step": 2490
     },
     {
       "epoch": 1.3353403579193845,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.156797697564029,
       "learning_rate": 6.789367763543292e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21002598106861115,
+      "logits/rejected": -0.19391609728336334,
+      "logps/chosen": -1.7901356220245361,
+      "logps/rejected": -2.3645081520080566,
+      "loss": 0.6519,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7901356220245361,
+      "rewards/margins": 0.5743725895881653,
+      "rewards/rejected": -2.3645081520080566,
       "step": 2495
     },
     {
       "epoch": 1.338016390700786,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.127954726212114,
       "learning_rate": 6.774816547081714e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21312396228313446,
+      "logits/rejected": -0.07053796201944351,
+      "logps/chosen": -1.7219873666763306,
+      "logps/rejected": -2.3269619941711426,
+      "loss": 0.6508,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7219873666763306,
+      "rewards/margins": 0.604974627494812,
+      "rewards/rejected": -2.3269619941711426,
       "step": 2500
     },
     {
       "epoch": 1.3406924234821878,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6087588370906456,
       "learning_rate": 6.760248111342211e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22121405601501465,
+      "logits/rejected": -0.06345383077859879,
+      "logps/chosen": -1.664574384689331,
+      "logps/rejected": -2.3759021759033203,
+      "loss": 0.641,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.664574384689331,
+      "rewards/margins": 0.7113279700279236,
+      "rewards/rejected": -2.3759021759033203,
       "step": 2505
     },
     {
       "epoch": 1.3433684562635893,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 10.249612405111915,
       "learning_rate": 6.745662597667813e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30030661821365356,
+      "logits/rejected": -0.17661242187023163,
+      "logps/chosen": -1.639116644859314,
+      "logps/rejected": -2.321545362472534,
+      "loss": 0.6376,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.639116644859314,
+      "rewards/margins": 0.6824290156364441,
+      "rewards/rejected": -2.321545362472534,
       "step": 2510
     },
     {
       "epoch": 1.3460444890449907,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.7183861484392768,
       "learning_rate": 6.731060147567236e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22120817005634308,
+      "logits/rejected": -0.11798825114965439,
+      "logps/chosen": -1.7695461511611938,
+      "logps/rejected": -2.413296937942505,
+      "loss": 0.6556,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7695461511611938,
+      "rewards/margins": 0.6437507271766663,
+      "rewards/rejected": -2.413296937942505,
       "step": 2515
     },
     {
       "epoch": 1.3487205218263925,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8940408412012624,
       "learning_rate": 6.716440902713515e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3133282959461212,
+      "logits/rejected": -0.23770251870155334,
+      "logps/chosen": -1.7753585577011108,
+      "logps/rejected": -2.304100275039673,
+      "loss": 0.6551,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.7753585577011108,
+      "rewards/margins": 0.5287418365478516,
+      "rewards/rejected": -2.304100275039673,
       "step": 2520
     },
     {
       "epoch": 1.351396554607794,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.581228343801736,
       "learning_rate": 6.701805004942627e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29815781116485596,
+      "logits/rejected": -0.23119676113128662,
+      "logps/chosen": -1.7896652221679688,
+      "logps/rejected": -2.4759063720703125,
+      "loss": 0.6324,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7896652221679688,
+      "rewards/margins": 0.6862412095069885,
+      "rewards/rejected": -2.4759063720703125,
       "step": 2525
     },
     {
       "epoch": 1.3540725873891954,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.676718501107866,
       "learning_rate": 6.687152596252119e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31035998463630676,
+      "logits/rejected": -0.2606969177722931,
+      "logps/chosen": -1.8071067333221436,
+      "logps/rejected": -2.2726917266845703,
+      "loss": 0.6653,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.8071067333221436,
+      "rewards/margins": 0.46558499336242676,
+      "rewards/rejected": -2.2726917266845703,
       "step": 2530
     },
     {
       "epoch": 1.3567486201705972,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.129110634203399,
       "learning_rate": 6.672483818799722e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3664632737636566,
+      "logits/rejected": -0.22831162810325623,
+      "logps/chosen": -1.6874622106552124,
+      "logps/rejected": -2.3388257026672363,
+      "loss": 0.654,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6874622106552124,
+      "rewards/margins": 0.651363730430603,
+      "rewards/rejected": -2.3388257026672363,
       "step": 2535
     },
     {
       "epoch": 1.3594246529519987,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 13.874159193367435,
       "learning_rate": 6.657798814901978e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2834777235984802,
+      "logits/rejected": -0.12068116664886475,
+      "logps/chosen": -1.9264991283416748,
+      "logps/rejected": -2.3612332344055176,
+      "loss": 0.6642,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.9264991283416748,
+      "rewards/margins": 0.43473386764526367,
+      "rewards/rejected": -2.3612332344055176,
       "step": 2540
     },
     {
       "epoch": 1.3621006857334002,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9086149982270437,
       "learning_rate": 6.643097727032863e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26425299048423767,
+      "logits/rejected": -0.09231270849704742,
+      "logps/chosen": -1.7324844598770142,
+      "logps/rejected": -2.5158865451812744,
+      "loss": 0.6468,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7324844598770142,
+      "rewards/margins": 0.7834024429321289,
+      "rewards/rejected": -2.5158865451812744,
       "step": 2545
     },
     {
       "epoch": 1.3647767185148019,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.49613954856616,
       "learning_rate": 6.628380697822392e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2774287164211273,
+      "logits/rejected": -0.12992632389068604,
+      "logps/chosen": -1.8103183507919312,
+      "logps/rejected": -2.2813520431518555,
+      "loss": 0.6583,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.8103183507919312,
+      "rewards/margins": 0.47103390097618103,
+      "rewards/rejected": -2.2813520431518555,
       "step": 2550
     },
     {
       "epoch": 1.3674527512962034,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.640954255450204,
       "learning_rate": 6.61364787005525e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23260538280010223,
+      "logits/rejected": -0.1453295201063156,
+      "logps/chosen": -1.6589101552963257,
+      "logps/rejected": -2.5150787830352783,
+      "loss": 0.6335,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6589101552963257,
+      "rewards/margins": 0.8561686277389526,
+      "rewards/rejected": -2.5150787830352783,
       "step": 2555
     },
     {
       "epoch": 1.3701287840776049,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7617067933650183,
       "learning_rate": 6.598899386669395e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23766915500164032,
+      "logits/rejected": -0.131187304854393,
+      "logps/chosen": -1.781348466873169,
+      "logps/rejected": -2.421043872833252,
+      "loss": 0.6471,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.781348466873169,
+      "rewards/margins": 0.6396957635879517,
+      "rewards/rejected": -2.421043872833252,
       "step": 2560
     },
     {
       "epoch": 1.3728048168590066,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.539480690974514,
       "learning_rate": 6.584135390754679e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25783127546310425,
+      "logits/rejected": -0.1402159333229065,
+      "logps/chosen": -1.8040361404418945,
+      "logps/rejected": -2.56550931930542,
+      "loss": 0.6477,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8040361404418945,
+      "rewards/margins": 0.7614732980728149,
+      "rewards/rejected": -2.56550931930542,
       "step": 2565
     },
     {
       "epoch": 1.375480849640408,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.514412136804674,
       "learning_rate": 6.569356025551454e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2026633769273758,
+      "logits/rejected": -0.13103753328323364,
+      "logps/chosen": -1.745004653930664,
+      "logps/rejected": -2.4098191261291504,
+      "loss": 0.6483,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.745004653930664,
+      "rewards/margins": 0.6648143529891968,
+      "rewards/rejected": -2.4098191261291504,
       "step": 2570
     },
     {
       "epoch": 1.3781568824218096,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.310789839258217,
       "learning_rate": 6.554561434449186e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3188205063343048,
+      "logits/rejected": -0.17965996265411377,
+      "logps/chosen": -1.7267307043075562,
+      "logps/rejected": -2.4178781509399414,
+      "loss": 0.6476,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7267307043075562,
+      "rewards/margins": 0.6911474466323853,
+      "rewards/rejected": -2.4178781509399414,
       "step": 2575
     },
     {
       "epoch": 1.3808329152032113,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.18253447147648,
       "learning_rate": 6.539751760985063e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24876229465007782,
+      "logits/rejected": -0.16214443743228912,
+      "logps/chosen": -1.7502638101577759,
+      "logps/rejected": -2.1803629398345947,
+      "loss": 0.6515,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7502638101577759,
+      "rewards/margins": 0.4300991892814636,
+      "rewards/rejected": -2.1803629398345947,
       "step": 2580
     },
     {
       "epoch": 1.3835089479846128,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 10.404966168029356,
       "learning_rate": 6.524927148842602e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15811878442764282,
+      "logits/rejected": -0.01163451373577118,
+      "logps/chosen": -1.662668228149414,
+      "logps/rejected": -2.353851079940796,
+      "loss": 0.6276,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.662668228149414,
+      "rewards/margins": 0.6911830902099609,
+      "rewards/rejected": -2.353851079940796,
       "step": 2585
     },
     {
       "epoch": 1.3861849807660143,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.555108056643559,
       "learning_rate": 6.510087741850254e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26308685541152954,
+      "logits/rejected": -0.13633175194263458,
+      "logps/chosen": -1.6702969074249268,
+      "logps/rejected": -2.2908263206481934,
+      "loss": 0.6486,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6702969074249268,
+      "rewards/margins": 0.620529294013977,
+      "rewards/rejected": -2.2908263206481934,
       "step": 2590
     },
     {
       "epoch": 1.388861013547416,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.992735933140492,
       "learning_rate": 6.495233683980012e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24364066123962402,
+      "logits/rejected": -0.207584410905838,
+      "logps/chosen": -1.7725086212158203,
+      "logps/rejected": -2.2910594940185547,
+      "loss": 0.6563,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.7725086212158203,
+      "rewards/margins": 0.5185508131980896,
+      "rewards/rejected": -2.2910594940185547,
       "step": 2595
     },
     {
       "epoch": 1.3915370463288175,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5411936614103667,
       "learning_rate": 6.480365119346011e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15943028032779694,
+      "logits/rejected": -0.04367305710911751,
+      "logps/chosen": -1.7259521484375,
+      "logps/rejected": -2.209993839263916,
+      "loss": 0.6518,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7259521484375,
+      "rewards/margins": 0.4840417802333832,
+      "rewards/rejected": -2.209993839263916,
       "step": 2600
     },
     {
       "epoch": 1.394213079110219,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0814659297496303,
       "learning_rate": 6.465482192203129e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15056212246418,
+      "logits/rejected": -0.09472917765378952,
+      "logps/chosen": -1.7878143787384033,
+      "logps/rejected": -2.3382906913757324,
+      "loss": 0.6571,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7878143787384033,
+      "rewards/margins": 0.5504765510559082,
+      "rewards/rejected": -2.3382906913757324,
       "step": 2605
     },
     {
       "epoch": 1.3968891118916207,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.160412298886795,
       "learning_rate": 6.45058504694559e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15479031205177307,
+      "logits/rejected": -0.08214214444160461,
+      "logps/chosen": -1.7428840398788452,
+      "logps/rejected": -2.331597089767456,
+      "loss": 0.6528,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.7428840398788452,
+      "rewards/margins": 0.5887130498886108,
+      "rewards/rejected": -2.331597089767456,
       "step": 2610
     },
     {
       "epoch": 1.3995651446730222,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.4120817727238295,
       "learning_rate": 6.435673828105564e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26009640097618103,
+      "logits/rejected": -0.13040851056575775,
+      "logps/chosen": -1.6906667947769165,
+      "logps/rejected": -2.451883554458618,
+      "loss": 0.6362,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.6906667947769165,
+      "rewards/margins": 0.7612167596817017,
+      "rewards/rejected": -2.451883554458618,
       "step": 2615
     },
     {
       "epoch": 1.402241177454424,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.395231691948201,
       "learning_rate": 6.420748680351763e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2514781653881073,
+      "logits/rejected": -0.26113006472587585,
+      "logps/chosen": -1.7800700664520264,
+      "logps/rejected": -2.1782071590423584,
+      "loss": 0.6595,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.7800700664520264,
+      "rewards/margins": 0.3981374204158783,
+      "rewards/rejected": -2.1782071590423584,
       "step": 2620
     },
     {
       "epoch": 1.4049172102358254,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.128482537350947,
       "learning_rate": 6.405809748488032e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23293261229991913,
+      "logits/rejected": -0.08996725082397461,
+      "logps/chosen": -1.8136482238769531,
+      "logps/rejected": -2.443168878555298,
+      "loss": 0.6502,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.8136482238769531,
+      "rewards/margins": 0.6295205354690552,
+      "rewards/rejected": -2.443168878555298,
       "step": 2625
     },
     {
       "epoch": 1.4075932430172269,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.07666378837188,
       "learning_rate": 6.390857177451956e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.37709859013557434,
+      "logits/rejected": -0.1935141235589981,
+      "logps/chosen": -1.771398901939392,
+      "logps/rejected": -2.5137712955474854,
+      "loss": 0.6636,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.771398901939392,
+      "rewards/margins": 0.7423726320266724,
+      "rewards/rejected": -2.5137712955474854,
       "step": 2630
     },
     {
       "epoch": 1.4102692757986286,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.099606861540803,
       "learning_rate": 6.375891112313445e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.316692590713501,
+      "logits/rejected": -0.24023547768592834,
+      "logps/chosen": -1.7382618188858032,
+      "logps/rejected": -2.4512152671813965,
+      "loss": 0.6449,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7382618188858032,
+      "rewards/margins": 0.712953507900238,
+      "rewards/rejected": -2.4512152671813965,
       "step": 2635
     },
     {
       "epoch": 1.41294530858003,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.539035189643531,
       "learning_rate": 6.360911698273326e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2363899052143097,
+      "logits/rejected": -0.15972241759300232,
+      "logps/chosen": -1.8156410455703735,
+      "logps/rejected": -2.361262798309326,
+      "loss": 0.6569,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.8156410455703735,
+      "rewards/margins": 0.5456218719482422,
+      "rewards/rejected": -2.361262798309326,
       "step": 2640
     },
     {
       "epoch": 1.4156213413614318,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.198020864457346,
       "learning_rate": 6.345919080661944e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26558759808540344,
+      "logits/rejected": -0.18720869719982147,
+      "logps/chosen": -1.7317609786987305,
+      "logps/rejected": -2.4149932861328125,
+      "loss": 0.6467,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7317609786987305,
+      "rewards/margins": 0.6832322478294373,
+      "rewards/rejected": -2.4149932861328125,
       "step": 2645
     },
     {
       "epoch": 1.4182973741428333,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0000568117130304,
       "learning_rate": 6.330913404937737e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3219813108444214,
+      "logits/rejected": -0.18093791604042053,
+      "logps/chosen": -1.7924048900604248,
+      "logps/rejected": -2.690263271331787,
+      "loss": 0.6574,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7924048900604248,
+      "rewards/margins": 0.8978585004806519,
+      "rewards/rejected": -2.690263271331787,
       "step": 2650
     },
     {
       "epoch": 1.4209734069242348,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.391385467316748,
       "learning_rate": 6.315894816685838e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26562362909317017,
+      "logits/rejected": -0.12827429175376892,
+      "logps/chosen": -1.6384748220443726,
+      "logps/rejected": -2.2689034938812256,
+      "loss": 0.6387,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6384748220443726,
+      "rewards/margins": 0.6304289102554321,
+      "rewards/rejected": -2.2689034938812256,
       "step": 2655
     },
     {
       "epoch": 1.4236494397056365,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6052175295137556,
       "learning_rate": 6.300863461616657e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19047358632087708,
+      "logits/rejected": -0.15413883328437805,
+      "logps/chosen": -1.651698112487793,
+      "logps/rejected": -2.405064344406128,
+      "loss": 0.6428,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.651698112487793,
+      "rewards/margins": 0.7533665895462036,
+      "rewards/rejected": -2.405064344406128,
       "step": 2660
     },
     {
       "epoch": 1.426325472487038,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.251379647365575,
       "learning_rate": 6.285819485564465e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3384145200252533,
+      "logits/rejected": -0.2251162827014923,
+      "logps/chosen": -1.7945671081542969,
+      "logps/rejected": -2.537292003631592,
+      "loss": 0.6354,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7945671081542969,
+      "rewards/margins": 0.7427248358726501,
+      "rewards/rejected": -2.537292003631592,
       "step": 2665
     },
     {
       "epoch": 1.4290015052684395,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.785573181370322,
       "learning_rate": 6.270763034485986e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1997223049402237,
+      "logits/rejected": -0.11220314353704453,
+      "logps/chosen": -1.8296387195587158,
+      "logps/rejected": -2.411137104034424,
+      "loss": 0.644,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8296387195587158,
+      "rewards/margins": 0.5814980864524841,
+      "rewards/rejected": -2.411137104034424,
       "step": 2670
     },
     {
       "epoch": 1.4316775380498412,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.774749873358035,
       "learning_rate": 6.255694254458972e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2656470835208893,
+      "logits/rejected": -0.1130392923951149,
+      "logps/chosen": -1.9110164642333984,
+      "logps/rejected": -2.334594249725342,
+      "loss": 0.6621,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9110164642333984,
+      "rewards/margins": 0.4235778748989105,
+      "rewards/rejected": -2.334594249725342,
       "step": 2675
     },
     {
       "epoch": 1.4343535708312427,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.403943401830967,
       "learning_rate": 6.240613291680795e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29431331157684326,
+      "logits/rejected": -0.13483694195747375,
+      "logps/chosen": -1.730508804321289,
+      "logps/rejected": -2.375779628753662,
+      "loss": 0.656,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.730508804321289,
+      "rewards/margins": 0.6452709436416626,
+      "rewards/rejected": -2.375779628753662,
       "step": 2680
     },
     {
       "epoch": 1.4370296036126442,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.518624044451933,
       "learning_rate": 6.225520292467021e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29283085465431213,
+      "logits/rejected": -0.10865583270788193,
+      "logps/chosen": -1.7070003747940063,
+      "logps/rejected": -2.3367650508880615,
+      "loss": 0.6518,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7070003747940063,
+      "rewards/margins": 0.6297644972801208,
+      "rewards/rejected": -2.3367650508880615,
       "step": 2685
     },
     {
       "epoch": 1.439705636394046,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.8354303600440884,
       "learning_rate": 6.210415403249993e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.39326462149620056,
+      "logits/rejected": -0.16955551505088806,
+      "logps/chosen": -1.7571204900741577,
+      "logps/rejected": -2.596787452697754,
+      "loss": 0.64,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.7571204900741577,
+      "rewards/margins": 0.8396671414375305,
+      "rewards/rejected": -2.596787452697754,
       "step": 2690
     },
     {
       "epoch": 1.4423816691754474,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2817661680057877,
       "learning_rate": 6.195298770577415e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2121725082397461,
+      "logits/rejected": -0.18434196710586548,
+      "logps/chosen": -1.7174314260482788,
+      "logps/rejected": -2.4146575927734375,
+      "loss": 0.6548,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7174314260482788,
+      "rewards/margins": 0.6972262263298035,
+      "rewards/rejected": -2.4146575927734375,
       "step": 2695
     },
     {
       "epoch": 1.445057701956849,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.387559609812957,
       "learning_rate": 6.180170541110923e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3020084500312805,
+      "logits/rejected": -0.1298791468143463,
+      "logps/chosen": -1.8188788890838623,
+      "logps/rejected": -2.406090259552002,
+      "loss": 0.6564,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.8188788890838623,
+      "rewards/margins": 0.5872111320495605,
+      "rewards/rejected": -2.406090259552002,
       "step": 2700
     },
     {
       "epoch": 1.4477337347382506,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.345593080632254,
       "learning_rate": 6.165030861624663e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3078783452510834,
+      "logits/rejected": -0.11023547500371933,
+      "logps/chosen": -1.6492578983306885,
+      "logps/rejected": -2.753251314163208,
+      "loss": 0.6253,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.6492578983306885,
+      "rewards/margins": 1.1039937734603882,
+      "rewards/rejected": -2.753251314163208,
       "step": 2705
     },
     {
       "epoch": 1.4504097675196521,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0128318476677247,
       "learning_rate": 6.149879879003876e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19196617603302002,
+      "logits/rejected": -0.16649788618087769,
+      "logps/chosen": -1.7526260614395142,
+      "logps/rejected": -2.351362943649292,
+      "loss": 0.6379,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7526260614395142,
+      "rewards/margins": 0.5987369418144226,
+      "rewards/rejected": -2.351362943649292,
       "step": 2710
     },
     {
       "epoch": 1.4530858003010536,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.095745684090834,
       "learning_rate": 6.13471774024346e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3329054117202759,
+      "logits/rejected": -0.23338384926319122,
+      "logps/chosen": -1.5861828327178955,
+      "logps/rejected": -2.19793438911438,
+      "loss": 0.6323,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.5861828327178955,
+      "rewards/margins": 0.6117514967918396,
+      "rewards/rejected": -2.19793438911438,
       "step": 2715
     },
     {
       "epoch": 1.4557618330824553,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.486532353735225,
       "learning_rate": 6.119544592446551e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2918776869773865,
+      "logits/rejected": -0.18322092294692993,
+      "logps/chosen": -1.6861388683319092,
+      "logps/rejected": -2.1504018306732178,
+      "loss": 0.6517,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6861388683319092,
+      "rewards/margins": 0.46426311135292053,
+      "rewards/rejected": -2.1504018306732178,
       "step": 2720
     },
     {
       "epoch": 1.4584378658638568,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8935251355263265,
       "learning_rate": 6.104360582823096e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24642066657543182,
+      "logits/rejected": -0.14998921751976013,
+      "logps/chosen": -1.6713050603866577,
+      "logps/rejected": -2.299609661102295,
+      "loss": 0.6371,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6713050603866577,
+      "rewards/margins": 0.6283043622970581,
+      "rewards/rejected": -2.299609661102295,
       "step": 2725
     },
     {
       "epoch": 1.4611138986452583,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.716992321069855,
       "learning_rate": 6.089165858688423e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2875072956085205,
+      "logits/rejected": -0.12147003412246704,
+      "logps/chosen": -1.6491838693618774,
+      "logps/rejected": -2.5092873573303223,
+      "loss": 0.6339,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6491838693618774,
+      "rewards/margins": 0.8601036071777344,
+      "rewards/rejected": -2.5092873573303223,
       "step": 2730
     },
     {
       "epoch": 1.46378993142666,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.642721008135586,
       "learning_rate": 6.073960567461811e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28647851943969727,
+      "logits/rejected": -0.09984248876571655,
+      "logps/chosen": -1.5842465162277222,
+      "logps/rejected": -2.3439853191375732,
+      "loss": 0.6352,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5842465162277222,
+      "rewards/margins": 0.7597386240959167,
+      "rewards/rejected": -2.3439853191375732,
       "step": 2735
     },
     {
       "epoch": 1.4664659642080615,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.407511033249579,
       "learning_rate": 6.058744856665065e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2942974269390106,
+      "logits/rejected": -0.19632267951965332,
+      "logps/chosen": -1.6867074966430664,
+      "logps/rejected": -2.6290409564971924,
+      "loss": 0.6361,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.6867074966430664,
+      "rewards/margins": 0.9423335194587708,
+      "rewards/rejected": -2.6290409564971924,
       "step": 2740
     },
     {
       "epoch": 1.469141996989463,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9732480232771046,
       "learning_rate": 6.043518873921074e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28983527421951294,
+      "logits/rejected": -0.16624179482460022,
+      "logps/chosen": -1.6274681091308594,
+      "logps/rejected": -2.2462551593780518,
+      "loss": 0.6367,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6274681091308594,
+      "rewards/margins": 0.6187870502471924,
+      "rewards/rejected": -2.2462551593780518,
       "step": 2745
     },
     {
       "epoch": 1.4718180297708647,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.865113043136241,
       "learning_rate": 6.028282766952393e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26166191697120667,
+      "logits/rejected": -0.15309420228004456,
+      "logps/chosen": -1.7772222757339478,
+      "logps/rejected": -2.521023988723755,
+      "loss": 0.629,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7772222757339478,
+      "rewards/margins": 0.7438015341758728,
+      "rewards/rejected": -2.521023988723755,
       "step": 2750
     },
     {
       "epoch": 1.4744940625522662,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.97388308241166,
       "learning_rate": 6.013036683579798e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22013874351978302,
+      "logits/rejected": -0.08937226235866547,
+      "logps/chosen": -1.6064167022705078,
+      "logps/rejected": -2.302889585494995,
+      "loss": 0.6463,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6064167022705078,
+      "rewards/margins": 0.6964728832244873,
+      "rewards/rejected": -2.302889585494995,
       "step": 2755
     },
     {
       "epoch": 1.4771700953336677,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4261148082335064,
       "learning_rate": 5.997780771720854e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3059960603713989,
+      "logits/rejected": -0.13851214945316315,
+      "logps/chosen": -1.7123925685882568,
+      "logps/rejected": -2.5452980995178223,
+      "loss": 0.6428,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7123925685882568,
+      "rewards/margins": 0.8329055905342102,
+      "rewards/rejected": -2.5452980995178223,
       "step": 2760
     },
     {
       "epoch": 1.4798461281150694,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.485222469671208,
       "learning_rate": 5.982515179388486e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23644042015075684,
+      "logits/rejected": -0.10643786191940308,
+      "logps/chosen": -1.701786994934082,
+      "logps/rejected": -2.3223700523376465,
+      "loss": 0.6272,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.701786994934082,
+      "rewards/margins": 0.6205830574035645,
+      "rewards/rejected": -2.3223700523376465,
       "step": 2765
     },
     {
       "epoch": 1.482522160896471,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.093056798887892,
       "learning_rate": 5.967240054689541e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30416834354400635,
+      "logits/rejected": -0.24106450378894806,
+      "logps/chosen": -1.595759391784668,
+      "logps/rejected": -2.070983648300171,
+      "loss": 0.6479,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.595759391784668,
+      "rewards/margins": 0.4752243161201477,
+      "rewards/rejected": -2.070983648300171,
       "step": 2770
     },
     {
       "epoch": 1.4851981936778724,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.037280279383754,
       "learning_rate": 5.951955545823342e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2683941721916199,
+      "logits/rejected": -0.22709858417510986,
+      "logps/chosen": -1.6916639804840088,
+      "logps/rejected": -2.398190975189209,
+      "loss": 0.6377,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6916639804840088,
+      "rewards/margins": 0.7065272927284241,
+      "rewards/rejected": -2.398190975189209,
       "step": 2775
     },
     {
       "epoch": 1.4878742264592741,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3794403371784507,
       "learning_rate": 5.936661801080263e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26575779914855957,
+      "logits/rejected": -0.17347212135791779,
+      "logps/chosen": -1.8421399593353271,
+      "logps/rejected": -2.4046874046325684,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.8421399593353271,
+      "rewards/margins": 0.5625476837158203,
+      "rewards/rejected": -2.4046874046325684,
       "step": 2780
     },
     {
       "epoch": 1.4905502592406756,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.420275137963173,
       "learning_rate": 5.92135896884028e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2970080077648163,
+      "logits/rejected": -0.16437241435050964,
+      "logps/chosen": -1.8791061639785767,
+      "logps/rejected": -2.571920871734619,
+      "loss": 0.6646,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.8791061639785767,
+      "rewards/margins": 0.6928142309188843,
+      "rewards/rejected": -2.571920871734619,
       "step": 2785
     },
     {
       "epoch": 1.4932262920220774,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.016483369213864,
       "learning_rate": 5.906047197571541e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2199886590242386,
+      "logits/rejected": -0.2385924756526947,
+      "logps/chosen": -1.7190725803375244,
+      "logps/rejected": -2.3302128314971924,
+      "loss": 0.6491,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7190725803375244,
+      "rewards/margins": 0.6111401319503784,
+      "rewards/rejected": -2.3302128314971924,
       "step": 2790
     },
     {
       "epoch": 1.4959023248034788,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.658766343356379,
       "learning_rate": 5.890726635828919e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13149581849575043,
+      "logits/rejected": -0.11898727715015411,
+      "logps/chosen": -1.6707260608673096,
+      "logps/rejected": -2.3066916465759277,
+      "loss": 0.6554,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.6707260608673096,
+      "rewards/margins": 0.6359657645225525,
+      "rewards/rejected": -2.3066916465759277,
       "step": 2795
     },
     {
       "epoch": 1.4985783575848803,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3405072873647477,
       "learning_rate": 5.875397432252569e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28285425901412964,
+      "logits/rejected": -0.1973208636045456,
+      "logps/chosen": -1.803589105606079,
+      "logps/rejected": -2.428412437438965,
+      "loss": 0.6444,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.803589105606079,
+      "rewards/margins": 0.6248234510421753,
+      "rewards/rejected": -2.428412437438965,
       "step": 2800
     },
     {
       "epoch": 1.4985783575848803,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.8388,
-      "eval_samples_per_second": 38.606,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.673,
+      "eval_logits/chosen": -0.014461087062954903,
+      "eval_logits/rejected": 0.06653594970703125,
+      "eval_logps/chosen": -1.781496524810791,
+      "eval_logps/rejected": -2.48334002494812,
+      "eval_loss": 0.6536964774131775,
+      "eval_rewards/accuracies": 0.6683976054191589,
+      "eval_rewards/chosen": -1.781496524810791,
+      "eval_rewards/margins": 0.7018433809280396,
+      "eval_rewards/rejected": -2.48334002494812,
+      "eval_runtime": 40.4028,
+      "eval_samples_per_second": 33.29,
+      "eval_steps_per_second": 8.341,
       "step": 2800
     },
     {
       "epoch": 1.5012543903662818,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9528437738998286,
       "learning_rate": 5.860059735566491e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.40563344955444336,
+      "logits/rejected": -0.2681305706501007,
+      "logps/chosen": -1.5786396265029907,
+      "logps/rejected": -2.2465896606445312,
+      "loss": 0.6426,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5786396265029907,
+      "rewards/margins": 0.6679500341415405,
+      "rewards/rejected": -2.2465896606445312,
       "step": 2805
     },
     {
       "epoch": 1.5039304231476835,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 16.121465915164105,
       "learning_rate": 5.844713694577087e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2658074200153351,
+      "logits/rejected": -0.22980177402496338,
+      "logps/chosen": -1.7288968563079834,
+      "logps/rejected": -2.309450149536133,
+      "loss": 0.6556,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7288968563079834,
+      "rewards/margins": 0.5805534720420837,
+      "rewards/rejected": -2.309450149536133,
       "step": 2810
     },
     {
       "epoch": 1.5066064559290853,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0458663361092837,
       "learning_rate": 5.829359458171714e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2191942036151886,
+      "logits/rejected": -0.10844508558511734,
+      "logps/chosen": -1.7707496881484985,
+      "logps/rejected": -2.522172451019287,
+      "loss": 0.6389,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7707496881484985,
+      "rewards/margins": 0.7514227032661438,
+      "rewards/rejected": -2.522172451019287,
       "step": 2815
     },
     {
       "epoch": 1.5092824887104868,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.832270184430611,
       "learning_rate": 5.81399717531724e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2655988931655884,
+      "logits/rejected": -0.10418226569890976,
+      "logps/chosen": -1.7698020935058594,
+      "logps/rejected": -2.4652791023254395,
+      "loss": 0.6489,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7698020935058594,
+      "rewards/margins": 0.6954768896102905,
+      "rewards/rejected": -2.4652791023254395,
       "step": 2820
     },
     {
       "epoch": 1.5119585214918883,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.383360382615581,
       "learning_rate": 5.798626995058602e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31909435987472534,
+      "logits/rejected": -0.14148707687854767,
+      "logps/chosen": -1.7953548431396484,
+      "logps/rejected": -2.6157522201538086,
+      "loss": 0.6518,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7953548431396484,
+      "rewards/margins": 0.8203975558280945,
+      "rewards/rejected": -2.6157522201538086,
       "step": 2825
     },
     {
       "epoch": 1.51463455427329,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.493188194544531,
       "learning_rate": 5.783249066517354e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29562944173812866,
+      "logits/rejected": -0.15610770881175995,
+      "logps/chosen": -1.8168258666992188,
+      "logps/rejected": -2.368178367614746,
+      "loss": 0.6783,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.8168258666992188,
+      "rewards/margins": 0.5513526201248169,
+      "rewards/rejected": -2.368178367614746,
       "step": 2830
     },
     {
       "epoch": 1.5173105870546915,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.71617662686016,
       "learning_rate": 5.767863538890228e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3051961064338684,
+      "logits/rejected": -0.16161957383155823,
+      "logps/chosen": -1.7413673400878906,
+      "logps/rejected": -2.5268642902374268,
+      "loss": 0.6426,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7413673400878906,
+      "rewards/margins": 0.7854971885681152,
+      "rewards/rejected": -2.5268642902374268,
       "step": 2835
     },
     {
       "epoch": 1.519986619836093,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.678733783167968,
       "learning_rate": 5.75247056144768e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2722797095775604,
+      "logits/rejected": -0.1833418309688568,
+      "logps/chosen": -1.7482802867889404,
+      "logps/rejected": -2.2249321937561035,
+      "loss": 0.6486,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7482802867889404,
+      "rewards/margins": 0.4766520857810974,
+      "rewards/rejected": -2.2249321937561035,
       "step": 2840
     },
     {
       "epoch": 1.5226626526174947,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.5798198175838785,
       "learning_rate": 5.737070283532444e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2515469789505005,
+      "logits/rejected": -0.17257753014564514,
+      "logps/chosen": -1.7867534160614014,
+      "logps/rejected": -2.3816041946411133,
+      "loss": 0.6569,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7867534160614014,
+      "rewards/margins": 0.594851016998291,
+      "rewards/rejected": -2.3816041946411133,
       "step": 2845
     },
     {
       "epoch": 1.5253386853988962,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.855052652874718,
       "learning_rate": 5.721662854558084e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33099642395973206,
+      "logits/rejected": -0.25515037775039673,
+      "logps/chosen": -1.728581190109253,
+      "logps/rejected": -2.4049243927001953,
+      "loss": 0.6553,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.728581190109253,
+      "rewards/margins": 0.6763433814048767,
+      "rewards/rejected": -2.4049243927001953,
       "step": 2850
     },
     {
       "epoch": 1.5280147181802977,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3234670995399287,
       "learning_rate": 5.706248424007545e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.35217684507369995,
+      "logits/rejected": -0.19632931053638458,
+      "logps/chosen": -1.9164234399795532,
+      "logps/rejected": -2.52608060836792,
+      "loss": 0.6609,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.9164234399795532,
+      "rewards/margins": 0.6096576452255249,
+      "rewards/rejected": -2.52608060836792,
       "step": 2855
     },
     {
       "epoch": 1.5306907509616994,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.004807791802777,
       "learning_rate": 5.690827141431699e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.38299429416656494,
+      "logits/rejected": -0.20971879363059998,
+      "logps/chosen": -1.822824478149414,
+      "logps/rejected": -2.438981771469116,
+      "loss": 0.6475,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.822824478149414,
+      "rewards/margins": 0.6161569952964783,
+      "rewards/rejected": -2.438981771469116,
       "step": 2860
     },
     {
       "epoch": 1.5333667837431009,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.058093940078705,
       "learning_rate": 5.675399156447897e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.42472419142723083,
+      "logits/rejected": -0.2947673499584198,
+      "logps/chosen": -1.753854751586914,
+      "logps/rejected": -2.3892388343811035,
+      "loss": 0.6634,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.753854751586914,
+      "rewards/margins": 0.6353837251663208,
+      "rewards/rejected": -2.3892388343811035,
       "step": 2865
     },
     {
       "epoch": 1.5360428165245024,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.046809019667335,
       "learning_rate": 5.659964618738515e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3400971293449402,
+      "logits/rejected": -0.23686620593070984,
+      "logps/chosen": -1.770789384841919,
+      "logps/rejected": -2.241206645965576,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.770789384841919,
+      "rewards/margins": 0.4704172611236572,
+      "rewards/rejected": -2.241206645965576,
       "step": 2870
     },
     {
       "epoch": 1.538718849305904,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5466154360262423,
       "learning_rate": 5.644523678049509e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34364116191864014,
+      "logits/rejected": -0.25228601694107056,
+      "logps/chosen": -1.7833263874053955,
+      "logps/rejected": -2.3740034103393555,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7833263874053955,
+      "rewards/margins": 0.5906770825386047,
+      "rewards/rejected": -2.3740034103393555,
       "step": 2875
     },
     {
       "epoch": 1.5413948820873056,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7430774099524915,
       "learning_rate": 5.629076484188952e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20157694816589355,
+      "logits/rejected": -0.09859408438205719,
+      "logps/chosen": -1.7882091999053955,
+      "logps/rejected": -2.4153380393981934,
+      "loss": 0.6416,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7882091999053955,
+      "rewards/margins": 0.6271288394927979,
+      "rewards/rejected": -2.4153380393981934,
       "step": 2880
     },
     {
       "epoch": 1.544070914868707,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7015724233887646,
       "learning_rate": 5.613623187025587e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3295699954032898,
+      "logits/rejected": -0.22217822074890137,
+      "logps/chosen": -1.8663032054901123,
+      "logps/rejected": -2.4661288261413574,
+      "loss": 0.651,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8663032054901123,
+      "rewards/margins": 0.5998259782791138,
+      "rewards/rejected": -2.4661288261413574,
       "step": 2885
     },
     {
       "epoch": 1.5467469476501088,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7587370506365856,
       "learning_rate": 5.598163936487369e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.375520795583725,
+      "logits/rejected": -0.20846684277057648,
+      "logps/chosen": -1.7613131999969482,
+      "logps/rejected": -2.459127902984619,
+      "loss": 0.655,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7613131999969482,
+      "rewards/margins": 0.6978144645690918,
+      "rewards/rejected": -2.459127902984619,
       "step": 2890
     },
     {
       "epoch": 1.5494229804315103,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2258686083752313,
       "learning_rate": 5.582698882560017e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3725875914096832,
+      "logits/rejected": -0.23959724605083466,
+      "logps/chosen": -1.7926946878433228,
+      "logps/rejected": -2.4823594093322754,
+      "loss": 0.6485,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7926946878433228,
+      "rewards/margins": 0.6896650195121765,
+      "rewards/rejected": -2.4823594093322754,
       "step": 2895
     },
     {
       "epoch": 1.5520990132129118,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.79073603790691,
       "learning_rate": 5.567228175285549e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31144049763679504,
+      "logits/rejected": -0.2203945815563202,
+      "logps/chosen": -1.727837324142456,
+      "logps/rejected": -2.5480780601501465,
+      "loss": 0.6455,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.727837324142456,
+      "rewards/margins": 0.82024085521698,
+      "rewards/rejected": -2.5480780601501465,
       "step": 2900
     },
     {
       "epoch": 1.5547750459943135,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.079433887544208,
       "learning_rate": 5.551751964760838e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2177608758211136,
+      "logits/rejected": -0.19566825032234192,
+      "logps/chosen": -1.779537558555603,
+      "logps/rejected": -2.4034876823425293,
+      "loss": 0.6438,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.779537558555603,
+      "rewards/margins": 0.6239501237869263,
+      "rewards/rejected": -2.4034876823425293,
       "step": 2905
     },
     {
       "epoch": 1.557451078775715,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.962793623693685,
       "learning_rate": 5.536270401136145e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32901883125305176,
+      "logits/rejected": -0.24412909150123596,
+      "logps/chosen": -1.619386911392212,
+      "logps/rejected": -2.1715385913848877,
+      "loss": 0.6415,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.619386911392212,
+      "rewards/margins": 0.5521514415740967,
+      "rewards/rejected": -2.1715385913848877,
       "step": 2910
     },
     {
       "epoch": 1.5601271115571165,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.568344513640716,
       "learning_rate": 5.520783634613667e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2727218568325043,
+      "logits/rejected": -0.11468567699193954,
+      "logps/chosen": -1.8145971298217773,
+      "logps/rejected": -2.4697604179382324,
+      "loss": 0.6493,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8145971298217773,
+      "rewards/margins": 0.6551633477210999,
+      "rewards/rejected": -2.4697604179382324,
       "step": 2915
     },
     {
       "epoch": 1.5628031443385182,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.6320156776669645,
       "learning_rate": 5.505291815446082e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2540712356567383,
+      "logits/rejected": -0.14601746201515198,
+      "logps/chosen": -1.798622727394104,
+      "logps/rejected": -2.4401488304138184,
+      "loss": 0.6628,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.798622727394104,
+      "rewards/margins": 0.6415263414382935,
+      "rewards/rejected": -2.4401488304138184,
       "step": 2920
     },
     {
       "epoch": 1.5654791771199197,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6021659082528203,
       "learning_rate": 5.489795093935089e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24974870681762695,
+      "logits/rejected": -0.1918577253818512,
+      "logps/chosen": -1.7381932735443115,
+      "logps/rejected": -2.3884787559509277,
+      "loss": 0.6298,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7381932735443115,
+      "rewards/margins": 0.6502853631973267,
+      "rewards/rejected": -2.3884787559509277,
       "step": 2925
     },
     {
       "epoch": 1.5681552099013212,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.416550920278306,
       "learning_rate": 5.474293620429946e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3967882990837097,
+      "logits/rejected": -0.2407037317752838,
+      "logps/chosen": -1.7135616540908813,
+      "logps/rejected": -2.8280446529388428,
+      "loss": 0.6315,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.7135616540908813,
+      "rewards/margins": 1.114483118057251,
+      "rewards/rejected": -2.8280446529388428,
       "step": 2930
     },
     {
       "epoch": 1.570831242682723,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4496825875692,
       "learning_rate": 5.458787545326018e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36053353548049927,
+      "logits/rejected": -0.23718604445457458,
+      "logps/chosen": -1.7853285074234009,
+      "logps/rejected": -2.433076858520508,
+      "loss": 0.6436,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7853285074234009,
+      "rewards/margins": 0.6477484107017517,
+      "rewards/rejected": -2.433076858520508,
       "step": 2935
     },
     {
       "epoch": 1.5735072754641244,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.260987468576226,
       "learning_rate": 5.443277019063311e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3389289677143097,
+      "logits/rejected": -0.18744170665740967,
+      "logps/chosen": -1.7133495807647705,
+      "logps/rejected": -2.5210843086242676,
+      "loss": 0.627,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7133495807647705,
+      "rewards/margins": 0.8077349662780762,
+      "rewards/rejected": -2.5210843086242676,
       "step": 2940
     },
     {
       "epoch": 1.5761833082455259,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.146633414659927,
       "learning_rate": 5.427762192125023e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3239956498146057,
+      "logits/rejected": -0.19952084124088287,
+      "logps/chosen": -1.8015220165252686,
+      "logps/rejected": -2.2534210681915283,
+      "loss": 0.6512,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.8015220165252686,
+      "rewards/margins": 0.4518989622592926,
+      "rewards/rejected": -2.2534210681915283,
       "step": 2945
     },
     {
       "epoch": 1.5788593410269276,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.252174528182001,
       "learning_rate": 5.41224321503607e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26125818490982056,
+      "logits/rejected": -0.04103229194879532,
+      "logps/chosen": -1.633354902267456,
+      "logps/rejected": -2.4924654960632324,
+      "loss": 0.6309,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.633354902267456,
+      "rewards/margins": 0.8591103553771973,
+      "rewards/rejected": -2.4924654960632324,
       "step": 2950
     },
     {
       "epoch": 1.5815353738083293,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1616595103203546,
       "learning_rate": 5.396720238361637e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.226335808634758,
+      "logits/rejected": -0.13410703837871552,
+      "logps/chosen": -1.6979429721832275,
+      "logps/rejected": -2.315345525741577,
+      "loss": 0.6506,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6979429721832275,
+      "rewards/margins": 0.6174028515815735,
+      "rewards/rejected": -2.315345525741577,
       "step": 2955
     },
     {
       "epoch": 1.5842114065897306,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.758345757107147,
       "learning_rate": 5.381193412705711e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31831610202789307,
+      "logits/rejected": -0.19526976346969604,
+      "logps/chosen": -1.7809349298477173,
+      "logps/rejected": -2.423278331756592,
+      "loss": 0.6377,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7809349298477173,
+      "rewards/margins": 0.6423431634902954,
+      "rewards/rejected": -2.423278331756592,
       "step": 2960
     },
     {
       "epoch": 1.5868874393711323,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.283529344729765,
       "learning_rate": 5.365662888709622e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.285057008266449,
+      "logits/rejected": -0.20029599964618683,
+      "logps/chosen": -1.7176443338394165,
+      "logps/rejected": -2.415170907974243,
+      "loss": 0.6416,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7176443338394165,
+      "rewards/margins": 0.6975264549255371,
+      "rewards/rejected": -2.415170907974243,
       "step": 2965
     },
     {
       "epoch": 1.589563472152534,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.637322214233277,
       "learning_rate": 5.350128817050585e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30295291543006897,
+      "logits/rejected": -0.13567155599594116,
+      "logps/chosen": -1.7915077209472656,
+      "logps/rejected": -2.5360145568847656,
+      "loss": 0.6554,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.7915077209472656,
+      "rewards/margins": 0.7445067763328552,
+      "rewards/rejected": -2.5360145568847656,
       "step": 2970
     },
     {
       "epoch": 1.5922395049339353,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.254667130092636,
       "learning_rate": 5.334591348440229e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2654396891593933,
+      "logits/rejected": -0.12772902846336365,
+      "logps/chosen": -1.7204933166503906,
+      "logps/rejected": -2.588350534439087,
+      "loss": 0.6432,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7204933166503906,
+      "rewards/margins": 0.8678572773933411,
+      "rewards/rejected": -2.588350534439087,
       "step": 2975
     },
     {
       "epoch": 1.594915537715337,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.355678534097844,
       "learning_rate": 5.319050633623141e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32967478036880493,
+      "logits/rejected": -0.18231257796287537,
+      "logps/chosen": -1.8578590154647827,
+      "logps/rejected": -2.4528942108154297,
+      "loss": 0.6656,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8578590154647827,
+      "rewards/margins": 0.5950349569320679,
+      "rewards/rejected": -2.4528942108154297,
       "step": 2980
     },
     {
       "epoch": 1.5975915704967387,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3063919577781276,
       "learning_rate": 5.303506823375409e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3028491735458374,
+      "logits/rejected": -0.1257626712322235,
+      "logps/chosen": -1.8272817134857178,
+      "logps/rejected": -2.5646255016326904,
+      "loss": 0.6568,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.8272817134857178,
+      "rewards/margins": 0.7373439073562622,
+      "rewards/rejected": -2.5646255016326904,
       "step": 2985
     },
     {
       "epoch": 1.60026760327814,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.8880444207421,
       "learning_rate": 5.287960068503143e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.296041876077652,
+      "logits/rejected": -0.12234993278980255,
+      "logps/chosen": -1.7051788568496704,
+      "logps/rejected": -2.5311193466186523,
+      "loss": 0.6457,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7051788568496704,
+      "rewards/margins": 0.8259404897689819,
+      "rewards/rejected": -2.5311193466186523,
       "step": 2990
     },
     {
       "epoch": 1.6029436360595417,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2164132362344326,
       "learning_rate": 5.272410519841032e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26291483640670776,
+      "logits/rejected": -0.15441301465034485,
+      "logps/chosen": -1.852125883102417,
+      "logps/rejected": -2.867703914642334,
+      "loss": 0.6337,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.852125883102417,
+      "rewards/margins": 1.0155781507492065,
+      "rewards/rejected": -2.867703914642334,
       "step": 2995
     },
     {
       "epoch": 1.6056196688409434,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.549126738758649,
       "learning_rate": 5.256858328250861e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3079647421836853,
+      "logits/rejected": -0.1680411398410797,
+      "logps/chosen": -1.7927579879760742,
+      "logps/rejected": -2.4446043968200684,
+      "loss": 0.6636,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7927579879760742,
+      "rewards/margins": 0.651846706867218,
+      "rewards/rejected": -2.4446043968200684,
       "step": 3000
     },
     {
       "epoch": 1.608295701622345,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.614817207153222,
       "learning_rate": 5.241303644620063e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3628775179386139,
+      "logits/rejected": -0.21877360343933105,
+      "logps/chosen": -1.6768512725830078,
+      "logps/rejected": -2.221369504928589,
+      "loss": 0.6544,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6768512725830078,
+      "rewards/margins": 0.544518232345581,
+      "rewards/rejected": -2.221369504928589,
       "step": 3005
     },
     {
       "epoch": 1.6109717344037464,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.933110621208879,
       "learning_rate": 5.225746619860248e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3393351137638092,
+      "logits/rejected": -0.22328808903694153,
+      "logps/chosen": -1.693605661392212,
+      "logps/rejected": -2.3360767364501953,
+      "loss": 0.6651,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.693605661392212,
+      "rewards/margins": 0.642471432685852,
+      "rewards/rejected": -2.3360767364501953,
       "step": 3010
     },
     {
       "epoch": 1.6136477671851481,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1185753121380775,
       "learning_rate": 5.210187404905735e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1582556664943695,
+      "logits/rejected": -0.0862528532743454,
+      "logps/chosen": -1.8426650762557983,
+      "logps/rejected": -2.362696886062622,
+      "loss": 0.6398,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.8426650762557983,
+      "rewards/margins": 0.5200316309928894,
+      "rewards/rejected": -2.362696886062622,
       "step": 3015
     },
     {
       "epoch": 1.6163237999665496,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0087483681299596,
       "learning_rate": 5.194626150712098e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31622034311294556,
+      "logits/rejected": -0.17147010564804077,
+      "logps/chosen": -1.7515758275985718,
+      "logps/rejected": -2.376955270767212,
+      "loss": 0.6494,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7515758275985718,
+      "rewards/margins": 0.6253794431686401,
+      "rewards/rejected": -2.376955270767212,
       "step": 3020
     },
     {
       "epoch": 1.6189998327479511,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.262704984458592,
       "learning_rate": 5.179063008254695e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2549213767051697,
+      "logits/rejected": -0.09584025293588638,
+      "logps/chosen": -1.6214898824691772,
+      "logps/rejected": -2.2347261905670166,
+      "loss": 0.6567,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6214898824691772,
+      "rewards/margins": 0.6132357120513916,
+      "rewards/rejected": -2.2347261905670166,
       "step": 3025
     },
     {
       "epoch": 1.6216758655293528,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.451830005709934,
       "learning_rate": 5.163498128527199e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2438913881778717,
+      "logits/rejected": -0.10928897559642792,
+      "logps/chosen": -1.9000005722045898,
+      "logps/rejected": -2.4701499938964844,
+      "loss": 0.6502,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.9000005722045898,
+      "rewards/margins": 0.5701490640640259,
+      "rewards/rejected": -2.4701499938964844,
       "step": 3030
     },
     {
       "epoch": 1.6243518983107543,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.418596830553681,
       "learning_rate": 5.147931662540144e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.14950978755950928,
+      "logits/rejected": -0.038586877286434174,
+      "logps/chosen": -1.7542774677276611,
+      "logps/rejected": -2.254215717315674,
+      "loss": 0.6551,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7542774677276611,
+      "rewards/margins": 0.4999381899833679,
+      "rewards/rejected": -2.254215717315674,
       "step": 3035
     },
     {
       "epoch": 1.6270279310921558,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.5987755848951535,
       "learning_rate": 5.132363761319449e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2564184367656708,
+      "logits/rejected": -0.1935768574476242,
+      "logps/chosen": -1.5337872505187988,
+      "logps/rejected": -2.384906768798828,
+      "loss": 0.6248,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5337872505187988,
+      "rewards/margins": 0.8511196374893188,
+      "rewards/rejected": -2.384906768798828,
       "step": 3040
     },
     {
       "epoch": 1.6297039638735575,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5501505584986974,
       "learning_rate": 5.116794575904962e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22907240688800812,
+      "logits/rejected": -0.13626930117607117,
+      "logps/chosen": -1.6575698852539062,
+      "logps/rejected": -2.1971399784088135,
+      "loss": 0.6475,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.6575698852539062,
+      "rewards/margins": 0.5395702123641968,
+      "rewards/rejected": -2.1971399784088135,
       "step": 3045
     },
     {
       "epoch": 1.632379996654959,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.3795075755536805,
       "learning_rate": 5.101224257348987e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2877201437950134,
+      "logits/rejected": -0.13610991835594177,
+      "logps/chosen": -1.796411156654358,
+      "logps/rejected": -2.4966630935668945,
+      "loss": 0.6406,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.796411156654358,
+      "rewards/margins": 0.7002518773078918,
+      "rewards/rejected": -2.4966630935668945,
       "step": 3050
     },
     {
       "epoch": 1.6350560294363605,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7629422001573984,
       "learning_rate": 5.085652956714823e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3086533844470978,
+      "logits/rejected": -0.1841677725315094,
+      "logps/chosen": -1.7766603231430054,
+      "logps/rejected": -2.4963510036468506,
+      "loss": 0.6387,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7766603231430054,
+      "rewards/margins": 0.7196909785270691,
+      "rewards/rejected": -2.4963510036468506,
       "step": 3055
     },
     {
       "epoch": 1.6377320622177622,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.257388270734288,
       "learning_rate": 5.070080825075298e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2867925763130188,
+      "logits/rejected": -0.11529000103473663,
+      "logps/chosen": -1.709903359413147,
+      "logps/rejected": -2.3392274379730225,
+      "loss": 0.6562,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.709903359413147,
+      "rewards/margins": 0.6293242573738098,
+      "rewards/rejected": -2.3392274379730225,
       "step": 3060
     },
     {
       "epoch": 1.6404080949991637,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.300899580181916,
       "learning_rate": 5.0545080135113e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.16866309940814972,
+      "logits/rejected": -0.1270056664943695,
+      "logps/chosen": -1.7282190322875977,
+      "logps/rejected": -2.479179859161377,
+      "loss": 0.6369,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7282190322875977,
+      "rewards/margins": 0.750961184501648,
+      "rewards/rejected": -2.479179859161377,
       "step": 3065
     },
     {
       "epoch": 1.6430841277805652,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6718695627131996,
       "learning_rate": 5.038934673110316e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33182889223098755,
+      "logits/rejected": -0.21299715340137482,
+      "logps/chosen": -1.78805410861969,
+      "logps/rejected": -2.4402472972869873,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.78805410861969,
+      "rewards/margins": 0.6521932482719421,
+      "rewards/rejected": -2.4402472972869873,
       "step": 3070
     },
     {
       "epoch": 1.645760160561967,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.656859606671018,
       "learning_rate": 5.023360954964963e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3329083323478699,
+      "logits/rejected": -0.2637431025505066,
+      "logps/chosen": -1.6994746923446655,
+      "logps/rejected": -2.359039783477783,
+      "loss": 0.6439,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.6994746923446655,
+      "rewards/margins": 0.6595650911331177,
+      "rewards/rejected": -2.359039783477783,
       "step": 3075
     },
     {
       "epoch": 1.6484361933433684,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9723817246651736,
       "learning_rate": 5.007787010171524e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3671368360519409,
+      "logits/rejected": -0.17530004680156708,
+      "logps/chosen": -1.5964505672454834,
+      "logps/rejected": -2.3508477210998535,
+      "loss": 0.6425,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5964505672454834,
+      "rewards/margins": 0.7543970942497253,
+      "rewards/rejected": -2.3508477210998535,
       "step": 3080
     },
     {
       "epoch": 1.65111222612477,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 10.346574464617618,
       "learning_rate": 4.992212989828477e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20340967178344727,
+      "logits/rejected": -0.18976713716983795,
+      "logps/chosen": -1.6913814544677734,
+      "logps/rejected": -2.3659720420837402,
+      "loss": 0.6501,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.6913814544677734,
+      "rewards/margins": 0.6745905876159668,
+      "rewards/rejected": -2.3659720420837402,
       "step": 3085
     },
     {
       "epoch": 1.6537882589061716,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.005395622103251,
       "learning_rate": 4.976639045035036e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1630939245223999,
+      "logits/rejected": -0.09154991805553436,
+      "logps/chosen": -1.687139868736267,
+      "logps/rejected": -2.183581829071045,
+      "loss": 0.6564,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.687139868736267,
+      "rewards/margins": 0.4964420795440674,
+      "rewards/rejected": -2.183581829071045,
       "step": 3090
     },
     {
       "epoch": 1.6564642916875731,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.582468706000511,
       "learning_rate": 4.961065326889683e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2233189344406128,
+      "logits/rejected": -0.08394929021596909,
+      "logps/chosen": -1.7342462539672852,
+      "logps/rejected": -2.404630661010742,
+      "loss": 0.6496,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7342462539672852,
+      "rewards/margins": 0.6703846454620361,
+      "rewards/rejected": -2.404630661010742,
       "step": 3095
     },
     {
       "epoch": 1.6591403244689746,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.909886043297479,
       "learning_rate": 4.9454919864887e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3707055151462555,
+      "logits/rejected": -0.23319748044013977,
+      "logps/chosen": -1.7295427322387695,
+      "logps/rejected": -2.330104351043701,
+      "loss": 0.6652,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7295427322387695,
+      "rewards/margins": 0.6005619168281555,
+      "rewards/rejected": -2.330104351043701,
       "step": 3100
     },
     {
       "epoch": 1.6618163572503764,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6100351950934906,
       "learning_rate": 4.929919174924701e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3370603024959564,
+      "logits/rejected": -0.16688783466815948,
+      "logps/chosen": -1.7489798069000244,
+      "logps/rejected": -2.3440399169921875,
+      "loss": 0.6525,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7489798069000244,
+      "rewards/margins": 0.5950601696968079,
+      "rewards/rejected": -2.3440399169921875,
       "step": 3105
     },
     {
       "epoch": 1.6644923900317778,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4496752173433496,
       "learning_rate": 4.914347043285177e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2490864247083664,
+      "logits/rejected": -0.139280766248703,
+      "logps/chosen": -1.752001404762268,
+      "logps/rejected": -2.393537998199463,
+      "loss": 0.6476,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.752001404762268,
+      "rewards/margins": 0.6415364146232605,
+      "rewards/rejected": -2.393537998199463,
       "step": 3110
     },
     {
       "epoch": 1.6671684228131793,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6294569399126453,
       "learning_rate": 4.898775742651013e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23234477639198303,
+      "logits/rejected": -0.14225415885448456,
+      "logps/chosen": -1.7735204696655273,
+      "logps/rejected": -2.529548168182373,
+      "loss": 0.6292,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7735204696655273,
+      "rewards/margins": 0.7560276985168457,
+      "rewards/rejected": -2.529548168182373,
       "step": 3115
     },
     {
       "epoch": 1.669844455594581,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.200298104958111,
       "learning_rate": 4.883205424095037e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33337435126304626,
+      "logits/rejected": -0.1985912322998047,
+      "logps/chosen": -1.8811705112457275,
+      "logps/rejected": -2.62290620803833,
+      "loss": 0.6699,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.8811705112457275,
+      "rewards/margins": 0.7417356371879578,
+      "rewards/rejected": -2.62290620803833,
       "step": 3120
     },
     {
       "epoch": 1.6725204883759828,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.29012459780223,
       "learning_rate": 4.86763623868055e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22427792847156525,
+      "logits/rejected": -0.11705160140991211,
+      "logps/chosen": -1.9192575216293335,
+      "logps/rejected": -2.738067388534546,
+      "loss": 0.6553,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.9192575216293335,
+      "rewards/margins": 0.8188098669052124,
+      "rewards/rejected": -2.738067388534546,
       "step": 3125
     },
     {
       "epoch": 1.675196521157384,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.331992935290061,
       "learning_rate": 4.852068337459856e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2532265782356262,
+      "logits/rejected": -0.11561497300863266,
+      "logps/chosen": -1.8707072734832764,
+      "logps/rejected": -2.6265602111816406,
+      "loss": 0.6603,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8707072734832764,
+      "rewards/margins": 0.7558528184890747,
+      "rewards/rejected": -2.6265602111816406,
       "step": 3130
     },
     {
       "epoch": 1.6778725539387858,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1599563057114115,
       "learning_rate": 4.8365018714728e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20253655314445496,
+      "logits/rejected": -0.1392373889684677,
+      "logps/chosen": -1.951327919960022,
+      "logps/rejected": -2.5465481281280518,
+      "loss": 0.6584,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.951327919960022,
+      "rewards/margins": 0.5952200889587402,
+      "rewards/rejected": -2.5465481281280518,
       "step": 3135
     },
     {
       "epoch": 1.6805485867201875,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.422459929603815,
       "learning_rate": 4.820936991745304e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4520641267299652,
+      "logits/rejected": -0.3145623207092285,
+      "logps/chosen": -1.704223871231079,
+      "logps/rejected": -2.3595499992370605,
+      "loss": 0.6449,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.704223871231079,
+      "rewards/margins": 0.655326247215271,
+      "rewards/rejected": -2.3595499992370605,
       "step": 3140
     },
     {
       "epoch": 1.6832246195015887,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.752725628275623,
       "learning_rate": 4.8053738492879e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2962173819541931,
+      "logits/rejected": -0.17013126611709595,
+      "logps/chosen": -1.7363189458847046,
+      "logps/rejected": -2.5431504249572754,
+      "loss": 0.6424,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7363189458847046,
+      "rewards/margins": 0.8068313598632812,
+      "rewards/rejected": -2.5431504249572754,
       "step": 3145
     },
     {
       "epoch": 1.6859006522829905,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.4759981283947585,
       "learning_rate": 4.789812595094265e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.38815784454345703,
+      "logits/rejected": -0.27954789996147156,
+      "logps/chosen": -1.8133283853530884,
+      "logps/rejected": -2.376929521560669,
+      "loss": 0.6472,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8133283853530884,
+      "rewards/margins": 0.5636009573936462,
+      "rewards/rejected": -2.376929521560669,
       "step": 3150
     },
     {
       "epoch": 1.6885766850643922,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.4009070167979765,
       "learning_rate": 4.774253380139752e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.39069780707359314,
+      "logits/rejected": -0.2762901484966278,
+      "logps/chosen": -1.6346584558486938,
+      "logps/rejected": -2.413119316101074,
+      "loss": 0.6305,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6346584558486938,
+      "rewards/margins": 0.7784609198570251,
+      "rewards/rejected": -2.413119316101074,
       "step": 3155
     },
     {
       "epoch": 1.6912527178457935,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6495952325250633,
       "learning_rate": 4.758696355379936e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.288690984249115,
+      "logits/rejected": -0.28015196323394775,
+      "logps/chosen": -1.7347129583358765,
+      "logps/rejected": -2.467343807220459,
+      "loss": 0.6379,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7347129583358765,
+      "rewards/margins": 0.7326309084892273,
+      "rewards/rejected": -2.467343807220459,
       "step": 3160
     },
     {
       "epoch": 1.6939287506271952,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6501042100155305,
       "learning_rate": 4.743141671749138e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.37108731269836426,
+      "logits/rejected": -0.2744086980819702,
+      "logps/chosen": -1.7751808166503906,
+      "logps/rejected": -2.294650077819824,
+      "loss": 0.6618,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.7751808166503906,
+      "rewards/margins": 0.5194692611694336,
+      "rewards/rejected": -2.294650077819824,
       "step": 3165
     },
     {
       "epoch": 1.6966047834085969,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1909132363978086,
       "learning_rate": 4.727589480158968e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3335801959037781,
+      "logits/rejected": -0.24491123855113983,
+      "logps/chosen": -1.7683846950531006,
+      "logps/rejected": -2.622194290161133,
+      "loss": 0.6525,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7683846950531006,
+      "rewards/margins": 0.8538094758987427,
+      "rewards/rejected": -2.622194290161133,
       "step": 3170
     },
     {
       "epoch": 1.6992808161899984,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.50818923629867,
       "learning_rate": 4.712039931496855e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.35254278779029846,
+      "logits/rejected": -0.24656379222869873,
+      "logps/chosen": -1.8637815713882446,
+      "logps/rejected": -2.275770664215088,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.8637815713882446,
+      "rewards/margins": 0.41198912262916565,
+      "rewards/rejected": -2.275770664215088,
       "step": 3175
     },
     {
       "epoch": 1.7019568489713999,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9680187680514973,
       "learning_rate": 4.6964931766245905e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21323034167289734,
+      "logits/rejected": -0.16191688179969788,
+      "logps/chosen": -1.7952502965927124,
+      "logps/rejected": -2.6401352882385254,
+      "loss": 0.6468,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7952502965927124,
+      "rewards/margins": 0.8448851704597473,
+      "rewards/rejected": -2.6401352882385254,
       "step": 3180
     },
     {
       "epoch": 1.7046328817528016,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.593121956712056,
       "learning_rate": 4.6809493663768575e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.287620484828949,
+      "logits/rejected": -0.2678523659706116,
+      "logps/chosen": -1.9240102767944336,
+      "logps/rejected": -2.2741775512695312,
+      "loss": 0.6543,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.9240102767944336,
+      "rewards/margins": 0.3501672148704529,
+      "rewards/rejected": -2.2741775512695312,
       "step": 3185
     },
     {
       "epoch": 1.707308914534203,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.57182422501537,
       "learning_rate": 4.6654086515597716e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3525840640068054,
+      "logits/rejected": -0.19801035523414612,
+      "logps/chosen": -1.7617498636245728,
+      "logps/rejected": -2.721325635910034,
+      "loss": 0.6332,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.7617498636245728,
+      "rewards/margins": 0.9595760107040405,
+      "rewards/rejected": -2.721325635910034,
       "step": 3190
     },
     {
       "epoch": 1.7099849473156046,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.536822044114573,
       "learning_rate": 4.6498711829494154e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3808261752128601,
+      "logits/rejected": -0.27642735838890076,
+      "logps/chosen": -1.7383216619491577,
+      "logps/rejected": -2.5078377723693848,
+      "loss": 0.6408,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7383216619491577,
+      "rewards/margins": 0.7695156931877136,
+      "rewards/rejected": -2.5078377723693848,
       "step": 3195
     },
     {
       "epoch": 1.7126609800970063,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.144335321984435,
       "learning_rate": 4.6343371112903777e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22988788783550262,
+      "logits/rejected": -0.0959499329328537,
+      "logps/chosen": -1.8510921001434326,
+      "logps/rejected": -2.664167881011963,
+      "loss": 0.6575,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.8510921001434326,
+      "rewards/margins": 0.8130757212638855,
+      "rewards/rejected": -2.664167881011963,
       "step": 3200
     },
     {
       "epoch": 1.7126609800970063,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.9666,
-      "eval_samples_per_second": 38.465,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.638,
+      "eval_logits/chosen": 0.026014843955636024,
+      "eval_logits/rejected": 0.11065405607223511,
+      "eval_logps/chosen": -1.7922301292419434,
+      "eval_logps/rejected": -2.5114057064056396,
+      "eval_loss": 0.652005672454834,
+      "eval_rewards/accuracies": 0.6654302477836609,
+      "eval_rewards/chosen": -1.7922301292419434,
+      "eval_rewards/margins": 0.7191754579544067,
+      "eval_rewards/rejected": -2.5114057064056396,
+      "eval_runtime": 40.4143,
+      "eval_samples_per_second": 33.28,
+      "eval_steps_per_second": 8.339,
       "step": 3200
     },
     {
       "epoch": 1.7153370128784078,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9386482376359186,
       "learning_rate": 4.618806587294291e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3730618357658386,
+      "logits/rejected": -0.2632077634334564,
+      "logps/chosen": -1.7395604848861694,
+      "logps/rejected": -2.465817451477051,
+      "loss": 0.645,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7395604848861694,
+      "rewards/margins": 0.7262569665908813,
+      "rewards/rejected": -2.465817451477051,
       "step": 3205
     },
     {
       "epoch": 1.7180130456598093,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.322324200965726,
       "learning_rate": 4.603279761638365e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33539479970932007,
+      "logits/rejected": -0.23140735924243927,
+      "logps/chosen": -1.7064640522003174,
+      "logps/rejected": -2.375399351119995,
+      "loss": 0.645,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7064640522003174,
+      "rewards/margins": 0.6689354777336121,
+      "rewards/rejected": -2.375399351119995,
       "step": 3210
     },
     {
       "epoch": 1.720689078441211,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7906797927408045,
       "learning_rate": 4.5877567849639315e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28344544768333435,
+      "logits/rejected": -0.16761143505573273,
+      "logps/chosen": -1.7362693548202515,
+      "logps/rejected": -2.434016466140747,
+      "loss": 0.6471,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7362693548202515,
+      "rewards/margins": 0.6977471113204956,
+      "rewards/rejected": -2.434016466140747,
       "step": 3215
     },
     {
       "epoch": 1.7233651112226125,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.4985932726889555,
       "learning_rate": 4.572237807874979e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3271297216415405,
+      "logits/rejected": -0.11220970004796982,
+      "logps/chosen": -2.007469892501831,
+      "logps/rejected": -2.6530420780181885,
+      "loss": 0.6561,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.007469892501831,
+      "rewards/margins": 0.6455718874931335,
+      "rewards/rejected": -2.6530420780181885,
       "step": 3220
     },
     {
       "epoch": 1.726041144004014,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7409685277627247,
       "learning_rate": 4.5567229809366895e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29304784536361694,
+      "logits/rejected": -0.1683131754398346,
+      "logps/chosen": -1.6578865051269531,
+      "logps/rejected": -2.4507012367248535,
+      "loss": 0.6394,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6578865051269531,
+      "rewards/margins": 0.7928145527839661,
+      "rewards/rejected": -2.4507012367248535,
       "step": 3225
     },
     {
       "epoch": 1.7287171767854157,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.965801308466731,
       "learning_rate": 4.541212454673984e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3313615918159485,
+      "logits/rejected": -0.18042519688606262,
+      "logps/chosen": -1.837599515914917,
+      "logps/rejected": -2.8138744831085205,
+      "loss": 0.6342,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.837599515914917,
+      "rewards/margins": 0.976274847984314,
+      "rewards/rejected": -2.8138744831085205,
       "step": 3230
     },
     {
       "epoch": 1.7313932095668172,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.6367643355923605,
       "learning_rate": 4.525706379570055e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2997831106185913,
+      "logits/rejected": -0.23231983184814453,
+      "logps/chosen": -1.7785308361053467,
+      "logps/rejected": -2.4512112140655518,
+      "loss": 0.6457,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7785308361053467,
+      "rewards/margins": 0.6726802587509155,
+      "rewards/rejected": -2.4512112140655518,
       "step": 3235
     },
     {
       "epoch": 1.7340692423482187,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5706669627269596,
       "learning_rate": 4.510204906064911e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21972158551216125,
+      "logits/rejected": -0.12045315653085709,
+      "logps/chosen": -1.8616631031036377,
+      "logps/rejected": -2.624119281768799,
+      "loss": 0.6355,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.8616631031036377,
+      "rewards/margins": 0.7624566555023193,
+      "rewards/rejected": -2.624119281768799,
       "step": 3240
     },
     {
       "epoch": 1.7367452751296204,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.3502282444920715,
       "learning_rate": 4.4947081845539177e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3875466287136078,
+      "logits/rejected": -0.26213622093200684,
+      "logps/chosen": -1.84352707862854,
+      "logps/rejected": -2.4913182258605957,
+      "loss": 0.657,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.84352707862854,
+      "rewards/margins": 0.6477912664413452,
+      "rewards/rejected": -2.4913182258605957,
       "step": 3245
     },
     {
       "epoch": 1.739421307911022,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0303865299650337,
       "learning_rate": 4.479216365386333e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.1975325495004654,
+      "logits/rejected": -0.062357962131500244,
+      "logps/chosen": -1.729016900062561,
+      "logps/rejected": -2.5500648021698,
+      "loss": 0.6464,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.729016900062561,
+      "rewards/margins": 0.8210482597351074,
+      "rewards/rejected": -2.5500648021698,
       "step": 3250
     },
     {
       "epoch": 1.7420973406924234,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.258236197001299,
       "learning_rate": 4.4637295988638555e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20882156491279602,
+      "logits/rejected": -0.1410001814365387,
+      "logps/chosen": -1.8104639053344727,
+      "logps/rejected": -2.3793251514434814,
+      "loss": 0.6449,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.8104639053344727,
+      "rewards/margins": 0.5688613653182983,
+      "rewards/rejected": -2.3793251514434814,
       "step": 3255
     },
     {
       "epoch": 1.744773373473825,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2933906559250024,
       "learning_rate": 4.4482480352391623e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33961421251296997,
+      "logits/rejected": -0.2018471509218216,
+      "logps/chosen": -1.7063655853271484,
+      "logps/rejected": -2.332014799118042,
+      "loss": 0.6402,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.7063655853271484,
+      "rewards/margins": 0.6256491541862488,
+      "rewards/rejected": -2.332014799118042,
       "step": 3260
     },
     {
       "epoch": 1.7474494062552266,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.717362945460071,
       "learning_rate": 4.4327718247144507e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23457305133342743,
+      "logits/rejected": -0.12549740076065063,
+      "logps/chosen": -1.6968790292739868,
+      "logps/rejected": -2.471815824508667,
+      "loss": 0.6307,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6968790292739868,
+      "rewards/margins": 0.7749370336532593,
+      "rewards/rejected": -2.471815824508667,
       "step": 3265
     },
     {
       "epoch": 1.750125439036628,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 10.806814666880591,
       "learning_rate": 4.417301117439984e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.264382541179657,
+      "logits/rejected": -0.13343074917793274,
+      "logps/chosen": -1.6032546758651733,
+      "logps/rejected": -2.289067029953003,
+      "loss": 0.6317,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.6032546758651733,
+      "rewards/margins": 0.6858124136924744,
+      "rewards/rejected": -2.289067029953003,
       "step": 3270
     },
     {
       "epoch": 1.7528014718180298,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.4067640100064756,
       "learning_rate": 4.401836063512631e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2986655831336975,
+      "logits/rejected": -0.008811624720692635,
+      "logps/chosen": -1.6880327463150024,
+      "logps/rejected": -2.468693494796753,
+      "loss": 0.6396,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6880327463150024,
+      "rewards/margins": 0.7806608080863953,
+      "rewards/rejected": -2.468693494796753,
       "step": 3275
     },
     {
       "epoch": 1.7554775045994313,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.210932371686974,
       "learning_rate": 4.386376812974413e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28283125162124634,
+      "logits/rejected": -0.19131247699260712,
+      "logps/chosen": -1.651431679725647,
+      "logps/rejected": -2.360840320587158,
+      "loss": 0.6358,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.651431679725647,
+      "rewards/margins": 0.7094087600708008,
+      "rewards/rejected": -2.360840320587158,
       "step": 3280
     },
     {
       "epoch": 1.7581535373808328,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8172174625261217,
       "learning_rate": 4.370923515811048e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29546689987182617,
+      "logits/rejected": -0.08221445977687836,
+      "logps/chosen": -1.7048591375350952,
+      "logps/rejected": -2.412416458129883,
+      "loss": 0.6341,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7048591375350952,
+      "rewards/margins": 0.7075574994087219,
+      "rewards/rejected": -2.412416458129883,
       "step": 3285
     },
     {
       "epoch": 1.7608295701622345,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.621545861274173,
       "learning_rate": 4.35547632195049e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26247555017471313,
+      "logits/rejected": -0.15861615538597107,
+      "logps/chosen": -1.806140661239624,
+      "logps/rejected": -2.5648446083068848,
+      "loss": 0.6513,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.806140661239624,
+      "rewards/margins": 0.7587036490440369,
+      "rewards/rejected": -2.5648446083068848,
       "step": 3290
     },
     {
       "epoch": 1.763505602943636,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.4124831731240395,
       "learning_rate": 4.340035381261484e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26630762219429016,
+      "logits/rejected": -0.1956515908241272,
+      "logps/chosen": -1.9975216388702393,
+      "logps/rejected": -2.5921995639801025,
+      "loss": 0.6604,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.9975216388702393,
+      "rewards/margins": 0.5946779251098633,
+      "rewards/rejected": -2.5921995639801025,
       "step": 3295
     },
     {
       "epoch": 1.7661816357250375,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8285839639470667,
       "learning_rate": 4.324600843552104e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3663349747657776,
+      "logits/rejected": -0.23267126083374023,
+      "logps/chosen": -2.1042354106903076,
+      "logps/rejected": -2.869649887084961,
+      "loss": 0.6541,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.1042354106903076,
+      "rewards/margins": 0.765414297580719,
+      "rewards/rejected": -2.869649887084961,
       "step": 3300
     },
     {
       "epoch": 1.7688576685064392,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.706231255758944,
       "learning_rate": 4.309172858568302e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3727497458457947,
+      "logits/rejected": -0.23155096173286438,
+      "logps/chosen": -1.8578596115112305,
+      "logps/rejected": -2.600282669067383,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.8578596115112305,
+      "rewards/margins": 0.7424228191375732,
+      "rewards/rejected": -2.600282669067383,
       "step": 3305
     },
     {
       "epoch": 1.771533701287841,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.548652123831599,
       "learning_rate": 4.293751575992455e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2010372430086136,
+      "logits/rejected": -0.15267989039421082,
+      "logps/chosen": -1.9395782947540283,
+      "logps/rejected": -2.575082302093506,
+      "loss": 0.6506,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.9395782947540283,
+      "rewards/margins": 0.6355037689208984,
+      "rewards/rejected": -2.575082302093506,
       "step": 3310
     },
     {
       "epoch": 1.7742097340692422,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 10.112796363795793,
       "learning_rate": 4.278337145441916e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36823052167892456,
+      "logits/rejected": -0.21823231875896454,
+      "logps/chosen": -1.8513453006744385,
+      "logps/rejected": -2.6169285774230957,
+      "loss": 0.6577,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.8513453006744385,
+      "rewards/margins": 0.7655830979347229,
+      "rewards/rejected": -2.6169285774230957,
       "step": 3315
     },
     {
       "epoch": 1.776885766850644,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.158199214192057,
       "learning_rate": 4.262929716467556e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2795010805130005,
+      "logits/rejected": -0.09490416198968887,
+      "logps/chosen": -1.790374755859375,
+      "logps/rejected": -2.794741153717041,
+      "loss": 0.6451,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.790374755859375,
+      "rewards/margins": 1.0043660402297974,
+      "rewards/rejected": -2.794741153717041,
       "step": 3320
     },
     {
       "epoch": 1.7795617996320456,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.37076873801921,
       "learning_rate": 4.247529438552321e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3242601752281189,
+      "logits/rejected": -0.13930591940879822,
+      "logps/chosen": -1.8662172555923462,
+      "logps/rejected": -2.58949613571167,
+      "loss": 0.6414,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.8662172555923462,
+      "rewards/margins": 0.7232788801193237,
+      "rewards/rejected": -2.58949613571167,
       "step": 3325
     },
     {
       "epoch": 1.782237832413447,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 10.125991528559716,
       "learning_rate": 4.232136461109773e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2393198311328888,
+      "logits/rejected": -0.13017411530017853,
+      "logps/chosen": -1.6411468982696533,
+      "logps/rejected": -2.501619338989258,
+      "loss": 0.621,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.6411468982696533,
+      "rewards/margins": 0.8604723811149597,
+      "rewards/rejected": -2.501619338989258,
       "step": 3330
     },
     {
       "epoch": 1.7849138651948486,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 10.038335644024933,
       "learning_rate": 4.216750933482646e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2740943729877472,
+      "logits/rejected": -0.12172722816467285,
+      "logps/chosen": -1.895138144493103,
+      "logps/rejected": -2.4407427310943604,
+      "loss": 0.6559,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.895138144493103,
+      "rewards/margins": 0.5456045269966125,
+      "rewards/rejected": -2.4407427310943604,
       "step": 3335
     },
     {
       "epoch": 1.7875898979762503,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.821661908915419,
       "learning_rate": 4.2013730049413986e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23169226944446564,
+      "logits/rejected": -0.09255462139844894,
+      "logps/chosen": -1.5660432577133179,
+      "logps/rejected": -2.4294326305389404,
+      "loss": 0.6331,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5660432577133179,
+      "rewards/margins": 0.8633894920349121,
+      "rewards/rejected": -2.4294326305389404,
       "step": 3340
     },
     {
       "epoch": 1.7902659307576518,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.167593939408226,
       "learning_rate": 4.1860028246827594e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24397608637809753,
+      "logits/rejected": -0.07166433334350586,
+      "logps/chosen": -1.5221376419067383,
+      "logps/rejected": -2.2187047004699707,
+      "loss": 0.6323,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5221376419067383,
+      "rewards/margins": 0.6965669393539429,
+      "rewards/rejected": -2.2187047004699707,
       "step": 3345
     },
     {
       "epoch": 1.7929419635390533,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.485404380553365,
       "learning_rate": 4.170640541828285e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3768085539340973,
+      "logits/rejected": -0.24453075230121613,
+      "logps/chosen": -1.7420793771743774,
+      "logps/rejected": -2.317716121673584,
+      "loss": 0.6603,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.7420793771743774,
+      "rewards/margins": 0.5756366848945618,
+      "rewards/rejected": -2.317716121673584,
       "step": 3350
     },
     {
       "epoch": 1.795617996320455,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 11.784110227028538,
       "learning_rate": 4.1552863054229116e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13424353301525116,
+      "logits/rejected": -0.09827093034982681,
+      "logps/chosen": -1.850396752357483,
+      "logps/rejected": -2.334486484527588,
+      "loss": 0.6776,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.850396752357483,
+      "rewards/margins": 0.48408985137939453,
+      "rewards/rejected": -2.334486484527588,
       "step": 3355
     },
     {
       "epoch": 1.7982940291018565,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.043129111559157,
       "learning_rate": 4.139940264433508e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3173399567604065,
+      "logits/rejected": -0.10785996913909912,
+      "logps/chosen": -1.5856767892837524,
+      "logps/rejected": -2.346341609954834,
+      "loss": 0.6489,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5856767892837524,
+      "rewards/margins": 0.7606646418571472,
+      "rewards/rejected": -2.346341609954834,
       "step": 3360
     },
     {
       "epoch": 1.800970061883258,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.096722674455591,
       "learning_rate": 4.1246025677474303e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.305219829082489,
+      "logits/rejected": -0.14943692088127136,
+      "logps/chosen": -1.705397367477417,
+      "logps/rejected": -2.42407488822937,
+      "loss": 0.6423,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.705397367477417,
+      "rewards/margins": 0.7186775207519531,
+      "rewards/rejected": -2.42407488822937,
       "step": 3365
     },
     {
       "epoch": 1.8036460946646597,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.652917793424408,
       "learning_rate": 4.10927336417108e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2808131277561188,
+      "logits/rejected": -0.12957140803337097,
+      "logps/chosen": -1.7117106914520264,
+      "logps/rejected": -2.3442375659942627,
+      "loss": 0.6467,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7117106914520264,
+      "rewards/margins": 0.632526695728302,
+      "rewards/rejected": -2.3442375659942627,
       "step": 3370
     },
     {
       "epoch": 1.8063221274460612,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.584796409405119,
       "learning_rate": 4.093952802428457e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.14948195219039917,
+      "logits/rejected": -0.09165170788764954,
+      "logps/chosen": -1.872698426246643,
+      "logps/rejected": -2.38480281829834,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.872698426246643,
+      "rewards/margins": 0.5121043920516968,
+      "rewards/rejected": -2.38480281829834,
       "step": 3375
     },
     {
       "epoch": 1.8089981602274627,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9941750837372663,
       "learning_rate": 4.0786410311597184e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34011146426200867,
+      "logits/rejected": -0.19623064994812012,
+      "logps/chosen": -1.7286345958709717,
+      "logps/rejected": -2.40024995803833,
+      "loss": 0.6553,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7286345958709717,
+      "rewards/margins": 0.6716152429580688,
+      "rewards/rejected": -2.40024995803833,
       "step": 3380
     },
     {
       "epoch": 1.8116741930088645,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.34099015216004,
       "learning_rate": 4.063338198919737e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31016573309898376,
+      "logits/rejected": -0.282024085521698,
+      "logps/chosen": -1.8196008205413818,
+      "logps/rejected": -2.3293144702911377,
+      "loss": 0.6604,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.8196008205413818,
+      "rewards/margins": 0.5097137689590454,
+      "rewards/rejected": -2.3293144702911377,
       "step": 3385
     },
     {
       "epoch": 1.814350225790266,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.210078949381032,
       "learning_rate": 4.0480444541766575e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2707437574863434,
+      "logits/rejected": -0.1467195451259613,
+      "logps/chosen": -1.9012434482574463,
+      "logps/rejected": -2.535709857940674,
+      "loss": 0.6602,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.9012434482574463,
+      "rewards/margins": 0.6344659924507141,
+      "rewards/rejected": -2.535709857940674,
       "step": 3390
     },
     {
       "epoch": 1.8170262585716674,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.476918970141886,
       "learning_rate": 4.0327599453104606e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.295647531747818,
+      "logits/rejected": -0.1946633756160736,
+      "logps/chosen": -1.6795024871826172,
+      "logps/rejected": -2.4840247631073,
+      "loss": 0.6352,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6795024871826172,
+      "rewards/margins": 0.8045223951339722,
+      "rewards/rejected": -2.4840247631073,
       "step": 3395
     },
     {
       "epoch": 1.8197022913530692,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.645715681009244,
       "learning_rate": 4.017484820611514e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27027496695518494,
+      "logits/rejected": -0.15165910124778748,
+      "logps/chosen": -1.8216352462768555,
+      "logps/rejected": -2.5530591011047363,
+      "loss": 0.6458,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.8216352462768555,
+      "rewards/margins": 0.7314238548278809,
+      "rewards/rejected": -2.5530591011047363,
       "step": 3400
     },
     {
       "epoch": 1.8223783241344707,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.0447864893579295,
       "learning_rate": 4.002219228279148e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28753283619880676,
+      "logits/rejected": -0.13703644275665283,
+      "logps/chosen": -1.7225971221923828,
+      "logps/rejected": -2.4299230575561523,
+      "loss": 0.6658,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7225971221923828,
+      "rewards/margins": 0.7073257565498352,
+      "rewards/rejected": -2.4299230575561523,
       "step": 3405
     },
     {
       "epoch": 1.8250543569158721,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1153570040462735,
       "learning_rate": 3.9869633164202045e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2813507914543152,
+      "logits/rejected": -0.06477998197078705,
+      "logps/chosen": -1.967111587524414,
+      "logps/rejected": -2.6441922187805176,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.967111587524414,
+      "rewards/margins": 0.6770805716514587,
+      "rewards/rejected": -2.6441922187805176,
       "step": 3410
     },
     {
       "epoch": 1.8277303896972739,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2439649459597475,
       "learning_rate": 3.9717172330476077e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27776211500167847,
+      "logits/rejected": -0.17397698760032654,
+      "logps/chosen": -1.8295953273773193,
+      "logps/rejected": -2.6177358627319336,
+      "loss": 0.662,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8295953273773193,
+      "rewards/margins": 0.7881405353546143,
+      "rewards/rejected": -2.6177358627319336,
       "step": 3415
     },
     {
       "epoch": 1.8304064224786754,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.251003597832578,
       "learning_rate": 3.956481126078927e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2295829802751541,
+      "logits/rejected": -0.10351558774709702,
+      "logps/chosen": -2.0241928100585938,
+      "logps/rejected": -2.7826108932495117,
+      "loss": 0.6611,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.0241928100585938,
+      "rewards/margins": 0.7584178447723389,
+      "rewards/rejected": -2.7826108932495117,
       "step": 3420
     },
     {
       "epoch": 1.8330824552600768,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8420838576074074,
       "learning_rate": 3.941255143334937e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3079303205013275,
+      "logits/rejected": -0.25670403242111206,
+      "logps/chosen": -1.7964670658111572,
+      "logps/rejected": -2.5763087272644043,
+      "loss": 0.6492,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7964670658111572,
+      "rewards/margins": 0.7798420190811157,
+      "rewards/rejected": -2.5763087272644043,
       "step": 3425
     },
     {
       "epoch": 1.8357584880414786,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 16.55509774924355,
       "learning_rate": 3.9260394325381895e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3037746250629425,
+      "logits/rejected": -0.17107325792312622,
+      "logps/chosen": -1.798728346824646,
+      "logps/rejected": -2.998878002166748,
+      "loss": 0.6606,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.798728346824646,
+      "rewards/margins": 1.2001495361328125,
+      "rewards/rejected": -2.998878002166748,
       "step": 3430
     },
     {
       "epoch": 1.83843452082288,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.360555405206908,
       "learning_rate": 3.9108341413115784e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2811899185180664,
+      "logits/rejected": -0.17460323870182037,
+      "logps/chosen": -1.707069754600525,
+      "logps/rejected": -2.543130397796631,
+      "loss": 0.6267,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.707069754600525,
+      "rewards/margins": 0.8360604047775269,
+      "rewards/rejected": -2.543130397796631,
       "step": 3435
     },
     {
       "epoch": 1.8411105536042816,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.522593103338406,
       "learning_rate": 3.895639417176905e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34337347745895386,
+      "logits/rejected": -0.2677827477455139,
+      "logps/chosen": -1.6915746927261353,
+      "logps/rejected": -2.7019612789154053,
+      "loss": 0.6541,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.6915746927261353,
+      "rewards/margins": 1.010386347770691,
+      "rewards/rejected": -2.7019612789154053,
       "step": 3440
     },
     {
       "epoch": 1.8437865863856833,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.48263004840494,
       "learning_rate": 3.8804554075534497e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33988475799560547,
+      "logits/rejected": -0.13183362782001495,
+      "logps/chosen": -1.8456426858901978,
+      "logps/rejected": -2.6009817123413086,
+      "loss": 0.6473,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.8456426858901978,
+      "rewards/margins": 0.7553392648696899,
+      "rewards/rejected": -2.6009817123413086,
       "step": 3445
     },
     {
       "epoch": 1.8464626191670848,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.595008907641011,
       "learning_rate": 3.8652822597565403e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.41368597745895386,
+      "logits/rejected": -0.23327429592609406,
+      "logps/chosen": -1.7736181020736694,
+      "logps/rejected": -2.571000576019287,
+      "loss": 0.6415,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7736181020736694,
+      "rewards/margins": 0.7973822355270386,
+      "rewards/rejected": -2.571000576019287,
       "step": 3450
     },
     {
       "epoch": 1.8491386519484863,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.902331194487124,
       "learning_rate": 3.850120120996123e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27916446328163147,
+      "logits/rejected": -0.10583939403295517,
+      "logps/chosen": -1.9777143001556396,
+      "logps/rejected": -2.704444169998169,
+      "loss": 0.6611,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.9777143001556396,
+      "rewards/margins": 0.7267298102378845,
+      "rewards/rejected": -2.704444169998169,
       "step": 3455
     },
     {
       "epoch": 1.851814684729888,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.227738188815731,
       "learning_rate": 3.8349691383753356e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19867241382598877,
+      "logits/rejected": -0.08401435613632202,
+      "logps/chosen": -1.8392589092254639,
+      "logps/rejected": -2.520399570465088,
+      "loss": 0.675,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.8392589092254639,
+      "rewards/margins": 0.6811404824256897,
+      "rewards/rejected": -2.520399570465088,
       "step": 3460
     },
     {
       "epoch": 1.8544907175112895,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4516270418409105,
       "learning_rate": 3.819829458889078e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3314283788204193,
+      "logits/rejected": -0.2166227400302887,
+      "logps/chosen": -1.6118297576904297,
+      "logps/rejected": -2.2584919929504395,
+      "loss": 0.6475,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6118297576904297,
+      "rewards/margins": 0.6466619968414307,
+      "rewards/rejected": -2.2584919929504395,
       "step": 3465
     },
     {
       "epoch": 1.857166750292691,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7327894004952773,
       "learning_rate": 3.804701229422585e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.35179510712623596,
+      "logits/rejected": -0.25943320989608765,
+      "logps/chosen": -1.9293190240859985,
+      "logps/rejected": -2.535024881362915,
+      "loss": 0.654,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.9293190240859985,
+      "rewards/margins": 0.605705738067627,
+      "rewards/rejected": -2.535024881362915,
       "step": 3470
     },
     {
       "epoch": 1.8598427830740927,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4881891868906605,
       "learning_rate": 3.789584596750007e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3454849123954773,
+      "logits/rejected": -0.2847757041454315,
+      "logps/chosen": -1.792888879776001,
+      "logps/rejected": -2.4048995971679688,
+      "loss": 0.6391,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.792888879776001,
+      "rewards/margins": 0.6120105981826782,
+      "rewards/rejected": -2.4048995971679688,
       "step": 3475
     },
     {
       "epoch": 1.8625188158554944,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5091771533748326,
       "learning_rate": 3.77447970753298e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23524720966815948,
+      "logits/rejected": -0.20086169242858887,
+      "logps/chosen": -1.8092292547225952,
+      "logps/rejected": -2.4469611644744873,
+      "loss": 0.6411,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8092292547225952,
+      "rewards/margins": 0.637732207775116,
+      "rewards/rejected": -2.4469611644744873,
       "step": 3480
     },
     {
       "epoch": 1.8651948486368957,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.2416801470382595,
       "learning_rate": 3.7593867083192057e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2664981484413147,
+      "logits/rejected": -0.14917102456092834,
+      "logps/chosen": -1.7971121072769165,
+      "logps/rejected": -2.480259895324707,
+      "loss": 0.6607,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7971121072769165,
+      "rewards/margins": 0.6831477880477905,
+      "rewards/rejected": -2.480259895324707,
       "step": 3485
     },
     {
       "epoch": 1.8678708814182974,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0659739759669735,
       "learning_rate": 3.7443057455410276e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25824278593063354,
+      "logits/rejected": -0.1416032463312149,
+      "logps/chosen": -1.6490789651870728,
+      "logps/rejected": -2.344550132751465,
+      "loss": 0.6445,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6490789651870728,
+      "rewards/margins": 0.6954714059829712,
+      "rewards/rejected": -2.344550132751465,
       "step": 3490
     },
     {
       "epoch": 1.870546914199699,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.891367150434508,
       "learning_rate": 3.7292369655140145e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.365573912858963,
+      "logits/rejected": -0.20765963196754456,
+      "logps/chosen": -1.853293776512146,
+      "logps/rejected": -2.431737184524536,
+      "loss": 0.6486,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.853293776512146,
+      "rewards/margins": 0.5784434080123901,
+      "rewards/rejected": -2.431737184524536,
       "step": 3495
     },
     {
       "epoch": 1.8732229469811004,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.758645628999563,
       "learning_rate": 3.714180514435534e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2675396502017975,
+      "logits/rejected": -0.11602430045604706,
+      "logps/chosen": -1.8958715200424194,
+      "logps/rejected": -2.7268686294555664,
+      "loss": 0.6377,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8958715200424194,
+      "rewards/margins": 0.8309972882270813,
+      "rewards/rejected": -2.7268686294555664,
       "step": 3500
     },
     {
       "epoch": 1.875898979762502,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9133179940146294,
       "learning_rate": 3.6991365383833426e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27402132749557495,
+      "logits/rejected": -0.15270105004310608,
+      "logps/chosen": -1.7927844524383545,
+      "logps/rejected": -2.54477858543396,
+      "loss": 0.6503,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7927844524383545,
+      "rewards/margins": 0.7519937753677368,
+      "rewards/rejected": -2.54477858543396,
       "step": 3505
     },
     {
       "epoch": 1.8785750125439038,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.174802020459413,
       "learning_rate": 3.684105183314162e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3005546033382416,
+      "logits/rejected": -0.22739270329475403,
+      "logps/chosen": -1.6991758346557617,
+      "logps/rejected": -2.440171003341675,
+      "loss": 0.6372,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.6991758346557617,
+      "rewards/margins": 0.7409950494766235,
+      "rewards/rejected": -2.440171003341675,
       "step": 3510
     },
     {
       "epoch": 1.881251045325305,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.884096496225365,
       "learning_rate": 3.669086595062263e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3000214993953705,
+      "logits/rejected": -0.11149589717388153,
+      "logps/chosen": -1.7275785207748413,
+      "logps/rejected": -2.4624385833740234,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7275785207748413,
+      "rewards/margins": 0.7348599433898926,
+      "rewards/rejected": -2.4624385833740234,
       "step": 3515
     },
     {
       "epoch": 1.8839270781067068,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.513962856595358,
       "learning_rate": 3.654080919338056e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3341074585914612,
+      "logits/rejected": -0.19895395636558533,
+      "logps/chosen": -1.7745752334594727,
+      "logps/rejected": -2.4967024326324463,
+      "loss": 0.6505,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7745752334594727,
+      "rewards/margins": 0.7221272587776184,
+      "rewards/rejected": -2.4967024326324463,
       "step": 3520
     },
     {
       "epoch": 1.8866031108881085,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.068376558366414,
       "learning_rate": 3.639088301726673e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25603416562080383,
+      "logits/rejected": -0.06728498637676239,
+      "logps/chosen": -1.7568079233169556,
+      "logps/rejected": -2.4309723377227783,
+      "loss": 0.6476,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7568079233169556,
+      "rewards/margins": 0.6741644144058228,
+      "rewards/rejected": -2.4309723377227783,
       "step": 3525
     },
     {
       "epoch": 1.88927914366951,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.937163442842716,
       "learning_rate": 3.624108887686556e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2659794092178345,
+      "logits/rejected": -0.19908373057842255,
+      "logps/chosen": -1.7431881427764893,
+      "logps/rejected": -2.332472562789917,
+      "loss": 0.6542,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7431881427764893,
+      "rewards/margins": 0.5892844796180725,
+      "rewards/rejected": -2.332472562789917,
       "step": 3530
     },
     {
       "epoch": 1.8919551764509115,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.097476914134114,
       "learning_rate": 3.6091428225480433e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.35586485266685486,
+      "logits/rejected": -0.23673877120018005,
+      "logps/chosen": -1.633011817932129,
+      "logps/rejected": -2.3713784217834473,
+      "loss": 0.649,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.633011817932129,
+      "rewards/margins": 0.7383664846420288,
+      "rewards/rejected": -2.3713784217834473,
       "step": 3535
     },
     {
       "epoch": 1.8946312092323132,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 13.39979705999114,
       "learning_rate": 3.5941902515119674e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32095763087272644,
+      "logits/rejected": -0.09967707097530365,
+      "logps/chosen": -1.7250210046768188,
+      "logps/rejected": -2.278024196624756,
+      "loss": 0.6484,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7250210046768188,
+      "rewards/margins": 0.5530033111572266,
+      "rewards/rejected": -2.278024196624756,
       "step": 3540
     },
     {
       "epoch": 1.8973072420137147,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.250817732773738,
       "learning_rate": 3.5792513196482373e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.445732980966568,
+      "logits/rejected": -0.20107467472553253,
+      "logps/chosen": -1.6788619756698608,
+      "logps/rejected": -2.362474203109741,
+      "loss": 0.6506,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6788619756698608,
+      "rewards/margins": 0.6836119890213013,
+      "rewards/rejected": -2.362474203109741,
       "step": 3545
     },
     {
       "epoch": 1.8999832747951162,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9289289196422272,
       "learning_rate": 3.5643261718944346e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2657968997955322,
+      "logits/rejected": -0.18451794981956482,
+      "logps/chosen": -1.774898886680603,
+      "logps/rejected": -2.333242416381836,
+      "loss": 0.6579,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.774898886680603,
+      "rewards/margins": 0.5583432912826538,
+      "rewards/rejected": -2.333242416381836,
       "step": 3550
     },
     {
       "epoch": 1.902659307576518,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.188780011159042,
       "learning_rate": 3.5494149530544087e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3926597833633423,
+      "logits/rejected": -0.26186755299568176,
+      "logps/chosen": -1.6639636754989624,
+      "logps/rejected": -2.427198886871338,
+      "loss": 0.6555,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6639636754989624,
+      "rewards/margins": 0.7632350921630859,
+      "rewards/rejected": -2.427198886871338,
       "step": 3555
     },
     {
       "epoch": 1.9053353403579194,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.784762444293204,
       "learning_rate": 3.534517807796871e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30003243684768677,
+      "logits/rejected": -0.21647843718528748,
+      "logps/chosen": -1.768978476524353,
+      "logps/rejected": -2.492739677429199,
+      "loss": 0.6518,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.768978476524353,
+      "rewards/margins": 0.7237612009048462,
+      "rewards/rejected": -2.492739677429199,
       "step": 3560
     },
     {
       "epoch": 1.908011373139321,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.274412352466071,
       "learning_rate": 3.519634880653988e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25581759214401245,
+      "logits/rejected": -0.18214426934719086,
+      "logps/chosen": -1.7457149028778076,
+      "logps/rejected": -2.645226240158081,
+      "loss": 0.641,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7457149028778076,
+      "rewards/margins": 0.8995113372802734,
+      "rewards/rejected": -2.645226240158081,
       "step": 3565
     },
     {
       "epoch": 1.9106874059207226,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.72129917983072,
       "learning_rate": 3.504766316019987e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33632293343544006,
+      "logits/rejected": -0.18968139588832855,
+      "logps/chosen": -1.7234680652618408,
+      "logps/rejected": -2.455012798309326,
+      "loss": 0.6407,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.7234680652618408,
+      "rewards/margins": 0.731544554233551,
+      "rewards/rejected": -2.455012798309326,
       "step": 3570
     },
     {
       "epoch": 1.913363438702124,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7090202357949495,
       "learning_rate": 3.489912258149745e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23553848266601562,
+      "logits/rejected": -0.11940286308526993,
+      "logps/chosen": -1.7654396295547485,
+      "logps/rejected": -2.5256149768829346,
+      "loss": 0.656,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7654396295547485,
+      "rewards/margins": 0.7601752877235413,
+      "rewards/rejected": -2.5256149768829346,
       "step": 3575
     },
     {
       "epoch": 1.9160394714835256,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.2459281506523165,
       "learning_rate": 3.475072851157397e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29631006717681885,
+      "logits/rejected": -0.24821333587169647,
+      "logps/chosen": -1.681279182434082,
+      "logps/rejected": -2.5801539421081543,
+      "loss": 0.6318,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.681279182434082,
+      "rewards/margins": 0.8988748788833618,
+      "rewards/rejected": -2.5801539421081543,
       "step": 3580
     },
     {
       "epoch": 1.9187155042649273,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6015231291000336,
       "learning_rate": 3.460248239014936e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2192951887845993,
+      "logits/rejected": -0.1593814492225647,
+      "logps/chosen": -1.8684778213500977,
+      "logps/rejected": -2.5493276119232178,
+      "loss": 0.6331,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8684778213500977,
+      "rewards/margins": 0.6808497309684753,
+      "rewards/rejected": -2.5493276119232178,
       "step": 3585
     },
     {
       "epoch": 1.9213915370463288,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.685389459836955,
       "learning_rate": 3.4454385655508134e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.257930725812912,
+      "logits/rejected": -0.1985943764448166,
+      "logps/chosen": -1.8187906742095947,
+      "logps/rejected": -2.2867140769958496,
+      "loss": 0.6536,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.8187906742095947,
+      "rewards/margins": 0.4679235517978668,
+      "rewards/rejected": -2.2867140769958496,
       "step": 3590
     },
     {
       "epoch": 1.9240675698277303,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.340606886725828,
       "learning_rate": 3.4306439744485447e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.39070749282836914,
+      "logits/rejected": -0.19286325573921204,
+      "logps/chosen": -1.8034741878509521,
+      "logps/rejected": -2.591022491455078,
+      "loss": 0.6492,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8034741878509521,
+      "rewards/margins": 0.7875484228134155,
+      "rewards/rejected": -2.591022491455078,
       "step": 3595
     },
     {
       "epoch": 1.926743602609132,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.7493777477915975,
       "learning_rate": 3.415864609245322e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22314944863319397,
+      "logits/rejected": -0.05440715700387955,
+      "logps/chosen": -1.8342784643173218,
+      "logps/rejected": -2.714042901992798,
+      "loss": 0.6481,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8342784643173218,
+      "rewards/margins": 0.8797640800476074,
+      "rewards/rejected": -2.714042901992798,
       "step": 3600
     },
     {
       "epoch": 1.926743602609132,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.9181,
-      "eval_samples_per_second": 38.519,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.651,
+      "eval_logits/chosen": 0.011269119568169117,
+      "eval_logits/rejected": 0.09392748773097992,
+      "eval_logps/chosen": -1.7357550859451294,
+      "eval_logps/rejected": -2.4632444381713867,
+      "eval_loss": 0.6507142782211304,
+      "eval_rewards/accuracies": 0.6735904812812805,
+      "eval_rewards/chosen": -1.7357550859451294,
+      "eval_rewards/margins": 0.7274892330169678,
+      "eval_rewards/rejected": -2.4632444381713867,
+      "eval_runtime": 40.502,
+      "eval_samples_per_second": 33.208,
+      "eval_steps_per_second": 8.321,
       "step": 3600
     },
     {
       "epoch": 1.9294196353905335,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.3333788987276565,
       "learning_rate": 3.401100613330605e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3214555084705353,
+      "logits/rejected": -0.28421396017074585,
+      "logps/chosen": -1.7438173294067383,
+      "logps/rejected": -2.2593116760253906,
+      "loss": 0.654,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7438173294067383,
+      "rewards/margins": 0.5154942870140076,
+      "rewards/rejected": -2.2593116760253906,
       "step": 3605
     },
     {
       "epoch": 1.932095668171935,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5679888364660473,
       "learning_rate": 3.3863521299447514e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.257505863904953,
+      "logits/rejected": -0.14152228832244873,
+      "logps/chosen": -1.7479839324951172,
+      "logps/rejected": -2.488645076751709,
+      "loss": 0.6407,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7479839324951172,
+      "rewards/margins": 0.7406615018844604,
+      "rewards/rejected": -2.488645076751709,
       "step": 3610
     },
     {
       "epoch": 1.9347717009533367,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2775350240591017,
       "learning_rate": 3.371619302177609e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19945763051509857,
+      "logits/rejected": -0.08808594942092896,
+      "logps/chosen": -1.7347023487091064,
+      "logps/rejected": -2.3722901344299316,
+      "loss": 0.6588,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.7347023487091064,
+      "rewards/margins": 0.6375876665115356,
+      "rewards/rejected": -2.3722901344299316,
       "step": 3615
     },
     {
       "epoch": 1.9374477337347382,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.59394591914742,
       "learning_rate": 3.3569022729671393e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2541313171386719,
+      "logits/rejected": -0.1862151324748993,
+      "logps/chosen": -1.8164138793945312,
+      "logps/rejected": -2.363574266433716,
+      "loss": 0.6438,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.8164138793945312,
+      "rewards/margins": 0.5471606850624084,
+      "rewards/rejected": -2.363574266433716,
       "step": 3620
     },
     {
       "epoch": 1.9401237665161397,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8652791572250735,
       "learning_rate": 3.342201185098024e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.20091740787029266,
+      "logits/rejected": -0.18552783131599426,
+      "logps/chosen": -1.7096580266952515,
+      "logps/rejected": -2.2215349674224854,
+      "loss": 0.6452,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7096580266952515,
+      "rewards/margins": 0.5118769407272339,
+      "rewards/rejected": -2.2215349674224854,
       "step": 3625
     },
     {
       "epoch": 1.9427997992975414,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.091003930110245,
       "learning_rate": 3.3275161812002807e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29093703627586365,
+      "logits/rejected": -0.24474754929542542,
+      "logps/chosen": -1.7577886581420898,
+      "logps/rejected": -2.37738299369812,
+      "loss": 0.6543,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.7577886581420898,
+      "rewards/margins": 0.619594395160675,
+      "rewards/rejected": -2.37738299369812,
       "step": 3630
     },
     {
       "epoch": 1.945475832078943,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.344477139273824,
       "learning_rate": 3.312847403747883e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30554112792015076,
+      "logits/rejected": -0.21518298983573914,
+      "logps/chosen": -1.7132161855697632,
+      "logps/rejected": -2.2776951789855957,
+      "loss": 0.6517,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.7132161855697632,
+      "rewards/margins": 0.5644789934158325,
+      "rewards/rejected": -2.2776951789855957,
       "step": 3635
     },
     {
       "epoch": 1.9481518648603444,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.589004826791331,
       "learning_rate": 3.2981949950573733e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25908124446868896,
+      "logits/rejected": -0.15341386198997498,
+      "logps/chosen": -1.8757848739624023,
+      "logps/rejected": -2.2695398330688477,
+      "loss": 0.6575,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.8757848739624023,
+      "rewards/margins": 0.3937550485134125,
+      "rewards/rejected": -2.2695398330688477,
       "step": 3640
     },
     {
       "epoch": 1.9508278976417461,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7514623237768485,
       "learning_rate": 3.283559097286486e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31307995319366455,
+      "logits/rejected": -0.1968092918395996,
+      "logps/chosen": -1.8629028797149658,
+      "logps/rejected": -2.231245517730713,
+      "loss": 0.6589,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.8629028797149658,
+      "rewards/margins": 0.36834239959716797,
+      "rewards/rejected": -2.231245517730713,
       "step": 3645
     },
     {
       "epoch": 1.9535039304231478,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.213092801643786,
       "learning_rate": 3.268939852432765e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.330477774143219,
+      "logits/rejected": -0.2381499707698822,
+      "logps/chosen": -1.7277549505233765,
+      "logps/rejected": -2.293978214263916,
+      "loss": 0.6414,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7277549505233765,
+      "rewards/margins": 0.5662233233451843,
+      "rewards/rejected": -2.293978214263916,
       "step": 3650
     },
     {
       "epoch": 1.9561799632045491,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.639030830121628,
       "learning_rate": 3.254337402332187e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27924010157585144,
+      "logits/rejected": -0.1512012779712677,
+      "logps/chosen": -1.8996235132217407,
+      "logps/rejected": -2.461909532546997,
+      "loss": 0.6654,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.8996235132217407,
+      "rewards/margins": 0.5622859597206116,
+      "rewards/rejected": -2.461909532546997,
       "step": 3655
     },
     {
       "epoch": 1.9588559959859508,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.080892743519266,
       "learning_rate": 3.239751888657788e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32951053977012634,
+      "logits/rejected": -0.21374821662902832,
+      "logps/chosen": -1.680132508277893,
+      "logps/rejected": -2.459019184112549,
+      "loss": 0.6446,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.680132508277893,
+      "rewards/margins": 0.7788869142532349,
+      "rewards/rejected": -2.459019184112549,
       "step": 3660
     },
     {
       "epoch": 1.9615320287673526,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.223143137395554,
       "learning_rate": 3.2251834529182856e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2693172097206116,
+      "logits/rejected": -0.15960440039634705,
+      "logps/chosen": -1.6137605905532837,
+      "logps/rejected": -2.4235005378723145,
+      "loss": 0.6396,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6137605905532837,
+      "rewards/margins": 0.8097399473190308,
+      "rewards/rejected": -2.4235005378723145,
       "step": 3665
     },
     {
       "epoch": 1.9642080615487538,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.384550045675292,
       "learning_rate": 3.2106322364567075e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31217432022094727,
+      "logits/rejected": -0.17984290421009064,
+      "logps/chosen": -1.6787456274032593,
+      "logps/rejected": -2.5007739067077637,
+      "loss": 0.6332,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6787456274032593,
+      "rewards/margins": 0.8220283389091492,
+      "rewards/rejected": -2.5007739067077637,
       "step": 3670
     },
     {
       "epoch": 1.9668840943301555,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.889443794599803,
       "learning_rate": 3.1960983804490183e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32620900869369507,
+      "logits/rejected": -0.1794060468673706,
+      "logps/chosen": -1.7892608642578125,
+      "logps/rejected": -2.613853931427002,
+      "loss": 0.644,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7892608642578125,
+      "rewards/margins": 0.8245927095413208,
+      "rewards/rejected": -2.613853931427002,
       "step": 3675
     },
     {
       "epoch": 1.9695601271115573,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.178528344668754,
       "learning_rate": 3.1815820259027537e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3021976351737976,
+      "logits/rejected": -0.1883808970451355,
+      "logps/chosen": -1.5749704837799072,
+      "logps/rejected": -2.2422590255737305,
+      "loss": 0.6282,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5749704837799072,
+      "rewards/margins": 0.6672886610031128,
+      "rewards/rejected": -2.2422590255737305,
       "step": 3680
     },
     {
       "epoch": 1.9722361598929585,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.791315159457073,
       "learning_rate": 3.16708331365565e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3310215473175049,
+      "logits/rejected": -0.24974718689918518,
+      "logps/chosen": -1.6960508823394775,
+      "logps/rejected": -2.441659688949585,
+      "loss": 0.6311,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6960508823394775,
+      "rewards/margins": 0.7456088066101074,
+      "rewards/rejected": -2.441659688949585,
       "step": 3685
     },
     {
       "epoch": 1.9749121926743602,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.078580813205106,
       "learning_rate": 3.152602384374275e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2878320813179016,
+      "logits/rejected": -0.12999853491783142,
+      "logps/chosen": -1.7542734146118164,
+      "logps/rejected": -2.547133684158325,
+      "loss": 0.6523,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7542734146118164,
+      "rewards/margins": 0.7928601503372192,
+      "rewards/rejected": -2.547133684158325,
       "step": 3690
     },
     {
       "epoch": 1.977588225455762,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8508594042174447,
       "learning_rate": 3.1381393785526697e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25654295086860657,
+      "logits/rejected": -0.20818956196308136,
+      "logps/chosen": -1.8498799800872803,
+      "logps/rejected": -2.6051557064056396,
+      "loss": 0.6428,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8498799800872803,
+      "rewards/margins": 0.7552758455276489,
+      "rewards/rejected": -2.6051557064056396,
       "step": 3695
     },
     {
       "epoch": 1.9802642582371635,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.084001666659813,
       "learning_rate": 3.123694436510979e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2247171401977539,
+      "logits/rejected": -0.11395609378814697,
+      "logps/chosen": -1.7278048992156982,
+      "logps/rejected": -2.371425151824951,
+      "loss": 0.643,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7278048992156982,
+      "rewards/margins": 0.6436203122138977,
+      "rewards/rejected": -2.371425151824951,
       "step": 3700
     },
     {
       "epoch": 1.982940291018565,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.445050161489767,
       "learning_rate": 3.1092676983940946e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2709904611110687,
+      "logits/rejected": -0.1995052844285965,
+      "logps/chosen": -1.7405210733413696,
+      "logps/rejected": -2.539783000946045,
+      "loss": 0.6446,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7405210733413696,
+      "rewards/margins": 0.7992619276046753,
+      "rewards/rejected": -2.539783000946045,
       "step": 3705
     },
     {
       "epoch": 1.9856163237999667,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.144902938696415,
       "learning_rate": 3.094859304170293e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.13747677206993103,
+      "logits/rejected": -0.0701674073934555,
+      "logps/chosen": -1.6963932514190674,
+      "logps/rejected": -2.339641809463501,
+      "loss": 0.6448,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6963932514190674,
+      "rewards/margins": 0.6432483196258545,
+      "rewards/rejected": -2.339641809463501,
       "step": 3710
     },
     {
       "epoch": 1.9882923565813682,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2071839029954963,
       "learning_rate": 3.0804693936298795e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19925697147846222,
+      "logits/rejected": -0.1288217008113861,
+      "logps/chosen": -1.776210069656372,
+      "logps/rejected": -2.592970132827759,
+      "loss": 0.6419,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.776210069656372,
+      "rewards/margins": 0.8167603611946106,
+      "rewards/rejected": -2.592970132827759,
       "step": 3715
     },
     {
       "epoch": 1.9909683893627697,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.853504747333402,
       "learning_rate": 3.066098106383826e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25840967893600464,
+      "logits/rejected": -0.1785687804222107,
+      "logps/chosen": -1.7179828882217407,
+      "logps/rejected": -2.30916690826416,
+      "loss": 0.6617,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7179828882217407,
+      "rewards/margins": 0.5911839604377747,
+      "rewards/rejected": -2.30916690826416,
       "step": 3720
     },
     {
       "epoch": 1.9936444221441714,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.261834457985911,
       "learning_rate": 3.0517455818624263e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33893775939941406,
+      "logits/rejected": -0.24609117209911346,
+      "logps/chosen": -1.755340814590454,
+      "logps/rejected": -2.4367339611053467,
+      "loss": 0.6516,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.755340814590454,
+      "rewards/margins": 0.6813931465148926,
+      "rewards/rejected": -2.4367339611053467,
       "step": 3725
     },
     {
       "epoch": 1.9963204549255729,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9129698055952233,
       "learning_rate": 3.037411959313936e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24086007475852966,
+      "logits/rejected": -0.11738632619380951,
+      "logps/chosen": -1.7126209735870361,
+      "logps/rejected": -2.386350393295288,
+      "loss": 0.6335,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7126209735870361,
+      "rewards/margins": 0.6737293004989624,
+      "rewards/rejected": -2.386350393295288,
       "step": 3730
     },
     {
       "epoch": 1.9989964877069744,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.562207923001707,
       "learning_rate": 3.023097377803224e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17555740475654602,
+      "logits/rejected": -0.09611587971448898,
+      "logps/chosen": -1.836539626121521,
+      "logps/rejected": -2.288254976272583,
+      "loss": 0.6751,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.836539626121521,
+      "rewards/margins": 0.45171523094177246,
+      "rewards/rejected": -2.288254976272583,
       "step": 3735
     },
     {
       "epoch": 2.001672520488376,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0079564409963946,
       "learning_rate": 3.008801976210423e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2095160037279129,
+      "logits/rejected": -0.1528283655643463,
+      "logps/chosen": -1.8421785831451416,
+      "logps/rejected": -2.438142776489258,
+      "loss": 0.649,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.8421785831451416,
+      "rewards/margins": 0.5959640741348267,
+      "rewards/rejected": -2.438142776489258,
       "step": 3740
     },
     {
       "epoch": 2.0043485532697773,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.687304265797114,
       "learning_rate": 2.994525893229581e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23508930206298828,
+      "logits/rejected": -0.14902694523334503,
+      "logps/chosen": -1.829967737197876,
+      "logps/rejected": -2.450139045715332,
+      "loss": 0.6456,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.829967737197876,
+      "rewards/margins": 0.6201715469360352,
+      "rewards/rejected": -2.450139045715332,
       "step": 3745
     },
     {
       "epoch": 2.007024586051179,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.45293244843537,
       "learning_rate": 2.98026926736732e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29503563046455383,
+      "logits/rejected": -0.20740416646003723,
+      "logps/chosen": -1.684440016746521,
+      "logps/rejected": -2.4244980812072754,
+      "loss": 0.6133,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.684440016746521,
+      "rewards/margins": 0.7400578260421753,
+      "rewards/rejected": -2.4244980812072754,
       "step": 3750
     },
     {
       "epoch": 2.0097006188325808,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.54819256702016,
       "learning_rate": 2.9660322369414846e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2591850161552429,
+      "logits/rejected": -0.15787804126739502,
+      "logps/chosen": -1.652494192123413,
+      "logps/rejected": -2.6242923736572266,
+      "loss": 0.6148,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.652494192123413,
+      "rewards/margins": 0.9717981219291687,
+      "rewards/rejected": -2.6242923736572266,
       "step": 3755
     },
     {
       "epoch": 2.0123766516139825,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.855377564723784,
       "learning_rate": 2.9518149400798063e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.35640662908554077,
+      "logits/rejected": -0.32270240783691406,
+      "logps/chosen": -1.718248963356018,
+      "logps/rejected": -2.658128261566162,
+      "loss": 0.6232,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.718248963356018,
+      "rewards/margins": 0.9398797154426575,
+      "rewards/rejected": -2.658128261566162,
       "step": 3760
     },
     {
       "epoch": 2.0150526843953838,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.180279692493725,
       "learning_rate": 2.9376175147185633e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23468975722789764,
+      "logits/rejected": -0.04748193547129631,
+      "logps/chosen": -1.8464765548706055,
+      "logps/rejected": -2.7129173278808594,
+      "loss": 0.6385,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8464765548706055,
+      "rewards/margins": 0.86644047498703,
+      "rewards/rejected": -2.7129173278808594,
       "step": 3765
     },
     {
       "epoch": 2.0177287171767855,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.308813260566101,
       "learning_rate": 2.9234400986012376e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36255311965942383,
+      "logits/rejected": -0.20302942395210266,
+      "logps/chosen": -1.6834214925765991,
+      "logps/rejected": -2.7001051902770996,
+      "loss": 0.6104,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.6834214925765991,
+      "rewards/margins": 1.01668381690979,
+      "rewards/rejected": -2.7001051902770996,
       "step": 3770
     },
     {
       "epoch": 2.020404749958187,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4890930767989867,
       "learning_rate": 2.9092828292771817e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27233198285102844,
+      "logits/rejected": -0.21808934211730957,
+      "logps/chosen": -1.857740044593811,
+      "logps/rejected": -2.4746203422546387,
+      "loss": 0.6491,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.857740044593811,
+      "rewards/margins": 0.6168803572654724,
+      "rewards/rejected": -2.4746203422546387,
       "step": 3775
     },
     {
       "epoch": 2.0230807827395885,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8618640832834905,
       "learning_rate": 2.8951458441002875e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2346903532743454,
+      "logits/rejected": -0.2048753798007965,
+      "logps/chosen": -1.785418152809143,
+      "logps/rejected": -2.528390645980835,
+      "loss": 0.6423,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.785418152809143,
+      "rewards/margins": 0.7429728507995605,
+      "rewards/rejected": -2.528390645980835,
       "step": 3780
     },
     {
       "epoch": 2.02575681552099,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 18.302283582327735,
       "learning_rate": 2.881029280227643e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2991737723350525,
+      "logits/rejected": -0.1729886829853058,
+      "logps/chosen": -1.738761305809021,
+      "logps/rejected": -2.631852626800537,
+      "loss": 0.6335,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.738761305809021,
+      "rewards/margins": 0.8930912017822266,
+      "rewards/rejected": -2.631852626800537,
       "step": 3785
     },
     {
       "epoch": 2.028432848302392,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.271616571922266,
       "learning_rate": 2.8669332746182177e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32839930057525635,
+      "logits/rejected": -0.16898956894874573,
+      "logps/chosen": -1.697758674621582,
+      "logps/rejected": -2.6285202503204346,
+      "loss": 0.609,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.697758674621582,
+      "rewards/margins": 0.9307616949081421,
+      "rewards/rejected": -2.6285202503204346,
       "step": 3790
     },
     {
       "epoch": 2.031108881083793,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8348231286472627,
       "learning_rate": 2.8528579640315156e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23574522137641907,
+      "logits/rejected": -0.20986194908618927,
+      "logps/chosen": -1.7152926921844482,
+      "logps/rejected": -2.314023971557617,
+      "loss": 0.6412,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7152926921844482,
+      "rewards/margins": 0.5987311601638794,
+      "rewards/rejected": -2.314023971557617,
       "step": 3795
     },
     {
       "epoch": 2.033784913865195,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.05055629285019,
       "learning_rate": 2.8388034850262646e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26121801137924194,
+      "logits/rejected": -0.13769063353538513,
+      "logps/chosen": -1.8534374237060547,
+      "logps/rejected": -2.484135150909424,
+      "loss": 0.6262,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8534374237060547,
+      "rewards/margins": 0.6306976675987244,
+      "rewards/rejected": -2.484135150909424,
       "step": 3800
     },
     {
       "epoch": 2.0364609466465966,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.205847010113773,
       "learning_rate": 2.824769973959079e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23032322525978088,
+      "logits/rejected": -0.11731795966625214,
+      "logps/chosen": -1.6673457622528076,
+      "logps/rejected": -2.311662197113037,
+      "loss": 0.6312,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6673457622528076,
+      "rewards/margins": 0.6443161964416504,
+      "rewards/rejected": -2.311662197113037,
       "step": 3805
     },
     {
       "epoch": 2.039136979427998,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.65018835313217,
       "learning_rate": 2.81075756698315e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.16865390539169312,
+      "logits/rejected": -0.08221783488988876,
+      "logps/chosen": -1.696155309677124,
+      "logps/rejected": -2.6400060653686523,
+      "loss": 0.6257,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.696155309677124,
+      "rewards/margins": 0.9438506364822388,
+      "rewards/rejected": -2.6400060653686523,
       "step": 3810
     },
     {
       "epoch": 2.0418130122093996,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.27721039587076,
       "learning_rate": 2.7967664000469035e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3687906861305237,
+      "logits/rejected": -0.251179039478302,
+      "logps/chosen": -1.8650439977645874,
+      "logps/rejected": -2.485954999923706,
+      "loss": 0.6459,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8650439977645874,
+      "rewards/margins": 0.6209110021591187,
+      "rewards/rejected": -2.485954999923706,
       "step": 3815
     },
     {
       "epoch": 2.0444890449908013,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5485927958773185,
       "learning_rate": 2.7827966088927095e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3417259752750397,
+      "logits/rejected": -0.14000238478183746,
+      "logps/chosen": -1.825007677078247,
+      "logps/rejected": -2.6428163051605225,
+      "loss": 0.6557,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.825007677078247,
+      "rewards/margins": 0.8178087472915649,
+      "rewards/rejected": -2.6428163051605225,
       "step": 3820
     },
     {
       "epoch": 2.0471650777722026,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1392775892188003,
       "learning_rate": 2.768848329055538e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2999024987220764,
+      "logits/rejected": -0.20472308993339539,
+      "logps/chosen": -1.7003463506698608,
+      "logps/rejected": -2.438305377960205,
+      "loss": 0.6342,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7003463506698608,
+      "rewards/margins": 0.7379589080810547,
+      "rewards/rejected": -2.438305377960205,
       "step": 3825
     },
     {
       "epoch": 2.0498411105536043,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6324302868681415,
       "learning_rate": 2.7549216958616657e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.37821680307388306,
+      "logits/rejected": -0.23961147665977478,
+      "logps/chosen": -1.7797120809555054,
+      "logps/rejected": -2.721122980117798,
+      "loss": 0.6254,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7797120809555054,
+      "rewards/margins": 0.9414108991622925,
+      "rewards/rejected": -2.721122980117798,
       "step": 3830
     },
     {
       "epoch": 2.052517143335006,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.169403061117983,
       "learning_rate": 2.741016844427344e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27757728099823,
+      "logits/rejected": -0.1233028769493103,
+      "logps/chosen": -1.7559255361557007,
+      "logps/rejected": -2.4776434898376465,
+      "loss": 0.6457,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7559255361557007,
+      "rewards/margins": 0.721717894077301,
+      "rewards/rejected": -2.4776434898376465,
       "step": 3835
     },
     {
       "epoch": 2.0551931761164073,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.280686471847997,
       "learning_rate": 2.7271339096575073e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22597940266132355,
+      "logits/rejected": -0.11347540467977524,
+      "logps/chosen": -1.6762222051620483,
+      "logps/rejected": -2.5189106464385986,
+      "loss": 0.6159,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.6762222051620483,
+      "rewards/margins": 0.8426884412765503,
+      "rewards/rejected": -2.5189106464385986,
       "step": 3840
     },
     {
       "epoch": 2.057869208897809,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6691712617071466,
       "learning_rate": 2.713273026244446e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.39662525057792664,
+      "logits/rejected": -0.16729433834552765,
+      "logps/chosen": -1.8051817417144775,
+      "logps/rejected": -2.53495717048645,
+      "loss": 0.6392,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8051817417144775,
+      "rewards/margins": 0.7297754287719727,
+      "rewards/rejected": -2.53495717048645,
       "step": 3845
     },
     {
       "epoch": 2.0605452416792107,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7351157450928256,
       "learning_rate": 2.6994343286665156e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29524707794189453,
+      "logits/rejected": -0.1351134330034256,
+      "logps/chosen": -1.8099720478057861,
+      "logps/rejected": -2.545163631439209,
+      "loss": 0.6436,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.8099720478057861,
+      "rewards/margins": 0.7351916432380676,
+      "rewards/rejected": -2.545163631439209,
       "step": 3850
     },
     {
       "epoch": 2.063221274460612,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.807429976307861,
       "learning_rate": 2.6856179511868156e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25214678049087524,
+      "logits/rejected": -0.07815800607204437,
+      "logps/chosen": -1.7499492168426514,
+      "logps/rejected": -2.6900038719177246,
+      "loss": 0.6452,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7499492168426514,
+      "rewards/margins": 0.9400542378425598,
+      "rewards/rejected": -2.6900038719177246,
       "step": 3855
     },
     {
       "epoch": 2.0658973072420137,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1866878006849384,
       "learning_rate": 2.6718240278519056e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2914854884147644,
+      "logits/rejected": -0.1527956873178482,
+      "logps/chosen": -1.7513301372528076,
+      "logps/rejected": -2.754714012145996,
+      "loss": 0.6433,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7513301372528076,
+      "rewards/margins": 1.0033838748931885,
+      "rewards/rejected": -2.754714012145996,
       "step": 3860
     },
     {
       "epoch": 2.0685733400234154,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.352208434947895,
       "learning_rate": 2.6580526924904866e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.38585910201072693,
+      "logits/rejected": -0.21896329522132874,
+      "logps/chosen": -1.8602120876312256,
+      "logps/rejected": -2.405473232269287,
+      "loss": 0.6389,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8602120876312256,
+      "rewards/margins": 0.545261025428772,
+      "rewards/rejected": -2.405473232269287,
       "step": 3865
     },
     {
       "epoch": 2.0712493728048167,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4065837113520185,
       "learning_rate": 2.6443040787121186e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30670252442359924,
+      "logits/rejected": -0.2517473101615906,
+      "logps/chosen": -1.7013648748397827,
+      "logps/rejected": -2.4678256511688232,
+      "loss": 0.626,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7013648748397827,
+      "rewards/margins": 0.7664605975151062,
+      "rewards/rejected": -2.4678256511688232,
       "step": 3870
     },
     {
       "epoch": 2.0739254055862184,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.064616974041105,
       "learning_rate": 2.6305783199059084e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.292243093252182,
+      "logits/rejected": -0.18742215633392334,
+      "logps/chosen": -1.779335379600525,
+      "logps/rejected": -2.5090296268463135,
+      "loss": 0.6216,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.779335379600525,
+      "rewards/margins": 0.7296940684318542,
+      "rewards/rejected": -2.5090296268463135,
       "step": 3875
     },
     {
       "epoch": 2.07660143836762,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8546715198223303,
       "learning_rate": 2.6168755492392324e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3231429159641266,
+      "logits/rejected": -0.16940495371818542,
+      "logps/chosen": -1.5702096223831177,
+      "logps/rejected": -2.556466817855835,
+      "loss": 0.6186,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5702096223831177,
+      "rewards/margins": 0.9862571954727173,
+      "rewards/rejected": -2.556466817855835,
       "step": 3880
     },
     {
       "epoch": 2.0792774711490214,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8200499875388334,
       "learning_rate": 2.6031958996564274e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32586389780044556,
+      "logits/rejected": -0.19205589592456818,
+      "logps/chosen": -1.652404546737671,
+      "logps/rejected": -2.6144018173217773,
+      "loss": 0.6246,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.652404546737671,
+      "rewards/margins": 0.9619973301887512,
+      "rewards/rejected": -2.6144018173217773,
       "step": 3885
     },
     {
       "epoch": 2.081953503930423,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.115376413580909,
       "learning_rate": 2.589539503877518e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23725812137126923,
+      "logits/rejected": -0.1469879150390625,
+      "logps/chosen": -1.7307205200195312,
+      "logps/rejected": -2.4744274616241455,
+      "loss": 0.6386,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7307205200195312,
+      "rewards/margins": 0.7437066435813904,
+      "rewards/rejected": -2.4744274616241455,
       "step": 3890
     },
     {
       "epoch": 2.084629536711825,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.088131805831307,
       "learning_rate": 2.5759064943969125e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36349180340766907,
+      "logits/rejected": -0.1418864130973816,
+      "logps/chosen": -1.6969791650772095,
+      "logps/rejected": -2.519658088684082,
+      "loss": 0.6373,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.6969791650772095,
+      "rewards/margins": 0.8226790428161621,
+      "rewards/rejected": -2.519658088684082,
       "step": 3895
     },
     {
       "epoch": 2.087305569493226,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.369496917792683,
       "learning_rate": 2.562297003482131e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19480571150779724,
+      "logits/rejected": -0.18047909438610077,
+      "logps/chosen": -1.713881254196167,
+      "logps/rejected": -2.477423667907715,
+      "loss": 0.6275,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.713881254196167,
+      "rewards/margins": 0.7635424137115479,
+      "rewards/rejected": -2.477423667907715,
       "step": 3900
     },
     {
       "epoch": 2.089981602274628,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 15.00766405216186,
       "learning_rate": 2.548711163172512e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2571311593055725,
+      "logits/rejected": -0.16614094376564026,
+      "logps/chosen": -1.8690626621246338,
+      "logps/rejected": -2.5330750942230225,
+      "loss": 0.6465,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8690626621246338,
+      "rewards/margins": 0.6640125513076782,
+      "rewards/rejected": -2.5330750942230225,
       "step": 3905
     },
     {
       "epoch": 2.0926576350560295,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0606362230185953,
       "learning_rate": 2.53514910527794e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22525492310523987,
+      "logits/rejected": -0.10693037509918213,
+      "logps/chosen": -1.5843796730041504,
+      "logps/rejected": -2.3118433952331543,
+      "loss": 0.619,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5843796730041504,
+      "rewards/margins": 0.7274640202522278,
+      "rewards/rejected": -2.3118433952331543,
       "step": 3910
     },
     {
       "epoch": 2.095333667837431,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.104775830505062,
       "learning_rate": 2.5216109613775573e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.312709242105484,
+      "logits/rejected": -0.1482788622379303,
+      "logps/chosen": -1.8243350982666016,
+      "logps/rejected": -2.6101794242858887,
+      "loss": 0.6548,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8243350982666016,
+      "rewards/margins": 0.7858443260192871,
+      "rewards/rejected": -2.6101794242858887,
       "step": 3915
     },
     {
       "epoch": 2.0980097006188325,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2419625881326306,
       "learning_rate": 2.5080968628184993e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3016451597213745,
+      "logits/rejected": -0.14085645973682404,
+      "logps/chosen": -1.7804778814315796,
+      "logps/rejected": -2.7533557415008545,
+      "loss": 0.6354,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7804778814315796,
+      "rewards/margins": 0.9728778600692749,
+      "rewards/rejected": -2.7533557415008545,
       "step": 3920
     },
     {
       "epoch": 2.1006857334002342,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.101486865027415,
       "learning_rate": 2.494606940714605e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30268168449401855,
+      "logits/rejected": -0.2054332196712494,
+      "logps/chosen": -1.616539716720581,
+      "logps/rejected": -2.4663467407226562,
+      "loss": 0.611,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.616539716720581,
+      "rewards/margins": 0.8498069047927856,
+      "rewards/rejected": -2.4663467407226562,
       "step": 3925
     },
     {
       "epoch": 2.103361766181636,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5081037128703696,
       "learning_rate": 2.4811413259451625e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36914071440696716,
+      "logits/rejected": -0.22562257945537567,
+      "logps/chosen": -1.704716444015503,
+      "logps/rejected": -2.4639034271240234,
+      "loss": 0.6356,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.704716444015503,
+      "rewards/margins": 0.7591866850852966,
+      "rewards/rejected": -2.4639034271240234,
       "step": 3930
     },
     {
       "epoch": 2.106037798963037,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6128351489667585,
       "learning_rate": 2.46770014915362e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2524584233760834,
+      "logits/rejected": -0.19292497634887695,
+      "logps/chosen": -1.7421573400497437,
+      "logps/rejected": -2.487544298171997,
+      "loss": 0.6263,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7421573400497437,
+      "rewards/margins": 0.7453868985176086,
+      "rewards/rejected": -2.487544298171997,
       "step": 3935
     },
     {
       "epoch": 2.108713831744439,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.77378035199757,
       "learning_rate": 2.45428354074634e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25665968656539917,
+      "logits/rejected": -0.2053678035736084,
+      "logps/chosen": -1.6815509796142578,
+      "logps/rejected": -2.56133770942688,
+      "loss": 0.6149,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6815509796142578,
+      "rewards/margins": 0.8797863721847534,
+      "rewards/rejected": -2.56133770942688,
       "step": 3940
     },
     {
       "epoch": 2.1113898645258407,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.073471114130495,
       "learning_rate": 2.4408916308913105e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2969088554382324,
+      "logits/rejected": -0.12912429869174957,
+      "logps/chosen": -1.8013194799423218,
+      "logps/rejected": -2.4130377769470215,
+      "loss": 0.646,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.8013194799423218,
+      "rewards/margins": 0.6117182970046997,
+      "rewards/rejected": -2.4130377769470215,
       "step": 3945
     },
     {
       "epoch": 2.114065897307242,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.613801474618704,
       "learning_rate": 2.4275245495169025e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21768684685230255,
+      "logits/rejected": -0.0687679797410965,
+      "logps/chosen": -1.7293579578399658,
+      "logps/rejected": -2.446604013442993,
+      "loss": 0.6422,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7293579578399658,
+      "rewards/margins": 0.7172459959983826,
+      "rewards/rejected": -2.446604013442993,
       "step": 3950
     },
     {
       "epoch": 2.1167419300886436,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.541708730430523,
       "learning_rate": 2.414182426310597e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3482029438018799,
+      "logits/rejected": -0.272969514131546,
+      "logps/chosen": -1.6908928155899048,
+      "logps/rejected": -2.601578950881958,
+      "loss": 0.6267,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.6908928155899048,
+      "rewards/margins": 0.9106863141059875,
+      "rewards/rejected": -2.601578950881958,
       "step": 3955
     },
     {
       "epoch": 2.1194179628700454,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.02749868884262,
       "learning_rate": 2.400865390717734e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2600652575492859,
+      "logits/rejected": -0.14817263185977936,
+      "logps/chosen": -1.732611894607544,
+      "logps/rejected": -2.7784550189971924,
+      "loss": 0.6184,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.732611894607544,
+      "rewards/margins": 1.0458433628082275,
+      "rewards/rejected": -2.7784550189971924,
       "step": 3960
     },
     {
       "epoch": 2.1220939956514466,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.178496083232117,
       "learning_rate": 2.3875735719402475e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22812554240226746,
+      "logits/rejected": -0.11873205751180649,
+      "logps/chosen": -1.6974542140960693,
+      "logps/rejected": -2.579291343688965,
+      "loss": 0.6203,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6974542140960693,
+      "rewards/margins": 0.8818371891975403,
+      "rewards/rejected": -2.579291343688965,
       "step": 3965
     },
     {
       "epoch": 2.1247700284328483,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4539356983288916,
       "learning_rate": 2.3743070989354258e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28308460116386414,
+      "logits/rejected": -0.19616027176380157,
+      "logps/chosen": -1.742536187171936,
+      "logps/rejected": -2.561159133911133,
+      "loss": 0.6219,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.742536187171936,
+      "rewards/margins": 0.8186231851577759,
+      "rewards/rejected": -2.561159133911133,
       "step": 3970
     },
     {
       "epoch": 2.12744606121425,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.688653120383383,
       "learning_rate": 2.3610661004146454e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23555810749530792,
+      "logits/rejected": -0.14716582000255585,
+      "logps/chosen": -1.5741770267486572,
+      "logps/rejected": -2.224228858947754,
+      "loss": 0.6204,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5741770267486572,
+      "rewards/margins": 0.6500517129898071,
+      "rewards/rejected": -2.224228858947754,
       "step": 3975
     },
     {
       "epoch": 2.1301220939956513,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8437774657556125,
       "learning_rate": 2.3478507048421314e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3129003345966339,
+      "logits/rejected": -0.2450159341096878,
+      "logps/chosen": -1.6318928003311157,
+      "logps/rejected": -2.4831271171569824,
+      "loss": 0.6024,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6318928003311157,
+      "rewards/margins": 0.8512343168258667,
+      "rewards/rejected": -2.4831271171569824,
       "step": 3980
     },
     {
       "epoch": 2.132798126777053,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.85102327480223,
       "learning_rate": 2.334661040433713e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36019739508628845,
+      "logits/rejected": -0.25447770953178406,
+      "logps/chosen": -1.8021939992904663,
+      "logps/rejected": -2.643709659576416,
+      "loss": 0.6341,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.8021939992904663,
+      "rewards/margins": 0.8415155410766602,
+      "rewards/rejected": -2.643709659576416,
       "step": 3985
     },
     {
       "epoch": 2.1354741595584548,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.212893831526301,
       "learning_rate": 2.321497235155568e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3910345137119293,
+      "logits/rejected": -0.26656073331832886,
+      "logps/chosen": -1.520722508430481,
+      "logps/rejected": -2.4742817878723145,
+      "loss": 0.6073,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.520722508430481,
+      "rewards/margins": 0.9535595178604126,
+      "rewards/rejected": -2.4742817878723145,
       "step": 3990
     },
     {
       "epoch": 2.138150192339856,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2631011890258073,
       "learning_rate": 2.3083594167229965e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4348251223564148,
+      "logits/rejected": -0.18633487820625305,
+      "logps/chosen": -1.7703431844711304,
+      "logps/rejected": -2.633288860321045,
+      "loss": 0.638,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7703431844711304,
+      "rewards/margins": 0.8629458546638489,
+      "rewards/rejected": -2.633288860321045,
       "step": 3995
     },
     {
       "epoch": 2.1408262251212578,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.583626985574007,
       "learning_rate": 2.295247712599167e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3090136647224426,
+      "logits/rejected": -0.2181253433227539,
+      "logps/chosen": -1.6136226654052734,
+      "logps/rejected": -2.570077419281006,
+      "loss": 0.607,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.6136226654052734,
+      "rewards/margins": 0.9564547538757324,
+      "rewards/rejected": -2.570077419281006,
       "step": 4000
     },
     {
       "epoch": 2.1408262251212578,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.9149,
-      "eval_samples_per_second": 38.522,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.652,
+      "eval_logits/chosen": 0.0005278190947137773,
+      "eval_logits/rejected": 0.08424228429794312,
+      "eval_logps/chosen": -1.768615484237671,
+      "eval_logps/rejected": -2.5161468982696533,
+      "eval_loss": 0.6506023406982422,
+      "eval_rewards/accuracies": 0.6750741600990295,
+      "eval_rewards/chosen": -1.768615484237671,
+      "eval_rewards/margins": 0.7475313544273376,
+      "eval_rewards/rejected": -2.5161468982696533,
+      "eval_runtime": 40.4601,
+      "eval_samples_per_second": 33.243,
+      "eval_steps_per_second": 8.329,
       "step": 4000
     },
     {
       "epoch": 2.1435022579026595,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.013761204337641,
       "learning_rate": 2.2821622499938948e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.320697158575058,
+      "logits/rejected": -0.12043861299753189,
+      "logps/chosen": -1.9164727926254272,
+      "logps/rejected": -2.6353962421417236,
+      "loss": 0.6553,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.9164727926254272,
+      "rewards/margins": 0.7189234495162964,
+      "rewards/rejected": -2.6353962421417236,
       "step": 4005
     },
     {
       "epoch": 2.1461782906840607,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8469565970805233,
       "learning_rate": 2.269103155862391e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3350791335105896,
+      "logits/rejected": -0.2439563274383545,
+      "logps/chosen": -1.7500808238983154,
+      "logps/rejected": -2.4539079666137695,
+      "loss": 0.6287,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7500808238983154,
+      "rewards/margins": 0.7038268446922302,
+      "rewards/rejected": -2.4539079666137695,
       "step": 4010
     },
     {
       "epoch": 2.1488543234654625,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5183245679737056,
       "learning_rate": 2.2560705569040483e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32507115602493286,
+      "logits/rejected": -0.07696642726659775,
+      "logps/chosen": -1.7587387561798096,
+      "logps/rejected": -2.514599561691284,
+      "loss": 0.6459,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7587387561798096,
+      "rewards/margins": 0.7558610439300537,
+      "rewards/rejected": -2.514599561691284,
       "step": 4015
     },
     {
       "epoch": 2.151530356246864,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.4845437027183,
       "learning_rate": 2.2430645795611963e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4072760045528412,
+      "logits/rejected": -0.2580171823501587,
+      "logps/chosen": -1.7949028015136719,
+      "logps/rejected": -2.452763557434082,
+      "loss": 0.6487,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7949028015136719,
+      "rewards/margins": 0.6578606367111206,
+      "rewards/rejected": -2.452763557434082,
       "step": 4020
     },
     {
       "epoch": 2.1542063890282654,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3543157023049988,
       "learning_rate": 2.230085350017884e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32240030169487,
+      "logits/rejected": -0.23190274834632874,
+      "logps/chosen": -1.726784348487854,
+      "logps/rejected": -2.546051025390625,
+      "loss": 0.6374,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.726784348487854,
+      "rewards/margins": 0.8192664384841919,
+      "rewards/rejected": -2.546051025390625,
       "step": 4025
     },
     {
       "epoch": 2.156882421809667,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.201413174537865,
       "learning_rate": 2.2171329941986554e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3496195673942566,
+      "logits/rejected": -0.26665931940078735,
+      "logps/chosen": -1.727299451828003,
+      "logps/rejected": -2.50832200050354,
+      "loss": 0.6243,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.727299451828003,
+      "rewards/margins": 0.7810222506523132,
+      "rewards/rejected": -2.50832200050354,
       "step": 4030
     },
     {
       "epoch": 2.159558454591069,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.320068970435404,
       "learning_rate": 2.2042076377673202e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34290963411331177,
+      "logits/rejected": -0.308978408575058,
+      "logps/chosen": -1.6266816854476929,
+      "logps/rejected": -2.1143691539764404,
+      "loss": 0.6397,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6266816854476929,
+      "rewards/margins": 0.48768752813339233,
+      "rewards/rejected": -2.1143691539764404,
       "step": 4035
     },
     {
       "epoch": 2.16223448737247,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.035863255371789,
       "learning_rate": 2.1913094061257476e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3375782370567322,
+      "logits/rejected": -0.3095376789569855,
+      "logps/chosen": -1.6361453533172607,
+      "logps/rejected": -2.42558217048645,
+      "loss": 0.6337,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6361453533172607,
+      "rewards/margins": 0.7894368171691895,
+      "rewards/rejected": -2.42558217048645,
       "step": 4040
     },
     {
       "epoch": 2.164910520153872,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7751505459950394,
       "learning_rate": 2.178438424412633e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28719958662986755,
+      "logits/rejected": -0.16218821704387665,
+      "logps/chosen": -1.7684729099273682,
+      "logps/rejected": -2.481672525405884,
+      "loss": 0.6419,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.7684729099273682,
+      "rewards/margins": 0.7131994962692261,
+      "rewards/rejected": -2.481672525405884,
       "step": 4045
     },
     {
       "epoch": 2.1675865529352736,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.542477918235696,
       "learning_rate": 2.165594817502302e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3617328405380249,
+      "logits/rejected": -0.2666403353214264,
+      "logps/chosen": -1.781402826309204,
+      "logps/rejected": -2.3657386302948,
+      "loss": 0.6363,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.781402826309204,
+      "rewards/margins": 0.5843357443809509,
+      "rewards/rejected": -2.3657386302948,
       "step": 4050
     },
     {
       "epoch": 2.170262585716675,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.424603880555552,
       "learning_rate": 2.1527787100034806e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2755914032459259,
+      "logits/rejected": -0.22461871802806854,
+      "logps/chosen": -1.7758814096450806,
+      "logps/rejected": -2.2319905757904053,
+      "loss": 0.6527,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7758814096450806,
+      "rewards/margins": 0.4561091363430023,
+      "rewards/rejected": -2.2319905757904053,
       "step": 4055
     },
     {
       "epoch": 2.1729386184980766,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0556734927316587,
       "learning_rate": 2.1399902262581037e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.19631099700927734,
+      "logits/rejected": -0.09221359342336655,
+      "logps/chosen": -1.6683521270751953,
+      "logps/rejected": -2.4084999561309814,
+      "loss": 0.6307,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.6683521270751953,
+      "rewards/margins": 0.7401479482650757,
+      "rewards/rejected": -2.4084999561309814,
       "step": 4060
     },
     {
       "epoch": 2.1756146512794783,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.660948281465448,
       "learning_rate": 2.127229490340094e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3749244809150696,
+      "logits/rejected": -0.2946448028087616,
+      "logps/chosen": -1.7683089971542358,
+      "logps/rejected": -2.5882973670959473,
+      "loss": 0.6236,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7683089971542358,
+      "rewards/margins": 0.8199883699417114,
+      "rewards/rejected": -2.5882973670959473,
       "step": 4065
     },
     {
       "epoch": 2.1782906840608796,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.428287873948031,
       "learning_rate": 2.1144966260541698e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2856397032737732,
+      "logits/rejected": -0.08739195019006729,
+      "logps/chosen": -1.771997094154358,
+      "logps/rejected": -2.6562013626098633,
+      "loss": 0.626,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.771997094154358,
+      "rewards/margins": 0.8842042684555054,
+      "rewards/rejected": -2.6562013626098633,
       "step": 4070
     },
     {
       "epoch": 2.1809667168422813,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.464580081345183,
       "learning_rate": 2.1017917569346332e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3391563594341278,
+      "logits/rejected": -0.1765109747648239,
+      "logps/chosen": -1.8340823650360107,
+      "logps/rejected": -2.531696081161499,
+      "loss": 0.6336,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.8340823650360107,
+      "rewards/margins": 0.6976135969161987,
+      "rewards/rejected": -2.531696081161499,
       "step": 4075
     },
     {
       "epoch": 2.183642749623683,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.819178343501842,
       "learning_rate": 2.0891150062441837e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3388068974018097,
+      "logits/rejected": -0.21791037917137146,
+      "logps/chosen": -1.8249047994613647,
+      "logps/rejected": -2.7282283306121826,
+      "loss": 0.6376,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.8249047994613647,
+      "rewards/margins": 0.9033233523368835,
+      "rewards/rejected": -2.7282283306121826,
       "step": 4080
     },
     {
       "epoch": 2.1863187824050843,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9104752602197554,
       "learning_rate": 2.0764664969727086e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29575809836387634,
+      "logits/rejected": -0.198848694562912,
+      "logps/chosen": -1.7234375476837158,
+      "logps/rejected": -2.3980133533477783,
+      "loss": 0.6249,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7234375476837158,
+      "rewards/margins": 0.6745756268501282,
+      "rewards/rejected": -2.3980133533477783,
       "step": 4085
     },
     {
       "epoch": 2.188994815186486,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.229708321349307,
       "learning_rate": 2.0638463518361033e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3777976632118225,
+      "logits/rejected": -0.1911180168390274,
+      "logps/chosen": -1.7214601039886475,
+      "logps/rejected": -2.48490834236145,
+      "loss": 0.6337,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7214601039886475,
+      "rewards/margins": 0.7634482383728027,
+      "rewards/rejected": -2.48490834236145,
       "step": 4090
     },
     {
       "epoch": 2.1916708479678877,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5897060161234817,
       "learning_rate": 2.0512546932750702e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3546110987663269,
+      "logits/rejected": -0.27483221888542175,
+      "logps/chosen": -1.7474124431610107,
+      "logps/rejected": -2.4874348640441895,
+      "loss": 0.6401,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7474124431610107,
+      "rewards/margins": 0.7400221824645996,
+      "rewards/rejected": -2.4874348640441895,
       "step": 4095
     },
     {
       "epoch": 2.194346880749289,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.904749483319274,
       "learning_rate": 2.0386916434539343e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2656613290309906,
+      "logits/rejected": -0.13175199925899506,
+      "logps/chosen": -1.5548592805862427,
+      "logps/rejected": -2.499527931213379,
+      "loss": 0.5947,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.5548592805862427,
+      "rewards/margins": 0.9446685910224915,
+      "rewards/rejected": -2.499527931213379,
       "step": 4100
     },
     {
       "epoch": 2.1970229135306907,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.001749144425705,
       "learning_rate": 2.0261573242594627e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2897736430168152,
+      "logits/rejected": -0.11054684966802597,
+      "logps/chosen": -1.8838201761245728,
+      "logps/rejected": -2.509253740310669,
+      "loss": 0.6438,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.8838201761245728,
+      "rewards/margins": 0.625433623790741,
+      "rewards/rejected": -2.509253740310669,
       "step": 4105
     },
     {
       "epoch": 2.1996989463120924,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.623500655594086,
       "learning_rate": 2.0136518572996724e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26431092619895935,
+      "logits/rejected": -0.08807923644781113,
+      "logps/chosen": -1.8124196529388428,
+      "logps/rejected": -2.4803731441497803,
+      "loss": 0.6332,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.8124196529388428,
+      "rewards/margins": 0.667953610420227,
+      "rewards/rejected": -2.4803731441497803,
       "step": 4110
     },
     {
       "epoch": 2.202374979093494,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.344311000662098,
       "learning_rate": 2.0011753639026617e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26585882902145386,
+      "logits/rejected": -0.22727441787719727,
+      "logps/chosen": -1.794546127319336,
+      "logps/rejected": -2.6541194915771484,
+      "loss": 0.6319,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.794546127319336,
+      "rewards/margins": 0.859573483467102,
+      "rewards/rejected": -2.6541194915771484,
       "step": 4115
     },
     {
       "epoch": 2.2050510118748954,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7525257613436995,
       "learning_rate": 1.988727965115421e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28927236795425415,
+      "logits/rejected": -0.21951599419116974,
+      "logps/chosen": -1.5871387720108032,
+      "logps/rejected": -2.46940541267395,
+      "loss": 0.6021,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5871387720108032,
+      "rewards/margins": 0.8822664022445679,
+      "rewards/rejected": -2.46940541267395,
       "step": 4120
     },
     {
       "epoch": 2.207727044656297,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9366205482203784,
       "learning_rate": 1.9763097817026713e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3674069941043854,
+      "logits/rejected": -0.16737040877342224,
+      "logps/chosen": -1.6832363605499268,
+      "logps/rejected": -2.6822304725646973,
+      "loss": 0.618,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6832363605499268,
+      "rewards/margins": 0.9989942312240601,
+      "rewards/rejected": -2.6822304725646973,
       "step": 4125
     },
     {
       "epoch": 2.210403077437699,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.412195137306042,
       "learning_rate": 1.9639209341456796e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29053226113319397,
+      "logits/rejected": -0.20578761398792267,
+      "logps/chosen": -1.7618839740753174,
+      "logps/rejected": -2.548367738723755,
+      "loss": 0.6383,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.7618839740753174,
+      "rewards/margins": 0.7864838242530823,
+      "rewards/rejected": -2.548367738723755,
       "step": 4130
     },
     {
       "epoch": 2.2130791102191,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8505355078278836,
       "learning_rate": 1.951561542641102e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28010624647140503,
+      "logits/rejected": -0.2799418568611145,
+      "logps/chosen": -1.8486697673797607,
+      "logps/rejected": -2.722203016281128,
+      "loss": 0.6264,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8486697673797607,
+      "rewards/margins": 0.8735335469245911,
+      "rewards/rejected": -2.722203016281128,
       "step": 4135
     },
     {
       "epoch": 2.215755143000502,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2189728840238225,
       "learning_rate": 1.939231727099806e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.43608731031417847,
+      "logits/rejected": -0.37845778465270996,
+      "logps/chosen": -1.6761572360992432,
+      "logps/rejected": -2.4600701332092285,
+      "loss": 0.6303,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.6761572360992432,
+      "rewards/margins": 0.7839127779006958,
+      "rewards/rejected": -2.4600701332092285,
       "step": 4140
     },
     {
       "epoch": 2.2184311757819035,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8589344382492614,
       "learning_rate": 1.926931607145719e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22892260551452637,
+      "logits/rejected": -0.09062418341636658,
+      "logps/chosen": -1.9290889501571655,
+      "logps/rejected": -2.709465265274048,
+      "loss": 0.6473,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.9290889501571655,
+      "rewards/margins": 0.7803763747215271,
+      "rewards/rejected": -2.709465265274048,
       "step": 4145
     },
     {
       "epoch": 2.221107208563305,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.032906800212431,
       "learning_rate": 1.9146613021146564e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29464372992515564,
+      "logits/rejected": -0.22168800234794617,
+      "logps/chosen": -1.6789153814315796,
+      "logps/rejected": -2.544818878173828,
+      "loss": 0.6227,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6789153814315796,
+      "rewards/margins": 0.8659032583236694,
+      "rewards/rejected": -2.544818878173828,
       "step": 4150
     },
     {
       "epoch": 2.2237832413447065,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7081916786587144,
       "learning_rate": 1.9024209310531736e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2532450258731842,
+      "logits/rejected": -0.2671474516391754,
+      "logps/chosen": -1.7272021770477295,
+      "logps/rejected": -2.366227626800537,
+      "loss": 0.617,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7272021770477295,
+      "rewards/margins": 0.6390254497528076,
+      "rewards/rejected": -2.366227626800537,
       "step": 4155
     },
     {
       "epoch": 2.2264592741261082,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6804232483320565,
       "learning_rate": 1.890210612717401e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29431647062301636,
+      "logits/rejected": -0.1643054187297821,
+      "logps/chosen": -1.885109543800354,
+      "logps/rejected": -2.5073468685150146,
+      "loss": 0.632,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.885109543800354,
+      "rewards/margins": 0.6222373843193054,
+      "rewards/rejected": -2.5073468685150146,
       "step": 4160
     },
     {
       "epoch": 2.2291353069075095,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9632252732313735,
       "learning_rate": 1.8780304655719054e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3045499324798584,
+      "logits/rejected": -0.20411646366119385,
+      "logps/chosen": -1.769202470779419,
+      "logps/rejected": -2.6223161220550537,
+      "loss": 0.628,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.769202470779419,
+      "rewards/margins": 0.8531135320663452,
+      "rewards/rejected": -2.6223161220550537,
       "step": 4165
     },
     {
       "epoch": 2.231811339688911,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.994701730847222,
       "learning_rate": 1.865880607788523e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2232269048690796,
+      "logits/rejected": -0.1632028967142105,
+      "logps/chosen": -1.733852744102478,
+      "logps/rejected": -2.5535037517547607,
+      "loss": 0.6343,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.733852744102478,
+      "rewards/margins": 0.8196510076522827,
+      "rewards/rejected": -2.5535037517547607,
       "step": 4170
     },
     {
       "epoch": 2.234487372470313,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1488306485829716,
       "learning_rate": 1.8537611572452316e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3247857689857483,
+      "logits/rejected": -0.2429112195968628,
+      "logps/chosen": -1.7657562494277954,
+      "logps/rejected": -2.355099678039551,
+      "loss": 0.6299,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7657562494277954,
+      "rewards/margins": 0.5893433094024658,
+      "rewards/rejected": -2.355099678039551,
       "step": 4175
     },
     {
       "epoch": 2.237163405251714,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.858500735406035,
       "learning_rate": 1.84167223152499e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3293859660625458,
+      "logits/rejected": -0.13067664206027985,
+      "logps/chosen": -1.6849706172943115,
+      "logps/rejected": -2.4818215370178223,
+      "loss": 0.6245,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.6849706172943115,
+      "rewards/margins": 0.7968508005142212,
+      "rewards/rejected": -2.4818215370178223,
       "step": 4180
     },
     {
       "epoch": 2.239839438033116,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.978710032336423,
       "learning_rate": 1.8296139479146112e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34674057364463806,
+      "logits/rejected": -0.33721351623535156,
+      "logps/chosen": -1.6453500986099243,
+      "logps/rejected": -2.402988910675049,
+      "loss": 0.6197,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6453500986099243,
+      "rewards/margins": 0.7576389312744141,
+      "rewards/rejected": -2.402988910675049,
       "step": 4185
     },
     {
       "epoch": 2.2425154708145176,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.355202272445241,
       "learning_rate": 1.8175864234036132e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21903762221336365,
+      "logits/rejected": -0.12407919019460678,
+      "logps/chosen": -1.6965036392211914,
+      "logps/rejected": -2.5763978958129883,
+      "loss": 0.6279,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6965036392211914,
+      "rewards/margins": 0.8798943758010864,
+      "rewards/rejected": -2.5763978958129883,
       "step": 4190
     },
     {
       "epoch": 2.245191503595919,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4585605705552074,
       "learning_rate": 1.805589774683094e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4161747395992279,
+      "logits/rejected": -0.2747994661331177,
+      "logps/chosen": -1.661920189857483,
+      "logps/rejected": -2.3085784912109375,
+      "loss": 0.6317,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.661920189857483,
+      "rewards/margins": 0.646658182144165,
+      "rewards/rejected": -2.3085784912109375,
       "step": 4195
     },
     {
       "epoch": 2.2478675363773206,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6519175872566785,
       "learning_rate": 1.79362411814459e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17609548568725586,
+      "logits/rejected": -0.19131174683570862,
+      "logps/chosen": -1.8170013427734375,
+      "logps/rejected": -2.480583667755127,
+      "loss": 0.6209,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8170013427734375,
+      "rewards/margins": 0.6635822057723999,
+      "rewards/rejected": -2.480583667755127,
       "step": 4200
     },
     {
       "epoch": 2.2505435691587223,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1454202676598557,
       "learning_rate": 1.7816895698789552e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3559998869895935,
+      "logits/rejected": -0.28615492582321167,
+      "logps/chosen": -1.671596884727478,
+      "logps/rejected": -2.333047389984131,
+      "loss": 0.6285,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.671596884727478,
+      "rewards/margins": 0.6614505052566528,
+      "rewards/rejected": -2.333047389984131,
       "step": 4205
     },
     {
       "epoch": 2.2532196019401236,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.625123436471169,
       "learning_rate": 1.7697862456752271e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33030959963798523,
+      "logits/rejected": -0.22413039207458496,
+      "logps/chosen": -1.784055471420288,
+      "logps/rejected": -2.834258556365967,
+      "loss": 0.6312,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.784055471420288,
+      "rewards/margins": 1.05020272731781,
+      "rewards/rejected": -2.834258556365967,
       "step": 4210
     },
     {
       "epoch": 2.2558956347215253,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.786339555979613,
       "learning_rate": 1.7579142610195124e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3069586455821991,
+      "logits/rejected": -0.16654400527477264,
+      "logps/chosen": -1.760841965675354,
+      "logps/rejected": -2.4419257640838623,
+      "loss": 0.6323,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.760841965675354,
+      "rewards/margins": 0.6810838580131531,
+      "rewards/rejected": -2.4419257640838623,
       "step": 4215
     },
     {
       "epoch": 2.258571667502927,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.929816210509115,
       "learning_rate": 1.7460737310938568e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3510569632053375,
+      "logits/rejected": -0.16214652359485626,
+      "logps/chosen": -1.7250341176986694,
+      "logps/rejected": -2.5300498008728027,
+      "loss": 0.6206,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7250341176986694,
+      "rewards/margins": 0.8050156831741333,
+      "rewards/rejected": -2.5300498008728027,
       "step": 4220
     },
     {
       "epoch": 2.2612477002843283,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.023835654973352,
       "learning_rate": 1.734264770775133e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34140121936798096,
+      "logits/rejected": -0.16098840534687042,
+      "logps/chosen": -1.7128225564956665,
+      "logps/rejected": -2.668806791305542,
+      "loss": 0.6244,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.7128225564956665,
+      "rewards/margins": 0.9559841156005859,
+      "rewards/rejected": -2.668806791305542,
       "step": 4225
     },
     {
       "epoch": 2.26392373306573,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.156738907336234,
       "learning_rate": 1.7224874946339241e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.37167495489120483,
+      "logits/rejected": -0.29459255933761597,
+      "logps/chosen": -1.9041416645050049,
+      "logps/rejected": -2.5558533668518066,
+      "loss": 0.6395,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.9041416645050049,
+      "rewards/margins": 0.6517120599746704,
+      "rewards/rejected": -2.5558533668518066,
       "step": 4230
     },
     {
       "epoch": 2.2665997658471317,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.119323606239518,
       "learning_rate": 1.7107420169334186e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2928672432899475,
+      "logits/rejected": -0.19220854341983795,
+      "logps/chosen": -1.8240137100219727,
+      "logps/rejected": -2.370568037033081,
+      "loss": 0.6502,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.8240137100219727,
+      "rewards/margins": 0.5465543866157532,
+      "rewards/rejected": -2.370568037033081,
       "step": 4235
     },
     {
       "epoch": 2.269275798628533,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0229365989461567,
       "learning_rate": 1.6990284516282893e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2844608724117279,
+      "logits/rejected": -0.17581160366535187,
+      "logps/chosen": -1.6708028316497803,
+      "logps/rejected": -2.359114170074463,
+      "loss": 0.6386,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6708028316497803,
+      "rewards/margins": 0.6883114576339722,
+      "rewards/rejected": -2.359114170074463,
       "step": 4240
     },
     {
       "epoch": 2.2719518314099347,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.698110253687053,
       "learning_rate": 1.687346912363602e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3674829602241516,
+      "logits/rejected": -0.259930819272995,
+      "logps/chosen": -1.751574158668518,
+      "logps/rejected": -2.547194480895996,
+      "loss": 0.629,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.751574158668518,
+      "rewards/margins": 0.795620322227478,
+      "rewards/rejected": -2.547194480895996,
       "step": 4245
     },
     {
       "epoch": 2.2746278641913364,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.335687100322479,
       "learning_rate": 1.675697512473697e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33533594012260437,
+      "logits/rejected": -0.17312833666801453,
+      "logps/chosen": -1.8092082738876343,
+      "logps/rejected": -2.6107614040374756,
+      "loss": 0.6387,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8092082738876343,
+      "rewards/margins": 0.8015531301498413,
+      "rewards/rejected": -2.6107614040374756,
       "step": 4250
     },
     {
       "epoch": 2.2773038969727377,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7708000036825875,
       "learning_rate": 1.6640803649811087e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33472856879234314,
+      "logits/rejected": -0.11561266332864761,
+      "logps/chosen": -1.7939739227294922,
+      "logps/rejected": -2.637749195098877,
+      "loss": 0.6309,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7939739227294922,
+      "rewards/margins": 0.8437749743461609,
+      "rewards/rejected": -2.637749195098877,
       "step": 4255
     },
     {
       "epoch": 2.2799799297541394,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8905748092488266,
       "learning_rate": 1.6524955825954472e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31425994634628296,
+      "logits/rejected": -0.21730037033557892,
+      "logps/chosen": -1.6536824703216553,
+      "logps/rejected": -2.4594016075134277,
+      "loss": 0.6168,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.6536824703216553,
+      "rewards/margins": 0.8057191967964172,
+      "rewards/rejected": -2.4594016075134277,
       "step": 4260
     },
     {
       "epoch": 2.282655962535541,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.65756600209929,
       "learning_rate": 1.6409432777123277e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.40557774901390076,
+      "logits/rejected": -0.24844405055046082,
+      "logps/chosen": -1.668541669845581,
+      "logps/rejected": -2.552429676055908,
+      "loss": 0.6296,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.668541669845581,
+      "rewards/margins": 0.8838878870010376,
+      "rewards/rejected": -2.552429676055908,
       "step": 4265
     },
     {
       "epoch": 2.285331995316943,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.133762420309421,
       "learning_rate": 1.6294235624122577e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24907204508781433,
+      "logits/rejected": -0.02573532983660698,
+      "logps/chosen": -1.7543748617172241,
+      "logps/rejected": -2.6755356788635254,
+      "loss": 0.6355,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7543748617172241,
+      "rewards/margins": 0.9211609959602356,
+      "rewards/rejected": -2.6755356788635254,
       "step": 4270
     },
     {
       "epoch": 2.288008028098344,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2645318881764314,
       "learning_rate": 1.6179365484595697e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28888121247291565,
+      "logits/rejected": -0.2120685875415802,
+      "logps/chosen": -1.790113091468811,
+      "logps/rejected": -2.546772003173828,
+      "loss": 0.6441,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.790113091468811,
+      "rewards/margins": 0.7566587924957275,
+      "rewards/rejected": -2.546772003173828,
       "step": 4275
     },
     {
       "epoch": 2.290684060879746,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3599584994210616,
       "learning_rate": 1.60648234730132e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3366296887397766,
+      "logits/rejected": -0.2522452771663666,
+      "logps/chosen": -1.6745474338531494,
+      "logps/rejected": -2.5388686656951904,
+      "loss": 0.6141,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.6745474338531494,
+      "rewards/margins": 0.8643213510513306,
+      "rewards/rejected": -2.5388686656951904,
       "step": 4280
     },
     {
       "epoch": 2.293360093661147,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.729080939044079,
       "learning_rate": 1.595061070066222e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25139468908309937,
+      "logits/rejected": -0.26244086027145386,
+      "logps/chosen": -1.711867094039917,
+      "logps/rejected": -2.4635210037231445,
+      "loss": 0.6283,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.711867094039917,
+      "rewards/margins": 0.7516540884971619,
+      "rewards/rejected": -2.4635210037231445,
       "step": 4285
     },
     {
       "epoch": 2.296036126442549,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.960484240939342,
       "learning_rate": 1.5836728275635542e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36737060546875,
+      "logits/rejected": -0.21788792312145233,
+      "logps/chosen": -1.7893348932266235,
+      "logps/rejected": -2.5447208881378174,
+      "loss": 0.6449,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7893348932266235,
+      "rewards/margins": 0.7553859949111938,
+      "rewards/rejected": -2.5447208881378174,
       "step": 4290
     },
     {
       "epoch": 2.2987121592239506,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5008414308453224,
       "learning_rate": 1.5723177302820984e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34719908237457275,
+      "logits/rejected": -0.29328644275665283,
+      "logps/chosen": -1.7602910995483398,
+      "logps/rejected": -2.4229378700256348,
+      "loss": 0.6514,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7602910995483398,
+      "rewards/margins": 0.6626469492912292,
+      "rewards/rejected": -2.4229378700256348,
       "step": 4295
     },
     {
       "epoch": 2.3013881920053523,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7199796225765804,
       "learning_rate": 1.5609958883890544e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3088197410106659,
+      "logits/rejected": -0.19304877519607544,
+      "logps/chosen": -1.7934868335723877,
+      "logps/rejected": -2.373642683029175,
+      "loss": 0.6303,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7934868335723877,
+      "rewards/margins": 0.5801559686660767,
+      "rewards/rejected": -2.373642683029175,
       "step": 4300
     },
     {
       "epoch": 2.3040642247867535,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7170254041948083,
       "learning_rate": 1.5497074117289865e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3929354250431061,
+      "logits/rejected": -0.2908039093017578,
+      "logps/chosen": -1.7100038528442383,
+      "logps/rejected": -2.6406426429748535,
+      "loss": 0.6056,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7100038528442383,
+      "rewards/margins": 0.9306389093399048,
+      "rewards/rejected": -2.6406426429748535,
       "step": 4305
     },
     {
       "epoch": 2.3067402575681553,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.666820652344712,
       "learning_rate": 1.5384524098227402e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3525611460208893,
+      "logits/rejected": -0.16577701270580292,
+      "logps/chosen": -1.7296491861343384,
+      "logps/rejected": -2.7785751819610596,
+      "loss": 0.6289,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.7296491861343384,
+      "rewards/margins": 1.0489258766174316,
+      "rewards/rejected": -2.7785751819610596,
       "step": 4310
     },
     {
       "epoch": 2.3094162903495565,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.15051940555315,
       "learning_rate": 1.5272309918663974e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33821356296539307,
+      "logits/rejected": -0.1929440051317215,
+      "logps/chosen": -1.8458083868026733,
+      "logps/rejected": -2.406405210494995,
+      "loss": 0.6505,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8458083868026733,
+      "rewards/margins": 0.5605968832969666,
+      "rewards/rejected": -2.406405210494995,
       "step": 4315
     },
     {
       "epoch": 2.3120923231309582,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.254727298870795,
       "learning_rate": 1.516043266730201e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3479671776294708,
+      "logits/rejected": -0.22230663895606995,
+      "logps/chosen": -1.8740215301513672,
+      "logps/rejected": -2.537930965423584,
+      "loss": 0.6268,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8740215301513672,
+      "rewards/margins": 0.6639095544815063,
+      "rewards/rejected": -2.537930965423584,
       "step": 4320
     },
     {
       "epoch": 2.31476835591236,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.0397662877029585,
       "learning_rate": 1.504889342957512e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33565694093704224,
+      "logits/rejected": -0.1851847916841507,
+      "logps/chosen": -1.8233845233917236,
+      "logps/rejected": -2.6876044273376465,
+      "loss": 0.6287,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8233845233917236,
+      "rewards/margins": 0.8642200231552124,
+      "rewards/rejected": -2.6876044273376465,
       "step": 4325
     },
     {
       "epoch": 2.3174443886937617,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.511294428591714,
       "learning_rate": 1.4937693287637453e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3516416847705841,
+      "logits/rejected": -0.25131863355636597,
+      "logps/chosen": -1.929541826248169,
+      "logps/rejected": -2.558314323425293,
+      "loss": 0.6607,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.929541826248169,
+      "rewards/margins": 0.6287727355957031,
+      "rewards/rejected": -2.558314323425293,
       "step": 4330
     },
     {
       "epoch": 2.320120421475163,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.304706192262088,
       "learning_rate": 1.4826833320353305e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3014776408672333,
+      "logits/rejected": -0.232911616563797,
+      "logps/chosen": -1.8749675750732422,
+      "logps/rejected": -2.678061008453369,
+      "loss": 0.6376,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.8749675750732422,
+      "rewards/margins": 0.8030937314033508,
+      "rewards/rejected": -2.678061008453369,
       "step": 4335
     },
     {
       "epoch": 2.3227964542565647,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.286428163572841,
       "learning_rate": 1.4716314603286528e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3378271460533142,
+      "logits/rejected": -0.1686471402645111,
+      "logps/chosen": -1.6964128017425537,
+      "logps/rejected": -2.6832313537597656,
+      "loss": 0.6216,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6964128017425537,
+      "rewards/margins": 0.9868181347846985,
+      "rewards/rejected": -2.6832313537597656,
       "step": 4340
     },
     {
       "epoch": 2.3254724870379664,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.630035199037854,
       "learning_rate": 1.4606138208690233e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3806575834751129,
+      "logits/rejected": -0.3197815418243408,
+      "logps/chosen": -2.0581846237182617,
+      "logps/rejected": -2.55918025970459,
+      "loss": 0.6529,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.0581846237182617,
+      "rewards/margins": 0.5009955167770386,
+      "rewards/rejected": -2.55918025970459,
       "step": 4345
     },
     {
       "epoch": 2.3281485198193677,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2731208540082215,
       "learning_rate": 1.4496305205496251e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30668050050735474,
+      "logits/rejected": -0.24796047806739807,
+      "logps/chosen": -1.8612480163574219,
+      "logps/rejected": -2.710714340209961,
+      "loss": 0.6394,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8612480163574219,
+      "rewards/margins": 0.8494661450386047,
+      "rewards/rejected": -2.710714340209961,
       "step": 4350
     },
     {
       "epoch": 2.3308245526007694,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8104400538314662,
       "learning_rate": 1.4386816659304895e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.37561193108558655,
+      "logits/rejected": -0.27570003271102905,
+      "logps/chosen": -1.7838554382324219,
+      "logps/rejected": -2.491302013397217,
+      "loss": 0.6325,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7838554382324219,
+      "rewards/margins": 0.7074465155601501,
+      "rewards/rejected": -2.491302013397217,
       "step": 4355
     },
     {
       "epoch": 2.333500585382171,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.457363565355267,
       "learning_rate": 1.4277673632374492e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.41690635681152344,
+      "logits/rejected": -0.22828006744384766,
+      "logps/chosen": -1.8884210586547852,
+      "logps/rejected": -2.624763250350952,
+      "loss": 0.6467,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.8884210586547852,
+      "rewards/margins": 0.7363423109054565,
+      "rewards/rejected": -2.624763250350952,
       "step": 4360
     },
     {
       "epoch": 2.3361766181635724,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.55419828721774,
       "learning_rate": 1.416887718361119e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.22773535549640656,
+      "logits/rejected": -0.23800663650035858,
+      "logps/chosen": -1.7300447225570679,
+      "logps/rejected": -2.5209877490997314,
+      "loss": 0.6285,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7300447225570679,
+      "rewards/margins": 0.7909430861473083,
+      "rewards/rejected": -2.5209877490997314,
       "step": 4365
     },
     {
       "epoch": 2.338852650944974,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.2899116870324505,
       "learning_rate": 1.406042836855859e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2845227122306824,
+      "logits/rejected": -0.17272140085697174,
+      "logps/chosen": -1.5957354307174683,
+      "logps/rejected": -2.6434693336486816,
+      "loss": 0.6137,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.5957354307174683,
+      "rewards/margins": 1.0477337837219238,
+      "rewards/rejected": -2.6434693336486816,
       "step": 4370
     },
     {
       "epoch": 2.341528683726376,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9954804658909118,
       "learning_rate": 1.3952328239387595e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4034675657749176,
+      "logits/rejected": -0.21865996718406677,
+      "logps/chosen": -1.8410793542861938,
+      "logps/rejected": -2.617647886276245,
+      "loss": 0.6262,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8410793542861938,
+      "rewards/margins": 0.776568591594696,
+      "rewards/rejected": -2.617647886276245,
       "step": 4375
     },
     {
       "epoch": 2.344204716507777,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.261412652982543,
       "learning_rate": 1.3844577844886109e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3604537844657898,
+      "logits/rejected": -0.18028056621551514,
+      "logps/chosen": -1.7703317403793335,
+      "logps/rejected": -2.54407000541687,
+      "loss": 0.6454,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7703317403793335,
+      "rewards/margins": 0.7737382650375366,
+      "rewards/rejected": -2.54407000541687,
       "step": 4380
     },
     {
       "epoch": 2.346880749289179,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.813064578337984,
       "learning_rate": 1.3737178230448955e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.38482218980789185,
+      "logits/rejected": -0.2585505545139313,
+      "logps/chosen": -1.7552177906036377,
+      "logps/rejected": -2.5703039169311523,
+      "loss": 0.6334,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7552177906036377,
+      "rewards/margins": 0.8150862455368042,
+      "rewards/rejected": -2.5703039169311523,
       "step": 4385
     },
     {
       "epoch": 2.3495567820705805,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6586530139071685,
       "learning_rate": 1.363013043806764e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33463671803474426,
+      "logits/rejected": -0.21315093338489532,
+      "logps/chosen": -1.6880172491073608,
+      "logps/rejected": -2.4300947189331055,
+      "loss": 0.6338,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6880172491073608,
+      "rewards/margins": 0.7420774698257446,
+      "rewards/rejected": -2.4300947189331055,
       "step": 4390
     },
     {
       "epoch": 2.3522328148519818,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.323964995470904,
       "learning_rate": 1.352343550632034e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3076775074005127,
+      "logits/rejected": -0.16306333243846893,
+      "logps/chosen": -1.7408854961395264,
+      "logps/rejected": -2.8347063064575195,
+      "loss": 0.635,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.7408854961395264,
+      "rewards/margins": 1.0938211679458618,
+      "rewards/rejected": -2.8347063064575195,
       "step": 4395
     },
     {
       "epoch": 2.3549088476333835,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7469462068936132,
       "learning_rate": 1.3417094470361722e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.407585084438324,
+      "logits/rejected": -0.2629408538341522,
+      "logps/chosen": -1.8576734066009521,
+      "logps/rejected": -2.590334415435791,
+      "loss": 0.6294,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.8576734066009521,
+      "rewards/margins": 0.7326608896255493,
+      "rewards/rejected": -2.590334415435791,
       "step": 4400
     },
     {
       "epoch": 2.3549088476333835,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.7377,
-      "eval_samples_per_second": 38.719,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.701,
+      "eval_logits/chosen": -0.07782063633203506,
+      "eval_logits/rejected": 0.0008049181196838617,
+      "eval_logps/chosen": -1.8215194940567017,
+      "eval_logps/rejected": -2.59859561920166,
+      "eval_loss": 0.6514319181442261,
+      "eval_rewards/accuracies": 0.671364963054657,
+      "eval_rewards/chosen": -1.8215194940567017,
+      "eval_rewards/margins": 0.7770758867263794,
+      "eval_rewards/rejected": -2.59859561920166,
+      "eval_runtime": 40.532,
+      "eval_samples_per_second": 33.184,
+      "eval_steps_per_second": 8.314,
       "step": 4400
     },
     {
       "epoch": 2.357584880414785,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.348547992407362,
       "learning_rate": 1.3311108361913015e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.41006383299827576,
+      "logits/rejected": -0.3769548833370209,
+      "logps/chosen": -1.792110800743103,
+      "logps/rejected": -2.376709461212158,
+      "loss": 0.6372,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.792110800743103,
+      "rewards/margins": 0.5845984220504761,
+      "rewards/rejected": -2.376709461212158,
       "step": 4405
     },
     {
       "epoch": 2.3602609131961865,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3704975197244442,
       "learning_rate": 1.3205478209251874e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3163067698478699,
+      "logits/rejected": -0.26261502504348755,
+      "logps/chosen": -1.918731689453125,
+      "logps/rejected": -2.773430585861206,
+      "loss": 0.6343,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.918731689453125,
+      "rewards/margins": 0.854698657989502,
+      "rewards/rejected": -2.773430585861206,
       "step": 4410
     },
     {
       "epoch": 2.362936945977588,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.977677818140991,
       "learning_rate": 1.310020503720254e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3101418614387512,
+      "logits/rejected": -0.14574924111366272,
+      "logps/chosen": -1.8429241180419922,
+      "logps/rejected": -2.534780263900757,
+      "loss": 0.6443,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8429241180419922,
+      "rewards/margins": 0.6918561458587646,
+      "rewards/rejected": -2.534780263900757,
       "step": 4415
     },
     {
       "epoch": 2.36561297875899,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.174781928490571,
       "learning_rate": 1.2995289867125752e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3415676951408386,
+      "logits/rejected": -0.27117228507995605,
+      "logps/chosen": -1.8236768245697021,
+      "logps/rejected": -2.3673901557922363,
+      "loss": 0.6474,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.8236768245697021,
+      "rewards/margins": 0.5437131524085999,
+      "rewards/rejected": -2.3673901557922363,
       "step": 4420
     },
     {
       "epoch": 2.368289011540391,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.513514245806839,
       "learning_rate": 1.2890733716908986e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3365481495857239,
+      "logits/rejected": -0.2380417138338089,
+      "logps/chosen": -1.5756289958953857,
+      "logps/rejected": -2.3698067665100098,
+      "loss": 0.5987,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.5756289958953857,
+      "rewards/margins": 0.7941776514053345,
+      "rewards/rejected": -2.3698067665100098,
       "step": 4425
     },
     {
       "epoch": 2.370965044321793,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8190032969066277,
       "learning_rate": 1.2786537600956454e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36630773544311523,
+      "logits/rejected": -0.21061179041862488,
+      "logps/chosen": -1.8328521251678467,
+      "logps/rejected": -2.6732728481292725,
+      "loss": 0.6301,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.8328521251678467,
+      "rewards/margins": 0.8404207229614258,
+      "rewards/rejected": -2.6732728481292725,
       "step": 4430
     },
     {
       "epoch": 2.3736410771031946,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3402373565726595,
       "learning_rate": 1.268270253017933e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4255373477935791,
+      "logits/rejected": -0.23574483394622803,
+      "logps/chosen": -1.7449229955673218,
+      "logps/rejected": -2.6929125785827637,
+      "loss": 0.6252,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.7449229955673218,
+      "rewards/margins": 0.9479901194572449,
+      "rewards/rejected": -2.6929125785827637,
       "step": 4435
     },
     {
       "epoch": 2.376317109884596,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.369978607429932,
       "learning_rate": 1.257922951198591e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4335462152957916,
+      "logits/rejected": -0.22237510979175568,
+      "logps/chosen": -1.779528260231018,
+      "logps/rejected": -2.541109561920166,
+      "loss": 0.642,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.779528260231018,
+      "rewards/margins": 0.761581301689148,
+      "rewards/rejected": -2.541109561920166,
       "step": 4440
     },
     {
       "epoch": 2.3789931426659976,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.674721276520078,
       "learning_rate": 1.24761195502719e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36738961935043335,
+      "logits/rejected": -0.18383683264255524,
+      "logps/chosen": -1.7971773147583008,
+      "logps/rejected": -2.4280996322631836,
+      "loss": 0.6386,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.7971773147583008,
+      "rewards/margins": 0.6309221982955933,
+      "rewards/rejected": -2.4280996322631836,
       "step": 4445
     },
     {
       "epoch": 2.3816691754473993,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9375767956543184,
       "learning_rate": 1.2373373645410573e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3088432550430298,
+      "logits/rejected": -0.18048100173473358,
+      "logps/chosen": -2.002148151397705,
+      "logps/rejected": -2.788806200027466,
+      "loss": 0.6374,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.002148151397705,
+      "rewards/margins": 0.7866576910018921,
+      "rewards/rejected": -2.788806200027466,
       "step": 4450
     },
     {
       "epoch": 2.384345208228801,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8672546418565616,
       "learning_rate": 1.2270992794243175e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.394756942987442,
+      "logits/rejected": -0.2997278571128845,
+      "logps/chosen": -1.7987737655639648,
+      "logps/rejected": -2.6619699001312256,
+      "loss": 0.6367,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7987737655639648,
+      "rewards/margins": 0.8631962537765503,
+      "rewards/rejected": -2.6619699001312256,
       "step": 4455
     },
     {
       "epoch": 2.3870212410102023,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8100553533776256,
       "learning_rate": 1.2168977990069147e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4052153527736664,
+      "logits/rejected": -0.2049853503704071,
+      "logps/chosen": -1.7444114685058594,
+      "logps/rejected": -2.714082717895508,
+      "loss": 0.623,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.7444114685058594,
+      "rewards/margins": 0.9696715474128723,
+      "rewards/rejected": -2.714082717895508,
       "step": 4460
     },
     {
       "epoch": 2.389697273791604,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2435896289873036,
       "learning_rate": 1.206733022263659e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.41493672132492065,
+      "logits/rejected": -0.24559709429740906,
+      "logps/chosen": -1.8315160274505615,
+      "logps/rejected": -2.5717101097106934,
+      "loss": 0.6592,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.8315160274505615,
+      "rewards/margins": 0.7401941418647766,
+      "rewards/rejected": -2.5717101097106934,
       "step": 4465
     },
     {
       "epoch": 2.3923733065730053,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.427008283444539,
       "learning_rate": 1.1966050478132572e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2819797992706299,
+      "logits/rejected": -0.21192362904548645,
+      "logps/chosen": -1.6955623626708984,
+      "logps/rejected": -2.344102382659912,
+      "loss": 0.6352,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6955623626708984,
+      "rewards/margins": 0.6485401391983032,
+      "rewards/rejected": -2.344102382659912,
       "step": 4470
     },
     {
       "epoch": 2.395049339354407,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4987511623225225,
       "learning_rate": 1.1865139739173635e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3589915335178375,
+      "logits/rejected": -0.1512741893529892,
+      "logps/chosen": -1.841543436050415,
+      "logps/rejected": -2.640795946121216,
+      "loss": 0.6316,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.841543436050415,
+      "rewards/margins": 0.7992528676986694,
+      "rewards/rejected": -2.640795946121216,
       "step": 4475
     },
     {
       "epoch": 2.3977253721358087,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.333908979526546,
       "learning_rate": 1.1764598984796187e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36414626240730286,
+      "logits/rejected": -0.28508707880973816,
+      "logps/chosen": -1.642447829246521,
+      "logps/rejected": -2.36337947845459,
+      "loss": 0.6378,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.642447829246521,
+      "rewards/margins": 0.7209314107894897,
+      "rewards/rejected": -2.36337947845459,
       "step": 4480
     },
     {
       "epoch": 2.4004014049172104,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.16535265869227,
       "learning_rate": 1.1664429190447095e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.35410210490226746,
+      "logits/rejected": -0.2717391848564148,
+      "logps/chosen": -1.7873741388320923,
+      "logps/rejected": -2.6225664615631104,
+      "loss": 0.6381,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7873741388320923,
+      "rewards/margins": 0.8351923227310181,
+      "rewards/rejected": -2.6225664615631104,
       "step": 4485
     },
     {
       "epoch": 2.4030774376986117,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.200744401628418,
       "learning_rate": 1.1564631327974122e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.37899166345596313,
+      "logits/rejected": -0.16827648878097534,
+      "logps/chosen": -1.739331841468811,
+      "logps/rejected": -2.8000082969665527,
+      "loss": 0.617,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.739331841468811,
+      "rewards/margins": 1.0606762170791626,
+      "rewards/rejected": -2.8000082969665527,
       "step": 4490
     },
     {
       "epoch": 2.4057534704800134,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.639898506055999,
       "learning_rate": 1.1465206365616587e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4435587525367737,
+      "logits/rejected": -0.2509711682796478,
+      "logps/chosen": -1.7548288106918335,
+      "logps/rejected": -2.5668725967407227,
+      "loss": 0.6357,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7548288106918335,
+      "rewards/margins": 0.8120437860488892,
+      "rewards/rejected": -2.5668725967407227,
       "step": 4495
     },
     {
       "epoch": 2.408429503261415,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0449975032286765,
       "learning_rate": 1.1366155267995887e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25938931107521057,
+      "logits/rejected": -0.2606748938560486,
+      "logps/chosen": -1.7342004776000977,
+      "logps/rejected": -2.494269371032715,
+      "loss": 0.6271,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7342004776000977,
+      "rewards/margins": 0.7600687742233276,
+      "rewards/rejected": -2.494269371032715,
       "step": 4500
     },
     {
       "epoch": 2.4111055360428164,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.664390651804298,
       "learning_rate": 1.1267478996106228e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3862311840057373,
+      "logits/rejected": -0.238784521818161,
+      "logps/chosen": -1.7857458591461182,
+      "logps/rejected": -2.538468360900879,
+      "loss": 0.6342,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7857458591461182,
+      "rewards/margins": 0.752722442150116,
+      "rewards/rejected": -2.538468360900879,
       "step": 4505
     },
     {
       "epoch": 2.413781568824218,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.602434978036108,
       "learning_rate": 1.116917850730521e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3959066867828369,
+      "logits/rejected": -0.2757801413536072,
+      "logps/chosen": -1.8242563009262085,
+      "logps/rejected": -2.4745359420776367,
+      "loss": 0.6372,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8242563009262085,
+      "rewards/margins": 0.6502797603607178,
+      "rewards/rejected": -2.4745359420776367,
       "step": 4510
     },
     {
       "epoch": 2.41645760160562,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3925774712755596,
       "learning_rate": 1.1071254755304637e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36941561102867126,
+      "logits/rejected": -0.30356502532958984,
+      "logps/chosen": -1.6827844381332397,
+      "logps/rejected": -2.5098698139190674,
+      "loss": 0.6254,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6827844381332397,
+      "rewards/margins": 0.8270853161811829,
+      "rewards/rejected": -2.5098698139190674,
       "step": 4515
     },
     {
       "epoch": 2.419133634387021,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4061608083625012,
       "learning_rate": 1.0973708690161143e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3405909240245819,
+      "logits/rejected": -0.2539195120334625,
+      "logps/chosen": -1.708552598953247,
+      "logps/rejected": -2.6818342208862305,
+      "loss": 0.6059,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.708552598953247,
+      "rewards/margins": 0.9732818603515625,
+      "rewards/rejected": -2.6818342208862305,
       "step": 4520
     },
     {
       "epoch": 2.421809667168423,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4873912661758304,
       "learning_rate": 1.0876541258267119e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.41522806882858276,
+      "logits/rejected": -0.24374647438526154,
+      "logps/chosen": -1.8222522735595703,
+      "logps/rejected": -2.660886764526367,
+      "loss": 0.6317,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.8222522735595703,
+      "rewards/margins": 0.8386346697807312,
+      "rewards/rejected": -2.660886764526367,
       "step": 4525
     },
     {
       "epoch": 2.4244856999498245,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6035424027112994,
       "learning_rate": 1.0779753402341379e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3976280689239502,
+      "logits/rejected": -0.3150855600833893,
+      "logps/chosen": -1.8207180500030518,
+      "logps/rejected": -2.4412295818328857,
+      "loss": 0.6563,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8207180500030518,
+      "rewards/margins": 0.620511531829834,
+      "rewards/rejected": -2.4412295818328857,
       "step": 4530
     },
     {
       "epoch": 2.427161732731226,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2948417353525725,
       "learning_rate": 1.0683346061420157e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2357984483242035,
+      "logits/rejected": -0.1362278014421463,
+      "logps/chosen": -1.7148891687393188,
+      "logps/rejected": -2.591066598892212,
+      "loss": 0.6334,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7148891687393188,
+      "rewards/margins": 0.876177191734314,
+      "rewards/rejected": -2.591066598892212,
       "step": 4535
     },
     {
       "epoch": 2.4298377655126275,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.700263642740025,
       "learning_rate": 1.0587320170847874e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2672288417816162,
+      "logits/rejected": -0.18598780035972595,
+      "logps/chosen": -1.6883213520050049,
+      "logps/rejected": -2.453092575073242,
+      "loss": 0.637,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6883213520050049,
+      "rewards/margins": 0.7647711634635925,
+      "rewards/rejected": -2.453092575073242,
       "step": 4540
     },
     {
       "epoch": 2.4325137982940293,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.279966397561868,
       "learning_rate": 1.0491676662268156e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30240780115127563,
+      "logits/rejected": -0.17786887288093567,
+      "logps/chosen": -1.768720269203186,
+      "logps/rejected": -2.544724702835083,
+      "loss": 0.6621,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.768720269203186,
+      "rewards/margins": 0.7760045528411865,
+      "rewards/rejected": -2.544724702835083,
       "step": 4545
     },
     {
       "epoch": 2.4351898310754305,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.92392914645922,
       "learning_rate": 1.0396416463614732e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3666709065437317,
+      "logits/rejected": -0.2662682831287384,
+      "logps/chosen": -1.6269958019256592,
+      "logps/rejected": -2.5118014812469482,
+      "loss": 0.6226,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6269958019256592,
+      "rewards/margins": 0.8848057985305786,
+      "rewards/rejected": -2.5118014812469482,
       "step": 4550
     },
     {
       "epoch": 2.4378658638568322,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.4302521387396046,
       "learning_rate": 1.0301540499102479e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3394717276096344,
+      "logits/rejected": -0.2763212323188782,
+      "logps/chosen": -1.896158218383789,
+      "logps/rejected": -2.540597438812256,
+      "loss": 0.642,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.896158218383789,
+      "rewards/margins": 0.6444392204284668,
+      "rewards/rejected": -2.540597438812256,
       "step": 4555
     },
     {
       "epoch": 2.440541896638234,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.920627472439535,
       "learning_rate": 1.0207049689218405e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3737161457538605,
+      "logits/rejected": -0.19910819828510284,
+      "logps/chosen": -1.7671979665756226,
+      "logps/rejected": -2.569070816040039,
+      "loss": 0.6271,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7671979665756226,
+      "rewards/margins": 0.8018728494644165,
+      "rewards/rejected": -2.569070816040039,
       "step": 4560
     },
     {
       "epoch": 2.4432179294196352,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 12.185605139377287,
       "learning_rate": 1.0112944950712782e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32298845052719116,
+      "logits/rejected": -0.22985847294330597,
+      "logps/chosen": -1.7550981044769287,
+      "logps/rejected": -2.5336966514587402,
+      "loss": 0.6351,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7550981044769287,
+      "rewards/margins": 0.778598427772522,
+      "rewards/rejected": -2.5336966514587402,
       "step": 4565
     },
     {
       "epoch": 2.445893962201037,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9484458715614665,
       "learning_rate": 1.0019227196590174e-07,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26166146993637085,
+      "logits/rejected": -0.1305924355983734,
+      "logps/chosen": -1.7308450937271118,
+      "logps/rejected": -2.6151154041290283,
+      "loss": 0.6267,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.7308450937271118,
+      "rewards/margins": 0.8842706680297852,
+      "rewards/rejected": -2.6151154041290283,
       "step": 4570
     },
     {
       "epoch": 2.4485699949824387,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.014459470694215,
       "learning_rate": 9.925897336100664e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2753921449184418,
+      "logits/rejected": -0.22050213813781738,
+      "logps/chosen": -1.6932579278945923,
+      "logps/rejected": -2.466698408126831,
+      "loss": 0.6298,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6932579278945923,
+      "rewards/margins": 0.7734404802322388,
+      "rewards/rejected": -2.466698408126831,
       "step": 4575
     },
     {
       "epoch": 2.45124602776384,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7454324560935106,
       "learning_rate": 9.832956274730946e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32033124566078186,
+      "logits/rejected": -0.2809397876262665,
+      "logps/chosen": -1.6984596252441406,
+      "logps/rejected": -2.413623094558716,
+      "loss": 0.6172,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6984596252441406,
+      "rewards/margins": 0.71516352891922,
+      "rewards/rejected": -2.413623094558716,
       "step": 4580
     },
     {
       "epoch": 2.4539220605452416,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.056389061609097,
       "learning_rate": 9.740404914195633e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3172834813594818,
+      "logits/rejected": -0.1805722713470459,
+      "logps/chosen": -1.7809635400772095,
+      "logps/rejected": -2.5508055686950684,
+      "loss": 0.6349,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7809635400772095,
+      "rewards/margins": 0.7698421478271484,
+      "rewards/rejected": -2.5508055686950684,
       "step": 4585
     },
     {
       "epoch": 2.4565980933266434,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2679102618397216,
       "learning_rate": 9.648244152428392e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4158778786659241,
+      "logits/rejected": -0.25018391013145447,
+      "logps/chosen": -1.665862798690796,
+      "logps/rejected": -2.305826425552368,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.665862798690796,
+      "rewards/margins": 0.6399635672569275,
+      "rewards/rejected": -2.305826425552368,
       "step": 4590
     },
     {
       "epoch": 2.4592741261080446,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.345755205913535,
       "learning_rate": 9.556474883573379e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3630771040916443,
+      "logits/rejected": -0.259331613779068,
+      "logps/chosen": -1.6674509048461914,
+      "logps/rejected": -2.608018398284912,
+      "loss": 0.6317,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6674509048461914,
+      "rewards/margins": 0.9405676126480103,
+      "rewards/rejected": -2.608018398284912,
       "step": 4595
     },
     {
       "epoch": 2.4619501588894463,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6662197291747596,
       "learning_rate": 9.465097997976412e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3668380677700043,
+      "logits/rejected": -0.13316704332828522,
+      "logps/chosen": -1.746704339981079,
+      "logps/rejected": -2.763484239578247,
+      "loss": 0.6268,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.746704339981079,
+      "rewards/margins": 1.0167797803878784,
+      "rewards/rejected": -2.763484239578247,
       "step": 4600
     },
     {
       "epoch": 2.464626191670848,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.188112810281958,
       "learning_rate": 9.374114382176457e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33177709579467773,
+      "logits/rejected": -0.19537650048732758,
+      "logps/chosen": -1.7065045833587646,
+      "logps/rejected": -2.606412887573242,
+      "loss": 0.6374,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7065045833587646,
+      "rewards/margins": 0.8999080657958984,
+      "rewards/rejected": -2.606412887573242,
       "step": 4605
     },
     {
       "epoch": 2.46730222445225,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9214710133608184,
       "learning_rate": 9.283524918896945e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.35641947388648987,
+      "logits/rejected": -0.22487279772758484,
+      "logps/chosen": -1.752122163772583,
+      "logps/rejected": -2.6588945388793945,
+      "loss": 0.6257,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.752122163772583,
+      "rewards/margins": 0.906772255897522,
+      "rewards/rejected": -2.6588945388793945,
       "step": 4610
     },
     {
       "epoch": 2.469978257233651,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7610085749246376,
       "learning_rate": 9.193330487037232e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2909630239009857,
+      "logits/rejected": -0.16076165437698364,
+      "logps/chosen": -1.8512153625488281,
+      "logps/rejected": -2.9460525512695312,
+      "loss": 0.6389,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8512153625488281,
+      "rewards/margins": 1.0948371887207031,
+      "rewards/rejected": -2.9460525512695312,
       "step": 4615
     },
     {
       "epoch": 2.4726542900150528,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6433408731960246,
       "learning_rate": 9.103531961664118e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28057029843330383,
+      "logits/rejected": -0.08383586257696152,
+      "logps/chosen": -1.6649610996246338,
+      "logps/rejected": -2.5462238788604736,
+      "loss": 0.6189,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6649610996246338,
+      "rewards/margins": 0.8812629580497742,
+      "rewards/rejected": -2.5462238788604736,
       "step": 4620
     },
     {
       "epoch": 2.475330322796454,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7371665361288917,
       "learning_rate": 9.014130214003269e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.40973979234695435,
+      "logits/rejected": -0.3713271915912628,
+      "logps/chosen": -1.7484760284423828,
+      "logps/rejected": -2.5309720039367676,
+      "loss": 0.6358,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7484760284423828,
+      "rewards/margins": 0.7824960350990295,
+      "rewards/rejected": -2.5309720039367676,
       "step": 4625
     },
     {
       "epoch": 2.4780063555778558,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.6585741530365095,
       "learning_rate": 8.925126111430848e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2794022262096405,
+      "logits/rejected": -0.20719806849956512,
+      "logps/chosen": -1.6633245944976807,
+      "logps/rejected": -2.517793655395508,
+      "loss": 0.6214,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.6633245944976807,
+      "rewards/margins": 0.8544691205024719,
+      "rewards/rejected": -2.517793655395508,
       "step": 4630
     },
     {
       "epoch": 2.4806823883592575,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.511257145990953,
       "learning_rate": 8.83652051746504e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23644717037677765,
+      "logits/rejected": -0.09185831248760223,
+      "logps/chosen": -1.8258545398712158,
+      "logps/rejected": -2.806856870651245,
+      "loss": 0.6313,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.8258545398712158,
+      "rewards/margins": 0.9810024499893188,
+      "rewards/rejected": -2.806856870651245,
       "step": 4635
     },
     {
       "epoch": 2.483358421140659,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.866039963469145,
       "learning_rate": 8.748314291757696e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.282148540019989,
+      "logits/rejected": -0.1820870339870453,
+      "logps/chosen": -1.774224042892456,
+      "logps/rejected": -2.6100430488586426,
+      "loss": 0.6307,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.774224042892456,
+      "rewards/margins": 0.8358189463615417,
+      "rewards/rejected": -2.6100430488586426,
       "step": 4640
     },
     {
       "epoch": 2.4860344539220605,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 10.545148457208,
       "learning_rate": 8.660508290086032e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31090161204338074,
+      "logits/rejected": -0.18742410838603973,
+      "logps/chosen": -1.6410408020019531,
+      "logps/rejected": -2.430260419845581,
+      "loss": 0.6283,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6410408020019531,
+      "rewards/margins": 0.7892197370529175,
+      "rewards/rejected": -2.430260419845581,
       "step": 4645
     },
     {
       "epoch": 2.488710486703462,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1454795705044396,
       "learning_rate": 8.573103364344231e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3407088816165924,
+      "logits/rejected": -0.10989546775817871,
+      "logps/chosen": -1.7280409336090088,
+      "logps/rejected": -2.641613006591797,
+      "loss": 0.6244,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7280409336090088,
+      "rewards/margins": 0.9135719537734985,
+      "rewards/rejected": -2.641613006591797,
       "step": 4650
     },
     {
       "epoch": 2.4913865194848634,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.969209891710649,
       "learning_rate": 8.486100362535292e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3544526994228363,
+      "logits/rejected": -0.20713789761066437,
+      "logps/chosen": -1.7289094924926758,
+      "logps/rejected": -2.421337842941284,
+      "loss": 0.6454,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7289094924926758,
+      "rewards/margins": 0.6924282908439636,
+      "rewards/rejected": -2.421337842941284,
       "step": 4655
     },
     {
       "epoch": 2.494062552266265,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.677187541212396,
       "learning_rate": 8.399500128762693e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.345185786485672,
+      "logits/rejected": -0.22764956951141357,
+      "logps/chosen": -1.8246192932128906,
+      "logps/rejected": -2.593273639678955,
+      "loss": 0.6316,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8246192932128906,
+      "rewards/margins": 0.7686543464660645,
+      "rewards/rejected": -2.593273639678955,
       "step": 4660
     },
     {
       "epoch": 2.496738585047667,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.777647780905034,
       "learning_rate": 8.313303503222313e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32963716983795166,
+      "logits/rejected": -0.2604052424430847,
+      "logps/chosen": -2.0204555988311768,
+      "logps/rejected": -2.6320033073425293,
+      "loss": 0.6553,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -2.0204555988311768,
+      "rewards/margins": 0.6115478873252869,
+      "rewards/rejected": -2.6320033073425293,
       "step": 4665
     },
     {
       "epoch": 2.4994146178290686,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9497337981984346,
       "learning_rate": 8.227511322194164e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3374949097633362,
+      "logits/rejected": -0.21395353972911835,
+      "logps/chosen": -1.761682152748108,
+      "logps/rejected": -2.3977866172790527,
+      "loss": 0.6391,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.761682152748108,
+      "rewards/margins": 0.6361044645309448,
+      "rewards/rejected": -2.3977866172790527,
       "step": 4670
     },
     {
       "epoch": 2.50209065061047,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6835180565996297,
       "learning_rate": 8.142124418034385e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27696865797042847,
+      "logits/rejected": -0.12812180817127228,
+      "logps/chosen": -1.7523448467254639,
+      "logps/rejected": -2.5539422035217285,
+      "loss": 0.6192,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7523448467254639,
+      "rewards/margins": 0.8015974760055542,
+      "rewards/rejected": -2.5539422035217285,
       "step": 4675
     },
     {
       "epoch": 2.5047666833918716,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0339168692205685,
       "learning_rate": 8.057143619167073e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26528993248939514,
+      "logits/rejected": -0.16851824522018433,
+      "logps/chosen": -1.7922909259796143,
+      "logps/rejected": -2.7119834423065186,
+      "loss": 0.6316,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.7922909259796143,
+      "rewards/margins": 0.9196925163269043,
+      "rewards/rejected": -2.7119834423065186,
       "step": 4680
     },
     {
       "epoch": 2.507442716173273,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0357810935054634,
       "learning_rate": 7.97256975007633e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.38172757625579834,
+      "logits/rejected": -0.17545638978481293,
+      "logps/chosen": -1.6785633563995361,
+      "logps/rejected": -2.4353890419006348,
+      "loss": 0.6328,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6785633563995361,
+      "rewards/margins": 0.7568257451057434,
+      "rewards/rejected": -2.4353890419006348,
       "step": 4685
     },
     {
       "epoch": 2.5101187489546746,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.05317203367443,
       "learning_rate": 7.888403631298186e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28789272904396057,
+      "logits/rejected": -0.2314242571592331,
+      "logps/chosen": -1.7204748392105103,
+      "logps/rejected": -2.5079309940338135,
+      "loss": 0.6415,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.7204748392105103,
+      "rewards/margins": 0.7874559760093689,
+      "rewards/rejected": -2.5079309940338135,
       "step": 4690
     },
     {
       "epoch": 2.5127947817360763,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.3683801965129994,
       "learning_rate": 7.804646079412719e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2871004641056061,
+      "logits/rejected": -0.1092192530632019,
+      "logps/chosen": -1.8037662506103516,
+      "logps/rejected": -2.557796001434326,
+      "loss": 0.6242,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8037662506103516,
+      "rewards/margins": 0.7540299296379089,
+      "rewards/rejected": -2.557796001434326,
       "step": 4695
     },
     {
       "epoch": 2.515470814517478,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.918309693516709,
       "learning_rate": 7.72129790703604e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3942047953605652,
+      "logits/rejected": -0.2772538959980011,
+      "logps/chosen": -1.6554441452026367,
+      "logps/rejected": -2.4783589839935303,
+      "loss": 0.6281,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6554441452026367,
+      "rewards/margins": 0.8229148983955383,
+      "rewards/rejected": -2.4783589839935303,
       "step": 4700
     },
     {
       "epoch": 2.5181468472988793,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.486871642431245,
       "learning_rate": 7.638359922812504e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26174527406692505,
+      "logits/rejected": -0.22303374111652374,
+      "logps/chosen": -1.8373247385025024,
+      "logps/rejected": -2.551281690597534,
+      "loss": 0.6358,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8373247385025024,
+      "rewards/margins": 0.7139570713043213,
+      "rewards/rejected": -2.551281690597534,
       "step": 4705
     },
     {
       "epoch": 2.520822880080281,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.907709067269801,
       "learning_rate": 7.555832931406774e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36601758003234863,
+      "logits/rejected": -0.208743616938591,
+      "logps/chosen": -1.8351609706878662,
+      "logps/rejected": -2.6942193508148193,
+      "loss": 0.6373,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.8351609706878662,
+      "rewards/margins": 0.8590584993362427,
+      "rewards/rejected": -2.6942193508148193,
       "step": 4710
     },
     {
       "epoch": 2.5234989128616827,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.060417505535391,
       "learning_rate": 7.47371773349611e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3157570958137512,
+      "logits/rejected": -0.28143349289894104,
+      "logps/chosen": -1.8886724710464478,
+      "logps/rejected": -2.7185275554656982,
+      "loss": 0.6299,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8886724710464478,
+      "rewards/margins": 0.8298551440238953,
+      "rewards/rejected": -2.7185275554656982,
       "step": 4715
     },
     {
       "epoch": 2.526174945643084,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.327913071257849,
       "learning_rate": 7.392015125762496e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3142890930175781,
+      "logits/rejected": -0.2092617005109787,
+      "logps/chosen": -1.7095396518707275,
+      "logps/rejected": -2.509830951690674,
+      "loss": 0.6254,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7095396518707275,
+      "rewards/margins": 0.8002913594245911,
+      "rewards/rejected": -2.509830951690674,
       "step": 4720
     },
     {
       "epoch": 2.5288509784244857,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.173455405255259,
       "learning_rate": 7.310725900885018e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34330859780311584,
+      "logits/rejected": -0.28681227564811707,
+      "logps/chosen": -1.7928367853164673,
+      "logps/rejected": -2.4322080612182617,
+      "loss": 0.6367,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7928367853164673,
+      "rewards/margins": 0.6393714547157288,
+      "rewards/rejected": -2.4322080612182617,
       "step": 4725
     },
     {
       "epoch": 2.5315270112058874,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.166693767189998,
       "learning_rate": 7.229850847532076e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3086932599544525,
+      "logits/rejected": -0.19496838748455048,
+      "logps/chosen": -1.6521517038345337,
+      "logps/rejected": -2.462780475616455,
+      "loss": 0.6206,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6521517038345337,
+      "rewards/margins": 0.8106290102005005,
+      "rewards/rejected": -2.462780475616455,
       "step": 4730
     },
     {
       "epoch": 2.5342030439872887,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6908243602282744,
       "learning_rate": 7.149390750353779e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2598186433315277,
+      "logits/rejected": -0.30689769983291626,
+      "logps/chosen": -1.7692327499389648,
+      "logps/rejected": -2.427434206008911,
+      "loss": 0.6232,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7692327499389648,
+      "rewards/margins": 0.6582015752792358,
+      "rewards/rejected": -2.427434206008911,
       "step": 4735
     },
     {
       "epoch": 2.5368790767686904,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.436770992930151,
       "learning_rate": 7.069346389974374e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3501608371734619,
+      "logits/rejected": -0.20892676711082458,
+      "logps/chosen": -1.754713773727417,
+      "logps/rejected": -2.601682186126709,
+      "loss": 0.632,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.754713773727417,
+      "rewards/margins": 0.8469686508178711,
+      "rewards/rejected": -2.601682186126709,
       "step": 4740
     },
     {
       "epoch": 2.539555109550092,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.360283351456191,
       "learning_rate": 6.989718542984563e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29178953170776367,
+      "logits/rejected": -0.24500393867492676,
+      "logps/chosen": -1.8699512481689453,
+      "logps/rejected": -2.608508348464966,
+      "loss": 0.6258,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.8699512481689453,
+      "rewards/margins": 0.7385571599006653,
+      "rewards/rejected": -2.608508348464966,
       "step": 4745
     },
     {
       "epoch": 2.5422311423314934,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2401680685994556,
       "learning_rate": 6.9105079819341e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2783927321434021,
+      "logits/rejected": -0.09226509183645248,
+      "logps/chosen": -1.7047332525253296,
+      "logps/rejected": -2.8983700275421143,
+      "loss": 0.6272,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.7047332525253296,
+      "rewards/margins": 1.1936366558074951,
+      "rewards/rejected": -2.8983700275421143,
       "step": 4750
     },
     {
       "epoch": 2.544907175112895,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5538424138026175,
       "learning_rate": 6.831715475324163e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.373802125453949,
+      "logits/rejected": -0.2126348465681076,
+      "logps/chosen": -1.6304079294204712,
+      "logps/rejected": -2.7318787574768066,
+      "loss": 0.6144,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6304079294204712,
+      "rewards/margins": 1.101471185684204,
+      "rewards/rejected": -2.7318787574768066,
       "step": 4755
     },
     {
       "epoch": 2.547583207894297,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5431125537403525,
       "learning_rate": 6.753341787600026e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4034843444824219,
+      "logits/rejected": -0.2755557894706726,
+      "logps/chosen": -1.6206445693969727,
+      "logps/rejected": -2.559770107269287,
+      "loss": 0.6253,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.6206445693969727,
+      "rewards/margins": 0.9391258358955383,
+      "rewards/rejected": -2.559770107269287,
       "step": 4760
     },
     {
       "epoch": 2.5502592406756985,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5231272695809017,
       "learning_rate": 6.67538767914353e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3814742863178253,
+      "logits/rejected": -0.20748881995677948,
+      "logps/chosen": -1.8639312982559204,
+      "logps/rejected": -2.5788328647613525,
+      "loss": 0.6494,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8639312982559204,
+      "rewards/margins": 0.7149016261100769,
+      "rewards/rejected": -2.5788328647613525,
       "step": 4765
     },
     {
       "epoch": 2.5529352734571,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.874316562941172,
       "learning_rate": 6.597853906265793e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3440282940864563,
+      "logits/rejected": -0.25217828154563904,
+      "logps/chosen": -1.780583143234253,
+      "logps/rejected": -2.7937779426574707,
+      "loss": 0.6322,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.780583143234253,
+      "rewards/margins": 1.0131947994232178,
+      "rewards/rejected": -2.7937779426574707,
       "step": 4770
     },
     {
       "epoch": 2.5556113062385015,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5588289691711203,
       "learning_rate": 6.5207412211998e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.21451349556446075,
+      "logits/rejected": -0.12699481844902039,
+      "logps/chosen": -1.724069356918335,
+      "logps/rejected": -2.590970516204834,
+      "loss": 0.6231,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.724069356918335,
+      "rewards/margins": 0.8669013977050781,
+      "rewards/rejected": -2.590970516204834,
       "step": 4775
     },
     {
       "epoch": 2.558287339019903,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.1357145353568905,
       "learning_rate": 6.444050372093186e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31357306241989136,
+      "logits/rejected": -0.21205756068229675,
+      "logps/chosen": -1.7464720010757446,
+      "logps/rejected": -2.536564350128174,
+      "loss": 0.6411,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7464720010757446,
+      "rewards/margins": 0.7900919914245605,
+      "rewards/rejected": -2.536564350128174,
       "step": 4780
     },
     {
       "epoch": 2.5609633718013045,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.059978278938339,
       "learning_rate": 6.367782103000873e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.301750123500824,
+      "logits/rejected": -0.23965826630592346,
+      "logps/chosen": -1.804858922958374,
+      "logps/rejected": -2.4217610359191895,
+      "loss": 0.6514,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.804858922958374,
+      "rewards/margins": 0.616902232170105,
+      "rewards/rejected": -2.4217610359191895,
       "step": 4785
     },
     {
       "epoch": 2.5636394045827062,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.0959807759992937,
       "learning_rate": 6.29193715387798e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4151396155357361,
+      "logits/rejected": -0.29026761651039124,
+      "logps/chosen": -1.7557621002197266,
+      "logps/rejected": -2.6925177574157715,
+      "loss": 0.633,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7557621002197266,
+      "rewards/margins": 0.9367557764053345,
+      "rewards/rejected": -2.6925177574157715,
       "step": 4790
     },
     {
       "epoch": 2.566315437364108,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.220349267952519,
       "learning_rate": 6.216516260572502e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27624744176864624,
+      "logits/rejected": -0.19688108563423157,
+      "logps/chosen": -1.8552882671356201,
+      "logps/rejected": -2.515388011932373,
+      "loss": 0.6436,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.8552882671356201,
+      "rewards/margins": 0.6600998044013977,
+      "rewards/rejected": -2.515388011932373,
       "step": 4795
     },
     {
       "epoch": 2.568991470145509,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6250602181832803,
       "learning_rate": 6.141520154818297e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34858226776123047,
+      "logits/rejected": -0.23849251866340637,
+      "logps/chosen": -1.69162118434906,
+      "logps/rejected": -2.5039455890655518,
+      "loss": 0.6098,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.69162118434906,
+      "rewards/margins": 0.8123245239257812,
+      "rewards/rejected": -2.5039455890655518,
       "step": 4800
     },
     {
       "epoch": 2.568991470145509,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.8057,
-      "eval_samples_per_second": 38.643,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.682,
+      "eval_logits/chosen": -0.010313918814063072,
+      "eval_logits/rejected": 0.07351107150316238,
+      "eval_logps/chosen": -1.7917885780334473,
+      "eval_logps/rejected": -2.569279432296753,
+      "eval_loss": 0.6506672501564026,
+      "eval_rewards/accuracies": 0.6765578389167786,
+      "eval_rewards/chosen": -1.7917885780334473,
+      "eval_rewards/margins": 0.7774907350540161,
+      "eval_rewards/rejected": -2.569279432296753,
+      "eval_runtime": 40.3989,
+      "eval_samples_per_second": 33.293,
+      "eval_steps_per_second": 8.342,
       "step": 4800
     },
     {
       "epoch": 2.571667502926911,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.084576775487708,
       "learning_rate": 6.066949564227897e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.39581921696662903,
+      "logits/rejected": -0.27536314725875854,
+      "logps/chosen": -1.7372856140136719,
+      "logps/rejected": -2.5802907943725586,
+      "loss": 0.6287,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7372856140136719,
+      "rewards/margins": 0.8430051803588867,
+      "rewards/rejected": -2.5802907943725586,
       "step": 4805
     },
     {
       "epoch": 2.574343535708312,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.119331803245935,
       "learning_rate": 5.992805212285523e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3052743077278137,
+      "logits/rejected": -0.1878601610660553,
+      "logps/chosen": -1.771254301071167,
+      "logps/rejected": -2.501477003097534,
+      "loss": 0.6458,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.771254301071167,
+      "rewards/margins": 0.7302228808403015,
+      "rewards/rejected": -2.501477003097534,
       "step": 4810
     },
     {
       "epoch": 2.577019568489714,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 11.996624413371373,
       "learning_rate": 5.9190878183399684e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3068881630897522,
+      "logits/rejected": -0.1794106662273407,
+      "logps/chosen": -1.6003291606903076,
+      "logps/rejected": -2.533151149749756,
+      "loss": 0.6061,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6003291606903076,
+      "rewards/margins": 0.9328222274780273,
+      "rewards/rejected": -2.533151149749756,
       "step": 4815
     },
     {
       "epoch": 2.5796956012711156,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5715388854664516,
       "learning_rate": 5.845798097597748e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26806825399398804,
+      "logits/rejected": -0.18667533993721008,
+      "logps/chosen": -1.8457601070404053,
+      "logps/rejected": -2.417269706726074,
+      "loss": 0.6342,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.8457601070404053,
+      "rewards/margins": 0.5715095400810242,
+      "rewards/rejected": -2.417269706726074,
       "step": 4820
     },
     {
       "epoch": 2.5823716340525174,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.67582474189255,
       "learning_rate": 5.772936761116026e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3103354573249817,
+      "logits/rejected": -0.18535049259662628,
+      "logps/chosen": -1.7792314291000366,
+      "logps/rejected": -2.4361772537231445,
+      "loss": 0.635,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7792314291000366,
+      "rewards/margins": 0.6569457054138184,
+      "rewards/rejected": -2.4361772537231445,
       "step": 4825
     },
     {
       "epoch": 2.5850476668339186,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.937664186917649,
       "learning_rate": 5.700504515795829e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34878450632095337,
+      "logits/rejected": -0.2155938595533371,
+      "logps/chosen": -1.8296024799346924,
+      "logps/rejected": -2.490471363067627,
+      "loss": 0.6415,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.8296024799346924,
+      "rewards/margins": 0.6608688235282898,
+      "rewards/rejected": -2.490471363067627,
       "step": 4830
     },
     {
       "epoch": 2.5877236996153203,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.081157776766907,
       "learning_rate": 5.628502064375101e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.45001688599586487,
+      "logits/rejected": -0.2697954475879669,
+      "logps/chosen": -1.7228597402572632,
+      "logps/rejected": -2.616790294647217,
+      "loss": 0.6384,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7228597402572632,
+      "rewards/margins": 0.8939305543899536,
+      "rewards/rejected": -2.616790294647217,
       "step": 4835
     },
     {
       "epoch": 2.5903997323967216,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.438335539055274,
       "learning_rate": 5.55693010542197e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.40222078561782837,
+      "logits/rejected": -0.21907205879688263,
+      "logps/chosen": -1.6957511901855469,
+      "logps/rejected": -2.626537561416626,
+      "loss": 0.6245,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6957511901855469,
+      "rewards/margins": 0.9307863116264343,
+      "rewards/rejected": -2.626537561416626,
       "step": 4840
     },
     {
       "epoch": 2.5930757651781233,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8559170709079096,
       "learning_rate": 5.485789333327856e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.35954198241233826,
+      "logits/rejected": -0.2648337185382843,
+      "logps/chosen": -1.7580009698867798,
+      "logps/rejected": -2.503173351287842,
+      "loss": 0.6363,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.7580009698867798,
+      "rewards/margins": 0.7451725006103516,
+      "rewards/rejected": -2.503173351287842,
       "step": 4845
     },
     {
       "epoch": 2.595751797959525,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.60765753506622,
       "learning_rate": 5.4150804383008675e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.46444255113601685,
+      "logits/rejected": -0.3219277262687683,
+      "logps/chosen": -1.7803875207901,
+      "logps/rejected": -2.632236957550049,
+      "loss": 0.64,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7803875207901,
+      "rewards/margins": 0.8518489599227905,
+      "rewards/rejected": -2.632236957550049,
       "step": 4850
     },
     {
       "epoch": 2.5984278307409268,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.266200978796011,
       "learning_rate": 5.344804106359002e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29651978611946106,
+      "logits/rejected": -0.15605640411376953,
+      "logps/chosen": -1.6830055713653564,
+      "logps/rejected": -2.5393669605255127,
+      "loss": 0.6268,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.6830055713653564,
+      "rewards/margins": 0.8563610315322876,
+      "rewards/rejected": -2.5393669605255127,
       "step": 4855
     },
     {
       "epoch": 2.601103863522328,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.651896173404922,
       "learning_rate": 5.274961019323559e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34556329250335693,
+      "logits/rejected": -0.29233765602111816,
+      "logps/chosen": -1.5563634634017944,
+      "logps/rejected": -2.5117478370666504,
+      "loss": 0.606,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.5563634634017944,
+      "rewards/margins": 0.9553844332695007,
+      "rewards/rejected": -2.5117478370666504,
       "step": 4860
     },
     {
       "epoch": 2.6037798963037297,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.953044290279977,
       "learning_rate": 5.205551854812451e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.41089048981666565,
+      "logits/rejected": -0.3245064616203308,
+      "logps/chosen": -1.689849853515625,
+      "logps/rejected": -2.626068353652954,
+      "loss": 0.6203,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.689849853515625,
+      "rewards/margins": 0.9362181425094604,
+      "rewards/rejected": -2.626068353652954,
       "step": 4865
     },
     {
       "epoch": 2.606455929085131,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.666276876681703,
       "learning_rate": 5.1365772862337177e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2882205843925476,
+      "logits/rejected": -0.16618773341178894,
+      "logps/chosen": -1.806471824645996,
+      "logps/rejected": -2.6302473545074463,
+      "loss": 0.628,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.806471824645996,
+      "rewards/margins": 0.8237756490707397,
+      "rewards/rejected": -2.6302473545074463,
       "step": 4870
     },
     {
       "epoch": 2.6091319618665327,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.796320496276626,
       "learning_rate": 5.068037982778905e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.17762644588947296,
+      "logits/rejected": -0.10517667233943939,
+      "logps/chosen": -1.6670891046524048,
+      "logps/rejected": -2.6319377422332764,
+      "loss": 0.6273,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6670891046524048,
+      "rewards/margins": 0.9648488163948059,
+      "rewards/rejected": -2.6319377422332764,
       "step": 4875
     },
     {
       "epoch": 2.6118079946479344,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.858519965771402,
       "learning_rate": 4.999934609416656e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.23198369145393372,
+      "logits/rejected": -0.12397295236587524,
+      "logps/chosen": -1.6499942541122437,
+      "logps/rejected": -2.618476629257202,
+      "loss": 0.6118,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.6499942541122437,
+      "rewards/margins": 0.9684826135635376,
+      "rewards/rejected": -2.618476629257202,
       "step": 4880
     },
     {
       "epoch": 2.614484027429336,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6533240513759493,
       "learning_rate": 4.932267826886183e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24396400153636932,
+      "logits/rejected": -0.1810634732246399,
+      "logps/chosen": -1.9138429164886475,
+      "logps/rejected": -2.6590218544006348,
+      "loss": 0.6381,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.9138429164886475,
+      "rewards/margins": 0.7451786994934082,
+      "rewards/rejected": -2.6590218544006348,
       "step": 4885
     },
     {
       "epoch": 2.6171600602107374,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3542466341157944,
       "learning_rate": 4.8650382916909206e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4177396893501282,
+      "logits/rejected": -0.2678728699684143,
+      "logps/chosen": -1.7374515533447266,
+      "logps/rejected": -2.5576536655426025,
+      "loss": 0.6277,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7374515533447266,
+      "rewards/margins": 0.8202022314071655,
+      "rewards/rejected": -2.5576536655426025,
       "step": 4890
     },
     {
       "epoch": 2.619836092992139,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.859673541397908,
       "learning_rate": 4.7982466560920976e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3425450325012207,
+      "logits/rejected": -0.2664199471473694,
+      "logps/chosen": -1.8227548599243164,
+      "logps/rejected": -2.470186948776245,
+      "loss": 0.6353,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8227548599243164,
+      "rewards/margins": 0.6474322080612183,
+      "rewards/rejected": -2.470186948776245,
       "step": 4895
     },
     {
       "epoch": 2.622512125773541,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0915578098638075,
       "learning_rate": 4.7318935681024685e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2890065610408783,
+      "logits/rejected": -0.15463271737098694,
+      "logps/chosen": -1.6751819849014282,
+      "logps/rejected": -2.5790317058563232,
+      "loss": 0.6234,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6751819849014282,
+      "rewards/margins": 0.9038499593734741,
+      "rewards/rejected": -2.5790317058563232,
       "step": 4900
     },
     {
       "epoch": 2.625188158554942,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.564372262104271,
       "learning_rate": 4.6659796714799745e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3123909533023834,
+      "logits/rejected": -0.16087953746318817,
+      "logps/chosen": -1.7844699621200562,
+      "logps/rejected": -2.687450885772705,
+      "loss": 0.6319,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7844699621200562,
+      "rewards/margins": 0.902981162071228,
+      "rewards/rejected": -2.687450885772705,
       "step": 4905
     },
     {
       "epoch": 2.627864191336344,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2292301811621194,
       "learning_rate": 4.60050560572155e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32884809374809265,
+      "logits/rejected": -0.3501850664615631,
+      "logps/chosen": -1.9155794382095337,
+      "logps/rejected": -3.009523630142212,
+      "loss": 0.6477,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9155794382095337,
+      "rewards/margins": 1.0939443111419678,
+      "rewards/rejected": -3.009523630142212,
       "step": 4910
     },
     {
       "epoch": 2.6305402241177456,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.333355059189466,
       "learning_rate": 4.535472006056834e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27297505736351013,
+      "logits/rejected": -0.1630019098520279,
+      "logps/chosen": -1.6011962890625,
+      "logps/rejected": -2.4325575828552246,
+      "loss": 0.6163,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.6011962890625,
+      "rewards/margins": 0.8313612937927246,
+      "rewards/rejected": -2.4325575828552246,
       "step": 4915
     },
     {
       "epoch": 2.6332162568991473,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.211808459034992,
       "learning_rate": 4.470879503442132e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2790857255458832,
+      "logits/rejected": -0.2088259905576706,
+      "logps/chosen": -1.835229516029358,
+      "logps/rejected": -2.508394479751587,
+      "loss": 0.6486,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.835229516029358,
+      "rewards/margins": 0.6731648445129395,
+      "rewards/rejected": -2.508394479751587,
       "step": 4920
     },
     {
       "epoch": 2.6358922896805486,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1986598231631036,
       "learning_rate": 4.406728724554154e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.47235098481178284,
+      "logits/rejected": -0.24175730347633362,
+      "logps/chosen": -1.647503137588501,
+      "logps/rejected": -2.5925686359405518,
+      "loss": 0.616,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.647503137588501,
+      "rewards/margins": 0.9450654983520508,
+      "rewards/rejected": -2.5925686359405518,
       "step": 4925
     },
     {
       "epoch": 2.6385683224619503,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8700445940608628,
       "learning_rate": 4.3430202917840664e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26749318838119507,
+      "logits/rejected": -0.10780428349971771,
+      "logps/chosen": -1.8831889629364014,
+      "logps/rejected": -2.9055662155151367,
+      "loss": 0.6357,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.8831889629364014,
+      "rewards/margins": 1.0223772525787354,
+      "rewards/rejected": -2.9055662155151367,
       "step": 4930
     },
     {
       "epoch": 2.6412443552433515,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.135193525190124,
       "learning_rate": 4.279754823231346e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3818560242652893,
+      "logits/rejected": -0.23750321567058563,
+      "logps/chosen": -1.754361867904663,
+      "logps/rejected": -2.462897300720215,
+      "loss": 0.6304,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.754361867904663,
+      "rewards/margins": 0.7085353136062622,
+      "rewards/rejected": -2.462897300720215,
       "step": 4935
     },
     {
       "epoch": 2.6439203880247533,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.595475208450434,
       "learning_rate": 4.216932932697859e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.313612163066864,
+      "logits/rejected": -0.24269525706768036,
+      "logps/chosen": -1.742937445640564,
+      "logps/rejected": -2.236677408218384,
+      "loss": 0.6305,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.742937445640564,
+      "rewards/margins": 0.49373990297317505,
+      "rewards/rejected": -2.236677408218384,
       "step": 4940
     },
     {
       "epoch": 2.646596420806155,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.785647771726111,
       "learning_rate": 4.154555229681844e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34462857246398926,
+      "logits/rejected": -0.15363140404224396,
+      "logps/chosen": -1.6797311305999756,
+      "logps/rejected": -2.7455055713653564,
+      "loss": 0.6238,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6797311305999756,
+      "rewards/margins": 1.0657740831375122,
+      "rewards/rejected": -2.7455055713653564,
       "step": 4945
     },
     {
       "epoch": 2.6492724535875567,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.396624747501437,
       "learning_rate": 4.092622319372069e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34628576040267944,
+      "logits/rejected": -0.22478561103343964,
+      "logps/chosen": -1.7646477222442627,
+      "logps/rejected": -2.523099899291992,
+      "loss": 0.63,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7646477222442627,
+      "rewards/margins": 0.7584523558616638,
+      "rewards/rejected": -2.523099899291992,
       "step": 4950
     },
     {
       "epoch": 2.651948486368958,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.109475177662839,
       "learning_rate": 4.031134802641889e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31815478205680847,
+      "logits/rejected": -0.31365352869033813,
+      "logps/chosen": -1.7963138818740845,
+      "logps/rejected": -2.4990620613098145,
+      "loss": 0.6192,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7963138818740845,
+      "rewards/margins": 0.7027480006217957,
+      "rewards/rejected": -2.4990620613098145,
       "step": 4955
     },
     {
       "epoch": 2.6546245191503597,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.370518956048994,
       "learning_rate": 3.970093276043468e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26316365599632263,
+      "logits/rejected": -0.17915061116218567,
+      "logps/chosen": -1.725598692893982,
+      "logps/rejected": -2.485387086868286,
+      "loss": 0.6314,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.725598692893982,
+      "rewards/margins": 0.7597883939743042,
+      "rewards/rejected": -2.485387086868286,
       "step": 4960
     },
     {
       "epoch": 2.657300551931761,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.621471099023893,
       "learning_rate": 3.9094983318019584e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3824637830257416,
+      "logits/rejected": -0.27842313051223755,
+      "logps/chosen": -1.5657049417495728,
+      "logps/rejected": -2.501459836959839,
+      "loss": 0.6044,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.5657049417495728,
+      "rewards/margins": 0.9357550740242004,
+      "rewards/rejected": -2.501459836959839,
       "step": 4965
     },
     {
       "epoch": 2.6599765847131627,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.34283551316149,
       "learning_rate": 3.849350557809789e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2758468687534332,
+      "logits/rejected": -0.2027125358581543,
+      "logps/chosen": -1.68268620967865,
+      "logps/rejected": -2.4562220573425293,
+      "loss": 0.6301,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.68268620967865,
+      "rewards/margins": 0.7735356092453003,
+      "rewards/rejected": -2.4562220573425293,
       "step": 4970
     },
     {
       "epoch": 2.6626526174945644,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3060589605353545,
       "learning_rate": 3.789650537620903e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3271946609020233,
+      "logits/rejected": -0.27639108896255493,
+      "logps/chosen": -1.8608283996582031,
+      "logps/rejected": -2.565107822418213,
+      "loss": 0.6426,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.8608283996582031,
+      "rewards/margins": 0.7042794227600098,
+      "rewards/rejected": -2.565107822418213,
       "step": 4975
     },
     {
       "epoch": 2.665328650275966,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.832514979396592,
       "learning_rate": 3.730398850445182e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2205376923084259,
+      "logits/rejected": -0.16105088591575623,
+      "logps/chosen": -1.9094688892364502,
+      "logps/rejected": -2.669527530670166,
+      "loss": 0.6562,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.9094688892364502,
+      "rewards/margins": 0.7600587606430054,
+      "rewards/rejected": -2.669527530670166,
       "step": 4980
     },
     {
       "epoch": 2.6680046830573674,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.7656383505843465,
       "learning_rate": 3.671596071142735e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2904699444770813,
+      "logits/rejected": -0.1263243556022644,
+      "logps/chosen": -1.6709270477294922,
+      "logps/rejected": -2.7014896869659424,
+      "loss": 0.6208,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6709270477294922,
+      "rewards/margins": 1.0305626392364502,
+      "rewards/rejected": -2.7014896869659424,
       "step": 4985
     },
     {
       "epoch": 2.670680715838769,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.502724017456988,
       "learning_rate": 3.6132427702183996e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3974912762641907,
+      "logits/rejected": -0.20451974868774414,
+      "logps/chosen": -1.7071950435638428,
+      "logps/rejected": -2.621973752975464,
+      "loss": 0.6263,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7071950435638428,
+      "rewards/margins": 0.9147785902023315,
+      "rewards/rejected": -2.621973752975464,
       "step": 4990
     },
     {
       "epoch": 2.6733567486201704,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6978750683672867,
       "learning_rate": 3.555339513816147e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3854199945926666,
+      "logits/rejected": -0.36574098467826843,
+      "logps/chosen": -1.793611764907837,
+      "logps/rejected": -2.4520514011383057,
+      "loss": 0.6475,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.793611764907837,
+      "rewards/margins": 0.6584394574165344,
+      "rewards/rejected": -2.4520514011383057,
       "step": 4995
     },
     {
       "epoch": 2.676032781401572,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.9896703079331335,
       "learning_rate": 3.497886863713639e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3461175560951233,
+      "logits/rejected": -0.3211877942085266,
+      "logps/chosen": -1.8304531574249268,
+      "logps/rejected": -2.606536626815796,
+      "loss": 0.6347,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8304531574249268,
+      "rewards/margins": 0.7760835289955139,
+      "rewards/rejected": -2.606536626815796,
       "step": 5000
     },
     {
       "epoch": 2.678708814182974,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.9183097447794735,
       "learning_rate": 3.440885377316721e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2895556390285492,
+      "logits/rejected": -0.2541015148162842,
+      "logps/chosen": -1.763859510421753,
+      "logps/rejected": -2.458897113800049,
+      "loss": 0.6388,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.763859510421753,
+      "rewards/margins": 0.6950375437736511,
+      "rewards/rejected": -2.458897113800049,
       "step": 5005
     },
     {
       "epoch": 2.6813848469643755,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.0311759120713315,
       "learning_rate": 3.384335607654082e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26786476373672485,
+      "logits/rejected": -0.17571647465229034,
+      "logps/chosen": -2.0086464881896973,
+      "logps/rejected": -2.774988889694214,
+      "loss": 0.6516,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.0086464881896973,
+      "rewards/margins": 0.7663424611091614,
+      "rewards/rejected": -2.774988889694214,
       "step": 5010
     },
     {
       "epoch": 2.684060879745777,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.309869886655922,
       "learning_rate": 3.328238103371811e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36966007947921753,
+      "logits/rejected": -0.3122475743293762,
+      "logps/chosen": -1.8556442260742188,
+      "logps/rejected": -2.631962537765503,
+      "loss": 0.628,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.8556442260742188,
+      "rewards/margins": 0.776318371295929,
+      "rewards/rejected": -2.631962537765503,
       "step": 5015
     },
     {
       "epoch": 2.6867369125271785,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8767575765217077,
       "learning_rate": 3.272593408728169e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3966383934020996,
+      "logits/rejected": -0.20989497005939484,
+      "logps/chosen": -1.7038034200668335,
+      "logps/rejected": -2.4656903743743896,
+      "loss": 0.6472,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7038034200668335,
+      "rewards/margins": 0.7618870139122009,
+      "rewards/rejected": -2.4656903743743896,
       "step": 5020
     },
     {
       "epoch": 2.6894129453085798,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7318508803226482,
       "learning_rate": 3.217402063588204e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.38230687379837036,
+      "logits/rejected": -0.25896474719047546,
+      "logps/chosen": -1.7870826721191406,
+      "logps/rejected": -2.7815194129943848,
+      "loss": 0.6472,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7870826721191406,
+      "rewards/margins": 0.9944368600845337,
+      "rewards/rejected": -2.7815194129943848,
       "step": 5025
     },
     {
       "epoch": 2.6920889780899815,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.628511850554065,
       "learning_rate": 3.162664603418608e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34512820839881897,
+      "logits/rejected": -0.30146753787994385,
+      "logps/chosen": -1.7304741144180298,
+      "logps/rejected": -2.740088701248169,
+      "loss": 0.6352,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7304741144180298,
+      "rewards/margins": 1.0096145868301392,
+      "rewards/rejected": -2.740088701248169,
       "step": 5030
     },
     {
       "epoch": 2.694765010871383,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.298711840422115,
       "learning_rate": 3.1083815592824416e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34689611196517944,
+      "logits/rejected": -0.23589608073234558,
+      "logps/chosen": -1.898553490638733,
+      "logps/rejected": -2.640455961227417,
+      "loss": 0.6419,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.898553490638733,
+      "rewards/margins": 0.7419024109840393,
+      "rewards/rejected": -2.640455961227417,
       "step": 5035
     },
     {
       "epoch": 2.697441043652785,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.311785519449887,
       "learning_rate": 3.054553457834053e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.16329729557037354,
+      "logits/rejected": -0.20507881045341492,
+      "logps/chosen": -1.7704591751098633,
+      "logps/rejected": -2.611961841583252,
+      "loss": 0.6247,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7704591751098633,
+      "rewards/margins": 0.8415029644966125,
+      "rewards/rejected": -2.611961841583252,
       "step": 5040
     },
     {
       "epoch": 2.700117076434186,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7825475756796214,
       "learning_rate": 3.0011808213139036e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2768876552581787,
+      "logits/rejected": -0.2575112283229828,
+      "logps/chosen": -1.8032324314117432,
+      "logps/rejected": -2.435812473297119,
+      "loss": 0.6279,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8032324314117432,
+      "rewards/margins": 0.6325801014900208,
+      "rewards/rejected": -2.435812473297119,
       "step": 5045
     },
     {
       "epoch": 2.702793109215588,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.792253110964795,
       "learning_rate": 2.948264167543568e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33796292543411255,
+      "logits/rejected": -0.26896563172340393,
+      "logps/chosen": -1.5676472187042236,
+      "logps/rejected": -2.3442723751068115,
+      "loss": 0.6127,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.5676472187042236,
+      "rewards/margins": 0.7766249775886536,
+      "rewards/rejected": -2.3442723751068115,
       "step": 5050
     },
     {
       "epoch": 2.7054691419969896,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.9634271942738835,
       "learning_rate": 2.8958040099206216e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4128356873989105,
+      "logits/rejected": -0.33461785316467285,
+      "logps/chosen": -1.6708858013153076,
+      "logps/rejected": -2.485339403152466,
+      "loss": 0.6226,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.6708858013153076,
+      "rewards/margins": 0.8144532442092896,
+      "rewards/rejected": -2.485339403152466,
       "step": 5055
     },
     {
       "epoch": 2.708145174778391,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.810898663277476,
       "learning_rate": 2.843800857413775e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3056473433971405,
+      "logits/rejected": -0.24220359325408936,
+      "logps/chosen": -1.7950751781463623,
+      "logps/rejected": -2.4210362434387207,
+      "loss": 0.6189,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7950751781463623,
+      "rewards/margins": 0.6259608268737793,
+      "rewards/rejected": -2.4210362434387207,
       "step": 5060
     },
     {
       "epoch": 2.7108212075597926,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.471893212977959,
       "learning_rate": 2.7922552145578203e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3553815484046936,
+      "logits/rejected": -0.13700377941131592,
+      "logps/chosen": -1.7302262783050537,
+      "logps/rejected": -2.5278847217559814,
+      "loss": 0.6253,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7302262783050537,
+      "rewards/margins": 0.7976580858230591,
+      "rewards/rejected": -2.5278847217559814,
       "step": 5065
     },
     {
       "epoch": 2.7134972403411943,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.802481417446605,
       "learning_rate": 2.7411675814488277e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24825115501880646,
+      "logits/rejected": -0.11393527686595917,
+      "logps/chosen": -1.6740573644638062,
+      "logps/rejected": -2.260146379470825,
+      "loss": 0.6389,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6740573644638062,
+      "rewards/margins": 0.5860888361930847,
+      "rewards/rejected": -2.260146379470825,
       "step": 5070
     },
     {
       "epoch": 2.7161732731225956,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2360440737603167,
       "learning_rate": 2.690538453739216e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.329376757144928,
+      "logits/rejected": -0.2595978379249573,
+      "logps/chosen": -1.7060918807983398,
+      "logps/rejected": -2.2983341217041016,
+      "loss": 0.6422,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.7060918807983398,
+      "rewards/margins": 0.5922420620918274,
+      "rewards/rejected": -2.2983341217041016,
       "step": 5075
     },
     {
       "epoch": 2.7188493059039973,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.955027894799977,
       "learning_rate": 2.6403683226330298e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3585333824157715,
+      "logits/rejected": -0.23486480116844177,
+      "logps/chosen": -1.752667784690857,
+      "logps/rejected": -2.538553476333618,
+      "loss": 0.6317,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.752667784690857,
+      "rewards/margins": 0.7858856320381165,
+      "rewards/rejected": -2.538553476333618,
       "step": 5080
     },
     {
       "epoch": 2.721525338685399,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 7.275448916186843,
       "learning_rate": 2.5906576748810804e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4088355004787445,
+      "logits/rejected": -0.3130089044570923,
+      "logps/chosen": -1.586399793624878,
+      "logps/rejected": -2.5133368968963623,
+      "loss": 0.6075,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.586399793624878,
+      "rewards/margins": 0.9269372224807739,
+      "rewards/rejected": -2.5133368968963623,
       "step": 5085
     },
     {
       "epoch": 2.7242013714668003,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.107394698335762,
       "learning_rate": 2.5414069927763016e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.43071407079696655,
+      "logits/rejected": -0.28995826840400696,
+      "logps/chosen": -1.7724056243896484,
+      "logps/rejected": -2.5769448280334473,
+      "loss": 0.6405,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.7724056243896484,
+      "rewards/margins": 0.8045392036437988,
+      "rewards/rejected": -2.5769448280334473,
       "step": 5090
     },
     {
       "epoch": 2.726877404248202,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.475896732568026,
       "learning_rate": 2.4926167541490185e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4667305052280426,
+      "logits/rejected": -0.28526026010513306,
+      "logps/chosen": -1.7191221714019775,
+      "logps/rejected": -2.8218188285827637,
+      "loss": 0.6227,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7191221714019775,
+      "rewards/margins": 1.1026967763900757,
+      "rewards/rejected": -2.8218188285827637,
       "step": 5095
     },
     {
       "epoch": 2.7295534370296037,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.224584824091027,
       "learning_rate": 2.4442874323623574e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27878162264823914,
+      "logits/rejected": -0.14085450768470764,
+      "logps/chosen": -1.8141734600067139,
+      "logps/rejected": -2.8471367359161377,
+      "loss": 0.6277,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8141734600067139,
+      "rewards/margins": 1.0329630374908447,
+      "rewards/rejected": -2.8471367359161377,
       "step": 5100
     },
     {
       "epoch": 2.7322294698110055,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.47832478584508,
       "learning_rate": 2.396419496307589e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3441801071166992,
+      "logits/rejected": -0.20141243934631348,
+      "logps/chosen": -1.7909717559814453,
+      "logps/rejected": -2.6419901847839355,
+      "loss": 0.6309,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7909717559814453,
+      "rewards/margins": 0.8510181307792664,
+      "rewards/rejected": -2.6419901847839355,
       "step": 5105
     },
     {
       "epoch": 2.7349055025924067,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.600085776053535,
       "learning_rate": 2.349013410399653e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.39745816588401794,
+      "logits/rejected": -0.2758747339248657,
+      "logps/chosen": -1.8012301921844482,
+      "logps/rejected": -2.585437774658203,
+      "loss": 0.6235,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.8012301921844482,
+      "rewards/margins": 0.7842077016830444,
+      "rewards/rejected": -2.585437774658203,
       "step": 5110
     },
     {
       "epoch": 2.7375815353738084,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.298936515366596,
       "learning_rate": 2.3020696345725954e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.415400892496109,
+      "logits/rejected": -0.23416920006275177,
+      "logps/chosen": -1.7437431812286377,
+      "logps/rejected": -2.718733072280884,
+      "loss": 0.6256,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7437431812286377,
+      "rewards/margins": 0.9749897122383118,
+      "rewards/rejected": -2.718733072280884,
       "step": 5115
     },
     {
       "epoch": 2.7402575681552097,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.8627520316538675,
       "learning_rate": 2.2555886242751398e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3631313145160675,
+      "logits/rejected": -0.31255537271499634,
+      "logps/chosen": -1.8130548000335693,
+      "logps/rejected": -2.6843059062957764,
+      "loss": 0.6307,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.8130548000335693,
+      "rewards/margins": 0.8712509274482727,
+      "rewards/rejected": -2.6843059062957764,
       "step": 5120
     },
     {
       "epoch": 2.7429336009366114,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.227169782200615,
       "learning_rate": 2.2095708304662453e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.44803112745285034,
+      "logits/rejected": -0.23682916164398193,
+      "logps/chosen": -1.7739585638046265,
+      "logps/rejected": -2.4262607097625732,
+      "loss": 0.6163,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7739585638046265,
+      "rewards/margins": 0.6523019075393677,
+      "rewards/rejected": -2.4262607097625732,
       "step": 5125
     },
     {
       "epoch": 2.745609633718013,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2688997900705727,
       "learning_rate": 2.16401669961076e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4778536260128021,
+      "logits/rejected": -0.3026002049446106,
+      "logps/chosen": -1.6862748861312866,
+      "logps/rejected": -2.659932851791382,
+      "loss": 0.6267,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6862748861312866,
+      "rewards/margins": 0.9736580848693848,
+      "rewards/rejected": -2.659932851791382,
       "step": 5130
     },
     {
       "epoch": 2.748285666499415,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.6045906628439885,
       "learning_rate": 2.1189266736750532e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2955007553100586,
+      "logits/rejected": -0.23071876168251038,
+      "logps/chosen": -1.772390604019165,
+      "logps/rejected": -2.5268969535827637,
+      "loss": 0.632,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.772390604019165,
+      "rewards/margins": 0.7545064687728882,
+      "rewards/rejected": -2.5268969535827637,
       "step": 5135
     },
     {
       "epoch": 2.750961699280816,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.612794177007736,
       "learning_rate": 2.0743011901227623e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3122073709964752,
+      "logits/rejected": -0.19321602582931519,
+      "logps/chosen": -1.8101962804794312,
+      "logps/rejected": -2.4867358207702637,
+      "loss": 0.6498,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8101962804794312,
+      "rewards/margins": 0.6765391826629639,
+      "rewards/rejected": -2.4867358207702637,
       "step": 5140
     },
     {
       "epoch": 2.753637732062218,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.3373232741845635,
       "learning_rate": 2.030140681910508e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3722541928291321,
+      "logits/rejected": -0.21742311120033264,
+      "logps/chosen": -1.6935880184173584,
+      "logps/rejected": -2.4045422077178955,
+      "loss": 0.6383,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6935880184173584,
+      "rewards/margins": 0.7109543085098267,
+      "rewards/rejected": -2.4045422077178955,
       "step": 5145
     },
     {
       "epoch": 2.756313764843619,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.300175970060646,
       "learning_rate": 1.986445577483753e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.40788954496383667,
+      "logits/rejected": -0.29361873865127563,
+      "logps/chosen": -1.7056297063827515,
+      "logps/rejected": -2.4532270431518555,
+      "loss": 0.625,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7056297063827515,
+      "rewards/margins": 0.7475974559783936,
+      "rewards/rejected": -2.4532270431518555,
       "step": 5150
     },
     {
       "epoch": 2.758989797625021,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.654475776953978,
       "learning_rate": 1.9432163007725765e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4219639301300049,
+      "logits/rejected": -0.3254469037055969,
+      "logps/chosen": -1.7963573932647705,
+      "logps/rejected": -2.447082281112671,
+      "loss": 0.6268,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7963573932647705,
+      "rewards/margins": 0.6507248282432556,
+      "rewards/rejected": -2.447082281112671,
       "step": 5155
     },
     {
       "epoch": 2.7616658304064226,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9859831555824488,
       "learning_rate": 1.9004532711876297e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3710784912109375,
+      "logits/rejected": -0.31647300720214844,
+      "logps/chosen": -1.6898996829986572,
+      "logps/rejected": -2.4750778675079346,
+      "loss": 0.5949,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6898996829986572,
+      "rewards/margins": 0.7851781845092773,
+      "rewards/rejected": -2.4750778675079346,
       "step": 5160
     },
     {
       "epoch": 2.7643418631878243,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.48983303380351,
       "learning_rate": 1.8581569036159928e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.35644659399986267,
+      "logits/rejected": -0.19706431031227112,
+      "logps/chosen": -1.6137230396270752,
+      "logps/rejected": -2.573822021484375,
+      "loss": 0.6151,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.6137230396270752,
+      "rewards/margins": 0.9600992202758789,
+      "rewards/rejected": -2.573822021484375,
       "step": 5165
     },
     {
       "epoch": 2.7670178959692255,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.775651879091801,
       "learning_rate": 1.8163276084172285e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34777694940567017,
+      "logits/rejected": -0.21423101425170898,
+      "logps/chosen": -1.789373755455017,
+      "logps/rejected": -2.504390239715576,
+      "loss": 0.6332,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.789373755455017,
+      "rewards/margins": 0.7150167226791382,
+      "rewards/rejected": -2.504390239715576,
       "step": 5170
     },
     {
       "epoch": 2.7696939287506273,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.64159372638769,
       "learning_rate": 1.7749657914193194e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.37370485067367554,
+      "logits/rejected": -0.28204071521759033,
+      "logps/chosen": -1.7653802633285522,
+      "logps/rejected": -2.593999147415161,
+      "loss": 0.6245,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7653802633285522,
+      "rewards/margins": 0.8286188244819641,
+      "rewards/rejected": -2.593999147415161,
       "step": 5175
     },
     {
       "epoch": 2.7723699615320285,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.480782287372028,
       "learning_rate": 1.7340718539148203e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.29181626439094543,
+      "logits/rejected": -0.25367361307144165,
+      "logps/chosen": -1.8553955554962158,
+      "logps/rejected": -2.572511911392212,
+      "loss": 0.6519,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.8553955554962158,
+      "rewards/margins": 0.7171164751052856,
+      "rewards/rejected": -2.572511911392212,
       "step": 5180
     },
     {
       "epoch": 2.7750459943134302,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.959165749123368,
       "learning_rate": 1.6936461926568724e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3373684883117676,
+      "logits/rejected": -0.23044605553150177,
+      "logps/chosen": -1.5884438753128052,
+      "logps/rejected": -2.4817638397216797,
+      "loss": 0.6286,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5884438753128052,
+      "rewards/margins": 0.8933199644088745,
+      "rewards/rejected": -2.4817638397216797,
       "step": 5185
     },
     {
       "epoch": 2.777722027094832,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.5948934098990115,
       "learning_rate": 1.6536891998554346e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.44566091895103455,
+      "logits/rejected": -0.2912386953830719,
+      "logps/chosen": -1.702091932296753,
+      "logps/rejected": -2.479557514190674,
+      "loss": 0.6135,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.702091932296753,
+      "rewards/margins": 0.7774654626846313,
+      "rewards/rejected": -2.479557514190674,
       "step": 5190
     },
     {
       "epoch": 2.7803980598762337,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.543006525967003,
       "learning_rate": 1.6142012631734093e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3275555670261383,
+      "logits/rejected": -0.20165352523326874,
+      "logps/chosen": -1.7679847478866577,
+      "logps/rejected": -2.466597318649292,
+      "loss": 0.6359,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7679847478866577,
+      "rewards/margins": 0.6986128091812134,
+      "rewards/rejected": -2.466597318649292,
       "step": 5195
     },
     {
       "epoch": 2.783074092657635,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.49756670071674,
       "learning_rate": 1.575182765722949e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.40978866815567017,
+      "logits/rejected": -0.26748016476631165,
+      "logps/chosen": -1.7002661228179932,
+      "logps/rejected": -2.566056728363037,
+      "loss": 0.6302,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7002661228179932,
+      "rewards/margins": 0.8657907247543335,
+      "rewards/rejected": -2.566056728363037,
       "step": 5200
     },
     {
       "epoch": 2.783074092657635,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.8408,
-      "eval_samples_per_second": 38.604,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.673,
+      "eval_logits/chosen": -0.041791804134845734,
+      "eval_logits/rejected": 0.03945944085717201,
+      "eval_logps/chosen": -1.7943087816238403,
+      "eval_logps/rejected": -2.5779664516448975,
+      "eval_loss": 0.6507219076156616,
+      "eval_rewards/accuracies": 0.6750741600990295,
+      "eval_rewards/chosen": -1.7943087816238403,
+      "eval_rewards/margins": 0.783657431602478,
+      "eval_rewards/rejected": -2.5779664516448975,
+      "eval_runtime": 40.6252,
+      "eval_samples_per_second": 33.108,
+      "eval_steps_per_second": 8.295,
       "step": 5200
     },
     {
       "epoch": 2.7857501254390367,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8313882656606384,
       "learning_rate": 1.536634086061672e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2882939875125885,
+      "logits/rejected": -0.2685455083847046,
+      "logps/chosen": -1.7234971523284912,
+      "logps/rejected": -2.5011818408966064,
+      "loss": 0.6334,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7234971523284912,
+      "rewards/margins": 0.7776845693588257,
+      "rewards/rejected": -2.5011818408966064,
       "step": 5205
     },
     {
       "epoch": 2.788426158220438,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.562017339829881,
       "learning_rate": 1.4985555981890495e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33882421255111694,
+      "logits/rejected": -0.2494608610868454,
+      "logps/chosen": -1.8340142965316772,
+      "logps/rejected": -2.65067458152771,
+      "loss": 0.6254,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.8340142965316772,
+      "rewards/margins": 0.8166602253913879,
+      "rewards/rejected": -2.65067458152771,
       "step": 5210
     },
     {
       "epoch": 2.7911021910018396,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2515452121348747,
       "learning_rate": 1.4609476715427226e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33114928007125854,
+      "logits/rejected": -0.2444652020931244,
+      "logps/chosen": -1.6127078533172607,
+      "logps/rejected": -2.581470251083374,
+      "loss": 0.6234,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.6127078533172607,
+      "rewards/margins": 0.9687622785568237,
+      "rewards/rejected": -2.581470251083374,
       "step": 5215
     },
     {
       "epoch": 2.7937782237832414,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.643595750458565,
       "learning_rate": 1.4238106709949792e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36710065603256226,
+      "logits/rejected": -0.30055347084999084,
+      "logps/chosen": -1.7153812646865845,
+      "logps/rejected": -2.6366658210754395,
+      "loss": 0.6103,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7153812646865845,
+      "rewards/margins": 0.921284556388855,
+      "rewards/rejected": -2.6366658210754395,
       "step": 5220
     },
     {
       "epoch": 2.796454256564643,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.796857519888347,
       "learning_rate": 1.3871449568491511e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31041473150253296,
+      "logits/rejected": -0.1830393373966217,
+      "logps/chosen": -1.826067566871643,
+      "logps/rejected": -2.6534905433654785,
+      "loss": 0.6393,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.826067566871643,
+      "rewards/margins": 0.827423095703125,
+      "rewards/rejected": -2.6534905433654785,
       "step": 5225
     },
     {
       "epoch": 2.7991302893460444,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.895548684924735,
       "learning_rate": 1.3509508848361606e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.45356059074401855,
+      "logits/rejected": -0.31752488017082214,
+      "logps/chosen": -1.7767022848129272,
+      "logps/rejected": -2.5571343898773193,
+      "loss": 0.6347,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7767022848129272,
+      "rewards/margins": 0.7804322242736816,
+      "rewards/rejected": -2.5571343898773193,
       "step": 5230
     },
     {
       "epoch": 2.801806322127446,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7897911651507616,
       "learning_rate": 1.3152288061110517e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4101329445838928,
+      "logits/rejected": -0.2947995066642761,
+      "logps/chosen": -1.6992151737213135,
+      "logps/rejected": -2.5177340507507324,
+      "loss": 0.6396,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6992151737213135,
+      "rewards/margins": 0.8185189962387085,
+      "rewards/rejected": -2.5177340507507324,
       "step": 5235
     },
     {
       "epoch": 2.804482354908848,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.200314707324688,
       "learning_rate": 1.2799790672495814e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.37742313742637634,
+      "logits/rejected": -0.1787382811307907,
+      "logps/chosen": -1.7562735080718994,
+      "logps/rejected": -2.650864362716675,
+      "loss": 0.6321,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7562735080718994,
+      "rewards/margins": 0.8945909738540649,
+      "rewards/rejected": -2.650864362716675,
       "step": 5240
     },
     {
       "epoch": 2.807158387690249,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.645305389040303,
       "learning_rate": 1.2452020102448835e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30662959814071655,
+      "logits/rejected": -0.2591291069984436,
+      "logps/chosen": -1.7684919834136963,
+      "logps/rejected": -2.3618083000183105,
+      "loss": 0.649,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7684919834136963,
+      "rewards/margins": 0.5933160781860352,
+      "rewards/rejected": -2.3618083000183105,
       "step": 5245
     },
     {
       "epoch": 2.8098344204716508,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.5325122563344284,
       "learning_rate": 1.2108979725041103e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.392932653427124,
+      "logits/rejected": -0.261179119348526,
+      "logps/chosen": -1.7438499927520752,
+      "logps/rejected": -2.506575345993042,
+      "loss": 0.6424,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7438499927520752,
+      "rewards/margins": 0.7627254724502563,
+      "rewards/rejected": -2.506575345993042,
       "step": 5250
     },
     {
       "epoch": 2.8125104532530525,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.161108759607861,
       "learning_rate": 1.1770672868451958e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3756434917449951,
+      "logits/rejected": -0.18598613142967224,
+      "logps/chosen": -1.7382304668426514,
+      "logps/rejected": -2.5941786766052246,
+      "loss": 0.631,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7382304668426514,
+      "rewards/margins": 0.8559479713439941,
+      "rewards/rejected": -2.5941786766052246,
       "step": 5255
     },
     {
       "epoch": 2.8151864860344538,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.8304987782352455,
       "learning_rate": 1.1437102814935872e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36808833479881287,
+      "logits/rejected": -0.3110390305519104,
+      "logps/chosen": -1.727137804031372,
+      "logps/rejected": -2.658583641052246,
+      "loss": 0.6165,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.727137804031372,
+      "rewards/margins": 0.9314458966255188,
+      "rewards/rejected": -2.658583641052246,
       "step": 5260
     },
     {
       "epoch": 2.8178625188158555,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.256486249943266,
       "learning_rate": 1.1108272800791018e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.45331114530563354,
+      "logits/rejected": -0.2555120587348938,
+      "logps/chosen": -1.9510552883148193,
+      "logps/rejected": -2.7148139476776123,
+      "loss": 0.6468,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.9510552883148193,
+      "rewards/margins": 0.763758659362793,
+      "rewards/rejected": -2.7148139476776123,
       "step": 5265
     },
     {
       "epoch": 2.820538551597257,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.353402464802818,
       "learning_rate": 1.078418601632769e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33222150802612305,
+      "logits/rejected": -0.1920807957649231,
+      "logps/chosen": -1.7499364614486694,
+      "logps/rejected": -2.5476667881011963,
+      "loss": 0.6231,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7499364614486694,
+      "rewards/margins": 0.7977302670478821,
+      "rewards/rejected": -2.5476667881011963,
       "step": 5270
     },
     {
       "epoch": 2.8232145843786585,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.82461810832671,
       "learning_rate": 1.0464845605837159e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3358948826789856,
+      "logits/rejected": -0.19161807000637054,
+      "logps/chosen": -1.7982944250106812,
+      "logps/rejected": -2.524439811706543,
+      "loss": 0.6357,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.7982944250106812,
+      "rewards/margins": 0.726145327091217,
+      "rewards/rejected": -2.524439811706543,
       "step": 5275
     },
     {
       "epoch": 2.82589061716006,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.146150102714037,
       "learning_rate": 1.0150254667561642e-08,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3473220467567444,
+      "logits/rejected": -0.20116400718688965,
+      "logps/chosen": -1.8950965404510498,
+      "logps/rejected": -2.8292934894561768,
+      "loss": 0.6393,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.8950965404510498,
+      "rewards/margins": 0.9341967701911926,
+      "rewards/rejected": -2.8292934894561768,
       "step": 5280
     },
     {
       "epoch": 2.828566649941462,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.6759755748077003,
       "learning_rate": 9.840416253663719e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3813789486885071,
+      "logits/rejected": -0.28502100706100464,
+      "logps/chosen": -1.6197478771209717,
+      "logps/rejected": -2.7194807529449463,
+      "loss": 0.6165,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.6197478771209717,
+      "rewards/margins": 1.0997326374053955,
+      "rewards/rejected": -2.7194807529449463,
       "step": 5285
     },
     {
       "epoch": 2.8312426827228636,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.293274971885987,
       "learning_rate": 9.535333370197074e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.35029035806655884,
+      "logits/rejected": -0.21218523383140564,
+      "logps/chosen": -1.8153654336929321,
+      "logps/rejected": -2.3945424556732178,
+      "loss": 0.6436,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.8153654336929321,
+      "rewards/margins": 0.5791771411895752,
+      "rewards/rejected": -2.3945424556732178,
       "step": 5290
     },
     {
       "epoch": 2.833918715504265,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2484986750655156,
       "learning_rate": 9.23500897707713e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4110341966152191,
+      "logits/rejected": -0.24050378799438477,
+      "logps/chosen": -1.8464577198028564,
+      "logps/rejected": -2.6592040061950684,
+      "loss": 0.6464,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.8464577198028564,
+      "rewards/margins": 0.8127459287643433,
+      "rewards/rejected": -2.6592040061950684,
       "step": 5295
     },
     {
       "epoch": 2.8365947482856666,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.554761313458533,
       "learning_rate": 8.939445988052574e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3608815670013428,
+      "logits/rejected": -0.3224570155143738,
+      "logps/chosen": -1.8052898645401,
+      "logps/rejected": -2.75120210647583,
+      "loss": 0.6242,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.8052898645401,
+      "rewards/margins": 0.94591224193573,
+      "rewards/rejected": -2.75120210647583,
       "step": 5300
     },
     {
       "epoch": 2.839270781067068,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.714461004484157,
       "learning_rate": 8.648647270676656e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34113526344299316,
+      "logits/rejected": -0.2237265557050705,
+      "logps/chosen": -1.7804982662200928,
+      "logps/rejected": -2.6945321559906006,
+      "loss": 0.6353,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.7804982662200928,
+      "rewards/margins": 0.9140337109565735,
+      "rewards/rejected": -2.6945321559906006,
       "step": 5305
     },
     {
       "epoch": 2.8419468138484696,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7896943223659934,
       "learning_rate": 8.362615646279991e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.47828373312950134,
+      "logits/rejected": -0.2530137002468109,
+      "logps/chosen": -1.7363154888153076,
+      "logps/rejected": -2.79663348197937,
+      "loss": 0.6202,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.7363154888153076,
+      "rewards/margins": 1.0603179931640625,
+      "rewards/rejected": -2.79663348197937,
       "step": 5310
     },
     {
       "epoch": 2.8446228466298713,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.277350691726261,
       "learning_rate": 8.081353889942466e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25181278586387634,
+      "logits/rejected": -0.1438337117433548,
+      "logps/chosen": -1.7717689275741577,
+      "logps/rejected": -2.5062098503112793,
+      "loss": 0.642,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7717689275741577,
+      "rewards/margins": 0.734440803527832,
+      "rewards/rejected": -2.5062098503112793,
       "step": 5315
     },
     {
       "epoch": 2.847298879411273,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.725172895237093,
       "learning_rate": 7.804864730467042e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2721318304538727,
+      "logits/rejected": -0.1980196237564087,
+      "logps/chosen": -1.774560570716858,
+      "logps/rejected": -2.3867430686950684,
+      "loss": 0.629,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.774560570716858,
+      "rewards/margins": 0.6121825575828552,
+      "rewards/rejected": -2.3867430686950684,
       "step": 5320
     },
     {
       "epoch": 2.8499749121926743,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.1482462896993693,
       "learning_rate": 7.533150850352665e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3267989754676819,
+      "logits/rejected": -0.1930588334798813,
+      "logps/chosen": -1.8621095418930054,
+      "logps/rejected": -2.8053512573242188,
+      "loss": 0.6298,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.8621095418930054,
+      "rewards/margins": 0.9432415962219238,
+      "rewards/rejected": -2.8053512573242188,
       "step": 5325
     },
     {
       "epoch": 2.852650944974076,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.02188822652614,
       "learning_rate": 7.2662148857686175e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2953232526779175,
+      "logits/rejected": -0.2269563227891922,
+      "logps/chosen": -1.8037335872650146,
+      "logps/rejected": -2.6073575019836426,
+      "loss": 0.6293,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8037335872650146,
+      "rewards/margins": 0.8036235570907593,
+      "rewards/rejected": -2.6073575019836426,
       "step": 5330
     },
     {
       "epoch": 2.8553269777554773,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.352592239513481,
       "learning_rate": 7.0040594265287635e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2404893934726715,
+      "logits/rejected": -0.26916104555130005,
+      "logps/chosen": -1.7544193267822266,
+      "logps/rejected": -2.3342835903167725,
+      "loss": 0.642,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7544193267822266,
+      "rewards/margins": 0.5798644423484802,
+      "rewards/rejected": -2.3342835903167725,
       "step": 5335
     },
     {
       "epoch": 2.858003010536879,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.338266669310113,
       "learning_rate": 6.746687016066566e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.307939738035202,
+      "logits/rejected": -0.2715227007865906,
+      "logps/chosen": -1.7190067768096924,
+      "logps/rejected": -2.463813066482544,
+      "loss": 0.6358,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7190067768096924,
+      "rewards/margins": 0.7448063492774963,
+      "rewards/rejected": -2.463813066482544,
       "step": 5340
     },
     {
       "epoch": 2.8606790433182807,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.216340778211539,
       "learning_rate": 6.494100151410276e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4319778382778168,
+      "logits/rejected": -0.2713262736797333,
+      "logps/chosen": -1.6416237354278564,
+      "logps/rejected": -2.529057741165161,
+      "loss": 0.6149,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.6416237354278564,
+      "rewards/margins": 0.8874341249465942,
+      "rewards/rejected": -2.529057741165161,
       "step": 5345
     },
     {
       "epoch": 2.8633550760996824,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.623455369596267,
       "learning_rate": 6.246301283158728e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25278955698013306,
+      "logits/rejected": -0.2800353765487671,
+      "logps/chosen": -1.8022266626358032,
+      "logps/rejected": -2.5044379234313965,
+      "loss": 0.6458,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8022266626358032,
+      "rewards/margins": 0.7022111415863037,
+      "rewards/rejected": -2.5044379234313965,
       "step": 5350
     },
     {
       "epoch": 2.8660311088810837,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7126552644509037,
       "learning_rate": 6.0032928154576944e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3647855222225189,
+      "logits/rejected": -0.28643468022346497,
+      "logps/chosen": -1.8026946783065796,
+      "logps/rejected": -2.5301718711853027,
+      "loss": 0.6501,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8026946783065796,
+      "rewards/margins": 0.7274774312973022,
+      "rewards/rejected": -2.5301718711853027,
       "step": 5355
     },
     {
       "epoch": 2.8687071416624854,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 9.126092194882341,
       "learning_rate": 5.76507710597629e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36469781398773193,
+      "logits/rejected": -0.19208720326423645,
+      "logps/chosen": -1.7601211071014404,
+      "logps/rejected": -2.574883222579956,
+      "loss": 0.626,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7601211071014404,
+      "rewards/margins": 0.8147621154785156,
+      "rewards/rejected": -2.574883222579956,
       "step": 5360
     },
     {
       "epoch": 2.8713831744438867,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.140324811947417,
       "learning_rate": 5.531656465884438e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4306454658508301,
+      "logits/rejected": -0.2866804003715515,
+      "logps/chosen": -1.8062137365341187,
+      "logps/rejected": -2.776937961578369,
+      "loss": 0.6315,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.8062137365341187,
+      "rewards/margins": 0.9707242846488953,
+      "rewards/rejected": -2.776937961578369,
       "step": 5365
     },
     {
       "epoch": 2.8740592072252884,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.702007510925298,
       "learning_rate": 5.303033159830217e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.2750771939754486,
+      "logits/rejected": -0.2480771541595459,
+      "logps/chosen": -1.8212366104125977,
+      "logps/rejected": -2.2720844745635986,
+      "loss": 0.6578,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.8212366104125977,
+      "rewards/margins": 0.4508478045463562,
+      "rewards/rejected": -2.2720844745635986,
       "step": 5370
     },
     {
       "epoch": 2.87673524000669,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.056523099584181,
       "learning_rate": 5.079209405917939e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3439173996448517,
+      "logits/rejected": -0.2712896764278412,
+      "logps/chosen": -1.6743265390396118,
+      "logps/rejected": -2.828094482421875,
+      "loss": 0.6213,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6743265390396118,
+      "rewards/margins": 1.1537678241729736,
+      "rewards/rejected": -2.828094482421875,
       "step": 5375
     },
     {
       "epoch": 2.879411272788092,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.540928425680103,
       "learning_rate": 4.860187375686664e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3923974633216858,
+      "logits/rejected": -0.1843414306640625,
+      "logps/chosen": -1.8921798467636108,
+      "logps/rejected": -2.8171045780181885,
+      "loss": 0.6302,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.8921798467636108,
+      "rewards/margins": 0.9249242544174194,
+      "rewards/rejected": -2.8171045780181885,
       "step": 5380
     },
     {
       "epoch": 2.882087305569493,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.1332579080089302,
       "learning_rate": 4.64596919408905e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3206592798233032,
+      "logits/rejected": -0.24186882376670837,
+      "logps/chosen": -1.7370660305023193,
+      "logps/rejected": -2.3389930725097656,
+      "loss": 0.6296,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7370660305023193,
+      "rewards/margins": 0.6019269227981567,
+      "rewards/rejected": -2.3389930725097656,
       "step": 5385
     },
     {
       "epoch": 2.884763338350895,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.593226667609051,
       "learning_rate": 4.436556939470814e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33029425144195557,
+      "logits/rejected": -0.21789279580116272,
+      "logps/chosen": -1.8678191900253296,
+      "logps/rejected": -2.526454210281372,
+      "loss": 0.6295,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8678191900253296,
+      "rewards/margins": 0.6586349606513977,
+      "rewards/rejected": -2.526454210281372,
       "step": 5390
     },
     {
       "epoch": 2.887439371132296,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.4409820062206378,
       "learning_rate": 4.23195264355064e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.482280969619751,
+      "logits/rejected": -0.30474457144737244,
+      "logps/chosen": -1.614304542541504,
+      "logps/rejected": -2.5113718509674072,
+      "loss": 0.6067,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.614304542541504,
+      "rewards/margins": 0.8970670700073242,
+      "rewards/rejected": -2.5113718509674072,
       "step": 5395
     },
     {
       "epoch": 2.890115403913698,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.246006739062337,
       "learning_rate": 4.032158291400245e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4062774181365967,
+      "logits/rejected": -0.19060960412025452,
+      "logps/chosen": -1.6950193643569946,
+      "logps/rejected": -2.932941436767578,
+      "loss": 0.6214,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.6950193643569946,
+      "rewards/margins": 1.237922191619873,
+      "rewards/rejected": -2.932941436767578,
       "step": 5400
     },
     {
       "epoch": 2.8927914366950995,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.5546208945096867,
       "learning_rate": 3.837175821425398e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30864930152893066,
+      "logits/rejected": -0.2632136940956116,
+      "logps/chosen": -2.0209739208221436,
+      "logps/rejected": -2.759159564971924,
+      "loss": 0.6399,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -2.0209739208221436,
+      "rewards/margins": 0.738185465335846,
+      "rewards/rejected": -2.759159564971924,
       "step": 5405
     },
     {
       "epoch": 2.8954674694765012,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.8775638360494074,
       "learning_rate": 3.6470071253467683e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30921876430511475,
+      "logits/rejected": -0.20783379673957825,
+      "logps/chosen": -1.7458614110946655,
+      "logps/rejected": -2.7438597679138184,
+      "loss": 0.6216,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7458614110946655,
+      "rewards/margins": 0.9979981184005737,
+      "rewards/rejected": -2.7438597679138184,
       "step": 5410
     },
     {
       "epoch": 2.8981435022579025,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.2479535503175287,
       "learning_rate": 3.461654048181939e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3934227526187897,
+      "logits/rejected": -0.2371637523174286,
+      "logps/chosen": -1.8393023014068604,
+      "logps/rejected": -2.5738048553466797,
+      "loss": 0.6394,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8393023014068604,
+      "rewards/margins": 0.7345024347305298,
+      "rewards/rejected": -2.5738048553466797,
       "step": 5415
     },
     {
       "epoch": 2.9008195350393042,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.3320285387435864,
       "learning_rate": 3.281118388227255e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.31701141595840454,
+      "logits/rejected": -0.24698016047477722,
+      "logps/chosen": -1.7885010242462158,
+      "logps/rejected": -2.455535411834717,
+      "loss": 0.6331,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.7885010242462158,
+      "rewards/margins": 0.6670345067977905,
+      "rewards/rejected": -2.455535411834717,
       "step": 5420
     },
     {
       "epoch": 2.903495567820706,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.443371596046379,
       "learning_rate": 3.1054018970405048e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.32353657484054565,
+      "logits/rejected": -0.21091696619987488,
+      "logps/chosen": -1.752562165260315,
+      "logps/rejected": -2.616198778152466,
+      "loss": 0.6283,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.752562165260315,
+      "rewards/margins": 0.8636366128921509,
+      "rewards/rejected": -2.616198778152466,
       "step": 5425
     },
     {
       "epoch": 2.906171600602107,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.274201572160154,
       "learning_rate": 2.9345062794238207e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.35556572675704956,
+      "logits/rejected": -0.19608236849308014,
+      "logps/chosen": -1.7497460842132568,
+      "logps/rejected": -2.6198620796203613,
+      "loss": 0.6187,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.7497460842132568,
+      "rewards/margins": 0.8701159358024597,
+      "rewards/rejected": -2.6198620796203613,
       "step": 5430
     },
     {
       "epoch": 2.908847633383509,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.549628084416386,
       "learning_rate": 2.7684331934072492e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.45467814803123474,
+      "logits/rejected": -0.3597552180290222,
+      "logps/chosen": -1.7424747943878174,
+      "logps/rejected": -2.6013827323913574,
+      "loss": 0.6268,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.7424747943878174,
+      "rewards/margins": 0.8589082956314087,
+      "rewards/rejected": -2.6013827323913574,
       "step": 5435
     },
     {
       "epoch": 2.9115236661649107,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.945475098520051,
       "learning_rate": 2.6071842502326526e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3801659345626831,
+      "logits/rejected": -0.2760412395000458,
+      "logps/chosen": -1.7551090717315674,
+      "logps/rejected": -2.4480719566345215,
+      "loss": 0.6313,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7551090717315674,
+      "rewards/margins": 0.6929631233215332,
+      "rewards/rejected": -2.4480719566345215,
       "step": 5440
     },
     {
       "epoch": 2.9141996989463124,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.717282843214742,
       "learning_rate": 2.450761014337888e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.15567722916603088,
+      "logits/rejected": -0.13922347128391266,
+      "logps/chosen": -1.8637468814849854,
+      "logps/rejected": -2.8638086318969727,
+      "loss": 0.6278,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.8637468814849854,
+      "rewards/margins": 1.0000616312026978,
+      "rewards/rejected": -2.8638086318969727,
       "step": 5445
     },
     {
       "epoch": 2.9168757317277136,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.270103313283751,
       "learning_rate": 2.299165003341985e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.24119620025157928,
+      "logits/rejected": -0.16210582852363586,
+      "logps/chosen": -1.7539758682250977,
+      "logps/rejected": -2.767542600631714,
+      "loss": 0.6347,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7539758682250977,
+      "rewards/margins": 1.0135668516159058,
+      "rewards/rejected": -2.767542600631714,
       "step": 5450
     },
     {
       "epoch": 2.9195517645091154,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.961711639946487,
       "learning_rate": 2.1523976880299945e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.37438976764678955,
+      "logits/rejected": -0.20710515975952148,
+      "logps/chosen": -1.803260087966919,
+      "logps/rejected": -2.5155234336853027,
+      "loss": 0.6493,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.803260087966919,
+      "rewards/margins": 0.7122634053230286,
+      "rewards/rejected": -2.5155234336853027,
       "step": 5455
     },
     {
       "epoch": 2.9222277972905166,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.579442951816846,
       "learning_rate": 2.010460492339161e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3413736820220947,
+      "logits/rejected": -0.2547042667865753,
+      "logps/chosen": -1.7281558513641357,
+      "logps/rejected": -2.7046637535095215,
+      "loss": 0.6202,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.7281558513641357,
+      "rewards/margins": 0.9765083193778992,
+      "rewards/rejected": -2.7046637535095215,
       "step": 5460
     },
     {
       "epoch": 2.9249038300719183,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.346646127871287,
       "learning_rate": 1.8733547933446614e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4199215769767761,
+      "logits/rejected": -0.2461710423231125,
+      "logps/chosen": -1.8513883352279663,
+      "logps/rejected": -2.6251227855682373,
+      "loss": 0.6552,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8513883352279663,
+      "rewards/margins": 0.7737346291542053,
+      "rewards/rejected": -2.6251227855682373,
       "step": 5465
     },
     {
       "epoch": 2.92757986285332,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.960926128229745,
       "learning_rate": 1.7410819212467231e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33259207010269165,
+      "logits/rejected": -0.23995885252952576,
+      "logps/chosen": -1.7798922061920166,
+      "logps/rejected": -2.5126891136169434,
+      "loss": 0.6385,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7798922061920166,
+      "rewards/margins": 0.732796847820282,
+      "rewards/rejected": -2.5126891136169434,
       "step": 5470
     },
     {
       "epoch": 2.9302558956347218,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.7514847974880556,
       "learning_rate": 1.613643159357192e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.28291112184524536,
+      "logits/rejected": -0.3352871537208557,
+      "logps/chosen": -1.7729923725128174,
+      "logps/rejected": -2.5206875801086426,
+      "loss": 0.6219,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7729923725128174,
+      "rewards/margins": 0.7476950883865356,
+      "rewards/rejected": -2.5206875801086426,
       "step": 5475
     },
     {
       "epoch": 2.932931928416123,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.8219758398255514,
       "learning_rate": 1.4910397440875967e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.330008864402771,
+      "logits/rejected": -0.2320789396762848,
+      "logps/chosen": -1.7890640497207642,
+      "logps/rejected": -2.4816558361053467,
+      "loss": 0.6357,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7890640497207642,
+      "rewards/margins": 0.6925919651985168,
+      "rewards/rejected": -2.4816558361053467,
       "step": 5480
     },
     {
       "epoch": 2.9356079611975248,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.06814612696574,
       "learning_rate": 1.3732728649368253e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.26871228218078613,
+      "logits/rejected": -0.11516822874546051,
+      "logps/chosen": -1.6910518407821655,
+      "logps/rejected": -2.56524658203125,
+      "loss": 0.6259,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.6910518407821655,
+      "rewards/margins": 0.8741944432258606,
+      "rewards/rejected": -2.56524658203125,
       "step": 5485
     },
     {
       "epoch": 2.938283993978926,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.431630569035244,
       "learning_rate": 1.260343664479524e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34523382782936096,
+      "logits/rejected": -0.3177355229854584,
+      "logps/chosen": -1.7829593420028687,
+      "logps/rejected": -2.4955010414123535,
+      "loss": 0.6364,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7829593420028687,
+      "rewards/margins": 0.7125414609909058,
+      "rewards/rejected": -2.4955010414123535,
       "step": 5490
     },
     {
       "epoch": 2.9409600267603278,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.830237277398031,
       "learning_rate": 1.1522532383554384e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.42663684487342834,
+      "logits/rejected": -0.25047892332077026,
+      "logps/chosen": -1.7500464916229248,
+      "logps/rejected": -2.6284983158111572,
+      "loss": 0.6181,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7500464916229248,
+      "rewards/margins": 0.8784518241882324,
+      "rewards/rejected": -2.6284983158111572,
       "step": 5495
     },
     {
       "epoch": 2.9436360595417295,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 1.9219940303624106,
       "learning_rate": 1.049002635258256e-09,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33516764640808105,
+      "logits/rejected": -0.24554689228534698,
+      "logps/chosen": -1.9226995706558228,
+      "logps/rejected": -2.7350268363952637,
+      "loss": 0.6423,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9226995706558228,
+      "rewards/margins": 0.81232750415802,
+      "rewards/rejected": -2.7350268363952637,
       "step": 5500
     },
     {
       "epoch": 2.946312092323131,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.0637238345071798,
       "learning_rate": 9.505928569258358e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.27503135800361633,
+      "logits/rejected": -0.2749641239643097,
+      "logps/chosen": -1.7228387594223022,
+      "logps/rejected": -2.5409157276153564,
+      "loss": 0.6163,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7228387594223022,
+      "rewards/margins": 0.8180767893791199,
+      "rewards/rejected": -2.5409157276153564,
       "step": 5505
     },
     {
       "epoch": 2.9489881251045325,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.431336456667168,
       "learning_rate": 8.57024858130273e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3942207098007202,
+      "logits/rejected": -0.28590455651283264,
+      "logps/chosen": -1.8086611032485962,
+      "logps/rejected": -2.9078149795532227,
+      "loss": 0.622,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.8086611032485962,
+      "rewards/margins": 1.099153995513916,
+      "rewards/rejected": -2.9078149795532227,
       "step": 5510
     },
     {
       "epoch": 2.951664157885934,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.933506200947653,
       "learning_rate": 7.682995466686826e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.44348087906837463,
+      "logits/rejected": -0.3183046281337738,
+      "logps/chosen": -1.8650305271148682,
+      "logps/rejected": -2.6847617626190186,
+      "loss": 0.6192,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.8650305271148682,
+      "rewards/margins": 0.8197313547134399,
+      "rewards/rejected": -2.6847617626190186,
       "step": 5515
     },
     {
       "epoch": 2.9543401906673354,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.0060448868683265,
       "learning_rate": 6.844177833543741e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3432375490665436,
+      "logits/rejected": -0.2900136113166809,
+      "logps/chosen": -1.7172647714614868,
+      "logps/rejected": -2.4098618030548096,
+      "loss": 0.6349,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7172647714614868,
+      "rewards/margins": 0.6925967931747437,
+      "rewards/rejected": -2.4098618030548096,
       "step": 5520
     },
     {
       "epoch": 2.957016223448737,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.8598113197225774,
       "learning_rate": 6.053803820087467e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33267855644226074,
+      "logits/rejected": -0.21917939186096191,
+      "logps/chosen": -1.8262615203857422,
+      "logps/rejected": -2.750936985015869,
+      "loss": 0.6229,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.8262615203857422,
+      "rewards/margins": 0.9246752858161926,
+      "rewards/rejected": -2.750936985015869,
       "step": 5525
     },
     {
       "epoch": 2.959692256230139,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9769868958306382,
       "learning_rate": 5.311881094528514e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.41620102524757385,
+      "logits/rejected": -0.21030232310295105,
+      "logps/chosen": -1.947731614112854,
+      "logps/rejected": -2.506101608276367,
+      "loss": 0.6572,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.947731614112854,
+      "rewards/margins": 0.5583699941635132,
+      "rewards/rejected": -2.506101608276367,
       "step": 5530
     },
     {
       "epoch": 2.9623682890115406,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.22692520824563,
       "learning_rate": 4.6184168550050806e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3624917268753052,
+      "logits/rejected": -0.3147834241390228,
+      "logps/chosen": -1.7690579891204834,
+      "logps/rejected": -2.4304842948913574,
+      "loss": 0.6309,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.7690579891204834,
+      "rewards/margins": 0.6614264845848083,
+      "rewards/rejected": -2.4304842948913574,
       "step": 5535
     },
     {
       "epoch": 2.965044321792942,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.520252468209945,
       "learning_rate": 3.973417829510328e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4425172209739685,
+      "logits/rejected": -0.296741247177124,
+      "logps/chosen": -1.835646629333496,
+      "logps/rejected": -2.4684407711029053,
+      "loss": 0.6449,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.835646629333496,
+      "rewards/margins": 0.6327940225601196,
+      "rewards/rejected": -2.4684407711029053,
       "step": 5540
     },
     {
       "epoch": 2.9677203545743436,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.6475002943971684,
       "learning_rate": 3.3768902758274377e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.34194228053092957,
+      "logits/rejected": -0.24907627701759338,
+      "logps/chosen": -1.6530265808105469,
+      "logps/rejected": -2.4179790019989014,
+      "loss": 0.6287,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.6530265808105469,
+      "rewards/margins": 0.7649524807929993,
+      "rewards/rejected": -2.4179790019989014,
       "step": 5545
     },
     {
       "epoch": 2.970396387355745,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.7401954810749816,
       "learning_rate": 2.8288399814691e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.25394099950790405,
+      "logits/rejected": -0.15671826899051666,
+      "logps/chosen": -1.8259217739105225,
+      "logps/rejected": -2.5749268531799316,
+      "loss": 0.6251,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.8259217739105225,
+      "rewards/margins": 0.7490050196647644,
+      "rewards/rejected": -2.5749268531799316,
       "step": 5550
     },
     {
       "epoch": 2.9730724201371466,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 5.3673662597401295,
       "learning_rate": 2.3292722636220066e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3811422288417816,
+      "logits/rejected": -0.20088335871696472,
+      "logps/chosen": -1.8389356136322021,
+      "logps/rejected": -2.839597225189209,
+      "loss": 0.6341,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8389356136322021,
+      "rewards/margins": 1.0006616115570068,
+      "rewards/rejected": -2.839597225189209,
       "step": 5555
     },
     {
       "epoch": 2.9757484529185483,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.85901463357802,
       "learning_rate": 1.8781919690946668e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.30960819125175476,
+      "logits/rejected": -0.2990754246711731,
+      "logps/chosen": -1.7924613952636719,
+      "logps/rejected": -2.329362630844116,
+      "loss": 0.6452,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7924613952636719,
+      "rewards/margins": 0.5369013547897339,
+      "rewards/rejected": -2.329362630844116,
       "step": 5560
     },
     {
       "epoch": 2.97842448569995,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.1379076159709,
       "learning_rate": 1.4756034742696711e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.40022993087768555,
+      "logits/rejected": -0.3548814356327057,
+      "logps/chosen": -1.6633554697036743,
+      "logps/rejected": -2.437417984008789,
+      "loss": 0.6362,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6633554697036743,
+      "rewards/margins": 0.7740625143051147,
+      "rewards/rejected": -2.437417984008789,
       "step": 5565
     },
     {
       "epoch": 2.9811005184813513,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.542214601950439,
       "learning_rate": 1.12151068506261e-10,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.33018016815185547,
+      "logits/rejected": -0.22013342380523682,
+      "logps/chosen": -1.6760002374649048,
+      "logps/rejected": -2.761202096939087,
+      "loss": 0.6097,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.6760002374649048,
+      "rewards/margins": 1.0852017402648926,
+      "rewards/rejected": -2.761202096939087,
       "step": 5570
     },
     {
       "epoch": 2.983776551262753,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 6.395232107023532,
       "learning_rate": 8.159170368826629e-11,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.36207816004753113,
+      "logits/rejected": -0.23169486224651337,
+      "logps/chosen": -1.6693875789642334,
+      "logps/rejected": -2.5817911624908447,
+      "loss": 0.6141,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.6693875789642334,
+      "rewards/margins": 0.9124037027359009,
+      "rewards/rejected": -2.5817911624908447,
       "step": 5575
     },
     {
       "epoch": 2.9864525840441547,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 3.2348839331300594,
       "learning_rate": 5.588254946015114e-11,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.45230990648269653,
+      "logits/rejected": -0.22150883078575134,
+      "logps/chosen": -1.6564671993255615,
+      "logps/rejected": -2.546064853668213,
+      "loss": 0.62,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6564671993255615,
+      "rewards/margins": 0.8895977735519409,
+      "rewards/rejected": -2.546064853668213,
       "step": 5580
     },
     {
       "epoch": 2.989128616825556,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.87468450412408,
       "learning_rate": 3.502385525216978e-11,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.4140097200870514,
+      "logits/rejected": -0.27461984753608704,
+      "logps/chosen": -1.7657220363616943,
+      "logps/rejected": -2.702287197113037,
+      "loss": 0.6267,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.7657220363616943,
+      "rewards/margins": 0.9365649223327637,
+      "rewards/rejected": -2.702287197113037,
       "step": 5585
     },
     {
       "epoch": 2.9918046496069577,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 2.9479480742081416,
       "learning_rate": 1.901582343555308e-11,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3398052752017975,
+      "logits/rejected": -0.28592461347579956,
+      "logps/chosen": -1.9114067554473877,
+      "logps/rejected": -2.6307592391967773,
+      "loss": 0.6509,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.9114067554473877,
+      "rewards/margins": 0.7193526029586792,
+      "rewards/rejected": -2.6307592391967773,
       "step": 5590
     },
     {
       "epoch": 2.9944806823883594,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 8.232123270484031,
       "learning_rate": 7.858609320232634e-12,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.35649338364601135,
+      "logits/rejected": -0.22360675036907196,
+      "logps/chosen": -1.712451696395874,
+      "logps/rejected": -2.4911859035491943,
+      "loss": 0.617,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.712451696395874,
+      "rewards/margins": 0.7787342667579651,
+      "rewards/rejected": -2.4911859035491943,
       "step": 5595
     },
     {
       "epoch": 2.9971567151697607,
-      "grad_norm": 1.4142135623730951,
+      "grad_norm": 4.6488917845362066,
       "learning_rate": 1.5523211535639624e-12,
-      "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": NaN,
-      "logps/rejected": NaN,
-      "loss": 0.0,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": NaN,
-      "rewards/margins": NaN,
-      "rewards/rejected": NaN,
-      "semantic_entropy": NaN,
+      "logits/chosen": -0.3544706404209137,
+      "logits/rejected": -0.26396384835243225,
+      "logps/chosen": -1.702840805053711,
+      "logps/rejected": -2.855792760848999,
+      "loss": 0.6181,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.702840805053711,
+      "rewards/margins": 1.152951955795288,
+      "rewards/rejected": -2.855792760848999,
       "step": 5600
     },
     {
       "epoch": 2.9971567151697607,
-      "eval_logits/chosen": NaN,
-      "eval_logits/rejected": NaN,
-      "eval_logps/chosen": NaN,
-      "eval_logps/rejected": NaN,
-      "eval_loss": NaN,
-      "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": NaN,
-      "eval_rewards/margins": NaN,
-      "eval_rewards/rejected": NaN,
-      "eval_runtime": 34.6428,
-      "eval_samples_per_second": 38.825,
-      "eval_semantic_entropy": NaN,
-      "eval_steps_per_second": 9.728,
+      "eval_logits/chosen": -0.060386497527360916,
+      "eval_logits/rejected": 0.01924128830432892,
+      "eval_logps/chosen": -1.79891836643219,
+      "eval_logps/rejected": -2.583002805709839,
+      "eval_loss": 0.6509584784507751,
+      "eval_rewards/accuracies": 0.6735904812812805,
+      "eval_rewards/chosen": -1.79891836643219,
+      "eval_rewards/margins": 0.7840846180915833,
+      "eval_rewards/rejected": -2.583002805709839,
+      "eval_runtime": 40.7815,
+      "eval_samples_per_second": 32.981,
+      "eval_steps_per_second": 8.264,
       "step": 5600
     },
     {
       "epoch": 2.999297541394882,
       "step": 5604,
       "total_flos": 0.0,
-      "train_loss": 0.4331439037054118,
-      "train_runtime": 28837.4505,
-      "train_samples_per_second": 6.22,
-      "train_steps_per_second": 0.194
+      "train_loss": 0.6513626721822561,
+      "train_runtime": 30253.6444,
+      "train_samples_per_second": 5.929,
+      "train_steps_per_second": 0.185
     }
   ],
   "logging_steps": 5,