{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 276,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "debug/policy_chosen_logits": 0.8079685568809509,
      "debug/policy_chosen_logps": -5.034485816955566,
      "debug/policy_rejected_logits": 0.6268295645713806,
      "debug/policy_rejected_logps": -2.0584616661071777,
      "debug/reference_chosen_logps": -5.034485816955566,
      "debug/reference_rejected_logps": -2.0584616661071777,
      "debug/sppo_chosen_loss": 2500.0,
      "debug/sppo_chosen_reward_in_loss": 0.0,
      "debug/sppo_rej_reward_in_loss": 0.0,
      "debug/sppo_reject_loss": 2500.0,
      "epoch": 0.0036231884057971015,
      "grad_norm": 47836.56654178361,
      "learning_rate": 1e-09,
      "logits/chosen": 0.8079685568809509,
      "logits/rejected": 0.6268295645713806,
      "logps/chosen": -5.034485816955566,
      "logps/rejected": -2.0584616661071777,
      "loss": 5000.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "debug/policy_chosen_logits": 1.1308157444000244,
      "debug/policy_chosen_logps": -5.128948211669922,
      "debug/policy_rejected_logits": 1.3690780401229858,
      "debug/policy_rejected_logps": -47.817047119140625,
      "debug/reference_chosen_logps": -5.141074180603027,
      "debug/reference_rejected_logps": -47.71269607543945,
      "debug/sppo_chosen_loss": 2498.794189453125,
      "debug/sppo_chosen_reward_in_loss": 0.012125834822654724,
      "debug/sppo_rej_reward_in_loss": -0.10434922575950623,
      "debug/sppo_reject_loss": 2489.93359375,
      "epoch": 0.018115942028985508,
      "grad_norm": 83041.00650934623,
      "learning_rate": 5e-09,
      "logits/chosen": 1.1308157444000244,
      "logits/rejected": 1.3690780401229858,
      "logps/chosen": -5.128948211669922,
      "logps/rejected": -47.817047119140625,
      "loss": 4999.4365,
      "rewards/accuracies": 0.15625,
      "rewards/chosen": 0.00012125834473408759,
      "rewards/margins": 0.0011647505452856421,
      "rewards/rejected": -0.0010434923460707068,
      "step": 5
    },
    {
      "debug/policy_chosen_logits": 1.3401418924331665,
      "debug/policy_chosen_logps": -37.56370544433594,
      "debug/policy_rejected_logits": 1.6091701984405518,
      "debug/policy_rejected_logps": -8.9139404296875,
      "debug/reference_chosen_logps": -37.657630920410156,
      "debug/reference_rejected_logps": -8.866057395935059,
      "debug/sppo_chosen_loss": 2490.720458984375,
      "debug/sppo_chosen_reward_in_loss": 0.09392313659191132,
      "debug/sppo_rej_reward_in_loss": -0.04788307100534439,
      "debug/sppo_reject_loss": 2495.249755859375,
      "epoch": 0.036231884057971016,
      "grad_norm": 25854.702147472322,
      "learning_rate": 1e-08,
      "logits/chosen": 1.3401418924331665,
      "logits/rejected": 1.6091701984405518,
      "logps/chosen": -37.56370544433594,
      "logps/rejected": -8.9139404296875,
      "loss": 4998.7039,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.0009392313659191132,
      "rewards/margins": 0.0014180620200932026,
      "rewards/rejected": -0.00047883071238175035,
      "step": 10
    },
    {
      "debug/policy_chosen_logits": 1.281254768371582,
      "debug/policy_chosen_logps": -7.3922319412231445,
      "debug/policy_rejected_logits": 1.6363356113433838,
      "debug/policy_rejected_logps": -5.406702518463135,
      "debug/reference_chosen_logps": -7.438336372375488,
      "debug/reference_rejected_logps": -5.3998122215271,
      "debug/sppo_chosen_loss": 2495.47998046875,
      "debug/sppo_chosen_reward_in_loss": 0.046104758977890015,
      "debug/sppo_rej_reward_in_loss": -0.006889957003295422,
      "debug/sppo_reject_loss": 2499.34228515625,
      "epoch": 0.05434782608695652,
      "grad_norm": 52134.81661267604,
      "learning_rate": 1.5e-08,
      "logits/chosen": 1.281254768371582,
      "logits/rejected": 1.6363356113433838,
      "logps/chosen": -7.3922319412231445,
      "logps/rejected": -5.406702518463135,
      "loss": 4994.4305,
      "rewards/accuracies": 0.2750000059604645,
      "rewards/chosen": 0.0004610475734807551,
      "rewards/margins": 0.0005299471667967737,
      "rewards/rejected": -6.889957876410335e-05,
      "step": 15
    },
    {
      "debug/policy_chosen_logits": 1.2531625032424927,
      "debug/policy_chosen_logps": -4.177506446838379,
      "debug/policy_rejected_logits": 1.600624680519104,
      "debug/policy_rejected_logps": -88.92843627929688,
      "debug/reference_chosen_logps": -4.198297023773193,
      "debug/reference_rejected_logps": -88.761962890625,
      "debug/sppo_chosen_loss": 2497.947021484375,
      "debug/sppo_chosen_reward_in_loss": 0.020790796726942062,
      "debug/sppo_rej_reward_in_loss": -0.16647538542747498,
      "debug/sppo_reject_loss": 2484.888427734375,
      "epoch": 0.07246376811594203,
      "grad_norm": 21278.15413555925,
      "learning_rate": 2e-08,
      "logits/chosen": 1.2531625032424927,
      "logits/rejected": 1.600624680519104,
      "logps/chosen": -4.177506446838379,
      "logps/rejected": -88.92843627929688,
      "loss": 5001.0859,
      "rewards/accuracies": 0.22499999403953552,
      "rewards/chosen": 0.00020790801499970257,
      "rewards/margins": 0.001872661872766912,
      "rewards/rejected": -0.001664753770455718,
      "step": 20
    },
    {
      "debug/policy_chosen_logits": 1.3284282684326172,
      "debug/policy_chosen_logps": -209.1931610107422,
      "debug/policy_rejected_logits": 1.7004191875457764,
      "debug/policy_rejected_logps": -268.02142333984375,
      "debug/reference_chosen_logps": -208.846923828125,
      "debug/reference_rejected_logps": -267.5577697753906,
      "debug/sppo_chosen_loss": 2539.96240234375,
      "debug/sppo_chosen_reward_in_loss": -0.3462248742580414,
      "debug/sppo_rej_reward_in_loss": -0.4636896550655365,
      "debug/sppo_reject_loss": 2462.03173828125,
      "epoch": 0.09057971014492754,
      "grad_norm": 18939.799089042473,
      "learning_rate": 2.5e-08,
      "logits/chosen": 1.3284282684326172,
      "logits/rejected": 1.7004191875457764,
      "logps/chosen": -209.1931610107422,
      "logps/rejected": -268.02142333984375,
      "loss": 4993.6285,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": -0.0034622489474713802,
      "rewards/margins": 0.0011746471282094717,
      "rewards/rejected": -0.004636895842850208,
      "step": 25
    },
    {
      "debug/policy_chosen_logits": 1.4937888383865356,
      "debug/policy_chosen_logps": -61.337852478027344,
      "debug/policy_rejected_logits": 1.8404518365859985,
      "debug/policy_rejected_logps": -5.3695268630981445,
      "debug/reference_chosen_logps": -61.356834411621094,
      "debug/reference_rejected_logps": -5.380406379699707,
      "debug/sppo_chosen_loss": 2498.155517578125,
      "debug/sppo_chosen_reward_in_loss": 0.01897408999502659,
      "debug/sppo_rej_reward_in_loss": 0.010879549197852612,
      "debug/sppo_reject_loss": 2501.10498046875,
      "epoch": 0.10869565217391304,
      "grad_norm": 37304.285865467384,
      "learning_rate": 3e-08,
      "logits/chosen": 1.4937888383865356,
      "logits/rejected": 1.8404518365859985,
      "logps/chosen": -61.337852478027344,
      "logps/rejected": -5.3695268630981445,
      "loss": 4995.4516,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.00018974088015966117,
      "rewards/margins": 8.094539953162894e-05,
      "rewards/rejected": 0.00010879549517994747,
      "step": 30
    },
    {
      "debug/policy_chosen_logits": 1.5908656120300293,
      "debug/policy_chosen_logps": -67.03999328613281,
      "debug/policy_rejected_logits": 1.9137623310089111,
      "debug/policy_rejected_logps": -7.439389228820801,
      "debug/reference_chosen_logps": -67.15159606933594,
      "debug/reference_rejected_logps": -7.461578369140625,
      "debug/sppo_chosen_loss": 2489.117431640625,
      "debug/sppo_chosen_reward_in_loss": 0.11160683631896973,
      "debug/sppo_rej_reward_in_loss": 0.022188443690538406,
      "debug/sppo_reject_loss": 2502.24462890625,
      "epoch": 0.12681159420289856,
      "grad_norm": 80189.68931153638,
      "learning_rate": 3.4999999999999996e-08,
      "logits/chosen": 1.5908656120300293,
      "logits/rejected": 1.9137623310089111,
      "logps/chosen": -67.03999328613281,
      "logps/rejected": -7.439389228820801,
      "loss": 4996.5711,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.0011160684516653419,
      "rewards/margins": 0.0008941839332692325,
      "rewards/rejected": 0.00022188443108461797,
      "step": 35
    },
    {
      "debug/policy_chosen_logits": 1.4925166368484497,
      "debug/policy_chosen_logps": -90.91502380371094,
      "debug/policy_rejected_logits": 1.8190809488296509,
      "debug/policy_rejected_logps": -80.93954467773438,
      "debug/reference_chosen_logps": -90.82229614257812,
      "debug/reference_rejected_logps": -81.13166809082031,
      "debug/sppo_chosen_loss": 2509.4033203125,
      "debug/sppo_chosen_reward_in_loss": -0.09272497892379761,
      "debug/sppo_rej_reward_in_loss": 0.19212593138217926,
      "debug/sppo_reject_loss": 2519.924560546875,
      "epoch": 0.14492753623188406,
      "grad_norm": 101789.85983014172,
      "learning_rate": 4e-08,
      "logits/chosen": 1.4925166368484497,
      "logits/rejected": 1.8190809488296509,
      "logps/chosen": -90.91502380371094,
      "logps/rejected": -80.93954467773438,
      "loss": 5008.0297,
      "rewards/accuracies": 0.10000000149011612,
      "rewards/chosen": -0.0009272497263737023,
      "rewards/margins": -0.0028485090006142855,
      "rewards/rejected": 0.0019212592160329223,
      "step": 40
    },
    {
      "debug/policy_chosen_logits": 1.4444226026535034,
      "debug/policy_chosen_logps": -43.32951736450195,
      "debug/policy_rejected_logits": 1.6251853704452515,
      "debug/policy_rejected_logps": -7.5910234451293945,
      "debug/reference_chosen_logps": -43.420326232910156,
      "debug/reference_rejected_logps": -7.612727165222168,
      "debug/sppo_chosen_loss": 2491.11474609375,
      "debug/sppo_chosen_reward_in_loss": 0.09080849587917328,
      "debug/sppo_rej_reward_in_loss": 0.021703582257032394,
      "debug/sppo_reject_loss": 2502.208251953125,
      "epoch": 0.16304347826086957,
      "grad_norm": 127325.37231669381,
      "learning_rate": 4.5e-08,
      "logits/chosen": 1.4444226026535034,
      "logits/rejected": 1.6251853704452515,
      "logps/chosen": -43.32951736450195,
      "logps/rejected": -7.5910234451293945,
      "loss": 4992.1121,
      "rewards/accuracies": 0.2750000059604645,
      "rewards/chosen": 0.0009080851450562477,
      "rewards/margins": 0.0006910492666065693,
      "rewards/rejected": 0.00021703580569010228,
      "step": 45
    },
    {
      "debug/policy_chosen_logits": 1.3567314147949219,
      "debug/policy_chosen_logps": -88.03765869140625,
      "debug/policy_rejected_logits": 1.7951405048370361,
      "debug/policy_rejected_logps": -3.4048943519592285,
      "debug/reference_chosen_logps": -88.64862823486328,
      "debug/reference_rejected_logps": -3.3678181171417236,
      "debug/sppo_chosen_loss": 2454.765380859375,
      "debug/sppo_chosen_reward_in_loss": 0.6109756231307983,
      "debug/sppo_rej_reward_in_loss": -0.03707638382911682,
      "debug/sppo_reject_loss": 2496.30859375,
      "epoch": 0.18115942028985507,
      "grad_norm": 58358.722910061,
      "learning_rate": 5e-08,
      "logits/chosen": 1.3567314147949219,
      "logits/rejected": 1.7951405048370361,
      "logps/chosen": -88.03765869140625,
      "logps/rejected": -3.4048943519592285,
      "loss": 4992.9215,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.006109756417572498,
      "rewards/margins": 0.006480520125478506,
      "rewards/rejected": -0.00037076379521749914,
      "step": 50
    },
    {
      "debug/policy_chosen_logits": 1.3366315364837646,
      "debug/policy_chosen_logps": -11.845144271850586,
      "debug/policy_rejected_logits": 1.8784109354019165,
      "debug/policy_rejected_logps": -3.4788289070129395,
      "debug/reference_chosen_logps": -11.87784194946289,
      "debug/reference_rejected_logps": -3.4915835857391357,
      "debug/sppo_chosen_loss": 2496.794677734375,
      "debug/sppo_chosen_reward_in_loss": 0.03269771859049797,
      "debug/sppo_rej_reward_in_loss": 0.012755215167999268,
      "debug/sppo_reject_loss": 2501.293212890625,
      "epoch": 0.19927536231884058,
      "grad_norm": 60865.57057492795,
      "learning_rate": 5.5e-08,
      "logits/chosen": 1.3366315364837646,
      "logits/rejected": 1.8784109354019165,
      "logps/chosen": -11.845144271850586,
      "logps/rejected": -3.4788289070129395,
      "loss": 4988.9117,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.0003269771987106651,
      "rewards/margins": 0.00019942503422498703,
      "rewards/rejected": 0.00012755214993376285,
      "step": 55
    },
    {
      "debug/policy_chosen_logits": 1.532639741897583,
      "debug/policy_chosen_logps": -4.54892635345459,
      "debug/policy_rejected_logits": 1.8314745426177979,
      "debug/policy_rejected_logps": -8.05273151397705,
      "debug/reference_chosen_logps": -4.53155517578125,
      "debug/reference_rejected_logps": -8.040096282958984,
      "debug/sppo_chosen_loss": 2501.75537109375,
      "debug/sppo_chosen_reward_in_loss": -0.017371535301208496,
      "debug/sppo_rej_reward_in_loss": -0.01263415813446045,
      "debug/sppo_reject_loss": 2498.755126953125,
      "epoch": 0.21739130434782608,
      "grad_norm": 81182.53900123628,
      "learning_rate": 6e-08,
      "logits/chosen": 1.532639741897583,
      "logits/rejected": 1.8314745426177979,
      "logps/chosen": -4.54892635345459,
      "logps/rejected": -8.05273151397705,
      "loss": 5001.6219,
      "rewards/accuracies": 0.2750000059604645,
      "rewards/chosen": -0.00017371535068377852,
      "rewards/margins": -4.7373774577863514e-05,
      "rewards/rejected": -0.000126341576105915,
      "step": 60
    },
    {
      "debug/policy_chosen_logits": 1.488297939300537,
      "debug/policy_chosen_logps": -133.06114196777344,
      "debug/policy_rejected_logits": 1.8658514022827148,
      "debug/policy_rejected_logps": -12.593046188354492,
      "debug/reference_chosen_logps": -133.0397491455078,
      "debug/reference_rejected_logps": -12.575475692749023,
      "debug/sppo_chosen_loss": 2502.216552734375,
      "debug/sppo_chosen_reward_in_loss": -0.0213775634765625,
      "debug/sppo_rej_reward_in_loss": -0.017570963129401207,
      "debug/sppo_reject_loss": 2498.254638671875,
      "epoch": 0.23550724637681159,
      "grad_norm": 17622.6879814254,
      "learning_rate": 6.5e-08,
      "logits/chosen": 1.488297939300537,
      "logits/rejected": 1.8658514022827148,
      "logps/chosen": -133.06114196777344,
      "logps/rejected": -12.593046188354492,
      "loss": 4999.2039,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.00021377562370616943,
      "rewards/margins": -3.806598761002533e-05,
      "rewards/rejected": -0.0001757096324581653,
      "step": 65
    },
    {
      "debug/policy_chosen_logits": 1.6522119045257568,
      "debug/policy_chosen_logps": -7.065809726715088,
      "debug/policy_rejected_logits": 1.638129472732544,
      "debug/policy_rejected_logps": -95.52651977539062,
      "debug/reference_chosen_logps": -7.0487470626831055,
      "debug/reference_rejected_logps": -95.66188049316406,
      "debug/sppo_chosen_loss": 2501.720947265625,
      "debug/sppo_chosen_reward_in_loss": -0.01706244982779026,
      "debug/sppo_rej_reward_in_loss": 0.135361909866333,
      "debug/sppo_reject_loss": 2514.490966796875,
      "epoch": 0.2536231884057971,
      "grad_norm": 69921.87787045751,
      "learning_rate": 6.999999999999999e-08,
      "logits/chosen": 1.6522119045257568,
      "logits/rejected": 1.638129472732544,
      "logps/chosen": -7.065809726715088,
      "logps/rejected": -95.52651977539062,
      "loss": 4999.0477,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": -0.0001706244656816125,
      "rewards/margins": -0.0015242435038089752,
      "rewards/rejected": 0.0013536190381273627,
      "step": 70
    },
    {
      "debug/policy_chosen_logits": 1.4129836559295654,
      "debug/policy_chosen_logps": -122.12205505371094,
      "debug/policy_rejected_logits": 1.7989412546157837,
      "debug/policy_rejected_logps": -41.31743621826172,
      "debug/reference_chosen_logps": -122.72642517089844,
      "debug/reference_rejected_logps": -41.365840911865234,
      "debug/sppo_chosen_loss": 2452.815673828125,
      "debug/sppo_chosen_reward_in_loss": 0.6043859720230103,
      "debug/sppo_rej_reward_in_loss": 0.04840099439024925,
      "debug/sppo_reject_loss": 2504.8818359375,
      "epoch": 0.2717391304347826,
      "grad_norm": 264335.1107357926,
      "learning_rate": 7.5e-08,
      "logits/chosen": 1.4129836559295654,
      "logits/rejected": 1.7989412546157837,
      "logps/chosen": -122.12205505371094,
      "logps/rejected": -41.31743621826172,
      "loss": 5007.7828,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.006043859291821718,
      "rewards/margins": 0.005559849552810192,
      "rewards/rejected": 0.00048400991363450885,
      "step": 75
    },
    {
      "debug/policy_chosen_logits": 1.3792654275894165,
      "debug/policy_chosen_logps": -5.836012840270996,
      "debug/policy_rejected_logits": 1.7591722011566162,
      "debug/policy_rejected_logps": -4.791333198547363,
      "debug/reference_chosen_logps": -5.829151630401611,
      "debug/reference_rejected_logps": -4.762129306793213,
      "debug/sppo_chosen_loss": 2500.70458984375,
      "debug/sppo_chosen_reward_in_loss": -0.006861341651529074,
      "debug/sppo_rej_reward_in_loss": -0.02920418418943882,
      "debug/sppo_reject_loss": 2497.136962890625,
      "epoch": 0.2898550724637681,
      "grad_norm": 15656.535742450906,
      "learning_rate": 8e-08,
      "logits/chosen": 1.3792654275894165,
      "logits/rejected": 1.7591722011566162,
      "logps/chosen": -5.836012840270996,
      "logps/rejected": -4.791333198547363,
      "loss": 4993.7707,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": -6.861341535113752e-05,
      "rewards/margins": 0.00022342840384226292,
      "rewards/rejected": -0.0002920418046414852,
      "step": 80
    },
    {
      "debug/policy_chosen_logits": 1.5190160274505615,
      "debug/policy_chosen_logps": -64.62407684326172,
      "debug/policy_rejected_logits": 2.1376373767852783,
      "debug/policy_rejected_logps": -7.913151741027832,
      "debug/reference_chosen_logps": -64.85469055175781,
      "debug/reference_rejected_logps": -7.927921295166016,
      "debug/sppo_chosen_loss": 2478.92041015625,
      "debug/sppo_chosen_reward_in_loss": 0.23062074184417725,
      "debug/sppo_rej_reward_in_loss": 0.014769583940505981,
      "debug/sppo_reject_loss": 2501.4853515625,
      "epoch": 0.3079710144927536,
      "grad_norm": 51152.42234215201,
      "learning_rate": 8.5e-08,
      "logits/chosen": 1.5190160274505615,
      "logits/rejected": 2.1376373767852783,
      "logps/chosen": -64.62407684326172,
      "logps/rejected": -7.913151741027832,
      "loss": 4991.4711,
      "rewards/accuracies": 0.2750000059604645,
      "rewards/chosen": 0.0023062073159962893,
      "rewards/margins": 0.002158511197194457,
      "rewards/rejected": 0.00014769584231544286,
      "step": 85
    },
    {
      "debug/policy_chosen_logits": 1.2532832622528076,
      "debug/policy_chosen_logps": -122.0447769165039,
      "debug/policy_rejected_logits": 1.4385173320770264,
      "debug/policy_rejected_logps": -7.544157981872559,
      "debug/reference_chosen_logps": -122.35246276855469,
      "debug/reference_rejected_logps": -7.5182623863220215,
      "debug/sppo_chosen_loss": 2472.504150390625,
      "debug/sppo_chosen_reward_in_loss": 0.3076968491077423,
      "debug/sppo_rej_reward_in_loss": -0.025895338505506516,
      "debug/sppo_reject_loss": 2497.43994140625,
      "epoch": 0.32608695652173914,
      "grad_norm": 15987.307319112604,
      "learning_rate": 9e-08,
      "logits/chosen": 1.2532832622528076,
      "logits/rejected": 1.4385173320770264,
      "logps/chosen": -122.0447769165039,
      "logps/rejected": -7.544157981872559,
      "loss": 4997.1215,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.0030769682489335537,
      "rewards/margins": 0.0033359217923134565,
      "rewards/rejected": -0.0002589533687569201,
      "step": 90
    },
    {
      "debug/policy_chosen_logits": 1.44913911819458,
      "debug/policy_chosen_logps": -149.0557098388672,
      "debug/policy_rejected_logits": 1.7728023529052734,
      "debug/policy_rejected_logps": -8.200785636901855,
      "debug/reference_chosen_logps": -149.43124389648438,
      "debug/reference_rejected_logps": -8.184698104858398,
      "debug/sppo_chosen_loss": 2464.93408203125,
      "debug/sppo_chosen_reward_in_loss": 0.3755301237106323,
      "debug/sppo_rej_reward_in_loss": -0.016088182106614113,
      "debug/sppo_reject_loss": 2498.406982421875,
      "epoch": 0.3442028985507246,
      "grad_norm": 13150.098708323369,
      "learning_rate": 9.499999999999999e-08,
      "logits/chosen": 1.44913911819458,
      "logits/rejected": 1.7728023529052734,
      "logps/chosen": -149.0557098388672,
      "logps/rejected": -8.200785636901855,
      "loss": 4994.3016,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.0037553017027676105,
      "rewards/margins": 0.003916183020919561,
      "rewards/rejected": -0.00016088179836515337,
      "step": 95
    },
    {
      "debug/policy_chosen_logits": 1.2099888324737549,
      "debug/policy_chosen_logps": -6.501370906829834,
      "debug/policy_rejected_logits": 1.892987847328186,
      "debug/policy_rejected_logps": -1.8146244287490845,
      "debug/reference_chosen_logps": -6.515327453613281,
      "debug/reference_rejected_logps": -1.8118356466293335,
      "debug/sppo_chosen_loss": 2498.66552734375,
      "debug/sppo_chosen_reward_in_loss": 0.013957425951957703,
      "debug/sppo_rej_reward_in_loss": -0.002788883401080966,
      "debug/sppo_reject_loss": 2499.729736328125,
      "epoch": 0.36231884057971014,
      "grad_norm": 48606.634339351476,
      "learning_rate": 1e-07,
      "logits/chosen": 1.2099888324737549,
      "logits/rejected": 1.892987847328186,
      "logps/chosen": -6.501370906829834,
      "logps/rejected": -1.8146244287490845,
      "loss": 4997.6852,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.0001395742583554238,
      "rewards/margins": 0.00016746309120208025,
      "rewards/rejected": -2.788883284665644e-05,
      "step": 100
    },
    {
      "epoch": 0.36231884057971014,
      "eval_debug/policy_chosen_logits": 1.6633050441741943,
      "eval_debug/policy_chosen_logps": -122.66744232177734,
      "eval_debug/policy_rejected_logits": 1.7245433330535889,
      "eval_debug/policy_rejected_logps": -63.6384391784668,
      "eval_debug/reference_chosen_logps": -123.14806365966797,
      "eval_debug/reference_rejected_logps": -63.887054443359375,
      "eval_debug/sppo_chosen_loss": 2455.7412109375,
      "eval_debug/sppo_chosen_reward_in_loss": 0.4806005656719208,
      "eval_debug/sppo_rej_reward_in_loss": 0.24862028658390045,
      "eval_debug/sppo_reject_loss": 2529.1591796875,
      "eval_logits/chosen": 1.6633050441741943,
      "eval_logits/rejected": 1.7245433330535889,
      "eval_logps/chosen": -122.66744232177734,
      "eval_logps/rejected": -63.6384391784668,
      "eval_loss": 4986.5,
      "eval_rewards/accuracies": 0.31578946113586426,
      "eval_rewards/chosen": 0.004806005861610174,
      "eval_rewards/margins": 0.0023198029957711697,
      "eval_rewards/rejected": 0.002486202633008361,
      "eval_runtime": 29.2776,
      "eval_samples_per_second": 20.494,
      "eval_steps_per_second": 0.649,
      "step": 100
    },
    {
      "debug/policy_chosen_logits": 1.1337151527404785,
      "debug/policy_chosen_logps": -5.464511871337891,
      "debug/policy_rejected_logits": 1.2518866062164307,
      "debug/policy_rejected_logps": -100.66749572753906,
      "debug/reference_chosen_logps": -5.478997230529785,
      "debug/reference_rejected_logps": -100.13005828857422,
      "debug/sppo_chosen_loss": 2498.566650390625,
      "debug/sppo_chosen_reward_in_loss": 0.014485633000731468,
      "debug/sppo_rej_reward_in_loss": -0.5374492406845093,
      "debug/sppo_reject_loss": 2460.12158203125,
      "epoch": 0.3804347826086957,
      "grad_norm": 233662.76041400852,
      "learning_rate": 9.98009940204023e-08,
      "logits/chosen": 1.1337151527404785,
      "logits/rejected": 1.2518866062164307,
      "logps/chosen": -5.464511871337891,
      "logps/rejected": -100.66749572753906,
      "loss": 5023.0688,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.00014485632709693164,
      "rewards/margins": 0.0055193486623466015,
      "rewards/rejected": -0.005374492146074772,
      "step": 105
    },
    {
      "debug/policy_chosen_logits": 1.2909049987792969,
      "debug/policy_chosen_logps": -69.8453598022461,
      "debug/policy_rejected_logits": 1.5784929990768433,
      "debug/policy_rejected_logps": -6.722166538238525,
      "debug/reference_chosen_logps": -70.15248107910156,
      "debug/reference_rejected_logps": -6.677098274230957,
      "debug/sppo_chosen_loss": 2473.166015625,
      "debug/sppo_chosen_reward_in_loss": 0.30711063742637634,
      "debug/sppo_rej_reward_in_loss": -0.045068077743053436,
      "debug/sppo_reject_loss": 2495.512451171875,
      "epoch": 0.39855072463768115,
      "grad_norm": 54187.6399779739,
      "learning_rate": 9.92055602168058e-08,
      "logits/chosen": 1.2909049987792969,
      "logits/rejected": 1.5784929990768433,
      "logps/chosen": -69.8453598022461,
      "logps/rejected": -6.722166538238525,
      "loss": 4988.0766,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003071106504648924,
      "rewards/margins": 0.003521787002682686,
      "rewards/rejected": -0.00045068078907206655,
      "step": 110
    },
    {
      "debug/policy_chosen_logits": 1.5610486268997192,
      "debug/policy_chosen_logps": -72.94502258300781,
      "debug/policy_rejected_logits": 1.9416284561157227,
      "debug/policy_rejected_logps": -17.85445213317871,
      "debug/reference_chosen_logps": -73.14714050292969,
      "debug/reference_rejected_logps": -17.959739685058594,
      "debug/sppo_chosen_loss": 2480.93798828125,
      "debug/sppo_chosen_reward_in_loss": 0.20211248099803925,
      "debug/sppo_rej_reward_in_loss": 0.10528746992349625,
      "debug/sppo_reject_loss": 2511.402587890625,
      "epoch": 0.4166666666666667,
      "grad_norm": 36462.2274872176,
      "learning_rate": 9.821843838470533e-08,
      "logits/chosen": 1.5610486268997192,
      "logits/rejected": 1.9416284561157227,
      "logps/chosen": -72.94502258300781,
      "logps/rejected": -17.85445213317871,
      "loss": 4989.1812,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": 0.002021125052124262,
      "rewards/margins": 0.0009682502713985741,
      "rewards/rejected": 0.0010528747225180268,
      "step": 115
    },
    {
      "debug/policy_chosen_logits": 1.3458185195922852,
      "debug/policy_chosen_logps": -47.522071838378906,
      "debug/policy_rejected_logits": 1.701849341392517,
      "debug/policy_rejected_logps": -2.4559662342071533,
      "debug/reference_chosen_logps": -47.64037322998047,
      "debug/reference_rejected_logps": -2.4745194911956787,
      "debug/sppo_chosen_loss": 2488.593017578125,
      "debug/sppo_chosen_reward_in_loss": 0.1182965412735939,
      "debug/sppo_rej_reward_in_loss": 0.018553102388978004,
      "debug/sppo_reject_loss": 2501.86083984375,
      "epoch": 0.43478260869565216,
      "grad_norm": 44388.4153538321,
      "learning_rate": 9.684748624998809e-08,
      "logits/chosen": 1.3458185195922852,
      "logits/rejected": 1.701849341392517,
      "logps/chosen": -47.522071838378906,
      "logps/rejected": -2.4559662342071533,
      "loss": 4995.3309,
      "rewards/accuracies": 0.17499999701976776,
      "rewards/chosen": 0.0011829653521999717,
      "rewards/margins": 0.0009974343702197075,
      "rewards/rejected": 0.00018553101108409464,
      "step": 120
    },
    {
      "debug/policy_chosen_logits": 1.4266709089279175,
      "debug/policy_chosen_logps": -6.328521251678467,
      "debug/policy_rejected_logits": 1.6656681299209595,
      "debug/policy_rejected_logps": -58.306007385253906,
      "debug/reference_chosen_logps": -6.291218280792236,
      "debug/reference_rejected_logps": -58.57655715942383,
      "debug/sppo_chosen_loss": 2503.767333984375,
      "debug/sppo_chosen_reward_in_loss": -0.037303246557712555,
      "debug/sppo_rej_reward_in_loss": 0.27054446935653687,
      "debug/sppo_reject_loss": 2530.11767578125,
      "epoch": 0.4528985507246377,
      "grad_norm": 35653.64462552226,
      "learning_rate": 9.510361691955607e-08,
      "logits/chosen": 1.4266709089279175,
      "logits/rejected": 1.6656681299209595,
      "logps/chosen": -6.328521251678467,
      "logps/rejected": -58.306007385253906,
      "loss": 4997.7953,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.000373032467905432,
      "rewards/margins": -0.0030784769915044308,
      "rewards/rejected": 0.0027054445818066597,
      "step": 125
    },
    {
      "debug/policy_chosen_logits": 1.2133262157440186,
      "debug/policy_chosen_logps": -4.814808368682861,
      "debug/policy_rejected_logits": 1.638085961341858,
      "debug/policy_rejected_logps": -7.694342136383057,
      "debug/reference_chosen_logps": -4.811456203460693,
      "debug/reference_rejected_logps": -7.753373622894287,
      "debug/sppo_chosen_loss": 2500.358154296875,
      "debug/sppo_chosen_reward_in_loss": -0.003351741936057806,
      "debug/sppo_rej_reward_in_loss": 0.059032391756772995,
      "debug/sppo_reject_loss": 2506.000732421875,
      "epoch": 0.47101449275362317,
      "grad_norm": 56980.429768390655,
      "learning_rate": 9.300071201038502e-08,
      "logits/chosen": 1.2133262157440186,
      "logits/rejected": 1.638085961341858,
      "logps/chosen": -4.814808368682861,
      "logps/rejected": -7.694342136383057,
      "loss": 4970.8211,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": -3.351741543156095e-05,
      "rewards/margins": -0.0006238413043320179,
      "rewards/rejected": 0.0005903238779865205,
      "step": 130
    },
    {
      "debug/policy_chosen_logits": 1.3553920984268188,
      "debug/policy_chosen_logps": -7.613694667816162,
      "debug/policy_rejected_logits": 1.8132812976837158,
      "debug/policy_rejected_logps": -45.774879455566406,
      "debug/reference_chosen_logps": -7.610405921936035,
      "debug/reference_rejected_logps": -46.041690826416016,
      "debug/sppo_chosen_loss": 2500.42138671875,
      "debug/sppo_chosen_reward_in_loss": -0.0032895803451538086,
      "debug/sppo_rej_reward_in_loss": 0.2668178975582123,
      "debug/sppo_reject_loss": 2528.033935546875,
      "epoch": 0.4891304347826087,
      "grad_norm": 22513.750392611728,
      "learning_rate": 9.055551114853295e-08,
      "logits/chosen": 1.3553920984268188,
      "logits/rejected": 1.8132812976837158,
      "logps/chosen": -7.613694667816162,
      "logps/rejected": -45.774879455566406,
      "loss": 5004.3156,
      "rewards/accuracies": 0.22499999403953552,
      "rewards/chosen": -3.289579035481438e-05,
      "rewards/margins": -0.0027010745834559202,
      "rewards/rejected": 0.0026681788731366396,
      "step": 135
    },
    {
      "debug/policy_chosen_logits": 1.7337901592254639,
      "debug/policy_chosen_logps": -10.878243446350098,
      "debug/policy_rejected_logits": 2.027592182159424,
      "debug/policy_rejected_logps": -1.9502204656600952,
      "debug/reference_chosen_logps": -10.920201301574707,
      "debug/reference_rejected_logps": -1.9509022235870361,
      "debug/sppo_chosen_loss": 2495.86865234375,
      "debug/sppo_chosen_reward_in_loss": 0.04195826128125191,
      "debug/sppo_rej_reward_in_loss": 0.0006817228859290481,
      "debug/sppo_reject_loss": 2500.08251953125,
      "epoch": 0.5072463768115942,
      "grad_norm": 207027.63645308473,
      "learning_rate": 8.778747871771291e-08,
      "logits/chosen": 1.7337901592254639,
      "logits/rejected": 2.027592182159424,
      "logps/chosen": -10.878243446350098,
      "logps/rejected": -1.9502204656600952,
      "loss": 5006.7344,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.00041958261863328516,
      "rewards/margins": 0.0004127653664909303,
      "rewards/rejected": 6.817223038524389e-06,
      "step": 140
    },
    {
      "debug/policy_chosen_logits": 1.2483991384506226,
      "debug/policy_chosen_logps": -226.301025390625,
      "debug/policy_rejected_logits": 1.5713260173797607,
      "debug/policy_rejected_logps": -5.900942802429199,
      "debug/reference_chosen_logps": -228.1366729736328,
      "debug/reference_rejected_logps": -5.889040946960449,
      "debug/sppo_chosen_loss": 2360.619873046875,
      "debug/sppo_chosen_reward_in_loss": 1.8356482982635498,
      "debug/sppo_rej_reward_in_loss": -0.011901888065040112,
      "debug/sppo_reject_loss": 2498.83056640625,
      "epoch": 0.5253623188405797,
      "grad_norm": 56481.61321339093,
      "learning_rate": 8.471864891814303e-08,
      "logits/chosen": 1.2483991384506226,
      "logits/rejected": 1.5713260173797607,
      "logps/chosen": -226.301025390625,
      "logps/rejected": -5.900942802429199,
      "loss": 4990.75,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.018356481567025185,
      "rewards/margins": 0.018475499004125595,
      "rewards/rejected": -0.00011901887774001807,
      "step": 145
    },
    {
      "debug/policy_chosen_logits": 1.6154155731201172,
      "debug/policy_chosen_logps": -195.41567993164062,
      "debug/policy_rejected_logits": 2.077685594558716,
      "debug/policy_rejected_logps": -69.73979187011719,
      "debug/reference_chosen_logps": -198.35165405273438,
      "debug/reference_rejected_logps": -70.21018981933594,
      "debug/sppo_chosen_loss": 2386.256103515625,
      "debug/sppo_chosen_reward_in_loss": 2.9359681606292725,
      "debug/sppo_rej_reward_in_loss": 0.47039833664894104,
      "debug/sppo_reject_loss": 2554.950439453125,
      "epoch": 0.5434782608695652,
      "grad_norm": 23877.971889635744,
      "learning_rate": 8.13734503690426e-08,
      "logits/chosen": 1.6154155731201172,
      "logits/rejected": 2.077685594558716,
      "logps/chosen": -195.41567993164062,
      "logps/rejected": -69.73979187011719,
      "loss": 4975.2414,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.02935967966914177,
      "rewards/margins": 0.024655699729919434,
      "rewards/rejected": 0.004703983664512634,
      "step": 150
    },
    {
      "debug/policy_chosen_logits": 1.3934733867645264,
      "debug/policy_chosen_logps": -186.84793090820312,
      "debug/policy_rejected_logits": 1.7428455352783203,
      "debug/policy_rejected_logps": -11.295183181762695,
      "debug/reference_chosen_logps": -188.67599487304688,
      "debug/reference_rejected_logps": -11.309015274047852,
      "debug/sppo_chosen_loss": 2406.025634765625,
      "debug/sppo_chosen_reward_in_loss": 1.828051209449768,
      "debug/sppo_rej_reward_in_loss": 0.013833016157150269,
      "debug/sppo_reject_loss": 2501.45263671875,
      "epoch": 0.5615942028985508,
      "grad_norm": 49089.76331500084,
      "learning_rate": 7.77785116509801e-08,
      "logits/chosen": 1.3934733867645264,
      "logits/rejected": 1.7428455352783203,
      "logps/chosen": -186.84793090820312,
      "logps/rejected": -11.295183181762695,
      "loss": 4959.9672,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": 0.0182805135846138,
      "rewards/margins": 0.018142182379961014,
      "rewards/rejected": 0.00013833014236297458,
      "step": 155
    },
    {
      "debug/policy_chosen_logits": 1.1241214275360107,
      "debug/policy_chosen_logps": -41.29949951171875,
      "debug/policy_rejected_logits": 1.587595820426941,
      "debug/policy_rejected_logps": -7.305385589599609,
      "debug/reference_chosen_logps": -41.46500778198242,
      "debug/reference_rejected_logps": -7.236231803894043,
      "debug/sppo_chosen_loss": 2484.546875,
      "debug/sppo_chosen_reward_in_loss": 0.16550657153129578,
      "debug/sppo_rej_reward_in_loss": -0.06915402412414551,
      "debug/sppo_reject_loss": 2493.12939453125,
      "epoch": 0.5797101449275363,
      "grad_norm": 14605.8114006113,
      "learning_rate": 7.396244933600284e-08,
      "logits/chosen": 1.1241214275360107,
      "logits/rejected": 1.587595820426941,
      "logps/chosen": -41.29949951171875,
      "logps/rejected": -7.305385589599609,
      "loss": 4964.507,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.0016550656873732805,
      "rewards/margins": 0.002346605993807316,
      "rewards/rejected": -0.0006915402482263744,
      "step": 160
    },
    {
      "debug/policy_chosen_logits": 1.6341183185577393,
      "debug/policy_chosen_logps": -46.92511749267578,
      "debug/policy_rejected_logits": 1.4523030519485474,
      "debug/policy_rejected_logps": -26.32564926147461,
      "debug/reference_chosen_logps": -47.207061767578125,
      "debug/reference_rejected_logps": -26.676687240600586,
      "debug/sppo_chosen_loss": 2474.646484375,
      "debug/sppo_chosen_reward_in_loss": 0.28194811940193176,
      "debug/sppo_rej_reward_in_loss": 0.35103607177734375,
      "debug/sppo_reject_loss": 2538.4462890625,
      "epoch": 0.5978260869565217,
      "grad_norm": 36718.92822244079,
      "learning_rate": 6.995564019287869e-08,
      "logits/chosen": 1.6341183185577393,
      "logits/rejected": 1.4523030519485474,
      "logps/chosen": -46.92511749267578,
      "logps/rejected": -26.32564926147461,
      "loss": 5053.6191,
      "rewards/accuracies": 0.125,
      "rewards/chosen": 0.002819481072947383,
      "rewards/margins": -0.0006908794166520238,
      "rewards/rejected": 0.003510360838845372,
      "step": 165
    },
    {
      "debug/policy_chosen_logits": 1.3338284492492676,
      "debug/policy_chosen_logps": -49.61326599121094,
      "debug/policy_rejected_logits": 1.6426509618759155,
      "debug/policy_rejected_logps": -6.317404270172119,
      "debug/reference_chosen_logps": -49.824684143066406,
      "debug/reference_rejected_logps": -6.327885627746582,
      "debug/sppo_chosen_loss": 2480.391845703125,
      "debug/sppo_chosen_reward_in_loss": 0.2114112377166748,
      "debug/sppo_rej_reward_in_loss": 0.010481464676558971,
      "debug/sppo_reject_loss": 2501.14501953125,
      "epoch": 0.6159420289855072,
      "grad_norm": 44446.380970079495,
      "learning_rate": 6.578997938075125e-08,
      "logits/chosen": 1.3338284492492676,
      "logits/rejected": 1.6426509618759155,
      "logps/chosen": -49.61326599121094,
      "logps/rejected": -6.317404270172119,
      "loss": 4995.0012,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.0021141120232641697,
      "rewards/margins": 0.0020092972554266453,
      "rewards/rejected": 0.00010481465142220259,
      "step": 170
    },
    {
      "debug/policy_chosen_logits": 1.471964716911316,
      "debug/policy_chosen_logps": -11.514124870300293,
      "debug/policy_rejected_logits": 1.9099857807159424,
      "debug/policy_rejected_logps": -112.86039733886719,
      "debug/reference_chosen_logps": -11.554305076599121,
      "debug/reference_rejected_logps": -113.70204162597656,
      "debug/sppo_chosen_loss": 2496.02197265625,
      "debug/sppo_chosen_reward_in_loss": 0.04018024355173111,
      "debug/sppo_rej_reward_in_loss": 0.8416390419006348,
      "debug/sppo_reject_loss": 2610.68212890625,
      "epoch": 0.6340579710144928,
      "grad_norm": 135545.85245016703,
      "learning_rate": 6.149862655604404e-08,
      "logits/chosen": 1.471964716911316,
      "logits/rejected": 1.9099857807159424,
      "logps/chosen": -11.514124870300293,
      "logps/rejected": -112.86039733886719,
      "loss": 4998.809,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00040180241921916604,
      "rewards/margins": -0.008014587685465813,
      "rewards/rejected": 0.008416390046477318,
      "step": 175
    },
    {
      "debug/policy_chosen_logits": 1.2563765048980713,
      "debug/policy_chosen_logps": -6.484707832336426,
      "debug/policy_rejected_logits": 1.6219933032989502,
      "debug/policy_rejected_logps": -11.174871444702148,
      "debug/reference_chosen_logps": -6.455367088317871,
      "debug/reference_rejected_logps": -11.164003372192383,
      "debug/sppo_chosen_loss": 2502.980712890625,
      "debug/sppo_chosen_reward_in_loss": -0.029340500012040138,
      "debug/sppo_rej_reward_in_loss": -0.010869455523788929,
      "debug/sppo_reject_loss": 2498.97802734375,
      "epoch": 0.6521739130434783,
      "grad_norm": 164996.56785625734,
      "learning_rate": 5.711574191366426e-08,
      "logits/chosen": 1.2563765048980713,
      "logits/rejected": 1.6219933032989502,
      "logps/chosen": -6.484707832336426,
      "logps/rejected": -11.174871444702148,
      "loss": 5019.5977,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": -0.0002934050280600786,
      "rewards/margins": -0.0001847104576881975,
      "rewards/rejected": -0.00010869452671613544,
      "step": 180
    },
    {
      "debug/policy_chosen_logits": 1.2139984369277954,
      "debug/policy_chosen_logps": -7.972718715667725,
      "debug/policy_rejected_logits": 1.6660735607147217,
      "debug/policy_rejected_logps": -15.723217964172363,
      "debug/reference_chosen_logps": -7.793820858001709,
      "debug/reference_rejected_logps": -15.452303886413574,
      "debug/sppo_chosen_loss": 2518.21337890625,
      "debug/sppo_chosen_reward_in_loss": -0.17889812588691711,
      "debug/sppo_rej_reward_in_loss": -0.2709135413169861,
      "debug/sppo_reject_loss": 2473.35009765625,
      "epoch": 0.6702898550724637,
      "grad_norm": 18624.21256617316,
      "learning_rate": 5.267621426368075e-08,
      "logits/chosen": 1.2139984369277954,
      "logits/rejected": 1.6660735607147217,
      "logps/chosen": -7.972718715667725,
      "logps/rejected": -15.723217964172363,
      "loss": 5039.2461,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": -0.0017889812588691711,
      "rewards/margins": 0.0009201543289236724,
      "rewards/rejected": -0.002709135413169861,
      "step": 185
    },
    {
      "debug/policy_chosen_logits": 1.2351247072219849,
      "debug/policy_chosen_logps": -55.260719299316406,
      "debug/policy_rejected_logits": 1.3063384294509888,
      "debug/policy_rejected_logps": -3.657384157180786,
      "debug/reference_chosen_logps": -55.711204528808594,
      "debug/reference_rejected_logps": -3.6667861938476562,
      "debug/sppo_chosen_loss": 2464.06689453125,
      "debug/sppo_chosen_reward_in_loss": 0.45049649477005005,
      "debug/sppo_rej_reward_in_loss": 0.009402072057127953,
      "debug/sppo_reject_loss": 2500.978271484375,
      "epoch": 0.6884057971014492,
      "grad_norm": 59339.47300560303,
      "learning_rate": 4.821538330805098e-08,
      "logits/chosen": 1.2351247072219849,
      "logits/rejected": 1.3063384294509888,
      "logps/chosen": -55.260719299316406,
      "logps/rejected": -3.657384157180786,
      "loss": 5023.3609,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.004504963755607605,
      "rewards/margins": 0.004410943016409874,
      "rewards/rejected": 9.402072464581579e-05,
      "step": 190
    },
    {
      "debug/policy_chosen_logits": 1.796984076499939,
      "debug/policy_chosen_logps": -100.9161605834961,
      "debug/policy_rejected_logits": 2.164973020553589,
      "debug/policy_rejected_logps": -83.23567199707031,
      "debug/reference_chosen_logps": -101.08064270019531,
      "debug/reference_rejected_logps": -83.28086853027344,
      "debug/sppo_chosen_loss": 2484.186767578125,
      "debug/sppo_chosen_reward_in_loss": 0.164472296833992,
      "debug/sppo_rej_reward_in_loss": 0.04519999027252197,
      "debug/sppo_reject_loss": 2504.991943359375,
      "epoch": 0.7065217391304348,
      "grad_norm": 55163.46876155982,
      "learning_rate": 4.3768758328140705e-08,
      "logits/chosen": 1.796984076499939,
      "logits/rejected": 2.164973020553589,
      "logps/chosen": -100.9161605834961,
      "logps/rejected": -83.23567199707031,
      "loss": 4972.7703,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.0016447228845208883,
      "rewards/margins": 0.0011927229352295399,
      "rewards/rejected": 0.00045199989108368754,
      "step": 195
    },
    {
      "debug/policy_chosen_logits": 1.2561403512954712,
      "debug/policy_chosen_logps": -4.826408863067627,
      "debug/policy_rejected_logits": 1.572509765625,
      "debug/policy_rejected_logps": -52.822608947753906,
      "debug/reference_chosen_logps": -4.822422027587891,
      "debug/reference_rejected_logps": -53.10357666015625,
      "debug/sppo_chosen_loss": 2500.411865234375,
      "debug/sppo_chosen_reward_in_loss": -0.003986963536590338,
      "debug/sppo_rej_reward_in_loss": 0.28096264600753784,
      "debug/sppo_reject_loss": 2531.43603515625,
      "epoch": 0.7246376811594203,
      "grad_norm": 73770.12364262494,
      "learning_rate": 3.9371735522351165e-08,
      "logits/chosen": 1.2561403512954712,
      "logits/rejected": 1.572509765625,
      "logps/chosen": -4.826408863067627,
      "logps/rejected": -52.822608947753906,
      "loss": 5015.0652,
      "rewards/accuracies": 0.22499999403953552,
      "rewards/chosen": -3.986962838098407e-05,
      "rewards/margins": -0.0028494957368820906,
      "rewards/rejected": 0.002809626515954733,
      "step": 200
    },
    {
      "epoch": 0.7246376811594203,
      "eval_debug/policy_chosen_logits": 1.6680212020874023,
      "eval_debug/policy_chosen_logps": -121.2494125366211,
      "eval_debug/policy_rejected_logits": 1.7270073890686035,
      "eval_debug/policy_rejected_logps": -63.196250915527344,
      "eval_debug/reference_chosen_logps": -123.14806365966797,
      "eval_debug/reference_rejected_logps": -63.887054443359375,
      "eval_debug/sppo_chosen_loss": 2390.288330078125,
      "eval_debug/sppo_chosen_reward_in_loss": 1.8986449241638184,
      "eval_debug/sppo_rej_reward_in_loss": 0.6908064484596252,
      "eval_debug/sppo_reject_loss": 2606.55419921875,
      "eval_logits/chosen": 1.6680212020874023,
      "eval_logits/rejected": 1.7270073890686035,
      "eval_logps/chosen": -121.2494125366211,
      "eval_logps/rejected": -63.196250915527344,
      "eval_loss": 5015.12939453125,
      "eval_rewards/accuracies": 0.2763157784938812,
      "eval_rewards/chosen": 0.018986448645591736,
      "eval_rewards/margins": 0.012078385800123215,
      "eval_rewards/rejected": 0.006908063776791096,
      "eval_runtime": 28.6889,
      "eval_samples_per_second": 20.914,
      "eval_steps_per_second": 0.662,
      "step": 200
    },
    {
      "debug/policy_chosen_logits": 1.3723523616790771,
      "debug/policy_chosen_logps": -4.869706153869629,
      "debug/policy_rejected_logits": 1.7145782709121704,
      "debug/policy_rejected_logps": -85.27532196044922,
      "debug/reference_chosen_logps": -4.833613872528076,
      "debug/reference_rejected_logps": -85.68836212158203,
      "debug/sppo_chosen_loss": 2503.64306640625,
      "debug/sppo_chosen_reward_in_loss": -0.03609180450439453,
      "debug/sppo_rej_reward_in_loss": 0.4130396842956543,
      "debug/sppo_reject_loss": 2549.91455078125,
      "epoch": 0.7427536231884058,
      "grad_norm": 167073.79540945674,
      "learning_rate": 3.505931624391355e-08,
      "logits/chosen": 1.3723523616790771,
      "logits/rejected": 1.7145782709121704,
      "logps/chosen": -4.869706153869629,
      "logps/rejected": -85.27532196044922,
      "loss": 5027.4824,
      "rewards/accuracies": 0.2750000059604645,
      "rewards/chosen": -0.0003609180566854775,
      "rewards/margins": -0.0044913142919540405,
      "rewards/rejected": 0.004130396526306868,
      "step": 205
    },
    {
      "debug/policy_chosen_logits": 1.2046802043914795,
      "debug/policy_chosen_logps": -63.47414016723633,
      "debug/policy_rejected_logits": 1.5040717124938965,
      "debug/policy_rejected_logps": -45.39781951904297,
      "debug/reference_chosen_logps": -63.378448486328125,
      "debug/reference_rejected_logps": -45.54474639892578,
      "debug/sppo_chosen_loss": 2509.64892578125,
      "debug/sppo_chosen_reward_in_loss": -0.0956823080778122,
      "debug/sppo_rej_reward_in_loss": 0.14692507684230804,
      "debug/sppo_reject_loss": 2515.404296875,
      "epoch": 0.7608695652173914,
      "grad_norm": 73818.32992301855,
      "learning_rate": 3.086582838174551e-08,
      "logits/chosen": 1.2046802043914795,
      "logits/rejected": 1.5040717124938965,
      "logps/chosen": -63.47414016723633,
      "logps/rejected": -45.39781951904297,
      "loss": 5026.3219,
      "rewards/accuracies": 0.07500000298023224,
      "rewards/chosen": -0.0009568231180310249,
      "rewards/margins": -0.0024260736536234617,
      "rewards/rejected": 0.0014692506520077586,
      "step": 210
    },
    {
      "debug/policy_chosen_logits": 1.4869812726974487,
      "debug/policy_chosen_logps": -46.38404083251953,
      "debug/policy_rejected_logits": 1.8799896240234375,
      "debug/policy_rejected_logps": -7.943442344665527,
      "debug/reference_chosen_logps": -46.571571350097656,
      "debug/reference_rejected_logps": -7.923023223876953,
      "debug/sppo_chosen_loss": 2482.71826171875,
      "debug/sppo_chosen_reward_in_loss": 0.18753060698509216,
      "debug/sppo_rej_reward_in_loss": -0.020419389009475708,
      "debug/sppo_reject_loss": 2498.00537109375,
      "epoch": 0.7789855072463768,
      "grad_norm": 30997.36116841476,
      "learning_rate": 2.6824653102244727e-08,
      "logits/chosen": 1.4869812726974487,
      "logits/rejected": 1.8799896240234375,
      "logps/chosen": -46.38404083251953,
      "logps/rejected": -7.943442344665527,
      "loss": 4958.6598,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": 0.0018753058975562453,
      "rewards/margins": 0.002079499652609229,
      "rewards/rejected": -0.0002041938714683056,
      "step": 215
    },
    {
      "debug/policy_chosen_logits": 0.9879446029663086,
      "debug/policy_chosen_logps": -26.743267059326172,
      "debug/policy_rejected_logits": 1.3120319843292236,
      "debug/policy_rejected_logps": -10.141648292541504,
      "debug/reference_chosen_logps": -26.971553802490234,
      "debug/reference_rejected_logps": -10.127599716186523,
      "debug/sppo_chosen_loss": 2479.09375,
      "debug/sppo_chosen_reward_in_loss": 0.2282869815826416,
      "debug/sppo_rej_reward_in_loss": -0.01404793281108141,
      "debug/sppo_reject_loss": 2498.64404296875,
      "epoch": 0.7971014492753623,
      "grad_norm": 26802.126181328487,
      "learning_rate": 2.2967959127220138e-08,
      "logits/chosen": 0.9879446029663086,
      "logits/rejected": 1.3120319843292236,
      "logps/chosen": -26.743267059326172,
      "logps/rejected": -10.141648292541504,
      "loss": 5009.0125,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.0022828695364296436,
      "rewards/margins": 0.002423349069431424,
      "rewards/rejected": -0.00014047934382688254,
      "step": 220
    },
    {
      "debug/policy_chosen_logits": 1.3579328060150146,
      "debug/policy_chosen_logps": -7.486691474914551,
      "debug/policy_rejected_logits": 1.647939920425415,
      "debug/policy_rejected_logps": -117.33280181884766,
      "debug/reference_chosen_logps": -7.492678165435791,
      "debug/reference_rejected_logps": -117.96882629394531,
      "debug/sppo_chosen_loss": 2499.44482421875,
      "debug/sppo_chosen_reward_in_loss": 0.005986439995467663,
      "debug/sppo_rej_reward_in_loss": 0.6360327005386353,
      "debug/sppo_reject_loss": 2580.13330078125,
      "epoch": 0.8152173913043478,
      "grad_norm": 31288.105349492,
      "learning_rate": 1.9326446663172035e-08,
      "logits/chosen": 1.3579328060150146,
      "logits/rejected": 1.647939920425415,
      "logps/chosen": -7.486691474914551,
      "logps/rejected": -117.33280181884766,
      "loss": 5015.4312,
      "rewards/accuracies": 0.2750000059604645,
      "rewards/chosen": 5.9864407376153395e-05,
      "rewards/margins": -0.006300461478531361,
      "rewards/rejected": 0.006360325962305069,
      "step": 225
    },
    {
      "debug/policy_chosen_logits": 1.215022325515747,
      "debug/policy_chosen_logps": -44.3668327331543,
      "debug/policy_rejected_logits": 1.4289997816085815,
      "debug/policy_rejected_logps": -9.595197677612305,
      "debug/reference_chosen_logps": -44.462120056152344,
      "debug/reference_rejected_logps": -9.55119514465332,
      "debug/sppo_chosen_loss": 2491.029052734375,
      "debug/sppo_chosen_reward_in_loss": 0.09528430551290512,
      "debug/sppo_rej_reward_in_loss": -0.04400312155485153,
      "debug/sppo_reject_loss": 2495.680419921875,
      "epoch": 0.8333333333333334,
      "grad_norm": 116537.70488652344,
      "learning_rate": 1.592910302030544e-08,
      "logits/chosen": 1.215022325515747,
      "logits/rejected": 1.4289997816085815,
      "logps/chosen": -44.3668327331543,
      "logps/rejected": -9.595197677612305,
      "loss": 4982.9453,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": 0.0009528429945930839,
      "rewards/margins": 0.0013928741682320833,
      "rewards/rejected": -0.0004400312027428299,
      "step": 230
    },
    {
      "debug/policy_chosen_logits": 1.398555040359497,
      "debug/policy_chosen_logps": -3.825951337814331,
      "debug/policy_rejected_logits": 1.5684314966201782,
      "debug/policy_rejected_logps": -11.321913719177246,
      "debug/reference_chosen_logps": -3.7865378856658936,
      "debug/reference_rejected_logps": -11.303999900817871,
      "debug/sppo_chosen_loss": 2503.960205078125,
      "debug/sppo_chosen_reward_in_loss": -0.039413463324308395,
      "debug/sppo_rej_reward_in_loss": -0.01791282929480076,
      "debug/sppo_reject_loss": 2498.244873046875,
      "epoch": 0.8514492753623188,
      "grad_norm": 118001.4885666594,
      "learning_rate": 1.2802971866607521e-08,
      "logits/chosen": 1.398555040359497,
      "logits/rejected": 1.5684314966201782,
      "logps/chosen": -3.825951337814331,
      "logps/rejected": -11.321913719177246,
      "loss": 4981.2648,
      "rewards/accuracies": 0.22499999403953552,
      "rewards/chosen": -0.0003941346367355436,
      "rewards/margins": -0.00021500633738469332,
      "rewards/rejected": -0.00017912828479893506,
      "step": 235
    },
    {
      "debug/policy_chosen_logits": 1.3944571018218994,
      "debug/policy_chosen_logps": -3.6208109855651855,
      "debug/policy_rejected_logits": 1.9865443706512451,
      "debug/policy_rejected_logps": -36.188865661621094,
      "debug/reference_chosen_logps": -3.5993995666503906,
      "debug/reference_rejected_logps": -36.39490509033203,
      "debug/sppo_chosen_loss": 2502.203857421875,
      "debug/sppo_chosen_reward_in_loss": -0.021411290392279625,
      "debug/sppo_rej_reward_in_loss": 0.20603516697883606,
      "debug/sppo_reject_loss": 2522.786376953125,
      "epoch": 0.8695652173913043,
      "grad_norm": 53766.29640847203,
      "learning_rate": 9.972937953781984e-09,
      "logits/chosen": 1.3944571018218994,
      "logits/rejected": 1.9865443706512451,
      "logps/chosen": -3.6208109855651855,
      "logps/rejected": -36.188865661621094,
      "loss": 5027.7012,
      "rewards/accuracies": 0.22499999403953552,
      "rewards/chosen": -0.00021411287889350206,
      "rewards/margins": -0.002274464350193739,
      "rewards/rejected": 0.002060351427644491,
      "step": 240
    },
    {
      "debug/policy_chosen_logits": 1.1183539628982544,
      "debug/policy_chosen_logps": -3.6978707313537598,
      "debug/policy_rejected_logits": 1.46969473361969,
      "debug/policy_rejected_logps": -170.7615203857422,
      "debug/reference_chosen_logps": -3.6889584064483643,
      "debug/reference_rejected_logps": -171.53671264648438,
      "debug/sppo_chosen_loss": 2500.911865234375,
      "debug/sppo_chosen_reward_in_loss": -0.008911940269172192,
      "debug/sppo_rej_reward_in_loss": 0.775180459022522,
      "debug/sppo_reject_loss": 2618.15576171875,
      "epoch": 0.8876811594202898,
      "grad_norm": 51804.200611742,
      "learning_rate": 7.461529028673463e-09,
      "logits/chosen": 1.1183539628982544,
      "logits/rejected": 1.46969473361969,
      "logps/chosen": -3.6978707313537598,
      "logps/rejected": -170.7615203857422,
      "loss": 5000.7332,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": -8.911939949030057e-05,
      "rewards/margins": -0.007840922102332115,
      "rewards/rejected": 0.00775180384516716,
      "step": 245
    },
    {
      "debug/policy_chosen_logits": 1.2443785667419434,
      "debug/policy_chosen_logps": -5.438248157501221,
      "debug/policy_rejected_logits": 1.60662043094635,
      "debug/policy_rejected_logps": -4.876561641693115,
      "debug/reference_chosen_logps": -5.442919731140137,
      "debug/reference_rejected_logps": -4.868186950683594,
      "debug/sppo_chosen_loss": 2499.578857421875,
      "debug/sppo_chosen_reward_in_loss": 0.004670926835387945,
      "debug/sppo_rej_reward_in_loss": -0.008374800905585289,
      "debug/sppo_reject_loss": 2499.17724609375,
      "epoch": 0.9057971014492754,
      "grad_norm": 64595.0866483288,
      "learning_rate": 5.288736507014435e-09,
      "logits/chosen": 1.2443785667419434,
      "logits/rejected": 1.60662043094635,
      "logps/chosen": -5.438248157501221,
      "logps/rejected": -4.876561641693115,
      "loss": 5046.0918,
      "rewards/accuracies": 0.22499999403953552,
      "rewards/chosen": 4.6709272282896563e-05,
      "rewards/margins": 0.00013045726518612355,
      "rewards/rejected": -8.374801836907864e-05,
      "step": 250
    },
    {
      "debug/policy_chosen_logits": 1.4383130073547363,
      "debug/policy_chosen_logps": -6.456732273101807,
      "debug/policy_rejected_logits": 1.6063964366912842,
      "debug/policy_rejected_logps": -2.8695645332336426,
      "debug/reference_chosen_logps": -6.483206748962402,
      "debug/reference_rejected_logps": -2.824536085128784,
      "debug/sppo_chosen_loss": 2497.39501953125,
      "debug/sppo_chosen_reward_in_loss": 0.026474904268980026,
      "debug/sppo_rej_reward_in_loss": -0.045027971267700195,
      "debug/sppo_reject_loss": 2495.526611328125,
      "epoch": 0.9239130434782609,
      "grad_norm": 93120.71264243545,
      "learning_rate": 3.4718563369743213e-09,
      "logits/chosen": 1.4383130073547363,
      "logits/rejected": 1.6063964366912842,
      "logps/chosen": -6.456732273101807,
      "logps/rejected": -2.8695645332336426,
      "loss": 5003.5242,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.00026474904734641314,
      "rewards/margins": 0.0007150286692194641,
      "rewards/rejected": -0.0004502796509768814,
      "step": 255
    },
    {
      "debug/policy_chosen_logits": 1.3548212051391602,
      "debug/policy_chosen_logps": -67.59104919433594,
      "debug/policy_rejected_logits": 1.9228832721710205,
      "debug/policy_rejected_logps": -47.09478759765625,
      "debug/reference_chosen_logps": -67.80909729003906,
      "debug/reference_rejected_logps": -47.41716384887695,
      "debug/sppo_chosen_loss": 2479.32470703125,
      "debug/sppo_chosen_reward_in_loss": 0.21804814040660858,
      "debug/sppo_rej_reward_in_loss": 0.32237640023231506,
      "debug/sppo_reject_loss": 2537.328369140625,
      "epoch": 0.9420289855072463,
      "grad_norm": 235843.0142935536,
      "learning_rate": 2.025351319275137e-09,
      "logits/chosen": 1.3548212051391602,
      "logits/rejected": 1.9228832721710205,
      "logps/chosen": -67.59104919433594,
      "logps/rejected": -47.09478759765625,
      "loss": 4997.7383,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.0021804815623909235,
      "rewards/margins": -0.0010432822164148092,
      "rewards/rejected": 0.0032237637788057327,
      "step": 260
    },
    {
      "debug/policy_chosen_logits": 1.5696322917938232,
      "debug/policy_chosen_logps": -38.5196647644043,
      "debug/policy_rejected_logits": 1.7193692922592163,
      "debug/policy_rejected_logps": -131.51307678222656,
      "debug/reference_chosen_logps": -38.708030700683594,
      "debug/reference_rejected_logps": -132.49644470214844,
      "debug/sppo_chosen_loss": 2481.755859375,
      "debug/sppo_chosen_reward_in_loss": 0.18836654722690582,
      "debug/sppo_rej_reward_in_loss": 0.9833652377128601,
      "debug/sppo_reject_loss": 2613.89111328125,
      "epoch": 0.9601449275362319,
      "grad_norm": 42745.49236462931,
      "learning_rate": 9.607359798384785e-10,
      "logits/chosen": 1.5696322917938232,
      "logits/rejected": 1.7193692922592163,
      "logps/chosen": -38.5196647644043,
      "logps/rejected": -131.51307678222656,
      "loss": 4985.8559,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.001883665332570672,
      "rewards/margins": -0.00794998649507761,
      "rewards/rejected": 0.009833652526140213,
      "step": 265
    },
    {
      "debug/policy_chosen_logits": 1.5161726474761963,
      "debug/policy_chosen_logps": -5.8967390060424805,
      "debug/policy_rejected_logits": 2.0397534370422363,
      "debug/policy_rejected_logps": -3.5706443786621094,
      "debug/reference_chosen_logps": -5.859536170959473,
      "debug/reference_rejected_logps": -3.5223605632781982,
      "debug/sppo_chosen_loss": 2503.755859375,
      "debug/sppo_chosen_reward_in_loss": -0.03720256686210632,
      "debug/sppo_rej_reward_in_loss": -0.04828394576907158,
      "debug/sppo_reject_loss": 2495.21142578125,
      "epoch": 0.9782608695652174,
      "grad_norm": 38783.68553474358,
      "learning_rate": 2.8648491140513267e-10,
      "logits/chosen": 1.5161726474761963,
      "logits/rejected": 2.0397534370422363,
      "logps/chosen": -5.8967390060424805,
      "logps/rejected": -3.5706443786621094,
      "loss": 4985.3109,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.00037202564999461174,
      "rewards/margins": 0.00011081376578658819,
      "rewards/rejected": -0.0004828394448850304,
      "step": 270
    },
    {
      "debug/policy_chosen_logits": 1.1615492105484009,
      "debug/policy_chosen_logps": -4.200676918029785,
      "debug/policy_rejected_logits": 1.4300708770751953,
      "debug/policy_rejected_logps": -11.9641752243042,
      "debug/reference_chosen_logps": -4.2250447273254395,
      "debug/reference_rejected_logps": -11.967859268188477,
      "debug/sppo_chosen_loss": 2497.57421875,
      "debug/sppo_chosen_reward_in_loss": 0.024368364363908768,
      "debug/sppo_rej_reward_in_loss": 0.003684788942337036,
      "debug/sppo_reject_loss": 2500.45068359375,
      "epoch": 0.9963768115942029,
      "grad_norm": 117686.2605109664,
      "learning_rate": 7.965313749835889e-12,
      "logits/chosen": 1.1615492105484009,
      "logits/rejected": 1.4300708770751953,
      "logps/chosen": -4.200676918029785,
      "logps/rejected": -11.9641752243042,
      "loss": 5020.2773,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.00024368362210225314,
      "rewards/margins": 0.00020683575712610036,
      "rewards/rejected": 3.6847894079983234e-05,
      "step": 275
    },
    {
      "epoch": 1.0,
      "step": 276,
      "total_flos": 0.0,
      "train_loss": 4999.9289533089905,
      "train_runtime": 2038.5346,
      "train_samples_per_second": 8.646,
      "train_steps_per_second": 0.135
    }
  ],
  "logging_steps": 5,
  "max_steps": 276,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}