| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 100, | |
| "global_step": 276, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "debug/policy_chosen_logits": 0.8079685568809509, | |
| "debug/policy_chosen_logps": -5.034485816955566, | |
| "debug/policy_rejected_logits": 0.6268295645713806, | |
| "debug/policy_rejected_logps": -2.0584616661071777, | |
| "debug/reference_chosen_logps": -5.034485816955566, | |
| "debug/reference_rejected_logps": -2.0584616661071777, | |
| "debug/sppo_chosen_loss": 2500.0, | |
| "debug/sppo_chosen_reward_in_loss": 0.0, | |
| "debug/sppo_rej_reward_in_loss": 0.0, | |
| "debug/sppo_reject_loss": 2500.0, | |
| "epoch": 0.0036231884057971015, | |
| "grad_norm": 47836.56654178361, | |
| "learning_rate": 1e-09, | |
| "logits/chosen": 0.8079685568809509, | |
| "logits/rejected": 0.6268295645713806, | |
| "logps/chosen": -5.034485816955566, | |
| "logps/rejected": -2.0584616661071777, | |
| "loss": 5000.0, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.1308157444000244, | |
| "debug/policy_chosen_logps": -5.128948211669922, | |
| "debug/policy_rejected_logits": 1.3690780401229858, | |
| "debug/policy_rejected_logps": -47.817047119140625, | |
| "debug/reference_chosen_logps": -5.141074180603027, | |
| "debug/reference_rejected_logps": -47.71269607543945, | |
| "debug/sppo_chosen_loss": 2498.794189453125, | |
| "debug/sppo_chosen_reward_in_loss": 0.012125834822654724, | |
| "debug/sppo_rej_reward_in_loss": -0.10434922575950623, | |
| "debug/sppo_reject_loss": 2489.93359375, | |
| "epoch": 0.018115942028985508, | |
| "grad_norm": 83041.00650934623, | |
| "learning_rate": 5e-09, | |
| "logits/chosen": 1.1308157444000244, | |
| "logits/rejected": 1.3690780401229858, | |
| "logps/chosen": -5.128948211669922, | |
| "logps/rejected": -47.817047119140625, | |
| "loss": 4999.4365, | |
| "rewards/accuracies": 0.15625, | |
| "rewards/chosen": 0.00012125834473408759, | |
| "rewards/margins": 0.0011647505452856421, | |
| "rewards/rejected": -0.0010434923460707068, | |
| "step": 5 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3401418924331665, | |
| "debug/policy_chosen_logps": -37.56370544433594, | |
| "debug/policy_rejected_logits": 1.6091701984405518, | |
| "debug/policy_rejected_logps": -8.9139404296875, | |
| "debug/reference_chosen_logps": -37.657630920410156, | |
| "debug/reference_rejected_logps": -8.866057395935059, | |
| "debug/sppo_chosen_loss": 2490.720458984375, | |
| "debug/sppo_chosen_reward_in_loss": 0.09392313659191132, | |
| "debug/sppo_rej_reward_in_loss": -0.04788307100534439, | |
| "debug/sppo_reject_loss": 2495.249755859375, | |
| "epoch": 0.036231884057971016, | |
| "grad_norm": 25854.702147472322, | |
| "learning_rate": 1e-08, | |
| "logits/chosen": 1.3401418924331665, | |
| "logits/rejected": 1.6091701984405518, | |
| "logps/chosen": -37.56370544433594, | |
| "logps/rejected": -8.9139404296875, | |
| "loss": 4998.7039, | |
| "rewards/accuracies": 0.42500001192092896, | |
| "rewards/chosen": 0.0009392313659191132, | |
| "rewards/margins": 0.0014180620200932026, | |
| "rewards/rejected": -0.00047883071238175035, | |
| "step": 10 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.281254768371582, | |
| "debug/policy_chosen_logps": -7.3922319412231445, | |
| "debug/policy_rejected_logits": 1.6363356113433838, | |
| "debug/policy_rejected_logps": -5.406702518463135, | |
| "debug/reference_chosen_logps": -7.438336372375488, | |
| "debug/reference_rejected_logps": -5.3998122215271, | |
| "debug/sppo_chosen_loss": 2495.47998046875, | |
| "debug/sppo_chosen_reward_in_loss": 0.046104758977890015, | |
| "debug/sppo_rej_reward_in_loss": -0.006889957003295422, | |
| "debug/sppo_reject_loss": 2499.34228515625, | |
| "epoch": 0.05434782608695652, | |
| "grad_norm": 52134.81661267604, | |
| "learning_rate": 1.5e-08, | |
| "logits/chosen": 1.281254768371582, | |
| "logits/rejected": 1.6363356113433838, | |
| "logps/chosen": -7.3922319412231445, | |
| "logps/rejected": -5.406702518463135, | |
| "loss": 4994.4305, | |
| "rewards/accuracies": 0.2750000059604645, | |
| "rewards/chosen": 0.0004610475734807551, | |
| "rewards/margins": 0.0005299471667967737, | |
| "rewards/rejected": -6.889957876410335e-05, | |
| "step": 15 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.2531625032424927, | |
| "debug/policy_chosen_logps": -4.177506446838379, | |
| "debug/policy_rejected_logits": 1.600624680519104, | |
| "debug/policy_rejected_logps": -88.92843627929688, | |
| "debug/reference_chosen_logps": -4.198297023773193, | |
| "debug/reference_rejected_logps": -88.761962890625, | |
| "debug/sppo_chosen_loss": 2497.947021484375, | |
| "debug/sppo_chosen_reward_in_loss": 0.020790796726942062, | |
| "debug/sppo_rej_reward_in_loss": -0.16647538542747498, | |
| "debug/sppo_reject_loss": 2484.888427734375, | |
| "epoch": 0.07246376811594203, | |
| "grad_norm": 21278.15413555925, | |
| "learning_rate": 2e-08, | |
| "logits/chosen": 1.2531625032424927, | |
| "logits/rejected": 1.600624680519104, | |
| "logps/chosen": -4.177506446838379, | |
| "logps/rejected": -88.92843627929688, | |
| "loss": 5001.0859, | |
| "rewards/accuracies": 0.22499999403953552, | |
| "rewards/chosen": 0.00020790801499970257, | |
| "rewards/margins": 0.001872661872766912, | |
| "rewards/rejected": -0.001664753770455718, | |
| "step": 20 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3284282684326172, | |
| "debug/policy_chosen_logps": -209.1931610107422, | |
| "debug/policy_rejected_logits": 1.7004191875457764, | |
| "debug/policy_rejected_logps": -268.02142333984375, | |
| "debug/reference_chosen_logps": -208.846923828125, | |
| "debug/reference_rejected_logps": -267.5577697753906, | |
| "debug/sppo_chosen_loss": 2539.96240234375, | |
| "debug/sppo_chosen_reward_in_loss": -0.3462248742580414, | |
| "debug/sppo_rej_reward_in_loss": -0.4636896550655365, | |
| "debug/sppo_reject_loss": 2462.03173828125, | |
| "epoch": 0.09057971014492754, | |
| "grad_norm": 18939.799089042473, | |
| "learning_rate": 2.5e-08, | |
| "logits/chosen": 1.3284282684326172, | |
| "logits/rejected": 1.7004191875457764, | |
| "logps/chosen": -209.1931610107422, | |
| "logps/rejected": -268.02142333984375, | |
| "loss": 4993.6285, | |
| "rewards/accuracies": 0.32499998807907104, | |
| "rewards/chosen": -0.0034622489474713802, | |
| "rewards/margins": 0.0011746471282094717, | |
| "rewards/rejected": -0.004636895842850208, | |
| "step": 25 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.4937888383865356, | |
| "debug/policy_chosen_logps": -61.337852478027344, | |
| "debug/policy_rejected_logits": 1.8404518365859985, | |
| "debug/policy_rejected_logps": -5.3695268630981445, | |
| "debug/reference_chosen_logps": -61.356834411621094, | |
| "debug/reference_rejected_logps": -5.380406379699707, | |
| "debug/sppo_chosen_loss": 2498.155517578125, | |
| "debug/sppo_chosen_reward_in_loss": 0.01897408999502659, | |
| "debug/sppo_rej_reward_in_loss": 0.010879549197852612, | |
| "debug/sppo_reject_loss": 2501.10498046875, | |
| "epoch": 0.10869565217391304, | |
| "grad_norm": 37304.285865467384, | |
| "learning_rate": 3e-08, | |
| "logits/chosen": 1.4937888383865356, | |
| "logits/rejected": 1.8404518365859985, | |
| "logps/chosen": -61.337852478027344, | |
| "logps/rejected": -5.3695268630981445, | |
| "loss": 4995.4516, | |
| "rewards/accuracies": 0.3499999940395355, | |
| "rewards/chosen": 0.00018974088015966117, | |
| "rewards/margins": 8.094539953162894e-05, | |
| "rewards/rejected": 0.00010879549517994747, | |
| "step": 30 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.5908656120300293, | |
| "debug/policy_chosen_logps": -67.03999328613281, | |
| "debug/policy_rejected_logits": 1.9137623310089111, | |
| "debug/policy_rejected_logps": -7.439389228820801, | |
| "debug/reference_chosen_logps": -67.15159606933594, | |
| "debug/reference_rejected_logps": -7.461578369140625, | |
| "debug/sppo_chosen_loss": 2489.117431640625, | |
| "debug/sppo_chosen_reward_in_loss": 0.11160683631896973, | |
| "debug/sppo_rej_reward_in_loss": 0.022188443690538406, | |
| "debug/sppo_reject_loss": 2502.24462890625, | |
| "epoch": 0.12681159420289856, | |
| "grad_norm": 80189.68931153638, | |
| "learning_rate": 3.4999999999999996e-08, | |
| "logits/chosen": 1.5908656120300293, | |
| "logits/rejected": 1.9137623310089111, | |
| "logps/chosen": -67.03999328613281, | |
| "logps/rejected": -7.439389228820801, | |
| "loss": 4996.5711, | |
| "rewards/accuracies": 0.3499999940395355, | |
| "rewards/chosen": 0.0011160684516653419, | |
| "rewards/margins": 0.0008941839332692325, | |
| "rewards/rejected": 0.00022188443108461797, | |
| "step": 35 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.4925166368484497, | |
| "debug/policy_chosen_logps": -90.91502380371094, | |
| "debug/policy_rejected_logits": 1.8190809488296509, | |
| "debug/policy_rejected_logps": -80.93954467773438, | |
| "debug/reference_chosen_logps": -90.82229614257812, | |
| "debug/reference_rejected_logps": -81.13166809082031, | |
| "debug/sppo_chosen_loss": 2509.4033203125, | |
| "debug/sppo_chosen_reward_in_loss": -0.09272497892379761, | |
| "debug/sppo_rej_reward_in_loss": 0.19212593138217926, | |
| "debug/sppo_reject_loss": 2519.924560546875, | |
| "epoch": 0.14492753623188406, | |
| "grad_norm": 101789.85983014172, | |
| "learning_rate": 4e-08, | |
| "logits/chosen": 1.4925166368484497, | |
| "logits/rejected": 1.8190809488296509, | |
| "logps/chosen": -90.91502380371094, | |
| "logps/rejected": -80.93954467773438, | |
| "loss": 5008.0297, | |
| "rewards/accuracies": 0.10000000149011612, | |
| "rewards/chosen": -0.0009272497263737023, | |
| "rewards/margins": -0.0028485090006142855, | |
| "rewards/rejected": 0.0019212592160329223, | |
| "step": 40 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.4444226026535034, | |
| "debug/policy_chosen_logps": -43.32951736450195, | |
| "debug/policy_rejected_logits": 1.6251853704452515, | |
| "debug/policy_rejected_logps": -7.5910234451293945, | |
| "debug/reference_chosen_logps": -43.420326232910156, | |
| "debug/reference_rejected_logps": -7.612727165222168, | |
| "debug/sppo_chosen_loss": 2491.11474609375, | |
| "debug/sppo_chosen_reward_in_loss": 0.09080849587917328, | |
| "debug/sppo_rej_reward_in_loss": 0.021703582257032394, | |
| "debug/sppo_reject_loss": 2502.208251953125, | |
| "epoch": 0.16304347826086957, | |
| "grad_norm": 127325.37231669381, | |
| "learning_rate": 4.5e-08, | |
| "logits/chosen": 1.4444226026535034, | |
| "logits/rejected": 1.6251853704452515, | |
| "logps/chosen": -43.32951736450195, | |
| "logps/rejected": -7.5910234451293945, | |
| "loss": 4992.1121, | |
| "rewards/accuracies": 0.2750000059604645, | |
| "rewards/chosen": 0.0009080851450562477, | |
| "rewards/margins": 0.0006910492666065693, | |
| "rewards/rejected": 0.00021703580569010228, | |
| "step": 45 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3567314147949219, | |
| "debug/policy_chosen_logps": -88.03765869140625, | |
| "debug/policy_rejected_logits": 1.7951405048370361, | |
| "debug/policy_rejected_logps": -3.4048943519592285, | |
| "debug/reference_chosen_logps": -88.64862823486328, | |
| "debug/reference_rejected_logps": -3.3678181171417236, | |
| "debug/sppo_chosen_loss": 2454.765380859375, | |
| "debug/sppo_chosen_reward_in_loss": 0.6109756231307983, | |
| "debug/sppo_rej_reward_in_loss": -0.03707638382911682, | |
| "debug/sppo_reject_loss": 2496.30859375, | |
| "epoch": 0.18115942028985507, | |
| "grad_norm": 58358.722910061, | |
| "learning_rate": 5e-08, | |
| "logits/chosen": 1.3567314147949219, | |
| "logits/rejected": 1.7951405048370361, | |
| "logps/chosen": -88.03765869140625, | |
| "logps/rejected": -3.4048943519592285, | |
| "loss": 4992.9215, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.006109756417572498, | |
| "rewards/margins": 0.006480520125478506, | |
| "rewards/rejected": -0.00037076379521749914, | |
| "step": 50 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3366315364837646, | |
| "debug/policy_chosen_logps": -11.845144271850586, | |
| "debug/policy_rejected_logits": 1.8784109354019165, | |
| "debug/policy_rejected_logps": -3.4788289070129395, | |
| "debug/reference_chosen_logps": -11.87784194946289, | |
| "debug/reference_rejected_logps": -3.4915835857391357, | |
| "debug/sppo_chosen_loss": 2496.794677734375, | |
| "debug/sppo_chosen_reward_in_loss": 0.03269771859049797, | |
| "debug/sppo_rej_reward_in_loss": 0.012755215167999268, | |
| "debug/sppo_reject_loss": 2501.293212890625, | |
| "epoch": 0.19927536231884058, | |
| "grad_norm": 60865.57057492795, | |
| "learning_rate": 5.5e-08, | |
| "logits/chosen": 1.3366315364837646, | |
| "logits/rejected": 1.8784109354019165, | |
| "logps/chosen": -11.845144271850586, | |
| "logps/rejected": -3.4788289070129395, | |
| "loss": 4988.9117, | |
| "rewards/accuracies": 0.30000001192092896, | |
| "rewards/chosen": 0.0003269771987106651, | |
| "rewards/margins": 0.00019942503422498703, | |
| "rewards/rejected": 0.00012755214993376285, | |
| "step": 55 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.532639741897583, | |
| "debug/policy_chosen_logps": -4.54892635345459, | |
| "debug/policy_rejected_logits": 1.8314745426177979, | |
| "debug/policy_rejected_logps": -8.05273151397705, | |
| "debug/reference_chosen_logps": -4.53155517578125, | |
| "debug/reference_rejected_logps": -8.040096282958984, | |
| "debug/sppo_chosen_loss": 2501.75537109375, | |
| "debug/sppo_chosen_reward_in_loss": -0.017371535301208496, | |
| "debug/sppo_rej_reward_in_loss": -0.01263415813446045, | |
| "debug/sppo_reject_loss": 2498.755126953125, | |
| "epoch": 0.21739130434782608, | |
| "grad_norm": 81182.53900123628, | |
| "learning_rate": 6e-08, | |
| "logits/chosen": 1.532639741897583, | |
| "logits/rejected": 1.8314745426177979, | |
| "logps/chosen": -4.54892635345459, | |
| "logps/rejected": -8.05273151397705, | |
| "loss": 5001.6219, | |
| "rewards/accuracies": 0.2750000059604645, | |
| "rewards/chosen": -0.00017371535068377852, | |
| "rewards/margins": -4.7373774577863514e-05, | |
| "rewards/rejected": -0.000126341576105915, | |
| "step": 60 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.488297939300537, | |
| "debug/policy_chosen_logps": -133.06114196777344, | |
| "debug/policy_rejected_logits": 1.8658514022827148, | |
| "debug/policy_rejected_logps": -12.593046188354492, | |
| "debug/reference_chosen_logps": -133.0397491455078, | |
| "debug/reference_rejected_logps": -12.575475692749023, | |
| "debug/sppo_chosen_loss": 2502.216552734375, | |
| "debug/sppo_chosen_reward_in_loss": -0.0213775634765625, | |
| "debug/sppo_rej_reward_in_loss": -0.017570963129401207, | |
| "debug/sppo_reject_loss": 2498.254638671875, | |
| "epoch": 0.23550724637681159, | |
| "grad_norm": 17622.6879814254, | |
| "learning_rate": 6.5e-08, | |
| "logits/chosen": 1.488297939300537, | |
| "logits/rejected": 1.8658514022827148, | |
| "logps/chosen": -133.06114196777344, | |
| "logps/rejected": -12.593046188354492, | |
| "loss": 4999.2039, | |
| "rewards/accuracies": 0.3499999940395355, | |
| "rewards/chosen": -0.00021377562370616943, | |
| "rewards/margins": -3.806598761002533e-05, | |
| "rewards/rejected": -0.0001757096324581653, | |
| "step": 65 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.6522119045257568, | |
| "debug/policy_chosen_logps": -7.065809726715088, | |
| "debug/policy_rejected_logits": 1.638129472732544, | |
| "debug/policy_rejected_logps": -95.52651977539062, | |
| "debug/reference_chosen_logps": -7.0487470626831055, | |
| "debug/reference_rejected_logps": -95.66188049316406, | |
| "debug/sppo_chosen_loss": 2501.720947265625, | |
| "debug/sppo_chosen_reward_in_loss": -0.01706244982779026, | |
| "debug/sppo_rej_reward_in_loss": 0.135361909866333, | |
| "debug/sppo_reject_loss": 2514.490966796875, | |
| "epoch": 0.2536231884057971, | |
| "grad_norm": 69921.87787045751, | |
| "learning_rate": 6.999999999999999e-08, | |
| "logits/chosen": 1.6522119045257568, | |
| "logits/rejected": 1.638129472732544, | |
| "logps/chosen": -7.065809726715088, | |
| "logps/rejected": -95.52651977539062, | |
| "loss": 4999.0477, | |
| "rewards/accuracies": 0.20000000298023224, | |
| "rewards/chosen": -0.0001706244656816125, | |
| "rewards/margins": -0.0015242435038089752, | |
| "rewards/rejected": 0.0013536190381273627, | |
| "step": 70 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.4129836559295654, | |
| "debug/policy_chosen_logps": -122.12205505371094, | |
| "debug/policy_rejected_logits": 1.7989412546157837, | |
| "debug/policy_rejected_logps": -41.31743621826172, | |
| "debug/reference_chosen_logps": -122.72642517089844, | |
| "debug/reference_rejected_logps": -41.365840911865234, | |
| "debug/sppo_chosen_loss": 2452.815673828125, | |
| "debug/sppo_chosen_reward_in_loss": 0.6043859720230103, | |
| "debug/sppo_rej_reward_in_loss": 0.04840099439024925, | |
| "debug/sppo_reject_loss": 2504.8818359375, | |
| "epoch": 0.2717391304347826, | |
| "grad_norm": 264335.1107357926, | |
| "learning_rate": 7.5e-08, | |
| "logits/chosen": 1.4129836559295654, | |
| "logits/rejected": 1.7989412546157837, | |
| "logps/chosen": -122.12205505371094, | |
| "logps/rejected": -41.31743621826172, | |
| "loss": 5007.7828, | |
| "rewards/accuracies": 0.3499999940395355, | |
| "rewards/chosen": 0.006043859291821718, | |
| "rewards/margins": 0.005559849552810192, | |
| "rewards/rejected": 0.00048400991363450885, | |
| "step": 75 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3792654275894165, | |
| "debug/policy_chosen_logps": -5.836012840270996, | |
| "debug/policy_rejected_logits": 1.7591722011566162, | |
| "debug/policy_rejected_logps": -4.791333198547363, | |
| "debug/reference_chosen_logps": -5.829151630401611, | |
| "debug/reference_rejected_logps": -4.762129306793213, | |
| "debug/sppo_chosen_loss": 2500.70458984375, | |
| "debug/sppo_chosen_reward_in_loss": -0.006861341651529074, | |
| "debug/sppo_rej_reward_in_loss": -0.02920418418943882, | |
| "debug/sppo_reject_loss": 2497.136962890625, | |
| "epoch": 0.2898550724637681, | |
| "grad_norm": 15656.535742450906, | |
| "learning_rate": 8e-08, | |
| "logits/chosen": 1.3792654275894165, | |
| "logits/rejected": 1.7591722011566162, | |
| "logps/chosen": -5.836012840270996, | |
| "logps/rejected": -4.791333198547363, | |
| "loss": 4993.7707, | |
| "rewards/accuracies": 0.20000000298023224, | |
| "rewards/chosen": -6.861341535113752e-05, | |
| "rewards/margins": 0.00022342840384226292, | |
| "rewards/rejected": -0.0002920418046414852, | |
| "step": 80 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.5190160274505615, | |
| "debug/policy_chosen_logps": -64.62407684326172, | |
| "debug/policy_rejected_logits": 2.1376373767852783, | |
| "debug/policy_rejected_logps": -7.913151741027832, | |
| "debug/reference_chosen_logps": -64.85469055175781, | |
| "debug/reference_rejected_logps": -7.927921295166016, | |
| "debug/sppo_chosen_loss": 2478.92041015625, | |
| "debug/sppo_chosen_reward_in_loss": 0.23062074184417725, | |
| "debug/sppo_rej_reward_in_loss": 0.014769583940505981, | |
| "debug/sppo_reject_loss": 2501.4853515625, | |
| "epoch": 0.3079710144927536, | |
| "grad_norm": 51152.42234215201, | |
| "learning_rate": 8.5e-08, | |
| "logits/chosen": 1.5190160274505615, | |
| "logits/rejected": 2.1376373767852783, | |
| "logps/chosen": -64.62407684326172, | |
| "logps/rejected": -7.913151741027832, | |
| "loss": 4991.4711, | |
| "rewards/accuracies": 0.2750000059604645, | |
| "rewards/chosen": 0.0023062073159962893, | |
| "rewards/margins": 0.002158511197194457, | |
| "rewards/rejected": 0.00014769584231544286, | |
| "step": 85 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.2532832622528076, | |
| "debug/policy_chosen_logps": -122.0447769165039, | |
| "debug/policy_rejected_logits": 1.4385173320770264, | |
| "debug/policy_rejected_logps": -7.544157981872559, | |
| "debug/reference_chosen_logps": -122.35246276855469, | |
| "debug/reference_rejected_logps": -7.5182623863220215, | |
| "debug/sppo_chosen_loss": 2472.504150390625, | |
| "debug/sppo_chosen_reward_in_loss": 0.3076968491077423, | |
| "debug/sppo_rej_reward_in_loss": -0.025895338505506516, | |
| "debug/sppo_reject_loss": 2497.43994140625, | |
| "epoch": 0.32608695652173914, | |
| "grad_norm": 15987.307319112604, | |
| "learning_rate": 9e-08, | |
| "logits/chosen": 1.2532832622528076, | |
| "logits/rejected": 1.4385173320770264, | |
| "logps/chosen": -122.0447769165039, | |
| "logps/rejected": -7.544157981872559, | |
| "loss": 4997.1215, | |
| "rewards/accuracies": 0.42500001192092896, | |
| "rewards/chosen": 0.0030769682489335537, | |
| "rewards/margins": 0.0033359217923134565, | |
| "rewards/rejected": -0.0002589533687569201, | |
| "step": 90 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.44913911819458, | |
| "debug/policy_chosen_logps": -149.0557098388672, | |
| "debug/policy_rejected_logits": 1.7728023529052734, | |
| "debug/policy_rejected_logps": -8.200785636901855, | |
| "debug/reference_chosen_logps": -149.43124389648438, | |
| "debug/reference_rejected_logps": -8.184698104858398, | |
| "debug/sppo_chosen_loss": 2464.93408203125, | |
| "debug/sppo_chosen_reward_in_loss": 0.3755301237106323, | |
| "debug/sppo_rej_reward_in_loss": -0.016088182106614113, | |
| "debug/sppo_reject_loss": 2498.406982421875, | |
| "epoch": 0.3442028985507246, | |
| "grad_norm": 13150.098708323369, | |
| "learning_rate": 9.499999999999999e-08, | |
| "logits/chosen": 1.44913911819458, | |
| "logits/rejected": 1.7728023529052734, | |
| "logps/chosen": -149.0557098388672, | |
| "logps/rejected": -8.200785636901855, | |
| "loss": 4994.3016, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": 0.0037553017027676105, | |
| "rewards/margins": 0.003916183020919561, | |
| "rewards/rejected": -0.00016088179836515337, | |
| "step": 95 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.2099888324737549, | |
| "debug/policy_chosen_logps": -6.501370906829834, | |
| "debug/policy_rejected_logits": 1.892987847328186, | |
| "debug/policy_rejected_logps": -1.8146244287490845, | |
| "debug/reference_chosen_logps": -6.515327453613281, | |
| "debug/reference_rejected_logps": -1.8118356466293335, | |
| "debug/sppo_chosen_loss": 2498.66552734375, | |
| "debug/sppo_chosen_reward_in_loss": 0.013957425951957703, | |
| "debug/sppo_rej_reward_in_loss": -0.002788883401080966, | |
| "debug/sppo_reject_loss": 2499.729736328125, | |
| "epoch": 0.36231884057971014, | |
| "grad_norm": 48606.634339351476, | |
| "learning_rate": 1e-07, | |
| "logits/chosen": 1.2099888324737549, | |
| "logits/rejected": 1.892987847328186, | |
| "logps/chosen": -6.501370906829834, | |
| "logps/rejected": -1.8146244287490845, | |
| "loss": 4997.6852, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.0001395742583554238, | |
| "rewards/margins": 0.00016746309120208025, | |
| "rewards/rejected": -2.788883284665644e-05, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.36231884057971014, | |
| "eval_debug/policy_chosen_logits": 1.6633050441741943, | |
| "eval_debug/policy_chosen_logps": -122.66744232177734, | |
| "eval_debug/policy_rejected_logits": 1.7245433330535889, | |
| "eval_debug/policy_rejected_logps": -63.6384391784668, | |
| "eval_debug/reference_chosen_logps": -123.14806365966797, | |
| "eval_debug/reference_rejected_logps": -63.887054443359375, | |
| "eval_debug/sppo_chosen_loss": 2455.7412109375, | |
| "eval_debug/sppo_chosen_reward_in_loss": 0.4806005656719208, | |
| "eval_debug/sppo_rej_reward_in_loss": 0.24862028658390045, | |
| "eval_debug/sppo_reject_loss": 2529.1591796875, | |
| "eval_logits/chosen": 1.6633050441741943, | |
| "eval_logits/rejected": 1.7245433330535889, | |
| "eval_logps/chosen": -122.66744232177734, | |
| "eval_logps/rejected": -63.6384391784668, | |
| "eval_loss": 4986.5, | |
| "eval_rewards/accuracies": 0.31578946113586426, | |
| "eval_rewards/chosen": 0.004806005861610174, | |
| "eval_rewards/margins": 0.0023198029957711697, | |
| "eval_rewards/rejected": 0.002486202633008361, | |
| "eval_runtime": 29.2776, | |
| "eval_samples_per_second": 20.494, | |
| "eval_steps_per_second": 0.649, | |
| "step": 100 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.1337151527404785, | |
| "debug/policy_chosen_logps": -5.464511871337891, | |
| "debug/policy_rejected_logits": 1.2518866062164307, | |
| "debug/policy_rejected_logps": -100.66749572753906, | |
| "debug/reference_chosen_logps": -5.478997230529785, | |
| "debug/reference_rejected_logps": -100.13005828857422, | |
| "debug/sppo_chosen_loss": 2498.566650390625, | |
| "debug/sppo_chosen_reward_in_loss": 0.014485633000731468, | |
| "debug/sppo_rej_reward_in_loss": -0.5374492406845093, | |
| "debug/sppo_reject_loss": 2460.12158203125, | |
| "epoch": 0.3804347826086957, | |
| "grad_norm": 233662.76041400852, | |
| "learning_rate": 9.98009940204023e-08, | |
| "logits/chosen": 1.1337151527404785, | |
| "logits/rejected": 1.2518866062164307, | |
| "logps/chosen": -5.464511871337891, | |
| "logps/rejected": -100.66749572753906, | |
| "loss": 5023.0688, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.00014485632709693164, | |
| "rewards/margins": 0.0055193486623466015, | |
| "rewards/rejected": -0.005374492146074772, | |
| "step": 105 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.2909049987792969, | |
| "debug/policy_chosen_logps": -69.8453598022461, | |
| "debug/policy_rejected_logits": 1.5784929990768433, | |
| "debug/policy_rejected_logps": -6.722166538238525, | |
| "debug/reference_chosen_logps": -70.15248107910156, | |
| "debug/reference_rejected_logps": -6.677098274230957, | |
| "debug/sppo_chosen_loss": 2473.166015625, | |
| "debug/sppo_chosen_reward_in_loss": 0.30711063742637634, | |
| "debug/sppo_rej_reward_in_loss": -0.045068077743053436, | |
| "debug/sppo_reject_loss": 2495.512451171875, | |
| "epoch": 0.39855072463768115, | |
| "grad_norm": 54187.6399779739, | |
| "learning_rate": 9.92055602168058e-08, | |
| "logits/chosen": 1.2909049987792969, | |
| "logits/rejected": 1.5784929990768433, | |
| "logps/chosen": -69.8453598022461, | |
| "logps/rejected": -6.722166538238525, | |
| "loss": 4988.0766, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.003071106504648924, | |
| "rewards/margins": 0.003521787002682686, | |
| "rewards/rejected": -0.00045068078907206655, | |
| "step": 110 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.5610486268997192, | |
| "debug/policy_chosen_logps": -72.94502258300781, | |
| "debug/policy_rejected_logits": 1.9416284561157227, | |
| "debug/policy_rejected_logps": -17.85445213317871, | |
| "debug/reference_chosen_logps": -73.14714050292969, | |
| "debug/reference_rejected_logps": -17.959739685058594, | |
| "debug/sppo_chosen_loss": 2480.93798828125, | |
| "debug/sppo_chosen_reward_in_loss": 0.20211248099803925, | |
| "debug/sppo_rej_reward_in_loss": 0.10528746992349625, | |
| "debug/sppo_reject_loss": 2511.402587890625, | |
| "epoch": 0.4166666666666667, | |
| "grad_norm": 36462.2274872176, | |
| "learning_rate": 9.821843838470533e-08, | |
| "logits/chosen": 1.5610486268997192, | |
| "logits/rejected": 1.9416284561157227, | |
| "logps/chosen": -72.94502258300781, | |
| "logps/rejected": -17.85445213317871, | |
| "loss": 4989.1812, | |
| "rewards/accuracies": 0.32499998807907104, | |
| "rewards/chosen": 0.002021125052124262, | |
| "rewards/margins": 0.0009682502713985741, | |
| "rewards/rejected": 0.0010528747225180268, | |
| "step": 115 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3458185195922852, | |
| "debug/policy_chosen_logps": -47.522071838378906, | |
| "debug/policy_rejected_logits": 1.701849341392517, | |
| "debug/policy_rejected_logps": -2.4559662342071533, | |
| "debug/reference_chosen_logps": -47.64037322998047, | |
| "debug/reference_rejected_logps": -2.4745194911956787, | |
| "debug/sppo_chosen_loss": 2488.593017578125, | |
| "debug/sppo_chosen_reward_in_loss": 0.1182965412735939, | |
| "debug/sppo_rej_reward_in_loss": 0.018553102388978004, | |
| "debug/sppo_reject_loss": 2501.86083984375, | |
| "epoch": 0.43478260869565216, | |
| "grad_norm": 44388.4153538321, | |
| "learning_rate": 9.684748624998809e-08, | |
| "logits/chosen": 1.3458185195922852, | |
| "logits/rejected": 1.701849341392517, | |
| "logps/chosen": -47.522071838378906, | |
| "logps/rejected": -2.4559662342071533, | |
| "loss": 4995.3309, | |
| "rewards/accuracies": 0.17499999701976776, | |
| "rewards/chosen": 0.0011829653521999717, | |
| "rewards/margins": 0.0009974343702197075, | |
| "rewards/rejected": 0.00018553101108409464, | |
| "step": 120 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.4266709089279175, | |
| "debug/policy_chosen_logps": -6.328521251678467, | |
| "debug/policy_rejected_logits": 1.6656681299209595, | |
| "debug/policy_rejected_logps": -58.306007385253906, | |
| "debug/reference_chosen_logps": -6.291218280792236, | |
| "debug/reference_rejected_logps": -58.57655715942383, | |
| "debug/sppo_chosen_loss": 2503.767333984375, | |
| "debug/sppo_chosen_reward_in_loss": -0.037303246557712555, | |
| "debug/sppo_rej_reward_in_loss": 0.27054446935653687, | |
| "debug/sppo_reject_loss": 2530.11767578125, | |
| "epoch": 0.4528985507246377, | |
| "grad_norm": 35653.64462552226, | |
| "learning_rate": 9.510361691955607e-08, | |
| "logits/chosen": 1.4266709089279175, | |
| "logits/rejected": 1.6656681299209595, | |
| "logps/chosen": -6.328521251678467, | |
| "logps/rejected": -58.306007385253906, | |
| "loss": 4997.7953, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": -0.000373032467905432, | |
| "rewards/margins": -0.0030784769915044308, | |
| "rewards/rejected": 0.0027054445818066597, | |
| "step": 125 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.2133262157440186, | |
| "debug/policy_chosen_logps": -4.814808368682861, | |
| "debug/policy_rejected_logits": 1.638085961341858, | |
| "debug/policy_rejected_logps": -7.694342136383057, | |
| "debug/reference_chosen_logps": -4.811456203460693, | |
| "debug/reference_rejected_logps": -7.753373622894287, | |
| "debug/sppo_chosen_loss": 2500.358154296875, | |
| "debug/sppo_chosen_reward_in_loss": -0.003351741936057806, | |
| "debug/sppo_rej_reward_in_loss": 0.059032391756772995, | |
| "debug/sppo_reject_loss": 2506.000732421875, | |
| "epoch": 0.47101449275362317, | |
| "grad_norm": 56980.429768390655, | |
| "learning_rate": 9.300071201038502e-08, | |
| "logits/chosen": 1.2133262157440186, | |
| "logits/rejected": 1.638085961341858, | |
| "logps/chosen": -4.814808368682861, | |
| "logps/rejected": -7.694342136383057, | |
| "loss": 4970.8211, | |
| "rewards/accuracies": 0.20000000298023224, | |
| "rewards/chosen": -3.351741543156095e-05, | |
| "rewards/margins": -0.0006238413043320179, | |
| "rewards/rejected": 0.0005903238779865205, | |
| "step": 130 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3553920984268188, | |
| "debug/policy_chosen_logps": -7.613694667816162, | |
| "debug/policy_rejected_logits": 1.8132812976837158, | |
| "debug/policy_rejected_logps": -45.774879455566406, | |
| "debug/reference_chosen_logps": -7.610405921936035, | |
| "debug/reference_rejected_logps": -46.041690826416016, | |
| "debug/sppo_chosen_loss": 2500.42138671875, | |
| "debug/sppo_chosen_reward_in_loss": -0.0032895803451538086, | |
| "debug/sppo_rej_reward_in_loss": 0.2668178975582123, | |
| "debug/sppo_reject_loss": 2528.033935546875, | |
| "epoch": 0.4891304347826087, | |
| "grad_norm": 22513.750392611728, | |
| "learning_rate": 9.055551114853295e-08, | |
| "logits/chosen": 1.3553920984268188, | |
| "logits/rejected": 1.8132812976837158, | |
| "logps/chosen": -7.613694667816162, | |
| "logps/rejected": -45.774879455566406, | |
| "loss": 5004.3156, | |
| "rewards/accuracies": 0.22499999403953552, | |
| "rewards/chosen": -3.289579035481438e-05, | |
| "rewards/margins": -0.0027010745834559202, | |
| "rewards/rejected": 0.0026681788731366396, | |
| "step": 135 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.7337901592254639, | |
| "debug/policy_chosen_logps": -10.878243446350098, | |
| "debug/policy_rejected_logits": 2.027592182159424, | |
| "debug/policy_rejected_logps": -1.9502204656600952, | |
| "debug/reference_chosen_logps": -10.920201301574707, | |
| "debug/reference_rejected_logps": -1.9509022235870361, | |
| "debug/sppo_chosen_loss": 2495.86865234375, | |
| "debug/sppo_chosen_reward_in_loss": 0.04195826128125191, | |
| "debug/sppo_rej_reward_in_loss": 0.0006817228859290481, | |
| "debug/sppo_reject_loss": 2500.08251953125, | |
| "epoch": 0.5072463768115942, | |
| "grad_norm": 207027.63645308473, | |
| "learning_rate": 8.778747871771291e-08, | |
| "logits/chosen": 1.7337901592254639, | |
| "logits/rejected": 2.027592182159424, | |
| "logps/chosen": -10.878243446350098, | |
| "logps/rejected": -1.9502204656600952, | |
| "loss": 5006.7344, | |
| "rewards/accuracies": 0.3499999940395355, | |
| "rewards/chosen": 0.00041958261863328516, | |
| "rewards/margins": 0.0004127653664909303, | |
| "rewards/rejected": 6.817223038524389e-06, | |
| "step": 140 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.2483991384506226, | |
| "debug/policy_chosen_logps": -226.301025390625, | |
| "debug/policy_rejected_logits": 1.5713260173797607, | |
| "debug/policy_rejected_logps": -5.900942802429199, | |
| "debug/reference_chosen_logps": -228.1366729736328, | |
| "debug/reference_rejected_logps": -5.889040946960449, | |
| "debug/sppo_chosen_loss": 2360.619873046875, | |
| "debug/sppo_chosen_reward_in_loss": 1.8356482982635498, | |
| "debug/sppo_rej_reward_in_loss": -0.011901888065040112, | |
| "debug/sppo_reject_loss": 2498.83056640625, | |
| "epoch": 0.5253623188405797, | |
| "grad_norm": 56481.61321339093, | |
| "learning_rate": 8.471864891814303e-08, | |
| "logits/chosen": 1.2483991384506226, | |
| "logits/rejected": 1.5713260173797607, | |
| "logps/chosen": -226.301025390625, | |
| "logps/rejected": -5.900942802429199, | |
| "loss": 4990.75, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": 0.018356481567025185, | |
| "rewards/margins": 0.018475499004125595, | |
| "rewards/rejected": -0.00011901887774001807, | |
| "step": 145 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.6154155731201172, | |
| "debug/policy_chosen_logps": -195.41567993164062, | |
| "debug/policy_rejected_logits": 2.077685594558716, | |
| "debug/policy_rejected_logps": -69.73979187011719, | |
| "debug/reference_chosen_logps": -198.35165405273438, | |
| "debug/reference_rejected_logps": -70.21018981933594, | |
| "debug/sppo_chosen_loss": 2386.256103515625, | |
| "debug/sppo_chosen_reward_in_loss": 2.9359681606292725, | |
| "debug/sppo_rej_reward_in_loss": 0.47039833664894104, | |
| "debug/sppo_reject_loss": 2554.950439453125, | |
| "epoch": 0.5434782608695652, | |
| "grad_norm": 23877.971889635744, | |
| "learning_rate": 8.13734503690426e-08, | |
| "logits/chosen": 1.6154155731201172, | |
| "logits/rejected": 2.077685594558716, | |
| "logps/chosen": -195.41567993164062, | |
| "logps/rejected": -69.73979187011719, | |
| "loss": 4975.2414, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.02935967966914177, | |
| "rewards/margins": 0.024655699729919434, | |
| "rewards/rejected": 0.004703983664512634, | |
| "step": 150 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3934733867645264, | |
| "debug/policy_chosen_logps": -186.84793090820312, | |
| "debug/policy_rejected_logits": 1.7428455352783203, | |
| "debug/policy_rejected_logps": -11.295183181762695, | |
| "debug/reference_chosen_logps": -188.67599487304688, | |
| "debug/reference_rejected_logps": -11.309015274047852, | |
| "debug/sppo_chosen_loss": 2406.025634765625, | |
| "debug/sppo_chosen_reward_in_loss": 1.828051209449768, | |
| "debug/sppo_rej_reward_in_loss": 0.013833016157150269, | |
| "debug/sppo_reject_loss": 2501.45263671875, | |
| "epoch": 0.5615942028985508, | |
| "grad_norm": 49089.76331500084, | |
| "learning_rate": 7.77785116509801e-08, | |
| "logits/chosen": 1.3934733867645264, | |
| "logits/rejected": 1.7428455352783203, | |
| "logps/chosen": -186.84793090820312, | |
| "logps/rejected": -11.295183181762695, | |
| "loss": 4959.9672, | |
| "rewards/accuracies": 0.32499998807907104, | |
| "rewards/chosen": 0.0182805135846138, | |
| "rewards/margins": 0.018142182379961014, | |
| "rewards/rejected": 0.00013833014236297458, | |
| "step": 155 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.1241214275360107, | |
| "debug/policy_chosen_logps": -41.29949951171875, | |
| "debug/policy_rejected_logits": 1.587595820426941, | |
| "debug/policy_rejected_logps": -7.305385589599609, | |
| "debug/reference_chosen_logps": -41.46500778198242, | |
| "debug/reference_rejected_logps": -7.236231803894043, | |
| "debug/sppo_chosen_loss": 2484.546875, | |
| "debug/sppo_chosen_reward_in_loss": 0.16550657153129578, | |
| "debug/sppo_rej_reward_in_loss": -0.06915402412414551, | |
| "debug/sppo_reject_loss": 2493.12939453125, | |
| "epoch": 0.5797101449275363, | |
| "grad_norm": 14605.8114006113, | |
| "learning_rate": 7.396244933600284e-08, | |
| "logits/chosen": 1.1241214275360107, | |
| "logits/rejected": 1.587595820426941, | |
| "logps/chosen": -41.29949951171875, | |
| "logps/rejected": -7.305385589599609, | |
| "loss": 4964.507, | |
| "rewards/accuracies": 0.3499999940395355, | |
| "rewards/chosen": 0.0016550656873732805, | |
| "rewards/margins": 0.002346605993807316, | |
| "rewards/rejected": -0.0006915402482263744, | |
| "step": 160 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.6341183185577393, | |
| "debug/policy_chosen_logps": -46.92511749267578, | |
| "debug/policy_rejected_logits": 1.4523030519485474, | |
| "debug/policy_rejected_logps": -26.32564926147461, | |
| "debug/reference_chosen_logps": -47.207061767578125, | |
| "debug/reference_rejected_logps": -26.676687240600586, | |
| "debug/sppo_chosen_loss": 2474.646484375, | |
| "debug/sppo_chosen_reward_in_loss": 0.28194811940193176, | |
| "debug/sppo_rej_reward_in_loss": 0.35103607177734375, | |
| "debug/sppo_reject_loss": 2538.4462890625, | |
| "epoch": 0.5978260869565217, | |
| "grad_norm": 36718.92822244079, | |
| "learning_rate": 6.995564019287869e-08, | |
| "logits/chosen": 1.6341183185577393, | |
| "logits/rejected": 1.4523030519485474, | |
| "logps/chosen": -46.92511749267578, | |
| "logps/rejected": -26.32564926147461, | |
| "loss": 5053.6191, | |
| "rewards/accuracies": 0.125, | |
| "rewards/chosen": 0.002819481072947383, | |
| "rewards/margins": -0.0006908794166520238, | |
| "rewards/rejected": 0.003510360838845372, | |
| "step": 165 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3338284492492676, | |
| "debug/policy_chosen_logps": -49.61326599121094, | |
| "debug/policy_rejected_logits": 1.6426509618759155, | |
| "debug/policy_rejected_logps": -6.317404270172119, | |
| "debug/reference_chosen_logps": -49.824684143066406, | |
| "debug/reference_rejected_logps": -6.327885627746582, | |
| "debug/sppo_chosen_loss": 2480.391845703125, | |
| "debug/sppo_chosen_reward_in_loss": 0.2114112377166748, | |
| "debug/sppo_rej_reward_in_loss": 0.010481464676558971, | |
| "debug/sppo_reject_loss": 2501.14501953125, | |
| "epoch": 0.6159420289855072, | |
| "grad_norm": 44446.380970079495, | |
| "learning_rate": 6.578997938075125e-08, | |
| "logits/chosen": 1.3338284492492676, | |
| "logits/rejected": 1.6426509618759155, | |
| "logps/chosen": -49.61326599121094, | |
| "logps/rejected": -6.317404270172119, | |
| "loss": 4995.0012, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.0021141120232641697, | |
| "rewards/margins": 0.0020092972554266453, | |
| "rewards/rejected": 0.00010481465142220259, | |
| "step": 170 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.471964716911316, | |
| "debug/policy_chosen_logps": -11.514124870300293, | |
| "debug/policy_rejected_logits": 1.9099857807159424, | |
| "debug/policy_rejected_logps": -112.86039733886719, | |
| "debug/reference_chosen_logps": -11.554305076599121, | |
| "debug/reference_rejected_logps": -113.70204162597656, | |
| "debug/sppo_chosen_loss": 2496.02197265625, | |
| "debug/sppo_chosen_reward_in_loss": 0.04018024355173111, | |
| "debug/sppo_rej_reward_in_loss": 0.8416390419006348, | |
| "debug/sppo_reject_loss": 2610.68212890625, | |
| "epoch": 0.6340579710144928, | |
| "grad_norm": 135545.85245016703, | |
| "learning_rate": 6.149862655604404e-08, | |
| "logits/chosen": 1.471964716911316, | |
| "logits/rejected": 1.9099857807159424, | |
| "logps/chosen": -11.514124870300293, | |
| "logps/rejected": -112.86039733886719, | |
| "loss": 4998.809, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.00040180241921916604, | |
| "rewards/margins": -0.008014587685465813, | |
| "rewards/rejected": 0.008416390046477318, | |
| "step": 175 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.2563765048980713, | |
| "debug/policy_chosen_logps": -6.484707832336426, | |
| "debug/policy_rejected_logits": 1.6219933032989502, | |
| "debug/policy_rejected_logps": -11.174871444702148, | |
| "debug/reference_chosen_logps": -6.455367088317871, | |
| "debug/reference_rejected_logps": -11.164003372192383, | |
| "debug/sppo_chosen_loss": 2502.980712890625, | |
| "debug/sppo_chosen_reward_in_loss": -0.029340500012040138, | |
| "debug/sppo_rej_reward_in_loss": -0.010869455523788929, | |
| "debug/sppo_reject_loss": 2498.97802734375, | |
| "epoch": 0.6521739130434783, | |
| "grad_norm": 164996.56785625734, | |
| "learning_rate": 5.711574191366426e-08, | |
| "logits/chosen": 1.2563765048980713, | |
| "logits/rejected": 1.6219933032989502, | |
| "logps/chosen": -6.484707832336426, | |
| "logps/rejected": -11.174871444702148, | |
| "loss": 5019.5977, | |
| "rewards/accuracies": 0.20000000298023224, | |
| "rewards/chosen": -0.0002934050280600786, | |
| "rewards/margins": -0.0001847104576881975, | |
| "rewards/rejected": -0.00010869452671613544, | |
| "step": 180 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.2139984369277954, | |
| "debug/policy_chosen_logps": -7.972718715667725, | |
| "debug/policy_rejected_logits": 1.6660735607147217, | |
| "debug/policy_rejected_logps": -15.723217964172363, | |
| "debug/reference_chosen_logps": -7.793820858001709, | |
| "debug/reference_rejected_logps": -15.452303886413574, | |
| "debug/sppo_chosen_loss": 2518.21337890625, | |
| "debug/sppo_chosen_reward_in_loss": -0.17889812588691711, | |
| "debug/sppo_rej_reward_in_loss": -0.2709135413169861, | |
| "debug/sppo_reject_loss": 2473.35009765625, | |
| "epoch": 0.6702898550724637, | |
| "grad_norm": 18624.21256617316, | |
| "learning_rate": 5.267621426368075e-08, | |
| "logits/chosen": 1.2139984369277954, | |
| "logits/rejected": 1.6660735607147217, | |
| "logps/chosen": -7.972718715667725, | |
| "logps/rejected": -15.723217964172363, | |
| "loss": 5039.2461, | |
| "rewards/accuracies": 0.32499998807907104, | |
| "rewards/chosen": -0.0017889812588691711, | |
| "rewards/margins": 0.0009201543289236724, | |
| "rewards/rejected": -0.002709135413169861, | |
| "step": 185 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.2351247072219849, | |
| "debug/policy_chosen_logps": -55.260719299316406, | |
| "debug/policy_rejected_logits": 1.3063384294509888, | |
| "debug/policy_rejected_logps": -3.657384157180786, | |
| "debug/reference_chosen_logps": -55.711204528808594, | |
| "debug/reference_rejected_logps": -3.6667861938476562, | |
| "debug/sppo_chosen_loss": 2464.06689453125, | |
| "debug/sppo_chosen_reward_in_loss": 0.45049649477005005, | |
| "debug/sppo_rej_reward_in_loss": 0.009402072057127953, | |
| "debug/sppo_reject_loss": 2500.978271484375, | |
| "epoch": 0.6884057971014492, | |
| "grad_norm": 59339.47300560303, | |
| "learning_rate": 4.821538330805098e-08, | |
| "logits/chosen": 1.2351247072219849, | |
| "logits/rejected": 1.3063384294509888, | |
| "logps/chosen": -55.260719299316406, | |
| "logps/rejected": -3.657384157180786, | |
| "loss": 5023.3609, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.004504963755607605, | |
| "rewards/margins": 0.004410943016409874, | |
| "rewards/rejected": 9.402072464581579e-05, | |
| "step": 190 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.796984076499939, | |
| "debug/policy_chosen_logps": -100.9161605834961, | |
| "debug/policy_rejected_logits": 2.164973020553589, | |
| "debug/policy_rejected_logps": -83.23567199707031, | |
| "debug/reference_chosen_logps": -101.08064270019531, | |
| "debug/reference_rejected_logps": -83.28086853027344, | |
| "debug/sppo_chosen_loss": 2484.186767578125, | |
| "debug/sppo_chosen_reward_in_loss": 0.164472296833992, | |
| "debug/sppo_rej_reward_in_loss": 0.04519999027252197, | |
| "debug/sppo_reject_loss": 2504.991943359375, | |
| "epoch": 0.7065217391304348, | |
| "grad_norm": 55163.46876155982, | |
| "learning_rate": 4.3768758328140705e-08, | |
| "logits/chosen": 1.796984076499939, | |
| "logits/rejected": 2.164973020553589, | |
| "logps/chosen": -100.9161605834961, | |
| "logps/rejected": -83.23567199707031, | |
| "loss": 4972.7703, | |
| "rewards/accuracies": 0.3499999940395355, | |
| "rewards/chosen": 0.0016447228845208883, | |
| "rewards/margins": 0.0011927229352295399, | |
| "rewards/rejected": 0.00045199989108368754, | |
| "step": 195 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.2561403512954712, | |
| "debug/policy_chosen_logps": -4.826408863067627, | |
| "debug/policy_rejected_logits": 1.572509765625, | |
| "debug/policy_rejected_logps": -52.822608947753906, | |
| "debug/reference_chosen_logps": -4.822422027587891, | |
| "debug/reference_rejected_logps": -53.10357666015625, | |
| "debug/sppo_chosen_loss": 2500.411865234375, | |
| "debug/sppo_chosen_reward_in_loss": -0.003986963536590338, | |
| "debug/sppo_rej_reward_in_loss": 0.28096264600753784, | |
| "debug/sppo_reject_loss": 2531.43603515625, | |
| "epoch": 0.7246376811594203, | |
| "grad_norm": 73770.12364262494, | |
| "learning_rate": 3.9371735522351165e-08, | |
| "logits/chosen": 1.2561403512954712, | |
| "logits/rejected": 1.572509765625, | |
| "logps/chosen": -4.826408863067627, | |
| "logps/rejected": -52.822608947753906, | |
| "loss": 5015.0652, | |
| "rewards/accuracies": 0.22499999403953552, | |
| "rewards/chosen": -3.986962838098407e-05, | |
| "rewards/margins": -0.0028494957368820906, | |
| "rewards/rejected": 0.002809626515954733, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.7246376811594203, | |
| "eval_debug/policy_chosen_logits": 1.6680212020874023, | |
| "eval_debug/policy_chosen_logps": -121.2494125366211, | |
| "eval_debug/policy_rejected_logits": 1.7270073890686035, | |
| "eval_debug/policy_rejected_logps": -63.196250915527344, | |
| "eval_debug/reference_chosen_logps": -123.14806365966797, | |
| "eval_debug/reference_rejected_logps": -63.887054443359375, | |
| "eval_debug/sppo_chosen_loss": 2390.288330078125, | |
| "eval_debug/sppo_chosen_reward_in_loss": 1.8986449241638184, | |
| "eval_debug/sppo_rej_reward_in_loss": 0.6908064484596252, | |
| "eval_debug/sppo_reject_loss": 2606.55419921875, | |
| "eval_logits/chosen": 1.6680212020874023, | |
| "eval_logits/rejected": 1.7270073890686035, | |
| "eval_logps/chosen": -121.2494125366211, | |
| "eval_logps/rejected": -63.196250915527344, | |
| "eval_loss": 5015.12939453125, | |
| "eval_rewards/accuracies": 0.2763157784938812, | |
| "eval_rewards/chosen": 0.018986448645591736, | |
| "eval_rewards/margins": 0.012078385800123215, | |
| "eval_rewards/rejected": 0.006908063776791096, | |
| "eval_runtime": 28.6889, | |
| "eval_samples_per_second": 20.914, | |
| "eval_steps_per_second": 0.662, | |
| "step": 200 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3723523616790771, | |
| "debug/policy_chosen_logps": -4.869706153869629, | |
| "debug/policy_rejected_logits": 1.7145782709121704, | |
| "debug/policy_rejected_logps": -85.27532196044922, | |
| "debug/reference_chosen_logps": -4.833613872528076, | |
| "debug/reference_rejected_logps": -85.68836212158203, | |
| "debug/sppo_chosen_loss": 2503.64306640625, | |
| "debug/sppo_chosen_reward_in_loss": -0.03609180450439453, | |
| "debug/sppo_rej_reward_in_loss": 0.4130396842956543, | |
| "debug/sppo_reject_loss": 2549.91455078125, | |
| "epoch": 0.7427536231884058, | |
| "grad_norm": 167073.79540945674, | |
| "learning_rate": 3.505931624391355e-08, | |
| "logits/chosen": 1.3723523616790771, | |
| "logits/rejected": 1.7145782709121704, | |
| "logps/chosen": -4.869706153869629, | |
| "logps/rejected": -85.27532196044922, | |
| "loss": 5027.4824, | |
| "rewards/accuracies": 0.2750000059604645, | |
| "rewards/chosen": -0.0003609180566854775, | |
| "rewards/margins": -0.0044913142919540405, | |
| "rewards/rejected": 0.004130396526306868, | |
| "step": 205 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.2046802043914795, | |
| "debug/policy_chosen_logps": -63.47414016723633, | |
| "debug/policy_rejected_logits": 1.5040717124938965, | |
| "debug/policy_rejected_logps": -45.39781951904297, | |
| "debug/reference_chosen_logps": -63.378448486328125, | |
| "debug/reference_rejected_logps": -45.54474639892578, | |
| "debug/sppo_chosen_loss": 2509.64892578125, | |
| "debug/sppo_chosen_reward_in_loss": -0.0956823080778122, | |
| "debug/sppo_rej_reward_in_loss": 0.14692507684230804, | |
| "debug/sppo_reject_loss": 2515.404296875, | |
| "epoch": 0.7608695652173914, | |
| "grad_norm": 73818.32992301855, | |
| "learning_rate": 3.086582838174551e-08, | |
| "logits/chosen": 1.2046802043914795, | |
| "logits/rejected": 1.5040717124938965, | |
| "logps/chosen": -63.47414016723633, | |
| "logps/rejected": -45.39781951904297, | |
| "loss": 5026.3219, | |
| "rewards/accuracies": 0.07500000298023224, | |
| "rewards/chosen": -0.0009568231180310249, | |
| "rewards/margins": -0.0024260736536234617, | |
| "rewards/rejected": 0.0014692506520077586, | |
| "step": 210 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.4869812726974487, | |
| "debug/policy_chosen_logps": -46.38404083251953, | |
| "debug/policy_rejected_logits": 1.8799896240234375, | |
| "debug/policy_rejected_logps": -7.943442344665527, | |
| "debug/reference_chosen_logps": -46.571571350097656, | |
| "debug/reference_rejected_logps": -7.923023223876953, | |
| "debug/sppo_chosen_loss": 2482.71826171875, | |
| "debug/sppo_chosen_reward_in_loss": 0.18753060698509216, | |
| "debug/sppo_rej_reward_in_loss": -0.020419389009475708, | |
| "debug/sppo_reject_loss": 2498.00537109375, | |
| "epoch": 0.7789855072463768, | |
| "grad_norm": 30997.36116841476, | |
| "learning_rate": 2.6824653102244727e-08, | |
| "logits/chosen": 1.4869812726974487, | |
| "logits/rejected": 1.8799896240234375, | |
| "logps/chosen": -46.38404083251953, | |
| "logps/rejected": -7.943442344665527, | |
| "loss": 4958.6598, | |
| "rewards/accuracies": 0.32499998807907104, | |
| "rewards/chosen": 0.0018753058975562453, | |
| "rewards/margins": 0.002079499652609229, | |
| "rewards/rejected": -0.0002041938714683056, | |
| "step": 215 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 0.9879446029663086, | |
| "debug/policy_chosen_logps": -26.743267059326172, | |
| "debug/policy_rejected_logits": 1.3120319843292236, | |
| "debug/policy_rejected_logps": -10.141648292541504, | |
| "debug/reference_chosen_logps": -26.971553802490234, | |
| "debug/reference_rejected_logps": -10.127599716186523, | |
| "debug/sppo_chosen_loss": 2479.09375, | |
| "debug/sppo_chosen_reward_in_loss": 0.2282869815826416, | |
| "debug/sppo_rej_reward_in_loss": -0.01404793281108141, | |
| "debug/sppo_reject_loss": 2498.64404296875, | |
| "epoch": 0.7971014492753623, | |
| "grad_norm": 26802.126181328487, | |
| "learning_rate": 2.2967959127220138e-08, | |
| "logits/chosen": 0.9879446029663086, | |
| "logits/rejected": 1.3120319843292236, | |
| "logps/chosen": -26.743267059326172, | |
| "logps/rejected": -10.141648292541504, | |
| "loss": 5009.0125, | |
| "rewards/accuracies": 0.30000001192092896, | |
| "rewards/chosen": 0.0022828695364296436, | |
| "rewards/margins": 0.002423349069431424, | |
| "rewards/rejected": -0.00014047934382688254, | |
| "step": 220 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3579328060150146, | |
| "debug/policy_chosen_logps": -7.486691474914551, | |
| "debug/policy_rejected_logits": 1.647939920425415, | |
| "debug/policy_rejected_logps": -117.33280181884766, | |
| "debug/reference_chosen_logps": -7.492678165435791, | |
| "debug/reference_rejected_logps": -117.96882629394531, | |
| "debug/sppo_chosen_loss": 2499.44482421875, | |
| "debug/sppo_chosen_reward_in_loss": 0.005986439995467663, | |
| "debug/sppo_rej_reward_in_loss": 0.6360327005386353, | |
| "debug/sppo_reject_loss": 2580.13330078125, | |
| "epoch": 0.8152173913043478, | |
| "grad_norm": 31288.105349492, | |
| "learning_rate": 1.9326446663172035e-08, | |
| "logits/chosen": 1.3579328060150146, | |
| "logits/rejected": 1.647939920425415, | |
| "logps/chosen": -7.486691474914551, | |
| "logps/rejected": -117.33280181884766, | |
| "loss": 5015.4312, | |
| "rewards/accuracies": 0.2750000059604645, | |
| "rewards/chosen": 5.9864407376153395e-05, | |
| "rewards/margins": -0.006300461478531361, | |
| "rewards/rejected": 0.006360325962305069, | |
| "step": 225 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.215022325515747, | |
| "debug/policy_chosen_logps": -44.3668327331543, | |
| "debug/policy_rejected_logits": 1.4289997816085815, | |
| "debug/policy_rejected_logps": -9.595197677612305, | |
| "debug/reference_chosen_logps": -44.462120056152344, | |
| "debug/reference_rejected_logps": -9.55119514465332, | |
| "debug/sppo_chosen_loss": 2491.029052734375, | |
| "debug/sppo_chosen_reward_in_loss": 0.09528430551290512, | |
| "debug/sppo_rej_reward_in_loss": -0.04400312155485153, | |
| "debug/sppo_reject_loss": 2495.680419921875, | |
| "epoch": 0.8333333333333334, | |
| "grad_norm": 116537.70488652344, | |
| "learning_rate": 1.592910302030544e-08, | |
| "logits/chosen": 1.215022325515747, | |
| "logits/rejected": 1.4289997816085815, | |
| "logps/chosen": -44.3668327331543, | |
| "logps/rejected": -9.595197677612305, | |
| "loss": 4982.9453, | |
| "rewards/accuracies": 0.32499998807907104, | |
| "rewards/chosen": 0.0009528429945930839, | |
| "rewards/margins": 0.0013928741682320833, | |
| "rewards/rejected": -0.0004400312027428299, | |
| "step": 230 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.398555040359497, | |
| "debug/policy_chosen_logps": -3.825951337814331, | |
| "debug/policy_rejected_logits": 1.5684314966201782, | |
| "debug/policy_rejected_logps": -11.321913719177246, | |
| "debug/reference_chosen_logps": -3.7865378856658936, | |
| "debug/reference_rejected_logps": -11.303999900817871, | |
| "debug/sppo_chosen_loss": 2503.960205078125, | |
| "debug/sppo_chosen_reward_in_loss": -0.039413463324308395, | |
| "debug/sppo_rej_reward_in_loss": -0.01791282929480076, | |
| "debug/sppo_reject_loss": 2498.244873046875, | |
| "epoch": 0.8514492753623188, | |
| "grad_norm": 118001.4885666594, | |
| "learning_rate": 1.2802971866607521e-08, | |
| "logits/chosen": 1.398555040359497, | |
| "logits/rejected": 1.5684314966201782, | |
| "logps/chosen": -3.825951337814331, | |
| "logps/rejected": -11.321913719177246, | |
| "loss": 4981.2648, | |
| "rewards/accuracies": 0.22499999403953552, | |
| "rewards/chosen": -0.0003941346367355436, | |
| "rewards/margins": -0.00021500633738469332, | |
| "rewards/rejected": -0.00017912828479893506, | |
| "step": 235 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3944571018218994, | |
| "debug/policy_chosen_logps": -3.6208109855651855, | |
| "debug/policy_rejected_logits": 1.9865443706512451, | |
| "debug/policy_rejected_logps": -36.188865661621094, | |
| "debug/reference_chosen_logps": -3.5993995666503906, | |
| "debug/reference_rejected_logps": -36.39490509033203, | |
| "debug/sppo_chosen_loss": 2502.203857421875, | |
| "debug/sppo_chosen_reward_in_loss": -0.021411290392279625, | |
| "debug/sppo_rej_reward_in_loss": 0.20603516697883606, | |
| "debug/sppo_reject_loss": 2522.786376953125, | |
| "epoch": 0.8695652173913043, | |
| "grad_norm": 53766.29640847203, | |
| "learning_rate": 9.972937953781984e-09, | |
| "logits/chosen": 1.3944571018218994, | |
| "logits/rejected": 1.9865443706512451, | |
| "logps/chosen": -3.6208109855651855, | |
| "logps/rejected": -36.188865661621094, | |
| "loss": 5027.7012, | |
| "rewards/accuracies": 0.22499999403953552, | |
| "rewards/chosen": -0.00021411287889350206, | |
| "rewards/margins": -0.002274464350193739, | |
| "rewards/rejected": 0.002060351427644491, | |
| "step": 240 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.1183539628982544, | |
| "debug/policy_chosen_logps": -3.6978707313537598, | |
| "debug/policy_rejected_logits": 1.46969473361969, | |
| "debug/policy_rejected_logps": -170.7615203857422, | |
| "debug/reference_chosen_logps": -3.6889584064483643, | |
| "debug/reference_rejected_logps": -171.53671264648438, | |
| "debug/sppo_chosen_loss": 2500.911865234375, | |
| "debug/sppo_chosen_reward_in_loss": -0.008911940269172192, | |
| "debug/sppo_rej_reward_in_loss": 0.775180459022522, | |
| "debug/sppo_reject_loss": 2618.15576171875, | |
| "epoch": 0.8876811594202898, | |
| "grad_norm": 51804.200611742, | |
| "learning_rate": 7.461529028673463e-09, | |
| "logits/chosen": 1.1183539628982544, | |
| "logits/rejected": 1.46969473361969, | |
| "logps/chosen": -3.6978707313537598, | |
| "logps/rejected": -170.7615203857422, | |
| "loss": 5000.7332, | |
| "rewards/accuracies": 0.32499998807907104, | |
| "rewards/chosen": -8.911939949030057e-05, | |
| "rewards/margins": -0.007840922102332115, | |
| "rewards/rejected": 0.00775180384516716, | |
| "step": 245 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.2443785667419434, | |
| "debug/policy_chosen_logps": -5.438248157501221, | |
| "debug/policy_rejected_logits": 1.60662043094635, | |
| "debug/policy_rejected_logps": -4.876561641693115, | |
| "debug/reference_chosen_logps": -5.442919731140137, | |
| "debug/reference_rejected_logps": -4.868186950683594, | |
| "debug/sppo_chosen_loss": 2499.578857421875, | |
| "debug/sppo_chosen_reward_in_loss": 0.004670926835387945, | |
| "debug/sppo_rej_reward_in_loss": -0.008374800905585289, | |
| "debug/sppo_reject_loss": 2499.17724609375, | |
| "epoch": 0.9057971014492754, | |
| "grad_norm": 64595.0866483288, | |
| "learning_rate": 5.288736507014435e-09, | |
| "logits/chosen": 1.2443785667419434, | |
| "logits/rejected": 1.60662043094635, | |
| "logps/chosen": -5.438248157501221, | |
| "logps/rejected": -4.876561641693115, | |
| "loss": 5046.0918, | |
| "rewards/accuracies": 0.22499999403953552, | |
| "rewards/chosen": 4.6709272282896563e-05, | |
| "rewards/margins": 0.00013045726518612355, | |
| "rewards/rejected": -8.374801836907864e-05, | |
| "step": 250 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.4383130073547363, | |
| "debug/policy_chosen_logps": -6.456732273101807, | |
| "debug/policy_rejected_logits": 1.6063964366912842, | |
| "debug/policy_rejected_logps": -2.8695645332336426, | |
| "debug/reference_chosen_logps": -6.483206748962402, | |
| "debug/reference_rejected_logps": -2.824536085128784, | |
| "debug/sppo_chosen_loss": 2497.39501953125, | |
| "debug/sppo_chosen_reward_in_loss": 0.026474904268980026, | |
| "debug/sppo_rej_reward_in_loss": -0.045027971267700195, | |
| "debug/sppo_reject_loss": 2495.526611328125, | |
| "epoch": 0.9239130434782609, | |
| "grad_norm": 93120.71264243545, | |
| "learning_rate": 3.4718563369743213e-09, | |
| "logits/chosen": 1.4383130073547363, | |
| "logits/rejected": 1.6063964366912842, | |
| "logps/chosen": -6.456732273101807, | |
| "logps/rejected": -2.8695645332336426, | |
| "loss": 5003.5242, | |
| "rewards/accuracies": 0.30000001192092896, | |
| "rewards/chosen": 0.00026474904734641314, | |
| "rewards/margins": 0.0007150286692194641, | |
| "rewards/rejected": -0.0004502796509768814, | |
| "step": 255 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3548212051391602, | |
| "debug/policy_chosen_logps": -67.59104919433594, | |
| "debug/policy_rejected_logits": 1.9228832721710205, | |
| "debug/policy_rejected_logps": -47.09478759765625, | |
| "debug/reference_chosen_logps": -67.80909729003906, | |
| "debug/reference_rejected_logps": -47.41716384887695, | |
| "debug/sppo_chosen_loss": 2479.32470703125, | |
| "debug/sppo_chosen_reward_in_loss": 0.21804814040660858, | |
| "debug/sppo_rej_reward_in_loss": 0.32237640023231506, | |
| "debug/sppo_reject_loss": 2537.328369140625, | |
| "epoch": 0.9420289855072463, | |
| "grad_norm": 235843.0142935536, | |
| "learning_rate": 2.025351319275137e-09, | |
| "logits/chosen": 1.3548212051391602, | |
| "logits/rejected": 1.9228832721710205, | |
| "logps/chosen": -67.59104919433594, | |
| "logps/rejected": -47.09478759765625, | |
| "loss": 4997.7383, | |
| "rewards/accuracies": 0.3499999940395355, | |
| "rewards/chosen": 0.0021804815623909235, | |
| "rewards/margins": -0.0010432822164148092, | |
| "rewards/rejected": 0.0032237637788057327, | |
| "step": 260 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.5696322917938232, | |
| "debug/policy_chosen_logps": -38.5196647644043, | |
| "debug/policy_rejected_logits": 1.7193692922592163, | |
| "debug/policy_rejected_logps": -131.51307678222656, | |
| "debug/reference_chosen_logps": -38.708030700683594, | |
| "debug/reference_rejected_logps": -132.49644470214844, | |
| "debug/sppo_chosen_loss": 2481.755859375, | |
| "debug/sppo_chosen_reward_in_loss": 0.18836654722690582, | |
| "debug/sppo_rej_reward_in_loss": 0.9833652377128601, | |
| "debug/sppo_reject_loss": 2613.89111328125, | |
| "epoch": 0.9601449275362319, | |
| "grad_norm": 42745.49236462931, | |
| "learning_rate": 9.607359798384785e-10, | |
| "logits/chosen": 1.5696322917938232, | |
| "logits/rejected": 1.7193692922592163, | |
| "logps/chosen": -38.5196647644043, | |
| "logps/rejected": -131.51307678222656, | |
| "loss": 4985.8559, | |
| "rewards/accuracies": 0.30000001192092896, | |
| "rewards/chosen": 0.001883665332570672, | |
| "rewards/margins": -0.00794998649507761, | |
| "rewards/rejected": 0.009833652526140213, | |
| "step": 265 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.5161726474761963, | |
| "debug/policy_chosen_logps": -5.8967390060424805, | |
| "debug/policy_rejected_logits": 2.0397534370422363, | |
| "debug/policy_rejected_logps": -3.5706443786621094, | |
| "debug/reference_chosen_logps": -5.859536170959473, | |
| "debug/reference_rejected_logps": -3.5223605632781982, | |
| "debug/sppo_chosen_loss": 2503.755859375, | |
| "debug/sppo_chosen_reward_in_loss": -0.03720256686210632, | |
| "debug/sppo_rej_reward_in_loss": -0.04828394576907158, | |
| "debug/sppo_reject_loss": 2495.21142578125, | |
| "epoch": 0.9782608695652174, | |
| "grad_norm": 38783.68553474358, | |
| "learning_rate": 2.8648491140513267e-10, | |
| "logits/chosen": 1.5161726474761963, | |
| "logits/rejected": 2.0397534370422363, | |
| "logps/chosen": -5.8967390060424805, | |
| "logps/rejected": -3.5706443786621094, | |
| "loss": 4985.3109, | |
| "rewards/accuracies": 0.30000001192092896, | |
| "rewards/chosen": -0.00037202564999461174, | |
| "rewards/margins": 0.00011081376578658819, | |
| "rewards/rejected": -0.0004828394448850304, | |
| "step": 270 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.1615492105484009, | |
| "debug/policy_chosen_logps": -4.200676918029785, | |
| "debug/policy_rejected_logits": 1.4300708770751953, | |
| "debug/policy_rejected_logps": -11.9641752243042, | |
| "debug/reference_chosen_logps": -4.2250447273254395, | |
| "debug/reference_rejected_logps": -11.967859268188477, | |
| "debug/sppo_chosen_loss": 2497.57421875, | |
| "debug/sppo_chosen_reward_in_loss": 0.024368364363908768, | |
| "debug/sppo_rej_reward_in_loss": 0.003684788942337036, | |
| "debug/sppo_reject_loss": 2500.45068359375, | |
| "epoch": 0.9963768115942029, | |
| "grad_norm": 117686.2605109664, | |
| "learning_rate": 7.965313749835889e-12, | |
| "logits/chosen": 1.1615492105484009, | |
| "logits/rejected": 1.4300708770751953, | |
| "logps/chosen": -4.200676918029785, | |
| "logps/rejected": -11.9641752243042, | |
| "loss": 5020.2773, | |
| "rewards/accuracies": 0.30000001192092896, | |
| "rewards/chosen": 0.00024368362210225314, | |
| "rewards/margins": 0.00020683575712610036, | |
| "rewards/rejected": 3.6847894079983234e-05, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 276, | |
| "total_flos": 0.0, | |
| "train_loss": 4999.9289533089905, | |
| "train_runtime": 2038.5346, | |
| "train_samples_per_second": 8.646, | |
| "train_steps_per_second": 0.135 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 276, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |