{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9952,
  "eval_steps": 62,
  "global_step": 936,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0032,
      "grad_norm": 173.8179841991536,
      "learning_rate": 5.3191489361702125e-09,
      "logits/generated": -3.044562816619873,
      "logits/real": -2.934573173522949,
      "logps/generated": -224.50445556640625,
      "logps/real": -215.07748413085938,
      "loss": 0.883,
      "rewards/accuracies": 0.0,
      "rewards/generated": 0.0,
      "rewards/margins": 0.0,
      "rewards/real": 0.0,
      "step": 1
    },
    {
      "epoch": 0.032,
      "grad_norm": 187.3286003673625,
      "learning_rate": 5.3191489361702123e-08,
      "logits/generated": -2.855332851409912,
      "logits/real": -2.901381254196167,
      "logps/generated": -219.83998107910156,
      "logps/real": -217.93182373046875,
      "loss": 0.8931,
      "rewards/accuracies": 0.4027777910232544,
      "rewards/generated": 0.011801423504948616,
      "rewards/margins": 0.00024285909603349864,
      "rewards/real": 0.01204428169876337,
      "step": 10
    },
    {
      "epoch": 0.064,
      "grad_norm": 182.15143154088975,
      "learning_rate": 1.0638297872340425e-07,
      "logits/generated": -2.875359296798706,
      "logits/real": -2.8933231830596924,
      "logps/generated": -215.1235809326172,
      "logps/real": -206.74008178710938,
      "loss": 0.8714,
      "rewards/accuracies": 0.612500011920929,
      "rewards/generated": 0.0668320506811142,
      "rewards/margins": 0.046206969767808914,
      "rewards/real": 0.113039031624794,
      "step": 20
    },
    {
      "epoch": 0.096,
      "grad_norm": 176.75915439125325,
      "learning_rate": 1.5957446808510638e-07,
      "logits/generated": -2.8365707397460938,
      "logits/real": -2.8767635822296143,
      "logps/generated": -219.2717742919922,
      "logps/real": -205.4467010498047,
      "loss": 0.8421,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/generated": 0.06644892692565918,
      "rewards/margins": 0.27186867594718933,
      "rewards/real": 0.3383175730705261,
      "step": 30
    },
    {
      "epoch": 0.128,
      "grad_norm": 152.11425016480666,
      "learning_rate": 2.127659574468085e-07,
      "logits/generated": -2.908146619796753,
      "logits/real": -2.9528186321258545,
      "logps/generated": -225.11038208007812,
      "logps/real": -210.19442749023438,
      "loss": 0.7927,
      "rewards/accuracies": 0.625,
      "rewards/generated": 0.17625515162944794,
      "rewards/margins": 0.23418943583965302,
      "rewards/real": 0.4104446470737457,
      "step": 40
    },
    {
      "epoch": 0.16,
      "grad_norm": 138.90547258018623,
      "learning_rate": 2.659574468085106e-07,
      "logits/generated": -2.779694080352783,
      "logits/real": -2.8061487674713135,
      "logps/generated": -224.0841522216797,
      "logps/real": -208.4123992919922,
      "loss": 0.7316,
      "rewards/accuracies": 0.762499988079071,
      "rewards/generated": -0.19016078114509583,
      "rewards/margins": 0.51332688331604,
      "rewards/real": 0.3231660723686218,
      "step": 50
    },
    {
      "epoch": 0.192,
      "grad_norm": 144.96875010439092,
      "learning_rate": 3.1914893617021275e-07,
      "logits/generated": -2.7721309661865234,
      "logits/real": -2.7228846549987793,
      "logps/generated": -226.5691375732422,
      "logps/real": -211.42724609375,
      "loss": 0.6898,
      "rewards/accuracies": 0.737500011920929,
      "rewards/generated": -0.36448976397514343,
      "rewards/margins": 0.680444598197937,
      "rewards/real": 0.3159547448158264,
      "step": 60
    },
    {
      "epoch": 0.1984,
      "eval_logits/generated": -2.613921642303467,
      "eval_logits/real": -2.5605974197387695,
      "eval_logps/generated": -246.04136657714844,
      "eval_logps/real": -211.02076721191406,
      "eval_loss": 0.6845277547836304,
      "eval_rewards/accuracies": 0.8269230723381042,
      "eval_rewards/generated": -1.2271192073822021,
      "eval_rewards/margins": 1.3587048053741455,
      "eval_rewards/real": 0.13158567249774933,
      "eval_runtime": 27.6315,
      "eval_samples_per_second": 7.238,
      "eval_steps_per_second": 0.47,
      "step": 62
    },
    {
      "epoch": 0.224,
      "grad_norm": 220.14070132691822,
      "learning_rate": 3.7234042553191484e-07,
      "logits/generated": -2.3481838703155518,
      "logits/real": -2.5661604404449463,
      "logps/generated": -213.6888885498047,
      "logps/real": -211.11782836914062,
      "loss": 0.6221,
      "rewards/accuracies": 0.824999988079071,
      "rewards/generated": -1.1715154647827148,
      "rewards/margins": 1.232180118560791,
      "rewards/real": 0.06066470220685005,
      "step": 70
    },
    {
      "epoch": 0.256,
      "grad_norm": 111.68074396571573,
      "learning_rate": 4.25531914893617e-07,
      "logits/generated": -2.302609920501709,
      "logits/real": -2.4217114448547363,
      "logps/generated": -227.2515411376953,
      "logps/real": -206.6273193359375,
      "loss": 0.6168,
      "rewards/accuracies": 0.800000011920929,
      "rewards/generated": -1.4599227905273438,
      "rewards/margins": 1.5430281162261963,
      "rewards/real": 0.08310532569885254,
      "step": 80
    },
    {
      "epoch": 0.288,
      "grad_norm": 144.7889636789569,
      "learning_rate": 4.787234042553192e-07,
      "logits/generated": -2.564621925354004,
      "logits/real": -2.6837966442108154,
      "logps/generated": -210.40658569335938,
      "logps/real": -194.1896514892578,
      "loss": 0.6247,
      "rewards/accuracies": 0.800000011920929,
      "rewards/generated": 0.20705755054950714,
      "rewards/margins": 1.2522168159484863,
      "rewards/real": 1.4592745304107666,
      "step": 90
    },
    {
      "epoch": 0.32,
      "grad_norm": 182.29029715268064,
      "learning_rate": 4.96437054631829e-07,
      "logits/generated": -2.6423590183258057,
      "logits/real": -2.5828661918640137,
      "logps/generated": -240.5973358154297,
      "logps/real": -204.6890106201172,
      "loss": 0.5837,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/generated": -0.983971118927002,
      "rewards/margins": 1.757327675819397,
      "rewards/real": 0.773356556892395,
      "step": 100
    },
    {
      "epoch": 0.352,
      "grad_norm": 127.78446444265371,
      "learning_rate": 4.904988123515439e-07,
      "logits/generated": -2.503037452697754,
      "logits/real": -2.5519888401031494,
      "logps/generated": -231.2539825439453,
      "logps/real": -182.541259765625,
      "loss": 0.4966,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/generated": -1.0986322164535522,
      "rewards/margins": 2.336041212081909,
      "rewards/real": 1.2374088764190674,
      "step": 110
    },
    {
      "epoch": 0.384,
      "grad_norm": 131.16498234786013,
      "learning_rate": 4.845605700712589e-07,
      "logits/generated": -2.506398916244507,
      "logits/real": -2.586212158203125,
      "logps/generated": -226.7395782470703,
      "logps/real": -199.89138793945312,
      "loss": 0.5637,
      "rewards/accuracies": 0.824999988079071,
      "rewards/generated": -1.1938557624816895,
      "rewards/margins": 2.05814790725708,
      "rewards/real": 0.8642921447753906,
      "step": 120
    },
    {
      "epoch": 0.3968,
      "eval_logits/generated": -2.9187638759613037,
      "eval_logits/real": -2.9273226261138916,
      "eval_logps/generated": -243.90988159179688,
      "eval_logps/real": -193.29714965820312,
      "eval_loss": 0.6090801358222961,
      "eval_rewards/accuracies": 0.9230769276618958,
      "eval_rewards/generated": -1.0139696598052979,
      "eval_rewards/margins": 2.917917013168335,
      "eval_rewards/real": 1.9039475917816162,
      "eval_runtime": 36.4245,
      "eval_samples_per_second": 5.491,
      "eval_steps_per_second": 0.357,
      "step": 124
    },
    {
      "epoch": 0.416,
      "grad_norm": 322.99075339676534,
      "learning_rate": 4.786223277909738e-07,
      "logits/generated": -2.9123921394348145,
      "logits/real": -2.95102596282959,
      "logps/generated": -231.7545928955078,
      "logps/real": -189.8568572998047,
      "loss": 0.5005,
      "rewards/accuracies": 0.875,
      "rewards/generated": -0.3055388331413269,
      "rewards/margins": 2.323357343673706,
      "rewards/real": 2.0178186893463135,
      "step": 130
    },
    {
      "epoch": 0.448,
      "grad_norm": 132.33253209314802,
      "learning_rate": 4.7268408551068883e-07,
      "logits/generated": -2.9417192935943604,
      "logits/real": -2.8813247680664062,
      "logps/generated": -233.68234252929688,
      "logps/real": -182.17855834960938,
      "loss": 0.5195,
      "rewards/accuracies": 0.875,
      "rewards/generated": -0.8809671401977539,
      "rewards/margins": 2.567401170730591,
      "rewards/real": 1.686434030532837,
      "step": 140
    },
    {
      "epoch": 0.48,
      "grad_norm": 111.61491160388277,
      "learning_rate": 4.667458432304038e-07,
      "logits/generated": -2.887362003326416,
      "logits/real": -2.8390328884124756,
      "logps/generated": -241.1803436279297,
      "logps/real": -189.00759887695312,
      "loss": 0.4997,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": -1.1145679950714111,
      "rewards/margins": 2.616346597671509,
      "rewards/real": 1.5017783641815186,
      "step": 150
    },
    {
      "epoch": 0.512,
      "grad_norm": 106.85713874892005,
      "learning_rate": 4.6080760095011875e-07,
      "logits/generated": -2.8256986141204834,
      "logits/real": -2.923900604248047,
      "logps/generated": -232.5541229248047,
      "logps/real": -191.5264129638672,
      "loss": 0.4483,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": -1.3177084922790527,
      "rewards/margins": 3.0125045776367188,
      "rewards/real": 1.6947962045669556,
      "step": 160
    },
    {
      "epoch": 0.544,
      "grad_norm": 107.15615684807614,
      "learning_rate": 4.548693586698337e-07,
      "logits/generated": -2.6653618812561035,
      "logits/real": -2.761810779571533,
      "logps/generated": -251.56607055664062,
      "logps/real": -205.3849334716797,
      "loss": 0.4207,
      "rewards/accuracies": 0.875,
      "rewards/generated": -2.9436697959899902,
      "rewards/margins": 4.08522891998291,
      "rewards/real": 1.14155912399292,
      "step": 170
    },
    {
      "epoch": 0.576,
      "grad_norm": 55.76896918076635,
      "learning_rate": 4.4893111638954866e-07,
      "logits/generated": -2.157531261444092,
      "logits/real": -2.4729576110839844,
      "logps/generated": -231.6317138671875,
      "logps/real": -206.774169921875,
      "loss": 0.4765,
      "rewards/accuracies": 0.862500011920929,
      "rewards/generated": -2.9688258171081543,
      "rewards/margins": 3.432262420654297,
      "rewards/real": 0.4634362757205963,
      "step": 180
    },
    {
      "epoch": 0.5952,
      "eval_logits/generated": -2.605010747909546,
      "eval_logits/real": -2.619339942932129,
      "eval_logps/generated": -262.9011535644531,
      "eval_logps/real": -196.02049255371094,
      "eval_loss": 0.4901280999183655,
      "eval_rewards/accuracies": 0.9615384340286255,
      "eval_rewards/generated": -2.9130964279174805,
      "eval_rewards/margins": 4.544712066650391,
      "eval_rewards/real": 1.631615400314331,
      "eval_runtime": 39.3785,
      "eval_samples_per_second": 5.079,
      "eval_steps_per_second": 0.33,
      "step": 186
    },
    {
      "epoch": 0.608,
      "grad_norm": 119.00624075528464,
      "learning_rate": 4.429928741092636e-07,
      "logits/generated": -2.3042728900909424,
      "logits/real": -2.5178918838500977,
      "logps/generated": -236.9404296875,
      "logps/real": -190.45376586914062,
      "loss": 0.4552,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": -2.7708938121795654,
      "rewards/margins": 4.090048789978027,
      "rewards/real": 1.3191547393798828,
      "step": 190
    },
    {
      "epoch": 0.64,
      "grad_norm": 129.91828086633245,
      "learning_rate": 4.3705463182897863e-07,
      "logits/generated": -2.398952007293701,
      "logits/real": -2.480475902557373,
      "logps/generated": -243.36251831054688,
      "logps/real": -192.48858642578125,
      "loss": 0.4558,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": -2.80302095413208,
      "rewards/margins": 4.1225690841674805,
      "rewards/real": 1.3195478916168213,
      "step": 200
    },
    {
      "epoch": 0.672,
      "grad_norm": 96.83142816186717,
      "learning_rate": 4.311163895486936e-07,
      "logits/generated": -2.4865310192108154,
      "logits/real": -2.585660457611084,
      "logps/generated": -250.3340301513672,
      "logps/real": -199.3630828857422,
      "loss": 0.3981,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": -2.9170405864715576,
      "rewards/margins": 3.853332996368408,
      "rewards/real": 0.9362924695014954,
      "step": 210
    },
    {
      "epoch": 0.704,
      "grad_norm": 144.03048951181958,
      "learning_rate": 4.251781472684085e-07,
      "logits/generated": -2.5723328590393066,
      "logits/real": -2.697535991668701,
      "logps/generated": -243.3986358642578,
      "logps/real": -194.57986450195312,
      "loss": 0.4331,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": -2.856130838394165,
      "rewards/margins": 4.286919593811035,
      "rewards/real": 1.4307892322540283,
      "step": 220
    },
    {
      "epoch": 0.736,
      "grad_norm": 144.96406268446455,
      "learning_rate": 4.192399049881235e-07,
      "logits/generated": -2.320840835571289,
      "logits/real": -2.6588072776794434,
      "logps/generated": -243.00350952148438,
      "logps/real": -194.57839965820312,
      "loss": 0.4393,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": -2.843940258026123,
      "rewards/margins": 4.278311729431152,
      "rewards/real": 1.4343717098236084,
      "step": 230
    },
    {
      "epoch": 0.768,
      "grad_norm": 100.44410130642852,
      "learning_rate": 4.1330166270783846e-07,
      "logits/generated": -2.5625109672546387,
      "logits/real": -2.7491722106933594,
      "logps/generated": -237.99240112304688,
      "logps/real": -195.57760620117188,
      "loss": 0.4421,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": -2.088608741760254,
      "rewards/margins": 3.674595355987549,
      "rewards/real": 1.5859863758087158,
      "step": 240
    },
    {
      "epoch": 0.7936,
      "eval_logits/generated": -2.547682285308838,
      "eval_logits/real": -2.5049173831939697,
      "eval_logps/generated": -271.4653015136719,
      "eval_logps/real": -203.58847045898438,
      "eval_loss": 0.4295947253704071,
      "eval_rewards/accuracies": 0.942307710647583,
      "eval_rewards/generated": -3.769508123397827,
      "eval_rewards/margins": 4.644325256347656,
      "eval_rewards/real": 0.8748170137405396,
      "eval_runtime": 37.4311,
      "eval_samples_per_second": 5.343,
      "eval_steps_per_second": 0.347,
      "step": 248
    },
    {
      "epoch": 0.8,
      "grad_norm": 85.68852683234162,
      "learning_rate": 4.0736342042755347e-07,
      "logits/generated": -2.3199338912963867,
      "logits/real": -2.4098474979400635,
      "logps/generated": -257.68133544921875,
      "logps/real": -199.05691528320312,
      "loss": 0.4287,
      "rewards/accuracies": 0.887499988079071,
      "rewards/generated": -3.8252933025360107,
      "rewards/margins": 4.427876949310303,
      "rewards/real": 0.6025834679603577,
      "step": 250
    },
    {
      "epoch": 0.832,
      "grad_norm": 106.66951246973989,
      "learning_rate": 4.0142517814726837e-07,
      "logits/generated": -2.3064651489257812,
      "logits/real": -2.488039970397949,
      "logps/generated": -258.024169921875,
      "logps/real": -201.6908416748047,
      "loss": 0.3807,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/generated": -4.099714756011963,
      "rewards/margins": 4.959208965301514,
      "rewards/real": 0.8594940304756165,
      "step": 260
    },
    {
      "epoch": 0.864,
      "grad_norm": 94.69352029008796,
      "learning_rate": 3.9548693586698333e-07,
      "logits/generated": -2.3191676139831543,
      "logits/real": -2.5164597034454346,
      "logps/generated": -251.24502563476562,
      "logps/real": -202.2902069091797,
      "loss": 0.392,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": -3.548872470855713,
      "rewards/margins": 4.431762218475342,
      "rewards/real": 0.8828901052474976,
      "step": 270
    },
    {
      "epoch": 0.896,
      "grad_norm": 101.12544596486202,
      "learning_rate": 3.8954869358669834e-07,
      "logits/generated": -2.3317694664001465,
      "logits/real": -2.4163994789123535,
      "logps/generated": -267.6225891113281,
      "logps/real": -196.59249877929688,
      "loss": 0.3761,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": -5.0150322914123535,
      "rewards/margins": 5.4737958908081055,
      "rewards/real": 0.4587637484073639,
      "step": 280
    },
    {
      "epoch": 0.928,
      "grad_norm": 58.98712901817461,
      "learning_rate": 3.836104513064133e-07,
      "logits/generated": -2.4430880546569824,
      "logits/real": -2.5380032062530518,
      "logps/generated": -254.8027801513672,
      "logps/real": -190.285888671875,
      "loss": 0.3646,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -2.9770331382751465,
      "rewards/margins": 4.5045061111450195,
      "rewards/real": 1.527472734451294,
      "step": 290
    },
    {
      "epoch": 0.96,
      "grad_norm": 100.96969095117763,
      "learning_rate": 3.7767220902612825e-07,
      "logits/generated": -2.366410970687866,
      "logits/real": -2.426678419113159,
      "logps/generated": -253.49923706054688,
      "logps/real": -190.0432891845703,
      "loss": 0.4157,
      "rewards/accuracies": 0.862500011920929,
      "rewards/generated": -3.769308567047119,
      "rewards/margins": 4.8213348388671875,
      "rewards/real": 1.0520257949829102,
      "step": 300
    },
    {
      "epoch": 0.992,
      "grad_norm": 91.25079784073122,
      "learning_rate": 3.717339667458432e-07,
      "logits/generated": -2.2679662704467773,
      "logits/real": -2.420544385910034,
      "logps/generated": -258.3539123535156,
      "logps/real": -192.3536834716797,
      "loss": 0.4329,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -4.084817886352539,
      "rewards/margins": 5.127682685852051,
      "rewards/real": 1.0428647994995117,
      "step": 310
    },
    {
      "epoch": 0.992,
      "eval_logits/generated": -2.4849164485931396,
      "eval_logits/real": -2.477924346923828,
      "eval_logps/generated": -266.6431884765625,
      "eval_logps/real": -195.02626037597656,
      "eval_loss": 0.388497531414032,
      "eval_rewards/accuracies": 0.9807692170143127,
      "eval_rewards/generated": -3.2873005867004395,
      "eval_rewards/margins": 5.018336772918701,
      "eval_rewards/real": 1.7310370206832886,
      "eval_runtime": 37.6914,
      "eval_samples_per_second": 5.306,
      "eval_steps_per_second": 0.345,
      "step": 310
    },
    {
      "epoch": 1.024,
      "grad_norm": 26.63030901621953,
      "learning_rate": 3.6579572446555817e-07,
      "logits/generated": -2.6036505699157715,
      "logits/real": -2.7402613162994385,
      "logps/generated": -266.7486572265625,
      "logps/real": -183.19754028320312,
      "loss": 0.2564,
      "rewards/accuracies": 1.0,
      "rewards/generated": -3.7132019996643066,
      "rewards/margins": 6.329413890838623,
      "rewards/real": 2.6162121295928955,
      "step": 320
    },
    {
      "epoch": 1.056,
      "grad_norm": 25.508302998825403,
      "learning_rate": 3.598574821852731e-07,
      "logits/generated": -2.413361072540283,
      "logits/real": -2.813431739807129,
      "logps/generated": -236.74295043945312,
      "logps/real": -174.03030395507812,
      "loss": 0.1948,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.714395523071289,
      "rewards/margins": 6.820935249328613,
      "rewards/real": 4.106539726257324,
      "step": 330
    },
    {
      "epoch": 1.088,
      "grad_norm": 6.903205755336687,
      "learning_rate": 3.5391923990498813e-07,
      "logits/generated": -2.4386377334594727,
      "logits/real": -2.5851504802703857,
      "logps/generated": -249.6878204345703,
      "logps/real": -167.23617553710938,
      "loss": 0.1944,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -2.9891231060028076,
      "rewards/margins": 6.840689659118652,
      "rewards/real": 3.851566791534424,
      "step": 340
    },
    {
      "epoch": 1.12,
      "grad_norm": 12.781012688853895,
      "learning_rate": 3.479809976247031e-07,
      "logits/generated": -2.563267469406128,
      "logits/real": -2.8669376373291016,
      "logps/generated": -256.77777099609375,
      "logps/real": -164.88247680664062,
      "loss": 0.1877,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -2.7024502754211426,
      "rewards/margins": 6.864438056945801,
      "rewards/real": 4.161987781524658,
      "step": 350
    },
    {
      "epoch": 1.152,
      "grad_norm": 6.4656216930208465,
      "learning_rate": 3.42042755344418e-07,
      "logits/generated": -2.5356485843658447,
      "logits/real": -2.7551302909851074,
      "logps/generated": -241.5117950439453,
      "logps/real": -155.2963409423828,
      "loss": 0.1866,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -2.0638256072998047,
      "rewards/margins": 6.775469779968262,
      "rewards/real": 4.711644172668457,
      "step": 360
    },
    {
      "epoch": 1.184,
      "grad_norm": 13.80481092376025,
      "learning_rate": 3.36104513064133e-07,
      "logits/generated": -2.444866418838501,
      "logits/real": -2.7210774421691895,
      "logps/generated": -237.6660919189453,
      "logps/real": -159.84698486328125,
      "loss": 0.192,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -2.4937832355499268,
      "rewards/margins": 7.103081703186035,
      "rewards/real": 4.609298229217529,
      "step": 370
    },
    {
      "epoch": 1.1904,
      "eval_logits/generated": -2.69924259185791,
      "eval_logits/real": -2.7275919914245605,
      "eval_logps/generated": -239.61849975585938,
      "eval_logps/real": -169.785888671875,
      "eval_loss": 0.4324784576892853,
      "eval_rewards/accuracies": 0.9230769276618958,
      "eval_rewards/generated": -0.5848276615142822,
      "eval_rewards/margins": 4.839901447296143,
      "eval_rewards/real": 4.255073547363281,
      "eval_runtime": 37.2563,
      "eval_samples_per_second": 5.368,
      "eval_steps_per_second": 0.349,
      "step": 372
    },
    {
      "epoch": 1.216,
      "grad_norm": 21.131804690810736,
      "learning_rate": 3.3016627078384796e-07,
      "logits/generated": -2.5392134189605713,
      "logits/real": -2.8252758979797363,
      "logps/generated": -240.2849884033203,
      "logps/real": -158.17660522460938,
      "loss": 0.1786,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.2793164253234863,
      "rewards/margins": 7.093780517578125,
      "rewards/real": 4.814464092254639,
      "step": 380
    },
    {
      "epoch": 1.248,
      "grad_norm": 18.209600390373854,
      "learning_rate": 3.2422802850356297e-07,
      "logits/generated": -2.569019317626953,
      "logits/real": -2.6191489696502686,
      "logps/generated": -244.01516723632812,
      "logps/real": -164.8138885498047,
      "loss": 0.1963,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -2.2556376457214355,
      "rewards/margins": 6.541670322418213,
      "rewards/real": 4.286032676696777,
      "step": 390
    },
    {
      "epoch": 1.28,
      "grad_norm": 52.08582313155134,
      "learning_rate": 3.182897862232779e-07,
      "logits/generated": -2.4035823345184326,
      "logits/real": -2.676548480987549,
      "logps/generated": -242.1820526123047,
      "logps/real": -156.36715698242188,
      "loss": 0.2042,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.7515151500701904,
      "rewards/margins": 7.663416385650635,
      "rewards/real": 4.911900520324707,
      "step": 400
    },
    {
      "epoch": 1.312,
      "grad_norm": 47.612591038684926,
      "learning_rate": 3.1235154394299283e-07,
      "logits/generated": -2.3149709701538086,
      "logits/real": -2.7124149799346924,
      "logps/generated": -240.0115509033203,
      "logps/real": -144.69625854492188,
      "loss": 0.1793,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.0378785133361816,
      "rewards/margins": 7.867486000061035,
      "rewards/real": 5.829607963562012,
      "step": 410
    },
    {
      "epoch": 1.3439999999999999,
      "grad_norm": 30.14318328882392,
      "learning_rate": 3.0641330166270784e-07,
      "logits/generated": -2.373650074005127,
      "logits/real": -2.6924972534179688,
      "logps/generated": -249.2195587158203,
      "logps/real": -161.0720672607422,
      "loss": 0.2033,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.716108798980713,
      "rewards/margins": 7.4836602210998535,
      "rewards/real": 4.767552375793457,
      "step": 420
    },
    {
      "epoch": 1.376,
      "grad_norm": 22.124267537528457,
      "learning_rate": 3.004750593824228e-07,
      "logits/generated": -2.4978554248809814,
      "logits/real": -2.710461139678955,
      "logps/generated": -239.83505249023438,
      "logps/real": -150.0258026123047,
      "loss": 0.1832,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -2.2577767372131348,
      "rewards/margins": 7.054965019226074,
      "rewards/real": 4.797187805175781,
      "step": 430
    },
    {
      "epoch": 1.3888,
      "eval_logits/generated": -2.635890483856201,
      "eval_logits/real": -2.659679412841797,
      "eval_logps/generated": -244.70217895507812,
      "eval_logps/real": -172.03485107421875,
      "eval_loss": 0.39648282527923584,
      "eval_rewards/accuracies": 0.9038461446762085,
      "eval_rewards/generated": -1.0931975841522217,
      "eval_rewards/margins": 5.1233744621276855,
      "eval_rewards/real": 4.030177116394043,
      "eval_runtime": 37.9803,
      "eval_samples_per_second": 5.266,
      "eval_steps_per_second": 0.342,
      "step": 434
    },
    {
      "epoch": 1.408,
      "grad_norm": 13.803067859455108,
      "learning_rate": 2.9453681710213776e-07,
      "logits/generated": -2.4936466217041016,
      "logits/real": -2.684986114501953,
      "logps/generated": -250.47024536132812,
      "logps/real": -158.733642578125,
      "loss": 0.1927,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.6509931087493896,
      "rewards/margins": 7.874148368835449,
      "rewards/real": 5.223154544830322,
      "step": 440
    },
    {
      "epoch": 1.44,
      "grad_norm": 18.24198614703468,
      "learning_rate": 2.885985748218527e-07,
      "logits/generated": -2.2589638233184814,
      "logits/real": -2.4937446117401123,
      "logps/generated": -234.1324462890625,
      "logps/real": -149.9835205078125,
      "loss": 0.1787,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -2.281383991241455,
      "rewards/margins": 7.3518171310424805,
      "rewards/real": 5.0704345703125,
      "step": 450
    },
    {
      "epoch": 1.472,
      "grad_norm": 36.549728538511495,
      "learning_rate": 2.8266033254156767e-07,
      "logits/generated": -2.4578120708465576,
      "logits/real": -2.6368680000305176,
      "logps/generated": -247.98818969726562,
      "logps/real": -158.1073455810547,
      "loss": 0.1795,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.7464351654052734,
      "rewards/margins": 7.739925384521484,
      "rewards/real": 4.993489742279053,
      "step": 460
    },
    {
      "epoch": 1.504,
      "grad_norm": 20.234495322356356,
      "learning_rate": 2.7672209026128263e-07,
      "logits/generated": -2.3000855445861816,
      "logits/real": -2.6723475456237793,
      "logps/generated": -242.532470703125,
      "logps/real": -154.8021697998047,
      "loss": 0.1684,
      "rewards/accuracies": 1.0,
      "rewards/generated": -3.049163341522217,
      "rewards/margins": 8.374896049499512,
      "rewards/real": 5.325732231140137,
      "step": 470
    },
    {
      "epoch": 1.536,
      "grad_norm": 59.79337914893289,
      "learning_rate": 2.7078384798099764e-07,
      "logits/generated": -2.4476964473724365,
      "logits/real": -2.6964683532714844,
      "logps/generated": -239.89083862304688,
      "logps/real": -158.37881469726562,
      "loss": 0.1846,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -2.601045608520508,
      "rewards/margins": 7.892436981201172,
      "rewards/real": 5.291390895843506,
      "step": 480
    },
    {
      "epoch": 1.568,
      "grad_norm": 5.685979081085808,
      "learning_rate": 2.648456057007126e-07,
      "logits/generated": -2.3954477310180664,
      "logits/real": -2.6281771659851074,
      "logps/generated": -239.33688354492188,
      "logps/real": -152.04110717773438,
      "loss": 0.1759,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -2.8380813598632812,
      "rewards/margins": 7.7762017250061035,
      "rewards/real": 4.938119888305664,
      "step": 490
    },
    {
      "epoch": 1.5872000000000002,
      "eval_logits/generated": -2.509535074234009,
      "eval_logits/real": -2.576845407485962,
      "eval_logps/generated": -246.48858642578125,
      "eval_logps/real": -166.0556640625,
      "eval_loss": 0.40290915966033936,
      "eval_rewards/accuracies": 0.9038461446762085,
      "eval_rewards/generated": -1.2718371152877808,
      "eval_rewards/margins": 5.899932861328125,
      "eval_rewards/real": 4.628096580505371,
      "eval_runtime": 38.6505,
      "eval_samples_per_second": 5.175,
      "eval_steps_per_second": 0.336,
      "step": 496
    },
    {
      "epoch": 1.6,
      "grad_norm": 30.570197415344,
      "learning_rate": 2.589073634204275e-07,
      "logits/generated": -2.497969150543213,
      "logits/real": -2.734889268875122,
      "logps/generated": -244.86740112304688,
      "logps/real": -145.41070556640625,
      "loss": 0.1762,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.2094335556030273,
      "rewards/margins": 8.094371795654297,
      "rewards/real": 5.8849382400512695,
      "step": 500
    },
    {
      "epoch": 1.6320000000000001,
      "grad_norm": 26.88929423699929,
      "learning_rate": 2.529691211401425e-07,
      "logits/generated": -2.2768752574920654,
      "logits/real": -2.5635409355163574,
      "logps/generated": -232.8406982421875,
      "logps/real": -154.79733276367188,
      "loss": 0.1739,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.6889169216156006,
      "rewards/margins": 7.117257118225098,
      "rewards/real": 5.428339958190918,
      "step": 510
    },
    {
      "epoch": 1.6640000000000001,
      "grad_norm": 31.657820457879357,
      "learning_rate": 2.4703087885985747e-07,
      "logits/generated": -2.25846791267395,
      "logits/real": -2.707723379135132,
      "logps/generated": -228.5704803466797,
      "logps/real": -151.35678100585938,
      "loss": 0.1881,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.787634253501892,
      "rewards/margins": 7.234114170074463,
      "rewards/real": 5.446479797363281,
      "step": 520
    },
    {
      "epoch": 1.696,
      "grad_norm": 8.396789968303967,
      "learning_rate": 2.410926365795724e-07,
      "logits/generated": -2.599165201187134,
      "logits/real": -2.841097354888916,
      "logps/generated": -236.2193603515625,
      "logps/real": -158.02024841308594,
      "loss": 0.1817,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.7233753204345703,
      "rewards/margins": 7.430352687835693,
      "rewards/real": 5.706976890563965,
      "step": 530
    },
    {
      "epoch": 1.728,
      "grad_norm": 6.004959519833077,
      "learning_rate": 2.351543942992874e-07,
      "logits/generated": -2.4057235717773438,
      "logits/real": -2.5695137977600098,
      "logps/generated": -234.90634155273438,
      "logps/real": -142.76551818847656,
      "loss": 0.1755,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -1.6140129566192627,
      "rewards/margins": 7.1941680908203125,
      "rewards/real": 5.580154895782471,
      "step": 540
    },
    {
      "epoch": 1.76,
      "grad_norm": 21.598338411496478,
      "learning_rate": 2.2921615201900234e-07,
      "logits/generated": -2.502744197845459,
      "logits/real": -2.7131333351135254,
      "logps/generated": -234.8448944091797,
      "logps/real": -151.5326385498047,
      "loss": 0.1911,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -1.6365852355957031,
      "rewards/margins": 6.960441589355469,
      "rewards/real": 5.323855400085449,
      "step": 550
    },
    {
      "epoch": 1.7856,
      "eval_logits/generated": -2.7026188373565674,
      "eval_logits/real": -2.8069300651550293,
      "eval_logps/generated": -243.658447265625,
      "eval_logps/real": -164.40818786621094,
      "eval_loss": 0.42809945344924927,
      "eval_rewards/accuracies": 0.9230769276618958,
      "eval_rewards/generated": -0.9888254404067993,
      "eval_rewards/margins": 5.781670570373535,
      "eval_rewards/real": 4.792846202850342,
      "eval_runtime": 36.19,
      "eval_samples_per_second": 5.526,
      "eval_steps_per_second": 0.359,
      "step": 558
    },
    {
      "epoch": 1.792,
      "grad_norm": 34.77016415698945,
      "learning_rate": 2.2327790973871732e-07,
      "logits/generated": -2.324758529663086,
      "logits/real": -2.811093330383301,
      "logps/generated": -231.7982177734375,
      "logps/real": -156.2529296875,
      "loss": 0.1897,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -2.3056435585021973,
      "rewards/margins": 7.733015537261963,
      "rewards/real": 5.427372932434082,
      "step": 560
    },
    {
      "epoch": 1.8239999999999998,
      "grad_norm": 26.476604255451427,
      "learning_rate": 2.173396674584323e-07,
      "logits/generated": -2.5880885124206543,
      "logits/real": -2.7699248790740967,
      "logps/generated": -230.7294158935547,
      "logps/real": -143.07369995117188,
      "loss": 0.1971,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -1.5589967966079712,
      "rewards/margins": 7.328855991363525,
      "rewards/real": 5.7698588371276855,
      "step": 570
    },
    {
      "epoch": 1.8559999999999999,
      "grad_norm": 58.58666966860945,
      "learning_rate": 2.1140142517814726e-07,
      "logits/generated": -2.570673942565918,
      "logits/real": -2.747009515762329,
      "logps/generated": -237.0471649169922,
      "logps/real": -140.8243408203125,
      "loss": 0.1734,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.633213758468628,
      "rewards/margins": 7.265746116638184,
      "rewards/real": 5.632533073425293,
      "step": 580
    },
    {
      "epoch": 1.888,
      "grad_norm": 31.35579764952576,
      "learning_rate": 2.0546318289786222e-07,
      "logits/generated": -2.578850269317627,
      "logits/real": -2.7570090293884277,
      "logps/generated": -233.3253631591797,
      "logps/real": -154.92361450195312,
      "loss": 0.1786,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.7253196239471436,
      "rewards/margins": 7.71604061126709,
      "rewards/real": 5.990720272064209,
      "step": 590
    },
    {
      "epoch": 1.92,
      "grad_norm": 9.736079557350774,
      "learning_rate": 1.9952494061757718e-07,
      "logits/generated": -2.5344178676605225,
      "logits/real": -2.8576979637145996,
      "logps/generated": -243.61709594726562,
      "logps/real": -163.37904357910156,
      "loss": 0.1789,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.3976175785064697,
      "rewards/margins": 8.051566123962402,
      "rewards/real": 5.6539483070373535,
      "step": 600
    },
    {
      "epoch": 1.952,
      "grad_norm": 6.250640631355344,
      "learning_rate": 1.9358669833729216e-07,
      "logits/generated": -2.4879064559936523,
      "logits/real": -2.5746703147888184,
      "logps/generated": -229.38784790039062,
      "logps/real": -137.59402465820312,
      "loss": 0.1627,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.879066824913025,
      "rewards/margins": 8.053318977355957,
      "rewards/real": 6.174252510070801,
      "step": 610
    },
    {
      "epoch": 1.984,
      "grad_norm": 41.72205488261495,
      "learning_rate": 1.876484560570071e-07,
      "logits/generated": -2.471391439437866,
      "logits/real": -2.729846239089966,
      "logps/generated": -242.6791534423828,
      "logps/real": -157.15383911132812,
      "loss": 0.1719,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -1.889248251914978,
      "rewards/margins": 7.926047325134277,
      "rewards/real": 6.036799430847168,
      "step": 620
    },
    {
      "epoch": 1.984,
      "eval_logits/generated": -2.633439779281616,
      "eval_logits/real": -2.674652576446533,
      "eval_logps/generated": -233.05728149414062,
      "eval_logps/real": -158.04676818847656,
      "eval_loss": 0.4521886110305786,
      "eval_rewards/accuracies": 0.8653846383094788,
      "eval_rewards/generated": 0.07129351049661636,
      "eval_rewards/margins": 5.357695579528809,
      "eval_rewards/real": 5.428988456726074,
      "eval_runtime": 36.8213,
      "eval_samples_per_second": 5.432,
      "eval_steps_per_second": 0.353,
      "step": 620
    },
    {
      "epoch": 2.016,
      "grad_norm": 8.304945281318759,
      "learning_rate": 1.8171021377672207e-07,
      "logits/generated": -2.511307954788208,
      "logits/real": -2.589796543121338,
      "logps/generated": -246.64413452148438,
      "logps/real": -139.1671142578125,
      "loss": 0.1429,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.2198785543441772,
      "rewards/margins": 7.896165370941162,
      "rewards/real": 6.676286220550537,
      "step": 630
    },
    {
      "epoch": 2.048,
      "grad_norm": 8.526179288519483,
      "learning_rate": 1.7577197149643706e-07,
      "logits/generated": -2.50537371635437,
      "logits/real": -2.675524950027466,
      "logps/generated": -236.70230102539062,
      "logps/real": -123.74015808105469,
      "loss": 0.1303,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.25620436668396,
      "rewards/margins": 8.749616622924805,
      "rewards/real": 7.493411064147949,
      "step": 640
    },
    {
      "epoch": 2.08,
      "grad_norm": 3.860348679481527,
      "learning_rate": 1.6983372921615202e-07,
      "logits/generated": -2.595600128173828,
      "logits/real": -2.7861080169677734,
      "logps/generated": -238.60336303710938,
      "logps/real": -130.31980895996094,
      "loss": 0.1339,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.3490166664123535,
      "rewards/margins": 9.323174476623535,
      "rewards/real": 7.974156379699707,
      "step": 650
    },
    {
      "epoch": 2.112,
      "grad_norm": 8.471083443166995,
      "learning_rate": 1.6389548693586697e-07,
      "logits/generated": -2.1889586448669434,
      "logits/real": -2.62569260597229,
      "logps/generated": -226.10330200195312,
      "logps/real": -125.58138275146484,
      "loss": 0.1306,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.2672169208526611,
      "rewards/margins": 8.960881233215332,
      "rewards/real": 7.693663597106934,
      "step": 660
    },
    {
      "epoch": 2.144,
      "grad_norm": 5.982391100393358,
      "learning_rate": 1.5795724465558193e-07,
      "logits/generated": -2.3514654636383057,
      "logits/real": -2.665621280670166,
      "logps/generated": -229.26901245117188,
      "logps/real": -131.5380859375,
      "loss": 0.1314,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.7971032857894897,
      "rewards/margins": 8.747750282287598,
      "rewards/real": 7.950647830963135,
      "step": 670
    },
    {
      "epoch": 2.176,
      "grad_norm": 21.360706528163764,
      "learning_rate": 1.520190023752969e-07,
      "logits/generated": -2.3648266792297363,
      "logits/real": -2.7507741451263428,
      "logps/generated": -224.5431671142578,
      "logps/real": -127.42989349365234,
      "loss": 0.1363,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.1734375953674316,
      "rewards/margins": 9.078557014465332,
      "rewards/real": 7.905119895935059,
      "step": 680
    },
    {
      "epoch": 2.1824,
      "eval_logits/generated": -2.51108980178833,
      "eval_logits/real": -2.632188320159912,
      "eval_logps/generated": -224.7698516845703,
      "eval_logps/real": -150.33514404296875,
      "eval_loss": 0.46486639976501465,
      "eval_rewards/accuracies": 0.8846153616905212,
      "eval_rewards/generated": 0.9000343084335327,
      "eval_rewards/margins": 5.300115585327148,
      "eval_rewards/real": 6.2001495361328125,
      "eval_runtime": 37.0488,
      "eval_samples_per_second": 5.398,
      "eval_steps_per_second": 0.351,
      "step": 682
    },
    {
      "epoch": 2.208,
      "grad_norm": 21.559711913039376,
      "learning_rate": 1.4608076009501184e-07,
      "logits/generated": -2.503950595855713,
      "logits/real": -2.7691397666931152,
      "logps/generated": -230.25411987304688,
      "logps/real": -133.43069458007812,
      "loss": 0.1345,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.6242657899856567,
      "rewards/margins": 8.57276439666748,
      "rewards/real": 7.948498725891113,
      "step": 690
    },
    {
      "epoch": 2.24,
      "grad_norm": 5.605321382680292,
      "learning_rate": 1.4014251781472683e-07,
      "logits/generated": -2.318974256515503,
      "logits/real": -2.6685843467712402,
      "logps/generated": -220.1696014404297,
      "logps/real": -121.99678802490234,
      "loss": 0.133,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.5589820146560669,
      "rewards/margins": 8.762539863586426,
      "rewards/real": 8.203557968139648,
      "step": 700
    },
    {
      "epoch": 2.2720000000000002,
      "grad_norm": 6.670392991647636,
      "learning_rate": 1.342042755344418e-07,
      "logits/generated": -2.238150119781494,
      "logits/real": -2.658259391784668,
      "logps/generated": -215.6110076904297,
      "logps/real": -129.4001007080078,
      "loss": 0.1283,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.6333814859390259,
      "rewards/margins": 8.75334644317627,
      "rewards/real": 8.119965553283691,
      "step": 710
    },
    {
      "epoch": 2.304,
      "grad_norm": 7.276655178443765,
      "learning_rate": 1.2826603325415677e-07,
      "logits/generated": -2.1864869594573975,
      "logits/real": -2.6987709999084473,
      "logps/generated": -214.52938842773438,
      "logps/real": -119.706787109375,
      "loss": 0.1287,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.8900312185287476,
      "rewards/margins": 8.763832092285156,
      "rewards/real": 7.873800754547119,
      "step": 720
    },
    {
      "epoch": 2.336,
      "grad_norm": 14.059086058443889,
      "learning_rate": 1.2232779097387173e-07,
      "logits/generated": -2.2249207496643066,
      "logits/real": -2.705604076385498,
      "logps/generated": -229.2352752685547,
      "logps/real": -130.58541870117188,
      "loss": 0.127,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.2050936222076416,
      "rewards/margins": 9.346543312072754,
      "rewards/real": 8.141449928283691,
      "step": 730
    },
    {
      "epoch": 2.368,
      "grad_norm": 8.492371245979857,
      "learning_rate": 1.163895486935867e-07,
      "logits/generated": -2.3090498447418213,
      "logits/real": -2.6790778636932373,
      "logps/generated": -227.0781707763672,
      "logps/real": -121.712646484375,
      "loss": 0.1349,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.9518004655838013,
      "rewards/margins": 9.113862037658691,
      "rewards/real": 8.16206169128418,
      "step": 740
    },
    {
      "epoch": 2.3808,
      "eval_logits/generated": -2.5129168033599854,
      "eval_logits/real": -2.6395812034606934,
      "eval_logps/generated": -220.21839904785156,
      "eval_logps/real": -146.43190002441406,
      "eval_loss": 0.4957815408706665,
      "eval_rewards/accuracies": 0.8846153616905212,
      "eval_rewards/generated": 1.3551806211471558,
      "eval_rewards/margins": 5.235291481018066,
      "eval_rewards/real": 6.590472221374512,
      "eval_runtime": 40.8531,
      "eval_samples_per_second": 4.896,
      "eval_steps_per_second": 0.318,
      "step": 744
    },
    {
      "epoch": 2.4,
      "grad_norm": 5.383272251217756,
      "learning_rate": 1.1045130641330165e-07,
      "logits/generated": -2.1900641918182373,
      "logits/real": -2.5824615955352783,
      "logps/generated": -231.75808715820312,
      "logps/real": -119.8072738647461,
      "loss": 0.1259,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.0860052108764648,
      "rewards/margins": 9.120469093322754,
      "rewards/real": 8.034463882446289,
      "step": 750
    },
    {
      "epoch": 2.432,
      "grad_norm": 12.328380398219704,
      "learning_rate": 1.0451306413301662e-07,
      "logits/generated": -2.0176100730895996,
      "logits/real": -2.6012253761291504,
      "logps/generated": -219.3955841064453,
      "logps/real": -122.1722640991211,
      "loss": 0.1278,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.8229852914810181,
      "rewards/margins": 8.620997428894043,
      "rewards/real": 7.798013210296631,
      "step": 760
    },
    {
      "epoch": 2.464,
      "grad_norm": 11.976725890098606,
      "learning_rate": 9.857482185273158e-08,
      "logits/generated": -2.0641188621520996,
      "logits/real": -2.63838529586792,
      "logps/generated": -227.0723114013672,
      "logps/real": -128.9047088623047,
      "loss": 0.1267,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.8111553192138672,
      "rewards/margins": 9.7754545211792,
      "rewards/real": 7.964300632476807,
      "step": 770
    },
    {
      "epoch": 2.496,
      "grad_norm": 5.483965203882113,
      "learning_rate": 9.263657957244655e-08,
      "logits/generated": -2.200010061264038,
      "logits/real": -2.684711217880249,
      "logps/generated": -225.25393676757812,
      "logps/real": -125.56937408447266,
      "loss": 0.1308,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.5697963833808899,
      "rewards/margins": 8.699732780456543,
      "rewards/real": 8.129937171936035,
      "step": 780
    },
    {
      "epoch": 2.528,
      "grad_norm": 4.434839042292689,
      "learning_rate": 8.669833729216151e-08,
      "logits/generated": -2.0318312644958496,
      "logits/real": -2.6771860122680664,
      "logps/generated": -237.12905883789062,
      "logps/real": -128.7871551513672,
      "loss": 0.1342,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.1906983852386475,
      "rewards/margins": 10.451623916625977,
      "rewards/real": 8.260926246643066,
      "step": 790
    },
    {
      "epoch": 2.56,
      "grad_norm": 4.373840018850373,
      "learning_rate": 8.076009501187649e-08,
      "logits/generated": -2.2422289848327637,
      "logits/real": -2.7076992988586426,
      "logps/generated": -238.6762237548828,
      "logps/real": -129.60299682617188,
      "loss": 0.1316,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.0829310417175293,
      "rewards/margins": 10.165118217468262,
      "rewards/real": 8.08218765258789,
      "step": 800
    },
    {
      "epoch": 2.5792,
      "eval_logits/generated": -2.537769317626953,
      "eval_logits/real": -2.684598684310913,
      "eval_logps/generated": -221.98573303222656,
      "eval_logps/real": -145.45445251464844,
      "eval_loss": 0.4795592129230499,
      "eval_rewards/accuracies": 0.9038461446762085,
      "eval_rewards/generated": 1.1784473657608032,
      "eval_rewards/margins": 5.50977087020874,
      "eval_rewards/real": 6.688217639923096,
      "eval_runtime": 36.9652,
      "eval_samples_per_second": 5.41,
      "eval_steps_per_second": 0.352,
      "step": 806
    },
    {
      "epoch": 2.592,
      "grad_norm": 14.115895104882506,
      "learning_rate": 7.482185273159145e-08,
      "logits/generated": -2.166250228881836,
      "logits/real": -2.6296944618225098,
      "logps/generated": -222.494873046875,
      "logps/real": -123.19234466552734,
      "loss": 0.1284,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.9841440320014954,
      "rewards/margins": 9.142965316772461,
      "rewards/real": 8.158821105957031,
      "step": 810
    },
    {
      "epoch": 2.624,
      "grad_norm": 11.503070892673891,
      "learning_rate": 6.88836104513064e-08,
      "logits/generated": -2.2634782791137695,
      "logits/real": -2.7158145904541016,
      "logps/generated": -226.20645141601562,
      "logps/real": -129.71890258789062,
      "loss": 0.1247,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.7692102193832397,
      "rewards/margins": 9.044401168823242,
      "rewards/real": 8.275190353393555,
      "step": 820
    },
    {
      "epoch": 2.656,
      "grad_norm": 15.379104856616065,
      "learning_rate": 6.294536817102138e-08,
      "logits/generated": -2.1017704010009766,
      "logits/real": -2.620785713195801,
      "logps/generated": -236.9258575439453,
      "logps/real": -125.70613098144531,
      "loss": 0.1273,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.9621328115463257,
      "rewards/margins": 10.335904121398926,
      "rewards/real": 8.373770713806152,
      "step": 830
    },
    {
      "epoch": 2.6879999999999997,
      "grad_norm": 5.570759346629193,
      "learning_rate": 5.700712589073634e-08,
      "logits/generated": -2.367926836013794,
      "logits/real": -2.703538417816162,
      "logps/generated": -239.62255859375,
      "logps/real": -120.87593078613281,
      "loss": 0.1271,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.3240606784820557,
      "rewards/margins": 10.333686828613281,
      "rewards/real": 9.009626388549805,
      "step": 840
    },
    {
      "epoch": 2.7199999999999998,
      "grad_norm": 5.160311965943695,
      "learning_rate": 5.10688836104513e-08,
      "logits/generated": -2.350095748901367,
      "logits/real": -2.683204174041748,
      "logps/generated": -231.53890991210938,
      "logps/real": -131.1325225830078,
      "loss": 0.1267,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.9219161868095398,
      "rewards/margins": 9.34153938293457,
      "rewards/real": 8.419622421264648,
      "step": 850
    },
    {
      "epoch": 2.752,
      "grad_norm": 4.79898172671313,
      "learning_rate": 4.5130641330166267e-08,
      "logits/generated": -2.4094595909118652,
      "logits/real": -2.782252073287964,
      "logps/generated": -225.58023071289062,
      "logps/real": -132.29019165039062,
      "loss": 0.1293,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.4511202871799469,
      "rewards/margins": 9.107528686523438,
      "rewards/real": 8.656408309936523,
      "step": 860
    },
    {
      "epoch": 2.7776,
      "eval_logits/generated": -2.4842541217803955,
      "eval_logits/real": -2.6386234760284424,
      "eval_logps/generated": -219.20919799804688,
      "eval_logps/real": -143.65853881835938,
      "eval_loss": 0.4937787652015686,
      "eval_rewards/accuracies": 0.8846153616905212,
      "eval_rewards/generated": 1.4561023712158203,
      "eval_rewards/margins": 5.411707401275635,
      "eval_rewards/real": 6.867809772491455,
      "eval_runtime": 37.1341,
      "eval_samples_per_second": 5.386,
      "eval_steps_per_second": 0.35,
      "step": 868
    },
    {
      "epoch": 2.784,
      "grad_norm": 18.38669622863717,
      "learning_rate": 3.919239904988123e-08,
      "logits/generated": -2.2608981132507324,
      "logits/real": -2.718656063079834,
      "logps/generated": -224.49014282226562,
      "logps/real": -121.529541015625,
      "loss": 0.1253,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.8186184763908386,
      "rewards/margins": 9.35456657409668,
      "rewards/real": 8.535947799682617,
      "step": 870
    },
    {
      "epoch": 2.816,
      "grad_norm": 3.9643247086668945,
      "learning_rate": 3.32541567695962e-08,
      "logits/generated": -2.301182508468628,
      "logits/real": -2.6465210914611816,
      "logps/generated": -237.81692504882812,
      "logps/real": -125.29216003417969,
      "loss": 0.1279,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.6119493246078491,
      "rewards/margins": 10.35554313659668,
      "rewards/real": 8.7435941696167,
      "step": 880
    },
    {
      "epoch": 2.848,
      "grad_norm": 13.130396849471675,
      "learning_rate": 2.7315914489311164e-08,
      "logits/generated": -2.229841709136963,
      "logits/real": -2.6934258937835693,
      "logps/generated": -222.32254028320312,
      "logps/real": -119.14742279052734,
      "loss": 0.1288,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.9097350835800171,
      "rewards/margins": 9.363935470581055,
      "rewards/real": 8.45419979095459,
      "step": 890
    },
    {
      "epoch": 2.88,
      "grad_norm": 5.4181285329894955,
      "learning_rate": 2.1377672209026125e-08,
      "logits/generated": -2.2020509243011475,
      "logits/real": -2.674494743347168,
      "logps/generated": -229.1976776123047,
      "logps/real": -124.3663101196289,
      "loss": 0.1247,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.1633367538452148,
      "rewards/margins": 9.624059677124023,
      "rewards/real": 8.460723876953125,
      "step": 900
    },
    {
      "epoch": 2.912,
      "grad_norm": 16.68936278463454,
      "learning_rate": 1.5439429928741092e-08,
      "logits/generated": -2.312161445617676,
      "logits/real": -2.5068631172180176,
      "logps/generated": -232.4893798828125,
      "logps/real": -122.8946304321289,
      "loss": 0.1264,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.6006708145141602,
      "rewards/margins": 8.866964340209961,
      "rewards/real": 8.2662935256958,
      "step": 910
    },
    {
      "epoch": 2.944,
      "grad_norm": 8.387314647468141,
      "learning_rate": 9.501187648456057e-09,
      "logits/generated": -2.2893476486206055,
      "logits/real": -2.6242709159851074,
      "logps/generated": -232.8568878173828,
      "logps/real": -114.94084167480469,
      "loss": 0.1206,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.4618213176727295,
      "rewards/margins": 10.02196979522705,
      "rewards/real": 8.560149192810059,
      "step": 920
    },
    {
      "epoch": 2.976,
      "grad_norm": 5.038389384372935,
      "learning_rate": 3.562945368171021e-09,
      "logits/generated": -2.3135273456573486,
      "logits/real": -2.751070022583008,
      "logps/generated": -226.42312622070312,
      "logps/real": -128.68014526367188,
      "loss": 0.1244,
      "rewards/accuracies": 1.0,
      "rewards/generated": -0.26498132944107056,
      "rewards/margins": 8.79969596862793,
      "rewards/real": 8.53471565246582,
      "step": 930
    },
    {
      "epoch": 2.976,
      "eval_logits/generated": -2.4267618656158447,
      "eval_logits/real": -2.5711991786956787,
      "eval_logps/generated": -226.38497924804688,
      "eval_logps/real": -146.0556640625,
      "eval_loss": 0.4841172695159912,
      "eval_rewards/accuracies": 0.9038461446762085,
      "eval_rewards/generated": 0.7385206818580627,
      "eval_rewards/margins": 5.889577388763428,
      "eval_rewards/real": 6.628098011016846,
      "eval_runtime": 36.827,
      "eval_samples_per_second": 5.431,
      "eval_steps_per_second": 0.353,
      "step": 930
    },
    {
      "epoch": 2.9952,
      "step": 936,
      "total_flos": 0.0,
      "train_loss": 0.28271993689047986,
      "train_runtime": 15010.7163,
      "train_samples_per_second": 1.997,
      "train_steps_per_second": 0.062
    }
  ],
  "logging_steps": 10,
  "max_steps": 936,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 62,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}