End of training

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.4979530750094233,
-    "train_runtime": 139.1789,
+    "train_loss": 0.4980396749200048,
+    "train_runtime": 139.9378,
     "train_samples": 2345,
-    "train_samples_per_second": 16.849,
-    "train_steps_per_second": 0.266
+    "train_samples_per_second": 16.757,
+    "train_steps_per_second": 0.264
 }

config.json

@@ -23,7 +23,7 @@
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.45.0",
-  "use_cache": false,
+  "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.4979530750094233,
-    "train_runtime": 139.1789,
+    "train_loss": 0.4980396749200048,
+    "train_runtime": 139.9378,
     "train_samples": 2345,
-    "train_samples_per_second": 16.849,
-    "train_steps_per_second": 0.266
+    "train_samples_per_second": 16.757,
+    "train_steps_per_second": 0.264
 }

trainer_state.json

@@ -289,7 +289,7 @@
       "debug/reference_chosen_logps": -165.61618041992188,
       "debug/reference_rejected_logps": -139.546142578125,
       "epoch": 0.3783783783783784,
-      "grad_norm": 3.271576173715924,
+      "grad_norm": 3.2715325631312377,
       "learning_rate": 1e-06,
       "logits/chosen": -3.4804084300994873,
       "logits/rejected": -3.5207834243774414,
@@ -303,496 +303,496 @@
       "step": 14
     },
     {
-      "debug/policy_chosen_logits": -3.371809244155884,
-      "debug/policy_chosen_logps": -138.2847900390625,
-      "debug/policy_rejected_logits": -3.2725796699523926,
-      "debug/policy_rejected_logps": -146.83644104003906,
+      "debug/policy_chosen_logits": -3.3723671436309814,
+      "debug/policy_chosen_logps": -138.40748596191406,
+      "debug/policy_rejected_logits": -3.2726492881774902,
+      "debug/policy_rejected_logps": -146.87710571289062,
       "debug/reference_chosen_logps": -138.04966735839844,
       "debug/reference_rejected_logps": -146.4534454345703,
       "epoch": 0.40540540540540543,
-      "grad_norm": 3.5549880331732084,
+      "grad_norm": 3.5479335407438923,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.371809244155884,
-      "logits/rejected": -3.2725796699523926,
-      "logps/chosen": -138.2847900390625,
-      "logps/rejected": -146.83644104003906,
-      "loss": 0.4978,
+      "logits/chosen": -3.3723671436309814,
+      "logits/rejected": -3.2726492881774902,
+      "logps/chosen": -138.40748596191406,
+      "logps/rejected": -146.87710571289062,
+      "loss": 0.4971,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.002351150382310152,
-      "rewards/margins": 0.0014788818079978228,
-      "rewards/rejected": -0.0038300324231386185,
+      "rewards/chosen": -0.003578138304874301,
+      "rewards/margins": 0.0006584453512914479,
+      "rewards/rejected": -0.004236583597958088,
       "step": 15
     },
     {
-      "debug/policy_chosen_logits": -3.2811360359191895,
-      "debug/policy_chosen_logps": -145.2406005859375,
-      "debug/policy_rejected_logits": -3.2666900157928467,
-      "debug/policy_rejected_logps": -136.08499145507812,
+      "debug/policy_chosen_logits": -3.280038356781006,
+      "debug/policy_chosen_logps": -144.99691772460938,
+      "debug/policy_rejected_logits": -3.265355110168457,
+      "debug/policy_rejected_logps": -136.1260223388672,
       "debug/reference_chosen_logps": -145.38519287109375,
       "debug/reference_rejected_logps": -136.1790313720703,
       "epoch": 0.43243243243243246,
-      "grad_norm": 3.390241199656903,
+      "grad_norm": 3.3905692790559057,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.2811360359191895,
-      "logits/rejected": -3.2666900157928467,
-      "logps/chosen": -145.2406005859375,
-      "logps/rejected": -136.08499145507812,
-      "loss": 0.4992,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": 0.0014458559453487396,
-      "rewards/margins": 0.0005054186331108212,
-      "rewards/rejected": 0.0009404371376149356,
+      "logits/chosen": -3.280038356781006,
+      "logits/rejected": -3.265355110168457,
+      "logps/chosen": -144.99691772460938,
+      "logps/rejected": -136.1260223388672,
+      "loss": 0.4987,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0038827608805149794,
+      "rewards/margins": 0.0033527181949466467,
+      "rewards/rejected": 0.0005300427437759936,
       "step": 16
     },
     {
-      "debug/policy_chosen_logits": -3.4209792613983154,
-      "debug/policy_chosen_logps": -154.63943481445312,
-      "debug/policy_rejected_logits": -3.3486948013305664,
-      "debug/policy_rejected_logps": -133.35781860351562,
+      "debug/policy_chosen_logits": -3.4201347827911377,
+      "debug/policy_chosen_logps": -154.7392578125,
+      "debug/policy_rejected_logits": -3.348952293395996,
+      "debug/policy_rejected_logps": -133.3173828125,
       "debug/reference_chosen_logps": -154.70895385742188,
       "debug/reference_rejected_logps": -133.4116668701172,
       "epoch": 0.4594594594594595,
-      "grad_norm": 3.6742049248327437,
+      "grad_norm": 3.6801385333166086,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.4209792613983154,
-      "logits/rejected": -3.3486948013305664,
-      "logps/chosen": -154.63943481445312,
-      "logps/rejected": -133.35781860351562,
-      "loss": 0.4989,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0006950664683245122,
-      "rewards/margins": 0.00015656478353776038,
-      "rewards/rejected": 0.0005385016556829214,
+      "logits/chosen": -3.4201347827911377,
+      "logits/rejected": -3.348952293395996,
+      "logps/chosen": -154.7392578125,
+      "logps/rejected": -133.3173828125,
+      "loss": 0.4999,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.0003031063242815435,
+      "rewards/margins": -0.0012458800338208675,
+      "rewards/rejected": 0.0009427738841623068,
       "step": 17
     },
     {
-      "debug/policy_chosen_logits": -3.298342704772949,
-      "debug/policy_chosen_logps": -146.52044677734375,
-      "debug/policy_rejected_logits": -3.3201894760131836,
-      "debug/policy_rejected_logps": -156.97808837890625,
+      "debug/policy_chosen_logits": -3.2982780933380127,
+      "debug/policy_chosen_logps": -146.35263061523438,
+      "debug/policy_rejected_logits": -3.3202931880950928,
+      "debug/policy_rejected_logps": -156.7283172607422,
       "debug/reference_chosen_logps": -146.34840393066406,
       "debug/reference_rejected_logps": -156.84378051757812,
       "epoch": 0.4864864864864865,
-      "grad_norm": 3.5194962148324103,
+      "grad_norm": 3.5196394680278327,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.298342704772949,
-      "logits/rejected": -3.3201894760131836,
-      "logps/chosen": -146.52044677734375,
-      "logps/rejected": -156.97808837890625,
-      "loss": 0.4973,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.0017202282324433327,
-      "rewards/margins": -0.0003771304036490619,
-      "rewards/rejected": -0.001343097654171288,
+      "logits/chosen": -3.2982780933380127,
+      "logits/rejected": -3.3202931880950928,
+      "logps/chosen": -146.35263061523438,
+      "logps/rejected": -156.7283172607422,
+      "loss": 0.4974,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -4.220963455736637e-05,
+      "rewards/margins": -0.0011968993349000812,
+      "rewards/rejected": 0.0011546898167580366,
       "step": 18
     },
     {
-      "debug/policy_chosen_logits": -3.3400371074676514,
-      "debug/policy_chosen_logps": -132.01437377929688,
-      "debug/policy_rejected_logits": -3.36023211479187,
-      "debug/policy_rejected_logps": -148.31887817382812,
+      "debug/policy_chosen_logits": -3.3398616313934326,
+      "debug/policy_chosen_logps": -131.99713134765625,
+      "debug/policy_rejected_logits": -3.3604841232299805,
+      "debug/policy_rejected_logps": -148.30227661132812,
       "debug/reference_chosen_logps": -131.88931274414062,
       "debug/reference_rejected_logps": -148.13345336914062,
       "epoch": 0.5135135135135135,
-      "grad_norm": 3.61313418435521,
+      "grad_norm": 3.609790844356815,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.3400371074676514,
-      "logits/rejected": -3.36023211479187,
-      "logps/chosen": -132.01437377929688,
-      "logps/rejected": -148.31887817382812,
-      "loss": 0.498,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.00125067715998739,
-      "rewards/margins": 0.0006034375401213765,
-      "rewards/rejected": -0.001854114467278123,
+      "logits/chosen": -3.3398616313934326,
+      "logits/rejected": -3.3604841232299805,
+      "logps/chosen": -131.99713134765625,
+      "logps/rejected": -148.30227661132812,
+      "loss": 0.4981,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.0010780429001897573,
+      "rewards/margins": 0.0006100271129980683,
+      "rewards/rejected": -0.0016880702460184693,
       "step": 19
     },
     {
-      "debug/policy_chosen_logits": -3.2641985416412354,
-      "debug/policy_chosen_logps": -134.37350463867188,
-      "debug/policy_rejected_logits": -3.255669355392456,
-      "debug/policy_rejected_logps": -124.11454772949219,
+      "debug/policy_chosen_logits": -3.2643423080444336,
+      "debug/policy_chosen_logps": -134.54617309570312,
+      "debug/policy_rejected_logits": -3.255173921585083,
+      "debug/policy_rejected_logps": -124.35269165039062,
       "debug/reference_chosen_logps": -134.53549194335938,
       "debug/reference_rejected_logps": -123.25413513183594,
       "epoch": 0.5405405405405406,
-      "grad_norm": 3.4240940627433663,
+      "grad_norm": 3.4308218789715035,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.2641985416412354,
-      "logits/rejected": -3.255669355392456,
-      "logps/chosen": -134.37350463867188,
-      "logps/rejected": -124.11454772949219,
+      "logits/chosen": -3.2643423080444336,
+      "logits/rejected": -3.255173921585083,
+      "logps/chosen": -134.54617309570312,
+      "logps/rejected": -124.35269165039062,
       "loss": 0.4969,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0016198731027543545,
-      "rewards/margins": 0.010224045254290104,
-      "rewards/rejected": -0.008604173548519611,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.00010671629570424557,
+      "rewards/margins": 0.010878859087824821,
+      "rewards/rejected": -0.010985574685037136,
       "step": 20
     },
     {
-      "debug/policy_chosen_logits": -3.389690637588501,
-      "debug/policy_chosen_logps": -144.95230102539062,
-      "debug/policy_rejected_logits": -3.3237528800964355,
-      "debug/policy_rejected_logps": -140.4608917236328,
+      "debug/policy_chosen_logits": -3.39021635055542,
+      "debug/policy_chosen_logps": -145.20547485351562,
+      "debug/policy_rejected_logits": -3.324305295944214,
+      "debug/policy_rejected_logps": -140.68399047851562,
       "debug/reference_chosen_logps": -145.19155883789062,
       "debug/reference_rejected_logps": -139.9624481201172,
       "epoch": 0.5675675675675675,
-      "grad_norm": 3.5636895297483804,
+      "grad_norm": 3.572312158938155,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.389690637588501,
-      "logits/rejected": -3.3237528800964355,
-      "logps/chosen": -144.95230102539062,
-      "logps/rejected": -140.4608917236328,
-      "loss": 0.4972,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": 0.0023925015702843666,
-      "rewards/margins": 0.007377022877335548,
-      "rewards/rejected": -0.004984522238373756,
+      "logits/chosen": -3.39021635055542,
+      "logits/rejected": -3.324305295944214,
+      "logps/chosen": -145.20547485351562,
+      "logps/rejected": -140.68399047851562,
+      "loss": 0.4975,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.00013918871991336346,
+      "rewards/margins": 0.007076186593621969,
+      "rewards/rejected": -0.007215375080704689,
       "step": 21
     },
     {
-      "debug/policy_chosen_logits": -3.2673821449279785,
-      "debug/policy_chosen_logps": -159.7883758544922,
-      "debug/policy_rejected_logits": -3.2593047618865967,
-      "debug/policy_rejected_logps": -141.72947692871094,
+      "debug/policy_chosen_logits": -3.267886161804199,
+      "debug/policy_chosen_logps": -160.2618865966797,
+      "debug/policy_rejected_logits": -3.2591264247894287,
+      "debug/policy_rejected_logps": -141.91876220703125,
       "debug/reference_chosen_logps": -159.80300903320312,
       "debug/reference_rejected_logps": -142.28695678710938,
       "epoch": 0.5945945945945946,
-      "grad_norm": 3.8014985351330988,
+      "grad_norm": 3.81196901542863,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.2673821449279785,
-      "logits/rejected": -3.2593047618865967,
-      "logps/chosen": -159.7883758544922,
-      "logps/rejected": -141.72947692871094,
-      "loss": 0.499,
+      "logits/chosen": -3.267886161804199,
+      "logits/rejected": -3.2591264247894287,
+      "logps/chosen": -160.2618865966797,
+      "logps/rejected": -141.91876220703125,
+      "loss": 0.4995,
       "rewards/accuracies": 0.125,
-      "rewards/chosen": 0.00014627468772232533,
-      "rewards/margins": -0.005428638309240341,
-      "rewards/rejected": 0.00557491322979331,
+      "rewards/chosen": -0.004588775336742401,
+      "rewards/margins": -0.008270883932709694,
+      "rewards/rejected": 0.0036821081303060055,
       "step": 22
     },
     {
-      "debug/policy_chosen_logits": -3.3076963424682617,
-      "debug/policy_chosen_logps": -147.4810791015625,
-      "debug/policy_rejected_logits": -3.3063511848449707,
-      "debug/policy_rejected_logps": -155.9129638671875,
+      "debug/policy_chosen_logits": -3.3076066970825195,
+      "debug/policy_chosen_logps": -147.66110229492188,
+      "debug/policy_rejected_logits": -3.3076066970825195,
+      "debug/policy_rejected_logps": -156.1669464111328,
       "debug/reference_chosen_logps": -147.6890869140625,
       "debug/reference_rejected_logps": -155.93359375,
       "epoch": 0.6216216216216216,
-      "grad_norm": 3.5943677168048143,
+      "grad_norm": 3.584513321934443,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.3076963424682617,
-      "logits/rejected": -3.3063511848449707,
-      "logps/chosen": -147.4810791015625,
-      "logps/rejected": -155.9129638671875,
-      "loss": 0.499,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.0020801923237740993,
-      "rewards/margins": 0.001873721950687468,
-      "rewards/rejected": 0.00020647048950195312,
+      "logits/chosen": -3.3076066970825195,
+      "logits/rejected": -3.3076066970825195,
+      "logps/chosen": -147.66110229492188,
+      "logps/rejected": -156.1669464111328,
+      "loss": 0.4981,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0002799700014293194,
+      "rewards/margins": 0.002613373100757599,
+      "rewards/rejected": -0.002333402633666992,
       "step": 23
     },
     {
-      "debug/policy_chosen_logits": -3.3162076473236084,
-      "debug/policy_chosen_logps": -146.12327575683594,
-      "debug/policy_rejected_logits": -3.2808167934417725,
-      "debug/policy_rejected_logps": -150.74459838867188,
+      "debug/policy_chosen_logits": -3.3157832622528076,
+      "debug/policy_chosen_logps": -145.93922424316406,
+      "debug/policy_rejected_logits": -3.2806382179260254,
+      "debug/policy_rejected_logps": -150.69491577148438,
       "debug/reference_chosen_logps": -146.20852661132812,
       "debug/reference_rejected_logps": -151.0212860107422,
       "epoch": 0.6486486486486487,
-      "grad_norm": 3.490255067904084,
+      "grad_norm": 3.4973421303279575,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.3162076473236084,
-      "logits/rejected": -3.2808167934417725,
-      "logps/chosen": -146.12327575683594,
-      "logps/rejected": -150.74459838867188,
+      "logits/chosen": -3.3157832622528076,
+      "logits/rejected": -3.2806382179260254,
+      "logps/chosen": -145.93922424316406,
+      "logps/rejected": -150.69491577148438,
       "loss": 0.4979,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.0008525657467544079,
-      "rewards/margins": -0.0019143482204526663,
-      "rewards/rejected": 0.002766914200037718,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0026929855812340975,
+      "rewards/margins": -0.0005707358941435814,
+      "rewards/rejected": 0.0032637212425470352,
       "step": 24
     },
     {
-      "debug/policy_chosen_logits": -3.341583490371704,
-      "debug/policy_chosen_logps": -145.74295043945312,
-      "debug/policy_rejected_logits": -3.2622642517089844,
-      "debug/policy_rejected_logps": -148.95095825195312,
+      "debug/policy_chosen_logits": -3.340818405151367,
+      "debug/policy_chosen_logps": -145.6500244140625,
+      "debug/policy_rejected_logits": -3.2608158588409424,
+      "debug/policy_rejected_logps": -148.91769409179688,
       "debug/reference_chosen_logps": -145.28700256347656,
       "debug/reference_rejected_logps": -148.38320922851562,
       "epoch": 0.6756756756756757,
-      "grad_norm": 3.4930826080691433,
+      "grad_norm": 3.493835228930821,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.341583490371704,
-      "logits/rejected": -3.2622642517089844,
-      "logps/chosen": -145.74295043945312,
-      "logps/rejected": -148.95095825195312,
-      "loss": 0.4992,
+      "logits/chosen": -3.340818405151367,
+      "logits/rejected": -3.2608158588409424,
+      "logps/chosen": -145.6500244140625,
+      "logps/rejected": -148.91769409179688,
+      "loss": 0.4997,
       "rewards/accuracies": 0.375,
-      "rewards/chosen": -0.0045594689436256886,
-      "rewards/margins": 0.001117925625294447,
-      "rewards/rejected": -0.0056773945689201355,
+      "rewards/chosen": -0.003630065592005849,
+      "rewards/margins": 0.0017146589234471321,
+      "rewards/rejected": -0.005344724282622337,
       "step": 25
     },
     {
-      "debug/policy_chosen_logits": -3.3710758686065674,
-      "debug/policy_chosen_logps": -134.76580810546875,
-      "debug/policy_rejected_logits": -3.268131971359253,
-      "debug/policy_rejected_logps": -164.87716674804688,
+      "debug/policy_chosen_logits": -3.3708367347717285,
+      "debug/policy_chosen_logps": -134.6257781982422,
+      "debug/policy_rejected_logits": -3.2685859203338623,
+      "debug/policy_rejected_logps": -164.87184143066406,
       "debug/reference_chosen_logps": -134.9357452392578,
       "debug/reference_rejected_logps": -164.06057739257812,
       "epoch": 0.7027027027027027,
-      "grad_norm": 3.5324806343379795,
+      "grad_norm": 3.5262056978807492,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.3710758686065674,
-      "logits/rejected": -3.268131971359253,
-      "logps/chosen": -134.76580810546875,
-      "logps/rejected": -164.87716674804688,
-      "loss": 0.4961,
+      "logits/chosen": -3.3708367347717285,
+      "logits/rejected": -3.2685859203338623,
+      "logps/chosen": -134.6257781982422,
+      "logps/rejected": -164.87184143066406,
+      "loss": 0.4954,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.0016993426252156496,
-      "rewards/margins": 0.009865359403192997,
-      "rewards/rejected": -0.008166017010807991,
+      "rewards/chosen": 0.0030995942652225494,
+      "rewards/margins": 0.011212329380214214,
+      "rewards/rejected": -0.008112735114991665,
       "step": 26
     },
     {
-      "debug/policy_chosen_logits": -3.276531934738159,
-      "debug/policy_chosen_logps": -128.13336181640625,
-      "debug/policy_rejected_logits": -3.2201032638549805,
-      "debug/policy_rejected_logps": -148.01954650878906,
+      "debug/policy_chosen_logits": -3.276146650314331,
+      "debug/policy_chosen_logps": -127.90383911132812,
+      "debug/policy_rejected_logits": -3.2199714183807373,
+      "debug/policy_rejected_logps": -148.08279418945312,
       "debug/reference_chosen_logps": -127.79987335205078,
       "debug/reference_rejected_logps": -148.602783203125,
       "epoch": 0.7297297297297297,
-      "grad_norm": 3.5504923092772165,
+      "grad_norm": 3.5419489962048973,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.276531934738159,
-      "logits/rejected": -3.2201032638549805,
-      "logps/chosen": -128.13336181640625,
-      "logps/rejected": -148.01954650878906,
-      "loss": 0.4978,
+      "logits/chosen": -3.276146650314331,
+      "logits/rejected": -3.2199714183807373,
+      "logps/chosen": -127.90383911132812,
+      "logps/rejected": -148.08279418945312,
+      "loss": 0.4972,
       "rewards/accuracies": 0.25,
-      "rewards/chosen": -0.0033348274882882833,
-      "rewards/margins": -0.009167090058326721,
-      "rewards/rejected": 0.005832261871546507,
+      "rewards/chosen": -0.0010396192083135247,
+      "rewards/margins": -0.006239423528313637,
+      "rewards/rejected": 0.005199803970754147,
       "step": 27
     },
     {
-      "debug/policy_chosen_logits": -3.3670849800109863,
-      "debug/policy_chosen_logps": -142.864013671875,
-      "debug/policy_rejected_logits": -3.314009428024292,
-      "debug/policy_rejected_logps": -159.29888916015625,
+      "debug/policy_chosen_logits": -3.3671457767486572,
+      "debug/policy_chosen_logps": -142.98072814941406,
+      "debug/policy_rejected_logits": -3.313788652420044,
+      "debug/policy_rejected_logps": -159.15054321289062,
       "debug/reference_chosen_logps": -143.54417419433594,
       "debug/reference_rejected_logps": -159.19894409179688,
       "epoch": 0.7567567567567568,
-      "grad_norm": 3.6333481085660533,
+      "grad_norm": 3.6334386387375948,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.3670849800109863,
-      "logits/rejected": -3.314009428024292,
-      "logps/chosen": -142.864013671875,
-      "logps/rejected": -159.29888916015625,
-      "loss": 0.4946,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.006801585666835308,
-      "rewards/margins": 0.007801036350429058,
-      "rewards/rejected": -0.0009994508000090718,
+      "logits/chosen": -3.3671457767486572,
+      "logits/rejected": -3.313788652420044,
+      "logps/chosen": -142.98072814941406,
+      "logps/rejected": -159.15054321289062,
+      "loss": 0.4957,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.005634431727230549,
+      "rewards/margins": 0.005150537472218275,
+      "rewards/rejected": 0.0004838943714275956,
       "step": 28
     },
     {
-      "debug/policy_chosen_logits": -3.4510252475738525,
-      "debug/policy_chosen_logps": -144.7528839111328,
-      "debug/policy_rejected_logits": -3.4992246627807617,
-      "debug/policy_rejected_logps": -129.56163024902344,
+      "debug/policy_chosen_logits": -3.451542377471924,
+      "debug/policy_chosen_logps": -144.62733459472656,
+      "debug/policy_rejected_logits": -3.500192403793335,
+      "debug/policy_rejected_logps": -129.41226196289062,
       "debug/reference_chosen_logps": -144.27435302734375,
       "debug/reference_rejected_logps": -128.7982177734375,
       "epoch": 0.7837837837837838,
-      "grad_norm": 3.519707420288352,
+      "grad_norm": 3.525191396719062,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.4510252475738525,
-      "logits/rejected": -3.4992246627807617,
-      "logps/chosen": -144.7528839111328,
-      "logps/rejected": -129.56163024902344,
-      "loss": 0.4917,
+      "logits/chosen": -3.451542377471924,
+      "logits/rejected": -3.500192403793335,
+      "logps/chosen": -144.62733459472656,
+      "logps/rejected": -129.41226196289062,
+      "loss": 0.4925,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.0047853561118245125,
-      "rewards/margins": 0.0028487201780080795,
-      "rewards/rejected": -0.007634077221155167,
+      "rewards/chosen": -0.003529853653162718,
+      "rewards/margins": 0.0026106643490493298,
+      "rewards/rejected": -0.006140518002212048,
       "step": 29
     },
     {
-      "debug/policy_chosen_logits": -3.442349910736084,
-      "debug/policy_chosen_logps": -129.01779174804688,
-      "debug/policy_rejected_logits": -3.355781078338623,
-      "debug/policy_rejected_logps": -125.97911071777344,
+      "debug/policy_chosen_logits": -3.4419167041778564,
+      "debug/policy_chosen_logps": -128.94613647460938,
+      "debug/policy_rejected_logits": -3.3551814556121826,
+      "debug/policy_rejected_logps": -125.87129974365234,
       "debug/reference_chosen_logps": -129.1351318359375,
       "debug/reference_rejected_logps": -125.61131286621094,
       "epoch": 0.8108108108108109,
-      "grad_norm": 3.436282122029211,
+      "grad_norm": 3.4357174715493812,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.442349910736084,
-      "logits/rejected": -3.355781078338623,
-      "logps/chosen": -129.01779174804688,
-      "logps/rejected": -125.97911071777344,
-      "loss": 0.4957,
+      "logits/chosen": -3.4419167041778564,
+      "logits/rejected": -3.3551814556121826,
+      "logps/chosen": -128.94613647460938,
+      "logps/rejected": -125.87129974365234,
+      "loss": 0.4956,
       "rewards/accuracies": 0.375,
-      "rewards/chosen": 0.0011734389699995518,
-      "rewards/margins": 0.004851359874010086,
-      "rewards/rejected": -0.003677921136841178,
+      "rewards/chosen": 0.001890067826025188,
+      "rewards/margins": 0.00448994617909193,
+      "rewards/rejected": -0.0025998782366514206,
       "step": 30
     },
     {
-      "debug/policy_chosen_logits": -3.46421480178833,
-      "debug/policy_chosen_logps": -148.1017608642578,
-      "debug/policy_rejected_logits": -3.4284045696258545,
-      "debug/policy_rejected_logps": -155.14401245117188,
+      "debug/policy_chosen_logits": -3.464578628540039,
+      "debug/policy_chosen_logps": -148.08114624023438,
+      "debug/policy_rejected_logits": -3.42940354347229,
+      "debug/policy_rejected_logps": -155.03309631347656,
       "debug/reference_chosen_logps": -147.857666015625,
       "debug/reference_rejected_logps": -153.7999267578125,
       "epoch": 0.8378378378378378,
-      "grad_norm": 3.573380219675295,
+      "grad_norm": 3.569444014406672,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.46421480178833,
-      "logits/rejected": -3.4284045696258545,
-      "logps/chosen": -148.1017608642578,
-      "logps/rejected": -155.14401245117188,
+      "logits/chosen": -3.464578628540039,
+      "logits/rejected": -3.42940354347229,
+      "logps/chosen": -148.08114624023438,
+      "logps/rejected": -155.03309631347656,
       "loss": 0.4994,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.002440805546939373,
-      "rewards/margins": 0.010999973863363266,
-      "rewards/rejected": -0.013440780341625214,
+      "rewards/chosen": -0.0022347732447087765,
+      "rewards/margins": 0.010096902027726173,
+      "rewards/rejected": -0.012331675738096237,
       "step": 31
     },
     {
-      "debug/policy_chosen_logits": -3.296452522277832,
-      "debug/policy_chosen_logps": -146.0049285888672,
-      "debug/policy_rejected_logits": -3.3844947814941406,
-      "debug/policy_rejected_logps": -145.01739501953125,
+      "debug/policy_chosen_logits": -3.2954604625701904,
+      "debug/policy_chosen_logps": -146.03802490234375,
+      "debug/policy_rejected_logits": -3.384243965148926,
+      "debug/policy_rejected_logps": -145.24456787109375,
       "debug/reference_chosen_logps": -146.03688049316406,
       "debug/reference_rejected_logps": -144.96478271484375,
       "epoch": 0.8648648648648649,
-      "grad_norm": 3.7844677501283552,
+      "grad_norm": 3.794862233420595,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.296452522277832,
-      "logits/rejected": -3.3844947814941406,
-      "logps/chosen": -146.0049285888672,
-      "logps/rejected": -145.01739501953125,
-      "loss": 0.4983,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.0003195476019755006,
-      "rewards/margins": 0.0008457758231088519,
-      "rewards/rejected": -0.000526227755472064,
+      "logits/chosen": -3.2954604625701904,
+      "logits/rejected": -3.384243965148926,
+      "logps/chosen": -146.03802490234375,
+      "logps/rejected": -145.24456787109375,
+      "loss": 0.4995,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.1482159607112408e-05,
+      "rewards/margins": 0.002786235883831978,
+      "rewards/rejected": -0.0027977179270237684,
       "step": 32
     },
     {
-      "debug/policy_chosen_logits": -3.393280506134033,
-      "debug/policy_chosen_logps": -135.49090576171875,
-      "debug/policy_rejected_logits": -3.4107859134674072,
-      "debug/policy_rejected_logps": -126.17267608642578,
+      "debug/policy_chosen_logits": -3.393167734146118,
+      "debug/policy_chosen_logps": -135.54547119140625,
+      "debug/policy_rejected_logits": -3.4107766151428223,
+      "debug/policy_rejected_logps": -126.21063995361328,
       "debug/reference_chosen_logps": -135.9659423828125,
       "debug/reference_rejected_logps": -123.26311492919922,
       "epoch": 0.8918918918918919,
-      "grad_norm": 3.6987210522943323,
+      "grad_norm": 3.7044090203550195,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.393280506134033,
-      "logits/rejected": -3.4107859134674072,
-      "logps/chosen": -135.49090576171875,
-      "logps/rejected": -126.17267608642578,
-      "loss": 0.4982,
+      "logits/chosen": -3.393167734146118,
+      "logits/rejected": -3.4107766151428223,
+      "logps/chosen": -135.54547119140625,
+      "logps/rejected": -126.21063995361328,
+      "loss": 0.4985,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.00475044222548604,
-      "rewards/margins": 0.03384600579738617,
-      "rewards/rejected": -0.029095562174916267,
+      "rewards/chosen": 0.004204845521599054,
+      "rewards/margins": 0.03368005529046059,
+      "rewards/rejected": -0.02947521209716797,
       "step": 33
     },
     {
-      "debug/policy_chosen_logits": -3.3232836723327637,
-      "debug/policy_chosen_logps": -159.23611450195312,
-      "debug/policy_rejected_logits": -3.3686249256134033,
-      "debug/policy_rejected_logps": -165.2159423828125,
+      "debug/policy_chosen_logits": -3.3235180377960205,
+      "debug/policy_chosen_logps": -159.286865234375,
+      "debug/policy_rejected_logits": -3.369253635406494,
+      "debug/policy_rejected_logps": -165.31944274902344,
       "debug/reference_chosen_logps": -158.15670776367188,
       "debug/reference_rejected_logps": -164.94976806640625,
       "epoch": 0.918918918918919,
-      "grad_norm": 3.553957133112115,
+      "grad_norm": 3.550860802017907,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.3232836723327637,
-      "logits/rejected": -3.3686249256134033,
-      "logps/chosen": -159.23611450195312,
-      "logps/rejected": -165.2159423828125,
-      "loss": 0.4998,
+      "logits/chosen": -3.3235180377960205,
+      "logits/rejected": -3.369253635406494,
+      "logps/chosen": -159.286865234375,
+      "logps/rejected": -165.31944274902344,
+      "loss": 0.4997,
       "rewards/accuracies": 0.25,
-      "rewards/chosen": -0.010794105939567089,
-      "rewards/margins": -0.008132400922477245,
-      "rewards/rejected": -0.0026617045514285564,
+      "rewards/chosen": -0.011301536113023758,
+      "rewards/margins": -0.007604693062603474,
+      "rewards/rejected": -0.0036968423519283533,
       "step": 34
     },
     {
-      "debug/policy_chosen_logits": -3.3675332069396973,
-      "debug/policy_chosen_logps": -129.2323455810547,
-      "debug/policy_rejected_logits": -3.39815092086792,
-      "debug/policy_rejected_logps": -157.40768432617188,
+      "debug/policy_chosen_logits": -3.367034673690796,
+      "debug/policy_chosen_logps": -129.437744140625,
+      "debug/policy_rejected_logits": -3.396902561187744,
+      "debug/policy_rejected_logps": -157.3302001953125,
       "debug/reference_chosen_logps": -129.46514892578125,
       "debug/reference_rejected_logps": -157.58755493164062,
       "epoch": 0.9459459459459459,
-      "grad_norm": 3.6687608874400897,
+      "grad_norm": 3.6785421821803044,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.3675332069396973,
-      "logits/rejected": -3.39815092086792,
-      "logps/chosen": -129.2323455810547,
-      "logps/rejected": -157.40768432617188,
-      "loss": 0.4972,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.0023280431050807238,
-      "rewards/margins": 0.0005296231247484684,
-      "rewards/rejected": 0.0017984198639169335,
+      "logits/chosen": -3.367034673690796,
+      "logits/rejected": -3.396902561187744,
+      "logps/chosen": -129.437744140625,
+      "logps/rejected": -157.3302001953125,
+      "loss": 0.498,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.00027409568428993225,
+      "rewards/margins": -0.002299328101798892,
+      "rewards/rejected": 0.002573424018919468,
       "step": 35
     },
     {
-      "debug/policy_chosen_logits": -3.2530903816223145,
-      "debug/policy_chosen_logps": -139.84768676757812,
-      "debug/policy_rejected_logits": -3.380523204803467,
-      "debug/policy_rejected_logps": -127.18656921386719,
+      "debug/policy_chosen_logits": -3.252328634262085,
+      "debug/policy_chosen_logps": -139.81039428710938,
+      "debug/policy_rejected_logits": -3.3797786235809326,
+      "debug/policy_rejected_logps": -127.12653350830078,
       "debug/reference_chosen_logps": -140.35733032226562,
       "debug/reference_rejected_logps": -127.09129333496094,
       "epoch": 0.972972972972973,
-      "grad_norm": 3.747493235351115,
+      "grad_norm": 3.7364723193101446,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.2530903816223145,
-      "logits/rejected": -3.380523204803467,
-      "logps/chosen": -139.84768676757812,
-      "logps/rejected": -127.18656921386719,
-      "loss": 0.4949,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.005096264183521271,
-      "rewards/margins": 0.006049060728400946,
-      "rewards/rejected": -0.0009527970105409622,
+      "logits/chosen": -3.252328634262085,
+      "logits/rejected": -3.3797786235809326,
+      "logps/chosen": -139.81039428710938,
+      "logps/rejected": -127.12653350830078,
+      "loss": 0.4951,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.005469245836138725,
+      "rewards/margins": 0.0058215707540512085,
+      "rewards/rejected": -0.0003523253835737705,
       "step": 36
     },
     {
-      "debug/policy_chosen_logits": -3.1672980785369873,
-      "debug/policy_chosen_logps": -165.81317138671875,
-      "debug/policy_rejected_logits": -3.1891133785247803,
-      "debug/policy_rejected_logps": -135.89328002929688,
+      "debug/policy_chosen_logits": -3.166999101638794,
+      "debug/policy_chosen_logps": -165.9640350341797,
+      "debug/policy_rejected_logits": -3.1885428428649902,
+      "debug/policy_rejected_logps": -135.82620239257812,
       "debug/reference_chosen_logps": -164.97256469726562,
       "debug/reference_rejected_logps": -135.76358032226562,
       "epoch": 1.0,
-      "grad_norm": 3.5111470282705084,
+      "grad_norm": 3.5142818885997116,
       "learning_rate": 1e-06,
-      "logits/chosen": -3.1672980785369873,
-      "logits/rejected": -3.1891133785247803,
-      "logps/chosen": -165.81317138671875,
-      "logps/rejected": -135.89328002929688,
-      "loss": 0.4809,
+      "logits/chosen": -3.166999101638794,
+      "logits/rejected": -3.1885428428649902,
+      "logps/chosen": -165.9640350341797,
+      "logps/rejected": -135.82620239257812,
+      "loss": 0.481,
       "rewards/accuracies": 0.375,
-      "rewards/chosen": -0.00840616226196289,
-      "rewards/margins": -0.007109222002327442,
-      "rewards/rejected": -0.0012969397939741611,
+      "rewards/chosen": -0.009914855472743511,
+      "rewards/margins": -0.009288596920669079,
+      "rewards/rejected": -0.0006262586684897542,
       "step": 37
     },
     {
       "epoch": 1.0,
       "step": 37,
       "total_flos": 0.0,
-      "train_loss": 0.4979530750094233,
-      "train_runtime": 139.1789,
-      "train_samples_per_second": 16.849,
-      "train_steps_per_second": 0.266
+      "train_loss": 0.4980396749200048,
+      "train_runtime": 139.9378,
+      "train_samples_per_second": 16.757,
+      "train_steps_per_second": 0.264
     }
   ],
   "logging_steps": 1,