Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6972
-- Rewards/chosen: -0.0013
-- Rewards/rejected: 0.0039
-- Rewards/accuracies: 0.4759
-- Rewards/margins: -0.0051
-- Logps/rejected: -37.5089
-- Logps/chosen: -34.0371
-- Logits/rejected: -2.2402
-- Logits/chosen: -2.2450
+- Loss: 0.6925
+- Rewards/chosen: 0.0061
+- Rewards/rejected: 0.0005
+- Rewards/accuracies: 0.5457
+- Rewards/margins: 0.0056
+- Logps/rejected: -37.5157
+- Logps/chosen: -34.0243
+- Logits/rejected: -2.2394
+- Logits/chosen: -2.2443
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6909        | 0.26  | 100  | 0.6898          | 0.0068         | -0.0023          | 0.5627             | 0.0091          | -37.5213       | -34.0210     | -2.2390         | -2.2439       |
-| 0.691         | 0.52  | 200  | 0.6955          | 0.0045         | 0.0068           | 0.4780             | -0.0023         | -37.5030       | -34.0255     | -2.2385         | -2.2434       |
-| 0.6819        | 0.78  | 300  | 0.6930          | 0.0088         | 0.0058           | 0.5046             | 0.0030          | -37.5050       | -34.0169     | -2.2388         | -2.2436       |
+| 0.6933        | 0.26  | 100  | 0.6912          | 0.0148         | 0.0075           | 0.4988             | 0.0073          | -37.5041       | -34.0098     | -2.2391         | -2.2439       |
+| 0.6827        | 0.52  | 200  | 0.6955          | 0.0061         | 0.0071           | 0.4813             | -0.0010         | -37.5048       | -34.0244     | -2.2385         | -2.2434       |
+| 0.6911        | 0.78  | 300  | 0.6925          | 0.0061         | 0.0005           | 0.5457             | 0.0056          | -37.5157       | -34.0243     | -2.2394         | -2.2443       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a27b61cf848beb8a190736e07a271658c8edeee6bf6fc70659c0de5c4e681060
+oid sha256:fcc2a0780b0e01a6ac0ca0c297e18ab145d3bb00342b9748f1b1f76ea3246fef
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2450356483459473,
-    "eval_logits/rejected": -2.2401509284973145,
-    "eval_logps/chosen": -34.03712844848633,
-    "eval_logps/rejected": -37.50889587402344,
-    "eval_loss": 0.6972227096557617,
-    "eval_rewards/accuracies": 0.4759136438369751,
-    "eval_rewards/chosen": -0.0012873795349150896,
-    "eval_rewards/margins": -0.005148096010088921,
-    "eval_rewards/rejected": 0.0038607169408351183,
-    "eval_runtime": 145.779,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.353,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.6914243283209862,
-    "train_runtime": 3256.6129,
+    "train_loss": 0.6911570282725544,
+    "train_runtime": 3257.344,
     "train_samples": 3079,
     "train_samples_per_second": 0.945,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6914243283209862,
-    "train_runtime": 3256.6129,
+    "train_loss": 0.6911570282725544,
+    "train_runtime": 3257.344,
     "train_samples": 3079,
     "train_samples_per_second": 0.945,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -25,311 +25,311 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.8662172555923462,
-      "logits/rejected": -1.8705354928970337,
-      "logps/chosen": -37.00023651123047,
-      "logps/rejected": -33.65288162231445,
-      "loss": 0.6868,
-      "rewards/accuracies": 0.4861111044883728,
-      "rewards/chosen": 0.0031818184070289135,
-      "rewards/margins": 0.014585819095373154,
-      "rewards/rejected": -0.011404001154005527,
+      "logits/chosen": -1.865976333618164,
+      "logits/rejected": -1.8702969551086426,
+      "logps/chosen": -36.97197723388672,
+      "logps/rejected": -33.66468811035156,
+      "loss": 0.6739,
+      "rewards/accuracies": 0.6388888955116272,
+      "rewards/chosen": 0.020770318806171417,
+      "rewards/margins": 0.041539546102285385,
+      "rewards/rejected": -0.020769227296113968,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.997016191482544,
-      "logits/rejected": -1.9996668100357056,
-      "logps/chosen": -29.636394500732422,
-      "logps/rejected": -29.053604125976562,
-      "loss": 0.6951,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.0028971880674362183,
-      "rewards/margins": -0.002024129033088684,
-      "rewards/rejected": 0.004921317100524902,
+      "logits/chosen": -1.9976377487182617,
+      "logits/rejected": -2.0002894401550293,
+      "logps/chosen": -29.63763427734375,
+      "logps/rejected": -29.050308227539062,
+      "loss": 0.6974,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.0027363565750420094,
+      "rewards/margins": -0.005146201699972153,
+      "rewards/rejected": 0.007882559671998024,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.9197543859481812,
-      "logits/rejected": -1.9170539379119873,
-      "logps/chosen": -31.429040908813477,
-      "logps/rejected": -33.19585418701172,
-      "loss": 0.7032,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.006471562199294567,
-      "rewards/margins": -0.017771439626812935,
-      "rewards/rejected": 0.011299876496195793,
+      "logits/chosen": -1.9201209545135498,
+      "logits/rejected": -1.917419195175171,
+      "logps/chosen": -31.423137664794922,
+      "logps/rejected": -33.23155975341797,
+      "loss": 0.6936,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.004226746968924999,
+      "rewards/margins": 0.0036326400004327297,
+      "rewards/rejected": -0.007859388366341591,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.016721248626709,
-      "logits/rejected": -2.007981300354004,
-      "logps/chosen": -32.594520568847656,
-      "logps/rejected": -32.49314498901367,
-      "loss": 0.7034,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": -0.00885552167892456,
-      "rewards/margins": -0.018619000911712646,
-      "rewards/rejected": 0.009763479232788086,
+      "logits/chosen": -2.0166988372802734,
+      "logits/rejected": -2.007939100265503,
+      "logps/chosen": -32.58073806762695,
+      "logps/rejected": -32.50107192993164,
+      "loss": 0.6996,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.0023586261086165905,
+      "rewards/margins": -0.009320550598204136,
+      "rewards/rejected": 0.006961924023926258,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8646072149276733,
-      "logits/rejected": -1.8538135290145874,
-      "logps/chosen": -33.57071304321289,
-      "logps/rejected": -35.4249267578125,
-      "loss": 0.7044,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.006614250130951405,
-      "rewards/margins": -0.020322056487202644,
-      "rewards/rejected": 0.01370780635625124,
+      "logits/chosen": -1.8645102977752686,
+      "logits/rejected": -1.8537269830703735,
+      "logps/chosen": -33.561344146728516,
+      "logps/rejected": -35.44156265258789,
+      "loss": 0.6992,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0023178723640739918,
+      "rewards/margins": -0.00878816656768322,
+      "rewards/rejected": 0.006470293737947941,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9454749822616577,
-      "logits/rejected": -1.9473978281021118,
-      "logps/chosen": -32.56868362426758,
-      "logps/rejected": -33.174442291259766,
-      "loss": 0.6856,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.015751874074339867,
-      "rewards/margins": 0.01728220283985138,
-      "rewards/rejected": -0.0015303284162655473,
+      "logits/chosen": -1.9450910091400146,
+      "logits/rejected": -1.947035551071167,
+      "logps/chosen": -32.59477233886719,
+      "logps/rejected": -33.191925048828125,
+      "loss": 0.6868,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.003247396554797888,
+      "rewards/margins": 0.015577316284179688,
+      "rewards/rejected": -0.012329919263720512,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.079904794692993,
-      "logits/rejected": -2.0848889350891113,
-      "logps/chosen": -34.005821228027344,
-      "logps/rejected": -36.594722747802734,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.008668524213135242,
-      "rewards/margins": 0.004255795385688543,
-      "rewards/rejected": -0.012924319133162498,
+      "logits/chosen": -2.0797348022460938,
+      "logits/rejected": -2.0847277641296387,
+      "logps/chosen": -34.00436782836914,
+      "logps/rejected": -36.588375091552734,
+      "loss": 0.6939,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.009527785703539848,
+      "rewards/margins": 0.002171074505895376,
+      "rewards/rejected": -0.011698859743773937,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9423730373382568,
-      "logits/rejected": -1.9455206394195557,
-      "logps/chosen": -34.40643310546875,
-      "logps/rejected": -34.57245635986328,
-      "loss": 0.6941,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.01068954449146986,
-      "rewards/margins": 0.000712524342816323,
-      "rewards/rejected": 0.00997702032327652,
+      "logits/chosen": -1.9422187805175781,
+      "logits/rejected": -1.945369005203247,
+      "logps/chosen": -34.411216735839844,
+      "logps/rejected": -34.58810806274414,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.009955625049769878,
+      "rewards/margins": 0.007375092711299658,
+      "rewards/rejected": 0.002580532105639577,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.9506025314331055,
-      "logits/rejected": -1.955121636390686,
-      "logps/chosen": -32.45918655395508,
-      "logps/rejected": -32.362247467041016,
-      "loss": 0.6914,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.004118821118026972,
-      "rewards/margins": 0.005789563059806824,
-      "rewards/rejected": -0.0016707420581951737,
+      "logits/chosen": -1.9506874084472656,
+      "logits/rejected": -1.9551897048950195,
+      "logps/chosen": -32.461769104003906,
+      "logps/rejected": -32.3617057800293,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0033923503942787647,
+      "rewards/margins": 0.005069470964372158,
+      "rewards/rejected": -0.0016771212685853243,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.0490012168884277,
-      "logits/rejected": -2.0470049381256104,
-      "logps/chosen": -32.25275802612305,
-      "logps/rejected": -31.29152488708496,
-      "loss": 0.6909,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.005962538532912731,
-      "rewards/margins": 0.006695576012134552,
-      "rewards/rejected": -0.012658114545047283,
+      "logits/chosen": -2.049078941345215,
+      "logits/rejected": -2.0470805168151855,
+      "logps/chosen": -32.23583984375,
+      "logps/rejected": -31.266429901123047,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.0029988770838826895,
+      "rewards/margins": 0.0031290887854993343,
+      "rewards/rejected": -0.0001302130549447611,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2439146041870117,
-      "eval_logits/rejected": -2.2390429973602295,
-      "eval_logps/chosen": -34.02096939086914,
-      "eval_logps/rejected": -37.521278381347656,
-      "eval_loss": 0.6897847056388855,
-      "eval_rewards/accuracies": 0.5627076625823975,
-      "eval_rewards/chosen": 0.006789046805351973,
-      "eval_rewards/margins": 0.009119975380599499,
-      "eval_rewards/rejected": -0.002330928109586239,
-      "eval_runtime": 146.2765,
-      "eval_samples_per_second": 2.345,
+      "eval_logits/chosen": -2.2439465522766113,
+      "eval_logits/rejected": -2.239065408706665,
+      "eval_logps/chosen": -34.00984573364258,
+      "eval_logps/rejected": -37.50405502319336,
+      "eval_loss": 0.6912049055099487,
+      "eval_rewards/accuracies": 0.4987541437149048,
+      "eval_rewards/chosen": 0.014824562706053257,
+      "eval_rewards/margins": 0.007284608669579029,
+      "eval_rewards/rejected": 0.007539951708167791,
+      "eval_runtime": 146.2285,
+      "eval_samples_per_second": 2.346,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.0054855346679688,
-      "logits/rejected": -2.0030674934387207,
-      "logps/chosen": -33.233333587646484,
-      "logps/rejected": -34.01185989379883,
-      "loss": 0.6959,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": 0.005242079496383667,
-      "rewards/margins": -0.003514623735100031,
-      "rewards/rejected": 0.00875670276582241,
+      "logits/chosen": -2.005307912826538,
+      "logits/rejected": -2.0029025077819824,
+      "logps/chosen": -33.245155334472656,
+      "logps/rejected": -34.01426696777344,
+      "loss": 0.6998,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.000803128641564399,
+      "rewards/margins": -0.009863952174782753,
+      "rewards/rejected": 0.009060824289917946,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.0167009830474854,
-      "logits/rejected": -2.008331537246704,
-      "logps/chosen": -32.438194274902344,
-      "logps/rejected": -32.18104934692383,
-      "loss": 0.6877,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.0033566683996468782,
-      "rewards/margins": 0.013127696700394154,
-      "rewards/rejected": -0.009771028533577919,
+      "logits/chosen": -2.0167527198791504,
+      "logits/rejected": -2.008373737335205,
+      "logps/chosen": -32.43388748168945,
+      "logps/rejected": -32.184364318847656,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.006612470839172602,
+      "rewards/margins": 0.020328599959611893,
+      "rewards/rejected": -0.013716126792132854,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.0462117195129395,
-      "logits/rejected": -2.0381579399108887,
-      "logps/chosen": -30.49118995666504,
-      "logps/rejected": -32.039451599121094,
-      "loss": 0.7014,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.0077363611198961735,
-      "rewards/margins": -0.014514627866446972,
-      "rewards/rejected": 0.006778267212212086,
+      "logits/chosen": -2.0465807914733887,
+      "logits/rejected": -2.03853178024292,
+      "logps/chosen": -30.49270248413086,
+      "logps/rejected": -32.03563690185547,
+      "loss": 0.7048,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.01019179355353117,
+      "rewards/margins": -0.020615343004465103,
+      "rewards/rejected": 0.010423547588288784,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.9772098064422607,
-      "logits/rejected": -1.9874696731567383,
-      "logps/chosen": -31.401031494140625,
-      "logps/rejected": -32.54351043701172,
-      "loss": 0.6877,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.009314531460404396,
-      "rewards/margins": 0.013629269786179066,
-      "rewards/rejected": -0.00431473832577467,
+      "logits/chosen": -1.9769595861434937,
+      "logits/rejected": -1.98722243309021,
+      "logps/chosen": -31.402965545654297,
+      "logps/rejected": -32.55664825439453,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.010017503052949905,
+      "rewards/margins": 0.02307732030749321,
+      "rewards/rejected": -0.013059815391898155,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8905785083770752,
-      "logits/rejected": -1.8916677236557007,
-      "logps/chosen": -34.182621002197266,
-      "logps/rejected": -34.750953674316406,
-      "loss": 0.6937,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.005035269074141979,
-      "rewards/margins": 0.0018822342390194535,
-      "rewards/rejected": 0.0031530349515378475,
+      "logits/chosen": -1.8905092477798462,
+      "logits/rejected": -1.8915998935699463,
+      "logps/chosen": -34.19499588012695,
+      "logps/rejected": -34.76214599609375,
+      "loss": 0.6941,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0013823460321873426,
+      "rewards/margins": 0.0015530474483966827,
+      "rewards/rejected": -0.00293539697304368,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.9425489902496338,
-      "logits/rejected": -1.939056634902954,
-      "logps/chosen": -36.16254425048828,
-      "logps/rejected": -32.711036682128906,
-      "loss": 0.6949,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": 0.0045667411759495735,
-      "rewards/margins": -0.0012899183202534914,
-      "rewards/rejected": 0.005856659263372421,
+      "logits/chosen": -1.9429140090942383,
+      "logits/rejected": -1.939447045326233,
+      "logps/chosen": -36.15991973876953,
+      "logps/rejected": -32.71558380126953,
+      "loss": 0.6935,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.007058164570480585,
+      "rewards/margins": 0.0027567949146032333,
+      "rewards/rejected": 0.004301370121538639,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.041844129562378,
-      "logits/rejected": -2.034442186355591,
-      "logps/chosen": -33.78926467895508,
-      "logps/rejected": -31.351390838623047,
-      "loss": 0.692,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.004301992245018482,
-      "rewards/margins": 0.004073685500770807,
-      "rewards/rejected": 0.00022830665693618357,
+      "logits/chosen": -2.042171001434326,
+      "logits/rejected": -2.0347886085510254,
+      "logps/chosen": -33.77357864379883,
+      "logps/rejected": -31.36397361755371,
+      "loss": 0.684,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.014572665095329285,
+      "rewards/margins": 0.021845515817403793,
+      "rewards/rejected": -0.007272848393768072,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.047542095184326,
-      "logits/rejected": -2.0527994632720947,
-      "logps/chosen": -32.531822204589844,
-      "logps/rejected": -32.49492645263672,
-      "loss": 0.6887,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.00648072361946106,
-      "rewards/margins": 0.01137510221451521,
-      "rewards/rejected": -0.00489437859505415,
+      "logits/chosen": -2.0476887226104736,
+      "logits/rejected": -2.05295729637146,
+      "logps/chosen": -32.52228546142578,
+      "logps/rejected": -32.505516052246094,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.013503499329090118,
+      "rewards/margins": 0.025730689987540245,
+      "rewards/rejected": -0.012227190658450127,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.048457622528076,
-      "logits/rejected": -2.0456814765930176,
-      "logps/chosen": -31.491973876953125,
-      "logps/rejected": -31.334081649780273,
-      "loss": 0.6912,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.00151053664740175,
-      "rewards/margins": 0.006422492675483227,
-      "rewards/rejected": -0.007933029904961586,
+      "logits/chosen": -2.0486130714416504,
+      "logits/rejected": -2.0458409786224365,
+      "logps/chosen": -31.47336196899414,
+      "logps/rejected": -31.337549209594727,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.009355323389172554,
+      "rewards/margins": 0.020955625921487808,
+      "rewards/rejected": -0.011600303463637829,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9186420440673828,
-      "logits/rejected": -1.9233074188232422,
-      "logps/chosen": -31.597881317138672,
-      "logps/rejected": -32.79570770263672,
-      "loss": 0.691,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.004199606366455555,
-      "rewards/margins": 0.006588342599570751,
-      "rewards/rejected": -0.0023887366987764835,
+      "logits/chosen": -1.9189808368682861,
+      "logits/rejected": -1.9236522912979126,
+      "logps/chosen": -31.58243179321289,
+      "logps/rejected": -32.80672073364258,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.014310021884739399,
+      "rewards/margins": 0.02378040924668312,
+      "rewards/rejected": -0.009470385499298573,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.243356466293335,
-      "eval_logits/rejected": -2.2384772300720215,
-      "eval_logps/chosen": -34.025508880615234,
-      "eval_logps/rejected": -37.50304412841797,
-      "eval_loss": 0.6954899430274963,
-      "eval_rewards/accuracies": 0.4779900312423706,
-      "eval_rewards/chosen": 0.004520992282778025,
-      "eval_rewards/margins": -0.0022661893162876368,
-      "eval_rewards/rejected": 0.006787182297557592,
-      "eval_runtime": 145.936,
+      "eval_logits/chosen": -2.243415355682373,
+      "eval_logits/rejected": -2.238534688949585,
+      "eval_logps/chosen": -34.02439880371094,
+      "eval_logps/rejected": -37.504756927490234,
+      "eval_loss": 0.6955486536026001,
+      "eval_rewards/accuracies": 0.48131227493286133,
+      "eval_rewards/chosen": 0.006092504132539034,
+      "eval_rewards/margins": -0.0010250143241137266,
+      "eval_rewards/rejected": 0.007117518223822117,
+      "eval_runtime": 145.9398,
       "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 200
@@ -337,155 +337,155 @@
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0320427417755127,
-      "logits/rejected": -2.0427355766296387,
-      "logps/chosen": -31.94189453125,
-      "logps/rejected": -33.89322280883789,
-      "loss": 0.6796,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.014702295884490013,
-      "rewards/margins": 0.029963891953229904,
-      "rewards/rejected": -0.015261595137417316,
+      "logits/chosen": -2.0316381454467773,
+      "logits/rejected": -2.0423309803009033,
+      "logps/chosen": -31.946680068969727,
+      "logps/rejected": -33.87198257446289,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.014773803763091564,
+      "rewards/margins": 0.02034572884440422,
+      "rewards/rejected": -0.005571924149990082,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9252369403839111,
-      "logits/rejected": -1.9400920867919922,
-      "logps/chosen": -30.095117568969727,
-      "logps/rejected": -31.542316436767578,
-      "loss": 0.6945,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.008213011547923088,
-      "rewards/margins": 0.0002824708935804665,
-      "rewards/rejected": 0.007930541411042213,
+      "logits/chosen": -1.9256389141082764,
+      "logits/rejected": -1.9404897689819336,
+      "logps/chosen": -30.095138549804688,
+      "logps/rejected": -31.5872802734375,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.009841076098382473,
+      "rewards/margins": 0.027301892638206482,
+      "rewards/rejected": -0.017460813745856285,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9828424453735352,
-      "logits/rejected": -1.9868125915527344,
-      "logps/chosen": -33.39949417114258,
-      "logps/rejected": -31.549747467041016,
-      "loss": 0.6899,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.00898725725710392,
-      "rewards/margins": 0.009209835901856422,
-      "rewards/rejected": -0.00022257864475250244,
+      "logits/chosen": -1.983253836631775,
+      "logits/rejected": -1.9872106313705444,
+      "logps/chosen": -33.377899169921875,
+      "logps/rejected": -31.550067901611328,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.02374371513724327,
+      "rewards/margins": 0.024203147739171982,
+      "rewards/rejected": -0.00045942998258396983,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.982708215713501,
-      "logits/rejected": -1.9607412815093994,
-      "logps/chosen": -34.14110565185547,
-      "logps/rejected": -34.958885192871094,
-      "loss": 0.6892,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.006322857923805714,
-      "rewards/margins": 0.01036941446363926,
-      "rewards/rejected": -0.004046556539833546,
+      "logits/chosen": -1.9831098318099976,
+      "logits/rejected": -1.9611488580703735,
+      "logps/chosen": -34.15457534790039,
+      "logps/rejected": -34.96303176879883,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.0004945084219798446,
+      "rewards/margins": 0.006849849130958319,
+      "rewards/rejected": -0.007344359997659922,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.0246503353118896,
-      "logits/rejected": -2.0213494300842285,
-      "logps/chosen": -32.90447998046875,
-      "logps/rejected": -36.232059478759766,
-      "loss": 0.6874,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.010292356833815575,
-      "rewards/margins": 0.013860592618584633,
-      "rewards/rejected": -0.0035682350862771273,
+      "logits/chosen": -2.0243515968322754,
+      "logits/rejected": -2.021042585372925,
+      "logps/chosen": -32.92682647705078,
+      "logps/rejected": -36.226043701171875,
+      "loss": 0.6951,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.001055869972333312,
+      "rewards/margins": -0.00038459646748378873,
+      "rewards/rejected": -0.000671274377964437,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8912708759307861,
-      "logits/rejected": -1.8888260126113892,
-      "logps/chosen": -34.188419342041016,
-      "logps/rejected": -35.51929473876953,
-      "loss": 0.6906,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -9.60230863711331e-06,
-      "rewards/margins": 0.0073286681436002254,
-      "rewards/rejected": -0.007338270545005798,
+      "logits/chosen": -1.8913053274154663,
+      "logits/rejected": -1.8888483047485352,
+      "logps/chosen": -34.18505096435547,
+      "logps/rejected": -35.491554260253906,
+      "loss": 0.6986,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.002010275376960635,
+      "rewards/margins": -0.005831328686326742,
+      "rewards/rejected": 0.007841606624424458,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8759148120880127,
-      "logits/rejected": -1.8733787536621094,
-      "logps/chosen": -34.3978157043457,
-      "logps/rejected": -31.752635955810547,
-      "loss": 0.6988,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.00010652244236553088,
-      "rewards/margins": -0.009455609135329723,
-      "rewards/rejected": 0.009349087253212929,
+      "logits/chosen": -1.8760089874267578,
+      "logits/rejected": -1.8734655380249023,
+      "logps/chosen": -34.394832611083984,
+      "logps/rejected": -31.769023895263672,
+      "loss": 0.6946,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0016577050555497408,
+      "rewards/margins": 0.00027335790218785405,
+      "rewards/rejected": 0.001384348259307444,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9801543951034546,
-      "logits/rejected": -1.9695091247558594,
-      "logps/chosen": -35.32206344604492,
-      "logps/rejected": -31.858333587646484,
-      "loss": 0.6816,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.017289450392127037,
-      "rewards/margins": 0.025958454236388206,
-      "rewards/rejected": -0.00866900384426117,
+      "logits/chosen": -1.9796526432037354,
+      "logits/rejected": -1.9690182209014893,
+      "logps/chosen": -35.34165954589844,
+      "logps/rejected": -31.8492488861084,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.008989132940769196,
+      "rewards/margins": 0.013942519202828407,
+      "rewards/rejected": -0.004953385330736637,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.07523775100708,
-      "logits/rejected": -2.0602152347564697,
-      "logps/chosen": -30.91782569885254,
-      "logps/rejected": -32.631629943847656,
-      "loss": 0.6977,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.0031501606572419405,
-      "rewards/margins": -0.006952463183552027,
-      "rewards/rejected": 0.010102624073624611,
+      "logits/chosen": -2.075491428375244,
+      "logits/rejected": -2.060464859008789,
+      "logps/chosen": -30.913305282592773,
+      "logps/rejected": -32.65747833251953,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.006493172142654657,
+      "rewards/margins": 0.009881803765892982,
+      "rewards/rejected": -0.003388631623238325,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.9465487003326416,
-      "logits/rejected": -1.9440209865570068,
-      "logps/chosen": -32.89127731323242,
-      "logps/rejected": -30.83047866821289,
-      "loss": 0.6819,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.017916660755872726,
-      "rewards/margins": 0.024751421064138412,
-      "rewards/rejected": -0.006834760308265686,
+      "logits/chosen": -1.946946144104004,
+      "logits/rejected": -1.9443925619125366,
+      "logps/chosen": -32.91460037231445,
+      "logps/rejected": -30.81583595275879,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.00750743318349123,
+      "rewards/margins": 0.00692352931946516,
+      "rewards/rejected": 0.0005839023506268859,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.243638753890991,
-      "eval_logits/rejected": -2.238759994506836,
-      "eval_logps/chosen": -34.016868591308594,
-      "eval_logps/rejected": -37.50496292114258,
-      "eval_loss": 0.6929702162742615,
-      "eval_rewards/accuracies": 0.5045680999755859,
-      "eval_rewards/chosen": 0.008841208182275295,
-      "eval_rewards/margins": 0.0030128166545182467,
-      "eval_rewards/rejected": 0.005828391760587692,
-      "eval_runtime": 145.9617,
+      "eval_logits/chosen": -2.244297504425049,
+      "eval_logits/rejected": -2.2394087314605713,
+      "eval_logps/chosen": -34.02434539794922,
+      "eval_logps/rejected": -37.515743255615234,
+      "eval_loss": 0.6924626231193542,
+      "eval_rewards/accuracies": 0.5456810593605042,
+      "eval_rewards/chosen": 0.006125003099441528,
+      "eval_rewards/margins": 0.00560029735788703,
+      "eval_rewards/rejected": 0.0005247062654234469,
+      "eval_runtime": 145.9317,
       "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 300
@@ -493,121 +493,121 @@
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.928741216659546,
-      "logits/rejected": -1.9254982471466064,
-      "logps/chosen": -31.564533233642578,
-      "logps/rejected": -33.7502555847168,
-      "loss": 0.6827,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.019408607855439186,
-      "rewards/margins": 0.0234612338244915,
-      "rewards/rejected": -0.004052626900374889,
+      "logits/chosen": -1.9294414520263672,
+      "logits/rejected": -1.9261829853057861,
+      "logps/chosen": -31.56203269958496,
+      "logps/rejected": -33.72837448120117,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.02479010447859764,
+      "rewards/margins": 0.01652434654533863,
+      "rewards/rejected": 0.008265760727226734,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9805328845977783,
-      "logits/rejected": -1.968227744102478,
-      "logps/chosen": -34.543617248535156,
-      "logps/rejected": -33.547271728515625,
-      "loss": 0.6797,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.02622898854315281,
-      "rewards/margins": 0.03044353798031807,
-      "rewards/rejected": -0.004214548971503973,
+      "logits/chosen": -1.980583906173706,
+      "logits/rejected": -1.9682567119598389,
+      "logps/chosen": -34.58124542236328,
+      "logps/rejected": -33.5312614440918,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.00889885425567627,
+      "rewards/margins": 0.004354512318968773,
+      "rewards/rejected": 0.004544342402368784,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.016136884689331,
-      "logits/rejected": -2.014674663543701,
-      "logps/chosen": -33.480918884277344,
-      "logps/rejected": -32.46858215332031,
-      "loss": 0.6958,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.0024364828132092953,
-      "rewards/margins": -0.0030642927158623934,
-      "rewards/rejected": 0.005500775761902332,
+      "logits/chosen": -2.0161619186401367,
+      "logits/rejected": -2.014702558517456,
+      "logps/chosen": -33.476417541503906,
+      "logps/rejected": -32.486053466796875,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.005623627454042435,
+      "rewards/margins": 0.009506640955805779,
+      "rewards/rejected": -0.0038830172270536423,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.1030609607696533,
-      "logits/rejected": -2.0872530937194824,
-      "logps/chosen": -34.16389083862305,
-      "logps/rejected": -33.10190963745117,
-      "loss": 0.695,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.007748591713607311,
-      "rewards/margins": -0.000315183395287022,
-      "rewards/rejected": 0.00806377548724413,
+      "logits/chosen": -2.103116512298584,
+      "logits/rejected": -2.087329149246216,
+      "logps/chosen": -34.173370361328125,
+      "logps/rejected": -33.09913635253906,
+      "loss": 0.699,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.0036146938800811768,
+      "rewards/margins": -0.007723378483206034,
+      "rewards/rejected": 0.011338073760271072,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9751472473144531,
-      "logits/rejected": -1.9742006063461304,
-      "logps/chosen": -33.24718475341797,
-      "logps/rejected": -32.467201232910156,
-      "loss": 0.6853,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.017456945031881332,
-      "rewards/margins": 0.01848423480987549,
-      "rewards/rejected": -0.001027289079502225,
+      "logits/chosen": -1.9745566844940186,
+      "logits/rejected": -1.9736087322235107,
+      "logps/chosen": -33.24656295776367,
+      "logps/rejected": -32.44744110107422,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.021319344639778137,
+      "rewards/margins": 0.01069412101060152,
+      "rewards/rejected": 0.010625220835208893,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9307329654693604,
-      "logits/rejected": -1.9411048889160156,
-      "logps/chosen": -32.18671798706055,
-      "logps/rejected": -35.27191162109375,
-      "loss": 0.6942,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.008985924534499645,
-      "rewards/margins": 0.0005646973731927574,
-      "rewards/rejected": 0.00842122733592987,
+      "logits/chosen": -1.9309154748916626,
+      "logits/rejected": -1.9412847757339478,
+      "logps/chosen": -32.214351654052734,
+      "logps/rejected": -35.293235778808594,
+      "loss": 0.6968,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.005797215737402439,
+      "rewards/margins": -0.003113214625045657,
+      "rewards/rejected": -0.0026840015780180693,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.0697169303894043,
-      "logits/rejected": -2.0631613731384277,
-      "logps/chosen": -33.653404235839844,
-      "logps/rejected": -29.2293701171875,
-      "loss": 0.6939,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.000717848539352417,
-      "rewards/margins": 0.0007542222738265991,
-      "rewards/rejected": -0.0014720708131790161,
+      "logits/chosen": -2.0695526599884033,
+      "logits/rejected": -2.0629897117614746,
+      "logps/chosen": -33.649295806884766,
+      "logps/rejected": -29.209609985351562,
+      "loss": 0.6991,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.0016033422434702516,
+      "rewards/margins": -0.008483831770718098,
+      "rewards/rejected": 0.01008717343211174,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.9293102025985718,
-      "logits/rejected": -1.931481122970581,
-      "logps/chosen": -34.22270965576172,
-      "logps/rejected": -30.89593505859375,
-      "loss": 0.6846,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.008169981651008129,
-      "rewards/margins": 0.019822364673018456,
-      "rewards/rejected": -0.011652383022010326,
+      "logits/chosen": -1.9294023513793945,
+      "logits/rejected": -1.9315516948699951,
+      "logps/chosen": -34.24349594116211,
+      "logps/rejected": -30.889087677001953,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.0026698322035372257,
+      "rewards/margins": 0.007205922156572342,
+      "rewards/rejected": -0.009875754825770855,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.6914243283209862,
-      "train_runtime": 3256.6129,
+      "train_loss": 0.6911570282725544,
+      "train_runtime": 3257.344,
       "train_samples_per_second": 0.945,
       "train_steps_per_second": 0.118
     }