Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.0139
-- Rewards/chosen: -0.0014
-- Rewards/rejected: 0.0126
-- Rewards/accuracies: 0.4664
-- Rewards/margins: -0.0140
-- Logps/rejected: -37.4987
-- Logps/chosen: -34.0366
-- Logits/rejected: -2.2391
-- Logits/chosen: -2.2439
+- Loss: 0.9998
+- Rewards/chosen: 0.0135
+- Rewards/rejected: 0.0132
+- Rewards/accuracies: 0.4904
+- Rewards/margins: 0.0004
+- Logps/rejected: -37.5002
+- Logps/chosen: -34.0177
+- Logits/rejected: -2.2386
+- Logits/chosen: -2.2435
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 1.0027        | 0.26  | 100  | 1.0043          | 0.0081         | 0.0123           | 0.4958             | -0.0042         | -37.4990       | -34.0229     | -2.2393         | -2.2442       |
-| 1.0046        | 0.52  | 200  | 1.0087          | 0.0001         | 0.0088           | 0.4668             | -0.0087         | -37.5040       | -34.0344     | -2.2385         | -2.2434       |
-| 0.9851        | 0.78  | 300  | 1.0123          | 0.0060         | 0.0180           | 0.4551             | -0.0120         | -37.4909       | -34.0260     | -2.2389         | -2.2438       |
+| 1.0026        | 0.26  | 100  | 1.0066          | 0.0091         | 0.0155           | 0.4983             | -0.0064         | -37.4972       | -34.0232     | -2.2384         | -2.2433       |
+| 0.9923        | 0.52  | 200  | 0.9864          | 0.0127         | -0.0007          | 0.5328             | 0.0134          | -37.5175       | -34.0187     | -2.2389         | -2.2438       |
+| 0.9775        | 0.78  | 300  | 0.9998          | 0.0135         | 0.0132           | 0.4904             | 0.0004          | -37.5002       | -34.0177     | -2.2386         | -2.2435       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53488c2422de288cb2e139b81ffe233e3575897d5fef05b0ee8a430b480bc772
+oid sha256:726700f20fcb28f7858cd1aa59e0cb2bd53a0fe4b7d2b662333531d42f0321bf
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.243932008743286,
-    "eval_logits/rejected": -2.239053249359131,
-    "eval_logps/chosen": -34.03658676147461,
-    "eval_logps/rejected": -37.49867630004883,
-    "eval_loss": 1.0139278173446655,
-    "eval_rewards/accuracies": 0.4663621187210083,
-    "eval_rewards/chosen": -0.0014253630070015788,
-    "eval_rewards/margins": -0.013985722325742245,
-    "eval_rewards/rejected": 0.01256035827100277,
-    "eval_runtime": 145.6648,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.355,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.9909477320584383,
-    "train_runtime": 3254.3492,
+    "train_loss": 0.9929689927534624,
+    "train_runtime": 3255.1859,
     "train_samples": 3079,
     "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.9909477320584383,
-    "train_runtime": 3254.3492,
+    "train_loss": 0.9929689927534624,
+    "train_runtime": 3255.1859,
     "train_samples": 3079,
     "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -25,589 +25,589 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.866518497467041,
-      "logits/rejected": -1.8708432912826538,
-      "logps/chosen": -36.97861862182617,
-      "logps/rejected": -33.65688705444336,
-      "loss": 0.9616,
-      "rewards/accuracies": 0.5416666865348816,
-      "rewards/chosen": 0.01958908699452877,
-      "rewards/margins": 0.038356635719537735,
-      "rewards/rejected": -0.018767546862363815,
+      "logits/chosen": -1.866065502166748,
+      "logits/rejected": -1.8703795671463013,
+      "logps/chosen": -36.988380432128906,
+      "logps/rejected": -33.66728210449219,
+      "loss": 0.9557,
+      "rewards/accuracies": 0.5694444179534912,
+      "rewards/chosen": 0.014575847424566746,
+      "rewards/margins": 0.04434243589639664,
+      "rewards/rejected": -0.02976658195257187,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.997629165649414,
-      "logits/rejected": -2.0002758502960205,
-      "logps/chosen": -29.631610870361328,
-      "logps/rejected": -29.059566497802734,
-      "loss": 0.9953,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.00741056352853775,
-      "rewards/margins": 0.004696969874203205,
-      "rewards/rejected": 0.002713592257350683,
+      "logits/chosen": -1.997332215309143,
+      "logits/rejected": -1.999983549118042,
+      "logps/chosen": -29.625896453857422,
+      "logps/rejected": -29.035802841186523,
+      "loss": 1.0091,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.013037634082138538,
+      "rewards/margins": -0.009074507281184196,
+      "rewards/rejected": 0.02211214043200016,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.9196974039077759,
-      "logits/rejected": -1.9170036315917969,
-      "logps/chosen": -31.402713775634766,
-      "logps/rejected": -33.229434967041016,
-      "loss": 0.9829,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.009367194958031178,
-      "rewards/margins": 0.01705181784927845,
-      "rewards/rejected": -0.007684621028602123,
+      "logits/chosen": -1.9199495315551758,
+      "logits/rejected": -1.917249321937561,
+      "logps/chosen": -31.421478271484375,
+      "logps/rejected": -33.2115364074707,
+      "loss": 1.0098,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.004308355040848255,
+      "rewards/margins": -0.009847955778241158,
+      "rewards/rejected": 0.005539602134376764,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.0161232948303223,
-      "logits/rejected": -2.0073835849761963,
-      "logps/chosen": -32.559471130371094,
-      "logps/rejected": -32.493953704833984,
-      "loss": 1.001,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.012133684009313583,
-      "rewards/margins": -0.0009662110242061317,
-      "rewards/rejected": 0.013099895790219307,
+      "logits/chosen": -2.0169284343719482,
+      "logits/rejected": -2.008178949356079,
+      "logps/chosen": -32.59435272216797,
+      "logps/rejected": -32.49193572998047,
+      "loss": 1.0306,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.014034454710781574,
+      "rewards/margins": -0.030623534694314003,
+      "rewards/rejected": 0.016589080914855003,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8646795749664307,
-      "logits/rejected": -1.8538821935653687,
-      "logps/chosen": -33.56920623779297,
-      "logps/rejected": -35.44689178466797,
-      "loss": 1.012,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.008206313475966454,
-      "rewards/margins": -0.012022162787616253,
-      "rewards/rejected": 0.003815849544480443,
+      "logits/chosen": -1.86457097530365,
+      "logits/rejected": -1.8537908792495728,
+      "logps/chosen": -33.56566619873047,
+      "logps/rejected": -35.423240661621094,
+      "loss": 1.0298,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.006550998892635107,
+      "rewards/margins": -0.029834842309355736,
+      "rewards/rejected": 0.02328384295105934,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9452946186065674,
-      "logits/rejected": -1.9472544193267822,
-      "logps/chosen": -32.58448791503906,
-      "logps/rejected": -33.194095611572266,
-      "loss": 0.9731,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.010985399596393108,
-      "rewards/margins": 0.02688519097864628,
-      "rewards/rejected": -0.015899788588285446,
+      "logits/chosen": -1.9449050426483154,
+      "logits/rejected": -1.9468472003936768,
+      "logps/chosen": -32.59955596923828,
+      "logps/rejected": -33.1828498840332,
+      "loss": 0.9903,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0005022765835747123,
+      "rewards/margins": 0.009675229899585247,
+      "rewards/rejected": -0.00917295552790165,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.079716682434082,
-      "logits/rejected": -2.084702730178833,
-      "logps/chosen": -33.99287033081055,
-      "logps/rejected": -36.58139419555664,
-      "loss": 0.9943,
+      "logits/chosen": -2.079878330230713,
+      "logits/rejected": -2.084862232208252,
+      "logps/chosen": -33.98878860473633,
+      "logps/rejected": -36.574462890625,
+      "loss": 0.9958,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.0030690780840814114,
-      "rewards/margins": 0.00569509994238615,
-      "rewards/rejected": -0.008764175698161125,
+      "rewards/chosen": -0.00024233898147940636,
+      "rewards/margins": 0.004231657367199659,
+      "rewards/rejected": -0.004473996348679066,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9423043727874756,
-      "logits/rejected": -1.9454625844955444,
-      "logps/chosen": -34.40251922607422,
-      "logps/rejected": -34.589447021484375,
-      "loss": 0.9844,
+      "logits/chosen": -1.9425569772720337,
+      "logits/rejected": -1.9457191228866577,
+      "logps/chosen": -34.40068054199219,
+      "logps/rejected": -34.5762939453125,
+      "loss": 0.9912,
       "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.017706653103232384,
-      "rewards/margins": 0.015634063631296158,
-      "rewards/rejected": 0.0020725864451378584,
+      "rewards/chosen": 0.021701809018850327,
+      "rewards/margins": 0.00881027989089489,
+      "rewards/rejected": 0.012891530990600586,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.950506567955017,
-      "logits/rejected": -1.955021619796753,
-      "logps/chosen": -32.44976806640625,
-      "logps/rejected": -32.36195373535156,
-      "loss": 0.9855,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.012356681749224663,
-      "rewards/margins": 0.01448807679116726,
-      "rewards/rejected": -0.0021313955076038837,
+      "logits/chosen": -1.9507396221160889,
+      "logits/rejected": -1.9552500247955322,
+      "logps/chosen": -32.460357666015625,
+      "logps/rejected": -32.354434967041016,
+      "loss": 0.9979,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.005654479376971722,
+      "rewards/margins": 0.002074276329949498,
+      "rewards/rejected": 0.0035802037455141544,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.0489630699157715,
-      "logits/rejected": -2.046970844268799,
-      "logps/chosen": -32.250099182128906,
-      "logps/rejected": -31.2716121673584,
-      "loss": 1.0027,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.006484628655016422,
-      "rewards/margins": -0.002702844562008977,
-      "rewards/rejected": -0.003781783627346158,
+      "logits/chosen": -2.0492873191833496,
+      "logits/rejected": -2.0472888946533203,
+      "logps/chosen": -32.23810958862305,
+      "logps/rejected": -31.260278701782227,
+      "loss": 1.0026,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.0021828364115208387,
+      "rewards/margins": -0.002563956892117858,
+      "rewards/rejected": 0.004746791906654835,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.244150400161743,
-      "eval_logits/rejected": -2.239272117614746,
-      "eval_logps/chosen": -34.02294921875,
-      "eval_logps/rejected": -37.49901580810547,
-      "eval_loss": 1.0042858123779297,
-      "eval_rewards/accuracies": 0.4958471655845642,
-      "eval_rewards/chosen": 0.00812594871968031,
-      "eval_rewards/margins": -0.004197616130113602,
-      "eval_rewards/rejected": 0.012323564849793911,
-      "eval_runtime": 146.0748,
-      "eval_samples_per_second": 2.348,
+      "eval_logits/chosen": -2.243312358856201,
+      "eval_logits/rejected": -2.238436222076416,
+      "eval_logps/chosen": -34.023216247558594,
+      "eval_logps/rejected": -37.49723434448242,
+      "eval_loss": 1.0065712928771973,
+      "eval_rewards/accuracies": 0.49833887815475464,
+      "eval_rewards/chosen": 0.009070919826626778,
+      "eval_rewards/margins": -0.006438401062041521,
+      "eval_rewards/rejected": 0.015509321354329586,
+      "eval_runtime": 146.1739,
+      "eval_samples_per_second": 2.347,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.006051540374756,
-      "logits/rejected": -2.003640651702881,
-      "logps/chosen": -33.229209899902344,
-      "logps/rejected": -34.01814651489258,
-      "loss": 0.9976,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.010226872749626637,
-      "rewards/margins": 0.0023667856585234404,
-      "rewards/rejected": 0.007860084064304829,
+      "logits/chosen": -2.005169153213501,
+      "logits/rejected": -2.0027499198913574,
+      "logps/chosen": -33.2365837097168,
+      "logps/rejected": -34.01953125,
+      "loss": 1.0021,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.005784005858004093,
+      "rewards/margins": -0.002089007291942835,
+      "rewards/rejected": 0.007873011752963066,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.016700506210327,
-      "logits/rejected": -2.0083200931549072,
-      "logps/chosen": -32.47074508666992,
-      "logps/rejected": -32.16743087768555,
-      "loss": 1.0139,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.018086189404129982,
-      "rewards/margins": -0.01393910776823759,
-      "rewards/rejected": -0.004147083964198828,
+      "logits/chosen": -2.0166728496551514,
+      "logits/rejected": -2.0083022117614746,
+      "logps/chosen": -32.457847595214844,
+      "logps/rejected": -32.18357467651367,
+      "loss": 0.9927,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.010353903286159039,
+      "rewards/margins": 0.0072973729111254215,
+      "rewards/rejected": -0.017651278525590897,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.046416759490967,
-      "logits/rejected": -2.0383784770965576,
-      "logps/chosen": -30.48056411743164,
-      "logps/rejected": -32.03020477294922,
-      "loss": 1.0194,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.003392170649021864,
-      "rewards/margins": -0.019355837255716324,
-      "rewards/rejected": 0.015963664278388023,
+      "logits/chosen": -2.0462448596954346,
+      "logits/rejected": -2.038203001022339,
+      "logps/chosen": -30.475027084350586,
+      "logps/rejected": -32.046302795410156,
+      "loss": 1.0048,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.0005506269517354667,
+      "rewards/margins": -0.004812211729586124,
+      "rewards/rejected": 0.005362839438021183,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.977251648902893,
-      "logits/rejected": -1.9875211715698242,
-      "logps/chosen": -31.414047241210938,
-      "logps/rejected": -32.53428268432617,
-      "loss": 0.9965,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.003930580802261829,
-      "rewards/margins": 0.0035118304658681154,
-      "rewards/rejected": 0.0004187510348856449,
+      "logits/chosen": -1.976986289024353,
+      "logits/rejected": -1.9872560501098633,
+      "logps/chosen": -31.407278060913086,
+      "logps/rejected": -32.543296813964844,
+      "loss": 0.9834,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.009907384403049946,
+      "rewards/margins": 0.016641682013869286,
+      "rewards/rejected": -0.006734299007803202,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8907463550567627,
-      "logits/rejected": -1.8918399810791016,
-      "logps/chosen": -34.17836380004883,
-      "logps/rejected": -34.76056671142578,
-      "loss": 0.9877,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.010030902922153473,
-      "rewards/margins": 0.012347406707704067,
-      "rewards/rejected": -0.002316503319889307,
+      "logits/chosen": -1.8905508518218994,
+      "logits/rejected": -1.891632080078125,
+      "logps/chosen": -34.20501708984375,
+      "logps/rejected": -34.77235412597656,
+      "loss": 0.9978,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.009861050173640251,
+      "rewards/margins": 0.0022162157110869884,
+      "rewards/rejected": -0.012077265419065952,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.9423500299453735,
-      "logits/rejected": -1.938869833946228,
-      "logps/chosen": -36.138328552246094,
-      "logps/rejected": -32.716270446777344,
-      "loss": 0.9812,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.023342430591583252,
-      "rewards/margins": 0.018807319924235344,
-      "rewards/rejected": 0.004535110667347908,
+      "logits/chosen": -1.94281005859375,
+      "logits/rejected": -1.939327597618103,
+      "logps/chosen": -36.144107818603516,
+      "logps/rejected": -32.72822570800781,
+      "loss": 0.9736,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.02205902896821499,
+      "rewards/margins": 0.026437148451805115,
+      "rewards/rejected": -0.004378115758299828,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.042358160018921,
-      "logits/rejected": -2.034965753555298,
-      "logps/chosen": -33.78316879272461,
-      "logps/rejected": -31.378530502319336,
-      "loss": 0.971,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.010285439901053905,
-      "rewards/margins": 0.028961649164557457,
-      "rewards/rejected": -0.018676208332180977,
+      "logits/chosen": -2.0420708656311035,
+      "logits/rejected": -2.0346803665161133,
+      "logps/chosen": -33.771934509277344,
+      "logps/rejected": -31.371145248413086,
+      "loss": 0.9638,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.020745161920785904,
+      "rewards/margins": 0.036180444061756134,
+      "rewards/rejected": -0.015435280278325081,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0474698543548584,
-      "logits/rejected": -2.0527453422546387,
-      "logps/chosen": -32.541080474853516,
-      "logps/rejected": -32.49051284790039,
-      "loss": 0.9936,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0025936015881597996,
-      "rewards/margins": 0.00635246466845274,
-      "rewards/rejected": -0.0037588649429380894,
+      "logits/chosen": -2.0475738048553467,
+      "logits/rejected": -2.0528526306152344,
+      "logps/chosen": -32.524593353271484,
+      "logps/rejected": -32.510643005371094,
+      "loss": 0.9634,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.016154326498508453,
+      "rewards/margins": 0.036555372178554535,
+      "rewards/rejected": -0.020401041954755783,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.048386573791504,
-      "logits/rejected": -2.045606851577759,
-      "logps/chosen": -31.502283096313477,
-      "logps/rejected": -31.347753524780273,
-      "loss": 0.9887,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.00933015439659357,
-      "rewards/margins": 0.011346332728862762,
-      "rewards/rejected": -0.02067648433148861,
+      "logits/chosen": -2.048490524291992,
+      "logits/rejected": -2.0457024574279785,
+      "logps/chosen": -31.492746353149414,
+      "logps/rejected": -31.319293975830078,
+      "loss": 1.0022,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.003033895045518875,
+      "rewards/margins": -0.002170243766158819,
+      "rewards/rejected": -0.0008636537822894752,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9189811944961548,
-      "logits/rejected": -1.9236608743667603,
-      "logps/chosen": -31.6005859375,
-      "logps/rejected": -32.77867889404297,
-      "loss": 1.0046,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.003988111391663551,
-      "rewards/margins": -0.004586963914334774,
-      "rewards/rejected": 0.0085750762373209,
+      "logits/chosen": -1.9185683727264404,
+      "logits/rejected": -1.9232347011566162,
+      "logps/chosen": -31.5926513671875,
+      "logps/rejected": -32.78697204589844,
+      "loss": 0.9923,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.01090637780725956,
+      "rewards/margins": 0.007740379776805639,
+      "rewards/rejected": 0.0031659987289458513,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.24337100982666,
-      "eval_logits/rejected": -2.238497257232666,
-      "eval_logps/chosen": -34.03440856933594,
-      "eval_logps/rejected": -37.50404357910156,
-      "eval_loss": 1.0087451934814453,
-      "eval_rewards/accuracies": 0.46677741408348083,
-      "eval_rewards/chosen": 0.00010354903497500345,
-      "eval_rewards/margins": -0.008702815510332584,
-      "eval_rewards/rejected": 0.008806362748146057,
-      "eval_runtime": 145.922,
-      "eval_samples_per_second": 2.351,
+      "eval_logits/chosen": -2.243807315826416,
+      "eval_logits/rejected": -2.2389235496520996,
+      "eval_logps/chosen": -34.018714904785156,
+      "eval_logps/rejected": -37.517478942871094,
+      "eval_loss": 0.9864435195922852,
+      "eval_rewards/accuracies": 0.5328072905540466,
+      "eval_rewards/chosen": 0.012671924196183681,
+      "eval_rewards/margins": 0.013361074030399323,
+      "eval_rewards/rejected": -0.0006891504744999111,
+      "eval_runtime": 145.9847,
+      "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0320065021514893,
-      "logits/rejected": -2.0426933765411377,
-      "logps/chosen": -31.95755386352539,
-      "logps/rejected": -33.87141036987305,
-      "loss": 0.9843,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.00962692778557539,
-      "rewards/margins": 0.015725601464509964,
-      "rewards/rejected": -0.006098673213273287,
+      "logits/chosen": -2.0318503379821777,
+      "logits/rejected": -2.042539596557617,
+      "logps/chosen": -31.948400497436523,
+      "logps/rejected": -33.86983871459961,
+      "loss": 0.976,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.018323851749300957,
+      "rewards/margins": 0.024035325273871422,
+      "rewards/rejected": -0.005711475852876902,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9255682229995728,
-      "logits/rejected": -1.9404188394546509,
-      "logps/chosen": -30.090320587158203,
-      "logps/rejected": -31.563533782958984,
-      "loss": 0.9814,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.014854127541184425,
-      "rewards/margins": 0.018603894859552383,
-      "rewards/rejected": -0.003749764757230878,
+      "logits/chosen": -1.9251388311386108,
+      "logits/rejected": -1.9399843215942383,
+      "logps/chosen": -30.099853515625,
+      "logps/rejected": -31.55409812927246,
+      "loss": 0.9939,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.009348717518150806,
+      "rewards/margins": 0.006087350193411112,
+      "rewards/rejected": 0.003261365694925189,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9833379983901978,
-      "logits/rejected": -1.987309455871582,
-      "logps/chosen": -33.385955810546875,
-      "logps/rejected": -31.533649444580078,
-      "loss": 0.9889,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.02206074632704258,
-      "rewards/margins": 0.011102994903922081,
-      "rewards/rejected": 0.010957752354443073,
+      "logits/chosen": -1.9835479259490967,
+      "logits/rejected": -1.9875112771987915,
+      "logps/chosen": -33.387638092041016,
+      "logps/rejected": -31.554845809936523,
+      "loss": 0.9717,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.023866940289735794,
+      "rewards/margins": 0.028299665078520775,
+      "rewards/rejected": -0.00443272665143013,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9829800128936768,
-      "logits/rejected": -1.9610090255737305,
-      "logps/chosen": -34.147132873535156,
-      "logps/rejected": -34.971717834472656,
-      "loss": 0.9807,
+      "logits/chosen": -1.9826898574829102,
+      "logits/rejected": -1.9607274532318115,
+      "logps/chosen": -34.158443450927734,
+      "logps/rejected": -34.963783264160156,
+      "loss": 0.9934,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.004631907679140568,
-      "rewards/margins": 0.019280528649687767,
-      "rewards/rejected": -0.0146486209705472,
+      "rewards/chosen": -0.0037571662105619907,
+      "rewards/margins": 0.0066327848471701145,
+      "rewards/rejected": -0.01038995012640953,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.024630069732666,
-      "logits/rejected": -2.021322011947632,
-      "logps/chosen": -32.90209197998047,
-      "logps/rejected": -36.2330207824707,
-      "loss": 0.9782,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.016082406044006348,
-      "rewards/margins": 0.021750962361693382,
-      "rewards/rejected": -0.005668557249009609,
+      "logits/chosen": -2.024381160736084,
+      "logits/rejected": -2.0210862159729004,
+      "logps/chosen": -32.9254035949707,
+      "logps/rejected": -36.251712799072266,
+      "loss": 0.9788,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.0002662956831045449,
+      "rewards/margins": 0.02116318605840206,
+      "rewards/rejected": -0.021429482847452164,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8910694122314453,
-      "logits/rejected": -1.888628602027893,
-      "logps/chosen": -34.16587448120117,
-      "logps/rejected": -35.542091369628906,
-      "loss": 0.958,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.015768656507134438,
-      "rewards/margins": 0.04199688881635666,
-      "rewards/rejected": -0.026228230446577072,
+      "logits/chosen": -1.8911311626434326,
+      "logits/rejected": -1.8886839151382446,
+      "logps/chosen": -34.194557189941406,
+      "logps/rejected": -35.51445770263672,
+      "loss": 0.9971,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.004926486872136593,
+      "rewards/margins": 0.002939151134341955,
+      "rewards/rejected": -0.007865638472139835,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8759692907333374,
-      "logits/rejected": -1.8734344244003296,
-      "logps/chosen": -34.40534973144531,
-      "logps/rejected": -31.75263023376465,
-      "loss": 1.0185,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.005422012414783239,
-      "rewards/margins": -0.01851092278957367,
-      "rewards/rejected": 0.013088913634419441,
+      "logits/chosen": -1.8759450912475586,
+      "logits/rejected": -1.8734045028686523,
+      "logps/chosen": -34.40558624267578,
+      "logps/rejected": -31.752349853515625,
+      "loss": 1.0216,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0063878437504172325,
+      "rewards/margins": -0.02157408930361271,
+      "rewards/rejected": 0.015186244621872902,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.979745626449585,
-      "logits/rejected": -1.9691104888916016,
-      "logps/chosen": -35.3358268737793,
-      "logps/rejected": -31.83913803100586,
-      "loss": 0.9867,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.014568204991519451,
-      "rewards/margins": 0.013271312229335308,
-      "rewards/rejected": 0.0012968936935067177,
+      "logits/chosen": -1.980015754699707,
+      "logits/rejected": -1.9693737030029297,
+      "logps/chosen": -35.33230209350586,
+      "logps/rejected": -31.845691680908203,
+      "loss": 0.9768,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.019471382722258568,
+      "rewards/margins": 0.023230455815792084,
+      "rewards/rejected": -0.0037590693682432175,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.075892210006714,
-      "logits/rejected": -2.060863494873047,
-      "logps/chosen": -30.9158935546875,
-      "logps/rejected": -32.63315963745117,
-      "loss": 1.0073,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.005761646665632725,
-      "rewards/margins": -0.007310979068279266,
-      "rewards/rejected": 0.013072624802589417,
+      "logits/chosen": -2.0756278038024902,
+      "logits/rejected": -2.060606002807617,
+      "logps/chosen": -30.907390594482422,
+      "logps/rejected": -32.64055252075195,
+      "loss": 0.9956,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.01338677667081356,
+      "rewards/margins": 0.0043600378558039665,
+      "rewards/rejected": 0.00902673788368702,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.9466651678085327,
-      "logits/rejected": -1.9441168308258057,
-      "logps/chosen": -32.88911056518555,
-      "logps/rejected": -30.80002784729004,
-      "loss": 0.9851,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.026600470766425133,
-      "rewards/margins": 0.014853144064545631,
-      "rewards/rejected": 0.011747325770556927,
+      "logits/chosen": -1.946616768836975,
+      "logits/rejected": -1.9440828561782837,
+      "logps/chosen": -32.894561767578125,
+      "logps/rejected": -30.812387466430664,
+      "loss": 0.9775,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.026041675359010696,
+      "rewards/margins": 0.022502180188894272,
+      "rewards/rejected": 0.003539492143318057,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2438066005706787,
-      "eval_logits/rejected": -2.238919258117676,
-      "eval_logps/chosen": -34.02598190307617,
-      "eval_logps/rejected": -37.4908561706543,
-      "eval_loss": 1.0122511386871338,
-      "eval_rewards/accuracies": 0.45514950156211853,
-      "eval_rewards/chosen": 0.005999694112688303,
-      "eval_rewards/margins": -0.01203568559139967,
-      "eval_rewards/rejected": 0.01803538203239441,
-      "eval_runtime": 145.863,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.2434821128845215,
+      "eval_logits/rejected": -2.2386035919189453,
+      "eval_logps/chosen": -34.017669677734375,
+      "eval_logps/rejected": -37.50018310546875,
+      "eval_loss": 0.9997690916061401,
+      "eval_rewards/accuracies": 0.490448534488678,
+      "eval_rewards/chosen": 0.013507848605513573,
+      "eval_rewards/margins": 0.000357679498847574,
+      "eval_rewards/rejected": 0.0131501704454422,
+      "eval_runtime": 145.7344,
+      "eval_samples_per_second": 2.354,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9287000894546509,
-      "logits/rejected": -1.9254179000854492,
-      "logps/chosen": -31.583938598632812,
-      "logps/rejected": -33.75525665283203,
-      "loss": 0.9772,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.013588106259703636,
-      "rewards/margins": 0.022759849205613136,
-      "rewards/rejected": -0.009171743877232075,
+      "logits/chosen": -1.9287067651748657,
+      "logits/rejected": -1.925451636314392,
+      "logps/chosen": -31.603496551513672,
+      "logps/rejected": -33.734046936035156,
+      "loss": 1.0066,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.00011934786743950099,
+      "rewards/margins": -0.006604696158319712,
+      "rewards/rejected": 0.006485348101705313,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9809232950210571,
-      "logits/rejected": -1.9685947895050049,
-      "logps/chosen": -34.56711196899414,
-      "logps/rejected": -33.54813003540039,
-      "loss": 0.9732,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.020275089889764786,
-      "rewards/margins": 0.026777852326631546,
-      "rewards/rejected": -0.006502761039882898,
+      "logits/chosen": -1.9802377223968506,
+      "logits/rejected": -1.9679291248321533,
+      "logps/chosen": -34.585323333740234,
+      "logps/rejected": -33.57084274291992,
+      "loss": 0.9658,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.00860034953802824,
+      "rewards/margins": 0.034203141927719116,
+      "rewards/rejected": -0.02560279332101345,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.0157439708709717,
-      "logits/rejected": -2.014296531677246,
-      "logps/chosen": -33.48149108886719,
-      "logps/rejected": -32.470550537109375,
-      "loss": 1.0033,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0030108082573860884,
-      "rewards/margins": -0.0033085450995713472,
-      "rewards/rejected": 0.00631935428828001,
+      "logits/chosen": -2.015906810760498,
+      "logits/rejected": -2.014427661895752,
+      "logps/chosen": -33.49116516113281,
+      "logps/rejected": -32.47978973388672,
+      "loss": 1.0041,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.0042965407483279705,
+      "rewards/margins": -0.004129170440137386,
+      "rewards/rejected": -0.00016737030819058418,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.1027872562408447,
-      "logits/rejected": -2.086984157562256,
-      "logps/chosen": -34.1781120300293,
-      "logps/rejected": -33.094017028808594,
-      "loss": 1.0159,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0008956976234912872,
-      "rewards/margins": -0.015917152166366577,
-      "rewards/rejected": 0.016812851652503014,
+      "logits/chosen": -2.1030631065368652,
+      "logits/rejected": -2.0872654914855957,
+      "logps/chosen": -34.18492889404297,
+      "logps/rejected": -33.08319854736328,
+      "loss": 1.0323,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.00442854966968298,
+      "rewards/margins": -0.03229869529604912,
+      "rewards/rejected": 0.027870142832398415,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9746129512786865,
-      "logits/rejected": -1.9736511707305908,
-      "logps/chosen": -33.25445556640625,
-      "logps/rejected": -32.4587287902832,
-      "loss": 0.9851,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.019348327070474625,
-      "rewards/margins": 0.014853959903120995,
-      "rewards/rejected": 0.004494364373385906,
+      "logits/chosen": -1.9745471477508545,
+      "logits/rejected": -1.9736032485961914,
+      "logps/chosen": -33.23271942138672,
+      "logps/rejected": -32.4765510559082,
+      "loss": 0.9514,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0395023450255394,
+      "rewards/margins": 0.04862459748983383,
+      "rewards/rejected": -0.009122253395617008,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9303343296051025,
-      "logits/rejected": -1.940706491470337,
-      "logps/chosen": -32.189186096191406,
-      "logps/rejected": -35.288429260253906,
-      "loss": 0.9894,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.010855492204427719,
-      "rewards/margins": 0.010628147050738335,
-      "rewards/rejected": 0.00022734627418685704,
+      "logits/chosen": -1.9305438995361328,
+      "logits/rejected": -1.940913438796997,
+      "logps/chosen": -32.22040939331055,
+      "logps/rejected": -35.28728103637695,
+      "loss": 1.0138,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.012576865963637829,
+      "rewards/margins": -0.013758530840277672,
+      "rewards/rejected": 0.0011816651094704866,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.0694239139556885,
-      "logits/rejected": -2.0628561973571777,
-      "logps/chosen": -33.65647506713867,
-      "logps/rejected": -29.229717254638672,
-      "loss": 1.0008,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.0031533471774309874,
-      "rewards/margins": -0.0008490110631100833,
-      "rewards/rejected": -0.0023043369874358177,
+      "logits/chosen": -2.069648265838623,
+      "logits/rejected": -2.0630898475646973,
+      "logps/chosen": -33.63695526123047,
+      "logps/rejected": -29.226470947265625,
+      "loss": 0.988,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.012009462341666222,
+      "rewards/margins": 0.012046033516526222,
+      "rewards/rejected": -3.6573597753886133e-05,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.9291013479232788,
-      "logits/rejected": -1.9312855005264282,
-      "logps/chosen": -34.26526641845703,
-      "logps/rejected": -30.913259506225586,
-      "loss": 0.9899,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.01835118792951107,
-      "rewards/margins": 0.010092146694660187,
-      "rewards/rejected": -0.028443332761526108,
+      "logits/chosen": -1.928865671157837,
+      "logits/rejected": -1.9310123920440674,
+      "logps/chosen": -34.243560791015625,
+      "logps/rejected": -30.892742156982422,
+      "loss": 0.9875,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.003606534795835614,
+      "rewards/margins": 0.012486499734222889,
+      "rewards/rejected": -0.01609303429722786,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.9909477320584383,
-      "train_runtime": 3254.3492,
+      "train_loss": 0.9929689927534624,
+      "train_runtime": 3255.1859,
       "train_samples_per_second": 0.946,
       "train_steps_per_second": 0.118
     }