Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.0085
-- Rewards/chosen: -0.0006
-- Rewards/rejected: 0.0077
-- Rewards/accuracies: 0.4817
-- Rewards/margins: -0.0083
-- Logps/rejected: -37.5037
-- Logps/chosen: -34.0355
-- Logits/rejected: -2.2391
-- Logits/chosen: -2.2440
+- Loss: 1.0123
+- Rewards/chosen: 0.0060
+- Rewards/rejected: 0.0180
+- Rewards/accuracies: 0.4551
+- Rewards/margins: -0.0120
+- Logps/rejected: -37.4909
+- Logps/chosen: -34.0260
+- Logits/rejected: -2.2389
+- Logits/chosen: -2.2438
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 1.0007        | 0.26  | 100  | 0.9989          | 0.0092         | 0.0079           | 0.5403             | 0.0013          | -37.5035       | -34.0192     | -2.2391         | -2.2440       |
-| 0.9898        | 0.52  | 200  | 0.9970          | 0.0124         | 0.0094           | 0.5307             | 0.0030          | -37.5009       | -34.0138     | -2.2391         | -2.2440       |
-| 0.9846        | 0.78  | 300  | 1.0020          | 0.0050         | 0.0068           | 0.4635             | -0.0019         | -37.5052       | -34.0263     | -2.2388         | -2.2437       |
+| 1.0027        | 0.26  | 100  | 1.0043          | 0.0081         | 0.0123           | 0.4958             | -0.0042         | -37.4990       | -34.0229     | -2.2393         | -2.2442       |
+| 1.0046        | 0.52  | 200  | 1.0087          | 0.0001         | 0.0088           | 0.4668             | -0.0087         | -37.5040       | -34.0344     | -2.2385         | -2.2434       |
+| 0.9851        | 0.78  | 300  | 1.0123          | 0.0060         | 0.0180           | 0.4551             | -0.0120         | -37.4909       | -34.0260     | -2.2389         | -2.2438       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a430e8c88a91fa43e2714883898b2f7ba08e7ee4cac27992c399c9f4bbd50ac6
+oid sha256:33c988b2a185b460a166f057b4a03ef055b164c7749bc3b291b623004c21414d
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.243980884552002,
-    "eval_logits/rejected": -2.2390992641448975,
-    "eval_logps/chosen": -34.03550720214844,
-    "eval_logps/rejected": -37.50373458862305,
-    "eval_loss": 1.0085219144821167,
-    "eval_rewards/accuracies": 0.48172760009765625,
-    "eval_rewards/chosen": -0.0005728387623094022,
-    "eval_rewards/margins": -0.008305290713906288,
-    "eval_rewards/rejected": 0.007732452359050512,
-    "eval_runtime": 145.8784,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.351,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.9932638019710393,
-    "train_runtime": 3254.1177,
+    "train_loss": 0.9909477320584383,
+    "train_runtime": 3254.3492,
     "train_samples": 3079,
     "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.9932638019710393,
-    "train_runtime": 3254.1177,
+    "train_loss": 0.9909477320584383,
+    "train_runtime": 3254.3492,
     "train_samples": 3079,
     "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -25,467 +25,467 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.866068720817566,
-      "logits/rejected": -1.8703863620758057,
-      "logps/chosen": -36.99105453491211,
-      "logps/rejected": -33.65992736816406,
-      "loss": 0.9728,
-      "rewards/accuracies": 0.5138888955116272,
-      "rewards/chosen": 0.009325359016656876,
-      "rewards/margins": 0.027240369468927383,
-      "rewards/rejected": -0.017915012314915657,
+      "logits/chosen": -1.866518497467041,
+      "logits/rejected": -1.8708432912826538,
+      "logps/chosen": -36.97861862182617,
+      "logps/rejected": -33.65688705444336,
+      "loss": 0.9616,
+      "rewards/accuracies": 0.5416666865348816,
+      "rewards/chosen": 0.01958908699452877,
+      "rewards/margins": 0.038356635719537735,
+      "rewards/rejected": -0.018767546862363815,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.9970887899398804,
-      "logits/rejected": -1.9997413158416748,
-      "logps/chosen": -29.63595199584961,
-      "logps/rejected": -29.070571899414062,
-      "loss": 0.992,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.003744622226804495,
-      "rewards/margins": 0.008021654561161995,
-      "rewards/rejected": -0.004277031868696213,
+      "logits/chosen": -1.997629165649414,
+      "logits/rejected": -2.0002758502960205,
+      "logps/chosen": -29.631610870361328,
+      "logps/rejected": -29.059566497802734,
+      "loss": 0.9953,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.00741056352853775,
+      "rewards/margins": 0.004696969874203205,
+      "rewards/rejected": 0.002713592257350683,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.9195963144302368,
-      "logits/rejected": -1.9168994426727295,
-      "logps/chosen": -31.414175033569336,
-      "logps/rejected": -33.25041961669922,
-      "loss": 0.9797,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0011531396303325891,
-      "rewards/margins": 0.02032935619354248,
-      "rewards/rejected": -0.019176214933395386,
+      "logits/chosen": -1.9196974039077759,
+      "logits/rejected": -1.9170036315917969,
+      "logps/chosen": -31.402713775634766,
+      "logps/rejected": -33.229434967041016,
+      "loss": 0.9829,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.009367194958031178,
+      "rewards/margins": 0.01705181784927845,
+      "rewards/rejected": -0.007684621028602123,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.01688814163208,
-      "logits/rejected": -2.0081212520599365,
-      "logps/chosen": -32.583614349365234,
-      "logps/rejected": -32.48966979980469,
-      "loss": 1.0179,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.004085154738277197,
-      "rewards/margins": -0.017886672168970108,
-      "rewards/rejected": 0.013801517896354198,
+      "logits/chosen": -2.0161232948303223,
+      "logits/rejected": -2.0073835849761963,
+      "logps/chosen": -32.559471130371094,
+      "logps/rejected": -32.493953704833984,
+      "loss": 1.001,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.012133684009313583,
+      "rewards/margins": -0.0009662110242061317,
+      "rewards/rejected": 0.013099895790219307,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8644500970840454,
-      "logits/rejected": -1.853655219078064,
-      "logps/chosen": -33.54804229736328,
-      "logps/rejected": -35.43767166137695,
-      "loss": 1.0031,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.005664288066327572,
-      "rewards/margins": -0.0031387731432914734,
-      "rewards/rejected": 0.00880306214094162,
+      "logits/chosen": -1.8646795749664307,
+      "logits/rejected": -1.8538821935653687,
+      "logps/chosen": -33.56920623779297,
+      "logps/rejected": -35.44689178466797,
+      "loss": 1.012,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.008206313475966454,
+      "rewards/margins": -0.012022162787616253,
+      "rewards/rejected": 0.003815849544480443,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9449504613876343,
-      "logits/rejected": -1.946890115737915,
-      "logps/chosen": -32.58049774169922,
-      "logps/rejected": -33.199378967285156,
-      "loss": 0.9714,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.011811850592494011,
-      "rewards/margins": 0.02860717847943306,
-      "rewards/rejected": -0.016795331612229347,
+      "logits/chosen": -1.9452946186065674,
+      "logits/rejected": -1.9472544193267822,
+      "logps/chosen": -32.58448791503906,
+      "logps/rejected": -33.194095611572266,
+      "loss": 0.9731,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.010985399596393108,
+      "rewards/margins": 0.02688519097864628,
+      "rewards/rejected": -0.015899788588285446,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.0795464515686035,
-      "logits/rejected": -2.0845184326171875,
-      "logps/chosen": -33.98798370361328,
-      "logps/rejected": -36.569732666015625,
-      "loss": 0.9992,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.0003018675488419831,
-      "rewards/margins": 0.0008206713828258216,
-      "rewards/rejected": -0.0005188033101148903,
+      "logits/chosen": -2.079716682434082,
+      "logits/rejected": -2.084702730178833,
+      "logps/chosen": -33.99287033081055,
+      "logps/rejected": -36.58139419555664,
+      "loss": 0.9943,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.0030690780840814114,
+      "rewards/margins": 0.00569509994238615,
+      "rewards/rejected": -0.008764175698161125,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9425585269927979,
-      "logits/rejected": -1.945704698562622,
-      "logps/chosen": -34.408721923828125,
-      "logps/rejected": -34.581504821777344,
-      "loss": 0.9951,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.011452903971076012,
-      "rewards/margins": 0.0049109989777207375,
-      "rewards/rejected": 0.006541903130710125,
+      "logits/chosen": -1.9423043727874756,
+      "logits/rejected": -1.9454625844955444,
+      "logps/chosen": -34.40251922607422,
+      "logps/rejected": -34.589447021484375,
+      "loss": 0.9844,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.017706653103232384,
+      "rewards/margins": 0.015634063631296158,
+      "rewards/rejected": 0.0020725864451378584,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.9505088329315186,
-      "logits/rejected": -1.9550243616104126,
-      "logps/chosen": -32.451255798339844,
-      "logps/rejected": -32.366790771484375,
-      "loss": 0.9856,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.009697502478957176,
-      "rewards/margins": 0.014425436034798622,
-      "rewards/rejected": -0.0047279344871640205,
+      "logits/chosen": -1.950506567955017,
+      "logits/rejected": -1.955021619796753,
+      "logps/chosen": -32.44976806640625,
+      "logps/rejected": -32.36195373535156,
+      "loss": 0.9855,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.012356681749224663,
+      "rewards/margins": 0.01448807679116726,
+      "rewards/rejected": -0.0021313955076038837,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.048947334289551,
-      "logits/rejected": -2.046935796737671,
-      "logps/chosen": -32.25363540649414,
-      "logps/rejected": -31.277917861938477,
-      "loss": 1.0007,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.007680465932935476,
-      "rewards/margins": -0.0006578800384886563,
-      "rewards/rejected": -0.007022587116807699,
+      "logits/chosen": -2.0489630699157715,
+      "logits/rejected": -2.046970844268799,
+      "logps/chosen": -32.250099182128906,
+      "logps/rejected": -31.2716121673584,
+      "loss": 1.0027,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.006484628655016422,
+      "rewards/margins": -0.002702844562008977,
+      "rewards/rejected": -0.003781783627346158,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.243957042694092,
-      "eval_logits/rejected": -2.2390689849853516,
-      "eval_logps/chosen": -34.019203186035156,
-      "eval_logps/rejected": -37.50347900390625,
-      "eval_loss": 0.998858630657196,
-      "eval_rewards/accuracies": 0.5402824282646179,
-      "eval_rewards/chosen": 0.009209612384438515,
-      "eval_rewards/margins": 0.0013267018366605043,
-      "eval_rewards/rejected": 0.00788290984928608,
-      "eval_runtime": 145.9923,
-      "eval_samples_per_second": 2.349,
-      "eval_steps_per_second": 0.295,
+      "eval_logits/chosen": -2.244150400161743,
+      "eval_logits/rejected": -2.239272117614746,
+      "eval_logps/chosen": -34.02294921875,
+      "eval_logps/rejected": -37.49901580810547,
+      "eval_loss": 1.0042858123779297,
+      "eval_rewards/accuracies": 0.4958471655845642,
+      "eval_rewards/chosen": 0.00812594871968031,
+      "eval_rewards/margins": -0.004197616130113602,
+      "eval_rewards/rejected": 0.012323564849793911,
+      "eval_runtime": 146.0748,
+      "eval_samples_per_second": 2.348,
+      "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.005302667617798,
-      "logits/rejected": -2.002894878387451,
-      "logps/chosen": -33.23944854736328,
-      "logps/rejected": -34.005104064941406,
-      "loss": 1.0119,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.002622986678034067,
-      "rewards/margins": -0.01194014959037304,
-      "rewards/rejected": 0.014563137665390968,
+      "logits/chosen": -2.006051540374756,
+      "logits/rejected": -2.003640651702881,
+      "logps/chosen": -33.229209899902344,
+      "logps/rejected": -34.01814651489258,
+      "loss": 0.9976,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.010226872749626637,
+      "rewards/margins": 0.0023667856585234404,
+      "rewards/rejected": 0.007860084064304829,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.0167059898376465,
-      "logits/rejected": -2.0083398818969727,
-      "logps/chosen": -32.45713424682617,
-      "logps/rejected": -32.17597198486328,
-      "loss": 0.9987,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.0073361145332455635,
-      "rewards/margins": 0.0013455990701913834,
-      "rewards/rejected": -0.008681714534759521,
+      "logits/chosen": -2.016700506210327,
+      "logits/rejected": -2.0083200931549072,
+      "logps/chosen": -32.47074508666992,
+      "logps/rejected": -32.16743087768555,
+      "loss": 1.0139,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.018086189404129982,
+      "rewards/margins": -0.01393910776823759,
+      "rewards/rejected": -0.004147083964198828,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.0464558601379395,
-      "logits/rejected": -2.038412094116211,
-      "logps/chosen": -30.482311248779297,
-      "logps/rejected": -32.04933166503906,
-      "loss": 1.0062,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.003956255037337542,
-      "rewards/margins": -0.006162940990179777,
-      "rewards/rejected": 0.002206685720011592,
+      "logits/chosen": -2.046416759490967,
+      "logits/rejected": -2.0383784770965576,
+      "logps/chosen": -30.48056411743164,
+      "logps/rejected": -32.03020477294922,
+      "loss": 1.0194,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.003392170649021864,
+      "rewards/margins": -0.019355837255716324,
+      "rewards/rejected": 0.015963664278388023,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.9769861698150635,
-      "logits/rejected": -1.9872652292251587,
-      "logps/chosen": -31.415592193603516,
-      "logps/rejected": -32.5562858581543,
-      "loss": 0.9847,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.0024404728319495916,
-      "rewards/margins": 0.015285758301615715,
-      "rewards/rejected": -0.01284528523683548,
+      "logits/chosen": -1.977251648902893,
+      "logits/rejected": -1.9875211715698242,
+      "logps/chosen": -31.414047241210938,
+      "logps/rejected": -32.53428268432617,
+      "loss": 0.9965,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.003930580802261829,
+      "rewards/margins": 0.0035118304658681154,
+      "rewards/rejected": 0.0004187510348856449,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.890750527381897,
-      "logits/rejected": -1.8918195962905884,
-      "logps/chosen": -34.196632385253906,
-      "logps/rejected": -34.74388885498047,
-      "loss": 1.0104,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.002364098560065031,
-      "rewards/margins": -0.0103833619505167,
-      "rewards/rejected": 0.008019264787435532,
+      "logits/chosen": -1.8907463550567627,
+      "logits/rejected": -1.8918399810791016,
+      "logps/chosen": -34.17836380004883,
+      "logps/rejected": -34.76056671142578,
+      "loss": 0.9877,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.010030902922153473,
+      "rewards/margins": 0.012347406707704067,
+      "rewards/rejected": -0.002316503319889307,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.9424158334732056,
-      "logits/rejected": -1.9389365911483765,
-      "logps/chosen": -36.13062286376953,
-      "logps/rejected": -32.720314025878906,
-      "loss": 0.9768,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.02463117241859436,
-      "rewards/margins": 0.023165332153439522,
-      "rewards/rejected": 0.0014658428262919188,
+      "logits/chosen": -1.9423500299453735,
+      "logits/rejected": -1.938869833946228,
+      "logps/chosen": -36.138328552246094,
+      "logps/rejected": -32.716270446777344,
+      "loss": 0.9812,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.023342430591583252,
+      "rewards/margins": 0.018807319924235344,
+      "rewards/rejected": 0.004535110667347908,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.04191517829895,
-      "logits/rejected": -2.034518003463745,
-      "logps/chosen": -33.780067443847656,
-      "logps/rejected": -31.35748863220215,
-      "loss": 0.9859,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.010676576755940914,
-      "rewards/margins": 0.014060018584132195,
-      "rewards/rejected": -0.0033834422938525677,
+      "logits/chosen": -2.042358160018921,
+      "logits/rejected": -2.034965753555298,
+      "logps/chosen": -33.78316879272461,
+      "logps/rejected": -31.378530502319336,
+      "loss": 0.971,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.010285439901053905,
+      "rewards/margins": 0.028961649164557457,
+      "rewards/rejected": -0.018676208332180977,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0474770069122314,
-      "logits/rejected": -2.0527572631835938,
-      "logps/chosen": -32.51685333251953,
-      "logps/rejected": -32.48542022705078,
-      "loss": 0.9831,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.01676076650619507,
-      "rewards/margins": 0.016928378492593765,
-      "rewards/rejected": -0.00016761067672632635,
+      "logits/chosen": -2.0474698543548584,
+      "logits/rejected": -2.0527453422546387,
+      "logps/chosen": -32.541080474853516,
+      "logps/rejected": -32.49051284790039,
+      "loss": 0.9936,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0025936015881597996,
+      "rewards/margins": 0.00635246466845274,
+      "rewards/rejected": -0.0037588649429380894,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.0486884117126465,
-      "logits/rejected": -2.045897960662842,
-      "logps/chosen": -31.490026473999023,
-      "logps/rejected": -31.336299896240234,
-      "loss": 0.9898,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.0006444025784730911,
-      "rewards/margins": 0.010206506587564945,
-      "rewards/rejected": -0.010850909166038036,
+      "logits/chosen": -2.048386573791504,
+      "logits/rejected": -2.045606851577759,
+      "logps/chosen": -31.502283096313477,
+      "logps/rejected": -31.347753524780273,
+      "loss": 0.9887,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.00933015439659357,
+      "rewards/margins": 0.011346332728862762,
+      "rewards/rejected": -0.02067648433148861,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9185329675674438,
-      "logits/rejected": -1.9231984615325928,
-      "logps/chosen": -31.579875946044922,
-      "logps/rejected": -32.781455993652344,
-      "loss": 0.9898,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.015846019610762596,
-      "rewards/margins": 0.010161412879824638,
-      "rewards/rejected": 0.005684606730937958,
+      "logits/chosen": -1.9189811944961548,
+      "logits/rejected": -1.9236608743667603,
+      "logps/chosen": -31.6005859375,
+      "logps/rejected": -32.77867889404297,
+      "loss": 1.0046,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.003988111391663551,
+      "rewards/margins": -0.004586963914334774,
+      "rewards/rejected": 0.0085750762373209,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2439582347869873,
-      "eval_logits/rejected": -2.239088296890259,
-      "eval_logps/chosen": -34.01380920410156,
-      "eval_logps/rejected": -37.500911712646484,
-      "eval_loss": 0.9969704151153564,
-      "eval_rewards/accuracies": 0.5307309031486511,
-      "eval_rewards/chosen": 0.01244510430842638,
-      "eval_rewards/margins": 0.0030194728169590235,
-      "eval_rewards/rejected": 0.009425631724298,
-      "eval_runtime": 145.8589,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.24337100982666,
+      "eval_logits/rejected": -2.238497257232666,
+      "eval_logps/chosen": -34.03440856933594,
+      "eval_logps/rejected": -37.50404357910156,
+      "eval_loss": 1.0087451934814453,
+      "eval_rewards/accuracies": 0.46677741408348083,
+      "eval_rewards/chosen": 0.00010354903497500345,
+      "eval_rewards/margins": -0.008702815510332584,
+      "eval_rewards/rejected": 0.008806362748146057,
+      "eval_runtime": 145.922,
+      "eval_samples_per_second": 2.351,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0318081378936768,
-      "logits/rejected": -2.0424768924713135,
-      "logps/chosen": -31.95905113220215,
-      "logps/rejected": -33.87731170654297,
-      "loss": 0.9839,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.007351105101406574,
-      "rewards/margins": 0.01611880213022232,
-      "rewards/rejected": -0.008767696097493172,
+      "logits/chosen": -2.0320065021514893,
+      "logits/rejected": -2.0426933765411377,
+      "logps/chosen": -31.95755386352539,
+      "logps/rejected": -33.87141036987305,
+      "loss": 0.9843,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.00962692778557539,
+      "rewards/margins": 0.015725601464509964,
+      "rewards/rejected": -0.006098673213273287,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9253685474395752,
-      "logits/rejected": -1.940233826637268,
-      "logps/chosen": -30.08770179748535,
-      "logps/rejected": -31.577301025390625,
-      "loss": 0.9742,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.014303619973361492,
-      "rewards/margins": 0.025777745991945267,
-      "rewards/rejected": -0.011474122293293476,
+      "logits/chosen": -1.9255682229995728,
+      "logits/rejected": -1.9404188394546509,
+      "logps/chosen": -30.090320587158203,
+      "logps/rejected": -31.563533782958984,
+      "loss": 0.9814,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.014854127541184425,
+      "rewards/margins": 0.018603894859552383,
+      "rewards/rejected": -0.003749764757230878,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9832963943481445,
-      "logits/rejected": -1.9872655868530273,
-      "logps/chosen": -33.39131546020508,
-      "logps/rejected": -31.548086166381836,
-      "loss": 0.985,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.015694385394454002,
-      "rewards/margins": 0.014964587986469269,
-      "rewards/rejected": 0.0007297966512851417,
+      "logits/chosen": -1.9833379983901978,
+      "logits/rejected": -1.987309455871582,
+      "logps/chosen": -33.385955810546875,
+      "logps/rejected": -31.533649444580078,
+      "loss": 0.9889,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.02206074632704258,
+      "rewards/margins": 0.011102994903922081,
+      "rewards/rejected": 0.010957752354443073,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9828859567642212,
-      "logits/rejected": -1.9609191417694092,
-      "logps/chosen": -34.16459655761719,
-      "logps/rejected": -34.95134735107422,
-      "loss": 1.0062,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.0065087564289569855,
-      "rewards/margins": -0.006178082898259163,
-      "rewards/rejected": -0.0003306727739982307,
+      "logits/chosen": -1.9829800128936768,
+      "logits/rejected": -1.9610090255737305,
+      "logps/chosen": -34.147132873535156,
+      "logps/rejected": -34.971717834472656,
+      "loss": 0.9807,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.004631907679140568,
+      "rewards/margins": 0.019280528649687767,
+      "rewards/rejected": -0.0146486209705472,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.0246593952178955,
-      "logits/rejected": -2.0213539600372314,
-      "logps/chosen": -32.9222412109375,
-      "logps/rejected": -36.228416442871094,
-      "loss": 0.9962,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.0016961356159299612,
-      "rewards/margins": 0.0037924889475107193,
-      "rewards/rejected": -0.0020963551942259073,
+      "logits/chosen": -2.024630069732666,
+      "logits/rejected": -2.021322011947632,
+      "logps/chosen": -32.90209197998047,
+      "logps/rejected": -36.2330207824707,
+      "loss": 0.9782,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.016082406044006348,
+      "rewards/margins": 0.021750962361693382,
+      "rewards/rejected": -0.005668557249009609,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8918907642364502,
-      "logits/rejected": -1.8894577026367188,
-      "logps/chosen": -34.189327239990234,
-      "logps/rejected": -35.505496978759766,
-      "loss": 1.0,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.0005569729837588966,
-      "rewards/margins": -2.999706157424953e-05,
-      "rewards/rejected": -0.0005269756657071412,
+      "logits/chosen": -1.8910694122314453,
+      "logits/rejected": -1.888628602027893,
+      "logps/chosen": -34.16587448120117,
+      "logps/rejected": -35.542091369628906,
+      "loss": 0.958,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.015768656507134438,
+      "rewards/margins": 0.04199688881635666,
+      "rewards/rejected": -0.026228230446577072,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8758213520050049,
-      "logits/rejected": -1.8732786178588867,
-      "logps/chosen": -34.38149642944336,
-      "logps/rejected": -31.752059936523438,
-      "loss": 1.0019,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.009660609066486359,
-      "rewards/margins": -0.0019017171580344439,
-      "rewards/rejected": 0.011562327854335308,
+      "logits/chosen": -1.8759692907333374,
+      "logits/rejected": -1.8734344244003296,
+      "logps/chosen": -34.40534973144531,
+      "logps/rejected": -31.75263023376465,
+      "loss": 1.0185,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.005422012414783239,
+      "rewards/margins": -0.01851092278957367,
+      "rewards/rejected": 0.013088913634419441,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9798027276992798,
-      "logits/rejected": -1.9691746234893799,
-      "logps/chosen": -35.328853607177734,
-      "logps/rejected": -31.868383407592773,
-      "loss": 0.9669,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.016671547666192055,
-      "rewards/margins": 0.03310702368617058,
-      "rewards/rejected": -0.016435474157333374,
+      "logits/chosen": -1.979745626449585,
+      "logits/rejected": -1.9691104888916016,
+      "logps/chosen": -35.3358268737793,
+      "logps/rejected": -31.83913803100586,
+      "loss": 0.9867,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.014568204991519451,
+      "rewards/margins": 0.013271312229335308,
+      "rewards/rejected": 0.0012968936935067177,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.075648546218872,
-      "logits/rejected": -2.0605978965759277,
-      "logps/chosen": -30.926809310913086,
-      "logps/rejected": -32.621070861816406,
-      "loss": 1.0201,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.0016108205309137702,
-      "rewards/margins": -0.020070534199476242,
-      "rewards/rejected": 0.018459713086485863,
+      "logits/chosen": -2.075892210006714,
+      "logits/rejected": -2.060863494873047,
+      "logps/chosen": -30.9158935546875,
+      "logps/rejected": -32.63315963745117,
+      "loss": 1.0073,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.005761646665632725,
+      "rewards/margins": -0.007310979068279266,
+      "rewards/rejected": 0.013072624802589417,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.9466993808746338,
-      "logits/rejected": -1.9441711902618408,
-      "logps/chosen": -32.90770721435547,
-      "logps/rejected": -30.82305335998535,
-      "loss": 0.9846,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.011643724516034126,
-      "rewards/margins": 0.015391023829579353,
-      "rewards/rejected": -0.0037472969852387905,
+      "logits/chosen": -1.9466651678085327,
+      "logits/rejected": -1.9441168308258057,
+      "logps/chosen": -32.88911056518555,
+      "logps/rejected": -30.80002784729004,
+      "loss": 0.9851,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.026600470766425133,
+      "rewards/margins": 0.014853144064545631,
+      "rewards/rejected": 0.011747325770556927,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.243670701980591,
-      "eval_logits/rejected": -2.2387943267822266,
-      "eval_logps/chosen": -34.02626419067383,
-      "eval_logps/rejected": -37.50523376464844,
-      "eval_loss": 1.001994013786316,
-      "eval_rewards/accuracies": 0.4634551703929901,
-      "eval_rewards/chosen": 0.0049731116741895676,
-      "eval_rewards/margins": -0.0018590801628306508,
-      "eval_rewards/rejected": 0.006832191254943609,
-      "eval_runtime": 145.8336,
+      "eval_logits/chosen": -2.2438066005706787,
+      "eval_logits/rejected": -2.238919258117676,
+      "eval_logps/chosen": -34.02598190307617,
+      "eval_logps/rejected": -37.4908561706543,
+      "eval_loss": 1.0122511386871338,
+      "eval_rewards/accuracies": 0.45514950156211853,
+      "eval_rewards/chosen": 0.005999694112688303,
+      "eval_rewards/margins": -0.01203568559139967,
+      "eval_rewards/rejected": 0.01803538203239441,
+      "eval_runtime": 145.863,
       "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 300
@@ -493,121 +493,121 @@
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9287983179092407,
-      "logits/rejected": -1.9255239963531494,
-      "logps/chosen": -31.573007583618164,
-      "logps/rejected": -33.742637634277344,
-      "loss": 0.9815,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.018206708133220673,
-      "rewards/margins": 0.018496429547667503,
-      "rewards/rejected": -0.0002897246740758419,
+      "logits/chosen": -1.9287000894546509,
+      "logits/rejected": -1.9254179000854492,
+      "logps/chosen": -31.583938598632812,
+      "logps/rejected": -33.75525665283203,
+      "loss": 0.9772,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.013588106259703636,
+      "rewards/margins": 0.022759849205613136,
+      "rewards/rejected": -0.009171743877232075,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9805524349212646,
-      "logits/rejected": -1.9682422876358032,
-      "logps/chosen": -34.57670211791992,
-      "logps/rejected": -33.561363220214844,
-      "loss": 0.9749,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.011624744161963463,
-      "rewards/margins": 0.0251374039798975,
-      "rewards/rejected": -0.013512656092643738,
+      "logits/chosen": -1.9809232950210571,
+      "logits/rejected": -1.9685947895050049,
+      "logps/chosen": -34.56711196899414,
+      "logps/rejected": -33.54813003540039,
+      "loss": 0.9732,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.020275089889764786,
+      "rewards/margins": 0.026777852326631546,
+      "rewards/rejected": -0.006502761039882898,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.016091823577881,
-      "logits/rejected": -2.0146238803863525,
-      "logps/chosen": -33.47222137451172,
-      "logps/rejected": -32.46577835083008,
-      "loss": 1.0001,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.008143280632793903,
-      "rewards/margins": -0.0001393534184899181,
-      "rewards/rejected": 0.008282631635665894,
+      "logits/chosen": -2.0157439708709717,
+      "logits/rejected": -2.014296531677246,
+      "logps/chosen": -33.48149108886719,
+      "logps/rejected": -32.470550537109375,
+      "loss": 1.0033,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0030108082573860884,
+      "rewards/margins": -0.0033085450995713472,
+      "rewards/rejected": 0.00631935428828001,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.1032156944274902,
-      "logits/rejected": -2.0874197483062744,
-      "logps/chosen": -34.174861907958984,
-      "logps/rejected": -33.088890075683594,
-      "loss": 1.0148,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.002716648392379284,
-      "rewards/margins": -0.014771336689591408,
-      "rewards/rejected": 0.017487986013293266,
+      "logits/chosen": -2.1027872562408447,
+      "logits/rejected": -2.086984157562256,
+      "logps/chosen": -34.1781120300293,
+      "logps/rejected": -33.094017028808594,
+      "loss": 1.0159,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0008956976234912872,
+      "rewards/margins": -0.015917152166366577,
+      "rewards/rejected": 0.016812851652503014,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9747329950332642,
-      "logits/rejected": -1.9737837314605713,
-      "logps/chosen": -33.25817108154297,
-      "logps/rejected": -32.459861755371094,
-      "loss": 0.9888,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.014356844127178192,
-      "rewards/margins": 0.01118617132306099,
-      "rewards/rejected": 0.003170671407133341,
+      "logits/chosen": -1.9746129512786865,
+      "logits/rejected": -1.9736511707305908,
+      "logps/chosen": -33.25445556640625,
+      "logps/rejected": -32.4587287902832,
+      "loss": 0.9851,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.019348327070474625,
+      "rewards/margins": 0.014853959903120995,
+      "rewards/rejected": 0.004494364373385906,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9303538799285889,
-      "logits/rejected": -1.94071364402771,
-      "logps/chosen": -32.20769500732422,
-      "logps/rejected": -35.273780822753906,
-      "loss": 1.0108,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.0018019669223576784,
-      "rewards/margins": -0.010786842554807663,
-      "rewards/rejected": 0.008984875865280628,
+      "logits/chosen": -1.9303343296051025,
+      "logits/rejected": -1.940706491470337,
+      "logps/chosen": -32.189186096191406,
+      "logps/rejected": -35.288429260253906,
+      "loss": 0.9894,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.010855492204427719,
+      "rewards/margins": 0.010628147050738335,
+      "rewards/rejected": 0.00022734627418685704,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.069509744644165,
-      "logits/rejected": -2.0629658699035645,
-      "logps/chosen": -33.647117614746094,
-      "logps/rejected": -29.217060089111328,
-      "loss": 1.0027,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.002911838237196207,
-      "rewards/margins": -0.0027081891894340515,
-      "rewards/rejected": 0.005620025563985109,
+      "logits/chosen": -2.0694239139556885,
+      "logits/rejected": -2.0628561973571777,
+      "logps/chosen": -33.65647506713867,
+      "logps/rejected": -29.229717254638672,
+      "loss": 1.0008,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0031533471774309874,
+      "rewards/margins": -0.0008490110631100833,
+      "rewards/rejected": -0.0023043369874358177,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.929449439048767,
-      "logits/rejected": -1.9315989017486572,
-      "logps/chosen": -34.25099563598633,
-      "logps/rejected": -30.906116485595703,
-      "loss": 0.9871,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.007165629416704178,
-      "rewards/margins": 0.012927901931107044,
-      "rewards/rejected": -0.020093530416488647,
+      "logits/chosen": -1.9291013479232788,
+      "logits/rejected": -1.9312855005264282,
+      "logps/chosen": -34.26526641845703,
+      "logps/rejected": -30.913259506225586,
+      "loss": 0.9899,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.01835118792951107,
+      "rewards/margins": 0.010092146694660187,
+      "rewards/rejected": -0.028443332761526108,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.9932638019710393,
-      "train_runtime": 3254.1177,
+      "train_loss": 0.9909477320584383,
+      "train_runtime": 3254.3492,
       "train_samples_per_second": 0.946,
       "train_steps_per_second": 0.118
     }