Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6928
-- Rewards/chosen: 0.0020
-- Rewards/rejected: 0.0008
-- Rewards/accuracies: 0.5100
-- Rewards/margins: 0.0011
-- Logps/rejected: -37.5125
-- Logps/chosen: -34.0248
-- Logits/rejected: -2.2389
-- Logits/chosen: -2.2438
+- Loss: 0.6949
+- Rewards/chosen: -0.0012
+- Rewards/rejected: 0.0013
+- Rewards/accuracies: 0.5162
+- Rewards/margins: -0.0026
+- Logps/rejected: -37.5122
+- Logps/chosen: -34.0387
+- Logits/rejected: -2.2399
+- Logits/chosen: -2.2448
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6918        | 0.26  | 100  | 0.6936          | -0.0007        | -0.0002          | 0.5079             | -0.0006         | -37.5174       | -34.0382     | -2.2392         | -2.2441       |
-| 0.6908        | 0.52  | 200  | 0.6932          | 0.0034         | 0.0031           | 0.5112             | 0.0002          | -37.5009       | -34.0177     | -2.2392         | -2.2440       |
-| 0.6895        | 0.78  | 300  | 0.6937          | 0.0015         | 0.0022           | 0.5050             | -0.0007         | -37.5058       | -34.0273     | -2.2390         | -2.2438       |
+| 0.6886        | 0.26  | 100  | 0.6950          | 0.0021         | 0.0049           | 0.4817             | -0.0028         | -37.5001       | -34.0275     | -2.2393         | -2.2442       |
+| 0.6909        | 0.52  | 200  | 0.6952          | 0.0005         | 0.0038           | 0.5191             | -0.0033         | -37.5039       | -34.0328     | -2.2391         | -2.2439       |
+| 0.6869        | 0.78  | 300  | 0.6949          | -0.0012        | 0.0013           | 0.5162             | -0.0026         | -37.5122       | -34.0387     | -2.2399         | -2.2448       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a430fc5bf78793bfed9f2f01cf0c273a5b0f4e1551593252cae780aeb18a1984
+oid sha256:a1c41272bfc20de1787ca81e4ccbee9baeb8b772de90ef90ec9ab66107b6cf9f
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.243804693222046,
-    "eval_logits/rejected": -2.238922119140625,
-    "eval_logps/chosen": -34.02475357055664,
-    "eval_logps/rejected": -37.51246643066406,
-    "eval_loss": 0.6927995681762695,
-    "eval_rewards/accuracies": 0.5099667906761169,
-    "eval_rewards/chosen": 0.0019583876710385084,
-    "eval_rewards/margins": 0.0011271554976701736,
-    "eval_rewards/rejected": 0.0008312325226143003,
-    "eval_runtime": 145.6648,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.355,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.6919509800997647,
-    "train_runtime": 3253.2141,
+    "train_loss": 0.691356255791404,
+    "train_runtime": 3254.7157,
     "train_samples": 3079,
     "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6919509800997647,
-    "train_runtime": 3253.2141,
+    "train_loss": 0.691356255791404,
+    "train_runtime": 3254.7157,
     "train_samples": 3079,
     "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -25,589 +25,589 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.8659842014312744,
-      "logits/rejected": -1.8702926635742188,
-      "logps/chosen": -36.97752380371094,
-      "logps/rejected": -33.65974426269531,
-      "loss": 0.6875,
-      "rewards/accuracies": 0.5277777910232544,
-      "rewards/chosen": 0.005815046839416027,
-      "rewards/margins": 0.011749334633350372,
-      "rewards/rejected": -0.005934285931289196,
+      "logits/chosen": -1.866300106048584,
+      "logits/rejected": -1.870603084564209,
+      "logps/chosen": -36.973060607910156,
+      "logps/rejected": -33.65780258178711,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.5138888955116272,
+      "rewards/chosen": 0.010061310604214668,
+      "rewards/margins": 0.018380172550678253,
+      "rewards/rejected": -0.008318860083818436,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.997287392616272,
-      "logits/rejected": -1.9999300241470337,
-      "logps/chosen": -29.62233543395996,
-      "logps/rejected": -29.058269500732422,
-      "loss": 0.6918,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.003971663769334555,
-      "rewards/margins": 0.002936883131042123,
-      "rewards/rejected": 0.0010347809875383973,
+      "logits/chosen": -1.9976227283477783,
+      "logits/rejected": -2.0002670288085938,
+      "logps/chosen": -29.652048110961914,
+      "logps/rejected": -29.051151275634766,
+      "loss": 0.6969,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.0029565312433987856,
+      "rewards/margins": -0.006644421722739935,
+      "rewards/rejected": 0.003687891410663724,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.9199504852294922,
-      "logits/rejected": -1.917252779006958,
-      "logps/chosen": -31.414203643798828,
-      "logps/rejected": -33.21183395385742,
-      "loss": 0.6938,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0003786637098528445,
-      "rewards/margins": -0.000946315354667604,
-      "rewards/rejected": 0.001324978657066822,
+      "logits/chosen": -1.9202871322631836,
+      "logits/rejected": -1.9175989627838135,
+      "logps/chosen": -31.403018951416016,
+      "logps/rejected": -33.22191619873047,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0039227609522640705,
+      "rewards/margins": 0.004958181641995907,
+      "rewards/rejected": -0.0010354205733165145,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.0170738697052,
-      "logits/rejected": -2.008314847946167,
-      "logps/chosen": -32.582008361816406,
-      "logps/rejected": -32.49999237060547,
-      "loss": 0.6951,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.0010401441249996424,
-      "rewards/margins": -0.003575131995603442,
-      "rewards/rejected": 0.0025349881034344435,
+      "logits/chosen": -2.016842842102051,
+      "logits/rejected": -2.008082151412964,
+      "logps/chosen": -32.58789825439453,
+      "logps/rejected": -32.494510650634766,
+      "loss": 0.6978,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.003327915444970131,
+      "rewards/margins": -0.008775560185313225,
+      "rewards/rejected": 0.005447645206004381,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8644828796386719,
-      "logits/rejected": -1.8536993265151978,
-      "logps/chosen": -33.572994232177734,
-      "logps/rejected": -35.421836853027344,
-      "loss": 0.6979,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.0031035186257213354,
-      "rewards/margins": -0.009205068461596966,
-      "rewards/rejected": 0.006101551000028849,
+      "logits/chosen": -1.8639650344848633,
+      "logits/rejected": -1.8531978130340576,
+      "logps/chosen": -33.55598831176758,
+      "logps/rejected": -35.437034606933594,
+      "loss": 0.6956,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.0004471995052881539,
+      "rewards/margins": -0.004145150538533926,
+      "rewards/rejected": 0.00459235068410635,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9452167749404907,
-      "logits/rejected": -1.9471642971038818,
-      "logps/chosen": -32.57027816772461,
-      "logps/rejected": -33.18465805053711,
-      "loss": 0.689,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.0059812054969370365,
-      "rewards/margins": 0.008636695332825184,
-      "rewards/rejected": -0.0026554896030575037,
+      "logits/chosen": -1.9451839923858643,
+      "logits/rejected": -1.94712233543396,
+      "logps/chosen": -32.599273681640625,
+      "logps/rejected": -33.205684661865234,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.000273245939752087,
+      "rewards/margins": 0.010566400364041328,
+      "rewards/rejected": -0.010293153114616871,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.0795772075653076,
-      "logits/rejected": -2.0845541954040527,
-      "logps/chosen": -33.98872375488281,
-      "logps/rejected": -36.58330535888672,
-      "loss": 0.692,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -4.708058986579999e-05,
-      "rewards/margins": 0.002839430468156934,
-      "rewards/rejected": -0.002886510454118252,
+      "logits/chosen": -2.0796267986297607,
+      "logits/rejected": -2.0845961570739746,
+      "logps/chosen": -33.98247528076172,
+      "logps/rejected": -36.595970153808594,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0018027916084975004,
+      "rewards/margins": 0.009932766668498516,
+      "rewards/rejected": -0.008129975758492947,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.942291498184204,
-      "logits/rejected": -1.945452094078064,
-      "logps/chosen": -34.40483474731445,
-      "logps/rejected": -34.58934020996094,
-      "loss": 0.6914,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.004595070146024227,
-      "rewards/margins": 0.003981433808803558,
-      "rewards/rejected": 0.0006136370939202607,
+      "logits/chosen": -1.9422763586044312,
+      "logits/rejected": -1.9454386234283447,
+      "logps/chosen": -34.4161491394043,
+      "logps/rejected": -34.59540939331055,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0034980489872395992,
+      "rewards/margins": 0.004399437457323074,
+      "rewards/rejected": -0.0009013883536681533,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.950508713722229,
-      "logits/rejected": -1.9550243616104126,
-      "logps/chosen": -32.44390106201172,
-      "logps/rejected": -32.375,
-      "loss": 0.6893,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.004703683778643608,
-      "rewards/margins": 0.007921957410871983,
-      "rewards/rejected": -0.003218273166567087,
+      "logits/chosen": -1.950871467590332,
+      "logits/rejected": -1.9553781747817993,
+      "logps/chosen": -32.455718994140625,
+      "logps/rejected": -32.37923812866211,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.003510843263939023,
+      "rewards/margins": 0.009610405191779137,
+      "rewards/rejected": -0.00609956169500947,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.048755645751953,
-      "logits/rejected": -2.046764850616455,
-      "logps/chosen": -32.22840118408203,
-      "logps/rejected": -31.269317626953125,
-      "loss": 0.6918,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0024873518850654364,
-      "rewards/margins": 0.0031086006201803684,
-      "rewards/rejected": -0.0006212486186996102,
+      "logits/chosen": -2.0493884086608887,
+      "logits/rejected": -2.0473878383636475,
+      "logps/chosen": -32.24263381958008,
+      "logps/rejected": -31.301403045654297,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0005389736033976078,
+      "rewards/margins": 0.010018542408943176,
+      "rewards/rejected": -0.010557514615356922,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2440829277038574,
-      "eval_logits/rejected": -2.2392072677612305,
-      "eval_logps/chosen": -34.038169860839844,
-      "eval_logps/rejected": -37.51740264892578,
-      "eval_loss": 0.6936388611793518,
-      "eval_rewards/accuracies": 0.5078904032707214,
-      "eval_rewards/chosen": -0.0007231775089167058,
-      "eval_rewards/margins": -0.0005666659562848508,
-      "eval_rewards/rejected": -0.00015651114517822862,
-      "eval_runtime": 146.0083,
+      "eval_logits/chosen": -2.244236707687378,
+      "eval_logits/rejected": -2.2393476963043213,
+      "eval_logps/chosen": -34.027488708496094,
+      "eval_logps/rejected": -37.500125885009766,
+      "eval_loss": 0.6949825882911682,
+      "eval_rewards/accuracies": 0.48172760009765625,
+      "eval_rewards/chosen": 0.0021188759710639715,
+      "eval_rewards/margins": -0.0028304157312959433,
+      "eval_rewards/rejected": 0.0049492912366986275,
+      "eval_runtime": 146.0369,
       "eval_samples_per_second": 2.349,
-      "eval_steps_per_second": 0.295,
+      "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.0054333209991455,
-      "logits/rejected": -2.0030055046081543,
-      "logps/chosen": -33.228519439697266,
-      "logps/rejected": -34.01723098754883,
-      "loss": 0.693,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.003059562761336565,
-      "rewards/margins": 0.0006311066681519151,
-      "rewards/rejected": 0.002428455278277397,
+      "logits/chosen": -2.005858898162842,
+      "logits/rejected": -2.0034613609313965,
+      "logps/chosen": -33.22675323486328,
+      "logps/rejected": -34.024375915527344,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.0051184119656682014,
+      "rewards/margins": 0.0036206122022122145,
+      "rewards/rejected": 0.0014977991813793778,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.0166068077087402,
-      "logits/rejected": -2.0082287788391113,
-      "logps/chosen": -32.45368194580078,
-      "logps/rejected": -32.192543029785156,
-      "loss": 0.6911,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.0017549397889524698,
-      "rewards/margins": 0.004452199675142765,
-      "rewards/rejected": -0.006207138299942017,
+      "logits/chosen": -2.0165371894836426,
+      "logits/rejected": -2.0081915855407715,
+      "logps/chosen": -32.43837356567383,
+      "logps/rejected": -32.1854362487793,
+      "loss": 0.689,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0019600242376327515,
+      "rewards/margins": 0.009139470756053925,
+      "rewards/rejected": -0.0071794455870985985,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.0461902618408203,
-      "logits/rejected": -2.038154125213623,
-      "logps/chosen": -30.499649047851562,
-      "logps/rejected": -32.04579544067383,
-      "loss": 0.6965,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.004786232020705938,
-      "rewards/margins": -0.00622799526900053,
-      "rewards/rejected": 0.0014417637139558792,
+      "logits/chosen": -2.046520709991455,
+      "logits/rejected": -2.0384721755981445,
+      "logps/chosen": -30.485082626342773,
+      "logps/rejected": -32.0345344543457,
+      "loss": 0.6976,
+      "rewards/accuracies": 0.38749998807907104,
+      "rewards/chosen": -0.002810357604175806,
+      "rewards/margins": -0.008352747187018394,
+      "rewards/rejected": 0.005542389117181301,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.976766586303711,
-      "logits/rejected": -1.9870338439941406,
-      "logps/chosen": -31.403955459594727,
-      "logps/rejected": -32.556556701660156,
-      "loss": 0.6896,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.0031408462673425674,
-      "rewards/margins": 0.007476066239178181,
-      "rewards/rejected": -0.004335219506174326,
+      "logits/chosen": -1.976843237876892,
+      "logits/rejected": -1.9871089458465576,
+      "logps/chosen": -31.4160099029541,
+      "logps/rejected": -32.535850524902344,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0010961029911413789,
+      "rewards/margins": 0.0013871133560314775,
+      "rewards/rejected": -0.0002910103357862681,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8904304504394531,
-      "logits/rejected": -1.8915287256240845,
-      "logps/chosen": -34.17884063720703,
-      "logps/rejected": -34.76021957397461,
-      "loss": 0.6917,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.0027701070066541433,
-      "rewards/margins": 0.0033626866061240435,
-      "rewards/rejected": -0.0005925801815465093,
+      "logits/chosen": -1.890434980392456,
+      "logits/rejected": -1.8915313482284546,
+      "logps/chosen": -34.165489196777344,
+      "logps/rejected": -34.74839782714844,
+      "loss": 0.691,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.008160250261425972,
+      "rewards/margins": 0.005502746906131506,
+      "rewards/rejected": 0.0026575028896331787,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.942710518836975,
-      "logits/rejected": -1.939239263534546,
-      "logps/chosen": -36.136741638183594,
-      "logps/rejected": -32.73341751098633,
-      "loss": 0.6888,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.006986373569816351,
-      "rewards/margins": 0.009119498543441296,
-      "rewards/rejected": -0.0021331266034394503,
+      "logits/chosen": -1.942751169204712,
+      "logits/rejected": -1.9392738342285156,
+      "logps/chosen": -36.15436553955078,
+      "logps/rejected": -32.734371185302734,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.005194104742258787,
+      "rewards/margins": 0.008680562488734722,
+      "rewards/rejected": -0.0034864575136452913,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.0422637462615967,
-      "logits/rejected": -2.0348782539367676,
-      "logps/chosen": -33.789520263671875,
-      "logps/rejected": -31.35744285583496,
-      "loss": 0.6919,
+      "logits/chosen": -2.042160749435425,
+      "logits/rejected": -2.0347483158111572,
+      "logps/chosen": -33.785484313964844,
+      "logps/rejected": -31.344274520874023,
+      "loss": 0.6928,
       "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.001668277895078063,
-      "rewards/margins": 0.002786512253805995,
-      "rewards/rejected": -0.001118234358727932,
+      "rewards/chosen": 0.003714872058480978,
+      "rewards/margins": 0.0014427527785301208,
+      "rewards/rejected": 0.002272119279950857,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0476248264312744,
-      "logits/rejected": -2.052889347076416,
-      "logps/chosen": -32.531455993652344,
-      "logps/rejected": -32.49399185180664,
-      "loss": 0.6911,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0026666896883398294,
-      "rewards/margins": 0.0044365981593728065,
-      "rewards/rejected": -0.0017699094023555517,
+      "logits/chosen": -2.0473780632019043,
+      "logits/rejected": -2.0526490211486816,
+      "logps/chosen": -32.52016830444336,
+      "logps/rejected": -32.495323181152344,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.007385374512523413,
+      "rewards/margins": 0.010440578684210777,
+      "rewards/rejected": -0.0030552041716873646,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.048638105392456,
-      "logits/rejected": -2.0458552837371826,
-      "logps/chosen": -31.49679946899414,
-      "logps/rejected": -31.326080322265625,
-      "loss": 0.6933,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.0015687488485127687,
-      "rewards/margins": 4.425132829055656e-06,
-      "rewards/rejected": -0.0015731739113107324,
+      "logits/chosen": -2.0484580993652344,
+      "logits/rejected": -2.045682191848755,
+      "logps/chosen": -31.508682250976562,
+      "logps/rejected": -31.3348388671875,
+      "loss": 0.694,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.005918826907873154,
+      "rewards/margins": -0.0009324293350800872,
+      "rewards/rejected": -0.004986398387700319,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9184449911117554,
-      "logits/rejected": -1.9231094121932983,
-      "logps/chosen": -31.588008880615234,
-      "logps/rejected": -32.797332763671875,
-      "loss": 0.6908,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.0036545295733958483,
-      "rewards/margins": 0.0049341581761837006,
-      "rewards/rejected": -0.001279628137126565,
+      "logits/chosen": -1.9187612533569336,
+      "logits/rejected": -1.9234310388565063,
+      "logps/chosen": -31.59661865234375,
+      "logps/rejected": -32.798248291015625,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.002899137092754245,
+      "rewards/margins": 0.00509423715993762,
+      "rewards/rejected": -0.002195100300014019,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2440476417541504,
-      "eval_logits/rejected": -2.239173650741577,
-      "eval_logps/chosen": -34.017669677734375,
-      "eval_logps/rejected": -37.50092697143555,
-      "eval_loss": 0.6932498216629028,
-      "eval_rewards/accuracies": 0.5112126469612122,
-      "eval_rewards/chosen": 0.003376541193574667,
-      "eval_rewards/margins": 0.00023789344413671643,
-      "eval_rewards/rejected": 0.0031386471819132566,
-      "eval_runtime": 145.8713,
-      "eval_samples_per_second": 2.351,
+      "eval_logits/chosen": -2.2439279556274414,
+      "eval_logits/rejected": -2.239055633544922,
+      "eval_logps/chosen": -34.03278350830078,
+      "eval_logps/rejected": -37.50389099121094,
+      "eval_loss": 0.6952112317085266,
+      "eval_rewards/accuracies": 0.5191029906272888,
+      "eval_rewards/chosen": 0.0005306191742420197,
+      "eval_rewards/margins": -0.0032888990826904774,
+      "eval_rewards/rejected": 0.0038195180241018534,
+      "eval_runtime": 145.8142,
+      "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0318806171417236,
-      "logits/rejected": -2.0425620079040527,
-      "logps/chosen": -31.950469970703125,
-      "logps/rejected": -33.88362503051758,
-      "loss": 0.6892,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.004167118109762669,
-      "rewards/margins": 0.008351770229637623,
-      "rewards/rejected": -0.00418465118855238,
+      "logits/chosen": -2.0318386554718018,
+      "logits/rejected": -2.0425145626068115,
+      "logps/chosen": -31.961750030517578,
+      "logps/rejected": -33.86240005493164,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0028654958587139845,
+      "rewards/margins": 0.002774373395368457,
+      "rewards/rejected": 9.112273983191699e-05,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9253804683685303,
-      "logits/rejected": -1.9402259588241577,
-      "logps/chosen": -30.09176254272461,
-      "logps/rejected": -31.568744659423828,
-      "loss": 0.6903,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.003955656662583351,
-      "rewards/margins": 0.006068618968129158,
-      "rewards/rejected": -0.002112963469699025,
+      "logits/chosen": -1.9253981113433838,
+      "logits/rejected": -1.9402506351470947,
+      "logps/chosen": -30.093517303466797,
+      "logps/rejected": -31.571548461914062,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.005405673757195473,
+      "rewards/margins": 0.009417025372385979,
+      "rewards/rejected": -0.004011353012174368,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.983178734779358,
-      "logits/rejected": -1.987159013748169,
-      "logps/chosen": -33.3923454284668,
-      "logps/rejected": -31.549015045166016,
-      "loss": 0.6909,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.00502450205385685,
-      "rewards/margins": 0.004966902546584606,
-      "rewards/rejected": 5.75995072722435e-05,
+      "logits/chosen": -1.9839330911636353,
+      "logits/rejected": -1.9879001379013062,
+      "logps/chosen": -33.4061279296875,
+      "logps/rejected": -31.564762115478516,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0034024070482701063,
+      "rewards/margins": 0.008040683344006538,
+      "rewards/rejected": -0.004638276062905788,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.982914686203003,
-      "logits/rejected": -1.960942268371582,
-      "logps/chosen": -34.15560531616211,
-      "logps/rejected": -34.942718505859375,
-      "loss": 0.6943,
+      "logits/chosen": -1.983189344406128,
+      "logits/rejected": -1.9612159729003906,
+      "logps/chosen": -34.15366744995117,
+      "logps/rejected": -34.95070266723633,
+      "loss": 0.6937,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.0003707931318785995,
-      "rewards/margins": -0.001986075658351183,
-      "rewards/rejected": 0.0016152828466147184,
+      "rewards/chosen": 2.4808850866975263e-05,
+      "rewards/margins": -1.6275793086606427e-06,
+      "rewards/rejected": 2.643577681737952e-05,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.024036407470703,
-      "logits/rejected": -2.0207295417785645,
-      "logps/chosen": -32.916961669921875,
-      "logps/rejected": -36.2316780090332,
-      "loss": 0.6919,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.0016211355105042458,
-      "rewards/margins": 0.00297213951125741,
-      "rewards/rejected": -0.0013510034186765552,
+      "logits/chosen": -2.0243947505950928,
+      "logits/rejected": -2.021087169647217,
+      "logps/chosen": -32.91602325439453,
+      "logps/rejected": -36.221763610839844,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.002715445589274168,
+      "rewards/margins": 0.001767707639373839,
+      "rewards/rejected": 0.0009477367857471108,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.891046166419983,
-      "logits/rejected": -1.888594627380371,
-      "logps/chosen": -34.18373489379883,
-      "logps/rejected": -35.52981948852539,
-      "loss": 0.6904,
+      "logits/chosen": -1.8911654949188232,
+      "logits/rejected": -1.8887317180633545,
+      "logps/chosen": -34.18123245239258,
+      "logps/rejected": -35.515174865722656,
+      "loss": 0.691,
       "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.0009327814914286137,
-      "rewards/margins": 0.005972124636173248,
-      "rewards/rejected": -0.005039343610405922,
+      "rewards/chosen": 0.0021498873829841614,
+      "rewards/margins": 0.005315869115293026,
+      "rewards/rejected": -0.0031659831292927265,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.876119613647461,
-      "logits/rejected": -1.8735746145248413,
-      "logps/chosen": -34.3980827331543,
-      "logps/rejected": -31.744619369506836,
-      "loss": 0.696,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -9.752502955961972e-05,
-      "rewards/margins": -0.0054392144083976746,
-      "rewards/rejected": 0.0053416891023516655,
+      "logits/chosen": -1.875580072402954,
+      "logits/rejected": -1.8730493783950806,
+      "logps/chosen": -34.37896728515625,
+      "logps/rejected": -31.76068687438965,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.005589136388152838,
+      "rewards/margins": 0.0023957251105457544,
+      "rewards/rejected": 0.0031934112776070833,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9795544147491455,
-      "logits/rejected": -1.9689334630966187,
-      "logps/chosen": -35.3333854675293,
-      "logps/rejected": -31.85133934020996,
-      "loss": 0.69,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.004650552291423082,
-      "rewards/margins": 0.006719636730849743,
-      "rewards/rejected": -0.002069085370749235,
+      "logits/chosen": -1.9798663854599,
+      "logits/rejected": -1.9692500829696655,
+      "logps/chosen": -35.32219696044922,
+      "logps/rejected": -31.857141494750977,
+      "loss": 0.6861,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.010332418605685234,
+      "rewards/margins": 0.015176350250840187,
+      "rewards/rejected": -0.004843929782509804,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.075632333755493,
-      "logits/rejected": -2.0606019496917725,
-      "logps/chosen": -30.9345645904541,
-      "logps/rejected": -32.642818450927734,
-      "loss": 0.6952,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.0020882338285446167,
-      "rewards/margins": -0.0038910664152354,
-      "rewards/rejected": 0.0018028330523520708,
+      "logits/chosen": -2.076137065887451,
+      "logits/rejected": -2.061100721359253,
+      "logps/chosen": -30.907434463500977,
+      "logps/rejected": -32.638729095458984,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.00500696524977684,
+      "rewards/margins": 0.0010755129624158144,
+      "rewards/rejected": 0.0039314525201916695,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.9465770721435547,
-      "logits/rejected": -1.9440364837646484,
-      "logps/chosen": -32.8905143737793,
-      "logps/rejected": -30.818241119384766,
-      "loss": 0.6895,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.007319308817386627,
-      "rewards/margins": 0.007605412509292364,
-      "rewards/rejected": -0.00028610360459424555,
+      "logits/chosen": -1.947422981262207,
+      "logits/rejected": -1.9448877573013306,
+      "logps/chosen": -32.89393997192383,
+      "logps/rejected": -30.828500747680664,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.009951007552444935,
+      "rewards/margins": 0.013458291999995708,
+      "rewards/rejected": -0.003507285611703992,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2438313961029053,
-      "eval_logits/rejected": -2.2389516830444336,
-      "eval_logps/chosen": -34.027259826660156,
-      "eval_logps/rejected": -37.505775451660156,
-      "eval_loss": 0.6937180161476135,
-      "eval_rewards/accuracies": 0.5049834251403809,
-      "eval_rewards/chosen": 0.0014584256568923593,
-      "eval_rewards/margins": -0.0007105680997483432,
-      "eval_rewards/rejected": 0.0021689936984330416,
-      "eval_runtime": 145.7574,
-      "eval_samples_per_second": 2.353,
+      "eval_logits/chosen": -2.244764566421509,
+      "eval_logits/rejected": -2.2398788928985596,
+      "eval_logps/chosen": -34.038692474365234,
+      "eval_logps/rejected": -37.512168884277344,
+      "eval_loss": 0.6949278712272644,
+      "eval_rewards/accuracies": 0.5161960124969482,
+      "eval_rewards/chosen": -0.0012418677797541022,
+      "eval_rewards/margins": -0.0025783225428313017,
+      "eval_rewards/rejected": 0.0013364545302465558,
+      "eval_runtime": 145.8034,
+      "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9287267923355103,
-      "logits/rejected": -1.9254738092422485,
-      "logps/chosen": -31.579418182373047,
-      "logps/rejected": -33.74348068237305,
-      "loss": 0.6908,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.0047865137457847595,
-      "rewards/margins": 0.005052282474935055,
-      "rewards/rejected": -0.0002657676232047379,
+      "logits/chosen": -1.9299108982086182,
+      "logits/rejected": -1.9266481399536133,
+      "logps/chosen": -31.572988510131836,
+      "logps/rejected": -33.727210998535156,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.009108476340770721,
+      "rewards/margins": 0.004626707639545202,
+      "rewards/rejected": 0.0044817691668868065,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9805068969726562,
-      "logits/rejected": -1.9681942462921143,
-      "logps/chosen": -34.57032012939453,
-      "logps/rejected": -33.55123519897461,
-      "loss": 0.6895,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.005150976125150919,
-      "rewards/margins": 0.007629568222910166,
-      "rewards/rejected": -0.0024785916320979595,
+      "logits/chosen": -1.9814649820327759,
+      "logits/rejected": -1.9691375494003296,
+      "logps/chosen": -34.551979064941406,
+      "logps/rejected": -33.541648864746094,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.013229632750153542,
+      "rewards/margins": 0.014072793535888195,
+      "rewards/rejected": -0.0008431615424342453,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.0162813663482666,
-      "logits/rejected": -2.0148308277130127,
-      "logps/chosen": -33.48984909057617,
-      "logps/rejected": -32.48566818237305,
-      "loss": 0.6931,
+      "logits/chosen": -2.017097234725952,
+      "logits/rejected": -2.0156359672546387,
+      "logps/chosen": -33.49595260620117,
+      "logps/rejected": -32.49913787841797,
+      "loss": 0.6922,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.0008109404589049518,
-      "rewards/margins": 0.0004063365049660206,
-      "rewards/rejected": -0.001217277254909277,
+      "rewards/chosen": -0.003048144979402423,
+      "rewards/margins": 0.002819380722939968,
+      "rewards/rejected": -0.00586752500385046,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.102883815765381,
-      "logits/rejected": -2.087068796157837,
-      "logps/chosen": -34.1587028503418,
-      "logps/rejected": -33.081783294677734,
-      "loss": 0.6949,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.0041370633989572525,
-      "rewards/margins": -0.003113438840955496,
-      "rewards/rejected": 0.007250501308590174,
+      "logits/chosen": -2.103210926055908,
+      "logits/rejected": -2.0874040126800537,
+      "logps/chosen": -34.16447830200195,
+      "logps/rejected": -33.07390594482422,
+      "loss": 0.6981,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.004473570734262466,
+      "rewards/margins": -0.008766504004597664,
+      "rewards/rejected": 0.01324007473886013,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9753484725952148,
-      "logits/rejected": -1.9743890762329102,
-      "logps/chosen": -33.23331832885742,
-      "logps/rejected": -32.45554733276367,
-      "loss": 0.6895,
+      "logits/chosen": -1.9749380350112915,
+      "logits/rejected": -1.9740018844604492,
+      "logps/chosen": -33.25703048706055,
+      "logps/rejected": -32.47133255004883,
+      "loss": 0.689,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.009755617938935757,
-      "rewards/margins": 0.007835443131625652,
-      "rewards/rejected": 0.0019201741088181734,
+      "rewards/chosen": 0.00751916179433465,
+      "rewards/margins": 0.009374773129820824,
+      "rewards/rejected": -0.0018556114519014955,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9319757223129272,
-      "logits/rejected": -1.9423389434814453,
-      "logps/chosen": -32.21473693847656,
-      "logps/rejected": -35.285762786865234,
-      "loss": 0.6947,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.0020089317113161087,
-      "rewards/margins": -0.0026077935472130775,
-      "rewards/rejected": 0.0005988621269352734,
+      "logits/chosen": -1.9304084777832031,
+      "logits/rejected": -1.9407765865325928,
+      "logps/chosen": -32.21118927001953,
+      "logps/rejected": -35.31554412841797,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0019482700154185295,
+      "rewards/margins": 0.0060885134153068066,
+      "rewards/rejected": -0.008036783896386623,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.070089340209961,
-      "logits/rejected": -2.0635385513305664,
-      "logps/chosen": -33.64582061767578,
-      "logps/rejected": -29.22507095336914,
-      "loss": 0.6929,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.0012301013339310884,
-      "rewards/margins": 0.0009590888512320817,
-      "rewards/rejected": 0.00027101286104880273,
+      "logits/chosen": -2.06982421875,
+      "logits/rejected": -2.0632567405700684,
+      "logps/chosen": -33.641204833984375,
+      "logps/rejected": -29.2253475189209,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.003229249268770218,
+      "rewards/margins": 0.0029065334238111973,
+      "rewards/rejected": 0.000322715932270512,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.929181456565857,
-      "logits/rejected": -1.9313499927520752,
-      "logps/chosen": -34.237857818603516,
-      "logps/rejected": -30.90460777282715,
-      "loss": 0.69,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.00023896321363281459,
-      "rewards/margins": 0.006635263562202454,
-      "rewards/rejected": -0.006396301090717316,
+      "logits/chosen": -1.929220199584961,
+      "logits/rejected": -1.9313905239105225,
+      "logps/chosen": -34.26762008666992,
+      "logps/rejected": -30.9031982421875,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.00857237633317709,
+      "rewards/margins": 0.0005976069951429963,
+      "rewards/rejected": -0.009169982746243477,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.6919509800997647,
-      "train_runtime": 3253.2141,
+      "train_loss": 0.691356255791404,
+      "train_runtime": 3254.7157,
       "train_samples_per_second": 0.946,
       "train_steps_per_second": 0.118
     }