Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6933
-- Rewards/chosen: 0.0028
-- Rewards/rejected: 0.0023
-- Rewards/accuracies: 0.5133
-- Rewards/margins: 0.0006
-- Logps/rejected: -37.5090
-- Logps/chosen: -34.0251
-- Logits/rejected: -2.2391
-- Logits/chosen: -2.2439
+- Loss: 0.6949
+- Rewards/chosen: 0.0029
+- Rewards/rejected: 0.0046
+- Rewards/accuracies: 0.5116
+- Rewards/margins: -0.0017
+- Logps/rejected: -37.5051
+- Logps/chosen: -34.0272
+- Logits/rejected: -2.2395
+- Logits/chosen: -2.2444
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6886        | 0.26  | 100  | 0.6950          | 0.0021         | 0.0049           | 0.4817             | -0.0028         | -37.5001       | -34.0275     | -2.2393         | -2.2442       |
-| 0.6909        | 0.52  | 200  | 0.6952          | 0.0005         | 0.0038           | 0.5191             | -0.0033         | -37.5039       | -34.0328     | -2.2391         | -2.2439       |
-| 0.6869        | 0.78  | 300  | 0.6949          | -0.0012        | 0.0013           | 0.5162             | -0.0026         | -37.5122       | -34.0387     | -2.2399         | -2.2448       |
+| 0.6904        | 0.26  | 100  | 0.6919          | 0.0034         | -0.0006          | 0.5428             | 0.0040          | -37.5181       | -34.0260     | -2.2385         | -2.2434       |
+| 0.6912        | 0.52  | 200  | 0.6924          | 0.0082         | 0.0051           | 0.4992             | 0.0031          | -37.5039       | -34.0141     | -2.2385         | -2.2434       |
+| 0.6888        | 0.78  | 300  | 0.6949          | 0.0029         | 0.0046           | 0.5116             | -0.0017         | -37.5051       | -34.0272     | -2.2395         | -2.2444       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:57cbdcb5b30275063557fbf7103b41d9f2311e3f6241c10fe1cfb8f5d243ee99
+oid sha256:7fe7bc89af8e89a3d125ab386a11cc9fa9f8a74741897371683394c3b1c07bc8
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2439303398132324,
-    "eval_logits/rejected": -2.2390527725219727,
-    "eval_logps/chosen": -34.02507400512695,
-    "eval_logps/rejected": -37.508995056152344,
-    "eval_loss": 0.6933426260948181,
-    "eval_rewards/accuracies": 0.5132890343666077,
-    "eval_rewards/chosen": 0.0028422565665096045,
-    "eval_rewards/margins": 0.0005549592897295952,
-    "eval_rewards/rejected": 0.002287296811118722,
-    "eval_runtime": 145.8223,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.352,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.691356255791404,
-    "train_runtime": 3254.7157,
+    "train_loss": 0.6908149576806403,
+    "train_runtime": 3255.681,
     "train_samples": 3079,
     "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.691356255791404,
-    "train_runtime": 3254.7157,
+    "train_loss": 0.6908149576806403,
+    "train_runtime": 3255.681,
     "train_samples": 3079,
     "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -25,589 +25,589 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.866300106048584,
-      "logits/rejected": -1.870603084564209,
-      "logps/chosen": -36.973060607910156,
-      "logps/rejected": -33.65780258178711,
-      "loss": 0.6843,
-      "rewards/accuracies": 0.5138888955116272,
-      "rewards/chosen": 0.010061310604214668,
-      "rewards/margins": 0.018380172550678253,
-      "rewards/rejected": -0.008318860083818436,
+      "logits/chosen": -1.8663291931152344,
+      "logits/rejected": -1.870633602142334,
+      "logps/chosen": -37.00596237182617,
+      "logps/rejected": -33.65876007080078,
+      "loss": 0.688,
+      "rewards/accuracies": 0.4722222089767456,
+      "rewards/chosen": 0.0002556463878136128,
+      "rewards/margins": 0.011729677207767963,
+      "rewards/rejected": -0.011474031955003738,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.9976227283477783,
-      "logits/rejected": -2.0002670288085938,
-      "logps/chosen": -29.652048110961914,
-      "logps/rejected": -29.051151275634766,
-      "loss": 0.6969,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -0.0029565312433987856,
-      "rewards/margins": -0.006644421722739935,
-      "rewards/rejected": 0.003687891410663724,
+      "logits/chosen": -1.997443437576294,
+      "logits/rejected": -2.000075340270996,
+      "logps/chosen": -29.63150405883789,
+      "logps/rejected": -29.064098358154297,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.004276981111615896,
+      "rewards/margins": 0.004539952147752047,
+      "rewards/rejected": -0.0002629714144859463,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.9202871322631836,
-      "logits/rejected": -1.9175989627838135,
-      "logps/chosen": -31.403018951416016,
-      "logps/rejected": -33.22191619873047,
-      "loss": 0.6911,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0039227609522640705,
-      "rewards/margins": 0.004958181641995907,
-      "rewards/rejected": -0.0010354205733165145,
+      "logits/chosen": -1.920101523399353,
+      "logits/rejected": -1.9174143075942993,
+      "logps/chosen": -31.427581787109375,
+      "logps/rejected": -33.214088439941406,
+      "loss": 0.6971,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.004594600293785334,
+      "rewards/margins": -0.006341943051666021,
+      "rewards/rejected": 0.0017473434563726187,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.016842842102051,
-      "logits/rejected": -2.008082151412964,
-      "logps/chosen": -32.58789825439453,
-      "logps/rejected": -32.494510650634766,
-      "loss": 0.6978,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.003327915444970131,
-      "rewards/margins": -0.008775560185313225,
-      "rewards/rejected": 0.005447645206004381,
+      "logits/chosen": -2.0168609619140625,
+      "logits/rejected": -2.008105754852295,
+      "logps/chosen": -32.59783935546875,
+      "logps/rejected": -32.4984130859375,
+      "loss": 0.701,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.008411901071667671,
+      "rewards/margins": -0.014114728197455406,
+      "rewards/rejected": 0.00570282619446516,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8639650344848633,
-      "logits/rejected": -1.8531978130340576,
-      "logps/chosen": -33.55598831176758,
-      "logps/rejected": -35.437034606933594,
-      "loss": 0.6956,
+      "logits/chosen": -1.864262342453003,
+      "logits/rejected": -1.8534845113754272,
+      "logps/chosen": -33.57244873046875,
+      "logps/rejected": -35.4432373046875,
+      "loss": 0.6987,
       "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.0004471995052881539,
-      "rewards/margins": -0.004145150538533926,
-      "rewards/rejected": 0.00459235068410635,
+      "rewards/chosen": -0.005986724980175495,
+      "rewards/margins": -0.00962964165955782,
+      "rewards/rejected": 0.0036429166793823242,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9451839923858643,
-      "logits/rejected": -1.94712233543396,
-      "logps/chosen": -32.599273681640625,
-      "logps/rejected": -33.205684661865234,
-      "loss": 0.6882,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.000273245939752087,
-      "rewards/margins": 0.010566400364041328,
-      "rewards/rejected": -0.010293153114616871,
+      "logits/chosen": -1.9451322555541992,
+      "logits/rejected": -1.9470863342285156,
+      "logps/chosen": -32.58552551269531,
+      "logps/rejected": -33.180747985839844,
+      "loss": 0.689,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.005862490274012089,
+      "rewards/margins": 0.009610554203391075,
+      "rewards/rejected": -0.003748063463717699,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.0796267986297607,
-      "logits/rejected": -2.0845961570739746,
-      "logps/chosen": -33.98247528076172,
-      "logps/rejected": -36.595970153808594,
-      "loss": 0.6887,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.0018027916084975004,
-      "rewards/margins": 0.009932766668498516,
-      "rewards/rejected": -0.008129975758492947,
+      "logits/chosen": -2.0796828269958496,
+      "logits/rejected": -2.0846545696258545,
+      "logps/chosen": -33.98699188232422,
+      "logps/rejected": -36.566322326660156,
+      "loss": 0.6942,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.000597482081502676,
+      "rewards/margins": -0.00042274221777915955,
+      "rewards/rejected": 0.0010202232515439391,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9422763586044312,
-      "logits/rejected": -1.9454386234283447,
-      "logps/chosen": -34.4161491394043,
-      "logps/rejected": -34.59540939331055,
-      "loss": 0.6915,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0034980489872395992,
-      "rewards/margins": 0.004399437457323074,
-      "rewards/rejected": -0.0009013883536681533,
+      "logits/chosen": -1.9422571659088135,
+      "logits/rejected": -1.945412278175354,
+      "logps/chosen": -34.415496826171875,
+      "logps/rejected": -34.577476501464844,
+      "loss": 0.6947,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.004924021661281586,
+      "rewards/margins": -0.0010487461695447564,
+      "rewards/rejected": 0.00597276771441102,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.950871467590332,
-      "logits/rejected": -1.9553781747817993,
-      "logps/chosen": -32.455718994140625,
-      "logps/rejected": -32.37923812866211,
-      "loss": 0.6888,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.003510843263939023,
-      "rewards/margins": 0.009610405191779137,
-      "rewards/rejected": -0.00609956169500947,
+      "logits/chosen": -1.9508498907089233,
+      "logits/rejected": -1.955348253250122,
+      "logps/chosen": -32.44896697998047,
+      "logps/rejected": -32.381385803222656,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0073806666769087315,
+      "rewards/margins": 0.01637081615626812,
+      "rewards/rejected": -0.00899015087634325,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.0493884086608887,
-      "logits/rejected": -2.0473878383636475,
-      "logps/chosen": -32.24263381958008,
-      "logps/rejected": -31.301403045654297,
-      "loss": 0.6886,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.0005389736033976078,
-      "rewards/margins": 0.010018542408943176,
-      "rewards/rejected": -0.010557514615356922,
+      "logits/chosen": -2.0491819381713867,
+      "logits/rejected": -2.0471725463867188,
+      "logps/chosen": -32.225120544433594,
+      "logps/rejected": -31.267704010009766,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0062875039875507355,
+      "rewards/margins": 0.006883688271045685,
+      "rewards/rejected": -0.0005961857968941331,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.244236707687378,
-      "eval_logits/rejected": -2.2393476963043213,
-      "eval_logps/chosen": -34.027488708496094,
-      "eval_logps/rejected": -37.500125885009766,
-      "eval_loss": 0.6949825882911682,
-      "eval_rewards/accuracies": 0.48172760009765625,
-      "eval_rewards/chosen": 0.0021188759710639715,
-      "eval_rewards/margins": -0.0028304157312959433,
-      "eval_rewards/rejected": 0.0049492912366986275,
-      "eval_runtime": 146.0369,
-      "eval_samples_per_second": 2.349,
+      "eval_logits/chosen": -2.243363618850708,
+      "eval_logits/rejected": -2.2384824752807617,
+      "eval_logps/chosen": -34.026004791259766,
+      "eval_logps/rejected": -37.51805114746094,
+      "eval_loss": 0.6918829679489136,
+      "eval_rewards/accuracies": 0.5427741408348083,
+      "eval_rewards/chosen": 0.0034199401270598173,
+      "eval_rewards/margins": 0.003991155419498682,
+      "eval_rewards/rejected": -0.0005712147103622556,
+      "eval_runtime": 146.1635,
+      "eval_samples_per_second": 2.347,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.005858898162842,
-      "logits/rejected": -2.0034613609313965,
-      "logps/chosen": -33.22675323486328,
-      "logps/rejected": -34.024375915527344,
-      "loss": 0.6918,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.0051184119656682014,
-      "rewards/margins": 0.0036206122022122145,
-      "rewards/rejected": 0.0014977991813793778,
+      "logits/chosen": -2.0055806636810303,
+      "logits/rejected": -2.003166675567627,
+      "logps/chosen": -33.227996826171875,
+      "logps/rejected": -34.02782440185547,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0063266693614423275,
+      "rewards/margins": 0.0057087307795882225,
+      "rewards/rejected": 0.0006179373594932258,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.0165371894836426,
-      "logits/rejected": -2.0081915855407715,
-      "logps/chosen": -32.43837356567383,
-      "logps/rejected": -32.1854362487793,
-      "loss": 0.689,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0019600242376327515,
-      "rewards/margins": 0.009139470756053925,
-      "rewards/rejected": -0.0071794455870985985,
+      "logits/chosen": -2.016270160675049,
+      "logits/rejected": -2.0078837871551514,
+      "logps/chosen": -32.447757720947266,
+      "logps/rejected": -32.19353485107422,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0011401770170778036,
+      "rewards/margins": 0.011673182249069214,
+      "rewards/rejected": -0.012813357636332512,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.046520709991455,
-      "logits/rejected": -2.0384721755981445,
-      "logps/chosen": -30.485082626342773,
-      "logps/rejected": -32.0345344543457,
-      "loss": 0.6976,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": -0.002810357604175806,
-      "rewards/margins": -0.008352747187018394,
-      "rewards/rejected": 0.005542389117181301,
+      "logits/chosen": -2.04642653465271,
+      "logits/rejected": -2.038364887237549,
+      "logps/chosen": -30.48968505859375,
+      "logps/rejected": -32.04593276977539,
+      "loss": 0.6979,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.005588879343122244,
+      "rewards/margins": -0.008417831733822823,
+      "rewards/rejected": 0.0028289512265473604,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.976843237876892,
-      "logits/rejected": -1.9871089458465576,
-      "logps/chosen": -31.4160099029541,
-      "logps/rejected": -32.535850524902344,
-      "loss": 0.6929,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.0010961029911413789,
-      "rewards/margins": 0.0013871133560314775,
-      "rewards/rejected": -0.0002910103357862681,
+      "logits/chosen": -1.9769046306610107,
+      "logits/rejected": -1.9871702194213867,
+      "logps/chosen": -31.396442413330078,
+      "logps/rejected": -32.558387756347656,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.009287373162806034,
+      "rewards/margins": 0.01869189366698265,
+      "rewards/rejected": -0.009404524229466915,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.890434980392456,
-      "logits/rejected": -1.8915313482284546,
-      "logps/chosen": -34.165489196777344,
-      "logps/rejected": -34.74839782714844,
-      "loss": 0.691,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.008160250261425972,
-      "rewards/margins": 0.005502746906131506,
-      "rewards/rejected": 0.0026575028896331787,
+      "logits/chosen": -1.8908485174179077,
+      "logits/rejected": -1.8919233083724976,
+      "logps/chosen": -34.16999053955078,
+      "logps/rejected": -34.78479766845703,
+      "loss": 0.6841,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.009080578573048115,
+      "rewards/margins": 0.02009742520749569,
+      "rewards/rejected": -0.011016845703125,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.942751169204712,
-      "logits/rejected": -1.9392738342285156,
-      "logps/chosen": -36.15436553955078,
-      "logps/rejected": -32.734371185302734,
-      "loss": 0.6893,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.005194104742258787,
-      "rewards/margins": 0.008680562488734722,
-      "rewards/rejected": -0.0034864575136452913,
+      "logits/chosen": -1.942665457725525,
+      "logits/rejected": -1.9391899108886719,
+      "logps/chosen": -36.12689971923828,
+      "logps/rejected": -32.72255325317383,
+      "loss": 0.685,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.017910802736878395,
+      "rewards/margins": 0.01783105731010437,
+      "rewards/rejected": 7.97472894191742e-05,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.042160749435425,
-      "logits/rejected": -2.0347483158111572,
-      "logps/chosen": -33.785484313964844,
-      "logps/rejected": -31.344274520874023,
-      "loss": 0.6928,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.003714872058480978,
-      "rewards/margins": 0.0014427527785301208,
-      "rewards/rejected": 0.002272119279950857,
+      "logits/chosen": -2.042165756225586,
+      "logits/rejected": -2.0347695350646973,
+      "logps/chosen": -33.787109375,
+      "logps/rejected": -31.368520736694336,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.004300374537706375,
+      "rewards/margins": 0.010968221351504326,
+      "rewards/rejected": -0.006667847745120525,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0473780632019043,
-      "logits/rejected": -2.0526490211486816,
-      "logps/chosen": -32.52016830444336,
-      "logps/rejected": -32.495323181152344,
-      "loss": 0.6883,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.007385374512523413,
-      "rewards/margins": 0.010440578684210777,
-      "rewards/rejected": -0.0030552041716873646,
+      "logits/chosen": -2.047497510910034,
+      "logits/rejected": -2.052760601043701,
+      "logps/chosen": -32.53215789794922,
+      "logps/rejected": -32.50490951538086,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0050529674626886845,
+      "rewards/margins": 0.01296031754463911,
+      "rewards/rejected": -0.007907351478934288,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.0484580993652344,
-      "logits/rejected": -2.045682191848755,
-      "logps/chosen": -31.508682250976562,
-      "logps/rejected": -31.3348388671875,
-      "loss": 0.694,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.005918826907873154,
-      "rewards/margins": -0.0009324293350800872,
-      "rewards/rejected": -0.004986398387700319,
+      "logits/chosen": -2.0487968921661377,
+      "logits/rejected": -2.0460152626037598,
+      "logps/chosen": -31.495594024658203,
+      "logps/rejected": -31.3470401763916,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.002655787393450737,
+      "rewards/margins": 0.008874570950865746,
+      "rewards/rejected": -0.011530356481671333,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9187612533569336,
-      "logits/rejected": -1.9234310388565063,
-      "logps/chosen": -31.59661865234375,
-      "logps/rejected": -32.798248291015625,
-      "loss": 0.6909,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.002899137092754245,
-      "rewards/margins": 0.00509423715993762,
-      "rewards/rejected": -0.002195100300014019,
+      "logits/chosen": -1.9187023639678955,
+      "logits/rejected": -1.9233903884887695,
+      "logps/chosen": -31.589069366455078,
+      "logps/rejected": -32.78667449951172,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.006885607726871967,
+      "rewards/margins": 0.0051818834617733955,
+      "rewards/rejected": 0.0017037258949130774,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2439279556274414,
-      "eval_logits/rejected": -2.239055633544922,
-      "eval_logps/chosen": -34.03278350830078,
-      "eval_logps/rejected": -37.50389099121094,
-      "eval_loss": 0.6952112317085266,
-      "eval_rewards/accuracies": 0.5191029906272888,
-      "eval_rewards/chosen": 0.0005306191742420197,
-      "eval_rewards/margins": -0.0032888990826904774,
-      "eval_rewards/rejected": 0.0038195180241018534,
-      "eval_runtime": 145.8142,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.2433600425720215,
+      "eval_logits/rejected": -2.238468885421753,
+      "eval_logps/chosen": -34.01414489746094,
+      "eval_logps/rejected": -37.50386428833008,
+      "eval_loss": 0.692382276058197,
+      "eval_rewards/accuracies": 0.4991694688796997,
+      "eval_rewards/chosen": 0.008161893114447594,
+      "eval_rewards/margins": 0.003057094058021903,
+      "eval_rewards/rejected": 0.00510479835793376,
+      "eval_runtime": 145.9559,
+      "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0318386554718018,
-      "logits/rejected": -2.0425145626068115,
-      "logps/chosen": -31.961750030517578,
-      "logps/rejected": -33.86240005493164,
-      "loss": 0.6923,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.0028654958587139845,
-      "rewards/margins": 0.002774373395368457,
-      "rewards/rejected": 9.112273983191699e-05,
+      "logits/chosen": -2.031435489654541,
+      "logits/rejected": -2.042121648788452,
+      "logps/chosen": -31.93819236755371,
+      "logps/rejected": -33.87244415283203,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.013244074769318104,
+      "rewards/margins": 0.017141219228506088,
+      "rewards/rejected": -0.003897144692018628,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9253981113433838,
-      "logits/rejected": -1.9402506351470947,
-      "logps/chosen": -30.093517303466797,
-      "logps/rejected": -31.571548461914062,
-      "loss": 0.6889,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.005405673757195473,
-      "rewards/margins": 0.009417025372385979,
-      "rewards/rejected": -0.004011353012174368,
+      "logits/chosen": -1.9252116680145264,
+      "logits/rejected": -1.9400631189346313,
+      "logps/chosen": -30.0819149017334,
+      "logps/rejected": -31.582805633544922,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.011850157752633095,
+      "rewards/margins": 0.021701235324144363,
+      "rewards/rejected": -0.009851074777543545,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9839330911636353,
-      "logits/rejected": -1.9879001379013062,
-      "logps/chosen": -33.4061279296875,
-      "logps/rejected": -31.564762115478516,
+      "logits/chosen": -1.9831390380859375,
+      "logits/rejected": -1.9871015548706055,
+      "logps/chosen": -33.370384216308594,
+      "logps/rejected": -31.524139404296875,
       "loss": 0.6896,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.0034024070482701063,
-      "rewards/margins": 0.008040683344006538,
-      "rewards/rejected": -0.004638276062905788,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.018833406269550323,
+      "rewards/margins": 0.008769527077674866,
+      "rewards/rejected": 0.010063880123198032,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.983189344406128,
-      "logits/rejected": -1.9612159729003906,
-      "logps/chosen": -34.15366744995117,
-      "logps/rejected": -34.95070266723633,
-      "loss": 0.6937,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 2.4808850866975263e-05,
-      "rewards/margins": -1.6275793086606427e-06,
-      "rewards/rejected": 2.643577681737952e-05,
+      "logits/chosen": -1.9828351736068726,
+      "logits/rejected": -1.9608606100082397,
+      "logps/chosen": -34.157318115234375,
+      "logps/rejected": -34.97474670410156,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.0014271701220422983,
+      "rewards/margins": 0.008153198286890984,
+      "rewards/rejected": -0.009580368176102638,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.0243947505950928,
-      "logits/rejected": -2.021087169647217,
-      "logps/chosen": -32.91602325439453,
-      "logps/rejected": -36.221763610839844,
-      "loss": 0.6928,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.002715445589274168,
-      "rewards/margins": 0.001767707639373839,
-      "rewards/rejected": 0.0009477367857471108,
+      "logits/chosen": -2.0243985652923584,
+      "logits/rejected": -2.0210859775543213,
+      "logps/chosen": -32.91604995727539,
+      "logps/rejected": -36.21431350708008,
+      "loss": 0.6945,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.003607477992773056,
+      "rewards/margins": -0.0006370179471559823,
+      "rewards/rejected": 0.004244496580213308,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8911654949188232,
-      "logits/rejected": -1.8887317180633545,
-      "logps/chosen": -34.18123245239258,
-      "logps/rejected": -35.515174865722656,
-      "loss": 0.691,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.0021498873829841614,
-      "rewards/margins": 0.005315869115293026,
-      "rewards/rejected": -0.0031659831292927265,
+      "logits/chosen": -1.8909612894058228,
+      "logits/rejected": -1.8885223865509033,
+      "logps/chosen": -34.173240661621094,
+      "logps/rejected": -35.51388931274414,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.006062888540327549,
+      "rewards/margins": 0.009769372642040253,
+      "rewards/rejected": -0.0037064836360514164,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.875580072402954,
-      "logits/rejected": -1.8730493783950806,
-      "logps/chosen": -34.37896728515625,
-      "logps/rejected": -31.76068687438965,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.005589136388152838,
-      "rewards/margins": 0.0023957251105457544,
-      "rewards/rejected": 0.0031934112776070833,
+      "logits/chosen": -1.8755378723144531,
+      "logits/rejected": -1.8729900121688843,
+      "logps/chosen": -34.416507720947266,
+      "logps/rejected": -31.75918197631836,
+      "loss": 0.7001,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.007562094368040562,
+      "rewards/margins": -0.012420646846294403,
+      "rewards/rejected": 0.004858553409576416,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9798663854599,
-      "logits/rejected": -1.9692500829696655,
-      "logps/chosen": -35.32219696044922,
-      "logps/rejected": -31.857141494750977,
-      "loss": 0.6861,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.010332418605685234,
-      "rewards/margins": 0.015176350250840187,
-      "rewards/rejected": -0.004843929782509804,
+      "logits/chosen": -1.979413628578186,
+      "logits/rejected": -1.9687983989715576,
+      "logps/chosen": -35.34419631958008,
+      "logps/rejected": -31.839731216430664,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.004978640470653772,
+      "rewards/margins": 0.004474071320146322,
+      "rewards/rejected": 0.0005045672878623009,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.076137065887451,
-      "logits/rejected": -2.061100721359253,
-      "logps/chosen": -30.907434463500977,
-      "logps/rejected": -32.638729095458984,
-      "loss": 0.693,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.00500696524977684,
-      "rewards/margins": 0.0010755129624158144,
-      "rewards/rejected": 0.0039314525201916695,
+      "logits/chosen": -2.075562000274658,
+      "logits/rejected": -2.060526132583618,
+      "logps/chosen": -30.924896240234375,
+      "logps/rejected": -32.64234161376953,
+      "loss": 0.696,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.0003082419862039387,
+      "rewards/margins": -0.004107682965695858,
+      "rewards/rejected": 0.0037994408048689365,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.947422981262207,
-      "logits/rejected": -1.9448877573013306,
-      "logps/chosen": -32.89393997192383,
-      "logps/rejected": -30.828500747680664,
-      "loss": 0.6869,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.009951007552444935,
-      "rewards/margins": 0.013458291999995708,
-      "rewards/rejected": -0.003507285611703992,
+      "logits/chosen": -1.947762131690979,
+      "logits/rejected": -1.9452297687530518,
+      "logps/chosen": -32.90521240234375,
+      "logps/rejected": -30.821033477783203,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.008759219199419022,
+      "rewards/margins": 0.010448457673192024,
+      "rewards/rejected": -0.0016892381245270371,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.244764566421509,
-      "eval_logits/rejected": -2.2398788928985596,
-      "eval_logps/chosen": -34.038692474365234,
-      "eval_logps/rejected": -37.512168884277344,
-      "eval_loss": 0.6949278712272644,
-      "eval_rewards/accuracies": 0.5161960124969482,
-      "eval_rewards/chosen": -0.0012418677797541022,
-      "eval_rewards/margins": -0.0025783225428313017,
-      "eval_rewards/rejected": 0.0013364545302465558,
-      "eval_runtime": 145.8034,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.2443997859954834,
+      "eval_logits/rejected": -2.2395174503326416,
+      "eval_logps/chosen": -34.02724838256836,
+      "eval_logps/rejected": -37.50510787963867,
+      "eval_loss": 0.6948643326759338,
+      "eval_rewards/accuracies": 0.5116279125213623,
+      "eval_rewards/chosen": 0.0029200678691267967,
+      "eval_rewards/margins": -0.0016844052588567138,
+      "eval_rewards/rejected": 0.00460447371006012,
+      "eval_runtime": 145.9303,
+      "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9299108982086182,
-      "logits/rejected": -1.9266481399536133,
-      "logps/chosen": -31.572988510131836,
-      "logps/rejected": -33.727210998535156,
-      "loss": 0.6912,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.009108476340770721,
-      "rewards/margins": 0.004626707639545202,
-      "rewards/rejected": 0.0044817691668868065,
+      "logits/chosen": -1.9295498132705688,
+      "logits/rejected": -1.9262762069702148,
+      "logps/chosen": -31.575878143310547,
+      "logps/rejected": -33.75602340698242,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.010988839901983738,
+      "rewards/margins": 0.0165377426892519,
+      "rewards/rejected": -0.0055489009246230125,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9814649820327759,
-      "logits/rejected": -1.9691375494003296,
-      "logps/chosen": -34.551979064941406,
-      "logps/rejected": -33.541648864746094,
-      "loss": 0.6866,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.013229632750153542,
-      "rewards/margins": 0.014072793535888195,
-      "rewards/rejected": -0.0008431615424342453,
+      "logits/chosen": -1.9806196689605713,
+      "logits/rejected": -1.9683196544647217,
+      "logps/chosen": -34.554603576660156,
+      "logps/rejected": -33.58423614501953,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.01658753491938114,
+      "rewards/margins": 0.034744132310152054,
+      "rewards/rejected": -0.01815659925341606,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.017097234725952,
-      "logits/rejected": -2.0156359672546387,
-      "logps/chosen": -33.49595260620117,
-      "logps/rejected": -32.49913787841797,
-      "loss": 0.6922,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.003048144979402423,
-      "rewards/margins": 0.002819380722939968,
-      "rewards/rejected": -0.00586752500385046,
+      "logits/chosen": -2.016143798828125,
+      "logits/rejected": -2.014680862426758,
+      "logps/chosen": -33.479766845703125,
+      "logps/rejected": -32.474754333496094,
+      "loss": 0.6937,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.002409630687907338,
+      "rewards/margins": 0.0004784400516655296,
+      "rewards/rejected": 0.0019311904907226562,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.103210926055908,
-      "logits/rejected": -2.0874040126800537,
-      "logps/chosen": -34.16447830200195,
-      "logps/rejected": -33.07390594482422,
-      "loss": 0.6981,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.004473570734262466,
-      "rewards/margins": -0.008766504004597664,
-      "rewards/rejected": 0.01324007473886013,
+      "logits/chosen": -2.103015899658203,
+      "logits/rejected": -2.08721923828125,
+      "logps/chosen": -34.18162155151367,
+      "logps/rejected": -33.10049819946289,
+      "loss": 0.6982,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.0008922239649109542,
+      "rewards/margins": -0.007907899096608162,
+      "rewards/rejected": 0.0070156739093363285,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9749380350112915,
-      "logits/rejected": -1.9740018844604492,
-      "logps/chosen": -33.25703048706055,
-      "logps/rejected": -32.47133255004883,
-      "loss": 0.689,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.00751916179433465,
-      "rewards/margins": 0.009374773129820824,
-      "rewards/rejected": -0.0018556114519014955,
+      "logits/chosen": -1.9744770526885986,
+      "logits/rejected": -1.9735443592071533,
+      "logps/chosen": -33.24970245361328,
+      "logps/rejected": -32.445960998535156,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.012956788763403893,
+      "rewards/margins": 0.00528245884925127,
+      "rewards/rejected": 0.007674329914152622,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9304084777832031,
-      "logits/rejected": -1.9407765865325928,
-      "logps/chosen": -32.21118927001953,
-      "logps/rejected": -35.31554412841797,
-      "loss": 0.6905,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.0019482700154185295,
-      "rewards/margins": 0.0060885134153068066,
-      "rewards/rejected": -0.008036783896386623,
+      "logits/chosen": -1.9305318593978882,
+      "logits/rejected": -1.9408973455429077,
+      "logps/chosen": -32.2140007019043,
+      "logps/rejected": -35.3133430480957,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.0037243987899273634,
+      "rewards/margins": 0.006108095403760672,
+      "rewards/rejected": -0.00983249582350254,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.06982421875,
-      "logits/rejected": -2.0632567405700684,
-      "logps/chosen": -33.641204833984375,
-      "logps/rejected": -29.2253475189209,
-      "loss": 0.6921,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.003229249268770218,
-      "rewards/margins": 0.0029065334238111973,
-      "rewards/rejected": 0.000322715932270512,
+      "logits/chosen": -2.069563150405884,
+      "logits/rejected": -2.063000202178955,
+      "logps/chosen": -33.635009765625,
+      "logps/rejected": -29.221439361572266,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.006782642100006342,
+      "rewards/margins": 0.004788074642419815,
+      "rewards/rejected": 0.0019945662934333086,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.929220199584961,
-      "logits/rejected": -1.9313905239105225,
-      "logps/chosen": -34.26762008666992,
-      "logps/rejected": -30.9031982421875,
-      "loss": 0.6934,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.00857237633317709,
-      "rewards/margins": 0.0005976069951429963,
-      "rewards/rejected": -0.009169982746243477,
+      "logits/chosen": -1.9291139841079712,
+      "logits/rejected": -1.9312822818756104,
+      "logps/chosen": -34.2513313293457,
+      "logps/rejected": -30.88556480407715,
+      "loss": 0.6938,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.004911360330879688,
+      "rewards/margins": 0.00026298072771169245,
+      "rewards/rejected": -0.005174341145902872,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.691356255791404,
-      "train_runtime": 3254.7157,
+      "train_loss": 0.6908149576806403,
+      "train_runtime": 3255.681,
       "train_samples_per_second": 0.946,
       "train_steps_per_second": 0.118
     }