Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9658
-- Rewards/chosen: -0.0301
-- Rewards/rejected: -0.0652
-- Rewards/accuracies: 0.5278
-- Rewards/margins: 0.0352
-- Logps/rejected: -37.6098
-- Logps/chosen: -34.0775
-- Logits/rejected: -2.2182
-- Logits/chosen: -2.2231
+- Loss: 0.6957
+- Rewards/chosen: -0.0451
+- Rewards/rejected: -0.0645
+- Rewards/accuracies: 0.5399
+- Rewards/margins: 0.0194
+- Logps/rejected: -37.5973
+- Logps/chosen: -34.0909
+- Logits/rejected: -2.2246
+- Logits/chosen: -2.2294
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.8961        | 0.26  | 100  | 0.9767          | 0.0065         | -0.0170          | 0.5365             | 0.0235          | -37.5409       | -34.0252     | -2.2266         | -2.2315       |
-| 0.7699        | 0.52  | 200  | 0.9742          | -0.0141        | -0.0400          | 0.5303             | 0.0259          | -37.5737       | -34.0547     | -2.2234         | -2.2282       |
-| 0.6723        | 0.78  | 300  | 0.9761          | -0.0366        | -0.0616          | 0.5299             | 0.0250          | -37.6047       | -34.0868     | -2.2186         | -2.2234       |
+| 0.6538        | 0.26  | 100  | 0.6955          | 0.0035         | -0.0085          | 0.5108             | 0.0120          | -37.5273       | -34.0302     | -2.2290         | -2.2339       |
+| 0.6015        | 0.52  | 200  | 0.6956          | -0.0209        | -0.0393          | 0.5249             | 0.0183          | -37.5657       | -34.0607     | -2.2274         | -2.2322       |
+| 0.5385        | 0.78  | 300  | 0.6957          | -0.0451        | -0.0645          | 0.5399             | 0.0194          | -37.5973       | -34.0909     | -2.2246         | -2.2294       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b67eba741e2020a4180a0fd9b6b098a84c42623f479ebff7d23f8e34d03ceef8
+oid sha256:e9f75024fb68f4db40ab121a28260c03b972557b8a61cd854612af6257348d54
 size 176183216

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2230520248413086,
-    "eval_logits/rejected": -2.2182328701019287,
-    "eval_logps/chosen": -34.07753372192383,
-    "eval_logps/rejected": -37.60981750488281,
-    "eval_loss": 0.9658033847808838,
-    "eval_rewards/accuracies": 0.5278239250183105,
-    "eval_rewards/chosen": -0.03008819743990898,
-    "eval_rewards/margins": 0.03515118733048439,
-    "eval_rewards/rejected": -0.06523937731981277,
-    "eval_runtime": 145.6671,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.355,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.8438688600218142,
-    "train_runtime": 3250.9917,
+    "train_loss": 0.6230236078237559,
+    "train_runtime": 3254.2307,
     "train_samples": 3079,
-    "train_samples_per_second": 0.947,
+    "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.8438688600218142,
-    "train_runtime": 3250.9917,
+    "train_loss": 0.6230236078237559,
+    "train_runtime": 3254.2307,
     "train_samples": 3079,
-    "train_samples_per_second": 0.947,
+    "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 1.0,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,590 +25,590 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8664777278900146,
-      "logits/rejected": -1.8707849979400635,
-      "logps/chosen": -36.99364471435547,
-      "logps/rejected": -33.650604248046875,
-      "loss": 0.9766,
-      "rewards/accuracies": 0.5277777910232544,
-      "rewards/chosen": 0.00906434003263712,
-      "rewards/margins": 0.023435616865754128,
-      "rewards/rejected": -0.014371277764439583,
+      "logits/chosen": -1.8663169145584106,
+      "logits/rejected": -1.870638370513916,
+      "logps/chosen": -36.98221206665039,
+      "logps/rejected": -33.6473503112793,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.5416666865348816,
+      "rewards/chosen": 0.01950961910188198,
+      "rewards/margins": 0.03332838416099548,
+      "rewards/rejected": -0.013818766921758652,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9982150793075562,
-      "logits/rejected": -2.0008621215820312,
-      "logps/chosen": -29.64394760131836,
-      "logps/rejected": -29.04986000061035,
-      "loss": 1.0107,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.0012267641723155975,
-      "rewards/margins": -0.010734880343079567,
-      "rewards/rejected": 0.00950811617076397,
+      "logits/chosen": -1.9978923797607422,
+      "logits/rejected": -2.0005345344543457,
+      "logps/chosen": -29.642324447631836,
+      "logps/rejected": -29.048343658447266,
+      "loss": 0.7013,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.00010492801811778918,
+      "rewards/margins": -0.012185259722173214,
+      "rewards/rejected": 0.012080332264304161,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.920768141746521,
-      "logits/rejected": -1.9180870056152344,
-      "logps/chosen": -31.416461944580078,
-      "logps/rejected": -33.2098274230957,
-      "loss": 1.0063,
+      "logits/chosen": -1.9209339618682861,
+      "logits/rejected": -1.9182507991790771,
+      "logps/chosen": -31.401519775390625,
+      "logps/rejected": -33.22309875488281,
+      "loss": 0.6891,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.000256747764069587,
-      "rewards/margins": -0.0063001858070492744,
-      "rewards/rejected": 0.00604343693703413,
+      "rewards/chosen": 0.011660982854664326,
+      "rewards/margins": 0.015369392931461334,
+      "rewards/rejected": -0.003708411008119583,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.0177602767944336,
-      "logits/rejected": -2.009014129638672,
-      "logps/chosen": -32.56236267089844,
-      "logps/rejected": -32.517822265625,
-      "loss": 0.9863,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.010110180824995041,
-      "rewards/margins": 0.013717299327254295,
-      "rewards/rejected": -0.003607118036597967,
+      "logits/chosen": -2.0180399417877197,
+      "logits/rejected": -2.009289264678955,
+      "logps/chosen": -32.559410095214844,
+      "logps/rejected": -32.52582550048828,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.013918718323111534,
+      "rewards/margins": 0.02444135770201683,
+      "rewards/rejected": -0.010522643104195595,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.862694501876831,
-      "logits/rejected": -1.8519262075424194,
-      "logps/chosen": -33.541160583496094,
-      "logps/rejected": -35.44048309326172,
-      "loss": 0.9969,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.011423684656620026,
-      "rewards/margins": 0.003121361369267106,
-      "rewards/rejected": 0.00830232072621584,
+      "logits/chosen": -1.8629716634750366,
+      "logits/rejected": -1.8522107601165771,
+      "logps/chosen": -33.554229736328125,
+      "logps/rejected": -35.44757080078125,
+      "loss": 0.6987,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.002600290346890688,
+      "rewards/margins": -0.0012185067171230912,
+      "rewards/rejected": 0.003818795783445239,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9417282342910767,
-      "logits/rejected": -1.9436867237091064,
-      "logps/chosen": -32.52958679199219,
-      "logps/rejected": -33.216880798339844,
-      "loss": 0.9296,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.04941769689321518,
-      "rewards/margins": 0.08126799017190933,
-      "rewards/rejected": -0.031850285828113556,
+      "logits/chosen": -1.9408414363861084,
+      "logits/rejected": -1.9427950382232666,
+      "logps/chosen": -32.56097412109375,
+      "logps/rejected": -33.213417053222656,
+      "loss": 0.6725,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.03136637061834335,
+      "rewards/margins": 0.06499636918306351,
+      "rewards/rejected": -0.03362999111413956,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.0729386806488037,
-      "logits/rejected": -2.0779125690460205,
-      "logps/chosen": -33.99254608154297,
-      "logps/rejected": -36.62586212158203,
-      "loss": 0.9629,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.0028400986921042204,
-      "rewards/margins": 0.037055134773254395,
-      "rewards/rejected": -0.03989524394273758,
+      "logits/chosen": -2.072221517562866,
+      "logits/rejected": -2.077198028564453,
+      "logps/chosen": -33.974578857421875,
+      "logps/rejected": -36.629173278808594,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.011124782264232635,
+      "rewards/margins": 0.059367585927248,
+      "rewards/rejected": -0.04824279993772507,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9335737228393555,
-      "logits/rejected": -1.9366981983184814,
-      "logps/chosen": -34.332157135009766,
-      "logps/rejected": -34.641021728515625,
-      "loss": 0.9026,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.06695752590894699,
-      "rewards/margins": 0.10098665952682495,
-      "rewards/rejected": -0.03402913734316826,
+      "logits/chosen": -1.9338233470916748,
+      "logits/rejected": -1.9369605779647827,
+      "logps/chosen": -34.30416488647461,
+      "logps/rejected": -34.634437561035156,
+      "loss": 0.6423,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.09891629219055176,
+      "rewards/margins": 0.132537841796875,
+      "rewards/rejected": -0.03362155705690384,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9401309490203857,
-      "logits/rejected": -1.9446433782577515,
-      "logps/chosen": -32.37213897705078,
-      "logps/rejected": -32.343849182128906,
-      "loss": 0.9438,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.06670050323009491,
-      "rewards/margins": 0.05615914613008499,
-      "rewards/rejected": 0.010541360825300217,
+      "logits/chosen": -1.9418385028839111,
+      "logits/rejected": -1.9463545083999634,
+      "logps/chosen": -32.39947509765625,
+      "logps/rejected": -32.35419464111328,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.05435952544212341,
+      "rewards/margins": 0.05058818310499191,
+      "rewards/rejected": 0.0037713423371315002,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.037087917327881,
-      "logits/rejected": -2.035101890563965,
-      "logps/chosen": -32.13945388793945,
-      "logps/rejected": -31.313283920288086,
-      "loss": 0.8961,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.07096613943576813,
-      "rewards/margins": 0.10391455888748169,
-      "rewards/rejected": -0.032948415726423264,
+      "logits/chosen": -2.039783000946045,
+      "logits/rejected": -2.037789821624756,
+      "logps/chosen": -32.164188385009766,
+      "logps/rejected": -31.309520721435547,
+      "loss": 0.6538,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.061319977045059204,
+      "rewards/margins": 0.09596750140190125,
+      "rewards/rejected": -0.03464752808213234,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.231482982635498,
-      "eval_logits/rejected": -2.226637363433838,
-      "eval_logps/chosen": -34.02524185180664,
-      "eval_logps/rejected": -37.54085159301758,
-      "eval_loss": 0.9766585230827332,
-      "eval_rewards/accuracies": 0.5365448594093323,
-      "eval_rewards/chosen": 0.006516099441796541,
-      "eval_rewards/margins": 0.02347717247903347,
-      "eval_rewards/rejected": -0.016961071640253067,
-      "eval_runtime": 145.8279,
-      "eval_samples_per_second": 2.352,
-      "eval_steps_per_second": 0.295,
+      "eval_logits/chosen": -2.2338831424713135,
+      "eval_logits/rejected": -2.229041814804077,
+      "eval_logps/chosen": -34.03020095825195,
+      "eval_logps/rejected": -37.52727127075195,
+      "eval_loss": 0.6954607963562012,
+      "eval_rewards/accuracies": 0.510797381401062,
+      "eval_rewards/chosen": 0.0034842013847082853,
+      "eval_rewards/margins": 0.012004716321825981,
+      "eval_rewards/rejected": -0.008520514704287052,
+      "eval_runtime": 146.115,
+      "eval_samples_per_second": 2.347,
+      "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9920060634613037,
-      "logits/rejected": -1.9896419048309326,
-      "logps/chosen": -33.146766662597656,
-      "logps/rejected": -34.02008819580078,
-      "loss": 0.9486,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.0679345652461052,
-      "rewards/margins": 0.061437882483005524,
-      "rewards/rejected": 0.006496679037809372,
+      "logits/chosen": -1.9946181774139404,
+      "logits/rejected": -1.992236852645874,
+      "logps/chosen": -33.117286682128906,
+      "logps/rejected": -34.00868225097656,
+      "loss": 0.6852,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.10122231394052505,
+      "rewards/margins": 0.0846698135137558,
+      "rewards/rejected": 0.016552483662962914,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.003952741622925,
-      "logits/rejected": -1.9956319332122803,
-      "logps/chosen": -32.33639144897461,
-      "logps/rejected": -32.133079528808594,
-      "loss": 0.9488,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.07595954835414886,
-      "rewards/margins": 0.05606143921613693,
-      "rewards/rejected": 0.019898109138011932,
+      "logits/chosen": -2.006875514984131,
+      "logits/rejected": -1.9985148906707764,
+      "logps/chosen": -32.336421966552734,
+      "logps/rejected": -32.137081146240234,
+      "loss": 0.675,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.08678452670574188,
+      "rewards/margins": 0.06724556535482407,
+      "rewards/rejected": 0.019538963213562965,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0316150188446045,
-      "logits/rejected": -2.0236544609069824,
-      "logps/chosen": -30.298206329345703,
-      "logps/rejected": -32.07080841064453,
-      "loss": 0.8911,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.12425784766674042,
-      "rewards/margins": 0.1367165446281433,
-      "rewards/rejected": -0.0124586820602417,
+      "logits/chosen": -2.0351502895355225,
+      "logits/rejected": -2.02717661857605,
+      "logps/chosen": -30.30923843383789,
+      "logps/rejected": -32.08501434326172,
+      "loss": 0.6398,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.13318516314029694,
+      "rewards/margins": 0.15878939628601074,
+      "rewards/rejected": -0.025604233145713806,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.962376356124878,
-      "logits/rejected": -1.9726108312606812,
-      "logps/chosen": -31.235275268554688,
-      "logps/rejected": -32.56925964355469,
-      "loss": 0.8524,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.1290682703256607,
-      "rewards/margins": 0.15313370525836945,
-      "rewards/rejected": -0.024065453559160233,
+      "logits/chosen": -1.966170072555542,
+      "logits/rejected": -1.9764087200164795,
+      "logps/chosen": -31.215194702148438,
+      "logps/rejected": -32.55674743652344,
+      "loss": 0.627,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.16357474029064178,
+      "rewards/margins": 0.18106886744499207,
+      "rewards/rejected": -0.017494117841124535,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8732143640518188,
-      "logits/rejected": -1.874371886253357,
-      "logps/chosen": -33.8985481262207,
-      "logps/rejected": -34.81908416748047,
-      "loss": 0.786,
+      "logits/chosen": -1.8775428533554077,
+      "logits/rejected": -1.8786998987197876,
+      "logps/chosen": -33.92055130004883,
+      "logps/rejected": -34.77721405029297,
+      "loss": 0.6168,
       "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.20590214431285858,
-      "rewards/margins": 0.24918103218078613,
-      "rewards/rejected": -0.04327889531850815,
+      "rewards/chosen": 0.21771302819252014,
+      "rewards/margins": 0.23367898166179657,
+      "rewards/rejected": -0.015965968370437622,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.924538016319275,
-      "logits/rejected": -1.921121597290039,
-      "logps/chosen": -36.01353454589844,
-      "logps/rejected": -32.723262786865234,
-      "loss": 0.8894,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.11069967597723007,
-      "rewards/margins": 0.11105670034885406,
-      "rewards/rejected": -0.0003570284752640873,
+      "logits/chosen": -1.9291296005249023,
+      "logits/rejected": -1.9257177114486694,
+      "logps/chosen": -36.01557922363281,
+      "logps/rejected": -32.72490692138672,
+      "loss": 0.6444,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.12488000094890594,
+      "rewards/margins": 0.12660440802574158,
+      "rewards/rejected": -0.0017244067275896668,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0247209072113037,
-      "logits/rejected": -2.017392635345459,
-      "logps/chosen": -33.49879455566406,
-      "logps/rejected": -31.44363784790039,
-      "loss": 0.7434,
+      "logits/chosen": -2.0295331478118896,
+      "logits/rejected": -2.0221762657165527,
+      "logps/chosen": -33.48248291015625,
+      "logps/rejected": -31.408077239990234,
+      "loss": 0.5816,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.2093469649553299,
-      "rewards/margins": 0.2735980153083801,
-      "rewards/rejected": -0.06425107270479202,
+      "rewards/chosen": 0.25230517983436584,
+      "rewards/margins": 0.2972865700721741,
+      "rewards/rejected": -0.04498137906193733,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.031324863433838,
-      "logits/rejected": -2.0365915298461914,
-      "logps/chosen": -32.253074645996094,
-      "logps/rejected": -32.45112609863281,
-      "loss": 0.8312,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.20420043170452118,
-      "rewards/margins": 0.18039169907569885,
-      "rewards/rejected": 0.023808732628822327,
+      "logits/chosen": -2.036190986633301,
+      "logits/rejected": -2.0414373874664307,
+      "logps/chosen": -32.22594451904297,
+      "logps/rejected": -32.46149444580078,
+      "loss": 0.5993,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.2550733685493469,
+      "rewards/margins": 0.2361568957567215,
+      "rewards/rejected": 0.018916476517915726,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.0317559242248535,
-      "logits/rejected": -2.0289719104766846,
-      "logps/chosen": -31.279537200927734,
-      "logps/rejected": -31.34115219116211,
-      "loss": 0.8405,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.14659160375595093,
-      "rewards/margins": 0.16264860332012177,
-      "rewards/rejected": -0.016057008877396584,
+      "logits/chosen": -2.0371222496032715,
+      "logits/rejected": -2.0343565940856934,
+      "logps/chosen": -31.28468894958496,
+      "logps/rejected": -31.336734771728516,
+      "loss": 0.6307,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.16341081261634827,
+      "rewards/margins": 0.17822694778442383,
+      "rewards/rejected": -0.014816122129559517,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9018064737319946,
-      "logits/rejected": -1.9064457416534424,
-      "logps/chosen": -31.301830291748047,
-      "logps/rejected": -32.8339729309082,
-      "loss": 0.7699,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.21311891078948975,
-      "rewards/margins": 0.2432461678981781,
-      "rewards/rejected": -0.03012726828455925,
+      "logits/chosen": -1.9067039489746094,
+      "logits/rejected": -1.91135573387146,
+      "logps/chosen": -31.312374114990234,
+      "logps/rejected": -32.82074737548828,
+      "loss": 0.6015,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.2351258099079132,
+      "rewards/margins": 0.25897616147994995,
+      "rewards/rejected": -0.0238503310829401,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2282192707061768,
-      "eval_logits/rejected": -2.223379611968994,
-      "eval_logps/chosen": -34.05467224121094,
-      "eval_logps/rejected": -37.57374572753906,
-      "eval_loss": 0.974229633808136,
-      "eval_rewards/accuracies": 0.530315637588501,
-      "eval_rewards/chosen": -0.01408342458307743,
-      "eval_rewards/margins": 0.02590302750468254,
-      "eval_rewards/rejected": -0.03998645395040512,
-      "eval_runtime": 145.7826,
-      "eval_samples_per_second": 2.353,
+      "eval_logits/chosen": -2.232191324234009,
+      "eval_logits/rejected": -2.22735333442688,
+      "eval_logps/chosen": -34.060691833496094,
+      "eval_logps/rejected": -37.56568908691406,
+      "eval_loss": 0.695566713809967,
+      "eval_rewards/accuracies": 0.52491694688797,
+      "eval_rewards/chosen": -0.020911961793899536,
+      "eval_rewards/margins": 0.01834380254149437,
+      "eval_rewards/rejected": -0.039255764335393906,
+      "eval_runtime": 145.8849,
+      "eval_samples_per_second": 2.351,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.014596462249756,
-      "logits/rejected": -2.0252418518066406,
-      "logps/chosen": -31.797557830810547,
-      "logps/rejected": -33.982398986816406,
-      "loss": 0.8139,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.12162177264690399,
-      "rewards/margins": 0.20541362464427948,
-      "rewards/rejected": -0.08379185199737549,
+      "logits/chosen": -2.0197176933288574,
+      "logits/rejected": -2.0303761959075928,
+      "logps/chosen": -31.751026153564453,
+      "logps/rejected": -33.96234893798828,
+      "loss": 0.5969,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.17622438073158264,
+      "rewards/margins": 0.25594404339790344,
+      "rewards/rejected": -0.0797196701169014,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9068737030029297,
-      "logits/rejected": -1.9216482639312744,
-      "logps/chosen": -29.838830947875977,
-      "logps/rejected": -31.62994956970215,
-      "loss": 0.766,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.1908968687057495,
-      "rewards/margins": 0.24113738536834717,
-      "rewards/rejected": -0.05024053901433945,
+      "logits/chosen": -1.9122480154037476,
+      "logits/rejected": -1.9270412921905518,
+      "logps/chosen": -29.86123275756836,
+      "logps/rejected": -31.612594604492188,
+      "loss": 0.6025,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.20024582743644714,
+      "rewards/margins": 0.24377915263175964,
+      "rewards/rejected": -0.04353334754705429,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9636850357055664,
-      "logits/rejected": -1.967655897140503,
-      "logps/chosen": -33.12433624267578,
-      "logps/rejected": -31.637094497680664,
-      "loss": 0.7833,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.20519034564495087,
-      "rewards/margins": 0.2666449546813965,
-      "rewards/rejected": -0.06145460531115532,
+      "logits/chosen": -1.9689687490463257,
+      "logits/rejected": -1.9729585647583008,
+      "logps/chosen": -33.11440658569336,
+      "logps/rejected": -31.650421142578125,
+      "loss": 0.5809,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.2424498349428177,
+      "rewards/margins": 0.3233444094657898,
+      "rewards/rejected": -0.08089461922645569,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9611847400665283,
-      "logits/rejected": -1.9393657445907593,
-      "logps/chosen": -33.867958068847656,
-      "logps/rejected": -35.12390899658203,
-      "loss": 0.737,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.2000524252653122,
-      "rewards/margins": 0.32123422622680664,
-      "rewards/rejected": -0.12118180096149445,
+      "logits/chosen": -1.9670231342315674,
+      "logits/rejected": -1.9451711177825928,
+      "logps/chosen": -33.830162048339844,
+      "logps/rejected": -35.1173095703125,
+      "loss": 0.558,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.2588713765144348,
+      "rewards/margins": 0.3920826315879822,
+      "rewards/rejected": -0.13321125507354736,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.001960277557373,
-      "logits/rejected": -1.9986454248428345,
-      "logps/chosen": -32.73499298095703,
-      "logps/rejected": -36.28093719482422,
-      "loss": 0.8434,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.13304933905601501,
-      "rewards/margins": 0.17225751280784607,
-      "rewards/rejected": -0.03920816630125046,
+      "logits/chosen": -2.0084152221679688,
+      "logits/rejected": -2.005080223083496,
+      "logps/chosen": -32.70518493652344,
+      "logps/rejected": -36.280517578125,
+      "loss": 0.6104,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.1759072244167328,
+      "rewards/margins": 0.22038432955741882,
+      "rewards/rejected": -0.044477105140686035,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8687576055526733,
-      "logits/rejected": -1.866320013999939,
-      "logps/chosen": -33.981781005859375,
-      "logps/rejected": -35.54584503173828,
-      "loss": 0.8296,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.14463281631469727,
-      "rewards/margins": 0.1734902262687683,
-      "rewards/rejected": -0.028857415542006493,
+      "logits/chosen": -1.8755052089691162,
+      "logits/rejected": -1.8730967044830322,
+      "logps/chosen": -33.984092712402344,
+      "logps/rejected": -35.538455963134766,
+      "loss": 0.622,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.16344432532787323,
+      "rewards/margins": 0.19051328301429749,
+      "rewards/rejected": -0.02706894651055336,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.853939414024353,
-      "logits/rejected": -1.8515437841415405,
-      "logps/chosen": -34.234046936035156,
-      "logps/rejected": -31.837631225585938,
-      "loss": 0.8487,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.11448518931865692,
-      "rewards/margins": 0.16089434921741486,
-      "rewards/rejected": -0.04640916362404823,
+      "logits/chosen": -1.8608484268188477,
+      "logits/rejected": -1.8584181070327759,
+      "logps/chosen": -34.17797088623047,
+      "logps/rejected": -31.830347061157227,
+      "loss": 0.6124,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.1757020801305771,
+      "rewards/margins": 0.22291450202465057,
+      "rewards/rejected": -0.047212425619363785,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9570705890655518,
-      "logits/rejected": -1.9465986490249634,
-      "logps/chosen": -35.030006408691406,
-      "logps/rejected": -31.88030433654785,
-      "loss": 0.7553,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.22864773869514465,
-      "rewards/margins": 0.2561652660369873,
-      "rewards/rejected": -0.02751758135855198,
+      "logits/chosen": -1.9641139507293701,
+      "logits/rejected": -1.9535942077636719,
+      "logps/chosen": -35.01939010620117,
+      "logps/rejected": -31.871440887451172,
+      "loss": 0.5852,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.26980119943618774,
+      "rewards/margins": 0.29416200518608093,
+      "rewards/rejected": -0.024360809475183487,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0522685050964355,
-      "logits/rejected": -2.0373730659484863,
-      "logps/chosen": -30.7352352142334,
-      "logps/rejected": -32.61699676513672,
-      "loss": 0.9072,
+      "logits/chosen": -2.0593204498291016,
+      "logits/rejected": -2.0443997383117676,
+      "logps/chosen": -30.722980499267578,
+      "logps/rejected": -32.61235809326172,
+      "loss": 0.6599,
       "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.13222160935401917,
-      "rewards/margins": 0.10783363878726959,
-      "rewards/rejected": 0.024387964978814125,
+      "rewards/chosen": 0.16091887652873993,
+      "rewards/margins": 0.12933868169784546,
+      "rewards/rejected": 0.031580209732055664,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9228973388671875,
-      "logits/rejected": -1.9203764200210571,
-      "logps/chosen": -32.44710159301758,
-      "logps/rejected": -30.934436798095703,
-      "loss": 0.6723,
+      "logits/chosen": -1.930450201034546,
+      "logits/rejected": -1.9279005527496338,
+      "logps/chosen": -32.415870666503906,
+      "logps/rejected": -30.882410049438477,
+      "loss": 0.5385,
       "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.3360074460506439,
-      "rewards/margins": 0.418344646692276,
-      "rewards/rejected": -0.08233721554279327,
+      "rewards/chosen": 0.4089924395084381,
+      "rewards/margins": 0.4614754319190979,
+      "rewards/rejected": -0.05248301103711128,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2234153747558594,
-      "eval_logits/rejected": -2.218602180480957,
-      "eval_logps/chosen": -34.08680725097656,
-      "eval_logps/rejected": -37.60466003417969,
-      "eval_loss": 0.976102888584137,
-      "eval_rewards/accuracies": 0.529900312423706,
-      "eval_rewards/chosen": -0.036580219864845276,
-      "eval_rewards/margins": 0.02504708059132099,
-      "eval_rewards/rejected": -0.06162729859352112,
-      "eval_runtime": 145.7665,
-      "eval_samples_per_second": 2.353,
+      "eval_logits/chosen": -2.229433536529541,
+      "eval_logits/rejected": -2.22458553314209,
+      "eval_logps/chosen": -34.090904235839844,
+      "eval_logps/rejected": -37.59726333618164,
+      "eval_loss": 0.69569993019104,
+      "eval_rewards/accuracies": 0.5398671627044678,
+      "eval_rewards/chosen": -0.04508008435368538,
+      "eval_rewards/margins": 0.019436603412032127,
+      "eval_rewards/rejected": -0.06451668590307236,
+      "eval_runtime": 145.8403,
+      "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.9082481861114502,
-      "logits/rejected": -1.9050118923187256,
-      "logps/chosen": -31.349285125732422,
-      "logps/rejected": -33.84658432006836,
-      "loss": 0.7796,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.17784307897090912,
-      "rewards/margins": 0.2509470283985138,
-      "rewards/rejected": -0.07310393452644348,
+      "logits/chosen": -1.9148633480072021,
+      "logits/rejected": -1.9115928411483765,
+      "logps/chosen": -31.324920654296875,
+      "logps/rejected": -33.81542205810547,
+      "loss": 0.5949,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.22274336218833923,
+      "rewards/margins": 0.28135946393013,
+      "rewards/rejected": -0.05861610919237137,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.9580894708633423,
-      "logits/rejected": -1.9458973407745361,
-      "logps/chosen": -34.3031005859375,
-      "logps/rejected": -33.67659378051758,
-      "loss": 0.7302,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.2050826996564865,
-      "rewards/margins": 0.3015114367008209,
-      "rewards/rejected": -0.09642868489027023,
+      "logits/chosen": -1.9647932052612305,
+      "logits/rejected": -1.9525552988052368,
+      "logps/chosen": -34.34864044189453,
+      "logps/rejected": -33.66791915893555,
+      "loss": 0.5831,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.19794727861881256,
+      "rewards/margins": 0.3012133836746216,
+      "rewards/rejected": -0.10326610505580902,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -1.9932842254638672,
-      "logits/rejected": -1.9918495416641235,
-      "logps/chosen": -33.17847442626953,
-      "logps/rejected": -32.54157638549805,
-      "loss": 0.7677,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.21512338519096375,
-      "rewards/margins": 0.2585209906101227,
-      "rewards/rejected": -0.04339758679270744,
+      "logits/chosen": -2.0001468658447266,
+      "logits/rejected": -1.9986999034881592,
+      "logps/chosen": -33.18779373168945,
+      "logps/rejected": -32.54129409790039,
+      "loss": 0.5895,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.23840396106243134,
+      "rewards/margins": 0.28777408599853516,
+      "rewards/rejected": -0.04937009885907173,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.0800719261169434,
-      "logits/rejected": -2.064396381378174,
-      "logps/chosen": -33.80484390258789,
-      "logps/rejected": -33.1123046875,
-      "loss": 0.7636,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.2621825039386749,
-      "rewards/margins": 0.25817227363586426,
-      "rewards/rejected": 0.004010227043181658,
+      "logits/chosen": -2.0865254402160645,
+      "logits/rejected": -2.0707924365997314,
+      "logps/chosen": -33.81252670288086,
+      "logps/rejected": -33.110015869140625,
+      "loss": 0.5883,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.2934878468513489,
+      "rewards/margins": 0.28707200288772583,
+      "rewards/rejected": 0.006415897514671087,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.9522559642791748,
-      "logits/rejected": -1.951424241065979,
-      "logps/chosen": -32.8499755859375,
-      "logps/rejected": -32.56407165527344,
-      "loss": 0.6881,
+      "logits/chosen": -1.9590953588485718,
+      "logits/rejected": -1.9582574367523193,
+      "logps/chosen": -32.849937438964844,
+      "logps/rejected": -32.53525161743164,
+      "loss": 0.5603,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.3024839758872986,
-      "rewards/margins": 0.3717316687107086,
-      "rewards/rejected": -0.06924761831760406,
+      "rewards/chosen": 0.34572547674179077,
+      "rewards/margins": 0.4018074870109558,
+      "rewards/rejected": -0.056082069873809814,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9075695276260376,
-      "logits/rejected": -1.917851209640503,
-      "logps/chosen": -31.882221221923828,
-      "logps/rejected": -35.31555938720703,
-      "loss": 0.7689,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.22572879493236542,
-      "rewards/margins": 0.24449090659618378,
-      "rewards/rejected": -0.018762132152915,
+      "logits/chosen": -1.914807677268982,
+      "logits/rejected": -1.9251015186309814,
+      "logps/chosen": -31.8874454498291,
+      "logps/rejected": -35.34430694580078,
+      "loss": 0.588,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.2537948489189148,
+      "rewards/margins": 0.2982342541217804,
+      "rewards/rejected": -0.0444394052028656,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.0478641986846924,
-      "logits/rejected": -2.041414737701416,
-      "logps/chosen": -33.331912994384766,
-      "logps/rejected": -29.259756088256836,
-      "loss": 0.7658,
+      "logits/chosen": -2.054311513900757,
+      "logits/rejected": -2.047823429107666,
+      "logps/chosen": -33.35334396362305,
+      "logps/rejected": -29.280254364013672,
+      "loss": 0.5847,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.2240387201309204,
-      "rewards/margins": 0.2473684549331665,
-      "rewards/rejected": -0.02332974039018154,
+      "rewards/chosen": 0.23889848589897156,
+      "rewards/margins": 0.2819606363773346,
+      "rewards/rejected": -0.04306213930249214,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.9081246852874756,
-      "logits/rejected": -1.9103105068206787,
-      "logps/chosen": -33.882568359375,
-      "logps/rejected": -30.96805191040039,
-      "loss": 0.741,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.24953576922416687,
-      "rewards/margins": 0.3163323998451233,
-      "rewards/rejected": -0.06679664552211761,
+      "logits/chosen": -1.9139082431793213,
+      "logits/rejected": -1.9161239862442017,
+      "logps/chosen": -33.855018615722656,
+      "logps/rejected": -30.981037139892578,
+      "loss": 0.5466,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.3072236478328705,
+      "rewards/margins": 0.3939489424228668,
+      "rewards/rejected": -0.08672530204057693,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.8438688600218142,
-      "train_runtime": 3250.9917,
-      "train_samples_per_second": 0.947,
+      "train_loss": 0.6230236078237559,
+      "train_runtime": 3254.2307,
+      "train_samples_per_second": 0.946,
       "train_steps_per_second": 0.118
     }
   ],