Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4947
-- Rewards/chosen: 0.1656
-- Rewards/rejected: 0.1437
-- Rewards/accuracies: 0.5365
-- Rewards/margins: 0.0219
-- Logps/rejected: -37.2771
-- Logps/chosen: -33.7585
-- Logits/rejected: -2.2259
-- Logits/chosen: -2.2307
+- Loss: 0.7655
+- Rewards/chosen: -0.0166
+- Rewards/rejected: -0.0295
+- Rewards/accuracies: 0.5071
+- Rewards/margins: 0.0129
+- Logps/rejected: -37.5657
+- Logps/chosen: -34.0622
+- Logits/rejected: -2.2285
+- Logits/chosen: -2.2333
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4799        | 0.26  | 100  | 0.4987          | 0.0790         | 0.0734           | 0.5104             | 0.0056          | -37.3943       | -33.9029     | -2.2301         | -2.2349       |
-| 0.4548        | 0.52  | 200  | 0.4956          | 0.1590         | 0.1392           | 0.5341             | 0.0198          | -37.2846       | -33.7696     | -2.2287         | -2.2335       |
-| 0.41          | 0.78  | 300  | 0.4937          | 0.1639         | 0.1391           | 0.5361             | 0.0248          | -37.2848       | -33.7614     | -2.2261         | -2.2309       |
+| 0.5903        | 0.26  | 100  | 0.7450          | 0.0123         | 0.0047           | 0.5241             | 0.0075          | -37.5087       | -34.0141     | -2.2318         | -2.2366       |
+| 0.59          | 0.52  | 200  | 0.7500          | -0.0208        | -0.0352          | 0.5365             | 0.0144          | -37.5752       | -34.0691     | -2.2284         | -2.2332       |
+| 0.3871        | 0.78  | 300  | 0.7655          | -0.0166        | -0.0295          | 0.5071             | 0.0129          | -37.5657       | -34.0622     | -2.2285         | -2.2333       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29d81ccb9aee98dd50644ca8a0d7a00dd0b18fd386da537748fc75ae8a58b388
+oid sha256:61fdd9123d70c8ba17ec6ad03dd0e97d84b7e40a5f415a1d5e1a038d4ac398a0
 size 176183216

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.230694055557251,
-    "eval_logits/rejected": -2.225895643234253,
-    "eval_logps/chosen": -33.75852966308594,
-    "eval_logps/rejected": -37.27712631225586,
-    "eval_loss": 0.4946732223033905,
-    "eval_rewards/accuracies": 0.5365448594093323,
-    "eval_rewards/chosen": 0.16561271250247955,
-    "eval_rewards/margins": 0.02191758342087269,
-    "eval_rewards/rejected": 0.14369513094425201,
-    "eval_runtime": 145.7007,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.354,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.46439293204963983,
-    "train_runtime": 3253.2697,
+    "train_loss": 0.5965218513042896,
+    "train_runtime": 3252.2803,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.46439293204963983,
-    "train_runtime": 3253.2697,
+    "train_loss": 0.5965218513042896,
+    "train_runtime": 3252.2803,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.5,
+      "loss": 0.6944,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,467 +25,467 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.866639494895935,
-      "logits/rejected": -1.870951533317566,
-      "logps/chosen": -36.991004943847656,
-      "logps/rejected": -33.64551544189453,
-      "loss": 0.4968,
-      "rewards/accuracies": 0.5694444179534912,
-      "rewards/chosen": 0.009355764836072922,
-      "rewards/margins": 0.018620232120156288,
-      "rewards/rejected": -0.009264465421438217,
+      "logits/chosen": -1.8664441108703613,
+      "logits/rejected": -1.8707667589187622,
+      "logps/chosen": -36.99871826171875,
+      "logps/rejected": -33.66032028198242,
+      "loss": 0.6698,
+      "rewards/accuracies": 0.5138888955116272,
+      "rewards/chosen": 0.0047301375307142735,
+      "rewards/margins": 0.022878725081682205,
+      "rewards/rejected": -0.018148591741919518,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9975944757461548,
-      "logits/rejected": -2.000239133834839,
-      "logps/chosen": -29.655420303344727,
-      "logps/rejected": -29.04937171936035,
-      "loss": 0.5034,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.007936288602650166,
-      "rewards/margins": -0.016377601772546768,
-      "rewards/rejected": 0.008441311307251453,
+      "logits/chosen": -1.9972795248031616,
+      "logits/rejected": -1.999922513961792,
+      "logps/chosen": -29.64887046813965,
+      "logps/rejected": -29.054330825805664,
+      "loss": 0.756,
+      "rewards/accuracies": 0.38749998807907104,
+      "rewards/chosen": -0.0040070666000247,
+      "rewards/margins": -0.009474089369177818,
+      "rewards/rejected": 0.0054670218378305435,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.920345664024353,
-      "logits/rejected": -1.917649507522583,
-      "logps/chosen": -31.40158462524414,
-      "logps/rejected": -33.22117614746094,
-      "loss": 0.499,
+      "logits/chosen": -1.9204769134521484,
+      "logits/rejected": -1.9177768230438232,
+      "logps/chosen": -31.404077529907227,
+      "logps/rejected": -33.22195053100586,
+      "loss": 0.7112,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.008706120774149895,
-      "rewards/margins": 0.010335424914956093,
-      "rewards/rejected": -0.0016293060034513474,
+      "rewards/chosen": 0.007212291471660137,
+      "rewards/margins": 0.009304212406277657,
+      "rewards/rejected": -0.0020919169764965773,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.016953706741333,
-      "logits/rejected": -2.008241891860962,
-      "logps/chosen": -32.55894088745117,
-      "logps/rejected": -32.48887634277344,
-      "loss": 0.501,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.010718915611505508,
-      "rewards/margins": -0.0035588580649346113,
-      "rewards/rejected": 0.014277773909270763,
+      "logits/chosen": -2.0172927379608154,
+      "logits/rejected": -2.008545398712158,
+      "logps/chosen": -32.57653045654297,
+      "logps/rejected": -32.520172119140625,
+      "loss": 0.7334,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.00016416943981312215,
+      "rewards/margins": 0.004665405489504337,
+      "rewards/rejected": -0.004501235671341419,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8621622323989868,
-      "logits/rejected": -1.851413369178772,
-      "logps/chosen": -33.5220832824707,
-      "logps/rejected": -35.4012565612793,
-      "loss": 0.5034,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.021240826696157455,
-      "rewards/margins": -0.009411826729774475,
-      "rewards/rejected": 0.03065265715122223,
+      "logits/chosen": -1.8620569705963135,
+      "logits/rejected": -1.8512712717056274,
+      "logps/chosen": -33.55234146118164,
+      "logps/rejected": -35.454689025878906,
+      "loss": 0.7458,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.003082751529291272,
+      "rewards/margins": 0.004490534774959087,
+      "rewards/rejected": -0.0014077809173613787,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9412288665771484,
-      "logits/rejected": -1.9431712627410889,
-      "logps/chosen": -32.506317138671875,
-      "logps/rejected": -33.202842712402344,
-      "loss": 0.4846,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.05631808191537857,
-      "rewards/margins": 0.07519452273845673,
-      "rewards/rejected": -0.018876442685723305,
+      "logits/chosen": -1.940212607383728,
+      "logits/rejected": -1.9421584606170654,
+      "logps/chosen": -32.56269454956055,
+      "logps/rejected": -33.21684265136719,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.022492902353405952,
+      "rewards/margins": 0.04977109283208847,
+      "rewards/rejected": -0.027278194203972816,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.0726475715637207,
-      "logits/rejected": -2.077623128890991,
-      "logps/chosen": -33.90425491333008,
-      "logps/rejected": -36.56460189819336,
-      "loss": 0.4903,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.05054063722491264,
-      "rewards/margins": 0.04797897860407829,
-      "rewards/rejected": 0.002561659086495638,
+      "logits/chosen": -2.0728747844696045,
+      "logits/rejected": -2.0778398513793945,
+      "logps/chosen": -33.98863983154297,
+      "logps/rejected": -36.62779235839844,
+      "loss": 0.7641,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -9.316801879322156e-05,
+      "rewards/margins": 0.035260457545518875,
+      "rewards/rejected": -0.035353630781173706,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9338359832763672,
-      "logits/rejected": -1.9369510412216187,
-      "logps/chosen": -34.21551513671875,
-      "logps/rejected": -34.527488708496094,
-      "loss": 0.4797,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.1273767203092575,
-      "rewards/margins": 0.08842838555574417,
-      "rewards/rejected": 0.038948334753513336,
+      "logits/chosen": -1.935132384300232,
+      "logits/rejected": -1.9382661581039429,
+      "logps/chosen": -34.321903228759766,
+      "logps/rejected": -34.638282775878906,
+      "loss": 0.6095,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.06354494392871857,
+      "rewards/margins": 0.09106875956058502,
+      "rewards/rejected": -0.027523810043931007,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9420888423919678,
-      "logits/rejected": -1.9465935230255127,
-      "logps/chosen": -32.2774772644043,
-      "logps/rejected": -32.245147705078125,
-      "loss": 0.4887,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.11396588385105133,
-      "rewards/margins": 0.04570900648832321,
-      "rewards/rejected": 0.06825686991214752,
+      "logits/chosen": -1.9444854259490967,
+      "logits/rejected": -1.9490203857421875,
+      "logps/chosen": -32.41173553466797,
+      "logps/rejected": -32.337852478027344,
+      "loss": 0.7499,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.033411428332328796,
+      "rewards/margins": 0.020780622959136963,
+      "rewards/rejected": 0.012630810961127281,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.040524482727051,
-      "logits/rejected": -2.0385289192199707,
-      "logps/chosen": -32.03118133544922,
-      "logps/rejected": -31.205692291259766,
-      "loss": 0.4799,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.12579473853111267,
-      "rewards/margins": 0.08948288857936859,
-      "rewards/rejected": 0.036311857402324677,
+      "logits/chosen": -2.042165994644165,
+      "logits/rejected": -2.0401768684387207,
+      "logps/chosen": -32.13690185546875,
+      "logps/rejected": -31.2872314453125,
+      "loss": 0.5903,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.06236031651496887,
+      "rewards/margins": 0.0749734491109848,
+      "rewards/rejected": -0.012613129802048206,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.234907865524292,
-      "eval_logits/rejected": -2.2300779819488525,
-      "eval_logps/chosen": -33.90293884277344,
-      "eval_logps/rejected": -37.394290924072266,
-      "eval_loss": 0.4987158477306366,
-      "eval_rewards/accuracies": 0.5103820562362671,
-      "eval_rewards/chosen": 0.0789671465754509,
-      "eval_rewards/margins": 0.005569713190197945,
-      "eval_rewards/rejected": 0.0733974277973175,
-      "eval_runtime": 145.9867,
-      "eval_samples_per_second": 2.35,
+      "eval_logits/chosen": -2.2366247177124023,
+      "eval_logits/rejected": -2.231769323348999,
+      "eval_logps/chosen": -34.01411819458008,
+      "eval_logps/rejected": -37.50873947143555,
+      "eval_loss": 0.7449910640716553,
+      "eval_rewards/accuracies": 0.5240863561630249,
+      "eval_rewards/chosen": 0.01226096786558628,
+      "eval_rewards/margins": 0.007533104624599218,
+      "eval_rewards/rejected": 0.0047278618440032005,
+      "eval_runtime": 145.783,
+      "eval_samples_per_second": 2.353,
       "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9966247081756592,
-      "logits/rejected": -1.9942585229873657,
-      "logps/chosen": -33.005943298339844,
-      "logps/rejected": -33.908599853515625,
-      "loss": 0.4799,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.14272192120552063,
-      "rewards/margins": 0.07025818526744843,
-      "rewards/rejected": 0.07246376574039459,
+      "logits/chosen": -1.9979403018951416,
+      "logits/rejected": -1.9955459833145142,
+      "logps/chosen": -33.12257766723633,
+      "logps/rejected": -33.99717712402344,
+      "loss": 0.8547,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.07274619489908218,
+      "rewards/margins": 0.05342693254351616,
+      "rewards/rejected": 0.019319258630275726,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.0064611434936523,
-      "logits/rejected": -1.9981132745742798,
-      "logps/chosen": -32.16605758666992,
-      "logps/rejected": -31.968135833740234,
-      "loss": 0.4883,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.16730833053588867,
-      "rewards/margins": 0.0512862391769886,
-      "rewards/rejected": 0.11602209508419037,
+      "logits/chosen": -2.009084463119507,
+      "logits/rejected": -2.0007264614105225,
+      "logps/chosen": -32.33968734741211,
+      "logps/rejected": -32.13513946533203,
+      "loss": 0.7104,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.06312941759824753,
+      "rewards/margins": 0.047309644520282745,
+      "rewards/rejected": 0.015819773077964783,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0353620052337646,
-      "logits/rejected": -2.0273985862731934,
-      "logps/chosen": -30.145355224609375,
-      "logps/rejected": -31.892078399658203,
-      "loss": 0.477,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.19821622967720032,
-      "rewards/margins": 0.10165951400995255,
-      "rewards/rejected": 0.09655670821666718,
+      "logits/chosen": -2.0365753173828125,
+      "logits/rejected": -2.028564453125,
+      "logps/chosen": -30.363454818725586,
+      "logps/rejected": -32.06770706176758,
+      "loss": 0.71,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.06735612452030182,
+      "rewards/margins": 0.07617705315351486,
+      "rewards/rejected": -0.008820934221148491,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.965121865272522,
-      "logits/rejected": -1.9753497838974,
-      "logps/chosen": -31.059722900390625,
-      "logps/rejected": -32.391807556152344,
-      "loss": 0.4695,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.21596479415893555,
-      "rewards/margins": 0.13012099266052246,
-      "rewards/rejected": 0.08584379404783249,
+      "logits/chosen": -1.9657623767852783,
+      "logits/rejected": -1.976011872291565,
+      "logps/chosen": -31.213130950927734,
+      "logps/rejected": -32.592655181884766,
+      "loss": 0.6309,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.12391922622919083,
+      "rewards/margins": 0.1585826873779297,
+      "rewards/rejected": -0.03466347977519035,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8772761821746826,
-      "logits/rejected": -1.8784434795379639,
-      "logps/chosen": -33.682029724121094,
-      "logps/rejected": -34.56599044799805,
-      "loss": 0.4566,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.30639737844467163,
-      "rewards/margins": 0.19163966178894043,
-      "rewards/rejected": 0.11475770175457001,
+      "logits/chosen": -1.8778575658798218,
+      "logits/rejected": -1.879020094871521,
+      "logps/chosen": -33.998287200927734,
+      "logps/rejected": -34.83546829223633,
+      "loss": 0.566,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.11664185672998428,
+      "rewards/margins": 0.1635705530643463,
+      "rewards/rejected": -0.04692870378494263,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9283088445663452,
-      "logits/rejected": -1.9249660968780518,
-      "logps/chosen": -35.78284454345703,
-      "logps/rejected": -32.46989059448242,
-      "loss": 0.4802,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.23330166935920715,
-      "rewards/margins": 0.08158595860004425,
-      "rewards/rejected": 0.1517156958580017,
+      "logits/chosen": -1.9302040338516235,
+      "logits/rejected": -1.9267680644989014,
+      "logps/chosen": -36.0306510925293,
+      "logps/rejected": -32.73199462890625,
+      "loss": 0.5795,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.08461670577526093,
+      "rewards/margins": 0.09016285836696625,
+      "rewards/rejected": -0.005546141881495714,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0310397148132324,
-      "logits/rejected": -2.0237185955047607,
-      "logps/chosen": -33.21892547607422,
-      "logps/rejected": -31.1767520904541,
-      "loss": 0.443,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.347359299659729,
-      "rewards/margins": 0.24229975044727325,
-      "rewards/rejected": 0.10505956411361694,
+      "logits/chosen": -2.031188726425171,
+      "logits/rejected": -2.0238261222839355,
+      "logps/chosen": -33.55583953857422,
+      "logps/rejected": -31.381237030029297,
+      "loss": 0.5041,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.14521491527557373,
+      "rewards/margins": 0.16284644603729248,
+      "rewards/rejected": -0.017631549388170242,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.037632703781128,
-      "logits/rejected": -2.0428080558776855,
-      "logps/chosen": -31.97260093688965,
-      "logps/rejected": -32.14643096923828,
-      "loss": 0.4662,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.34331104159355164,
-      "rewards/margins": 0.1400861144065857,
-      "rewards/rejected": 0.20322492718696594,
+      "logits/chosen": -2.037135362625122,
+      "logits/rejected": -2.0424118041992188,
+      "logps/chosen": -32.33055114746094,
+      "logps/rejected": -32.500144958496094,
+      "loss": 0.5257,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.12854143977165222,
+      "rewards/margins": 0.13754449784755707,
+      "rewards/rejected": -0.009003056213259697,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.0373456478118896,
-      "logits/rejected": -2.0346312522888184,
-      "logps/chosen": -31.077922821044922,
-      "logps/rejected": -31.089244842529297,
-      "loss": 0.4725,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.24661760032176971,
-      "rewards/margins": 0.10923590511083603,
-      "rewards/rejected": 0.13738170266151428,
+      "logits/chosen": -2.0384888648986816,
+      "logits/rejected": -2.0357367992401123,
+      "logps/chosen": -31.326950073242188,
+      "logps/rejected": -31.338672637939453,
+      "loss": 0.6504,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.09720095247030258,
+      "rewards/margins": 0.10947537422180176,
+      "rewards/rejected": -0.012274429202079773,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9075931310653687,
-      "logits/rejected": -1.9122419357299805,
-      "logps/chosen": -31.088512420654297,
-      "logps/rejected": -32.591243743896484,
-      "loss": 0.4548,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.3106623888015747,
-      "rewards/margins": 0.190850168466568,
-      "rewards/rejected": 0.11981221288442612,
+      "logits/chosen": -1.908755898475647,
+      "logits/rejected": -1.9134235382080078,
+      "logps/chosen": -31.36275863647461,
+      "logps/rejected": -32.82594680786133,
+      "loss": 0.59,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.14611336588859558,
+      "rewards/margins": 0.16712167859077454,
+      "rewards/rejected": -0.02100830152630806,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.233458995819092,
-      "eval_logits/rejected": -2.2286527156829834,
-      "eval_logps/chosen": -33.769569396972656,
-      "eval_logps/rejected": -37.28458786010742,
-      "eval_loss": 0.4955619275569916,
-      "eval_rewards/accuracies": 0.5340532064437866,
-      "eval_rewards/chosen": 0.15899108350276947,
-      "eval_rewards/margins": 0.019772635772824287,
-      "eval_rewards/rejected": 0.13921843469142914,
-      "eval_runtime": 145.6335,
-      "eval_samples_per_second": 2.355,
+      "eval_logits/chosen": -2.233200788497925,
+      "eval_logits/rejected": -2.228351354598999,
+      "eval_logps/chosen": -34.069149017333984,
+      "eval_logps/rejected": -37.57521057128906,
+      "eval_loss": 0.750046968460083,
+      "eval_rewards/accuracies": 0.5365448594093323,
+      "eval_rewards/chosen": -0.020758595317602158,
+      "eval_rewards/margins": 0.014396170154213905,
+      "eval_rewards/rejected": -0.035154763609170914,
+      "eval_runtime": 145.6956,
+      "eval_samples_per_second": 2.354,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.020599365234375,
-      "logits/rejected": -2.0312187671661377,
-      "logps/chosen": -31.51775550842285,
-      "logps/rejected": -33.68301773071289,
-      "loss": 0.462,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.2721289098262787,
-      "rewards/margins": 0.16431835293769836,
-      "rewards/rejected": 0.10781057924032211,
+      "logits/chosen": -2.0207877159118652,
+      "logits/rejected": -2.0314865112304688,
+      "logps/chosen": -31.7946834564209,
+      "logps/rejected": -33.94416046142578,
+      "loss": 0.544,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.10597337782382965,
+      "rewards/margins": 0.1548517644405365,
+      "rewards/rejected": -0.04887838289141655,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9131078720092773,
-      "logits/rejected": -1.9278091192245483,
-      "logps/chosen": -29.563827514648438,
-      "logps/rejected": -31.399028778076172,
-      "loss": 0.4463,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.3286263942718506,
-      "rewards/margins": 0.23313641548156738,
-      "rewards/rejected": 0.09548995643854141,
+      "logits/chosen": -1.9139292240142822,
+      "logits/rejected": -1.9287306070327759,
+      "logps/chosen": -29.91892433166504,
+      "logps/rejected": -31.5737247467041,
+      "loss": 0.5694,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.11556863784790039,
+      "rewards/margins": 0.12489677965641022,
+      "rewards/rejected": -0.00932813435792923,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.969555139541626,
-      "logits/rejected": -1.9735543727874756,
-      "logps/chosen": -32.82552719116211,
-      "logps/rejected": -31.39508628845215,
-      "loss": 0.4396,
+      "logits/chosen": -1.9701712131500244,
+      "logits/rejected": -1.974172592163086,
+      "logps/chosen": -33.17791748046875,
+      "logps/rejected": -31.603836059570312,
+      "loss": 0.4933,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.3551639914512634,
-      "rewards/margins": 0.26263612508773804,
-      "rewards/rejected": 0.0925278514623642,
+      "rewards/chosen": 0.1437317281961441,
+      "rewards/margins": 0.1764504611492157,
+      "rewards/rejected": -0.03271873667836189,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9689340591430664,
-      "logits/rejected": -1.947165846824646,
-      "logps/chosen": -33.572689056396484,
-      "logps/rejected": -34.84184265136719,
-      "loss": 0.4355,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.3486354947090149,
-      "rewards/margins": 0.2832663357257843,
-      "rewards/rejected": 0.0653691440820694,
+      "logits/chosen": -1.9691722393035889,
+      "logits/rejected": -1.9472358226776123,
+      "logps/chosen": -33.92795181274414,
+      "logps/rejected": -35.06544876098633,
+      "loss": 0.4966,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.13547900319099426,
+      "rewards/margins": 0.20427203178405762,
+      "rewards/rejected": -0.06879300624132156,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.0102758407592773,
-      "logits/rejected": -2.006990909576416,
-      "logps/chosen": -32.49989700317383,
-      "logps/rejected": -35.99126434326172,
-      "loss": 0.472,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.2551037669181824,
-      "rewards/margins": 0.11490776389837265,
-      "rewards/rejected": 0.1401960253715515,
+      "logits/chosen": -2.010617971420288,
+      "logits/rejected": -2.0072851181030273,
+      "logps/chosen": -32.785011291503906,
+      "logps/rejected": -36.283348083496094,
+      "loss": 0.5823,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.08403309434652328,
+      "rewards/margins": 0.11908762156963348,
+      "rewards/rejected": -0.035054538398981094,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.877345323562622,
-      "logits/rejected": -1.8749182224273682,
-      "logps/chosen": -33.7043571472168,
-      "logps/rejected": -35.259422302246094,
-      "loss": 0.4671,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.2904246747493744,
-      "rewards/margins": 0.14330394566059113,
-      "rewards/rejected": 0.14712072908878326,
+      "logits/chosen": -1.8787453174591064,
+      "logits/rejected": -1.8763244152069092,
+      "logps/chosen": -34.000633239746094,
+      "logps/rejected": -35.53338623046875,
+      "loss": 0.5277,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.11265683174133301,
+      "rewards/margins": 0.1299169510602951,
+      "rewards/rejected": -0.017260130494832993,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8626378774642944,
-      "logits/rejected": -1.8601328134536743,
-      "logps/chosen": -33.90363311767578,
-      "logps/rejected": -31.588598251342773,
-      "loss": 0.457,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.29638105630874634,
-      "rewards/margins": 0.1867421716451645,
-      "rewards/rejected": 0.10963888466358185,
+      "logits/chosen": -1.864057183265686,
+      "logits/rejected": -1.861581802368164,
+      "logps/chosen": -34.23812484741211,
+      "logps/rejected": -31.777530670166016,
+      "loss": 0.6307,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.09568403661251068,
+      "rewards/margins": 0.09940369427204132,
+      "rewards/rejected": -0.003719669533893466,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.965528130531311,
-      "logits/rejected": -1.9551260471343994,
-      "logps/chosen": -34.74420166015625,
-      "logps/rejected": -31.622207641601562,
-      "loss": 0.4442,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.3674633204936981,
-      "rewards/margins": 0.23619195818901062,
-      "rewards/rejected": 0.1312713772058487,
+      "logits/chosen": -1.9680726528167725,
+      "logits/rejected": -1.957493782043457,
+      "logps/chosen": -35.05353546142578,
+      "logps/rejected": -31.8691463470459,
+      "loss": 0.4252,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.18186399340629578,
+      "rewards/margins": 0.19875726103782654,
+      "rewards/rejected": -0.01689324900507927,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0614726543426514,
-      "logits/rejected": -2.0466229915618896,
-      "logps/chosen": -30.37356948852539,
-      "logps/rejected": -32.3351936340332,
-      "loss": 0.4671,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.33033472299575806,
-      "rewards/margins": 0.14034779369831085,
-      "rewards/rejected": 0.1899869292974472,
+      "logits/chosen": -2.0636587142944336,
+      "logits/rejected": -2.048659563064575,
+      "logps/chosen": -30.76055908203125,
+      "logps/rejected": -32.65589141845703,
+      "loss": 0.6667,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.09814020991325378,
+      "rewards/margins": 0.10057313740253448,
+      "rewards/rejected": -0.0024329207371920347,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9327377080917358,
-      "logits/rejected": -1.9302759170532227,
-      "logps/chosen": -32.0928955078125,
-      "logps/rejected": -30.652236938476562,
-      "loss": 0.41,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.5005279779434204,
-      "rewards/margins": 0.4017825126647949,
-      "rewards/rejected": 0.09874548017978668,
+      "logits/chosen": -1.9355440139770508,
+      "logits/rejected": -1.9329853057861328,
+      "logps/chosen": -32.57345199584961,
+      "logps/rejected": -30.86557388305664,
+      "loss": 0.3871,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.21219508349895477,
+      "rewards/margins": 0.24145305156707764,
+      "rewards/rejected": -0.029257934540510178,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2308623790740967,
-      "eval_logits/rejected": -2.226059913635254,
-      "eval_logps/chosen": -33.76142501831055,
-      "eval_logps/rejected": -37.284828186035156,
-      "eval_loss": 0.4937135577201843,
-      "eval_rewards/accuracies": 0.5361295938491821,
-      "eval_rewards/chosen": 0.16387464106082916,
-      "eval_rewards/margins": 0.02479831501841545,
-      "eval_rewards/rejected": 0.1390763521194458,
-      "eval_runtime": 145.6766,
+      "eval_logits/chosen": -2.233344554901123,
+      "eval_logits/rejected": -2.2284841537475586,
+      "eval_logps/chosen": -34.06221008300781,
+      "eval_logps/rejected": -37.56574249267578,
+      "eval_loss": 0.7655012607574463,
+      "eval_rewards/accuracies": 0.5070598125457764,
+      "eval_rewards/chosen": -0.016592998057603836,
+      "eval_rewards/margins": 0.012880506925284863,
+      "eval_rewards/rejected": -0.029473504051566124,
+      "eval_runtime": 145.6769,
       "eval_samples_per_second": 2.355,
       "eval_steps_per_second": 0.295,
       "step": 300
@@ -493,122 +493,122 @@
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.916757345199585,
-      "logits/rejected": -1.9136326313018799,
-      "logps/chosen": -31.0240421295166,
-      "logps/rejected": -33.553138732910156,
-      "loss": 0.4465,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.3475847542285919,
-      "rewards/margins": 0.23417536914348602,
-      "rewards/rejected": 0.1134093850851059,
+      "logits/chosen": -1.9186370372772217,
+      "logits/rejected": -1.9153635501861572,
+      "logps/chosen": -31.363632202148438,
+      "logps/rejected": -33.790061950683594,
+      "loss": 0.5232,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.1438317447900772,
+      "rewards/margins": 0.17257718741893768,
+      "rewards/rejected": -0.02874544821679592,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.9669981002807617,
-      "logits/rejected": -1.9548496007919312,
-      "logps/chosen": -34.02072525024414,
-      "logps/rejected": -33.40859603881836,
-      "loss": 0.4379,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.3452104926109314,
-      "rewards/margins": 0.26706355810165405,
-      "rewards/rejected": 0.07814691960811615,
+      "logits/chosen": -1.9695978164672852,
+      "logits/rejected": -1.9573185443878174,
+      "logps/chosen": -34.42280960083008,
+      "logps/rejected": -33.63968276977539,
+      "loss": 0.4864,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.103958860039711,
+      "rewards/margins": 0.1644643247127533,
+      "rewards/rejected": -0.06050548702478409,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -2.0023112297058105,
-      "logits/rejected": -2.0009615421295166,
-      "logps/chosen": -32.86361312866211,
-      "logps/rejected": -32.265541076660156,
-      "loss": 0.4439,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.37330904603004456,
-      "rewards/margins": 0.24488253891468048,
-      "rewards/rejected": 0.12842652201652527,
+      "logits/chosen": -2.0046210289001465,
+      "logits/rejected": -2.003176689147949,
+      "logps/chosen": -33.34724426269531,
+      "logps/rejected": -32.53419876098633,
+      "loss": 0.5611,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.08312927186489105,
+      "rewards/margins": 0.11589743942022324,
+      "rewards/rejected": -0.032768163830041885,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.0890090465545654,
-      "logits/rejected": -2.073373317718506,
-      "logps/chosen": -33.47917556762695,
-      "logps/rejected": -32.80946350097656,
-      "loss": 0.4446,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.42013058066368103,
-      "rewards/margins": 0.23498812317848206,
-      "rewards/rejected": 0.18514248728752136,
+      "logits/chosen": -2.091379404067993,
+      "logits/rejected": -2.0756208896636963,
+      "logps/chosen": -33.90237808227539,
+      "logps/rejected": -33.127288818359375,
+      "loss": 0.4859,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.16620711982250214,
+      "rewards/margins": 0.17176242172718048,
+      "rewards/rejected": -0.005555284209549427,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.9615952968597412,
-      "logits/rejected": -1.9607601165771484,
-      "logps/chosen": -32.57384490966797,
-      "logps/rejected": -32.25366973876953,
-      "loss": 0.4326,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.4249538779258728,
-      "rewards/margins": 0.2980697453022003,
-      "rewards/rejected": 0.12688413262367249,
+      "logits/chosen": -1.9639079570770264,
+      "logits/rejected": -1.9629888534545898,
+      "logps/chosen": -32.960304260253906,
+      "logps/rejected": -32.5081787109375,
+      "loss": 0.5048,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.1930757313966751,
+      "rewards/margins": 0.21889439225196838,
+      "rewards/rejected": -0.02581869065761566,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9177497625350952,
-      "logits/rejected": -1.9280321598052979,
-      "logps/chosen": -31.600200653076172,
-      "logps/rejected": -35.003135681152344,
-      "loss": 0.4555,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.3626941442489624,
-      "rewards/margins": 0.19132137298583984,
-      "rewards/rejected": 0.17137278616428375,
+      "logits/chosen": -1.9195839166641235,
+      "logits/rejected": -1.929896593093872,
+      "logps/chosen": -31.997264862060547,
+      "logps/rejected": -35.334999084472656,
+      "loss": 0.5364,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.12445466220378876,
+      "rewards/margins": 0.15219925343990326,
+      "rewards/rejected": -0.027744591236114502,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.0559232234954834,
-      "logits/rejected": -2.0493996143341064,
-      "logps/chosen": -33.01537322998047,
-      "logps/rejected": -28.992477416992188,
-      "loss": 0.4418,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.3819587230682373,
-      "rewards/margins": 0.2415906935930252,
-      "rewards/rejected": 0.14036807417869568,
+      "logits/chosen": -2.058486223220825,
+      "logits/rejected": -2.051971912384033,
+      "logps/chosen": -33.44487380981445,
+      "logps/rejected": -29.261219024658203,
+      "loss": 0.4692,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.12425780296325684,
+      "rewards/margins": 0.1451319307088852,
+      "rewards/rejected": -0.020874138921499252,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.9163579940795898,
-      "logits/rejected": -1.9185268878936768,
-      "logps/chosen": -33.650962829589844,
-      "logps/rejected": -30.753406524658203,
-      "loss": 0.4369,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.3528522849082947,
-      "rewards/margins": 0.28131863474845886,
-      "rewards/rejected": 0.07153360545635223,
+      "logits/chosen": -1.9182488918304443,
+      "logits/rejected": -1.9204342365264893,
+      "logps/chosen": -33.968048095703125,
+      "logps/rejected": -30.890918731689453,
+      "loss": 0.4679,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.16260387003421783,
+      "rewards/margins": 0.17357683181762695,
+      "rewards/rejected": -0.010972961783409119,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.46439293204963983,
-      "train_runtime": 3253.2697,
-      "train_samples_per_second": 0.946,
+      "train_loss": 0.5965218513042896,
+      "train_runtime": 3252.2803,
+      "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }
   ],