Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4934
-- Rewards/chosen: 0.2139
-- Rewards/rejected: 0.1872
-- Rewards/accuracies: 0.5457
-- Rewards/margins: 0.0267
-- Logps/rejected: -37.2826
-- Logps/chosen: -33.7672
-- Logits/rejected: -2.2262
-- Logits/chosen: -2.2310
+- Loss: 0.4464
+- Rewards/chosen: 0.0179
+- Rewards/rejected: -0.0124
+- Rewards/accuracies: 0.5428
+- Rewards/margins: 0.0304
+- Logps/rejected: -37.5321
+- Logps/chosen: -34.0121
+- Logits/rejected: -2.2303
+- Logits/chosen: -2.2352
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4749        | 0.26  | 100  | 0.4963          | 0.1467         | 0.1303           | 0.5336             | 0.0164          | -37.3537       | -33.8512     | -2.2327         | -2.2375       |
-| 0.4376        | 0.52  | 200  | 0.4956          | 0.1959         | 0.1769           | 0.5486             | 0.0191          | -37.2955       | -33.7896     | -2.2291         | -2.2339       |
-| 0.3835        | 0.78  | 300  | 0.4950          | 0.2045         | 0.1836           | 0.5245             | 0.0210          | -37.2872       | -33.7789     | -2.2264         | -2.2312       |
+| 0.3817        | 0.26  | 100  | 0.4287          | 0.0507         | 0.0294           | 0.5577             | 0.0213          | -37.4799       | -33.9712     | -2.2329         | -2.2378       |
+| 0.3991        | 0.52  | 200  | 0.4536          | -0.0010        | -0.0197          | 0.5374             | 0.0187          | -37.5412       | -34.0358     | -2.2300         | -2.2349       |
+| 0.2816        | 0.78  | 300  | 0.4464          | 0.0179         | -0.0124          | 0.5428             | 0.0304          | -37.5321       | -34.0121     | -2.2303         | -2.2352       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d5d4c9c7297ee45694cd2242c380ff348f1878684b2ee08ac91a39f76a9730d
+oid sha256:9323562170691969f33b21814d35e8b1eb7c2c35b3293abb03e7e74ca2e2815d
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2309982776641846,
-    "eval_logits/rejected": -2.2261931896209717,
-    "eval_logps/chosen": -33.767173767089844,
-    "eval_logps/rejected": -37.282623291015625,
-    "eval_loss": 0.49339157342910767,
-    "eval_rewards/accuracies": 0.5456810593605042,
-    "eval_rewards/chosen": 0.21390400826931,
-    "eval_rewards/margins": 0.026708098128437996,
-    "eval_rewards/rejected": 0.18719588220119476,
-    "eval_runtime": 145.5786,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.356,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.4538224170734356,
-    "train_runtime": 3252.427,
+    "train_loss": 0.3756082092012678,
+    "train_runtime": 3251.4223,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.4538224170734356,
-    "train_runtime": 3252.427,
+    "train_loss": 0.3756082092012678,
+    "train_runtime": 3251.4223,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.5,
+      "loss": 0.3906,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,589 +25,589 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.866452693939209,
-      "logits/rejected": -1.8707809448242188,
-      "logps/chosen": -36.98366165161133,
-      "logps/rejected": -33.666290283203125,
-      "loss": 0.4914,
-      "rewards/accuracies": 0.5555555820465088,
-      "rewards/chosen": 0.01834939420223236,
-      "rewards/margins": 0.04732292518019676,
-      "rewards/rejected": -0.02897353284060955,
+      "logits/chosen": -1.8662821054458618,
+      "logits/rejected": -1.8706018924713135,
+      "logps/chosen": -36.98260498046875,
+      "logps/rejected": -33.66376876831055,
+      "loss": 0.3501,
+      "rewards/accuracies": 0.5694444179534912,
+      "rewards/chosen": 0.01919599249958992,
+      "rewards/margins": 0.04614981636404991,
+      "rewards/rejected": -0.02695382386445999,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9971332550048828,
-      "logits/rejected": -1.999770164489746,
-      "logps/chosen": -29.6362247467041,
-      "logps/rejected": -29.044448852539062,
-      "loss": 0.5009,
+      "logits/chosen": -1.9981708526611328,
+      "logits/rejected": -2.000824451446533,
+      "logps/chosen": -29.6453914642334,
+      "logps/rejected": -29.056461334228516,
+      "loss": 0.4314,
       "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.004776014480739832,
-      "rewards/margins": -0.01042011845856905,
-      "rewards/rejected": 0.015196132473647594,
+      "rewards/chosen": -0.0025594178587198257,
+      "rewards/margins": -0.008148794062435627,
+      "rewards/rejected": 0.005589376203715801,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9208379983901978,
-      "logits/rejected": -1.9181442260742188,
-      "logps/chosen": -31.425155639648438,
-      "logps/rejected": -33.237945556640625,
-      "loss": 0.4997,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.007249836809933186,
-      "rewards/margins": 0.008341209962964058,
-      "rewards/rejected": -0.015591045841574669,
+      "logits/chosen": -1.9205211400985718,
+      "logits/rejected": -1.917824149131775,
+      "logps/chosen": -31.402332305908203,
+      "logps/rejected": -33.20569610595703,
+      "loss": 0.4366,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.011007689870893955,
+      "rewards/margins": 0.0007972270250320435,
+      "rewards/rejected": 0.010210464708507061,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.0180625915527344,
-      "logits/rejected": -2.009308338165283,
-      "logps/chosen": -32.559749603271484,
-      "logps/rejected": -32.524681091308594,
-      "loss": 0.4962,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.013646525330841541,
-      "rewards/margins": 0.023250887170433998,
-      "rewards/rejected": -0.009604359045624733,
+      "logits/chosen": -2.0170958042144775,
+      "logits/rejected": -2.0083703994750977,
+      "logps/chosen": -32.547943115234375,
+      "logps/rejected": -32.49810791015625,
+      "loss": 0.4286,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.023093106225132942,
+      "rewards/margins": 0.011441526934504509,
+      "rewards/rejected": 0.011651577427983284,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8639278411865234,
-      "logits/rejected": -1.8531490564346313,
-      "logps/chosen": -33.53788375854492,
-      "logps/rejected": -35.42852020263672,
-      "loss": 0.5015,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.015677569434046745,
-      "rewards/margins": -0.0033827773295342922,
-      "rewards/rejected": 0.019060343503952026,
+      "logits/chosen": -1.8613898754119873,
+      "logits/rejected": -1.8506320714950562,
+      "logps/chosen": -33.56399154663086,
+      "logps/rejected": -35.46310043334961,
+      "loss": 0.4498,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.005208463408052921,
+      "rewards/margins": 0.0033974028192460537,
+      "rewards/rejected": -0.008605867624282837,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9435577392578125,
-      "logits/rejected": -1.9454940557479858,
-      "logps/chosen": -32.49355697631836,
-      "logps/rejected": -33.1856575012207,
-      "loss": 0.4817,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.08530057966709137,
-      "rewards/margins": 0.09672373533248901,
-      "rewards/rejected": -0.011423162184655666,
+      "logits/chosen": -1.9389715194702148,
+      "logits/rejected": -1.9409143924713135,
+      "logps/chosen": -32.57710266113281,
+      "logps/rejected": -33.21430206298828,
+      "loss": 0.3914,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.018466468900442123,
+      "rewards/margins": 0.05280427262187004,
+      "rewards/rejected": -0.03433779999613762,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.075368881225586,
-      "logits/rejected": -2.0803427696228027,
-      "logps/chosen": -33.89010238647461,
-      "logps/rejected": -36.549957275390625,
-      "loss": 0.4868,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.07870620489120483,
-      "rewards/margins": 0.06357409805059433,
-      "rewards/rejected": 0.015132094733417034,
+      "logits/chosen": -2.0723679065704346,
+      "logits/rejected": -2.0773472785949707,
+      "logps/chosen": -33.946128845214844,
+      "logps/rejected": -36.60929489135742,
+      "loss": 0.4672,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.033884815871715546,
+      "rewards/margins": 0.0662197396159172,
+      "rewards/rejected": -0.03233493119478226,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9372116327285767,
-      "logits/rejected": -1.9403302669525146,
-      "logps/chosen": -34.205604553222656,
-      "logps/rejected": -34.49314880371094,
-      "loss": 0.4788,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.17776472866535187,
-      "rewards/margins": 0.09835849702358246,
-      "rewards/rejected": 0.0794062465429306,
+      "logits/chosen": -1.9366763830184937,
+      "logits/rejected": -1.9398372173309326,
+      "logps/chosen": -34.32460403442383,
+      "logps/rejected": -34.59550094604492,
+      "loss": 0.3932,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0825641006231308,
+      "rewards/margins": 0.08503931760787964,
+      "rewards/rejected": -0.002475212560966611,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.945416808128357,
-      "logits/rejected": -1.9499365091323853,
-      "logps/chosen": -32.26136016845703,
-      "logps/rejected": -32.26020050048828,
-      "loss": 0.4788,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.16484788060188293,
-      "rewards/margins": 0.08588258177042007,
-      "rewards/rejected": 0.07896529138088226,
+      "logits/chosen": -1.9469963312149048,
+      "logits/rejected": -1.9515202045440674,
+      "logps/chosen": -32.401084899902344,
+      "logps/rejected": -32.32483673095703,
+      "loss": 0.4402,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.05306895822286606,
+      "rewards/margins": 0.02581069990992546,
+      "rewards/rejected": 0.027258265763521194,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.043586254119873,
-      "logits/rejected": -2.0416104793548584,
-      "logps/chosen": -31.968154907226562,
-      "logps/rejected": -31.14252281188965,
-      "loss": 0.4749,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.21814338862895966,
-      "rewards/margins": 0.11919467151165009,
-      "rewards/rejected": 0.09894871711730957,
+      "logits/chosen": -2.0446877479553223,
+      "logits/rejected": -2.0427000522613525,
+      "logps/chosen": -32.168331146240234,
+      "logps/rejected": -31.263940811157227,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.05800473690032959,
+      "rewards/margins": 0.056188035756349564,
+      "rewards/rejected": 0.0018166989320889115,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2375268936157227,
-      "eval_logits/rejected": -2.2326817512512207,
-      "eval_logps/chosen": -33.851219177246094,
-      "eval_logps/rejected": -37.3537483215332,
-      "eval_loss": 0.49629145860671997,
-      "eval_rewards/accuracies": 0.5336378812789917,
-      "eval_rewards/chosen": 0.14666710793972015,
-      "eval_rewards/margins": 0.016369162127375603,
-      "eval_rewards/rejected": 0.1302979290485382,
-      "eval_runtime": 145.7938,
-      "eval_samples_per_second": 2.353,
-      "eval_steps_per_second": 0.295,
+      "eval_logits/chosen": -2.2377800941467285,
+      "eval_logits/rejected": -2.2329328060150146,
+      "eval_logps/chosen": -33.97116470336914,
+      "eval_logps/rejected": -37.47990036010742,
+      "eval_loss": 0.42868927121162415,
+      "eval_rewards/accuracies": 0.5577242374420166,
+      "eval_rewards/chosen": 0.05071057379245758,
+      "eval_rewards/margins": 0.021336428821086884,
+      "eval_rewards/rejected": 0.029374146834015846,
+      "eval_runtime": 146.0732,
+      "eval_samples_per_second": 2.348,
+      "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.998853087425232,
-      "logits/rejected": -1.9964749813079834,
-      "logps/chosen": -32.937965393066406,
-      "logps/rejected": -33.846004486083984,
-      "loss": 0.474,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.2446821928024292,
-      "rewards/margins": 0.09798828512430191,
-      "rewards/rejected": 0.1466939002275467,
+      "logits/chosen": -1.9993603229522705,
+      "logits/rejected": -1.996971845626831,
+      "logps/chosen": -33.112144470214844,
+      "logps/rejected": -33.986610412597656,
+      "loss": 0.5297,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.105336032807827,
+      "rewards/margins": 0.07112538814544678,
+      "rewards/rejected": 0.03421063348650932,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.0093610286712646,
-      "logits/rejected": -2.0010437965393066,
-      "logps/chosen": -32.1511116027832,
-      "logps/rejected": -31.939533233642578,
-      "loss": 0.4873,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.2350343018770218,
-      "rewards/margins": 0.057456426322460175,
-      "rewards/rejected": 0.17757786810398102,
+      "logits/chosen": -2.009766101837158,
+      "logits/rejected": -2.0014333724975586,
+      "logps/chosen": -32.351280212402344,
+      "logps/rejected": -32.099830627441406,
+      "loss": 0.5288,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.07490243762731552,
+      "rewards/margins": 0.025563379749655724,
+      "rewards/rejected": 0.04933905601501465,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0374419689178467,
-      "logits/rejected": -2.029500961303711,
-      "logps/chosen": -30.157424926757812,
-      "logps/rejected": -31.90500831604004,
-      "loss": 0.4716,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.2546332776546478,
-      "rewards/margins": 0.1362351030111313,
-      "rewards/rejected": 0.11839816719293594,
+      "logits/chosen": -2.0372567176818848,
+      "logits/rejected": -2.029240369796753,
+      "logps/chosen": -30.34758949279785,
+      "logps/rejected": -32.03752899169922,
+      "loss": 0.4171,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.10250232368707657,
+      "rewards/margins": 0.09011942148208618,
+      "rewards/rejected": 0.01238289289176464,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9675471782684326,
-      "logits/rejected": -1.9777450561523438,
-      "logps/chosen": -31.059162139892578,
-      "logps/rejected": -32.384300231933594,
-      "loss": 0.4616,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.28839823603630066,
-      "rewards/margins": 0.16793587803840637,
-      "rewards/rejected": 0.12046238034963608,
+      "logits/chosen": -1.966684341430664,
+      "logits/rejected": -1.9769500494003296,
+      "logps/chosen": -31.219501495361328,
+      "logps/rejected": -32.558082580566406,
+      "loss": 0.4241,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.1601306051015854,
+      "rewards/margins": 0.178691104054451,
+      "rewards/rejected": -0.018560513854026794,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8793624639511108,
-      "logits/rejected": -1.8805418014526367,
-      "logps/chosen": -33.646915435791016,
-      "logps/rejected": -34.56272888183594,
-      "loss": 0.4385,
+      "logits/chosen": -1.8799800872802734,
+      "logits/rejected": -1.8811286687850952,
+      "logps/chosen": -34.00492858886719,
+      "logps/rejected": -34.78579330444336,
+      "loss": 0.3742,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.4366222321987152,
-      "rewards/margins": 0.2809991240501404,
-      "rewards/rejected": 0.15562310814857483,
+      "rewards/chosen": 0.15021029114723206,
+      "rewards/margins": 0.17304366827011108,
+      "rewards/rejected": -0.02283337712287903,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9307949542999268,
-      "logits/rejected": -1.9274256229400635,
-      "logps/chosen": -35.765926361083984,
-      "logps/rejected": -32.48193359375,
-      "loss": 0.4685,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.32460084557533264,
-      "rewards/margins": 0.13194666802883148,
-      "rewards/rejected": 0.19265416264533997,
+      "logits/chosen": -1.932443618774414,
+      "logits/rejected": -1.9290469884872437,
+      "logps/chosen": -36.061256408691406,
+      "logps/rejected": -32.7225456237793,
+      "loss": 0.377,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0883389413356781,
+      "rewards/margins": 0.0881747230887413,
+      "rewards/rejected": 0.00016421750478912145,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0318870544433594,
-      "logits/rejected": -2.0245723724365234,
-      "logps/chosen": -33.25593566894531,
-      "logps/rejected": -31.189266204833984,
-      "loss": 0.4315,
+      "logits/chosen": -2.0334668159484863,
+      "logits/rejected": -2.026076078414917,
+      "logps/chosen": -33.546730041503906,
+      "logps/rejected": -31.355152130126953,
+      "loss": 0.3457,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.4335424304008484,
-      "rewards/margins": 0.3034728467464447,
-      "rewards/rejected": 0.1300695389509201,
+      "rewards/chosen": 0.2009061872959137,
+      "rewards/margins": 0.2035447061061859,
+      "rewards/rejected": -0.0026385621167719364,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.0387840270996094,
-      "logits/rejected": -2.0439517498016357,
-      "logps/chosen": -31.977941513061523,
-      "logps/rejected": -32.183441162109375,
-      "loss": 0.4499,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.4534761905670166,
-      "rewards/margins": 0.2121172845363617,
-      "rewards/rejected": 0.2413589060306549,
+      "logits/chosen": -2.0394930839538574,
+      "logits/rejected": -2.044752359390259,
+      "logps/chosen": -32.33013153076172,
+      "logps/rejected": -32.470909118652344,
+      "loss": 0.2919,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.17172439396381378,
+      "rewards/margins": 0.1603413224220276,
+      "rewards/rejected": 0.01138305850327015,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.039411783218384,
-      "logits/rejected": -2.036684513092041,
-      "logps/chosen": -31.094594955444336,
-      "logps/rejected": -31.10666275024414,
-      "loss": 0.4653,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.3154846429824829,
-      "rewards/margins": 0.1462438404560089,
-      "rewards/rejected": 0.169240802526474,
+      "logits/chosen": -2.0403592586517334,
+      "logits/rejected": -2.0375704765319824,
+      "logps/chosen": -31.314708709716797,
+      "logps/rejected": -31.29607582092285,
+      "loss": 0.4191,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.13939552009105682,
+      "rewards/margins": 0.12168798595666885,
+      "rewards/rejected": 0.017707547172904015,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.909691572189331,
-      "logits/rejected": -1.914358139038086,
-      "logps/chosen": -31.08693504333496,
-      "logps/rejected": -32.61079406738281,
-      "loss": 0.4376,
+      "logits/chosen": -1.9107002019882202,
+      "logits/rejected": -1.915356993675232,
+      "logps/chosen": -31.39776611328125,
+      "logps/rejected": -32.7841682434082,
+      "loss": 0.3991,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.41547876596450806,
-      "rewards/margins": 0.27136775851249695,
-      "rewards/rejected": 0.14411096274852753,
+      "rewards/chosen": 0.1668148785829544,
+      "rewards/margins": 0.16140125691890717,
+      "rewards/rejected": 0.005413623061031103,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.233895778656006,
-      "eval_logits/rejected": -2.2290823459625244,
-      "eval_logps/chosen": -33.78962326049805,
-      "eval_logps/rejected": -37.29551696777344,
-      "eval_loss": 0.4955826997756958,
-      "eval_rewards/accuracies": 0.5485880374908447,
-      "eval_rewards/chosen": 0.19594170153141022,
-      "eval_rewards/margins": 0.019058095291256905,
-      "eval_rewards/rejected": 0.17688362300395966,
-      "eval_runtime": 145.7262,
-      "eval_samples_per_second": 2.354,
-      "eval_steps_per_second": 0.295,
+      "eval_logits/chosen": -2.234880208969116,
+      "eval_logits/rejected": -2.2300188541412354,
+      "eval_logps/chosen": -34.03581619262695,
+      "eval_logps/rejected": -37.54121017456055,
+      "eval_loss": 0.4536176025867462,
+      "eval_rewards/accuracies": 0.5373754501342773,
+      "eval_rewards/chosen": -0.00101160176564008,
+      "eval_rewards/margins": 0.01865854486823082,
+      "eval_rewards/rejected": -0.019670147448778152,
+      "eval_runtime": 145.4825,
+      "eval_samples_per_second": 2.358,
+      "eval_steps_per_second": 0.296,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.0220179557800293,
-      "logits/rejected": -2.0326101779937744,
-      "logps/chosen": -31.51509666442871,
-      "logps/rejected": -33.655662536621094,
-      "loss": 0.4554,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.36496439576148987,
-      "rewards/margins": 0.19933317601680756,
-      "rewards/rejected": 0.16563120484352112,
+      "logits/chosen": -2.022460460662842,
+      "logits/rejected": -2.03314471244812,
+      "logps/chosen": -31.764179229736328,
+      "logps/rejected": -33.924598693847656,
+      "loss": 0.2827,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1656998097896576,
+      "rewards/margins": 0.21521887183189392,
+      "rewards/rejected": -0.049519073218107224,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9142780303955078,
-      "logits/rejected": -1.9289791584014893,
-      "logps/chosen": -29.601001739501953,
-      "logps/rejected": -31.418704986572266,
-      "loss": 0.4339,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.4084321856498718,
-      "rewards/margins": 0.2968555688858032,
-      "rewards/rejected": 0.11157669872045517,
+      "logits/chosen": -1.9155023097991943,
+      "logits/rejected": -1.9303117990493774,
+      "logps/chosen": -29.97748374938965,
+      "logps/rejected": -31.563989639282227,
+      "loss": 0.3625,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.10724345594644547,
+      "rewards/margins": 0.11189230531454086,
+      "rewards/rejected": -0.004648865200579166,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9722967147827148,
-      "logits/rejected": -1.9762967824935913,
-      "logps/chosen": -32.825157165527344,
-      "logps/rejected": -31.445331573486328,
-      "loss": 0.4142,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.47385087609291077,
-      "rewards/margins": 0.39067354798316956,
-      "rewards/rejected": 0.0831773579120636,
+      "logits/chosen": -1.9727697372436523,
+      "logits/rejected": -1.9767513275146484,
+      "logps/chosen": -33.23460006713867,
+      "logps/rejected": -31.581985473632812,
+      "loss": 0.3234,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.1462947130203247,
+      "rewards/margins": 0.17244157195091248,
+      "rewards/rejected": -0.026146870106458664,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9708006381988525,
-      "logits/rejected": -1.9490963220596313,
-      "logps/chosen": -33.599205017089844,
-      "logps/rejected": -34.856910705566406,
-      "loss": 0.4183,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.4436335563659668,
-      "rewards/margins": 0.36852845549583435,
-      "rewards/rejected": 0.07510510087013245,
+      "logits/chosen": -1.971836805343628,
+      "logits/rejected": -1.9498882293701172,
+      "logps/chosen": -33.9459228515625,
+      "logps/rejected": -35.02843475341797,
+      "loss": 0.3771,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.1662607342004776,
+      "rewards/margins": 0.2283717393875122,
+      "rewards/rejected": -0.0621110200881958,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.011307954788208,
-      "logits/rejected": -2.008012533187866,
-      "logps/chosen": -32.494956970214844,
-      "logps/rejected": -35.98812484741211,
-      "loss": 0.4641,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.3440879285335541,
-      "rewards/margins": 0.15464913845062256,
-      "rewards/rejected": 0.18943879008293152,
+      "logits/chosen": -2.0139522552490234,
+      "logits/rejected": -2.010632038116455,
+      "logps/chosen": -32.76514434814453,
+      "logps/rejected": -36.231590270996094,
+      "loss": 0.3601,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.1279398649930954,
+      "rewards/margins": 0.1332733929157257,
+      "rewards/rejected": -0.005333536770194769,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8780018091201782,
-      "logits/rejected": -1.8755817413330078,
-      "logps/chosen": -33.72126388549805,
-      "logps/rejected": -35.290069580078125,
-      "loss": 0.4544,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.37370795011520386,
-      "rewards/margins": 0.20206721127033234,
-      "rewards/rejected": 0.17164072394371033,
+      "logits/chosen": -1.8817164897918701,
+      "logits/rejected": -1.8792623281478882,
+      "logps/chosen": -34.03675842285156,
+      "logps/rejected": -35.52077102661133,
+      "loss": 0.3873,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.12131254374980927,
+      "rewards/margins": 0.13423141837120056,
+      "rewards/rejected": -0.012918862514197826,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8628206253051758,
-      "logits/rejected": -1.8603187799453735,
-      "logps/chosen": -33.929344177246094,
-      "logps/rejected": -31.61127281188965,
-      "loss": 0.4451,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.3746057152748108,
-      "rewards/margins": 0.246560737490654,
-      "rewards/rejected": 0.128044992685318,
+      "logits/chosen": -1.8678665161132812,
+      "logits/rejected": -1.8653860092163086,
+      "logps/chosen": -34.24772262573242,
+      "logps/rejected": -31.75213050842285,
+      "loss": 0.3738,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.11989933252334595,
+      "rewards/margins": 0.1045403853058815,
+      "rewards/rejected": 0.015358942560851574,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9666433334350586,
-      "logits/rejected": -1.9561984539031982,
-      "logps/chosen": -34.74022674560547,
-      "logps/rejected": -31.632495880126953,
-      "loss": 0.4239,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.4931296706199646,
-      "rewards/margins": 0.3263343572616577,
-      "rewards/rejected": 0.16679534316062927,
+      "logits/chosen": -1.9715077877044678,
+      "logits/rejected": -1.9609073400497437,
+      "logps/chosen": -35.06243133544922,
+      "logps/rejected": -31.837047576904297,
+      "loss": 0.2652,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.2353697121143341,
+      "rewards/margins": 0.2322143018245697,
+      "rewards/rejected": 0.003155359299853444,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0619094371795654,
-      "logits/rejected": -2.0470869541168213,
-      "logps/chosen": -30.43317222595215,
-      "logps/rejected": -32.3420524597168,
-      "loss": 0.4679,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.39276057481765747,
-      "rewards/margins": 0.1449340283870697,
-      "rewards/rejected": 0.24782654643058777,
+      "logits/chosen": -2.0671467781066895,
+      "logits/rejected": -2.052150011062622,
+      "logps/chosen": -30.7277774810791,
+      "logps/rejected": -32.63950729370117,
+      "loss": 0.3926,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.15707775950431824,
+      "rewards/margins": 0.14721640944480896,
+      "rewards/rejected": 0.009861335158348083,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9332126379013062,
-      "logits/rejected": -1.9307750463485718,
-      "logps/chosen": -32.100502014160156,
-      "logps/rejected": -30.6780948638916,
-      "loss": 0.3835,
+      "logits/chosen": -1.9383538961410522,
+      "logits/rejected": -1.935782790184021,
+      "logps/chosen": -32.601158142089844,
+      "logps/rejected": -30.8317813873291,
+      "loss": 0.2816,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.6612862348556519,
-      "rewards/margins": 0.5503143072128296,
-      "rewards/rejected": 0.1109718531370163,
+      "rewards/chosen": 0.26075875759124756,
+      "rewards/margins": 0.2727365493774414,
+      "rewards/rejected": -0.011977789923548698,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.231203556060791,
-      "eval_logits/rejected": -2.226395606994629,
-      "eval_logps/chosen": -33.778892517089844,
-      "eval_logps/rejected": -37.28717803955078,
-      "eval_loss": 0.4949820339679718,
-      "eval_rewards/accuracies": 0.5245016813278198,
-      "eval_rewards/chosen": 0.204526886343956,
-      "eval_rewards/margins": 0.020971858873963356,
-      "eval_rewards/rejected": 0.1835550218820572,
-      "eval_runtime": 145.6189,
-      "eval_samples_per_second": 2.355,
+      "eval_logits/chosen": -2.2351884841918945,
+      "eval_logits/rejected": -2.230333089828491,
+      "eval_logps/chosen": -34.01213836669922,
+      "eval_logps/rejected": -37.532142639160156,
+      "eval_loss": 0.4463596045970917,
+      "eval_rewards/accuracies": 0.5427741408348083,
+      "eval_rewards/chosen": 0.017930733039975166,
+      "eval_rewards/margins": 0.03035038523375988,
+      "eval_rewards/rejected": -0.012419654987752438,
+      "eval_runtime": 145.7115,
+      "eval_samples_per_second": 2.354,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.9173246622085571,
-      "logits/rejected": -1.9141733646392822,
-      "logps/chosen": -31.059444427490234,
-      "logps/rejected": -33.56504440307617,
-      "loss": 0.4356,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.4351249635219574,
-      "rewards/margins": 0.293440043926239,
-      "rewards/rejected": 0.141684889793396,
+      "logits/chosen": -1.9204727411270142,
+      "logits/rejected": -1.9172074794769287,
+      "logps/chosen": -31.360204696655273,
+      "logps/rejected": -33.759212493896484,
+      "loss": 0.3011,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.19451384246349335,
+      "rewards/margins": 0.20816302299499512,
+      "rewards/rejected": -0.01364919263869524,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.9676717519760132,
-      "logits/rejected": -1.9555410146713257,
-      "logps/chosen": -34.03219985961914,
-      "logps/rejected": -33.442317962646484,
-      "loss": 0.417,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.4510994851589203,
-      "rewards/margins": 0.37388378381729126,
-      "rewards/rejected": 0.07721573859453201,
+      "logits/chosen": -1.9719558954238892,
+      "logits/rejected": -1.959651231765747,
+      "logps/chosen": -34.44158935546875,
+      "logps/rejected": -33.596771240234375,
+      "loss": 0.2835,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.12358621507883072,
+      "rewards/margins": 0.1699295938014984,
+      "rewards/rejected": -0.0463433675467968,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -2.002480983734131,
-      "logits/rejected": -2.0011117458343506,
-      "logps/chosen": -32.882102966308594,
-      "logps/rejected": -32.251502990722656,
-      "loss": 0.4339,
+      "logits/chosen": -2.0068559646606445,
+      "logits/rejected": -2.00539231300354,
+      "logps/chosen": -33.331153869628906,
+      "logps/rejected": -32.49522018432617,
+      "loss": 0.3707,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.4829506278038025,
-      "rewards/margins": 0.30048781633377075,
-      "rewards/rejected": 0.18246281147003174,
+      "rewards/chosen": 0.12371524423360825,
+      "rewards/margins": 0.13622507452964783,
+      "rewards/rejected": -0.012509837746620178,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.0899081230163574,
-      "logits/rejected": -2.0742757320404053,
-      "logps/chosen": -33.487709045410156,
-      "logps/rejected": -32.8193359375,
-      "loss": 0.4286,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.5533460974693298,
-      "rewards/margins": 0.3143841624259949,
-      "rewards/rejected": 0.23896190524101257,
+      "logits/chosen": -2.0939385890960693,
+      "logits/rejected": -2.0781521797180176,
+      "logps/chosen": -33.88405227661133,
+      "logps/rejected": -33.090179443359375,
+      "loss": 0.36,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.23627004027366638,
+      "rewards/margins": 0.21398480236530304,
+      "rewards/rejected": 0.02228522300720215,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.9617509841918945,
-      "logits/rejected": -1.9609159231185913,
-      "logps/chosen": -32.60249710083008,
-      "logps/rejected": -32.25555419921875,
-      "loss": 0.418,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.5436802506446838,
-      "rewards/margins": 0.37600547075271606,
-      "rewards/rejected": 0.16767482459545135,
+      "logits/chosen": -1.9664535522460938,
+      "logits/rejected": -1.9655389785766602,
+      "logps/chosen": -32.96236038208008,
+      "logps/rejected": -32.478477478027344,
+      "loss": 0.3,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.2557886242866516,
+      "rewards/margins": 0.2664529085159302,
+      "rewards/rejected": -0.0106642572209239,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9185073375701904,
-      "logits/rejected": -1.9287922382354736,
-      "logps/chosen": -31.57277488708496,
-      "logps/rejected": -35.039085388183594,
-      "loss": 0.4304,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.505532443523407,
-      "rewards/margins": 0.305793821811676,
-      "rewards/rejected": 0.19973860681056976,
+      "logits/chosen": -1.921805739402771,
+      "logits/rejected": -1.9321292638778687,
+      "logps/chosen": -32.022247314453125,
+      "logps/rejected": -35.31328582763672,
+      "loss": 0.3671,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.14595167338848114,
+      "rewards/margins": 0.16557307541370392,
+      "rewards/rejected": -0.01962139829993248,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.0573890209198,
-      "logits/rejected": -2.0508790016174316,
-      "logps/chosen": -33.04835891723633,
-      "logps/rejected": -28.99324607849121,
-      "loss": 0.4308,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.4828890860080719,
-      "rewards/margins": 0.2963466942310333,
-      "rewards/rejected": 0.18654237687587738,
+      "logits/chosen": -2.060645580291748,
+      "logits/rejected": -2.0541434288024902,
+      "logps/chosen": -33.4498405456543,
+      "logps/rejected": -29.211456298828125,
+      "loss": 0.3226,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.16169998049736023,
+      "rewards/margins": 0.1497238427400589,
+      "rewards/rejected": 0.011976108886301517,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.9167007207870483,
-      "logits/rejected": -1.9188674688339233,
-      "logps/chosen": -33.65839385986328,
-      "logps/rejected": -30.719829559326172,
-      "loss": 0.4247,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.46452397108078003,
-      "rewards/margins": 0.34228652715682983,
-      "rewards/rejected": 0.12223746627569199,
+      "logits/chosen": -1.920902967453003,
+      "logits/rejected": -1.9230811595916748,
+      "logps/chosen": -33.97188186645508,
+      "logps/rejected": -30.862768173217773,
+      "loss": 0.3117,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.21373608708381653,
+      "rewards/margins": 0.20584869384765625,
+      "rewards/rejected": 0.00788736529648304,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.4538224170734356,
-      "train_runtime": 3252.427,
+      "train_loss": 0.3756082092012678,
+      "train_runtime": 3251.4223,
       "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }