Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9894
-- Rewards/chosen: -0.0296
-- Rewards/rejected: -0.0401
-- Rewards/accuracies: 0.5125
-- Rewards/margins: 0.0105
-- Logps/rejected: -37.6168
-- Logps/chosen: -34.1085
-- Logits/rejected: -2.2185
-- Logits/chosen: -2.2233
+- Loss: 0.6920
+- Rewards/chosen: -0.0273
+- Rewards/rejected: -0.0406
+- Rewards/accuracies: 0.5278
+- Rewards/margins: 0.0133
+- Logps/rejected: -37.5978
+- Logps/chosen: -34.0891
+- Logits/rejected: -2.2231
+- Logits/chosen: -2.2279
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.9521        | 0.26  | 100  | 0.9920          | -0.0014        | -0.0094          | 0.5361             | 0.0080          | -37.5401       | -34.0381     | -2.2279         | -2.2327       |
-| 0.8515        | 0.52  | 200  | 0.9817          | -0.0042        | -0.0226          | 0.5594             | 0.0184          | -37.5730       | -34.0451     | -2.2226         | -2.2274       |
-| 0.7473        | 0.78  | 300  | 0.9830          | -0.0231        | -0.0399          | 0.5390             | 0.0169          | -37.6165       | -34.0922     | -2.2186         | -2.2234       |
+| 0.6634        | 0.26  | 100  | 0.6931          | 0.0028         | -0.0041          | 0.5216             | 0.0069          | -37.5249       | -34.0290     | -2.2276         | -2.2324       |
+| 0.6329        | 0.52  | 200  | 0.6905          | -0.0127        | -0.0275          | 0.5274             | 0.0148          | -37.5716       | -34.0600     | -2.2255         | -2.2304       |
+| 0.5742        | 0.78  | 300  | 0.6920          | -0.0273        | -0.0406          | 0.5278             | 0.0133          | -37.5978       | -34.0891     | -2.2231         | -2.2279       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6c9ab0b5a807529fb4eda32e797f68875c5e54ce248261258a517a6afb9dd75
+oid sha256:0aa405f5b86503e02e2b370eae5754be314df4fd8079a6033e67d561adc170c8
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.223268747329712,
-    "eval_logits/rejected": -2.218452215194702,
-    "eval_logps/chosen": -34.1085205078125,
-    "eval_logps/rejected": -37.61680603027344,
-    "eval_loss": 0.9893841743469238,
-    "eval_rewards/accuracies": 0.5124584436416626,
-    "eval_rewards/chosen": -0.02958693355321884,
-    "eval_rewards/margins": 0.010487398132681847,
-    "eval_rewards/rejected": -0.04007433354854584,
-    "eval_runtime": 145.4187,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.359,
-    "eval_steps_per_second": 0.296,
-    "train_loss": 0.8990784768934373,
-    "train_runtime": 3250.3805,
+    "train_loss": 0.64145151051608,
+    "train_runtime": 3249.8987,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.8990784768934373,
-    "train_runtime": 3250.3805,
+    "train_loss": 0.64145151051608,
+    "train_runtime": 3249.8987,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 1.0,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,589 +25,589 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8667426109313965,
-      "logits/rejected": -1.8710602521896362,
-      "logps/chosen": -36.991912841796875,
-      "logps/rejected": -33.67206954956055,
-      "loss": 0.9773,
-      "rewards/accuracies": 0.5694444179534912,
-      "rewards/chosen": 0.0058750128373503685,
-      "rewards/margins": 0.022671451792120934,
-      "rewards/rejected": -0.01679643802344799,
+      "logits/chosen": -1.8664319515228271,
+      "logits/rejected": -1.8707623481750488,
+      "logps/chosen": -36.98527526855469,
+      "logps/rejected": -33.654090881347656,
+      "loss": 0.6829,
+      "rewards/accuracies": 0.5416666865348816,
+      "rewards/chosen": 0.010662304237484932,
+      "rewards/margins": 0.02267039567232132,
+      "rewards/rejected": -0.012008090503513813,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9977442026138306,
-      "logits/rejected": -2.0003952980041504,
-      "logps/chosen": -29.659366607666016,
-      "logps/rejected": -29.05437660217285,
-      "loss": 1.0105,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.006868218071758747,
-      "rewards/margins": -0.010493903420865536,
-      "rewards/rejected": 0.003625686513260007,
+      "logits/chosen": -1.9980642795562744,
+      "logits/rejected": -2.0007288455963135,
+      "logps/chosen": -29.634414672851562,
+      "logps/rejected": -29.0543270111084,
+      "loss": 0.6943,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0038894296158105135,
+      "rewards/margins": -0.000667938613332808,
+      "rewards/rejected": 0.004557368345558643,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.920693039894104,
-      "logits/rejected": -1.91802179813385,
-      "logps/chosen": -31.39971351623535,
-      "logps/rejected": -33.21495819091797,
-      "loss": 0.9948,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.006551843136548996,
-      "rewards/margins": 0.0051523735746741295,
-      "rewards/rejected": 0.001399471191689372,
+      "logits/chosen": -1.9210376739501953,
+      "logits/rejected": -1.9183601140975952,
+      "logps/chosen": -31.391239166259766,
+      "logps/rejected": -33.24319076538086,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.012426799163222313,
+      "rewards/margins": 0.024792592972517014,
+      "rewards/rejected": -0.0123657938092947,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.018057107925415,
-      "logits/rejected": -2.0093047618865967,
-      "logps/chosen": -32.565284729003906,
-      "logps/rejected": -32.50053405761719,
-      "loss": 1.0002,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.0046076299622654915,
-      "rewards/margins": -0.00024683662923052907,
-      "rewards/rejected": 0.004854466766119003,
+      "logits/chosen": -2.017646312713623,
+      "logits/rejected": -2.0089142322540283,
+      "logps/chosen": -32.557518005371094,
+      "logps/rejected": -32.51502227783203,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.009644975885748863,
+      "rewards/margins": 0.010818523354828358,
+      "rewards/rejected": -0.0011735468870028853,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8627817630767822,
-      "logits/rejected": -1.851999044418335,
-      "logps/chosen": -33.549964904785156,
-      "logps/rejected": -35.44340896606445,
-      "loss": 1.0006,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.0030060340650379658,
-      "rewards/margins": -0.0005688609671778977,
-      "rewards/rejected": 0.003574896603822708,
+      "logits/chosen": -1.8620023727416992,
+      "logits/rejected": -1.8512481451034546,
+      "logps/chosen": -33.577735900878906,
+      "logps/rejected": -35.46040344238281,
+      "loss": 0.6982,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.010127579793334007,
+      "rewards/margins": -0.0060965316370129585,
+      "rewards/rejected": -0.004031048621982336,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9416770935058594,
-      "logits/rejected": -1.9436094760894775,
-      "logps/chosen": -32.53351593017578,
-      "logps/rejected": -33.217529296875,
-      "loss": 0.9561,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.026666466146707535,
-      "rewards/margins": 0.04512657970190048,
-      "rewards/rejected": -0.018460111692547798,
+      "logits/chosen": -1.9402154684066772,
+      "logits/rejected": -1.9421701431274414,
+      "logps/chosen": -32.552555084228516,
+      "logps/rejected": -33.22978973388672,
+      "loss": 0.6723,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.023814614862203598,
+      "rewards/margins": 0.053018856793642044,
+      "rewards/rejected": -0.029204240068793297,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.072779655456543,
-      "logits/rejected": -2.077756881713867,
-      "logps/chosen": -34.002342224121094,
-      "logps/rejected": -36.633216857910156,
-      "loss": 0.9798,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.0055419160053133965,
-      "rewards/margins": 0.020196745172142982,
-      "rewards/rejected": -0.025738662108778954,
+      "logits/chosen": -2.071889877319336,
+      "logits/rejected": -2.0768685340881348,
+      "logps/chosen": -33.997718811035156,
+      "logps/rejected": -36.63623809814453,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.004616844467818737,
+      "rewards/margins": 0.02906452678143978,
+      "rewards/rejected": -0.033681370317935944,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9332294464111328,
-      "logits/rejected": -1.9363486766815186,
-      "logps/chosen": -34.33064651489258,
-      "logps/rejected": -34.64745330810547,
-      "loss": 0.9391,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.03886651247739792,
-      "rewards/margins": 0.06088464334607124,
-      "rewards/rejected": -0.022018127143383026,
+      "logits/chosen": -1.9326432943344116,
+      "logits/rejected": -1.935786247253418,
+      "logps/chosen": -34.30440902709961,
+      "logps/rejected": -34.659637451171875,
+      "loss": 0.6521,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.06169893592596054,
+      "rewards/margins": 0.09531383961439133,
+      "rewards/rejected": -0.03361489623785019,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9409536123275757,
-      "logits/rejected": -1.9454677104949951,
-      "logps/chosen": -32.381752014160156,
-      "logps/rejected": -32.35805892944336,
-      "loss": 0.9661,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0342683270573616,
-      "rewards/margins": 0.033929694443941116,
-      "rewards/rejected": 0.0003386303724255413,
+      "logits/chosen": -1.9409675598144531,
+      "logits/rejected": -1.9454774856567383,
+      "logps/chosen": -32.3830680847168,
+      "logps/rejected": -32.33238983154297,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.042176127433776855,
+      "rewards/margins": 0.028917592018842697,
+      "rewards/rejected": 0.013258534483611584,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.0382819175720215,
-      "logits/rejected": -2.036304473876953,
-      "logps/chosen": -32.15763473510742,
-      "logps/rejected": -31.302764892578125,
-      "loss": 0.9521,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.033279187977313995,
-      "rewards/margins": 0.047900475561618805,
-      "rewards/rejected": -0.014621290378272533,
+      "logits/chosen": -2.038121461868286,
+      "logits/rejected": -2.036132574081421,
+      "logps/chosen": -32.12568664550781,
+      "logps/rejected": -31.2890567779541,
+      "loss": 0.6634,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.05757413059473038,
+      "rewards/margins": 0.06899620592594147,
+      "rewards/rejected": -0.011422084644436836,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2327499389648438,
-      "eval_logits/rejected": -2.227902889251709,
-      "eval_logps/chosen": -34.038063049316406,
-      "eval_logps/rejected": -37.54012680053711,
-      "eval_loss": 0.9920137524604797,
-      "eval_rewards/accuracies": 0.5361295938491821,
-      "eval_rewards/chosen": -0.0014053798513486981,
-      "eval_rewards/margins": 0.007998107932507992,
-      "eval_rewards/rejected": -0.009403487667441368,
-      "eval_runtime": 146.1087,
-      "eval_samples_per_second": 2.348,
-      "eval_steps_per_second": 0.294,
+      "eval_logits/chosen": -2.232415199279785,
+      "eval_logits/rejected": -2.2275755405426025,
+      "eval_logps/chosen": -34.029048919677734,
+      "eval_logps/rejected": -37.52485275268555,
+      "eval_loss": 0.693107545375824,
+      "eval_rewards/accuracies": 0.5215947031974792,
+      "eval_rewards/chosen": 0.0027513643726706505,
+      "eval_rewards/margins": 0.006868092343211174,
+      "eval_rewards/rejected": -0.0041167279705405235,
+      "eval_runtime": 145.7484,
+      "eval_samples_per_second": 2.353,
+      "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9931761026382446,
-      "logits/rejected": -1.9907801151275635,
-      "logps/chosen": -33.120845794677734,
-      "logps/rejected": -34.02234649658203,
-      "loss": 0.9536,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.04918808490037918,
-      "rewards/margins": 0.046377379447221756,
-      "rewards/rejected": 0.0028106991667300463,
+      "logits/chosen": -1.9926633834838867,
+      "logits/rejected": -1.9902803897857666,
+      "logps/chosen": -33.11687088012695,
+      "logps/rejected": -34.01213836669922,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.06347335875034332,
+      "rewards/margins": 0.054856397211551666,
+      "rewards/rejected": 0.008616959676146507,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.004035472869873,
-      "logits/rejected": -1.9957201480865479,
-      "logps/chosen": -32.31081771850586,
-      "logps/rejected": -32.129127502441406,
-      "loss": 0.9593,
+      "logits/chosen": -2.0042788982391357,
+      "logits/rejected": -1.9959495067596436,
+      "logps/chosen": -32.306739807128906,
+      "logps/rejected": -32.13039779663086,
+      "loss": 0.6734,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.05363499000668526,
-      "rewards/margins": 0.04068244248628616,
-      "rewards/rejected": 0.012952548451721668,
+      "rewards/chosen": 0.06908417493104935,
+      "rewards/margins": 0.05353052541613579,
+      "rewards/rejected": 0.015553650446236134,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0312001705169678,
-      "logits/rejected": -2.023239850997925,
-      "logps/chosen": -30.317270278930664,
-      "logps/rejected": -32.05809783935547,
-      "loss": 0.9384,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.06337819993495941,
-      "rewards/margins": 0.06541591882705688,
-      "rewards/rejected": -0.0020377314649522305,
+      "logits/chosen": -2.0326714515686035,
+      "logits/rejected": -2.0247092247009277,
+      "logps/chosen": -30.308746337890625,
+      "logps/rejected": -32.05224609375,
+      "loss": 0.6637,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.08348459005355835,
+      "rewards/margins": 0.08310474455356598,
+      "rewards/rejected": 0.00037985146627761424,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9611848592758179,
-      "logits/rejected": -1.9714059829711914,
-      "logps/chosen": -31.235523223876953,
-      "logps/rejected": -32.55936813354492,
-      "loss": 0.9165,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.07365532219409943,
-      "rewards/margins": 0.08345074951648712,
-      "rewards/rejected": -0.009795431979000568,
+      "logits/chosen": -1.9627164602279663,
+      "logits/rejected": -1.9729163646697998,
+      "logps/chosen": -31.189788818359375,
+      "logps/rejected": -32.54594421386719,
+      "loss": 0.6424,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.1149359717965126,
+      "rewards/margins": 0.12046756595373154,
+      "rewards/rejected": -0.005531603004783392,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8718490600585938,
-      "logits/rejected": -1.8730093240737915,
-      "logps/chosen": -33.873046875,
-      "logps/rejected": -34.80980682373047,
-      "loss": 0.8563,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.12785716354846954,
-      "rewards/margins": 0.1488770991563797,
-      "rewards/rejected": -0.021019931882619858,
+      "logits/chosen": -1.8740726709365845,
+      "logits/rejected": -1.875239372253418,
+      "logps/chosen": -33.88011932373047,
+      "logps/rejected": -34.779319763183594,
+      "loss": 0.6271,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.15628577768802643,
+      "rewards/margins": 0.1673184335231781,
+      "rewards/rejected": -0.011032682843506336,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9235217571258545,
-      "logits/rejected": -1.9201278686523438,
-      "logps/chosen": -36.01029968261719,
-      "logps/rejected": -32.690834045410156,
-      "loss": 0.9482,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.06455211341381073,
-      "rewards/margins": 0.05178683251142502,
-      "rewards/rejected": 0.012765283696353436,
+      "logits/chosen": -1.9257261753082275,
+      "logits/rejected": -1.922323226928711,
+      "logps/chosen": -35.9793586730957,
+      "logps/rejected": -32.714969635009766,
+      "loss": 0.6539,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.09615939855575562,
+      "rewards/margins": 0.09226818382740021,
+      "rewards/rejected": 0.003891219152137637,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0244345664978027,
-      "logits/rejected": -2.0171027183532715,
-      "logps/chosen": -33.474937438964844,
-      "logps/rejected": -31.42409324645996,
-      "loss": 0.8419,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.12916772067546844,
-      "rewards/margins": 0.15806543827056885,
-      "rewards/rejected": -0.028897713869810104,
+      "logits/chosen": -2.027190923690796,
+      "logits/rejected": -2.019850254058838,
+      "logps/chosen": -33.4937629699707,
+      "logps/rejected": -31.404333114624023,
+      "loss": 0.6193,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.15204860270023346,
+      "rewards/margins": 0.17829009890556335,
+      "rewards/rejected": -0.026241496205329895,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.0310873985290527,
-      "logits/rejected": -2.036334276199341,
-      "logps/chosen": -32.2062873840332,
-      "logps/rejected": -32.42302703857422,
-      "loss": 0.8919,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.1354007124900818,
-      "rewards/margins": 0.11055666208267212,
-      "rewards/rejected": 0.024844054132699966,
+      "logits/chosen": -2.033841609954834,
+      "logits/rejected": -2.039079427719116,
+      "logps/chosen": -32.22673797607422,
+      "logps/rejected": -32.453857421875,
+      "loss": 0.632,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.1590258777141571,
+      "rewards/margins": 0.1433834582567215,
+      "rewards/rejected": 0.015642408281564713,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.0314042568206787,
-      "logits/rejected": -2.02862286567688,
-      "logps/chosen": -31.278457641601562,
-      "logps/rejected": -31.348251342773438,
-      "loss": 0.9038,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.0841989517211914,
-      "rewards/margins": 0.09621445834636688,
-      "rewards/rejected": -0.012015508487820625,
+      "logits/chosen": -2.03458833694458,
+      "logits/rejected": -2.031813144683838,
+      "logps/chosen": -31.249963760375977,
+      "logps/rejected": -31.329097747802734,
+      "loss": 0.6439,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.11949291080236435,
+      "rewards/margins": 0.12493407726287842,
+      "rewards/rejected": -0.005441152956336737,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9020229578018188,
-      "logits/rejected": -1.906660795211792,
-      "logps/chosen": -31.28672218322754,
-      "logps/rejected": -32.84270477294922,
-      "loss": 0.8515,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.12782469391822815,
-      "rewards/margins": 0.14853176474571228,
-      "rewards/rejected": -0.020707078278064728,
+      "logits/chosen": -1.9052807092666626,
+      "logits/rejected": -1.9099184274673462,
+      "logps/chosen": -31.314193725585938,
+      "logps/rejected": -32.81206512451172,
+      "loss": 0.6329,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.14604374766349792,
+      "rewards/margins": 0.15660937130451202,
+      "rewards/rejected": -0.010565629228949547,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.22743558883667,
-      "eval_logits/rejected": -2.2226204872131348,
-      "eval_logps/chosen": -34.04505920410156,
-      "eval_logps/rejected": -37.573036193847656,
-      "eval_loss": 0.9816663265228271,
-      "eval_rewards/accuracies": 0.559385359287262,
-      "eval_rewards/chosen": -0.004204160068184137,
-      "eval_rewards/margins": 0.018361244350671768,
-      "eval_rewards/rejected": -0.022565403953194618,
-      "eval_runtime": 145.491,
-      "eval_samples_per_second": 2.358,
-      "eval_steps_per_second": 0.296,
+      "eval_logits/chosen": -2.230377674102783,
+      "eval_logits/rejected": -2.225529193878174,
+      "eval_logps/chosen": -34.05998992919922,
+      "eval_logps/rejected": -37.57161331176758,
+      "eval_loss": 0.6905081868171692,
+      "eval_rewards/accuracies": 0.5274086594581604,
+      "eval_rewards/chosen": -0.012718739919364452,
+      "eval_rewards/margins": 0.014778696931898594,
+      "eval_rewards/rejected": -0.027497438713908195,
+      "eval_runtime": 145.701,
+      "eval_samples_per_second": 2.354,
+      "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.0146515369415283,
-      "logits/rejected": -2.025285005569458,
-      "logps/chosen": -31.747217178344727,
-      "logps/rejected": -33.96654510498047,
-      "loss": 0.8755,
+      "logits/chosen": -2.017439603805542,
+      "logits/rejected": -2.0280823707580566,
+      "logps/chosen": -31.72454261779785,
+      "logps/rejected": -33.935951232910156,
+      "loss": 0.6269,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.08963600546121597,
-      "rewards/margins": 0.13117524981498718,
-      "rewards/rejected": -0.04153924435377121,
+      "rewards/chosen": 0.12337962538003922,
+      "rewards/margins": 0.16000542044639587,
+      "rewards/rejected": -0.036625780165195465,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9072997570037842,
-      "logits/rejected": -1.922041654586792,
-      "logps/chosen": -29.81575584411621,
-      "logps/rejected": -31.636306762695312,
-      "loss": 0.8509,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.11831430345773697,
-      "rewards/margins": 0.14956556260585785,
-      "rewards/rejected": -0.03125125169754028,
+      "logits/chosen": -1.909328818321228,
+      "logits/rejected": -1.924088716506958,
+      "logps/chosen": -29.841415405273438,
+      "logps/rejected": -31.60904884338379,
+      "loss": 0.6274,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.13506175577640533,
+      "rewards/margins": 0.16049805283546448,
+      "rewards/rejected": -0.02543630823493004,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9637253284454346,
-      "logits/rejected": -1.9677000045776367,
-      "logps/chosen": -33.084877014160156,
-      "logps/rejected": -31.639108657836914,
-      "loss": 0.8476,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.13303686678409576,
-      "rewards/margins": 0.16895917057991028,
-      "rewards/rejected": -0.03592229634523392,
+      "logits/chosen": -1.966265320777893,
+      "logits/rejected": -1.970245361328125,
+      "logps/chosen": -33.091209411621094,
+      "logps/rejected": -31.639759063720703,
+      "loss": 0.6126,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.16312837600708008,
+      "rewards/margins": 0.20835788547992706,
+      "rewards/rejected": -0.045229505747556686,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9612106084823608,
-      "logits/rejected": -1.9394094944000244,
-      "logps/chosen": -33.84415817260742,
-      "logps/rejected": -35.132205963134766,
-      "loss": 0.8154,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.1238357201218605,
-      "rewards/margins": 0.19639968872070312,
-      "rewards/rejected": -0.07256398350000381,
+      "logits/chosen": -1.9648067951202393,
+      "logits/rejected": -1.9429614543914795,
+      "logps/chosen": -33.82001495361328,
+      "logps/rejected": -35.11749267578125,
+      "loss": 0.5941,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.16686691343784332,
+      "rewards/margins": 0.25021862983703613,
+      "rewards/rejected": -0.08335171639919281,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.002776861190796,
-      "logits/rejected": -1.999464988708496,
-      "logps/chosen": -32.74934387207031,
-      "logps/rejected": -36.28064727783203,
-      "loss": 0.9096,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.0702909380197525,
-      "rewards/margins": 0.09257940202951431,
-      "rewards/rejected": -0.02228846587240696,
+      "logits/chosen": -2.005873441696167,
+      "logits/rejected": -2.002545118331909,
+      "logps/chosen": -32.70961380004883,
+      "logps/rejected": -36.252098083496094,
+      "loss": 0.6448,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.10772605985403061,
+      "rewards/margins": 0.12131496518850327,
+      "rewards/rejected": -0.013588905334472656,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8692924976348877,
-      "logits/rejected": -1.8668625354766846,
-      "logps/chosen": -33.982818603515625,
-      "logps/rejected": -35.535736083984375,
-      "loss": 0.9053,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.08223171532154083,
-      "rewards/margins": 0.09467832744121552,
-      "rewards/rejected": -0.012446624226868153,
+      "logits/chosen": -1.8737099170684814,
+      "logits/rejected": -1.8713098764419556,
+      "logps/chosen": -33.96501922607422,
+      "logps/rejected": -35.54829025268555,
+      "loss": 0.6377,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.11169042438268661,
+      "rewards/margins": 0.13352498412132263,
+      "rewards/rejected": -0.02183455601334572,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8543907403945923,
-      "logits/rejected": -1.8520078659057617,
-      "logps/chosen": -34.20638656616211,
-      "logps/rejected": -31.873388290405273,
-      "loss": 0.8923,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.07648433744907379,
-      "rewards/margins": 0.11730633676052094,
-      "rewards/rejected": -0.040821999311447144,
+      "logits/chosen": -1.858642578125,
+      "logits/rejected": -1.8562240600585938,
+      "logps/chosen": -34.18030548095703,
+      "logps/rejected": -31.82675552368164,
+      "loss": 0.6384,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.1086462140083313,
+      "rewards/margins": 0.1363571435213089,
+      "rewards/rejected": -0.027710938826203346,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9576927423477173,
-      "logits/rejected": -1.9472328424453735,
-      "logps/chosen": -34.999000549316406,
-      "logps/rejected": -31.88382911682129,
-      "loss": 0.8398,
+      "logits/chosen": -1.9618957042694092,
+      "logits/rejected": -1.9513881206512451,
+      "logps/chosen": -35.000816345214844,
+      "logps/rejected": -31.879558563232422,
+      "loss": 0.6107,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.1430555135011673,
-      "rewards/margins": 0.16019006073474884,
-      "rewards/rejected": -0.017134560272097588,
+      "rewards/chosen": 0.17791253328323364,
+      "rewards/margins": 0.1971966028213501,
+      "rewards/rejected": -0.019284065812826157,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0523831844329834,
-      "logits/rejected": -2.037504196166992,
-      "logps/chosen": -30.712234497070312,
-      "logps/rejected": -32.659637451171875,
-      "loss": 0.9146,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.08475608378648758,
-      "rewards/margins": 0.08787757158279419,
-      "rewards/rejected": -0.0031214915215969086,
+      "logits/chosen": -2.0570178031921387,
+      "logits/rejected": -2.04209566116333,
+      "logps/chosen": -30.695226669311523,
+      "logps/rejected": -32.64103317260742,
+      "loss": 0.654,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.11444780975580215,
+      "rewards/margins": 0.10904743522405624,
+      "rewards/rejected": 0.005400371737778187,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9236303567886353,
-      "logits/rejected": -1.9211324453353882,
-      "logps/chosen": -32.370330810546875,
-      "logps/rejected": -30.952068328857422,
-      "loss": 0.7473,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.2227112501859665,
-      "rewards/margins": 0.27681466937065125,
-      "rewards/rejected": -0.054103411734104156,
+      "logits/chosen": -1.9285688400268555,
+      "logits/rejected": -1.9260343313217163,
+      "logps/chosen": -32.38969039916992,
+      "logps/rejected": -30.898773193359375,
+      "loss": 0.5742,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.2687075138092041,
+      "rewards/margins": 0.3096885085105896,
+      "rewards/rejected": -0.04098101332783699,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.223449468612671,
-      "eval_logits/rejected": -2.2186241149902344,
-      "eval_logps/chosen": -34.09219741821289,
-      "eval_logps/rejected": -37.61648178100586,
-      "eval_loss": 0.9830461740493774,
-      "eval_rewards/accuracies": 0.5390365719795227,
-      "eval_rewards/chosen": -0.023058738559484482,
-      "eval_rewards/margins": 0.016884688287973404,
-      "eval_rewards/rejected": -0.03994342312216759,
-      "eval_runtime": 145.6848,
-      "eval_samples_per_second": 2.354,
+      "eval_logits/chosen": -2.2279160022735596,
+      "eval_logits/rejected": -2.223081350326538,
+      "eval_logps/chosen": -34.089088439941406,
+      "eval_logps/rejected": -37.59783935546875,
+      "eval_loss": 0.6919631958007812,
+      "eval_rewards/accuracies": 0.5278239250183105,
+      "eval_rewards/chosen": -0.027267219498753548,
+      "eval_rewards/margins": 0.013343668542802334,
+      "eval_rewards/rejected": -0.040610890835523605,
+      "eval_runtime": 145.7459,
+      "eval_samples_per_second": 2.353,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.908721923828125,
-      "logits/rejected": -1.9054863452911377,
-      "logps/chosen": -31.30953025817871,
-      "logps/rejected": -33.8243408203125,
-      "loss": 0.8639,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.1175277829170227,
-      "rewards/margins": 0.15040358901023865,
-      "rewards/rejected": -0.032875802367925644,
+      "logits/chosen": -1.9126602411270142,
+      "logits/rejected": -1.9093825817108154,
+      "logps/chosen": -31.319168090820312,
+      "logps/rejected": -33.805519104003906,
+      "loss": 0.624,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.1420917958021164,
+      "rewards/margins": 0.1737762689590454,
+      "rewards/rejected": -0.03168448060750961,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.958335518836975,
-      "logits/rejected": -1.946170449256897,
-      "logps/chosen": -34.308921813964844,
-      "logps/rejected": -33.70027542114258,
-      "loss": 0.8275,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.11486158519983292,
-      "rewards/margins": 0.17943526804447174,
-      "rewards/rejected": -0.06457368284463882,
+      "logits/chosen": -1.9627044200897217,
+      "logits/rejected": -1.9504749774932861,
+      "logps/chosen": -34.31007385253906,
+      "logps/rejected": -33.66672134399414,
+      "loss": 0.6084,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.1430002748966217,
+      "rewards/margins": 0.20694026350975037,
+      "rewards/rejected": -0.06393997371196747,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -1.9931294918060303,
-      "logits/rejected": -1.9917207956314087,
-      "logps/chosen": -33.1971321105957,
-      "logps/rejected": -32.582763671875,
-      "loss": 0.8474,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.11546528339385986,
-      "rewards/margins": 0.15673772990703583,
-      "rewards/rejected": -0.04127243906259537,
+      "logits/chosen": -1.9980976581573486,
+      "logits/rejected": -1.9966537952423096,
+      "logps/chosen": -33.16533660888672,
+      "logps/rejected": -32.55678939819336,
+      "loss": 0.6136,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.1602279245853424,
+      "rewards/margins": 0.19883206486701965,
+      "rewards/rejected": -0.03860412910580635,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.0800864696502686,
-      "logits/rejected": -2.064441204071045,
-      "logps/chosen": -33.79554748535156,
-      "logps/rejected": -33.09272384643555,
-      "loss": 0.8566,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.15353715419769287,
-      "rewards/margins": 0.14341332018375397,
-      "rewards/rejected": 0.010123846121132374,
+      "logits/chosen": -2.0848796367645264,
+      "logits/rejected": -2.069186210632324,
+      "logps/chosen": -33.787841796875,
+      "logps/rejected": -33.07987976074219,
+      "loss": 0.6229,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.19577431678771973,
+      "rewards/margins": 0.17669571936130524,
+      "rewards/rejected": 0.01907859742641449,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.9522091150283813,
-      "logits/rejected": -1.9513927698135376,
-      "logps/chosen": -32.85493469238281,
-      "logps/rejected": -32.53038787841797,
-      "loss": 0.818,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.170864075422287,
-      "rewards/margins": 0.19696101546287537,
-      "rewards/rejected": -0.02609694004058838,
+      "logits/chosen": -1.957275390625,
+      "logits/rejected": -1.9564218521118164,
+      "logps/chosen": -32.81622314453125,
+      "logps/rejected": -32.52650833129883,
+      "loss": 0.5923,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.23293733596801758,
+      "rewards/margins": 0.2636169195175171,
+      "rewards/rejected": -0.030679568648338318,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9075958728790283,
-      "logits/rejected": -1.9178472757339478,
-      "logps/chosen": -31.86941909790039,
-      "logps/rejected": -35.343692779541016,
-      "loss": 0.8461,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.1341095268726349,
-      "rewards/margins": 0.1560816764831543,
-      "rewards/rejected": -0.021972158923745155,
+      "logits/chosen": -1.9124408960342407,
+      "logits/rejected": -1.92275071144104,
+      "logps/chosen": -31.859888076782227,
+      "logps/rejected": -35.33869934082031,
+      "loss": 0.6119,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.17240020632743835,
+      "rewards/margins": 0.19736871123313904,
+      "rewards/rejected": -0.024968529120087624,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.047290325164795,
-      "logits/rejected": -2.0408453941345215,
-      "logps/chosen": -33.33329772949219,
-      "logps/rejected": -29.256567001342773,
-      "loss": 0.8605,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.12746797502040863,
-      "rewards/margins": 0.1395258903503418,
-      "rewards/rejected": -0.012057906948029995,
+      "logits/chosen": -2.0522782802581787,
+      "logits/rejected": -2.045797824859619,
+      "logps/chosen": -33.34915542602539,
+      "logps/rejected": -29.27215576171875,
+      "loss": 0.6194,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.15140748023986816,
+      "rewards/margins": 0.1742721050977707,
+      "rewards/rejected": -0.022864630445837975,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.9072849750518799,
-      "logits/rejected": -1.909480333328247,
-      "logps/chosen": -33.89991760253906,
-      "logps/rejected": -31.013565063476562,
-      "loss": 0.8133,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.13565407693386078,
-      "rewards/margins": 0.1920291632413864,
-      "rewards/rejected": -0.05637507513165474,
+      "logits/chosen": -1.9120715856552124,
+      "logits/rejected": -1.9142844676971436,
+      "logps/chosen": -33.86906051635742,
+      "logps/rejected": -30.961559295654297,
+      "loss": 0.5996,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.1849948763847351,
+      "rewards/margins": 0.22946183383464813,
+      "rewards/rejected": -0.04446694999933243,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.8990784768934373,
-      "train_runtime": 3250.3805,
+      "train_loss": 0.64145151051608,
+      "train_runtime": 3249.8987,
       "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }