Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6984
-- Rewards/chosen: 0.0026
-- Rewards/rejected: 0.0097
-- Rewards/accuracies: 0.4988
-- Rewards/margins: -0.0071
-- Logps/rejected: -37.5004
-- Logps/chosen: -34.0302
-- Logits/rejected: -2.2394
-- Logits/chosen: -2.2443
+- Loss: 0.6902
+- Rewards/chosen: 0.0156
+- Rewards/rejected: 0.0052
+- Rewards/accuracies: 0.5133
+- Rewards/margins: 0.0103
+- Logps/rejected: -37.5091
+- Logps/chosen: -34.0123
+- Logits/rejected: -2.2392
+- Logits/chosen: -2.2441
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6933        | 0.26  | 100  | 0.6912          | 0.0148         | 0.0075           | 0.4988             | 0.0073          | -37.5041       | -34.0098     | -2.2391         | -2.2439       |
-| 0.6827        | 0.52  | 200  | 0.6955          | 0.0061         | 0.0071           | 0.4813             | -0.0010         | -37.5048       | -34.0244     | -2.2385         | -2.2434       |
-| 0.6911        | 0.78  | 300  | 0.6925          | 0.0061         | 0.0005           | 0.5457             | 0.0056          | -37.5157       | -34.0243     | -2.2394         | -2.2443       |
+| 0.6821        | 0.26  | 100  | 0.6996          | 0.0061         | 0.0150           | 0.4805             | -0.0088         | -37.4952       | -34.0258     | -2.2393         | -2.2442       |
+| 0.6938        | 0.52  | 200  | 0.6978          | 0.0050         | 0.0098           | 0.4817             | -0.0048         | -37.5027       | -34.0275     | -2.2389         | -2.2438       |
+| 0.681         | 0.78  | 300  | 0.6902          | 0.0156         | 0.0052           | 0.5133             | 0.0103          | -37.5091       | -34.0123     | -2.2392         | -2.2441       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d13b57097c6671d36ee4e8ee37b83f4236995b779904559de03402b737d30b0d
+oid sha256:fad833e10db9e8b4254a027fa666bb7c4f87d8c271d8f457b338262c20a03e39
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.244286060333252,
-    "eval_logits/rejected": -2.2393975257873535,
-    "eval_logps/chosen": -34.03020095825195,
-    "eval_logps/rejected": -37.50043869018555,
-    "eval_loss": 0.6983639597892761,
-    "eval_rewards/accuracies": 0.4987541437149048,
-    "eval_rewards/chosen": 0.0026094578206539154,
-    "eval_rewards/margins": -0.007098525296896696,
-    "eval_rewards/rejected": 0.00970798172056675,
-    "eval_runtime": 145.7128,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.354,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.6911570282725544,
-    "train_runtime": 3257.344,
+    "train_loss": 0.6898305428492558,
+    "train_runtime": 3256.6227,
     "train_samples": 3079,
     "train_samples_per_second": 0.945,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6911570282725544,
-    "train_runtime": 3257.344,
+    "train_loss": 0.6898305428492558,
+    "train_runtime": 3256.6227,
     "train_samples": 3079,
     "train_samples_per_second": 0.945,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -25,589 +25,589 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.865976333618164,
-      "logits/rejected": -1.8702969551086426,
-      "logps/chosen": -36.97197723388672,
-      "logps/rejected": -33.66468811035156,
-      "loss": 0.6739,
-      "rewards/accuracies": 0.6388888955116272,
-      "rewards/chosen": 0.020770318806171417,
-      "rewards/margins": 0.041539546102285385,
-      "rewards/rejected": -0.020769227296113968,
+      "logits/chosen": -1.866014003753662,
+      "logits/rejected": -1.8703253269195557,
+      "logps/chosen": -36.98957061767578,
+      "logps/rejected": -33.655086517333984,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.5555555820465088,
+      "rewards/chosen": 0.01192101463675499,
+      "rewards/margins": 0.029430679976940155,
+      "rewards/rejected": -0.017509667202830315,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.9976377487182617,
-      "logits/rejected": -2.0002894401550293,
-      "logps/chosen": -29.63763427734375,
-      "logps/rejected": -29.050308227539062,
-      "loss": 0.6974,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.0027363565750420094,
-      "rewards/margins": -0.005146201699972153,
-      "rewards/rejected": 0.007882559671998024,
+      "logits/chosen": -1.9975181818008423,
+      "logits/rejected": -2.000157356262207,
+      "logps/chosen": -29.643035888671875,
+      "logps/rejected": -29.068897247314453,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0005901128170080483,
+      "rewards/margins": 0.003226220142096281,
+      "rewards/rejected": -0.003816334530711174,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.9201209545135498,
-      "logits/rejected": -1.917419195175171,
-      "logps/chosen": -31.423137664794922,
-      "logps/rejected": -33.23155975341797,
-      "loss": 0.6936,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.004226746968924999,
-      "rewards/margins": 0.0036326400004327297,
-      "rewards/rejected": -0.007859388366341591,
+      "logits/chosen": -1.919750452041626,
+      "logits/rejected": -1.9170730113983154,
+      "logps/chosen": -31.426767349243164,
+      "logps/rejected": -33.24191665649414,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.00747058168053627,
+      "rewards/margins": 0.008947307243943214,
+      "rewards/rejected": -0.016417888924479485,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.0166988372802734,
-      "logits/rejected": -2.007939100265503,
-      "logps/chosen": -32.58073806762695,
-      "logps/rejected": -32.50107192993164,
-      "loss": 0.6996,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.0023586261086165905,
-      "rewards/margins": -0.009320550598204136,
-      "rewards/rejected": 0.006961924023926258,
+      "logits/chosen": -2.0170185565948486,
+      "logits/rejected": -2.008249282836914,
+      "logps/chosen": -32.58369827270508,
+      "logps/rejected": -32.508426666259766,
+      "loss": 0.6989,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.004824736155569553,
+      "rewards/margins": -0.007793365977704525,
+      "rewards/rejected": 0.0029686305206269026,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8645102977752686,
-      "logits/rejected": -1.8537269830703735,
-      "logps/chosen": -33.561344146728516,
-      "logps/rejected": -35.44156265258789,
-      "loss": 0.6992,
+      "logits/chosen": -1.864098310470581,
+      "logits/rejected": -1.8533239364624023,
+      "logps/chosen": -33.5538444519043,
+      "logps/rejected": -35.4467658996582,
+      "loss": 0.6956,
       "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.0023178723640739918,
-      "rewards/margins": -0.00878816656768322,
-      "rewards/rejected": 0.006470293737947941,
+      "rewards/chosen": 0.002545009134337306,
+      "rewards/margins": -0.0013612977927550673,
+      "rewards/rejected": 0.003906308673322201,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9450910091400146,
-      "logits/rejected": -1.947035551071167,
-      "logps/chosen": -32.59477233886719,
-      "logps/rejected": -33.191925048828125,
-      "loss": 0.6868,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.003247396554797888,
-      "rewards/margins": 0.015577316284179688,
-      "rewards/rejected": -0.012329919263720512,
+      "logits/chosen": -1.9450995922088623,
+      "logits/rejected": -1.9470382928848267,
+      "logps/chosen": -32.56639862060547,
+      "logps/rejected": -33.19929122924805,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.02365189418196678,
+      "rewards/margins": 0.043189071118831635,
+      "rewards/rejected": -0.019537178799510002,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.0797348022460938,
-      "logits/rejected": -2.0847277641296387,
-      "logps/chosen": -34.00436782836914,
-      "logps/rejected": -36.588375091552734,
-      "loss": 0.6939,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.009527785703539848,
-      "rewards/margins": 0.002171074505895376,
-      "rewards/rejected": -0.011698859743773937,
+      "logits/chosen": -2.0796637535095215,
+      "logits/rejected": -2.084643602371216,
+      "logps/chosen": -33.98316955566406,
+      "logps/rejected": -36.59341049194336,
+      "loss": 0.686,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.00371909374371171,
+      "rewards/margins": 0.020896537229418755,
+      "rewards/rejected": -0.01717744767665863,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9422187805175781,
-      "logits/rejected": -1.945369005203247,
-      "logps/chosen": -34.411216735839844,
-      "logps/rejected": -34.58810806274414,
-      "loss": 0.6913,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.009955625049769878,
-      "rewards/margins": 0.007375092711299658,
-      "rewards/rejected": 0.002580532105639577,
+      "logits/chosen": -1.9420992136001587,
+      "logits/rejected": -1.9452590942382812,
+      "logps/chosen": -34.40605545043945,
+      "logps/rejected": -34.58306884765625,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.015227687545120716,
+      "rewards/margins": 0.008690384216606617,
+      "rewards/rejected": 0.006537304259836674,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.9506874084472656,
-      "logits/rejected": -1.9551897048950195,
-      "logps/chosen": -32.461769104003906,
-      "logps/rejected": -32.3617057800293,
-      "loss": 0.6923,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0033923503942787647,
-      "rewards/margins": 0.005069470964372158,
-      "rewards/rejected": -0.0016771212685853243,
+      "logits/chosen": -1.950547456741333,
+      "logits/rejected": -1.9550635814666748,
+      "logps/chosen": -32.4520378112793,
+      "logps/rejected": -32.369686126708984,
+      "loss": 0.6861,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.010769412852823734,
+      "rewards/margins": 0.018315035849809647,
+      "rewards/rejected": -0.007545621134340763,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.049078941345215,
-      "logits/rejected": -2.0470805168151855,
-      "logps/chosen": -32.23583984375,
-      "logps/rejected": -31.266429901123047,
-      "loss": 0.6933,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.0029988770838826895,
-      "rewards/margins": 0.0031290887854993343,
-      "rewards/rejected": -0.0001302130549447611,
+      "logits/chosen": -2.0487563610076904,
+      "logits/rejected": -2.046765089035034,
+      "logps/chosen": -32.21302032470703,
+      "logps/rejected": -31.275714874267578,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.019469190388917923,
+      "rewards/margins": 0.02612192928791046,
+      "rewards/rejected": -0.006652742624282837,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2439465522766113,
-      "eval_logits/rejected": -2.239065408706665,
-      "eval_logps/chosen": -34.00984573364258,
-      "eval_logps/rejected": -37.50405502319336,
-      "eval_loss": 0.6912049055099487,
-      "eval_rewards/accuracies": 0.4987541437149048,
-      "eval_rewards/chosen": 0.014824562706053257,
-      "eval_rewards/margins": 0.007284608669579029,
-      "eval_rewards/rejected": 0.007539951708167791,
-      "eval_runtime": 146.2285,
-      "eval_samples_per_second": 2.346,
+      "eval_logits/chosen": -2.244154930114746,
+      "eval_logits/rejected": -2.2392725944519043,
+      "eval_logps/chosen": -34.02579116821289,
+      "eval_logps/rejected": -37.495243072509766,
+      "eval_loss": 0.6996207237243652,
+      "eval_rewards/accuracies": 0.48048171401023865,
+      "eval_rewards/chosen": 0.006134189199656248,
+      "eval_rewards/margins": -0.008827367797493935,
+      "eval_rewards/rejected": 0.014961558394134045,
+      "eval_runtime": 146.1618,
+      "eval_samples_per_second": 2.347,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.005307912826538,
-      "logits/rejected": -2.0029025077819824,
-      "logps/chosen": -33.245155334472656,
-      "logps/rejected": -34.01426696777344,
-      "loss": 0.6998,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.000803128641564399,
-      "rewards/margins": -0.009863952174782753,
-      "rewards/rejected": 0.009060824289917946,
+      "logits/chosen": -2.005396604537964,
+      "logits/rejected": -2.00298810005188,
+      "logps/chosen": -33.235660552978516,
+      "logps/rejected": -34.023067474365234,
+      "loss": 0.6946,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0057089305482804775,
+      "rewards/margins": 0.001296796603128314,
+      "rewards/rejected": 0.0044121332466602325,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.0167527198791504,
-      "logits/rejected": -2.008373737335205,
-      "logps/chosen": -32.43388748168945,
-      "logps/rejected": -32.184364318847656,
-      "loss": 0.6844,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.006612470839172602,
-      "rewards/margins": 0.020328599959611893,
-      "rewards/rejected": -0.013716126792132854,
+      "logits/chosen": -2.0165247917175293,
+      "logits/rejected": -2.0081515312194824,
+      "logps/chosen": -32.45998764038086,
+      "logps/rejected": -32.18000793457031,
+      "loss": 0.6936,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.010559305548667908,
+      "rewards/margins": 0.0023918983060866594,
+      "rewards/rejected": -0.012951202690601349,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.0465807914733887,
-      "logits/rejected": -2.03853178024292,
-      "logps/chosen": -30.49270248413086,
-      "logps/rejected": -32.03563690185547,
-      "loss": 0.7048,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.01019179355353117,
-      "rewards/margins": -0.020615343004465103,
-      "rewards/rejected": 0.010423547588288784,
+      "logits/chosen": -2.0464999675750732,
+      "logits/rejected": -2.0384459495544434,
+      "logps/chosen": -30.482324600219727,
+      "logps/rejected": -32.024261474609375,
+      "loss": 0.7073,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.004625456873327494,
+      "rewards/margins": -0.02474728412926197,
+      "rewards/rejected": 0.020121825858950615,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.9769595861434937,
-      "logits/rejected": -1.98722243309021,
-      "logps/chosen": -31.402965545654297,
-      "logps/rejected": -32.55664825439453,
-      "loss": 0.6836,
+      "logits/chosen": -1.976978063583374,
+      "logits/rejected": -1.9872522354125977,
+      "logps/chosen": -31.402069091796875,
+      "logps/rejected": -32.54044723510742,
+      "loss": 0.6869,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.010017503052949905,
-      "rewards/margins": 0.02307732030749321,
-      "rewards/rejected": -0.013059815391898155,
+      "rewards/chosen": 0.012314395979046822,
+      "rewards/margins": 0.01621088758111,
+      "rewards/rejected": -0.0038964927662163973,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8905092477798462,
-      "logits/rejected": -1.8915998935699463,
-      "logps/chosen": -34.19499588012695,
-      "logps/rejected": -34.76214599609375,
-      "loss": 0.6941,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.0013823460321873426,
-      "rewards/margins": 0.0015530474483966827,
-      "rewards/rejected": -0.00293539697304368,
+      "logits/chosen": -1.8904860019683838,
+      "logits/rejected": -1.8915693759918213,
+      "logps/chosen": -34.20112609863281,
+      "logps/rejected": -34.765201568603516,
+      "loss": 0.6961,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.005904030986130238,
+      "rewards/margins": -0.0003432026132941246,
+      "rewards/rejected": -0.005560829304158688,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.9429140090942383,
-      "logits/rejected": -1.939447045326233,
-      "logps/chosen": -36.15991973876953,
-      "logps/rejected": -32.71558380126953,
-      "loss": 0.6935,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.007058164570480585,
-      "rewards/margins": 0.0027567949146032333,
-      "rewards/rejected": 0.004301370121538639,
+      "logits/chosen": -1.9423534870147705,
+      "logits/rejected": -1.938856840133667,
+      "logps/chosen": -36.14204025268555,
+      "logps/rejected": -32.7347412109375,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.020746376365423203,
+      "rewards/margins": 0.029139723628759384,
+      "rewards/rejected": -0.008393346332013607,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.042171001434326,
-      "logits/rejected": -2.0347886085510254,
-      "logps/chosen": -33.77357864379883,
-      "logps/rejected": -31.36397361755371,
-      "loss": 0.684,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.014572665095329285,
-      "rewards/margins": 0.021845515817403793,
-      "rewards/rejected": -0.007272848393768072,
+      "logits/chosen": -2.0420148372650146,
+      "logits/rejected": -2.034619092941284,
+      "logps/chosen": -33.79521942138672,
+      "logps/rejected": -31.3667049407959,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0018501380691304803,
+      "rewards/margins": 0.012249214574694633,
+      "rewards/rejected": -0.010399078950285912,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0476887226104736,
-      "logits/rejected": -2.05295729637146,
-      "logps/chosen": -32.52228546142578,
-      "logps/rejected": -32.505516052246094,
-      "loss": 0.6821,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.013503499329090118,
-      "rewards/margins": 0.025730689987540245,
-      "rewards/rejected": -0.012227190658450127,
+      "logits/chosen": -2.0474741458892822,
+      "logits/rejected": -2.052746295928955,
+      "logps/chosen": -32.533199310302734,
+      "logps/rejected": -32.49622344970703,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.008109848015010357,
+      "rewards/margins": 0.015868691727519035,
+      "rewards/rejected": -0.007758840918540955,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.0486130714416504,
-      "logits/rejected": -2.0458409786224365,
-      "logps/chosen": -31.47336196899414,
-      "logps/rejected": -31.337549209594727,
-      "loss": 0.6856,
+      "logits/chosen": -2.048755407333374,
+      "logits/rejected": -2.045989513397217,
+      "logps/chosen": -31.495113372802734,
+      "logps/rejected": -31.329687118530273,
+      "loss": 0.6936,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.009355323389172554,
-      "rewards/margins": 0.020955625921487808,
-      "rewards/rejected": -0.011600303463637829,
+      "rewards/chosen": -0.004311963450163603,
+      "rewards/margins": 0.003718080697581172,
+      "rewards/rejected": -0.008030043914914131,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9189808368682861,
-      "logits/rejected": -1.9236522912979126,
-      "logps/chosen": -31.58243179321289,
-      "logps/rejected": -32.80672073364258,
-      "loss": 0.6827,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.014310021884739399,
-      "rewards/margins": 0.02378040924668312,
-      "rewards/rejected": -0.009470385499298573,
+      "logits/chosen": -1.9183790683746338,
+      "logits/rejected": -1.9230560064315796,
+      "logps/chosen": -31.61056137084961,
+      "logps/rejected": -32.79776382446289,
+      "loss": 0.6938,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.002994548762217164,
+      "rewards/margins": 0.001788289868272841,
+      "rewards/rejected": -0.004782836884260178,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.243415355682373,
-      "eval_logits/rejected": -2.238534688949585,
-      "eval_logps/chosen": -34.02439880371094,
-      "eval_logps/rejected": -37.504756927490234,
-      "eval_loss": 0.6955486536026001,
-      "eval_rewards/accuracies": 0.48131227493286133,
-      "eval_rewards/chosen": 0.006092504132539034,
-      "eval_rewards/margins": -0.0010250143241137266,
-      "eval_rewards/rejected": 0.007117518223822117,
-      "eval_runtime": 145.9398,
-      "eval_samples_per_second": 2.35,
+      "eval_logits/chosen": -2.2438013553619385,
+      "eval_logits/rejected": -2.2389278411865234,
+      "eval_logps/chosen": -34.02748107910156,
+      "eval_logps/rejected": -37.502662658691406,
+      "eval_loss": 0.6977797150611877,
+      "eval_rewards/accuracies": 0.48172760009765625,
+      "eval_rewards/chosen": 0.004952840972691774,
+      "eval_rewards/margins": -0.004817771725356579,
+      "eval_rewards/rejected": 0.009770614095032215,
+      "eval_runtime": 146.0018,
+      "eval_samples_per_second": 2.349,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0316381454467773,
-      "logits/rejected": -2.0423309803009033,
-      "logps/chosen": -31.946680068969727,
-      "logps/rejected": -33.87198257446289,
-      "loss": 0.6848,
+      "logits/chosen": -2.0318925380706787,
+      "logits/rejected": -2.0425820350646973,
+      "logps/chosen": -31.96123695373535,
+      "logps/rejected": -33.87626647949219,
+      "loss": 0.6874,
       "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.014773803763091564,
-      "rewards/margins": 0.02034572884440422,
-      "rewards/rejected": -0.005571924149990082,
+      "rewards/chosen": 0.007046692073345184,
+      "rewards/margins": 0.01654236949980259,
+      "rewards/rejected": -0.009495675563812256,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9256389141082764,
-      "logits/rejected": -1.9404897689819336,
-      "logps/chosen": -30.095138549804688,
-      "logps/rejected": -31.5872802734375,
-      "loss": 0.6813,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.009841076098382473,
-      "rewards/margins": 0.027301892638206482,
-      "rewards/rejected": -0.017460813745856285,
+      "logits/chosen": -1.9252264499664307,
+      "logits/rejected": -1.940086007118225,
+      "logps/chosen": -30.087350845336914,
+      "logps/rejected": -31.552490234375,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.01693141832947731,
+      "rewards/margins": 0.012951460666954517,
+      "rewards/rejected": 0.003979954868555069,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.983253836631775,
-      "logits/rejected": -1.9872106313705444,
-      "logps/chosen": -33.377899169921875,
-      "logps/rejected": -31.550067901611328,
-      "loss": 0.6835,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.02374371513724327,
-      "rewards/margins": 0.024203147739171982,
-      "rewards/rejected": -0.00045942998258396983,
+      "logits/chosen": -1.9830553531646729,
+      "logits/rejected": -1.9870340824127197,
+      "logps/chosen": -33.37425231933594,
+      "logps/rejected": -31.55866050720215,
+      "loss": 0.6771,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.030249694362282753,
+      "rewards/margins": 0.036800943315029144,
+      "rewards/rejected": -0.006551248021423817,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9831098318099976,
-      "logits/rejected": -1.9611488580703735,
-      "logps/chosen": -34.15457534790039,
-      "logps/rejected": -34.96303176879883,
-      "loss": 0.6918,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.0004945084219798446,
-      "rewards/margins": 0.006849849130958319,
-      "rewards/rejected": -0.007344359997659922,
+      "logits/chosen": -1.9825870990753174,
+      "logits/rejected": -1.9606201648712158,
+      "logps/chosen": -34.172298431396484,
+      "logps/rejected": -34.95369338989258,
+      "loss": 0.7012,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.012983955442905426,
+      "rewards/margins": -0.010954022407531738,
+      "rewards/rejected": -0.002029931638389826,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.0243515968322754,
-      "logits/rejected": -2.021042585372925,
-      "logps/chosen": -32.92682647705078,
-      "logps/rejected": -36.226043701171875,
-      "loss": 0.6951,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.001055869972333312,
-      "rewards/margins": -0.00038459646748378873,
-      "rewards/rejected": -0.000671274377964437,
+      "logits/chosen": -2.0240674018859863,
+      "logits/rejected": -2.020759105682373,
+      "logps/chosen": -32.91376495361328,
+      "logps/rejected": -36.22763442993164,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.007910441607236862,
+      "rewards/margins": 0.009807790629565716,
+      "rewards/rejected": -0.0018973512342199683,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8913053274154663,
-      "logits/rejected": -1.8888483047485352,
-      "logps/chosen": -34.18505096435547,
-      "logps/rejected": -35.491554260253906,
-      "loss": 0.6986,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.002010275376960635,
-      "rewards/margins": -0.005831328686326742,
-      "rewards/rejected": 0.007841606624424458,
+      "logits/chosen": -1.891227126121521,
+      "logits/rejected": -1.8887770175933838,
+      "logps/chosen": -34.196754455566406,
+      "logps/rejected": -35.51189422607422,
+      "loss": 0.6959,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.005844797007739544,
+      "rewards/margins": -0.0007547728600911796,
+      "rewards/rejected": -0.005090024787932634,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8760089874267578,
-      "logits/rejected": -1.8734655380249023,
-      "logps/chosen": -34.394832611083984,
-      "logps/rejected": -31.769023895263672,
-      "loss": 0.6946,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0016577050555497408,
-      "rewards/margins": 0.00027335790218785405,
-      "rewards/rejected": 0.001384348259307444,
+      "logits/chosen": -1.875758409500122,
+      "logits/rejected": -1.8732105493545532,
+      "logps/chosen": -34.38095474243164,
+      "logps/rejected": -31.74618148803711,
+      "loss": 0.6979,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.011651447974145412,
+      "rewards/margins": -0.005950801074504852,
+      "rewards/rejected": 0.01760224997997284,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9796526432037354,
-      "logits/rejected": -1.9690182209014893,
-      "logps/chosen": -35.34165954589844,
-      "logps/rejected": -31.8492488861084,
-      "loss": 0.6882,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.008989132940769196,
-      "rewards/margins": 0.013942519202828407,
-      "rewards/rejected": -0.004953385330736637,
+      "logits/chosen": -1.9794175624847412,
+      "logits/rejected": -1.968788504600525,
+      "logps/chosen": -35.32674026489258,
+      "logps/rejected": -31.84674072265625,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.020933207124471664,
+      "rewards/margins": 0.024957498535513878,
+      "rewards/rejected": -0.004024294205009937,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.075491428375244,
-      "logits/rejected": -2.060464859008789,
-      "logps/chosen": -30.913305282592773,
-      "logps/rejected": -32.65747833251953,
-      "loss": 0.6898,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.006493172142654657,
-      "rewards/margins": 0.009881803765892982,
-      "rewards/rejected": -0.003388631623238325,
+      "logits/chosen": -2.075899600982666,
+      "logits/rejected": -2.0608580112457275,
+      "logps/chosen": -30.886249542236328,
+      "logps/rejected": -32.66474533081055,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.026513904333114624,
+      "rewards/margins": 0.03555149585008621,
+      "rewards/rejected": -0.009037593379616737,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.946946144104004,
-      "logits/rejected": -1.9443925619125366,
-      "logps/chosen": -32.91460037231445,
-      "logps/rejected": -30.81583595275879,
-      "loss": 0.6911,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.00750743318349123,
-      "rewards/margins": 0.00692352931946516,
-      "rewards/rejected": 0.0005839023506268859,
+      "logits/chosen": -1.9468847513198853,
+      "logits/rejected": -1.9443557262420654,
+      "logps/chosen": -32.885986328125,
+      "logps/rejected": -30.816707611083984,
+      "loss": 0.681,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.02878658100962639,
+      "rewards/margins": 0.028715472668409348,
+      "rewards/rejected": 7.110489241313189e-05,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.244297504425049,
-      "eval_logits/rejected": -2.2394087314605713,
-      "eval_logps/chosen": -34.02434539794922,
-      "eval_logps/rejected": -37.515743255615234,
-      "eval_loss": 0.6924626231193542,
-      "eval_rewards/accuracies": 0.5456810593605042,
-      "eval_rewards/chosen": 0.006125003099441528,
-      "eval_rewards/margins": 0.00560029735788703,
-      "eval_rewards/rejected": 0.0005247062654234469,
-      "eval_runtime": 145.9317,
-      "eval_samples_per_second": 2.35,
+      "eval_logits/chosen": -2.244056224822998,
+      "eval_logits/rejected": -2.2391812801361084,
+      "eval_logps/chosen": -34.01231384277344,
+      "eval_logps/rejected": -37.509132385253906,
+      "eval_loss": 0.6901772618293762,
+      "eval_rewards/accuracies": 0.5132890343666077,
+      "eval_rewards/chosen": 0.015566298738121986,
+      "eval_rewards/margins": 0.01032568421214819,
+      "eval_rewards/rejected": 0.005240614525973797,
+      "eval_runtime": 145.9901,
+      "eval_samples_per_second": 2.349,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9294414520263672,
-      "logits/rejected": -1.9261829853057861,
-      "logps/chosen": -31.56203269958496,
-      "logps/rejected": -33.72837448120117,
-      "loss": 0.6871,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.02479010447859764,
-      "rewards/margins": 0.01652434654533863,
-      "rewards/rejected": 0.008265760727226734,
+      "logits/chosen": -1.9292223453521729,
+      "logits/rejected": -1.9259742498397827,
+      "logps/chosen": -31.571924209594727,
+      "logps/rejected": -33.74208450317383,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.021998558193445206,
+      "rewards/margins": 0.02195030450820923,
+      "rewards/rejected": 4.825703945243731e-05,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.980583906173706,
-      "logits/rejected": -1.9682567119598389,
-      "logps/chosen": -34.58124542236328,
-      "logps/rejected": -33.5312614440918,
-      "loss": 0.6926,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.00889885425567627,
-      "rewards/margins": 0.004354512318968773,
-      "rewards/rejected": 0.004544342402368784,
+      "logits/chosen": -1.980800986289978,
+      "logits/rejected": -1.9684827327728271,
+      "logps/chosen": -34.57365417480469,
+      "logps/rejected": -33.5623779296875,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.015695111826062202,
+      "rewards/margins": 0.032171379774808884,
+      "rewards/rejected": -0.01647626794874668,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.0161619186401367,
-      "logits/rejected": -2.014702558517456,
-      "logps/chosen": -33.476417541503906,
-      "logps/rejected": -32.486053466796875,
-      "loss": 0.6904,
+      "logits/chosen": -2.016021490097046,
+      "logits/rejected": -2.0145699977874756,
+      "logps/chosen": -33.47764205932617,
+      "logps/rejected": -32.47804260253906,
+      "loss": 0.6932,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.005623627454042435,
-      "rewards/margins": 0.009506640955805779,
-      "rewards/rejected": -0.0038830172270536423,
+      "rewards/chosen": 0.005706763360649347,
+      "rewards/margins": 0.00462835980579257,
+      "rewards/rejected": 0.0010784054175019264,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.103116512298584,
-      "logits/rejected": -2.087329149246216,
-      "logps/chosen": -34.173370361328125,
-      "logps/rejected": -33.09913635253906,
-      "loss": 0.699,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.0036146938800811768,
-      "rewards/margins": -0.007723378483206034,
-      "rewards/rejected": 0.011338073760271072,
+      "logits/chosen": -2.1033365726470947,
+      "logits/rejected": -2.087541103363037,
+      "logps/chosen": -34.160888671875,
+      "logps/rejected": -33.08522415161133,
+      "loss": 0.7012,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.012950187548995018,
+      "rewards/margins": -0.010019468143582344,
+      "rewards/rejected": 0.022969653829932213,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9745566844940186,
-      "logits/rejected": -1.9736087322235107,
-      "logps/chosen": -33.24656295776367,
-      "logps/rejected": -32.44744110107422,
-      "loss": 0.6899,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.021319344639778137,
-      "rewards/margins": 0.01069412101060152,
-      "rewards/rejected": 0.010625220835208893,
+      "logits/chosen": -1.974867582321167,
+      "logits/rejected": -1.9739230871200562,
+      "logps/chosen": -33.23981857299805,
+      "logps/rejected": -32.456180572509766,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.029593482613563538,
+      "rewards/margins": 0.023315051570534706,
+      "rewards/rejected": 0.006278430111706257,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9309154748916626,
-      "logits/rejected": -1.9412847757339478,
-      "logps/chosen": -32.214351654052734,
-      "logps/rejected": -35.293235778808594,
-      "loss": 0.6968,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.005797215737402439,
-      "rewards/margins": -0.003113214625045657,
-      "rewards/rejected": -0.0026840015780180693,
+      "logits/chosen": -1.9302256107330322,
+      "logits/rejected": -1.940580129623413,
+      "logps/chosen": -32.2147102355957,
+      "logps/rejected": -35.29616165161133,
+      "loss": 0.6963,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.007009805645793676,
+      "rewards/margins": -0.0018279661890119314,
+      "rewards/rejected": -0.005181840155273676,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.0695526599884033,
-      "logits/rejected": -2.0629897117614746,
-      "logps/chosen": -33.649295806884766,
-      "logps/rejected": -29.209609985351562,
-      "loss": 0.6991,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": 0.0016033422434702516,
-      "rewards/margins": -0.008483831770718098,
-      "rewards/rejected": 0.01008717343211174,
+      "logits/chosen": -2.069504737854004,
+      "logits/rejected": -2.062945604324341,
+      "logps/chosen": -33.646629333496094,
+      "logps/rejected": -29.217721939086914,
+      "loss": 0.6968,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.003734806552529335,
+      "rewards/margins": -0.0023581120185554028,
+      "rewards/rejected": 0.0060929167084395885,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.9294023513793945,
-      "logits/rejected": -1.9315516948699951,
-      "logps/chosen": -34.24349594116211,
-      "logps/rejected": -30.889087677001953,
-      "loss": 0.6916,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.0026698322035372257,
-      "rewards/margins": 0.007205922156572342,
-      "rewards/rejected": -0.009875754825770855,
+      "logits/chosen": -1.9288103580474854,
+      "logits/rejected": -1.9309613704681396,
+      "logps/chosen": -34.23542022705078,
+      "logps/rejected": -30.89495277404785,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0025402153842151165,
+      "rewards/margins": 0.018168382346630096,
+      "rewards/rejected": -0.015628164634108543,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.6911570282725544,
-      "train_runtime": 3257.344,
+      "train_loss": 0.6898305428492558,
+      "train_runtime": 3256.6227,
       "train_samples_per_second": 0.945,
       "train_steps_per_second": 0.118
     }