Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.7015
-- Rewards/chosen: -0.0021
+- Loss: 0.6981
+- Rewards/chosen: 0.0062
 - Rewards/rejected: 0.0101
-- Rewards/accuracies: 0.4925
-- Rewards/margins: -0.0122
-- Logps/rejected: -37.5022
-- Logps/chosen: -34.0376
-- Logits/rejected: -2.2391
-- Logits/chosen: -2.2440
+- Rewards/accuracies: 0.4983
+- Rewards/margins: -0.0039
+- Logps/rejected: -37.5040
+- Logps/chosen: -34.0268
+- Logits/rejected: -2.2390
+- Logits/chosen: -2.2439
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6821        | 0.26  | 100  | 0.6996          | 0.0061         | 0.0150           | 0.4805             | -0.0088         | -37.4952       | -34.0258     | -2.2393         | -2.2442       |
-| 0.6938        | 0.52  | 200  | 0.6978          | 0.0050         | 0.0098           | 0.4817             | -0.0048         | -37.5027       | -34.0275     | -2.2389         | -2.2438       |
-| 0.681         | 0.78  | 300  | 0.6902          | 0.0156         | 0.0052           | 0.5133             | 0.0103          | -37.5091       | -34.0123     | -2.2392         | -2.2441       |
+| 0.6927        | 0.26  | 100  | 0.6927          | 0.0150         | 0.0088           | 0.5187             | 0.0062          | -37.5056       | -34.0159     | -2.2389         | -2.2438       |
+| 0.6949        | 0.52  | 200  | 0.6928          | 0.0118         | 0.0055           | 0.5274             | 0.0063          | -37.5098       | -34.0198     | -2.2387         | -2.2436       |
+| 0.6798        | 0.78  | 300  | 0.6981          | 0.0062         | 0.0101           | 0.4983             | -0.0039         | -37.5040       | -34.0268     | -2.2390         | -2.2439       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f21e1ad286bb220a190c74f4dc67fc7f9dada19f7d8259ccc88b5305f920592
+oid sha256:4241a5bac4adc39debfd72b4e4d2124e198b356cd95c7245382199b8383c449c
 size 176183216

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2439703941345215,
-    "eval_logits/rejected": -2.2390873432159424,
-    "eval_logps/chosen": -34.037574768066406,
-    "eval_logps/rejected": -37.50224685668945,
-    "eval_loss": 0.7015302777290344,
-    "eval_rewards/accuracies": 0.4925249218940735,
-    "eval_rewards/chosen": -0.002115294337272644,
-    "eval_rewards/margins": -0.012174182571470737,
-    "eval_rewards/rejected": 0.010058889165520668,
-    "eval_runtime": 145.8307,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.352,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.6898305428492558,
-    "train_runtime": 3256.6227,
+    "train_loss": 0.6928803004227675,
+    "train_runtime": 3255.7403,
     "train_samples": 3079,
-    "train_samples_per_second": 0.945,
+    "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6898305428492558,
-    "train_runtime": 3256.6227,
+    "train_loss": 0.6928803004227675,
+    "train_runtime": 3255.7403,
     "train_samples": 3079,
-    "train_samples_per_second": 0.945,
+    "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118
 }

trainer_state.json

@@ -25,590 +25,590 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.866014003753662,
-      "logits/rejected": -1.8703253269195557,
-      "logps/chosen": -36.98957061767578,
-      "logps/rejected": -33.655086517333984,
-      "loss": 0.6802,
-      "rewards/accuracies": 0.5555555820465088,
-      "rewards/chosen": 0.01192101463675499,
-      "rewards/margins": 0.029430679976940155,
-      "rewards/rejected": -0.017509667202830315,
+      "logits/chosen": -1.8660534620285034,
+      "logits/rejected": -1.870363712310791,
+      "logps/chosen": -37.0097770690918,
+      "logps/rejected": -33.656063079833984,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": -0.0025405611377209425,
+      "rewards/margins": 0.018248552456498146,
+      "rewards/rejected": -0.020789114758372307,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.9975181818008423,
-      "logits/rejected": -2.000157356262207,
-      "logps/chosen": -29.643035888671875,
-      "logps/rejected": -29.068897247314453,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.0005901128170080483,
-      "rewards/margins": 0.003226220142096281,
-      "rewards/rejected": -0.003816334530711174,
+      "logits/chosen": -1.9975675344467163,
+      "logits/rejected": -2.0001988410949707,
+      "logps/chosen": -29.643625259399414,
+      "logps/rejected": -29.03875732421875,
+      "loss": 0.7064,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.0011415861081331968,
+      "rewards/margins": -0.02089018002152443,
+      "rewards/rejected": 0.019748592749238014,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.919750452041626,
-      "logits/rejected": -1.9170730113983154,
-      "logps/chosen": -31.426767349243164,
-      "logps/rejected": -33.24191665649414,
-      "loss": 0.6908,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.00747058168053627,
-      "rewards/margins": 0.008947307243943214,
-      "rewards/rejected": -0.016417888924479485,
+      "logits/chosen": -1.919878363609314,
+      "logits/rejected": -1.9171901941299438,
+      "logps/chosen": -31.411758422851562,
+      "logps/rejected": -33.21895980834961,
+      "loss": 0.6945,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.0034693297930061817,
+      "rewards/margins": 0.003873053938150406,
+      "rewards/rejected": -0.0004037246108055115,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.0170185565948486,
-      "logits/rejected": -2.008249282836914,
-      "logps/chosen": -32.58369827270508,
-      "logps/rejected": -32.508426666259766,
-      "loss": 0.6989,
+      "logits/chosen": -2.0166752338409424,
+      "logits/rejected": -2.007917881011963,
+      "logps/chosen": -32.59661102294922,
+      "logps/rejected": -32.506187438964844,
+      "loss": 0.7064,
       "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.004824736155569553,
-      "rewards/margins": -0.007793365977704525,
-      "rewards/rejected": 0.0029686305206269026,
+      "rewards/chosen": -0.01584548130631447,
+      "rewards/margins": -0.02103354223072529,
+      "rewards/rejected": 0.005188059527426958,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.864098310470581,
-      "logits/rejected": -1.8533239364624023,
-      "logps/chosen": -33.5538444519043,
-      "logps/rejected": -35.4467658996582,
-      "loss": 0.6956,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.002545009134337306,
-      "rewards/margins": -0.0013612977927550673,
-      "rewards/rejected": 0.003906308673322201,
+      "logits/chosen": -1.8648042678833008,
+      "logits/rejected": -1.8540267944335938,
+      "logps/chosen": -33.56422805786133,
+      "logps/rejected": -35.431602478027344,
+      "loss": 0.7067,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.005400418769568205,
+      "rewards/margins": -0.02199491299688816,
+      "rewards/rejected": 0.01659449376165867,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9450995922088623,
-      "logits/rejected": -1.9470382928848267,
-      "logps/chosen": -32.56639862060547,
-      "logps/rejected": -33.19929122924805,
-      "loss": 0.6743,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.02365189418196678,
-      "rewards/margins": 0.043189071118831635,
-      "rewards/rejected": -0.019537178799510002,
+      "logits/chosen": -1.9453144073486328,
+      "logits/rejected": -1.9472593069076538,
+      "logps/chosen": -32.59089279174805,
+      "logps/rejected": -33.192405700683594,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.007432769052684307,
+      "rewards/margins": 0.024254899471998215,
+      "rewards/rejected": -0.016822131350636482,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.0796637535095215,
-      "logits/rejected": -2.084643602371216,
-      "logps/chosen": -33.98316955566406,
-      "logps/rejected": -36.59341049194336,
-      "loss": 0.686,
+      "logits/chosen": -2.079728364944458,
+      "logits/rejected": -2.0847129821777344,
+      "logps/chosen": -33.983558654785156,
+      "logps/rejected": -36.584938049316406,
+      "loss": 0.688,
       "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.00371909374371171,
-      "rewards/margins": 0.020896537229418755,
-      "rewards/rejected": -0.01717744767665863,
+      "rewards/chosen": 0.003943340387195349,
+      "rewards/margins": 0.016796352341771126,
+      "rewards/rejected": -0.01285301148891449,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9420992136001587,
-      "logits/rejected": -1.9452590942382812,
-      "logps/chosen": -34.40605545043945,
-      "logps/rejected": -34.58306884765625,
-      "loss": 0.6914,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.015227687545120716,
-      "rewards/margins": 0.008690384216606617,
-      "rewards/rejected": 0.006537304259836674,
+      "logits/chosen": -1.9422305822372437,
+      "logits/rejected": -1.9453967809677124,
+      "logps/chosen": -34.4206657409668,
+      "logps/rejected": -34.566123962402344,
+      "loss": 0.7053,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.005717824678868055,
+      "rewards/margins": -0.015306837856769562,
+      "rewards/rejected": 0.021024659276008606,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.950547456741333,
-      "logits/rejected": -1.9550635814666748,
-      "logps/chosen": -32.4520378112793,
-      "logps/rejected": -32.369686126708984,
-      "loss": 0.6861,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.010769412852823734,
-      "rewards/margins": 0.018315035849809647,
-      "rewards/rejected": -0.007545621134340763,
+      "logits/chosen": -1.950453519821167,
+      "logits/rejected": -1.9549684524536133,
+      "logps/chosen": -32.457984924316406,
+      "logps/rejected": -32.36473083496094,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.007545561995357275,
+      "rewards/margins": 0.012203911319375038,
+      "rewards/rejected": -0.004658351186662912,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.0487563610076904,
-      "logits/rejected": -2.046765089035034,
-      "logps/chosen": -32.21302032470703,
-      "logps/rejected": -31.275714874267578,
-      "loss": 0.6821,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.019469190388917923,
-      "rewards/margins": 0.02612192928791046,
-      "rewards/rejected": -0.006652742624282837,
+      "logits/chosen": -2.048896312713623,
+      "logits/rejected": -2.0468926429748535,
+      "logps/chosen": -32.23190689086914,
+      "logps/rejected": -31.26666831970215,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.007141563110053539,
+      "rewards/margins": 0.007507171481847763,
+      "rewards/rejected": -0.0003656085464172065,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.244154930114746,
-      "eval_logits/rejected": -2.2392725944519043,
-      "eval_logps/chosen": -34.02579116821289,
-      "eval_logps/rejected": -37.495243072509766,
-      "eval_loss": 0.6996207237243652,
-      "eval_rewards/accuracies": 0.48048171401023865,
-      "eval_rewards/chosen": 0.006134189199656248,
-      "eval_rewards/margins": -0.008827367797493935,
-      "eval_rewards/rejected": 0.014961558394134045,
-      "eval_runtime": 146.1618,
-      "eval_samples_per_second": 2.347,
+      "eval_logits/chosen": -2.2437734603881836,
+      "eval_logits/rejected": -2.2388932704925537,
+      "eval_logps/chosen": -34.01585006713867,
+      "eval_logps/rejected": -37.50562286376953,
+      "eval_loss": 0.6927134394645691,
+      "eval_rewards/accuracies": 0.5186877250671387,
+      "eval_rewards/chosen": 0.014961617067456245,
+      "eval_rewards/margins": 0.006164299789816141,
+      "eval_rewards/rejected": 0.008797316811978817,
+      "eval_runtime": 146.2315,
+      "eval_samples_per_second": 2.346,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.005396604537964,
-      "logits/rejected": -2.00298810005188,
-      "logps/chosen": -33.235660552978516,
-      "logps/rejected": -34.023067474365234,
-      "loss": 0.6946,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.0057089305482804775,
-      "rewards/margins": 0.001296796603128314,
-      "rewards/rejected": 0.0044121332466602325,
+      "logits/chosen": -2.005382537841797,
+      "logits/rejected": -2.002962589263916,
+      "logps/chosen": -33.231327056884766,
+      "logps/rejected": -34.03110885620117,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.009991822764277458,
+      "rewards/margins": 0.01138134766370058,
+      "rewards/rejected": -0.0013895228039473295,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.0165247917175293,
-      "logits/rejected": -2.0081515312194824,
-      "logps/chosen": -32.45998764038086,
-      "logps/rejected": -32.18000793457031,
-      "loss": 0.6936,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.010559305548667908,
-      "rewards/margins": 0.0023918983060866594,
-      "rewards/rejected": -0.012951202690601349,
+      "logits/chosen": -2.01668643951416,
+      "logits/rejected": -2.0083160400390625,
+      "logps/chosen": -32.46741485595703,
+      "logps/rejected": -32.169979095458984,
+      "loss": 0.7019,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.018008466809988022,
+      "rewards/margins": -0.011230994947254658,
+      "rewards/rejected": -0.006777471862733364,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.0464999675750732,
-      "logits/rejected": -2.0384459495544434,
-      "logps/chosen": -30.482324600219727,
-      "logps/rejected": -32.024261474609375,
-      "loss": 0.7073,
-      "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": -0.004625456873327494,
-      "rewards/margins": -0.02474728412926197,
-      "rewards/rejected": 0.020121825858950615,
+      "logits/chosen": -2.046079158782959,
+      "logits/rejected": -2.038038969039917,
+      "logps/chosen": -30.489391326904297,
+      "logps/rejected": -32.04558563232422,
+      "loss": 0.7042,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.010940661653876305,
+      "rewards/margins": -0.01687880977988243,
+      "rewards/rejected": 0.005938149057328701,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.976978063583374,
-      "logits/rejected": -1.9872522354125977,
-      "logps/chosen": -31.402069091796875,
-      "logps/rejected": -32.54044723510742,
-      "loss": 0.6869,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.012314395979046822,
-      "rewards/margins": 0.01621088758111,
-      "rewards/rejected": -0.0038964927662163973,
+      "logits/chosen": -1.9770376682281494,
+      "logits/rejected": -1.9873006343841553,
+      "logps/chosen": -31.392288208007812,
+      "logps/rejected": -32.56155014038086,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.021898187696933746,
+      "rewards/margins": 0.043236203491687775,
+      "rewards/rejected": -0.02133801393210888,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8904860019683838,
-      "logits/rejected": -1.8915693759918213,
-      "logps/chosen": -34.20112609863281,
-      "logps/rejected": -34.765201568603516,
-      "loss": 0.6961,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.005904030986130238,
-      "rewards/margins": -0.0003432026132941246,
-      "rewards/rejected": -0.005560829304158688,
+      "logits/chosen": -1.8905258178710938,
+      "logits/rejected": -1.891603708267212,
+      "logps/chosen": -34.185401916503906,
+      "logps/rejected": -34.75876998901367,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.005832195747643709,
+      "rewards/margins": 0.007043843157589436,
+      "rewards/rejected": -0.0012116472935304046,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.9423534870147705,
-      "logits/rejected": -1.938856840133667,
-      "logps/chosen": -36.14204025268555,
-      "logps/rejected": -32.7347412109375,
-      "loss": 0.6817,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.020746376365423203,
-      "rewards/margins": 0.029139723628759384,
-      "rewards/rejected": -0.008393346332013607,
+      "logits/chosen": -1.942525863647461,
+      "logits/rejected": -1.9390523433685303,
+      "logps/chosen": -36.15251922607422,
+      "logps/rejected": -32.721561431884766,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.01532670110464096,
+      "rewards/margins": 0.014371859841048717,
+      "rewards/rejected": 0.0009548395755700767,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.0420148372650146,
-      "logits/rejected": -2.034619092941284,
-      "logps/chosen": -33.79521942138672,
-      "logps/rejected": -31.3667049407959,
-      "loss": 0.6897,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0018501380691304803,
-      "rewards/margins": 0.012249214574694633,
-      "rewards/rejected": -0.010399078950285912,
+      "logits/chosen": -2.042137622833252,
+      "logits/rejected": -2.0347514152526855,
+      "logps/chosen": -33.78362274169922,
+      "logps/rejected": -31.34653091430664,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.011391163803637028,
+      "rewards/margins": 0.007135935127735138,
+      "rewards/rejected": 0.004255227744579315,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0474741458892822,
-      "logits/rejected": -2.052746295928955,
-      "logps/chosen": -32.533199310302734,
-      "logps/rejected": -32.49622344970703,
-      "loss": 0.6872,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.008109848015010357,
-      "rewards/margins": 0.015868691727519035,
-      "rewards/rejected": -0.007758840918540955,
+      "logits/chosen": -2.047314405441284,
+      "logits/rejected": -2.052593231201172,
+      "logps/chosen": -32.52313995361328,
+      "logps/rejected": -32.499427795410156,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.01731465384364128,
+      "rewards/margins": 0.028742337599396706,
+      "rewards/rejected": -0.011427680030465126,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.048755407333374,
-      "logits/rejected": -2.045989513397217,
-      "logps/chosen": -31.495113372802734,
-      "logps/rejected": -31.329687118530273,
-      "loss": 0.6936,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.004311963450163603,
-      "rewards/margins": 0.003718080697581172,
-      "rewards/rejected": -0.008030043914914131,
+      "logits/chosen": -2.048703908920288,
+      "logits/rejected": -2.0459342002868652,
+      "logps/chosen": -31.495092391967773,
+      "logps/rejected": -31.32244300842285,
+      "loss": 0.6967,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.004913434386253357,
+      "rewards/margins": -0.0015291940653696656,
+      "rewards/rejected": -0.0033842422999441624,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9183790683746338,
-      "logits/rejected": -1.9230560064315796,
-      "logps/chosen": -31.61056137084961,
-      "logps/rejected": -32.79776382446289,
-      "loss": 0.6938,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.002994548762217164,
-      "rewards/margins": 0.001788289868272841,
-      "rewards/rejected": -0.004782836884260178,
+      "logits/chosen": -1.9186630249023438,
+      "logits/rejected": -1.9233324527740479,
+      "logps/chosen": -31.604812622070312,
+      "logps/rejected": -32.791358947753906,
+      "loss": 0.6949,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.0011774369049817324,
+      "rewards/margins": 0.0015200242633000016,
+      "rewards/rejected": -0.00034258683444932103,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2438013553619385,
-      "eval_logits/rejected": -2.2389278411865234,
-      "eval_logps/chosen": -34.02748107910156,
-      "eval_logps/rejected": -37.502662658691406,
-      "eval_loss": 0.6977797150611877,
-      "eval_rewards/accuracies": 0.48172760009765625,
-      "eval_rewards/chosen": 0.004952840972691774,
-      "eval_rewards/margins": -0.004817771725356579,
-      "eval_rewards/rejected": 0.009770614095032215,
-      "eval_runtime": 146.0018,
-      "eval_samples_per_second": 2.349,
+      "eval_logits/chosen": -2.2436025142669678,
+      "eval_logits/rejected": -2.2387301921844482,
+      "eval_logps/chosen": -34.01983642578125,
+      "eval_logps/rejected": -37.50978469848633,
+      "eval_loss": 0.6928001642227173,
+      "eval_rewards/accuracies": 0.5274086594581604,
+      "eval_rewards/chosen": 0.011776229366660118,
+      "eval_rewards/margins": 0.006309796124696732,
+      "eval_rewards/rejected": 0.0054664346389472485,
+      "eval_runtime": 145.9171,
+      "eval_samples_per_second": 2.351,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0318925380706787,
-      "logits/rejected": -2.0425820350646973,
-      "logps/chosen": -31.96123695373535,
-      "logps/rejected": -33.87626647949219,
-      "loss": 0.6874,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.007046692073345184,
-      "rewards/margins": 0.01654236949980259,
-      "rewards/rejected": -0.009495675563812256,
+      "logits/chosen": -2.0316667556762695,
+      "logits/rejected": -2.0423641204833984,
+      "logps/chosen": -31.9429874420166,
+      "logps/rejected": -33.863956451416016,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.022652912884950638,
+      "rewards/margins": 0.023658016696572304,
+      "rewards/rejected": -0.0010051012504845858,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9252264499664307,
-      "logits/rejected": -1.940086007118225,
-      "logps/chosen": -30.087350845336914,
-      "logps/rejected": -31.552490234375,
-      "loss": 0.6894,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.01693141832947731,
-      "rewards/margins": 0.012951460666954517,
-      "rewards/rejected": 0.003979954868555069,
+      "logits/chosen": -1.9254566431045532,
+      "logits/rejected": -1.9402967691421509,
+      "logps/chosen": -30.087352752685547,
+      "logps/rejected": -31.55807113647461,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.019350551068782806,
+      "rewards/margins": 0.019265536218881607,
+      "rewards/rejected": 8.501634147251025e-05,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9830553531646729,
-      "logits/rejected": -1.9870340824127197,
-      "logps/chosen": -33.37425231933594,
-      "logps/rejected": -31.55866050720215,
-      "loss": 0.6771,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.030249694362282753,
-      "rewards/margins": 0.036800943315029144,
-      "rewards/rejected": -0.006551248021423817,
+      "logits/chosen": -1.9833059310913086,
+      "logits/rejected": -1.9872699975967407,
+      "logps/chosen": -33.40017318725586,
+      "logps/rejected": -31.5333309173584,
+      "loss": 0.6963,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.013836635276675224,
+      "rewards/margins": 0.0010608620941638947,
+      "rewards/rejected": 0.012775774113833904,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9825870990753174,
-      "logits/rejected": -1.9606201648712158,
-      "logps/chosen": -34.172298431396484,
-      "logps/rejected": -34.95369338989258,
-      "loss": 0.7012,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.012983955442905426,
-      "rewards/margins": -0.010954022407531738,
-      "rewards/rejected": -0.002029931638389826,
+      "logits/chosen": -1.983289122581482,
+      "logits/rejected": -1.9613111019134521,
+      "logps/chosen": -34.167720794677734,
+      "logps/rejected": -34.96076965332031,
+      "loss": 0.6978,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.011179247871041298,
+      "rewards/margins": -0.0031959381885826588,
+      "rewards/rejected": -0.007983307354152203,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.0240674018859863,
-      "logits/rejected": -2.020759105682373,
-      "logps/chosen": -32.91376495361328,
-      "logps/rejected": -36.22763442993164,
-      "loss": 0.6907,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.007910441607236862,
-      "rewards/margins": 0.009807790629565716,
-      "rewards/rejected": -0.0018973512342199683,
+      "logits/chosen": -2.0248379707336426,
+      "logits/rejected": -2.0215275287628174,
+      "logps/chosen": -32.91853713989258,
+      "logps/rejected": -36.22992706298828,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.005225626286119223,
+      "rewards/margins": 0.009226549416780472,
+      "rewards/rejected": -0.004000924527645111,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.891227126121521,
-      "logits/rejected": -1.8887770175933838,
-      "logps/chosen": -34.196754455566406,
-      "logps/rejected": -35.51189422607422,
-      "loss": 0.6959,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.005844797007739544,
-      "rewards/margins": -0.0007547728600911796,
-      "rewards/rejected": -0.005090024787932634,
+      "logits/chosen": -1.8912242650985718,
+      "logits/rejected": -1.8887859582901,
+      "logps/chosen": -34.17466735839844,
+      "logps/rejected": -35.498863220214844,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.01098463125526905,
+      "rewards/margins": 0.006378169171512127,
+      "rewards/rejected": 0.004606460686773062,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.875758409500122,
-      "logits/rejected": -1.8732105493545532,
-      "logps/chosen": -34.38095474243164,
-      "logps/rejected": -31.74618148803711,
-      "loss": 0.6979,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.011651447974145412,
-      "rewards/margins": -0.005950801074504852,
-      "rewards/rejected": 0.01760224997997284,
+      "logits/chosen": -1.8761816024780273,
+      "logits/rejected": -1.8736416101455688,
+      "logps/chosen": -34.40642166137695,
+      "logps/rejected": -31.732311248779297,
+      "loss": 0.7147,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.007056808564811945,
+      "rewards/margins": -0.03827430680394173,
+      "rewards/rejected": 0.03121749497950077,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9794175624847412,
-      "logits/rejected": -1.968788504600525,
-      "logps/chosen": -35.32674026489258,
-      "logps/rejected": -31.84674072265625,
-      "loss": 0.6832,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.020933207124471664,
-      "rewards/margins": 0.024957498535513878,
-      "rewards/rejected": -0.004024294205009937,
+      "logits/chosen": -1.9794687032699585,
+      "logits/rejected": -1.9688432216644287,
+      "logps/chosen": -35.32493591308594,
+      "logps/rejected": -31.86197853088379,
+      "loss": 0.6757,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.025365391746163368,
+      "rewards/margins": 0.04215434193611145,
+      "rewards/rejected": -0.016788948327302933,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.075899600982666,
-      "logits/rejected": -2.0608580112457275,
-      "logps/chosen": -30.886249542236328,
-      "logps/rejected": -32.66474533081055,
-      "loss": 0.6773,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.026513904333114624,
-      "rewards/margins": 0.03555149585008621,
-      "rewards/rejected": -0.009037593379616737,
+      "logits/chosen": -2.0760245323181152,
+      "logits/rejected": -2.0610060691833496,
+      "logps/chosen": -30.925411224365234,
+      "logps/rejected": -32.648170471191406,
+      "loss": 0.6978,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0010284179588779807,
+      "rewards/margins": -0.003961210139095783,
+      "rewards/rejected": 0.0029327913653105497,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.9468847513198853,
-      "logits/rejected": -1.9443557262420654,
-      "logps/chosen": -32.885986328125,
-      "logps/rejected": -30.816707611083984,
-      "loss": 0.681,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.02878658100962639,
-      "rewards/margins": 0.028715472668409348,
-      "rewards/rejected": 7.110489241313189e-05,
+      "logits/chosen": -1.946812629699707,
+      "logits/rejected": -1.9442851543426514,
+      "logps/chosen": -32.90645217895508,
+      "logps/rejected": -30.836864471435547,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.016527386382222176,
+      "rewards/margins": 0.032571423798799515,
+      "rewards/rejected": -0.016044041141867638,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.244056224822998,
-      "eval_logits/rejected": -2.2391812801361084,
-      "eval_logps/chosen": -34.01231384277344,
-      "eval_logps/rejected": -37.509132385253906,
-      "eval_loss": 0.6901772618293762,
-      "eval_rewards/accuracies": 0.5132890343666077,
-      "eval_rewards/chosen": 0.015566298738121986,
-      "eval_rewards/margins": 0.01032568421214819,
-      "eval_rewards/rejected": 0.005240614525973797,
-      "eval_runtime": 145.9901,
-      "eval_samples_per_second": 2.349,
+      "eval_logits/chosen": -2.243879556655884,
+      "eval_logits/rejected": -2.239006996154785,
+      "eval_logps/chosen": -34.026798248291016,
+      "eval_logps/rejected": -37.50396728515625,
+      "eval_loss": 0.6981266140937805,
+      "eval_rewards/accuracies": 0.49833887815475464,
+      "eval_rewards/chosen": 0.006204271223396063,
+      "eval_rewards/margins": -0.003920448012650013,
+      "eval_rewards/rejected": 0.010124719701707363,
+      "eval_runtime": 145.9481,
+      "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9292223453521729,
-      "logits/rejected": -1.9259742498397827,
-      "logps/chosen": -31.571924209594727,
-      "logps/rejected": -33.74208450317383,
-      "loss": 0.6842,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.021998558193445206,
-      "rewards/margins": 0.02195030450820923,
-      "rewards/rejected": 4.825703945243731e-05,
+      "logits/chosen": -1.9286400079727173,
+      "logits/rejected": -1.92538583278656,
+      "logps/chosen": -31.571802139282227,
+      "logps/rejected": -33.7106819152832,
+      "loss": 0.6961,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.025238346308469772,
+      "rewards/margins": 5.8975441788788885e-05,
+      "rewards/rejected": 0.025179371237754822,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.980800986289978,
-      "logits/rejected": -1.9684827327728271,
-      "logps/chosen": -34.57365417480469,
-      "logps/rejected": -33.5623779296875,
-      "loss": 0.6796,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.015695111826062202,
-      "rewards/margins": 0.032171379774808884,
-      "rewards/rejected": -0.01647626794874668,
+      "logits/chosen": -1.9804341793060303,
+      "logits/rejected": -1.9681293964385986,
+      "logps/chosen": -34.57010269165039,
+      "logps/rejected": -33.552330017089844,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.020780155435204506,
+      "rewards/margins": 0.031573377549648285,
+      "rewards/rejected": -0.01079322025179863,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.016021490097046,
-      "logits/rejected": -2.0145699977874756,
-      "logps/chosen": -33.47764205932617,
-      "logps/rejected": -32.47804260253906,
-      "loss": 0.6932,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.005706763360649347,
-      "rewards/margins": 0.00462835980579257,
-      "rewards/rejected": 0.0010784054175019264,
+      "logits/chosen": -2.0159571170806885,
+      "logits/rejected": -2.0144896507263184,
+      "logps/chosen": -33.452049255371094,
+      "logps/rejected": -32.46733856201172,
+      "loss": 0.687,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.026994183659553528,
+      "rewards/margins": 0.017200354486703873,
+      "rewards/rejected": 0.009793824516236782,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.1033365726470947,
-      "logits/rejected": -2.087541103363037,
-      "logps/chosen": -34.160888671875,
-      "logps/rejected": -33.08522415161133,
-      "loss": 0.7012,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.012950187548995018,
-      "rewards/margins": -0.010019468143582344,
-      "rewards/rejected": 0.022969653829932213,
+      "logits/chosen": -2.103339433670044,
+      "logits/rejected": -2.0875279903411865,
+      "logps/chosen": -34.17384719848633,
+      "logps/rejected": -33.08417510986328,
+      "loss": 0.7089,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.004437476396560669,
+      "rewards/margins": -0.02264832705259323,
+      "rewards/rejected": 0.02708580158650875,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.974867582321167,
-      "logits/rejected": -1.9739230871200562,
-      "logps/chosen": -33.23981857299805,
-      "logps/rejected": -32.456180572509766,
-      "loss": 0.6846,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.029593482613563538,
-      "rewards/margins": 0.023315051570534706,
-      "rewards/rejected": 0.006278430111706257,
+      "logits/chosen": -1.974784255027771,
+      "logits/rejected": -1.9738404750823975,
+      "logps/chosen": -33.24315643310547,
+      "logps/rejected": -32.44605255126953,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.031151747331023216,
+      "rewards/margins": 0.01587487757205963,
+      "rewards/rejected": 0.015276871621608734,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9302256107330322,
-      "logits/rejected": -1.940580129623413,
-      "logps/chosen": -32.2147102355957,
-      "logps/rejected": -35.29616165161133,
-      "loss": 0.6963,
+      "logits/chosen": -1.9305660724639893,
+      "logits/rejected": -1.940930724143982,
+      "logps/chosen": -32.193580627441406,
+      "logps/rejected": -35.30970764160156,
+      "loss": 0.6828,
       "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.007009805645793676,
-      "rewards/margins": -0.0018279661890119314,
-      "rewards/rejected": -0.005181840155273676,
+      "rewards/chosen": 0.008886909112334251,
+      "rewards/margins": 0.025648051872849464,
+      "rewards/rejected": -0.016761142760515213,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.069504737854004,
-      "logits/rejected": -2.062945604324341,
-      "logps/chosen": -33.646629333496094,
-      "logps/rejected": -29.217721939086914,
-      "loss": 0.6968,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.003734806552529335,
-      "rewards/margins": -0.0023581120185554028,
-      "rewards/rejected": 0.0060929167084395885,
+      "logits/chosen": -2.069566249847412,
+      "logits/rejected": -2.0630078315734863,
+      "logps/chosen": -33.643856048583984,
+      "logps/rejected": -29.229293823242188,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.00649127596989274,
+      "rewards/margins": 0.008784732781350613,
+      "rewards/rejected": -0.0022934596054255962,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.9288103580474854,
-      "logits/rejected": -1.9309613704681396,
-      "logps/chosen": -34.23542022705078,
-      "logps/rejected": -30.89495277404785,
-      "loss": 0.6866,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0025402153842151165,
-      "rewards/margins": 0.018168382346630096,
-      "rewards/rejected": -0.015628164634108543,
+      "logits/chosen": -1.929059386253357,
+      "logits/rejected": -1.931227445602417,
+      "logps/chosen": -34.235416412353516,
+      "logps/rejected": -30.882593154907227,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0029094829224050045,
+      "rewards/margins": 0.010880164802074432,
+      "rewards/rejected": -0.00797068141400814,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.6898305428492558,
-      "train_runtime": 3256.6227,
-      "train_samples_per_second": 0.945,
+      "train_loss": 0.6928803004227675,
+      "train_runtime": 3255.7403,
+      "train_samples_per_second": 0.946,
       "train_steps_per_second": 0.118
     }
   ],