Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6950
-- Rewards/chosen: 0.0036
-- Rewards/rejected: 0.0057
-- Rewards/accuracies: 0.4900
-- Rewards/margins: -0.0021
-- Logps/rejected: -37.5024
-- Logps/chosen: -34.0256
-- Logits/rejected: -2.2389
-- Logits/chosen: -2.2438
+- Loss: 0.6930
+- Rewards/chosen: 0.0088
+- Rewards/rejected: 0.0058
+- Rewards/accuracies: 0.5046
+- Rewards/margins: 0.0030
+- Logps/rejected: -37.5050
+- Logps/chosen: -34.0169
+- Logits/rejected: -2.2388
+- Logits/chosen: -2.2436
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6904        | 0.26  | 100  | 0.6919          | 0.0034         | -0.0006          | 0.5428             | 0.0040          | -37.5181       | -34.0260     | -2.2385         | -2.2434       |
-| 0.6912        | 0.52  | 200  | 0.6924          | 0.0082         | 0.0051           | 0.4992             | 0.0031          | -37.5039       | -34.0141     | -2.2385         | -2.2434       |
-| 0.6888        | 0.78  | 300  | 0.6949          | 0.0029         | 0.0046           | 0.5116             | -0.0017         | -37.5051       | -34.0272     | -2.2395         | -2.2444       |
+| 0.6909        | 0.26  | 100  | 0.6898          | 0.0068         | -0.0023          | 0.5627             | 0.0091          | -37.5213       | -34.0210     | -2.2390         | -2.2439       |
+| 0.691         | 0.52  | 200  | 0.6955          | 0.0045         | 0.0068           | 0.4780             | -0.0023         | -37.5030       | -34.0255     | -2.2385         | -2.2434       |
+| 0.6819        | 0.78  | 300  | 0.6930          | 0.0088         | 0.0058           | 0.5046             | 0.0030          | -37.5050       | -34.0169     | -2.2388         | -2.2436       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f9bc927d6d400daece3ba556cf1bbbd7d87d92ff0be93b981778fd7af9e2a3d
+oid sha256:940058e5096675bffbc9ca99595396cf55d790ddf03efeadee7adf5e04b9d514
 size 176183216

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2437589168548584,
-    "eval_logits/rejected": -2.2388815879821777,
-    "eval_logps/chosen": -34.025550842285156,
-    "eval_logps/rejected": -37.502403259277344,
-    "eval_loss": 0.6950117945671082,
-    "eval_rewards/accuracies": 0.49003320932388306,
-    "eval_rewards/chosen": 0.0036009380128234625,
-    "eval_rewards/margins": -0.0020864540711045265,
-    "eval_rewards/rejected": 0.005687391385436058,
-    "eval_runtime": 145.772,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.353,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.6908149576806403,
-    "train_runtime": 3255.681,
+    "train_loss": 0.6914243283209862,
+    "train_runtime": 3256.6129,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.945,
     "train_steps_per_second": 0.118
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6908149576806403,
-    "train_runtime": 3255.681,
+    "train_loss": 0.6914243283209862,
+    "train_runtime": 3256.6129,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.945,
     "train_steps_per_second": 0.118
 }

trainer_state.json

@@ -25,311 +25,311 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.8663291931152344,
-      "logits/rejected": -1.870633602142334,
-      "logps/chosen": -37.00596237182617,
-      "logps/rejected": -33.65876007080078,
-      "loss": 0.688,
-      "rewards/accuracies": 0.4722222089767456,
-      "rewards/chosen": 0.0002556463878136128,
-      "rewards/margins": 0.011729677207767963,
-      "rewards/rejected": -0.011474031955003738,
+      "logits/chosen": -1.8662172555923462,
+      "logits/rejected": -1.8705354928970337,
+      "logps/chosen": -37.00023651123047,
+      "logps/rejected": -33.65288162231445,
+      "loss": 0.6868,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": 0.0031818184070289135,
+      "rewards/margins": 0.014585819095373154,
+      "rewards/rejected": -0.011404001154005527,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.997443437576294,
-      "logits/rejected": -2.000075340270996,
-      "logps/chosen": -29.63150405883789,
-      "logps/rejected": -29.064098358154297,
-      "loss": 0.6915,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": 0.004276981111615896,
-      "rewards/margins": 0.004539952147752047,
-      "rewards/rejected": -0.0002629714144859463,
+      "logits/chosen": -1.997016191482544,
+      "logits/rejected": -1.9996668100357056,
+      "logps/chosen": -29.636394500732422,
+      "logps/rejected": -29.053604125976562,
+      "loss": 0.6951,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.0028971880674362183,
+      "rewards/margins": -0.002024129033088684,
+      "rewards/rejected": 0.004921317100524902,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.920101523399353,
-      "logits/rejected": -1.9174143075942993,
-      "logps/chosen": -31.427581787109375,
-      "logps/rejected": -33.214088439941406,
-      "loss": 0.6971,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.004594600293785334,
-      "rewards/margins": -0.006341943051666021,
-      "rewards/rejected": 0.0017473434563726187,
+      "logits/chosen": -1.9197543859481812,
+      "logits/rejected": -1.9170539379119873,
+      "logps/chosen": -31.429040908813477,
+      "logps/rejected": -33.19585418701172,
+      "loss": 0.7032,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.006471562199294567,
+      "rewards/margins": -0.017771439626812935,
+      "rewards/rejected": 0.011299876496195793,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.0168609619140625,
-      "logits/rejected": -2.008105754852295,
-      "logps/chosen": -32.59783935546875,
-      "logps/rejected": -32.4984130859375,
-      "loss": 0.701,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.008411901071667671,
-      "rewards/margins": -0.014114728197455406,
-      "rewards/rejected": 0.00570282619446516,
+      "logits/chosen": -2.016721248626709,
+      "logits/rejected": -2.007981300354004,
+      "logps/chosen": -32.594520568847656,
+      "logps/rejected": -32.49314498901367,
+      "loss": 0.7034,
+      "rewards/accuracies": 0.38749998807907104,
+      "rewards/chosen": -0.00885552167892456,
+      "rewards/margins": -0.018619000911712646,
+      "rewards/rejected": 0.009763479232788086,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.864262342453003,
-      "logits/rejected": -1.8534845113754272,
-      "logps/chosen": -33.57244873046875,
-      "logps/rejected": -35.4432373046875,
-      "loss": 0.6987,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.005986724980175495,
-      "rewards/margins": -0.00962964165955782,
-      "rewards/rejected": 0.0036429166793823242,
+      "logits/chosen": -1.8646072149276733,
+      "logits/rejected": -1.8538135290145874,
+      "logps/chosen": -33.57071304321289,
+      "logps/rejected": -35.4249267578125,
+      "loss": 0.7044,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.006614250130951405,
+      "rewards/margins": -0.020322056487202644,
+      "rewards/rejected": 0.01370780635625124,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9451322555541992,
-      "logits/rejected": -1.9470863342285156,
-      "logps/chosen": -32.58552551269531,
-      "logps/rejected": -33.180747985839844,
-      "loss": 0.689,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.005862490274012089,
-      "rewards/margins": 0.009610554203391075,
-      "rewards/rejected": -0.003748063463717699,
+      "logits/chosen": -1.9454749822616577,
+      "logits/rejected": -1.9473978281021118,
+      "logps/chosen": -32.56868362426758,
+      "logps/rejected": -33.174442291259766,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.015751874074339867,
+      "rewards/margins": 0.01728220283985138,
+      "rewards/rejected": -0.0015303284162655473,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.0796828269958496,
-      "logits/rejected": -2.0846545696258545,
-      "logps/chosen": -33.98699188232422,
-      "logps/rejected": -36.566322326660156,
-      "loss": 0.6942,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.000597482081502676,
-      "rewards/margins": -0.00042274221777915955,
-      "rewards/rejected": 0.0010202232515439391,
+      "logits/chosen": -2.079904794692993,
+      "logits/rejected": -2.0848889350891113,
+      "logps/chosen": -34.005821228027344,
+      "logps/rejected": -36.594722747802734,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.008668524213135242,
+      "rewards/margins": 0.004255795385688543,
+      "rewards/rejected": -0.012924319133162498,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9422571659088135,
-      "logits/rejected": -1.945412278175354,
-      "logps/chosen": -34.415496826171875,
-      "logps/rejected": -34.577476501464844,
-      "loss": 0.6947,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.004924021661281586,
-      "rewards/margins": -0.0010487461695447564,
-      "rewards/rejected": 0.00597276771441102,
+      "logits/chosen": -1.9423730373382568,
+      "logits/rejected": -1.9455206394195557,
+      "logps/chosen": -34.40643310546875,
+      "logps/rejected": -34.57245635986328,
+      "loss": 0.6941,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.01068954449146986,
+      "rewards/margins": 0.000712524342816323,
+      "rewards/rejected": 0.00997702032327652,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.9508498907089233,
-      "logits/rejected": -1.955348253250122,
-      "logps/chosen": -32.44896697998047,
-      "logps/rejected": -32.381385803222656,
-      "loss": 0.6855,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0073806666769087315,
-      "rewards/margins": 0.01637081615626812,
-      "rewards/rejected": -0.00899015087634325,
+      "logits/chosen": -1.9506025314331055,
+      "logits/rejected": -1.955121636390686,
+      "logps/chosen": -32.45918655395508,
+      "logps/rejected": -32.362247467041016,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.004118821118026972,
+      "rewards/margins": 0.005789563059806824,
+      "rewards/rejected": -0.0016707420581951737,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.0491819381713867,
-      "logits/rejected": -2.0471725463867188,
-      "logps/chosen": -32.225120544433594,
-      "logps/rejected": -31.267704010009766,
-      "loss": 0.6904,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.0062875039875507355,
-      "rewards/margins": 0.006883688271045685,
-      "rewards/rejected": -0.0005961857968941331,
+      "logits/chosen": -2.0490012168884277,
+      "logits/rejected": -2.0470049381256104,
+      "logps/chosen": -32.25275802612305,
+      "logps/rejected": -31.29152488708496,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.005962538532912731,
+      "rewards/margins": 0.006695576012134552,
+      "rewards/rejected": -0.012658114545047283,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.243363618850708,
-      "eval_logits/rejected": -2.2384824752807617,
-      "eval_logps/chosen": -34.026004791259766,
-      "eval_logps/rejected": -37.51805114746094,
-      "eval_loss": 0.6918829679489136,
-      "eval_rewards/accuracies": 0.5427741408348083,
-      "eval_rewards/chosen": 0.0034199401270598173,
-      "eval_rewards/margins": 0.003991155419498682,
-      "eval_rewards/rejected": -0.0005712147103622556,
-      "eval_runtime": 146.1635,
-      "eval_samples_per_second": 2.347,
+      "eval_logits/chosen": -2.2439146041870117,
+      "eval_logits/rejected": -2.2390429973602295,
+      "eval_logps/chosen": -34.02096939086914,
+      "eval_logps/rejected": -37.521278381347656,
+      "eval_loss": 0.6897847056388855,
+      "eval_rewards/accuracies": 0.5627076625823975,
+      "eval_rewards/chosen": 0.006789046805351973,
+      "eval_rewards/margins": 0.009119975380599499,
+      "eval_rewards/rejected": -0.002330928109586239,
+      "eval_runtime": 146.2765,
+      "eval_samples_per_second": 2.345,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.0055806636810303,
-      "logits/rejected": -2.003166675567627,
-      "logps/chosen": -33.227996826171875,
-      "logps/rejected": -34.02782440185547,
-      "loss": 0.6912,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.0063266693614423275,
-      "rewards/margins": 0.0057087307795882225,
-      "rewards/rejected": 0.0006179373594932258,
+      "logits/chosen": -2.0054855346679688,
+      "logits/rejected": -2.0030674934387207,
+      "logps/chosen": -33.233333587646484,
+      "logps/rejected": -34.01185989379883,
+      "loss": 0.6959,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": 0.005242079496383667,
+      "rewards/margins": -0.003514623735100031,
+      "rewards/rejected": 0.00875670276582241,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.016270160675049,
-      "logits/rejected": -2.0078837871551514,
-      "logps/chosen": -32.447757720947266,
-      "logps/rejected": -32.19353485107422,
-      "loss": 0.6881,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.0011401770170778036,
-      "rewards/margins": 0.011673182249069214,
-      "rewards/rejected": -0.012813357636332512,
+      "logits/chosen": -2.0167009830474854,
+      "logits/rejected": -2.008331537246704,
+      "logps/chosen": -32.438194274902344,
+      "logps/rejected": -32.18104934692383,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0033566683996468782,
+      "rewards/margins": 0.013127696700394154,
+      "rewards/rejected": -0.009771028533577919,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.04642653465271,
-      "logits/rejected": -2.038364887237549,
-      "logps/chosen": -30.48968505859375,
-      "logps/rejected": -32.04593276977539,
-      "loss": 0.6979,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.005588879343122244,
-      "rewards/margins": -0.008417831733822823,
-      "rewards/rejected": 0.0028289512265473604,
+      "logits/chosen": -2.0462117195129395,
+      "logits/rejected": -2.0381579399108887,
+      "logps/chosen": -30.49118995666504,
+      "logps/rejected": -32.039451599121094,
+      "loss": 0.7014,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.0077363611198961735,
+      "rewards/margins": -0.014514627866446972,
+      "rewards/rejected": 0.006778267212212086,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.9769046306610107,
-      "logits/rejected": -1.9871702194213867,
-      "logps/chosen": -31.396442413330078,
-      "logps/rejected": -32.558387756347656,
-      "loss": 0.6847,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.009287373162806034,
-      "rewards/margins": 0.01869189366698265,
-      "rewards/rejected": -0.009404524229466915,
+      "logits/chosen": -1.9772098064422607,
+      "logits/rejected": -1.9874696731567383,
+      "logps/chosen": -31.401031494140625,
+      "logps/rejected": -32.54351043701172,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.009314531460404396,
+      "rewards/margins": 0.013629269786179066,
+      "rewards/rejected": -0.00431473832577467,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8908485174179077,
-      "logits/rejected": -1.8919233083724976,
-      "logps/chosen": -34.16999053955078,
-      "logps/rejected": -34.78479766845703,
-      "loss": 0.6841,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.009080578573048115,
-      "rewards/margins": 0.02009742520749569,
-      "rewards/rejected": -0.011016845703125,
+      "logits/chosen": -1.8905785083770752,
+      "logits/rejected": -1.8916677236557007,
+      "logps/chosen": -34.182621002197266,
+      "logps/rejected": -34.750953674316406,
+      "loss": 0.6937,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.005035269074141979,
+      "rewards/margins": 0.0018822342390194535,
+      "rewards/rejected": 0.0031530349515378475,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.942665457725525,
-      "logits/rejected": -1.9391899108886719,
-      "logps/chosen": -36.12689971923828,
-      "logps/rejected": -32.72255325317383,
-      "loss": 0.685,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.017910802736878395,
-      "rewards/margins": 0.01783105731010437,
-      "rewards/rejected": 7.97472894191742e-05,
+      "logits/chosen": -1.9425489902496338,
+      "logits/rejected": -1.939056634902954,
+      "logps/chosen": -36.16254425048828,
+      "logps/rejected": -32.711036682128906,
+      "loss": 0.6949,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.0045667411759495735,
+      "rewards/margins": -0.0012899183202534914,
+      "rewards/rejected": 0.005856659263372421,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.042165756225586,
-      "logits/rejected": -2.0347695350646973,
-      "logps/chosen": -33.787109375,
-      "logps/rejected": -31.368520736694336,
-      "loss": 0.6889,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.004300374537706375,
-      "rewards/margins": 0.010968221351504326,
-      "rewards/rejected": -0.006667847745120525,
+      "logits/chosen": -2.041844129562378,
+      "logits/rejected": -2.034442186355591,
+      "logps/chosen": -33.78926467895508,
+      "logps/rejected": -31.351390838623047,
+      "loss": 0.692,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.004301992245018482,
+      "rewards/margins": 0.004073685500770807,
+      "rewards/rejected": 0.00022830665693618357,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.047497510910034,
-      "logits/rejected": -2.052760601043701,
-      "logps/chosen": -32.53215789794922,
-      "logps/rejected": -32.50490951538086,
-      "loss": 0.6874,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0050529674626886845,
-      "rewards/margins": 0.01296031754463911,
-      "rewards/rejected": -0.007907351478934288,
+      "logits/chosen": -2.047542095184326,
+      "logits/rejected": -2.0527994632720947,
+      "logps/chosen": -32.531822204589844,
+      "logps/rejected": -32.49492645263672,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.00648072361946106,
+      "rewards/margins": 0.01137510221451521,
+      "rewards/rejected": -0.00489437859505415,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.0487968921661377,
-      "logits/rejected": -2.0460152626037598,
-      "logps/chosen": -31.495594024658203,
-      "logps/rejected": -31.3470401763916,
-      "loss": 0.6896,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.002655787393450737,
-      "rewards/margins": 0.008874570950865746,
-      "rewards/rejected": -0.011530356481671333,
+      "logits/chosen": -2.048457622528076,
+      "logits/rejected": -2.0456814765930176,
+      "logps/chosen": -31.491973876953125,
+      "logps/rejected": -31.334081649780273,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.00151053664740175,
+      "rewards/margins": 0.006422492675483227,
+      "rewards/rejected": -0.007933029904961586,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9187023639678955,
-      "logits/rejected": -1.9233903884887695,
-      "logps/chosen": -31.589069366455078,
-      "logps/rejected": -32.78667449951172,
-      "loss": 0.6912,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.006885607726871967,
-      "rewards/margins": 0.0051818834617733955,
-      "rewards/rejected": 0.0017037258949130774,
+      "logits/chosen": -1.9186420440673828,
+      "logits/rejected": -1.9233074188232422,
+      "logps/chosen": -31.597881317138672,
+      "logps/rejected": -32.79570770263672,
+      "loss": 0.691,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.004199606366455555,
+      "rewards/margins": 0.006588342599570751,
+      "rewards/rejected": -0.0023887366987764835,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2433600425720215,
-      "eval_logits/rejected": -2.238468885421753,
-      "eval_logps/chosen": -34.01414489746094,
-      "eval_logps/rejected": -37.50386428833008,
-      "eval_loss": 0.692382276058197,
-      "eval_rewards/accuracies": 0.4991694688796997,
-      "eval_rewards/chosen": 0.008161893114447594,
-      "eval_rewards/margins": 0.003057094058021903,
-      "eval_rewards/rejected": 0.00510479835793376,
-      "eval_runtime": 145.9559,
+      "eval_logits/chosen": -2.243356466293335,
+      "eval_logits/rejected": -2.2384772300720215,
+      "eval_logps/chosen": -34.025508880615234,
+      "eval_logps/rejected": -37.50304412841797,
+      "eval_loss": 0.6954899430274963,
+      "eval_rewards/accuracies": 0.4779900312423706,
+      "eval_rewards/chosen": 0.004520992282778025,
+      "eval_rewards/margins": -0.0022661893162876368,
+      "eval_rewards/rejected": 0.006787182297557592,
+      "eval_runtime": 145.936,
       "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 200
@@ -337,155 +337,155 @@
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.031435489654541,
-      "logits/rejected": -2.042121648788452,
-      "logps/chosen": -31.93819236755371,
-      "logps/rejected": -33.87244415283203,
-      "loss": 0.6855,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.013244074769318104,
-      "rewards/margins": 0.017141219228506088,
-      "rewards/rejected": -0.003897144692018628,
+      "logits/chosen": -2.0320427417755127,
+      "logits/rejected": -2.0427355766296387,
+      "logps/chosen": -31.94189453125,
+      "logps/rejected": -33.89322280883789,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.014702295884490013,
+      "rewards/margins": 0.029963891953229904,
+      "rewards/rejected": -0.015261595137417316,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9252116680145264,
-      "logits/rejected": -1.9400631189346313,
-      "logps/chosen": -30.0819149017334,
-      "logps/rejected": -31.582805633544922,
-      "loss": 0.6833,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.011850157752633095,
-      "rewards/margins": 0.021701235324144363,
-      "rewards/rejected": -0.009851074777543545,
+      "logits/chosen": -1.9252369403839111,
+      "logits/rejected": -1.9400920867919922,
+      "logps/chosen": -30.095117568969727,
+      "logps/rejected": -31.542316436767578,
+      "loss": 0.6945,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.008213011547923088,
+      "rewards/margins": 0.0002824708935804665,
+      "rewards/rejected": 0.007930541411042213,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9831390380859375,
-      "logits/rejected": -1.9871015548706055,
-      "logps/chosen": -33.370384216308594,
-      "logps/rejected": -31.524139404296875,
-      "loss": 0.6896,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.018833406269550323,
-      "rewards/margins": 0.008769527077674866,
-      "rewards/rejected": 0.010063880123198032,
+      "logits/chosen": -1.9828424453735352,
+      "logits/rejected": -1.9868125915527344,
+      "logps/chosen": -33.39949417114258,
+      "logps/rejected": -31.549747467041016,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.00898725725710392,
+      "rewards/margins": 0.009209835901856422,
+      "rewards/rejected": -0.00022257864475250244,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9828351736068726,
-      "logits/rejected": -1.9608606100082397,
-      "logps/chosen": -34.157318115234375,
-      "logps/rejected": -34.97474670410156,
-      "loss": 0.6898,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.0014271701220422983,
-      "rewards/margins": 0.008153198286890984,
-      "rewards/rejected": -0.009580368176102638,
+      "logits/chosen": -1.982708215713501,
+      "logits/rejected": -1.9607412815093994,
+      "logps/chosen": -34.14110565185547,
+      "logps/rejected": -34.958885192871094,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.006322857923805714,
+      "rewards/margins": 0.01036941446363926,
+      "rewards/rejected": -0.004046556539833546,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.0243985652923584,
-      "logits/rejected": -2.0210859775543213,
-      "logps/chosen": -32.91604995727539,
-      "logps/rejected": -36.21431350708008,
-      "loss": 0.6945,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.003607477992773056,
-      "rewards/margins": -0.0006370179471559823,
-      "rewards/rejected": 0.004244496580213308,
+      "logits/chosen": -2.0246503353118896,
+      "logits/rejected": -2.0213494300842285,
+      "logps/chosen": -32.90447998046875,
+      "logps/rejected": -36.232059478759766,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.010292356833815575,
+      "rewards/margins": 0.013860592618584633,
+      "rewards/rejected": -0.0035682350862771273,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8909612894058228,
-      "logits/rejected": -1.8885223865509033,
-      "logps/chosen": -34.173240661621094,
-      "logps/rejected": -35.51388931274414,
-      "loss": 0.6892,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.006062888540327549,
-      "rewards/margins": 0.009769372642040253,
-      "rewards/rejected": -0.0037064836360514164,
+      "logits/chosen": -1.8912708759307861,
+      "logits/rejected": -1.8888260126113892,
+      "logps/chosen": -34.188419342041016,
+      "logps/rejected": -35.51929473876953,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -9.60230863711331e-06,
+      "rewards/margins": 0.0073286681436002254,
+      "rewards/rejected": -0.007338270545005798,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8755378723144531,
-      "logits/rejected": -1.8729900121688843,
-      "logps/chosen": -34.416507720947266,
-      "logps/rejected": -31.75918197631836,
-      "loss": 0.7001,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.007562094368040562,
-      "rewards/margins": -0.012420646846294403,
-      "rewards/rejected": 0.004858553409576416,
+      "logits/chosen": -1.8759148120880127,
+      "logits/rejected": -1.8733787536621094,
+      "logps/chosen": -34.3978157043457,
+      "logps/rejected": -31.752635955810547,
+      "loss": 0.6988,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.00010652244236553088,
+      "rewards/margins": -0.009455609135329723,
+      "rewards/rejected": 0.009349087253212929,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.979413628578186,
-      "logits/rejected": -1.9687983989715576,
-      "logps/chosen": -35.34419631958008,
-      "logps/rejected": -31.839731216430664,
-      "loss": 0.6917,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.004978640470653772,
-      "rewards/margins": 0.004474071320146322,
-      "rewards/rejected": 0.0005045672878623009,
+      "logits/chosen": -1.9801543951034546,
+      "logits/rejected": -1.9695091247558594,
+      "logps/chosen": -35.32206344604492,
+      "logps/rejected": -31.858333587646484,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.017289450392127037,
+      "rewards/margins": 0.025958454236388206,
+      "rewards/rejected": -0.00866900384426117,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.075562000274658,
-      "logits/rejected": -2.060526132583618,
-      "logps/chosen": -30.924896240234375,
-      "logps/rejected": -32.64234161376953,
-      "loss": 0.696,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.0003082419862039387,
-      "rewards/margins": -0.004107682965695858,
-      "rewards/rejected": 0.0037994408048689365,
+      "logits/chosen": -2.07523775100708,
+      "logits/rejected": -2.0602152347564697,
+      "logps/chosen": -30.91782569885254,
+      "logps/rejected": -32.631629943847656,
+      "loss": 0.6977,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.0031501606572419405,
+      "rewards/margins": -0.006952463183552027,
+      "rewards/rejected": 0.010102624073624611,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.947762131690979,
-      "logits/rejected": -1.9452297687530518,
-      "logps/chosen": -32.90521240234375,
-      "logps/rejected": -30.821033477783203,
-      "loss": 0.6888,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.008759219199419022,
-      "rewards/margins": 0.010448457673192024,
-      "rewards/rejected": -0.0016892381245270371,
+      "logits/chosen": -1.9465487003326416,
+      "logits/rejected": -1.9440209865570068,
+      "logps/chosen": -32.89127731323242,
+      "logps/rejected": -30.83047866821289,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.017916660755872726,
+      "rewards/margins": 0.024751421064138412,
+      "rewards/rejected": -0.006834760308265686,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2443997859954834,
-      "eval_logits/rejected": -2.2395174503326416,
-      "eval_logps/chosen": -34.02724838256836,
-      "eval_logps/rejected": -37.50510787963867,
-      "eval_loss": 0.6948643326759338,
-      "eval_rewards/accuracies": 0.5116279125213623,
-      "eval_rewards/chosen": 0.0029200678691267967,
-      "eval_rewards/margins": -0.0016844052588567138,
-      "eval_rewards/rejected": 0.00460447371006012,
-      "eval_runtime": 145.9303,
+      "eval_logits/chosen": -2.243638753890991,
+      "eval_logits/rejected": -2.238759994506836,
+      "eval_logps/chosen": -34.016868591308594,
+      "eval_logps/rejected": -37.50496292114258,
+      "eval_loss": 0.6929702162742615,
+      "eval_rewards/accuracies": 0.5045680999755859,
+      "eval_rewards/chosen": 0.008841208182275295,
+      "eval_rewards/margins": 0.0030128166545182467,
+      "eval_rewards/rejected": 0.005828391760587692,
+      "eval_runtime": 145.9617,
       "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 300
@@ -493,122 +493,122 @@
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9295498132705688,
-      "logits/rejected": -1.9262762069702148,
-      "logps/chosen": -31.575878143310547,
-      "logps/rejected": -33.75602340698242,
-      "loss": 0.6856,
+      "logits/chosen": -1.928741216659546,
+      "logits/rejected": -1.9254982471466064,
+      "logps/chosen": -31.564533233642578,
+      "logps/rejected": -33.7502555847168,
+      "loss": 0.6827,
       "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.010988839901983738,
-      "rewards/margins": 0.0165377426892519,
-      "rewards/rejected": -0.0055489009246230125,
+      "rewards/chosen": 0.019408607855439186,
+      "rewards/margins": 0.0234612338244915,
+      "rewards/rejected": -0.004052626900374889,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9806196689605713,
-      "logits/rejected": -1.9683196544647217,
-      "logps/chosen": -34.554603576660156,
-      "logps/rejected": -33.58423614501953,
-      "loss": 0.6768,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.01658753491938114,
-      "rewards/margins": 0.034744132310152054,
-      "rewards/rejected": -0.01815659925341606,
+      "logits/chosen": -1.9805328845977783,
+      "logits/rejected": -1.968227744102478,
+      "logps/chosen": -34.543617248535156,
+      "logps/rejected": -33.547271728515625,
+      "loss": 0.6797,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.02622898854315281,
+      "rewards/margins": 0.03044353798031807,
+      "rewards/rejected": -0.004214548971503973,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.016143798828125,
-      "logits/rejected": -2.014680862426758,
-      "logps/chosen": -33.479766845703125,
-      "logps/rejected": -32.474754333496094,
-      "loss": 0.6937,
+      "logits/chosen": -2.016136884689331,
+      "logits/rejected": -2.014674663543701,
+      "logps/chosen": -33.480918884277344,
+      "logps/rejected": -32.46858215332031,
+      "loss": 0.6958,
       "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.002409630687907338,
-      "rewards/margins": 0.0004784400516655296,
-      "rewards/rejected": 0.0019311904907226562,
+      "rewards/chosen": 0.0024364828132092953,
+      "rewards/margins": -0.0030642927158623934,
+      "rewards/rejected": 0.005500775761902332,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.103015899658203,
-      "logits/rejected": -2.08721923828125,
-      "logps/chosen": -34.18162155151367,
-      "logps/rejected": -33.10049819946289,
-      "loss": 0.6982,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.0008922239649109542,
-      "rewards/margins": -0.007907899096608162,
-      "rewards/rejected": 0.0070156739093363285,
+      "logits/chosen": -2.1030609607696533,
+      "logits/rejected": -2.0872530937194824,
+      "logps/chosen": -34.16389083862305,
+      "logps/rejected": -33.10190963745117,
+      "loss": 0.695,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.007748591713607311,
+      "rewards/margins": -0.000315183395287022,
+      "rewards/rejected": 0.00806377548724413,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9744770526885986,
-      "logits/rejected": -1.9735443592071533,
-      "logps/chosen": -33.24970245361328,
-      "logps/rejected": -32.445960998535156,
-      "loss": 0.6916,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.012956788763403893,
-      "rewards/margins": 0.00528245884925127,
-      "rewards/rejected": 0.007674329914152622,
+      "logits/chosen": -1.9751472473144531,
+      "logits/rejected": -1.9742006063461304,
+      "logps/chosen": -33.24718475341797,
+      "logps/rejected": -32.467201232910156,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.017456945031881332,
+      "rewards/margins": 0.01848423480987549,
+      "rewards/rejected": -0.001027289079502225,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9305318593978882,
-      "logits/rejected": -1.9408973455429077,
-      "logps/chosen": -32.2140007019043,
-      "logps/rejected": -35.3133430480957,
-      "loss": 0.6908,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.0037243987899273634,
-      "rewards/margins": 0.006108095403760672,
-      "rewards/rejected": -0.00983249582350254,
+      "logits/chosen": -1.9307329654693604,
+      "logits/rejected": -1.9411048889160156,
+      "logps/chosen": -32.18671798706055,
+      "logps/rejected": -35.27191162109375,
+      "loss": 0.6942,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.008985924534499645,
+      "rewards/margins": 0.0005646973731927574,
+      "rewards/rejected": 0.00842122733592987,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.069563150405884,
-      "logits/rejected": -2.063000202178955,
-      "logps/chosen": -33.635009765625,
-      "logps/rejected": -29.221439361572266,
-      "loss": 0.6915,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.006782642100006342,
-      "rewards/margins": 0.004788074642419815,
-      "rewards/rejected": 0.0019945662934333086,
+      "logits/chosen": -2.0697169303894043,
+      "logits/rejected": -2.0631613731384277,
+      "logps/chosen": -33.653404235839844,
+      "logps/rejected": -29.2293701171875,
+      "loss": 0.6939,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.000717848539352417,
+      "rewards/margins": 0.0007542222738265991,
+      "rewards/rejected": -0.0014720708131790161,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.9291139841079712,
-      "logits/rejected": -1.9312822818756104,
-      "logps/chosen": -34.2513313293457,
-      "logps/rejected": -30.88556480407715,
-      "loss": 0.6938,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.004911360330879688,
-      "rewards/margins": 0.00026298072771169245,
-      "rewards/rejected": -0.005174341145902872,
+      "logits/chosen": -1.9293102025985718,
+      "logits/rejected": -1.931481122970581,
+      "logps/chosen": -34.22270965576172,
+      "logps/rejected": -30.89593505859375,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.008169981651008129,
+      "rewards/margins": 0.019822364673018456,
+      "rewards/rejected": -0.011652383022010326,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.6908149576806403,
-      "train_runtime": 3255.681,
-      "train_samples_per_second": 0.946,
+      "train_loss": 0.6914243283209862,
+      "train_runtime": 3256.6129,
+      "train_samples_per_second": 0.945,
       "train_steps_per_second": 0.118
     }
   ],