Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9980
-- Rewards/chosen: 0.0048
-- Rewards/rejected: 0.0026
-- Rewards/accuracies: 0.5141
-- Rewards/margins: 0.0022
-- Logps/rejected: -37.5100
-- Logps/chosen: -34.0224
-- Logits/rejected: -2.2385
-- Logits/chosen: -2.2434
+- Loss: 1.0014
+- Rewards/chosen: 0.0061
+- Rewards/rejected: 0.0075
+- Rewards/accuracies: 0.5104
+- Rewards/margins: -0.0014
+- Logps/rejected: -37.5016
+- Logps/chosen: -34.0223
+- Logits/rejected: -2.2389
+- Logits/chosen: -2.2438
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 1.0013        | 0.26  | 100  | 1.0033          | 0.0045         | 0.0078           | 0.4983             | -0.0032         | -37.4972       | -34.0232     | -2.2384         | -2.2433       |
-| 0.9961        | 0.52  | 200  | 0.9932          | 0.0063         | -0.0003          | 0.5328             | 0.0067          | -37.5175       | -34.0187     | -2.2389         | -2.2438       |
-| 0.9887        | 0.78  | 300  | 0.9999          | 0.0068         | 0.0066           | 0.4904             | 0.0002          | -37.5002       | -34.0177     | -2.2386         | -2.2435       |
+| 1.007         | 0.26  | 100  | 0.9963          | 0.0083         | 0.0045           | 0.5166             | 0.0039          | -37.5077       | -34.0179     | -2.2391         | -2.2440       |
+| 0.9725        | 0.52  | 200  | 1.0085          | 0.0008         | 0.0092           | 0.4992             | -0.0084         | -37.4982       | -34.0330     | -2.2385         | -2.2434       |
+| 0.982         | 0.78  | 300  | 1.0014          | 0.0061         | 0.0075           | 0.5104             | -0.0014         | -37.5016       | -34.0223     | -2.2389         | -2.2438       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:02cacc09633e03fcc85711e6b53860606ec088a35954fb973272bc1fbfae32be
+oid sha256:ad191a64493e95cbde4ef60db35429a7160e9cf1c921a1af3f24538d7fe45068
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2434139251708984,
-    "eval_logits/rejected": -2.2385354042053223,
-    "eval_logps/chosen": -34.0224494934082,
-    "eval_logps/rejected": -37.51002883911133,
-    "eval_loss": 0.9980219602584839,
-    "eval_rewards/accuracies": 0.5141196250915527,
-    "eval_rewards/chosen": 0.0048410315066576,
-    "eval_rewards/margins": 0.002204673830419779,
-    "eval_rewards/rejected": 0.0026363590732216835,
-    "eval_runtime": 145.7351,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.354,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.9964844827528124,
-    "train_runtime": 3255.8213,
+    "train_loss": 0.9930156608680626,
+    "train_runtime": 3255.7181,
     "train_samples": 3079,
     "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.9964844827528124,
-    "train_runtime": 3255.8213,
+    "train_loss": 0.9930156608680626,
+    "train_runtime": 3255.7181,
     "train_samples": 3079,
     "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -25,589 +25,589 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.866065502166748,
-      "logits/rejected": -1.8703795671463013,
-      "logps/chosen": -36.988380432128906,
-      "logps/rejected": -33.66728210449219,
-      "loss": 0.9778,
-      "rewards/accuracies": 0.5694444179534912,
-      "rewards/chosen": 0.007287923712283373,
-      "rewards/margins": 0.02217121794819832,
-      "rewards/rejected": -0.014883290976285934,
+      "logits/chosen": -1.8661351203918457,
+      "logits/rejected": -1.8704447746276855,
+      "logps/chosen": -36.994537353515625,
+      "logps/rejected": -33.65791320800781,
+      "loss": 0.98,
+      "rewards/accuracies": 0.5138888955116272,
+      "rewards/chosen": 0.006032215431332588,
+      "rewards/margins": 0.019952965900301933,
+      "rewards/rejected": -0.013920750468969345,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.997332215309143,
-      "logits/rejected": -1.999983549118042,
-      "logps/chosen": -29.625896453857422,
-      "logps/rejected": -29.035802841186523,
-      "loss": 1.0045,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.006518817041069269,
-      "rewards/margins": -0.004537253640592098,
-      "rewards/rejected": 0.01105607021600008,
+      "logits/chosen": -1.9972000122070312,
+      "logits/rejected": -1.9998512268066406,
+      "logps/chosen": -29.651025772094727,
+      "logps/rejected": -29.047237396240234,
+      "loss": 1.0125,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.004414811730384827,
+      "rewards/margins": -0.012516966089606285,
+      "rewards/rejected": 0.008102154359221458,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.9199495315551758,
-      "logits/rejected": -1.917249321937561,
-      "logps/chosen": -31.421478271484375,
-      "logps/rejected": -33.2115364074707,
-      "loss": 1.0049,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.0021541775204241276,
-      "rewards/margins": -0.004923977889120579,
-      "rewards/rejected": 0.002769801067188382,
+      "logits/chosen": -1.919605016708374,
+      "logits/rejected": -1.9169189929962158,
+      "logps/chosen": -31.410541534423828,
+      "logps/rejected": -33.22766876220703,
+      "loss": 0.9926,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.002776858163997531,
+      "rewards/margins": 0.007382071111351252,
+      "rewards/rejected": -0.004605212714523077,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.0169284343719482,
-      "logits/rejected": -2.008178949356079,
-      "logps/chosen": -32.59435272216797,
-      "logps/rejected": -32.49193572998047,
-      "loss": 1.0153,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.007017227355390787,
-      "rewards/margins": -0.015311767347157001,
-      "rewards/rejected": 0.008294540457427502,
+      "logits/chosen": -2.016796827316284,
+      "logits/rejected": -2.0080485343933105,
+      "logps/chosen": -32.59518814086914,
+      "logps/rejected": -32.49793243408203,
+      "loss": 1.0166,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.00919206440448761,
+      "rewards/margins": -0.016562189906835556,
+      "rewards/rejected": 0.007370126433670521,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.86457097530365,
-      "logits/rejected": -1.8537908792495728,
-      "logps/chosen": -33.56566619873047,
-      "logps/rejected": -35.423240661621094,
-      "loss": 1.0149,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.0032754994463175535,
-      "rewards/margins": -0.014917421154677868,
-      "rewards/rejected": 0.01164192147552967,
+      "logits/chosen": -1.8643512725830078,
+      "logits/rejected": -1.853578805923462,
+      "logps/chosen": -33.55515670776367,
+      "logps/rejected": -35.426795959472656,
+      "loss": 1.0116,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0011615634430199862,
+      "rewards/margins": -0.011613734066486359,
+      "rewards/rejected": 0.012775297276675701,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9449050426483154,
-      "logits/rejected": -1.9468472003936768,
-      "logps/chosen": -32.59955596923828,
-      "logps/rejected": -33.1828498840332,
-      "loss": 0.9952,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.00025113829178735614,
-      "rewards/margins": 0.0048376149497926235,
-      "rewards/rejected": -0.004586477763950825,
+      "logits/chosen": -1.9452184438705444,
+      "logits/rejected": -1.9471704959869385,
+      "logps/chosen": -32.57328414916992,
+      "logps/rejected": -33.17564010620117,
+      "loss": 0.9844,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.013449115678668022,
+      "rewards/margins": 0.015578977763652802,
+      "rewards/rejected": -0.002129861619323492,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.079878330230713,
-      "logits/rejected": -2.084862232208252,
-      "logps/chosen": -33.98878860473633,
-      "logps/rejected": -36.574462890625,
-      "loss": 0.9979,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.00012116949073970318,
-      "rewards/margins": 0.0021158286835998297,
-      "rewards/rejected": -0.002236998174339533,
+      "logits/chosen": -2.079322338104248,
+      "logits/rejected": -2.0843043327331543,
+      "logps/chosen": -33.998104095458984,
+      "logps/rejected": -36.586326599121094,
+      "loss": 0.9961,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.004809260368347168,
+      "rewards/margins": 0.003915563225746155,
+      "rewards/rejected": -0.008724823594093323,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9425569772720337,
-      "logits/rejected": -1.9457191228866577,
-      "logps/chosen": -34.40068054199219,
-      "logps/rejected": -34.5762939453125,
-      "loss": 0.9956,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.010850904509425163,
-      "rewards/margins": 0.004405139945447445,
-      "rewards/rejected": 0.006445765495300293,
+      "logits/chosen": -1.9419012069702148,
+      "logits/rejected": -1.9450546503067017,
+      "logps/chosen": -34.40644836425781,
+      "logps/rejected": -34.56013107299805,
+      "loss": 1.0055,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.010679833590984344,
+      "rewards/margins": -0.0054560392163693905,
+      "rewards/rejected": 0.016135873273015022,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.9507396221160889,
-      "logits/rejected": -1.9552500247955322,
-      "logps/chosen": -32.460357666015625,
-      "logps/rejected": -32.354434967041016,
-      "loss": 0.999,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.002827239688485861,
-      "rewards/margins": 0.001037138164974749,
-      "rewards/rejected": 0.0017901018727570772,
+      "logits/chosen": -1.9507659673690796,
+      "logits/rejected": -1.9552879333496094,
+      "logps/chosen": -32.45410919189453,
+      "logps/rejected": -32.34144973754883,
+      "loss": 1.0021,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.006657355930656195,
+      "rewards/margins": -0.002072322415187955,
+      "rewards/rejected": 0.008729678578674793,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.0492873191833496,
-      "logits/rejected": -2.0472888946533203,
-      "logps/chosen": -32.23810958862305,
-      "logps/rejected": -31.260278701782227,
-      "loss": 1.0013,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.0010914182057604194,
-      "rewards/margins": -0.001281978446058929,
-      "rewards/rejected": 0.0023733959533274174,
+      "logits/chosen": -2.0492916107177734,
+      "logits/rejected": -2.047295093536377,
+      "logps/chosen": -32.23640441894531,
+      "logps/rejected": -31.247751235961914,
+      "loss": 1.007,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.002216553781181574,
+      "rewards/margins": -0.007013450376689434,
+      "rewards/rejected": 0.009230004623532295,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.243312358856201,
-      "eval_logits/rejected": -2.238436222076416,
-      "eval_logps/chosen": -34.023216247558594,
-      "eval_logps/rejected": -37.49723434448242,
-      "eval_loss": 1.0032857656478882,
-      "eval_rewards/accuracies": 0.49833887815475464,
-      "eval_rewards/chosen": 0.004535459913313389,
-      "eval_rewards/margins": -0.0032192005310207605,
-      "eval_rewards/rejected": 0.007754660677164793,
-      "eval_runtime": 146.0197,
-      "eval_samples_per_second": 2.349,
+      "eval_logits/chosen": -2.2440249919891357,
+      "eval_logits/rejected": -2.239149808883667,
+      "eval_logps/chosen": -34.017887115478516,
+      "eval_logps/rejected": -37.50767517089844,
+      "eval_loss": 0.9963483810424805,
+      "eval_rewards/accuracies": 0.5166113376617432,
+      "eval_rewards/chosen": 0.008332207798957825,
+      "eval_rewards/margins": 0.0038595683872699738,
+      "eval_rewards/rejected": 0.004472639411687851,
+      "eval_runtime": 146.1438,
+      "eval_samples_per_second": 2.347,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.005169153213501,
-      "logits/rejected": -2.0027499198913574,
-      "logps/chosen": -33.2365837097168,
-      "logps/rejected": -34.01953125,
-      "loss": 1.001,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.0028920029290020466,
-      "rewards/margins": -0.0010445036459714174,
-      "rewards/rejected": 0.003936505876481533,
+      "logits/chosen": -2.005338191986084,
+      "logits/rejected": -2.002920627593994,
+      "logps/chosen": -33.22484588623047,
+      "logps/rejected": -34.01424026489258,
+      "loss": 0.9981,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.009485405869781971,
+      "rewards/margins": 0.001919901347719133,
+      "rewards/rejected": 0.007565504405647516,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.0166728496551514,
-      "logits/rejected": -2.0083022117614746,
-      "logps/chosen": -32.457847595214844,
-      "logps/rejected": -32.18357467651367,
-      "loss": 0.9964,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.005176951643079519,
-      "rewards/margins": 0.0036486864555627108,
-      "rewards/rejected": -0.008825639262795448,
+      "logits/chosen": -2.0164568424224854,
+      "logits/rejected": -2.008070468902588,
+      "logps/chosen": -32.454193115234375,
+      "logps/rejected": -32.175106048583984,
+      "loss": 0.9978,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.004645171575248241,
+      "rewards/margins": 0.0021557719446718693,
+      "rewards/rejected": -0.006800943519920111,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.0462448596954346,
-      "logits/rejected": -2.038203001022339,
-      "logps/chosen": -30.475027084350586,
-      "logps/rejected": -32.046302795410156,
-      "loss": 1.0024,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.00027531347586773336,
-      "rewards/margins": -0.002406105864793062,
-      "rewards/rejected": 0.0026814197190105915,
+      "logits/chosen": -2.046295642852783,
+      "logits/rejected": -2.03825044631958,
+      "logps/chosen": -30.491985321044922,
+      "logps/rejected": -32.04901885986328,
+      "loss": 1.0101,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.008134904317557812,
+      "rewards/margins": -0.010127933695912361,
+      "rewards/rejected": 0.001993028912693262,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.976986289024353,
-      "logits/rejected": -1.9872560501098633,
-      "logps/chosen": -31.407278060913086,
-      "logps/rejected": -32.543296813964844,
-      "loss": 0.9917,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.004953692201524973,
-      "rewards/margins": 0.008320841006934643,
-      "rewards/rejected": -0.003367149503901601,
+      "logits/chosen": -1.9772052764892578,
+      "logits/rejected": -1.9874699115753174,
+      "logps/chosen": -31.385913848876953,
+      "logps/rejected": -32.56010818481445,
+      "loss": 0.9705,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.016873350366950035,
+      "rewards/margins": 0.02948569692671299,
+      "rewards/rejected": -0.01261234562844038,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8905508518218994,
-      "logits/rejected": -1.891632080078125,
-      "logps/chosen": -34.20501708984375,
-      "logps/rejected": -34.77235412597656,
-      "loss": 0.9989,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.004930525086820126,
-      "rewards/margins": 0.0011081078555434942,
-      "rewards/rejected": -0.006038632709532976,
+      "logits/chosen": -1.8907018899917603,
+      "logits/rejected": -1.8917919397354126,
+      "logps/chosen": -34.191062927246094,
+      "logps/rejected": -34.757171630859375,
+      "loss": 0.9992,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0008138298871926963,
+      "rewards/margins": 0.0007720142602920532,
+      "rewards/rejected": 4.181563781457953e-05,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.94281005859375,
-      "logits/rejected": -1.939327597618103,
-      "logps/chosen": -36.144107818603516,
-      "logps/rejected": -32.72822570800781,
-      "loss": 0.9868,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.011029514484107494,
-      "rewards/margins": 0.013218574225902557,
-      "rewards/rejected": -0.002189057879149914,
+      "logits/chosen": -1.9424282312393188,
+      "logits/rejected": -1.9389431476593018,
+      "logps/chosen": -36.157745361328125,
+      "logps/rejected": -32.73271942138672,
+      "loss": 0.988,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0069676609709858894,
+      "rewards/margins": 0.011951583437621593,
+      "rewards/rejected": -0.004983922932296991,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.0420708656311035,
-      "logits/rejected": -2.0346803665161133,
-      "logps/chosen": -33.771934509277344,
-      "logps/rejected": -31.371145248413086,
-      "loss": 0.9819,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.010372580960392952,
-      "rewards/margins": 0.018090222030878067,
-      "rewards/rejected": -0.0077176401391625404,
+      "logits/chosen": -2.0421340465545654,
+      "logits/rejected": -2.0347371101379395,
+      "logps/chosen": -33.798614501953125,
+      "logps/rejected": -31.370548248291016,
+      "loss": 0.991,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.00037657321081496775,
+      "rewards/margins": 0.008971447125077248,
+      "rewards/rejected": -0.00934801995754242,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0475738048553467,
-      "logits/rejected": -2.0528526306152344,
-      "logps/chosen": -32.524593353271484,
-      "logps/rejected": -32.510643005371094,
-      "loss": 0.9817,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.008077163249254227,
-      "rewards/margins": 0.018277686089277267,
-      "rewards/rejected": -0.010200520977377892,
+      "logits/chosen": -2.0471675395965576,
+      "logits/rejected": -2.0524353981018066,
+      "logps/chosen": -32.519779205322266,
+      "logps/rejected": -32.49123001098633,
+      "loss": 0.9845,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.012504413723945618,
+      "rewards/margins": 0.015548673458397388,
+      "rewards/rejected": -0.003044259501621127,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.048490524291992,
-      "logits/rejected": -2.0457024574279785,
-      "logps/chosen": -31.492746353149414,
-      "logps/rejected": -31.319293975830078,
-      "loss": 1.0011,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.0015169475227594376,
-      "rewards/margins": -0.0010851218830794096,
-      "rewards/rejected": -0.0004318268911447376,
+      "logits/chosen": -2.0482120513916016,
+      "logits/rejected": -2.0454375743865967,
+      "logps/chosen": -31.485937118530273,
+      "logps/rejected": -31.332433700561523,
+      "loss": 0.9914,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0015064775943756104,
+      "rewards/margins": 0.008616042323410511,
+      "rewards/rejected": -0.007109564729034901,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9185683727264404,
-      "logits/rejected": -1.9232347011566162,
-      "logps/chosen": -31.5926513671875,
-      "logps/rejected": -32.78697204589844,
-      "loss": 0.9961,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.00545318890362978,
-      "rewards/margins": 0.0038701898884028196,
-      "rewards/rejected": 0.0015829993644729257,
+      "logits/chosen": -1.9187748432159424,
+      "logits/rejected": -1.9234533309936523,
+      "logps/chosen": -31.564071655273438,
+      "logps/rejected": -32.80377960205078,
+      "loss": 0.9725,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.021104933694005013,
+      "rewards/margins": 0.027528975158929825,
+      "rewards/rejected": -0.006424039602279663,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.243807315826416,
-      "eval_logits/rejected": -2.2389235496520996,
-      "eval_logps/chosen": -34.018714904785156,
-      "eval_logps/rejected": -37.517478942871094,
-      "eval_loss": 0.9932218790054321,
-      "eval_rewards/accuracies": 0.5328072905540466,
-      "eval_rewards/chosen": 0.006335962098091841,
-      "eval_rewards/margins": 0.006680537015199661,
-      "eval_rewards/rejected": -0.00034457523724995553,
-      "eval_runtime": 145.8414,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.24338698387146,
+      "eval_logits/rejected": -2.2385094165802,
+      "eval_logps/chosen": -34.032955169677734,
+      "eval_logps/rejected": -37.49824142456055,
+      "eval_loss": 1.0085184574127197,
+      "eval_rewards/accuracies": 0.4991694688796997,
+      "eval_rewards/chosen": 0.0007996229687705636,
+      "eval_rewards/margins": -0.008389986120164394,
+      "eval_rewards/rejected": 0.009189609438180923,
+      "eval_runtime": 145.9067,
+      "eval_samples_per_second": 2.351,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0318503379821777,
-      "logits/rejected": -2.042539596557617,
-      "logps/chosen": -31.948400497436523,
-      "logps/rejected": -33.86983871459961,
-      "loss": 0.988,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.009161925874650478,
-      "rewards/margins": 0.012017662636935711,
-      "rewards/rejected": -0.002855737926438451,
+      "logits/chosen": -2.0320358276367188,
+      "logits/rejected": -2.042719602584839,
+      "logps/chosen": -31.940053939819336,
+      "logps/rejected": -33.88956069946289,
+      "loss": 0.9709,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.015625199303030968,
+      "rewards/margins": 0.029055744409561157,
+      "rewards/rejected": -0.01343054324388504,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9251388311386108,
-      "logits/rejected": -1.9399843215942383,
-      "logps/chosen": -30.099853515625,
-      "logps/rejected": -31.55409812927246,
-      "loss": 0.997,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.004674358759075403,
-      "rewards/margins": 0.003043675096705556,
-      "rewards/rejected": 0.0016306828474625945,
+      "logits/chosen": -1.9255237579345703,
+      "logits/rejected": -1.9403839111328125,
+      "logps/chosen": -30.091089248657227,
+      "logps/rejected": -31.54927635192871,
+      "loss": 0.9942,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.010226013138890266,
+      "rewards/margins": 0.005776937119662762,
+      "rewards/rejected": 0.004449075553566217,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9835479259490967,
-      "logits/rejected": -1.9875112771987915,
-      "logps/chosen": -33.387638092041016,
-      "logps/rejected": -31.554845809936523,
-      "loss": 0.9859,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.011933470144867897,
-      "rewards/margins": 0.014149832539260387,
-      "rewards/rejected": -0.002216363325715065,
+      "logits/chosen": -1.9833370447158813,
+      "logits/rejected": -1.9873158931732178,
+      "logps/chosen": -33.393131256103516,
+      "logps/rejected": -31.559478759765625,
+      "loss": 0.9827,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.012168514542281628,
+      "rewards/margins": 0.017256852239370346,
+      "rewards/rejected": -0.005088338162750006,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9826898574829102,
-      "logits/rejected": -1.9607274532318115,
-      "logps/chosen": -34.158443450927734,
-      "logps/rejected": -34.963783264160156,
-      "loss": 0.9967,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.0018785831052809954,
-      "rewards/margins": 0.0033163924235850573,
-      "rewards/rejected": -0.005194975063204765,
+      "logits/chosen": -1.9829031229019165,
+      "logits/rejected": -1.9609248638153076,
+      "logps/chosen": -34.161617279052734,
+      "logps/rejected": -34.973785400390625,
+      "loss": 0.9924,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.003932595252990723,
+      "rewards/margins": 0.007563352584838867,
+      "rewards/rejected": -0.01149594783782959,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.024381160736084,
-      "logits/rejected": -2.0210862159729004,
-      "logps/chosen": -32.9254035949707,
-      "logps/rejected": -36.251712799072266,
-      "loss": 0.9894,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.00013314784155227244,
-      "rewards/margins": 0.01058159302920103,
-      "rewards/rejected": -0.010714741423726082,
+      "logits/chosen": -2.0244410037994385,
+      "logits/rejected": -2.0211281776428223,
+      "logps/chosen": -32.90896987915039,
+      "logps/rejected": -36.21443557739258,
+      "loss": 0.9972,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.008049068041145802,
+      "rewards/margins": 0.0028051852714270353,
+      "rewards/rejected": 0.0052438825368881226,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8911311626434326,
-      "logits/rejected": -1.8886839151382446,
-      "logps/chosen": -34.194557189941406,
-      "logps/rejected": -35.51445770263672,
-      "loss": 0.9985,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.0024632434360682964,
-      "rewards/margins": 0.0014695755671709776,
-      "rewards/rejected": -0.003932819236069918,
+      "logits/chosen": -1.8918297290802002,
+      "logits/rejected": -1.8893934488296509,
+      "logps/chosen": -34.178001403808594,
+      "logps/rejected": -35.52972412109375,
+      "loss": 0.9823,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.00519949197769165,
+      "rewards/margins": 0.017749812453985214,
+      "rewards/rejected": -0.012550321407616138,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8759450912475586,
-      "logits/rejected": -1.8734045028686523,
-      "logps/chosen": -34.40558624267578,
-      "logps/rejected": -31.752349853515625,
-      "loss": 1.0108,
+      "logits/chosen": -1.875998854637146,
+      "logits/rejected": -1.873445749282837,
+      "logps/chosen": -34.39997100830078,
+      "logps/rejected": -31.749908447265625,
+      "loss": 1.0119,
       "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.0031939218752086163,
-      "rewards/margins": -0.010787044651806355,
-      "rewards/rejected": 0.007593122310936451,
+      "rewards/chosen": -0.0011831462616100907,
+      "rewards/margins": -0.011894017457962036,
+      "rewards/rejected": 0.01071087084710598,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.980015754699707,
-      "logits/rejected": -1.9693737030029297,
-      "logps/chosen": -35.33230209350586,
-      "logps/rejected": -31.845691680908203,
-      "loss": 0.9884,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.009735691361129284,
-      "rewards/margins": 0.011615227907896042,
-      "rewards/rejected": -0.0018795346841216087,
+      "logits/chosen": -1.9797567129135132,
+      "logits/rejected": -1.9691368341445923,
+      "logps/chosen": -35.324066162109375,
+      "logps/rejected": -31.857070922851562,
+      "loss": 0.9757,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.016287770122289658,
+      "rewards/margins": 0.024326175451278687,
+      "rewards/rejected": -0.008038404397666454,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.0756278038024902,
-      "logits/rejected": -2.060606002807617,
-      "logps/chosen": -30.907390594482422,
-      "logps/rejected": -32.64055252075195,
-      "loss": 0.9978,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.00669338833540678,
-      "rewards/margins": 0.0021800189279019833,
-      "rewards/rejected": 0.00451336894184351,
+      "logits/chosen": -2.0752615928649902,
+      "logits/rejected": -2.06022310256958,
+      "logps/chosen": -30.904525756835938,
+      "logps/rejected": -32.66613006591797,
+      "loss": 0.9831,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.00980077963322401,
+      "rewards/margins": 0.016948198899626732,
+      "rewards/rejected": -0.007147419266402721,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.946616768836975,
-      "logits/rejected": -1.9440828561782837,
-      "logps/chosen": -32.894561767578125,
-      "logps/rejected": -30.812387466430664,
-      "loss": 0.9887,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.013020837679505348,
-      "rewards/margins": 0.011251090094447136,
-      "rewards/rejected": 0.0017697460716590285,
+      "logits/chosen": -1.946509599685669,
+      "logits/rejected": -1.9439618587493896,
+      "logps/chosen": -32.89145278930664,
+      "logps/rejected": -30.81719398498535,
+      "loss": 0.982,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.017829114571213722,
+      "rewards/margins": 0.018021035939455032,
+      "rewards/rejected": -0.00019192098989151418,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2434821128845215,
-      "eval_logits/rejected": -2.2386035919189453,
-      "eval_logps/chosen": -34.017669677734375,
-      "eval_logps/rejected": -37.50018310546875,
-      "eval_loss": 0.9998844861984253,
-      "eval_rewards/accuracies": 0.490448534488678,
-      "eval_rewards/chosen": 0.006753924302756786,
-      "eval_rewards/margins": 0.000178839749423787,
-      "eval_rewards/rejected": 0.0065750852227211,
-      "eval_runtime": 145.9162,
-      "eval_samples_per_second": 2.351,
+      "eval_logits/chosen": -2.2438137531280518,
+      "eval_logits/rejected": -2.2389373779296875,
+      "eval_logps/chosen": -34.02228546142578,
+      "eval_logps/rejected": -37.50155258178711,
+      "eval_loss": 1.0014283657073975,
+      "eval_rewards/accuracies": 0.5103820562362671,
+      "eval_rewards/chosen": 0.006133119110018015,
+      "eval_rewards/margins": -0.001401092391461134,
+      "eval_rewards/rejected": 0.007534211501479149,
+      "eval_runtime": 145.9391,
+      "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9287067651748657,
-      "logits/rejected": -1.925451636314392,
-      "logps/chosen": -31.603496551513672,
-      "logps/rejected": -33.734046936035156,
-      "loss": 1.0033,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -5.9673933719750494e-05,
-      "rewards/margins": -0.003302348079159856,
-      "rewards/rejected": 0.0032426740508526564,
+      "logits/chosen": -1.9287340641021729,
+      "logits/rejected": -1.9254703521728516,
+      "logps/chosen": -31.576568603515625,
+      "logps/rejected": -33.73060607910156,
+      "loss": 0.9924,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.013391390442848206,
+      "rewards/margins": 0.00761887151747942,
+      "rewards/rejected": 0.005772518925368786,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9802377223968506,
-      "logits/rejected": -1.9679291248321533,
-      "logps/chosen": -34.585323333740234,
-      "logps/rejected": -33.57084274291992,
-      "loss": 0.9829,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.00430017476901412,
-      "rewards/margins": 0.017101570963859558,
-      "rewards/rejected": -0.012801396660506725,
+      "logits/chosen": -1.980047583580017,
+      "logits/rejected": -1.9677263498306274,
+      "logps/chosen": -34.565635681152344,
+      "logps/rejected": -33.560508728027344,
+      "loss": 0.9739,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.015218675136566162,
+      "rewards/margins": 0.02605234459042549,
+      "rewards/rejected": -0.010833668522536755,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.015906810760498,
-      "logits/rejected": -2.014427661895752,
-      "logps/chosen": -33.49116516113281,
-      "logps/rejected": -32.47978973388672,
-      "loss": 1.0021,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.0021482703741639853,
-      "rewards/margins": -0.002064585220068693,
-      "rewards/rejected": -8.368515409529209e-05,
+      "logits/chosen": -2.0154004096984863,
+      "logits/rejected": -2.013948917388916,
+      "logps/chosen": -33.46211624145508,
+      "logps/rejected": -32.472103118896484,
+      "loss": 0.9919,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.011838942766189575,
+      "rewards/margins": 0.008100616745650768,
+      "rewards/rejected": 0.0037383257877081633,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.1030631065368652,
-      "logits/rejected": -2.0872654914855957,
-      "logps/chosen": -34.18492889404297,
-      "logps/rejected": -33.08319854736328,
-      "loss": 1.0161,
+      "logits/chosen": -2.1025614738464355,
+      "logits/rejected": -2.086764097213745,
+      "logps/chosen": -34.168495178222656,
+      "logps/rejected": -33.07737731933594,
+      "loss": 1.0149,
       "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.00221427483484149,
-      "rewards/margins": -0.01614934764802456,
-      "rewards/rejected": 0.013935071416199207,
+      "rewards/chosen": 0.005447447299957275,
+      "rewards/margins": -0.014881876297295094,
+      "rewards/rejected": 0.020329322665929794,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9745471477508545,
-      "logits/rejected": -1.9736032485961914,
-      "logps/chosen": -33.23271942138672,
-      "logps/rejected": -32.4765510559082,
-      "loss": 0.9757,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0197511725127697,
-      "rewards/margins": 0.024312298744916916,
-      "rewards/rejected": -0.004561126697808504,
+      "logits/chosen": -1.9740177392959595,
+      "logits/rejected": -1.9730663299560547,
+      "logps/chosen": -33.242271423339844,
+      "logps/rejected": -32.460182189941406,
+      "loss": 0.9826,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.019913053140044212,
+      "rewards/margins": 0.017432674765586853,
+      "rewards/rejected": 0.0024803788401186466,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9305438995361328,
-      "logits/rejected": -1.940913438796997,
-      "logps/chosen": -32.22040939331055,
-      "logps/rejected": -35.28728103637695,
-      "loss": 1.0069,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.006288432981818914,
-      "rewards/margins": -0.006879265420138836,
-      "rewards/rejected": 0.0005908325547352433,
+      "logits/chosen": -1.9304364919662476,
+      "logits/rejected": -1.9407978057861328,
+      "logps/chosen": -32.19029998779297,
+      "logps/rejected": -35.303955078125,
+      "loss": 0.9852,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.007193857338279486,
+      "rewards/margins": 0.014792281202971935,
+      "rewards/rejected": -0.00759842386469245,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.069648265838623,
-      "logits/rejected": -2.0630898475646973,
-      "logps/chosen": -33.63695526123047,
-      "logps/rejected": -29.226470947265625,
-      "loss": 0.994,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.006004731170833111,
-      "rewards/margins": 0.006023016758263111,
-      "rewards/rejected": -1.8286798876943067e-05,
+      "logits/chosen": -2.069343090057373,
+      "logits/rejected": -2.062788724899292,
+      "logps/chosen": -33.646358489990234,
+      "logps/rejected": -29.213809967041016,
+      "loss": 1.0035,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.0028047324158251286,
+      "rewards/margins": -0.0035038769710808992,
+      "rewards/rejected": 0.006308609154075384,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.928865671157837,
-      "logits/rejected": -1.9310123920440674,
-      "logps/chosen": -34.243560791015625,
-      "logps/rejected": -30.892742156982422,
-      "loss": 0.9938,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.001803267397917807,
-      "rewards/margins": 0.0062432498671114445,
-      "rewards/rejected": -0.00804651714861393,
+      "logits/chosen": -1.9290740489959717,
+      "logits/rejected": -1.9312365055084229,
+      "logps/chosen": -34.245155334472656,
+      "logps/rejected": -30.889087677001953,
+      "loss": 0.9948,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.003053072141483426,
+      "rewards/margins": 0.005176535341888666,
+      "rewards/rejected": -0.008229607716202736,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.9964844827528124,
-      "train_runtime": 3255.8213,
+      "train_loss": 0.9930156608680626,
+      "train_runtime": 3255.7181,
       "train_samples_per_second": 0.946,
       "train_steps_per_second": 0.118
     }