Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4937
-- Rewards/chosen: 0.2548
-- Rewards/rejected: 0.2285
-- Rewards/accuracies: 0.5274
-- Rewards/margins: 0.0263
-- Logps/rejected: -37.2627
-- Logps/chosen: -33.7514
-- Logits/rejected: -2.2263
-- Logits/chosen: -2.2311
+- Loss: 0.3845
+- Rewards/chosen: 0.0182
+- Rewards/rejected: 0.0177
+- Rewards/accuracies: 0.4934
+- Rewards/margins: 0.0005
+- Logps/rejected: -37.4970
+- Logps/chosen: -34.0143
+- Logits/rejected: -2.2316
+- Logits/chosen: -2.2364
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4639        | 0.26  | 100  | 0.4939          | 0.1483         | 0.1205           | 0.5627             | 0.0278          | -37.3828       | -33.8698     | -2.2305         | -2.2354       |
-| 0.4308        | 0.52  | 200  | 0.4894          | 0.2606         | 0.2153           | 0.5544             | 0.0453          | -37.2774       | -33.7450     | -2.2290         | -2.2338       |
-| 0.374         | 0.78  | 300  | 0.4904          | 0.2570         | 0.2171           | 0.5220             | 0.0399          | -37.2754       | -33.7490     | -2.2259         | -2.2308       |
+| 0.3038        | 0.26  | 100  | 0.3512          | 0.0196         | 0.0046           | 0.5424             | 0.0150          | -37.5115       | -34.0128     | -2.2324         | -2.2372       |
+| 0.3157        | 0.52  | 200  | 0.3716          | 0.0148         | -0.0016          | 0.5245             | 0.0164          | -37.5184       | -34.0181     | -2.2322         | -2.2371       |
+| 0.2156        | 0.78  | 300  | 0.3845          | 0.0182         | 0.0177           | 0.4934             | 0.0005          | -37.4970       | -34.0143     | -2.2316         | -2.2364       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c8bed71b5fca3295de74b9fc47c63b4ec9ac3352521fdd45d6a34763bb36821
+oid sha256:1697658a6a16b5b6fa72dbed3433705c0a58db99550d85b605badd7b5de5b313
 size 176183216

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2310752868652344,
-    "eval_logits/rejected": -2.226254940032959,
-    "eval_logps/chosen": -33.75144958496094,
-    "eval_logps/rejected": -37.262733459472656,
-    "eval_loss": 0.49370110034942627,
-    "eval_rewards/accuracies": 0.5274086594581604,
-    "eval_rewards/chosen": 0.25479215383529663,
-    "eval_rewards/margins": 0.02629559487104416,
-    "eval_rewards/rejected": 0.22849655151367188,
-    "eval_runtime": 145.6018,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.356,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.447351616079157,
-    "train_runtime": 3253.4458,
+    "train_loss": 0.31703355428460356,
+    "train_runtime": 3251.5033,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.447351616079157,
-    "train_runtime": 3253.4458,
+    "train_loss": 0.31703355428460356,
+    "train_runtime": 3251.5033,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.5,
+      "loss": 0.3086,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,590 +25,590 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8665987253189087,
-      "logits/rejected": -1.8709272146224976,
-      "logps/chosen": -36.985595703125,
-      "logps/rejected": -33.68160629272461,
-      "loss": 0.4886,
-      "rewards/accuracies": 0.5694444179534912,
-      "rewards/chosen": 0.018904482945799828,
-      "rewards/margins": 0.06528304517269135,
-      "rewards/rejected": -0.04637856408953667,
+      "logits/chosen": -1.8665881156921387,
+      "logits/rejected": -1.8709055185317993,
+      "logps/chosen": -36.99662399291992,
+      "logps/rejected": -33.65571594238281,
+      "loss": 0.3052,
+      "rewards/accuracies": 0.5416666865348816,
+      "rewards/chosen": 0.008980684913694859,
+      "rewards/margins": 0.032059140503406525,
+      "rewards/rejected": -0.02307845838367939,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.997780203819275,
-      "logits/rejected": -2.000434398651123,
-      "logps/chosen": -29.643661499023438,
-      "logps/rejected": -29.043325424194336,
-      "loss": 0.5031,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.001316396868787706,
-      "rewards/margins": -0.019422104582190514,
-      "rewards/rejected": 0.018105709925293922,
+      "logits/chosen": -1.9977455139160156,
+      "logits/rejected": -2.000382423400879,
+      "logps/chosen": -29.642925262451172,
+      "logps/rejected": -29.056737899780273,
+      "loss": 0.3525,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.0006573178106918931,
+      "rewards/margins": -0.00669272243976593,
+      "rewards/rejected": 0.006035405211150646,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9207446575164795,
-      "logits/rejected": -1.918060064315796,
-      "logps/chosen": -31.41064453125,
-      "logps/rejected": -33.227088928222656,
-      "loss": 0.4976,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.004905471112579107,
-      "rewards/margins": 0.012669263407588005,
-      "rewards/rejected": -0.007763790898025036,
+      "logits/chosen": -1.9204607009887695,
+      "logits/rejected": -1.9177772998809814,
+      "logps/chosen": -31.42336654663086,
+      "logps/rejected": -33.22785568237305,
+      "loss": 0.3603,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.006546213291585445,
+      "rewards/margins": 0.0019128695130348206,
+      "rewards/rejected": -0.00845908559858799,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.017446517944336,
-      "logits/rejected": -2.0087125301361084,
-      "logps/chosen": -32.553016662597656,
-      "logps/rejected": -32.50551986694336,
-      "loss": 0.4977,
+      "logits/chosen": -2.0172362327575684,
+      "logits/rejected": -2.008507251739502,
+      "logps/chosen": -32.56964874267578,
+      "logps/rejected": -32.50572967529297,
+      "loss": 0.3558,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.021415216848254204,
-      "rewards/margins": 0.014982220716774464,
-      "rewards/rejected": 0.006432999856770039,
+      "rewards/chosen": 0.006439635064452887,
+      "rewards/margins": 0.00019515231542754918,
+      "rewards/rejected": 0.006244482938200235,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8627235889434814,
-      "logits/rejected": -1.851959228515625,
-      "logps/chosen": -33.5064697265625,
-      "logps/rejected": -35.43267059326172,
-      "loss": 0.4951,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.0459100641310215,
-      "rewards/margins": 0.02820250764489174,
-      "rewards/rejected": 0.017707552760839462,
+      "logits/chosen": -1.8618619441986084,
+      "logits/rejected": -1.8510783910751343,
+      "logps/chosen": -33.56026077270508,
+      "logps/rejected": -35.45254898071289,
+      "loss": 0.3691,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.0025013976264744997,
+      "rewards/margins": -0.002315213903784752,
+      "rewards/rejected": -0.00018618404283188283,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9425691366195679,
-      "logits/rejected": -1.94449782371521,
-      "logps/chosen": -32.46650695800781,
-      "logps/rejected": -33.15652847290039,
-      "loss": 0.4765,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.12030963599681854,
-      "rewards/margins": 0.10694190114736557,
-      "rewards/rejected": 0.013367725536227226,
+      "logits/chosen": -1.9393202066421509,
+      "logits/rejected": -1.9412600994110107,
+      "logps/chosen": -32.57838439941406,
+      "logps/rejected": -33.215576171875,
+      "loss": 0.3063,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.01962057128548622,
+      "rewards/margins": 0.05940054729580879,
+      "rewards/rejected": -0.03977997973561287,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.073408842086792,
-      "logits/rejected": -2.078367233276367,
-      "logps/chosen": -33.917694091796875,
-      "logps/rejected": -36.547218322753906,
-      "loss": 0.4901,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.06371410191059113,
-      "rewards/margins": 0.04422418028116226,
-      "rewards/rejected": 0.019489921629428864,
+      "logits/chosen": -2.0718436241149902,
+      "logits/rejected": -2.0768017768859863,
+      "logps/chosen": -33.97806167602539,
+      "logps/rejected": -36.63082504272461,
+      "loss": 0.4257,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.00938049890100956,
+      "rewards/margins": 0.06513925641775131,
+      "rewards/rejected": -0.0557587556540966,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9349607229232788,
-      "logits/rejected": -1.9380786418914795,
-      "logps/chosen": -34.223785400390625,
-      "logps/rejected": -34.53069305419922,
-      "loss": 0.4713,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.1836203634738922,
-      "rewards/margins": 0.12807974219322205,
-      "rewards/rejected": 0.05554063245654106,
+      "logits/chosen": -1.9351739883422852,
+      "logits/rejected": -1.9383188486099243,
+      "logps/chosen": -34.33073043823242,
+      "logps/rejected": -34.61904525756836,
+      "loss": 0.2902,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.08736880123615265,
+      "rewards/margins": 0.11134655773639679,
+      "rewards/rejected": -0.023977745324373245,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9439691305160522,
-      "logits/rejected": -1.9484784603118896,
-      "logps/chosen": -32.27050018310547,
-      "logps/rejected": -32.26476287841797,
-      "loss": 0.4779,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.1772255003452301,
-      "rewards/margins": 0.09250012785196304,
-      "rewards/rejected": 0.08472537249326706,
+      "logits/chosen": -1.944392204284668,
+      "logits/rejected": -1.9489190578460693,
+      "logps/chosen": -32.419586181640625,
+      "logps/rejected": -32.3698844909668,
+      "loss": 0.3611,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.043051257729530334,
+      "rewards/margins": 0.052930813282728195,
+      "rewards/rejected": -0.009879561141133308,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.0411603450775146,
-      "logits/rejected": -2.039163112640381,
-      "logps/chosen": -31.98573875427246,
-      "logps/rejected": -31.193227767944336,
-      "loss": 0.4639,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.22958631813526154,
-      "rewards/margins": 0.16390272974967957,
-      "rewards/rejected": 0.06568360328674316,
+      "logits/chosen": -2.0419769287109375,
+      "logits/rejected": -2.0399627685546875,
+      "logps/chosen": -32.174407958984375,
+      "logps/rejected": -31.26608657836914,
+      "loss": 0.3038,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.059785228222608566,
+      "rewards/margins": 0.05967242643237114,
+      "rewards/rejected": 0.00011279433965682983,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.235391855239868,
-      "eval_logits/rejected": -2.2305493354797363,
-      "eval_logps/chosen": -33.869815826416016,
-      "eval_logps/rejected": -37.382774353027344,
-      "eval_loss": 0.4939241409301758,
-      "eval_rewards/accuracies": 0.5627076625823975,
-      "eval_rewards/chosen": 0.1482628434896469,
-      "eval_rewards/margins": 0.02780282311141491,
-      "eval_rewards/rejected": 0.12046003341674805,
-      "eval_runtime": 145.9747,
-      "eval_samples_per_second": 2.35,
+      "eval_logits/chosen": -2.2372143268585205,
+      "eval_logits/rejected": -2.2323503494262695,
+      "eval_logps/chosen": -34.01276779174805,
+      "eval_logps/rejected": -37.51152420043945,
+      "eval_loss": 0.3511974811553955,
+      "eval_rewards/accuracies": 0.5423588156700134,
+      "eval_rewards/chosen": 0.01960929110646248,
+      "eval_rewards/margins": 0.01502405758947134,
+      "eval_rewards/rejected": 0.004585230257362127,
+      "eval_runtime": 145.9032,
+      "eval_samples_per_second": 2.351,
       "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.997287392616272,
-      "logits/rejected": -1.9949369430541992,
-      "logps/chosen": -32.96843719482422,
-      "logps/rejected": -33.866310119628906,
-      "loss": 0.4739,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.24784216284751892,
-      "rewards/margins": 0.10108550637960434,
-      "rewards/rejected": 0.14675670862197876,
+      "logits/chosen": -1.998891830444336,
+      "logits/rejected": -1.996492624282837,
+      "logps/chosen": -33.14598083496094,
+      "logps/rejected": -34.020729064941406,
+      "loss": 0.4689,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.08805312216281891,
+      "rewards/margins": 0.08027410507202148,
+      "rewards/rejected": 0.007779018487781286,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.008091688156128,
-      "logits/rejected": -1.9997599124908447,
-      "logps/chosen": -32.20352554321289,
-      "logps/rejected": -31.995223999023438,
-      "loss": 0.485,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.2172430008649826,
-      "rewards/margins": 0.06759083271026611,
-      "rewards/rejected": 0.1496521681547165,
+      "logits/chosen": -2.010932445526123,
+      "logits/rejected": -2.0025696754455566,
+      "logps/chosen": -32.37172317504883,
+      "logps/rejected": -32.118797302246094,
+      "loss": 0.4465,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.06586603820323944,
+      "rewards/margins": 0.02742874063551426,
+      "rewards/rejected": 0.03843729570508003,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.035614490509033,
-      "logits/rejected": -2.027682304382324,
-      "logps/chosen": -30.1588077545166,
-      "logps/rejected": -31.886260986328125,
-      "loss": 0.4717,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.2852162718772888,
-      "rewards/margins": 0.1351451873779297,
-      "rewards/rejected": 0.15007111430168152,
+      "logits/chosen": -2.0387539863586426,
+      "logits/rejected": -2.030724287033081,
+      "logps/chosen": -30.41655921936035,
+      "logps/rejected": -32.060333251953125,
+      "loss": 0.3844,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.05324209854006767,
+      "rewards/margins": 0.05983499437570572,
+      "rewards/rejected": -0.00659290561452508,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.965490698814392,
-      "logits/rejected": -1.9756921529769897,
-      "logps/chosen": -31.065088272094727,
-      "logps/rejected": -32.42934036254883,
-      "loss": 0.4482,
+      "logits/chosen": -1.967858076095581,
+      "logits/rejected": -1.9781148433685303,
+      "logps/chosen": -31.223413467407227,
+      "logps/rejected": -32.55517578125,
+      "loss": 0.3904,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.3191176950931549,
-      "rewards/margins": 0.22413134574890137,
-      "rewards/rejected": 0.09498633444309235,
+      "rewards/chosen": 0.1766217201948166,
+      "rewards/margins": 0.1948881596326828,
+      "rewards/rejected": -0.01826643943786621,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8782259225845337,
-      "logits/rejected": -1.8793823719024658,
-      "logps/chosen": -33.68832778930664,
-      "logps/rejected": -34.58278274536133,
-      "loss": 0.4367,
+      "logits/chosen": -1.880910873413086,
+      "logits/rejected": -1.8820507526397705,
+      "logps/chosen": -34.01464080810547,
+      "logps/rejected": -34.783546447753906,
+      "loss": 0.3871,
       "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.45392999053001404,
-      "rewards/margins": 0.2969031035900116,
-      "rewards/rejected": 0.15702682733535767,
+      "rewards/chosen": 0.160243421792984,
+      "rewards/margins": 0.1839032918214798,
+      "rewards/rejected": -0.023659853264689445,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9295704364776611,
-      "logits/rejected": -1.9261808395385742,
-      "logps/chosen": -35.74212646484375,
-      "logps/rejected": -32.51028060913086,
-      "loss": 0.4538,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.38659390807151794,
-      "rewards/margins": 0.1953679323196411,
-      "rewards/rejected": 0.1912260353565216,
+      "logits/chosen": -1.933895468711853,
+      "logits/rejected": -1.9304730892181396,
+      "logps/chosen": -36.02853775024414,
+      "logps/rejected": -32.699058532714844,
+      "loss": 0.2729,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.1288261115550995,
+      "rewards/margins": 0.10750452429056168,
+      "rewards/rejected": 0.021321602165699005,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.031176805496216,
-      "logits/rejected": -2.023855686187744,
-      "logps/chosen": -33.24225616455078,
-      "logps/rejected": -31.193195343017578,
-      "loss": 0.42,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.5000473260879517,
-      "rewards/margins": 0.3572581112384796,
-      "rewards/rejected": 0.1427893042564392,
+      "logits/chosen": -2.0341715812683105,
+      "logits/rejected": -2.0267820358276367,
+      "logps/chosen": -33.55347442626953,
+      "logps/rejected": -31.3526554107666,
+      "loss": 0.294,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.21994857490062714,
+      "rewards/margins": 0.22067300975322723,
+      "rewards/rejected": -0.0007244400912895799,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.038222074508667,
-      "logits/rejected": -2.0434067249298096,
-      "logps/chosen": -31.95560646057129,
-      "logps/rejected": -32.17836380004883,
-      "loss": 0.4404,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.5302629470825195,
-      "rewards/margins": 0.2541634440422058,
-      "rewards/rejected": 0.2760995924472809,
+      "logits/chosen": -2.0400891304016113,
+      "logits/rejected": -2.045360565185547,
+      "logps/chosen": -32.370338439941406,
+      "logps/rejected": -32.4719123840332,
+      "loss": 0.2829,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.15700635313987732,
+      "rewards/margins": 0.145101398229599,
+      "rewards/rejected": 0.011904975399374962,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.0387401580810547,
-      "logits/rejected": -2.036006450653076,
-      "logps/chosen": -31.0674991607666,
-      "logps/rejected": -31.083877563476562,
-      "loss": 0.4607,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.3793107569217682,
-      "rewards/margins": 0.16841106116771698,
-      "rewards/rejected": 0.21089968085289001,
+      "logits/chosen": -2.041393280029297,
+      "logits/rejected": -2.038623809814453,
+      "logps/chosen": -31.328174591064453,
+      "logps/rejected": -31.316492080688477,
+      "loss": 0.3044,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.14470075070858002,
+      "rewards/margins": 0.1431477963924408,
+      "rewards/rejected": 0.0015529401134699583,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9085681438446045,
-      "logits/rejected": -1.9132543802261353,
-      "logps/chosen": -31.083459854125977,
-      "logps/rejected": -32.602638244628906,
-      "loss": 0.4308,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.4705420434474945,
-      "rewards/margins": 0.301074743270874,
-      "rewards/rejected": 0.16946731507778168,
+      "logits/chosen": -1.9122215509414673,
+      "logits/rejected": -1.9168663024902344,
+      "logps/chosen": -31.424020767211914,
+      "logps/rejected": -32.784080505371094,
+      "loss": 0.3157,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.16403506696224213,
+      "rewards/margins": 0.1578713059425354,
+      "rewards/rejected": 0.0061637843027710915,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2338287830352783,
-      "eval_logits/rejected": -2.229020357131958,
-      "eval_logps/chosen": -33.7449951171875,
-      "eval_logps/rejected": -37.27743911743164,
-      "eval_loss": 0.48942965269088745,
-      "eval_rewards/accuracies": 0.5544019937515259,
-      "eval_rewards/chosen": 0.26059985160827637,
-      "eval_rewards/margins": 0.0453372597694397,
-      "eval_rewards/rejected": 0.21526260673999786,
-      "eval_runtime": 145.8953,
-      "eval_samples_per_second": 2.351,
-      "eval_steps_per_second": 0.295,
+      "eval_logits/chosen": -2.237051010131836,
+      "eval_logits/rejected": -2.2321863174438477,
+      "eval_logps/chosen": -34.018070220947266,
+      "eval_logps/rejected": -37.51838684082031,
+      "eval_loss": 0.3716273605823517,
+      "eval_rewards/accuracies": 0.5245016813278198,
+      "eval_rewards/chosen": 0.014834923669695854,
+      "eval_rewards/margins": 0.016425320878624916,
+      "eval_rewards/rejected": -0.0015903981402516365,
+      "eval_runtime": 145.5151,
+      "eval_samples_per_second": 2.357,
+      "eval_steps_per_second": 0.296,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.0216596126556396,
-      "logits/rejected": -2.032275915145874,
-      "logps/chosen": -31.500268936157227,
-      "logps/rejected": -33.663352966308594,
-      "loss": 0.4458,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.42393389344215393,
-      "rewards/margins": 0.2445230931043625,
-      "rewards/rejected": 0.17941072583198547,
+      "logits/chosen": -2.023789882659912,
+      "logits/rejected": -2.034484386444092,
+      "logps/chosen": -31.767370223999023,
+      "logps/rejected": -33.890621185302734,
+      "loss": 0.275,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.18354059755802155,
+      "rewards/margins": 0.20867136120796204,
+      "rewards/rejected": -0.025130782276391983,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9136396646499634,
-      "logits/rejected": -1.928344964981079,
-      "logps/chosen": -29.588964462280273,
-      "logps/rejected": -31.396224975585938,
-      "loss": 0.4269,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.4703185558319092,
-      "rewards/margins": 0.3245617151260376,
-      "rewards/rejected": 0.1457568258047104,
+      "logits/chosen": -1.916685700416565,
+      "logits/rejected": -1.9314892292022705,
+      "logps/chosen": -29.956628799438477,
+      "logps/rejected": -31.564035415649414,
+      "loss": 0.2958,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.13942097127437592,
+      "rewards/margins": 0.14469322562217712,
+      "rewards/rejected": -0.005272268317639828,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.970298171043396,
-      "logits/rejected": -1.974283218383789,
-      "logps/chosen": -32.81959915161133,
-      "logps/rejected": -31.408565521240234,
-      "loss": 0.4109,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.5380831956863403,
-      "rewards/margins": 0.4114208221435547,
-      "rewards/rejected": 0.12666237354278564,
+      "logits/chosen": -1.9737945795059204,
+      "logits/rejected": -1.9777710437774658,
+      "logps/chosen": -33.19129180908203,
+      "logps/rejected": -31.5566463470459,
+      "loss": 0.287,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.20355579257011414,
+      "rewards/margins": 0.21016716957092285,
+      "rewards/rejected": -0.0066113718785345554,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9695065021514893,
-      "logits/rejected": -1.9477574825286865,
-      "logps/chosen": -33.58247756958008,
-      "logps/rejected": -34.828121185302734,
-      "loss": 0.4129,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.5141419172286987,
-      "rewards/margins": 0.403735876083374,
-      "rewards/rejected": 0.11040612310171127,
+      "logits/chosen": -1.9743419885635376,
+      "logits/rejected": -1.9523779153823853,
+      "logps/chosen": -33.9401969909668,
+      "logps/rejected": -35.008758544921875,
+      "loss": 0.3185,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.1921955645084381,
+      "rewards/margins": 0.24436470866203308,
+      "rewards/rejected": -0.052169155329465866,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.0098202228546143,
-      "logits/rejected": -2.0065340995788574,
-      "logps/chosen": -32.43529510498047,
-      "logps/rejected": -35.97461700439453,
-      "loss": 0.4514,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.44079580903053284,
-      "rewards/margins": 0.2155180424451828,
-      "rewards/rejected": 0.22527781128883362,
+      "logits/chosen": -2.0161709785461426,
+      "logits/rejected": -2.0128414630889893,
+      "logps/chosen": -32.74829864501953,
+      "logps/rejected": -36.236392974853516,
+      "loss": 0.2847,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.15909257531166077,
+      "rewards/margins": 0.1694144755601883,
+      "rewards/rejected": -0.010321905836462975,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8770506381988525,
-      "logits/rejected": -1.8746120929718018,
-      "logps/chosen": -33.7199821472168,
-      "logps/rejected": -35.28092575073242,
-      "loss": 0.4498,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.42157459259033203,
-      "rewards/margins": 0.2202514111995697,
-      "rewards/rejected": 0.2013232260942459,
+      "logits/chosen": -1.8847742080688477,
+      "logits/rejected": -1.8823268413543701,
+      "logps/chosen": -34.01182174682617,
+      "logps/rejected": -35.481346130371094,
+      "loss": 0.3073,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.1589193046092987,
+      "rewards/margins": 0.13797220587730408,
+      "rewards/rejected": 0.020947108045220375,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8618510961532593,
-      "logits/rejected": -1.8593294620513916,
-      "logps/chosen": -33.92017364501953,
-      "logps/rejected": -31.6002197265625,
-      "loss": 0.4397,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.42968273162841797,
-      "rewards/margins": 0.27568089962005615,
-      "rewards/rejected": 0.15400180220603943,
+      "logits/chosen": -1.8693536520004272,
+      "logits/rejected": -1.866838812828064,
+      "logps/chosen": -34.20549392700195,
+      "logps/rejected": -31.726673126220703,
+      "loss": 0.2733,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.17289286851882935,
+      "rewards/margins": 0.1327010840177536,
+      "rewards/rejected": 0.040191780775785446,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9657011032104492,
-      "logits/rejected": -1.9552650451660156,
-      "logps/chosen": -34.72232437133789,
-      "logps/rejected": -31.632369995117188,
-      "loss": 0.4114,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.5708868503570557,
-      "rewards/margins": 0.3831265866756439,
-      "rewards/rejected": 0.18776027858257294,
+      "logits/chosen": -1.9730831384658813,
+      "logits/rejected": -1.962480902671814,
+      "logps/chosen": -35.06049728393555,
+      "logps/rejected": -31.792781829833984,
+      "loss": 0.2523,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2665289342403412,
+      "rewards/margins": 0.22314274311065674,
+      "rewards/rejected": 0.04338619112968445,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0614376068115234,
-      "logits/rejected": -2.046600341796875,
-      "logps/chosen": -30.400625228881836,
-      "logps/rejected": -32.34136199951172,
-      "loss": 0.456,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.4711507260799408,
-      "rewards/margins": 0.19172403216362,
-      "rewards/rejected": 0.2794266939163208,
+      "logits/chosen": -2.0680813789367676,
+      "logits/rejected": -2.053079128265381,
+      "logps/chosen": -30.738479614257812,
+      "logps/rejected": -32.61243438720703,
+      "loss": 0.3194,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.16707859933376312,
+      "rewards/margins": 0.13161785900592804,
+      "rewards/rejected": 0.035460732877254486,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9332094192504883,
-      "logits/rejected": -1.9307467937469482,
-      "logps/chosen": -32.10976028442383,
-      "logps/rejected": -30.661523818969727,
-      "loss": 0.374,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.7356175184249878,
-      "rewards/margins": 0.5958597660064697,
-      "rewards/rejected": 0.13975778222084045,
+      "logits/chosen": -1.9398882389068604,
+      "logits/rejected": -1.9373395442962646,
+      "logps/chosen": -32.6181640625,
+      "logps/rejected": -30.843700408935547,
+      "loss": 0.2156,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2780519425868988,
+      "rewards/margins": 0.3022567331790924,
+      "rewards/rejected": -0.024204757064580917,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2307660579681396,
-      "eval_logits/rejected": -2.22594952583313,
-      "eval_logps/chosen": -33.74896240234375,
-      "eval_logps/rejected": -37.275413513183594,
-      "eval_loss": 0.49038100242614746,
-      "eval_rewards/accuracies": 0.5220099687576294,
-      "eval_rewards/chosen": 0.2570302486419678,
-      "eval_rewards/margins": 0.039943769574165344,
-      "eval_rewards/rejected": 0.21708647906780243,
-      "eval_runtime": 145.8077,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.2364187240600586,
+      "eval_logits/rejected": -2.2315518856048584,
+      "eval_logps/chosen": -34.01428985595703,
+      "eval_logps/rejected": -37.496952056884766,
+      "eval_loss": 0.38450533151626587,
+      "eval_rewards/accuracies": 0.49335551261901855,
+      "eval_rewards/chosen": 0.018236981704831123,
+      "eval_rewards/margins": 0.0005384809919632971,
+      "eval_rewards/rejected": 0.01769850216805935,
+      "eval_runtime": 145.7485,
+      "eval_samples_per_second": 2.353,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.9164836406707764,
-      "logits/rejected": -1.913336992263794,
-      "logps/chosen": -31.014041900634766,
-      "logps/rejected": -33.548377990722656,
-      "loss": 0.4218,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.5303782224655151,
-      "rewards/margins": 0.3559793531894684,
-      "rewards/rejected": 0.1743989735841751,
+      "logits/chosen": -1.922579050064087,
+      "logits/rejected": -1.9193273782730103,
+      "logps/chosen": -31.345911026000977,
+      "logps/rejected": -33.72126007080078,
+      "loss": 0.2917,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.23169513046741486,
+      "rewards/margins": 0.21289470791816711,
+      "rewards/rejected": 0.018800420686602592,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.967760682106018,
-      "logits/rejected": -1.955615758895874,
-      "logps/chosen": -34.05602264404297,
-      "logps/rejected": -33.42683410644531,
-      "loss": 0.4147,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.48604661226272583,
-      "rewards/margins": 0.385240375995636,
-      "rewards/rejected": 0.10080619156360626,
+      "logits/chosen": -1.9754743576049805,
+      "logits/rejected": -1.9631853103637695,
+      "logps/chosen": -34.408077239990234,
+      "logps/rejected": -33.58232879638672,
+      "loss": 0.2394,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.1691979616880417,
+      "rewards/margins": 0.20833876729011536,
+      "rewards/rejected": -0.03914082050323486,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -2.003138780593872,
-      "logits/rejected": -2.001786470413208,
-      "logps/chosen": -32.86919403076172,
-      "logps/rejected": -32.247493743896484,
-      "loss": 0.4259,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.5549365878105164,
-      "rewards/margins": 0.3460560441017151,
-      "rewards/rejected": 0.20888061821460724,
+      "logits/chosen": -2.0105607509613037,
+      "logits/rejected": -2.009115219116211,
+      "logps/chosen": -33.31591033935547,
+      "logps/rejected": -32.47368621826172,
+      "loss": 0.2769,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.15289874374866486,
+      "rewards/margins": 0.14759239554405212,
+      "rewards/rejected": 0.005306343547999859,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.0895984172821045,
-      "logits/rejected": -2.073963165283203,
-      "logps/chosen": -33.46659469604492,
-      "logps/rejected": -32.82307815551758,
-      "loss": 0.4171,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.6415189504623413,
-      "rewards/margins": 0.37605759501457214,
-      "rewards/rejected": 0.26546135544776917,
+      "logits/chosen": -2.096872091293335,
+      "logits/rejected": -2.0811073780059814,
+      "logps/chosen": -33.87510681152344,
+      "logps/rejected": -33.06427764892578,
+      "loss": 0.2792,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.27385497093200684,
+      "rewards/margins": 0.2254684418439865,
+      "rewards/rejected": 0.04838654398918152,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.9623206853866577,
-      "logits/rejected": -1.9614824056625366,
-      "logps/chosen": -32.549312591552734,
-      "logps/rejected": -32.24496078491211,
-      "loss": 0.4,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.6595046520233154,
-      "rewards/margins": 0.46133819222450256,
-      "rewards/rejected": 0.19816650450229645,
+      "logits/chosen": -1.969496488571167,
+      "logits/rejected": -1.9685735702514648,
+      "logps/chosen": -32.98945999145508,
+      "logps/rejected": -32.4643440246582,
+      "loss": 0.2958,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.26337358355522156,
+      "rewards/margins": 0.26264840364456177,
+      "rewards/rejected": 0.0007252089562825859,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9189163446426392,
-      "logits/rejected": -1.9292027950286865,
-      "logps/chosen": -31.60186767578125,
-      "logps/rejected": -34.987525939941406,
-      "loss": 0.4401,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.5425392985343933,
-      "rewards/margins": 0.27143171429634094,
-      "rewards/rejected": 0.27110758423805237,
+      "logits/chosen": -1.9255645275115967,
+      "logits/rejected": -1.9359004497528076,
+      "logps/chosen": -32.013362884521484,
+      "logps/rejected": -35.26326370239258,
+      "loss": 0.3425,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.1721937507390976,
+      "rewards/margins": 0.14924712479114532,
+      "rewards/rejected": 0.022946633398532867,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.0577945709228516,
-      "logits/rejected": -2.051274061203003,
-      "logps/chosen": -33.05121612548828,
-      "logps/rejected": -28.990320205688477,
-      "loss": 0.4233,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.5406805276870728,
-      "rewards/margins": 0.3281847834587097,
-      "rewards/rejected": 0.21249575912952423,
+      "logits/chosen": -2.0643956661224365,
+      "logits/rejected": -2.057886838912964,
+      "logps/chosen": -33.48772430419922,
+      "logps/rejected": -29.191638946533203,
+      "loss": 0.298,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.14782151579856873,
+      "rewards/margins": 0.11651048809289932,
+      "rewards/rejected": 0.03131101652979851,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.917284607887268,
-      "logits/rejected": -1.9194421768188477,
-      "logps/chosen": -33.59749984741211,
-      "logps/rejected": -30.708057403564453,
-      "loss": 0.4061,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.5773938298225403,
-      "rewards/margins": 0.429278701543808,
-      "rewards/rejected": 0.14811506867408752,
+      "logits/chosen": -1.9235725402832031,
+      "logits/rejected": -1.9257465600967407,
+      "logps/chosen": -33.965919494628906,
+      "logps/rejected": -30.839218139648438,
+      "loss": 0.2616,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.24581687152385712,
+      "rewards/margins": 0.2157471626996994,
+      "rewards/rejected": 0.03006969951093197,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.447351616079157,
-      "train_runtime": 3253.4458,
-      "train_samples_per_second": 0.946,
+      "train_loss": 0.31703355428460356,
+      "train_runtime": 3251.5033,
+      "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }
   ],