Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NorLLM-AI/NorMistral-7B
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: norllm-ai-normistral-7b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # norllm-ai-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/norllm-ai-normistral-7b-sft-qlora](https://huggingface.co/data/norllm-ai-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NorLLM-AI/NorMistral-7B](https://huggingface.co/NorLLM-AI/NorMistral-7B) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9071
-- Rewards/chosen: -0.0857
-- Rewards/rejected: -0.1842
-- Rewards/accuracies: 0.5714
-- Rewards/margins: 0.0984
-- Logps/rejected: -35.0665
-- Logps/chosen: -31.4539
-- Logits/rejected: -2.8132
-- Logits/chosen: -2.8155
+- Loss: 0.9279
+- Rewards/chosen: -0.0837
+- Rewards/rejected: -0.1567
+- Rewards/accuracies: 0.6125
+- Rewards/margins: 0.0730
+- Logps/rejected: -35.4817
+- Logps/chosen: -31.7010
+- Logits/rejected: -2.8247
+- Logits/chosen: -2.8267
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.841         | 0.26  | 100  | 0.9188          | -0.0118        | -0.0997          | 0.6100             | 0.0879          | -34.8976       | -31.3062     | -2.8098         | -2.8128       |
-| 0.7163        | 0.52  | 200  | 0.9221          | -0.0953        | -0.1781          | 0.5515             | 0.0829          | -35.0544       | -31.4730     | -2.8173         | -2.8196       |
-| 0.6439        | 0.78  | 300  | 0.9216          | -0.0864        | -0.1686          | 0.5573             | 0.0822          | -35.0354       | -31.4553     | -2.8133         | -2.8155       |
+| 0.9239        | 0.26  | 100  | 0.9661          | -0.0010        | -0.0351          | 0.6013             | 0.0342          | -34.8738       | -31.2872     | -2.8027         | -2.8055       |
+| 0.8146        | 0.52  | 200  | 0.9363          | -0.0747        | -0.1389          | 0.6184             | 0.0641          | -35.3925       | -31.6561     | -2.8206         | -2.8233       |
+| 0.7173        | 0.78  | 300  | 0.9279          | -0.0837        | -0.1567          | 0.6125             | 0.0730          | -35.4817       | -31.7010     | -2.8247         | -2.8267       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9892a403bb903d767ac251f52b187cda0e9f4702ada6dcc0a1d1f03e137824f7
+oid sha256:8c7928c54c9e4c718d39565f712549f734a55bc008dc207e23f0ce021bf92210
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.8154520988464355,
-    "eval_logits/rejected": -2.8132472038269043,
-    "eval_logps/chosen": -31.453943252563477,
-    "eval_logps/rejected": -35.06647872924805,
-    "eval_loss": 0.907057523727417,
-    "eval_rewards/accuracies": 0.5714285373687744,
-    "eval_rewards/chosen": -0.08574554324150085,
-    "eval_rewards/margins": 0.0984167754650116,
-    "eval_rewards/rejected": -0.18416231870651245,
-    "eval_runtime": 112.8077,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.041,
-    "eval_steps_per_second": 0.381,
-    "train_loss": 0.694122297732861,
-    "train_runtime": 2717.9333,
+    "train_loss": 0.8011291454364727,
+    "train_runtime": 2726.0249,
     "train_samples": 3079,
-    "train_samples_per_second": 1.133,
-    "train_steps_per_second": 0.142
+    "train_samples_per_second": 1.129,
+    "train_steps_per_second": 0.141
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.694122297732861,
-    "train_runtime": 2717.9333,
+    "train_loss": 0.8011291454364727,
+    "train_runtime": 2726.0249,
     "train_samples": 3079,
-    "train_samples_per_second": 1.133,
-    "train_steps_per_second": 0.142
+    "train_samples_per_second": 1.129,
+    "train_steps_per_second": 0.141
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 39.5,
+      "grad_norm": 15.8125,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": -2.7358343601226807,
       "logits/rejected": -2.7480404376983643,
@@ -25,630 +25,630 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 67.5,
+      "grad_norm": 27.375,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -3.0102524757385254,
-      "logits/rejected": -2.9992215633392334,
-      "logps/chosen": -33.198204040527344,
-      "logps/rejected": -31.97161865234375,
-      "loss": 1.0048,
-      "rewards/accuracies": 0.3888888955116272,
-      "rewards/chosen": -0.008629633113741875,
-      "rewards/margins": -0.00477217324078083,
-      "rewards/rejected": -0.0038574603386223316,
+      "logits/chosen": -3.0097196102142334,
+      "logits/rejected": -2.998093605041504,
+      "logps/chosen": -33.20063781738281,
+      "logps/rejected": -31.99871826171875,
+      "loss": 0.997,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": -0.003939596936106682,
+      "rewards/margins": 0.0030229436233639717,
+      "rewards/rejected": -0.006962540093809366,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 45.5,
+      "grad_norm": 18.375,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -2.8995842933654785,
-      "logits/rejected": -2.8944945335388184,
-      "logps/chosen": -32.47186279296875,
-      "logps/rejected": -28.95808982849121,
-      "loss": 1.0103,
+      "logits/chosen": -2.899087429046631,
+      "logits/rejected": -2.8937907218933105,
+      "logps/chosen": -32.46463394165039,
+      "logps/rejected": -28.943897247314453,
+      "loss": 1.0055,
       "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.0016174853080883622,
-      "rewards/margins": -0.01026993989944458,
-      "rewards/rejected": 0.011887425556778908,
+      "rewards/chosen": 0.0020922694820910692,
+      "rewards/margins": -0.005501673556864262,
+      "rewards/rejected": 0.0075939432717859745,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 44.25,
+      "grad_norm": 17.625,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -3.0970377922058105,
-      "logits/rejected": -3.1085267066955566,
-      "logps/chosen": -32.78118133544922,
-      "logps/rejected": -30.125417709350586,
-      "loss": 0.9716,
+      "logits/chosen": -3.096761465072632,
+      "logits/rejected": -3.1090149879455566,
+      "logps/chosen": -32.796714782714844,
+      "logps/rejected": -30.1170597076416,
+      "loss": 0.9934,
       "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.06375427544116974,
-      "rewards/margins": 0.02843797765672207,
-      "rewards/rejected": 0.03531629964709282,
+      "rewards/chosen": 0.022394269704818726,
+      "rewards/margins": 0.006596512161195278,
+      "rewards/rejected": 0.015797756612300873,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 36.0,
+      "grad_norm": 18.125,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.8625752925872803,
-      "logits/rejected": -2.853350877761841,
-      "logps/chosen": -31.52760887145996,
-      "logps/rejected": -32.38980484008789,
-      "loss": 0.8635,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.15135782957077026,
-      "rewards/margins": 0.16753093898296356,
-      "rewards/rejected": -0.016173111274838448,
+      "logits/chosen": -2.862929105758667,
+      "logits/rejected": -2.8537039756774902,
+      "logps/chosen": -31.520278930664062,
+      "logps/rejected": -32.38086700439453,
+      "loss": 0.9333,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.062008898705244064,
+      "rewards/margins": 0.06669147312641144,
+      "rewards/rejected": -0.004682569764554501,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 28.0,
+      "grad_norm": 12.4375,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -2.880800724029541,
-      "logits/rejected": -2.8784618377685547,
-      "logps/chosen": -29.383716583251953,
-      "logps/rejected": -30.082469940185547,
-      "loss": 0.8801,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.1782219111919403,
-      "rewards/margins": 0.17498424649238586,
-      "rewards/rejected": 0.00323770334944129,
+      "logits/chosen": -2.8800158500671387,
+      "logits/rejected": -2.877554416656494,
+      "logps/chosen": -29.325756072998047,
+      "logps/rejected": -30.063373565673828,
+      "loss": 0.9279,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.08288151025772095,
+      "rewards/margins": 0.07776711881160736,
+      "rewards/rejected": 0.005114386323839426,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 41.5,
+      "grad_norm": 16.125,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -2.9095096588134766,
-      "logits/rejected": -2.9110829830169678,
-      "logps/chosen": -29.854755401611328,
-      "logps/rejected": -28.0576114654541,
-      "loss": 0.8396,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.13750770688056946,
-      "rewards/margins": 0.18847230076789856,
-      "rewards/rejected": -0.0509646013379097,
+      "logits/chosen": -2.905949115753174,
+      "logits/rejected": -2.9077162742614746,
+      "logps/chosen": -29.794702529907227,
+      "logps/rejected": -27.984210968017578,
+      "loss": 0.9273,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.06701362133026123,
+      "rewards/margins": 0.07271867990493774,
+      "rewards/rejected": -0.005705048330128193,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 54.75,
+      "grad_norm": 21.5,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.9918501377105713,
-      "logits/rejected": -2.998004913330078,
-      "logps/chosen": -29.19577407836914,
-      "logps/rejected": -31.001514434814453,
-      "loss": 0.9064,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.042597077786922455,
-      "rewards/margins": 0.0974539965391159,
-      "rewards/rejected": -0.05485691875219345,
+      "logits/chosen": -2.988873243331909,
+      "logits/rejected": -2.995783567428589,
+      "logps/chosen": -29.117477416992188,
+      "logps/rejected": -30.8828067779541,
+      "loss": 0.9691,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.03269868344068527,
+      "rewards/margins": 0.030899733304977417,
+      "rewards/rejected": 0.001798946992494166,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 47.25,
+      "grad_norm": 18.75,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -2.8070733547210693,
-      "logits/rejected": -2.8222880363464355,
-      "logps/chosen": -29.365554809570312,
-      "logps/rejected": -29.91384506225586,
-      "loss": 0.8182,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.08758720755577087,
-      "rewards/margins": 0.20736832916736603,
-      "rewards/rejected": -0.11978113651275635,
+      "logits/chosen": -2.804410457611084,
+      "logits/rejected": -2.819267988204956,
+      "logps/chosen": -29.285558700561523,
+      "logps/rejected": -29.831363677978516,
+      "loss": 0.9175,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0510343499481678,
+      "rewards/margins": 0.0824500098824501,
+      "rewards/rejected": -0.0314156636595726,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 35.25,
+      "grad_norm": 18.875,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -2.8982958793640137,
-      "logits/rejected": -2.880619525909424,
-      "logps/chosen": -32.76674270629883,
-      "logps/rejected": -30.40105628967285,
-      "loss": 0.8554,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.03484407439827919,
-      "rewards/margins": 0.2245481014251709,
-      "rewards/rejected": -0.1897040456533432,
+      "logits/chosen": -2.893660068511963,
+      "logits/rejected": -2.876103401184082,
+      "logps/chosen": -32.681396484375,
+      "logps/rejected": -30.351022720336914,
+      "loss": 0.9037,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.03100699558854103,
+      "rewards/margins": 0.09688206017017365,
+      "rewards/rejected": -0.06587505340576172,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 34.5,
+      "grad_norm": 16.75,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -3.005671501159668,
-      "logits/rejected": -3.0055477619171143,
-      "logps/chosen": -31.862071990966797,
-      "logps/rejected": -30.99982261657715,
-      "loss": 0.841,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.09740196168422699,
-      "rewards/margins": 0.18922677636146545,
-      "rewards/rejected": -0.09182481467723846,
+      "logits/chosen": -2.998610734939575,
+      "logits/rejected": -3.0000457763671875,
+      "logps/chosen": -31.857208251953125,
+      "logps/rejected": -30.99690818786621,
+      "loss": 0.9239,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0399332270026207,
+      "rewards/margins": 0.07608047872781754,
+      "rewards/rejected": -0.03614724427461624,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.812790870666504,
-      "eval_logits/rejected": -2.8098013401031494,
-      "eval_logps/chosen": -31.30615234375,
-      "eval_logps/rejected": -34.89762496948242,
-      "eval_loss": 0.9188198447227478,
-      "eval_rewards/accuracies": 0.610049843788147,
-      "eval_rewards/chosen": -0.01184946671128273,
-      "eval_rewards/margins": 0.08788519352674484,
-      "eval_rewards/rejected": -0.09973466396331787,
-      "eval_runtime": 113.3423,
-      "eval_samples_per_second": 3.026,
-      "eval_steps_per_second": 0.379,
+      "eval_logits/chosen": -2.8054628372192383,
+      "eval_logits/rejected": -2.8027477264404297,
+      "eval_logps/chosen": -31.287212371826172,
+      "eval_logps/rejected": -34.87384033203125,
+      "eval_loss": 0.9661039710044861,
+      "eval_rewards/accuracies": 0.6013289093971252,
+      "eval_rewards/chosen": -0.0009517524158582091,
+      "eval_rewards/margins": 0.03418472036719322,
+      "eval_rewards/rejected": -0.035136472433805466,
+      "eval_runtime": 113.2847,
+      "eval_samples_per_second": 3.028,
+      "eval_steps_per_second": 0.38,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 52.75,
+      "grad_norm": 20.875,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -2.962007999420166,
-      "logits/rejected": -2.937967538833618,
-      "logps/chosen": -31.860488891601562,
-      "logps/rejected": -31.523914337158203,
-      "loss": 0.724,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.17815347015857697,
-      "rewards/margins": 0.3606840968132019,
-      "rewards/rejected": -0.18253062665462494,
+      "logits/chosen": -2.9547083377838135,
+      "logits/rejected": -2.9303781986236572,
+      "logps/chosen": -31.887340545654297,
+      "logps/rejected": -31.57730484008789,
+      "loss": 0.8618,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.0658910945057869,
+      "rewards/margins": 0.1495821326971054,
+      "rewards/rejected": -0.08369103074073792,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 44.5,
+      "grad_norm": 17.75,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -3.046175241470337,
-      "logits/rejected": -3.074599027633667,
-      "logps/chosen": -28.74391746520996,
-      "logps/rejected": -34.43209457397461,
-      "loss": 0.7306,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.2099936455488205,
-      "rewards/margins": 0.3522658348083496,
-      "rewards/rejected": -0.14227214455604553,
+      "logits/chosen": -3.036729335784912,
+      "logits/rejected": -3.066075563430786,
+      "logps/chosen": -28.683155059814453,
+      "logps/rejected": -34.55553436279297,
+      "loss": 0.8329,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.09614940732717514,
+      "rewards/margins": 0.1777469962835312,
+      "rewards/rejected": -0.08159758895635605,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 33.75,
+      "grad_norm": 17.0,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.750256061553955,
-      "logits/rejected": -2.7464325428009033,
-      "logps/chosen": -28.576923370361328,
-      "logps/rejected": -30.405838012695312,
-      "loss": 0.7558,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.19887787103652954,
-      "rewards/margins": 0.3348928391933441,
-      "rewards/rejected": -0.13601499795913696,
+      "logits/chosen": -2.742605209350586,
+      "logits/rejected": -2.7394351959228516,
+      "logps/chosen": -28.72283935546875,
+      "logps/rejected": -30.73993492126465,
+      "loss": 0.8454,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.050367455929517746,
+      "rewards/margins": 0.17159387469291687,
+      "rewards/rejected": -0.12122640758752823,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 29.625,
+      "grad_norm": 17.25,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -3.026742458343506,
-      "logits/rejected": -3.02447247505188,
-      "logps/chosen": -27.27823829650879,
-      "logps/rejected": -32.02156066894531,
-      "loss": 0.805,
+      "logits/chosen": -3.0202817916870117,
+      "logits/rejected": -3.0177478790283203,
+      "logps/chosen": -27.38101577758789,
+      "logps/rejected": -32.12105178833008,
+      "loss": 0.9013,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.10399170964956284,
-      "rewards/margins": 0.28638824820518494,
-      "rewards/rejected": -0.1823965609073639,
+      "rewards/chosen": 0.021041864529252052,
+      "rewards/margins": 0.11389818042516708,
+      "rewards/rejected": -0.09285632520914078,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 34.25,
+      "grad_norm": 17.5,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -2.8219058513641357,
-      "logits/rejected": -2.816573143005371,
-      "logps/chosen": -27.304168701171875,
-      "logps/rejected": -31.6512451171875,
-      "loss": 0.6938,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.26066717505455017,
-      "rewards/margins": 0.4744674563407898,
-      "rewards/rejected": -0.21380028128623962,
+      "logits/chosen": -2.815894603729248,
+      "logits/rejected": -2.8112242221832275,
+      "logps/chosen": -27.407995223999023,
+      "logps/rejected": -31.938213348388672,
+      "loss": 0.785,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.083502396941185,
+      "rewards/margins": 0.22641611099243164,
+      "rewards/rejected": -0.14291372895240784,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 47.0,
+      "grad_norm": 20.25,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -3.1381523609161377,
-      "logits/rejected": -3.121229648590088,
-      "logps/chosen": -31.75846290588379,
-      "logps/rejected": -29.593923568725586,
-      "loss": 0.5556,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.36662742495536804,
-      "rewards/margins": 0.6737016439437866,
-      "rewards/rejected": -0.30707424879074097,
+      "logits/chosen": -3.133810043334961,
+      "logits/rejected": -3.1171538829803467,
+      "logps/chosen": -31.85055160522461,
+      "logps/rejected": -30.0231990814209,
+      "loss": 0.7145,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.1282336711883545,
+      "rewards/margins": 0.33691832423210144,
+      "rewards/rejected": -0.20868465304374695,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 41.75,
+      "grad_norm": 18.875,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.9501750469207764,
-      "logits/rejected": -2.957343578338623,
-      "logps/chosen": -29.471771240234375,
-      "logps/rejected": -31.817256927490234,
-      "loss": 0.6244,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.21142145991325378,
-      "rewards/margins": 0.570308268070221,
-      "rewards/rejected": -0.35888683795928955,
+      "logits/chosen": -2.946866989135742,
+      "logits/rejected": -2.954603910446167,
+      "logps/chosen": -29.613689422607422,
+      "logps/rejected": -32.301788330078125,
+      "loss": 0.7392,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.05618465691804886,
+      "rewards/margins": 0.2966458201408386,
+      "rewards/rejected": -0.24046114087104797,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 37.25,
+      "grad_norm": 16.375,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.802713394165039,
-      "logits/rejected": -2.8004953861236572,
-      "logps/chosen": -29.250396728515625,
-      "logps/rejected": -30.246570587158203,
-      "loss": 0.6894,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.20972609519958496,
-      "rewards/margins": 0.44892430305480957,
-      "rewards/rejected": -0.2391982078552246,
+      "logits/chosen": -2.803523540496826,
+      "logits/rejected": -2.8009111881256104,
+      "logps/chosen": -29.574106216430664,
+      "logps/rejected": -30.58365249633789,
+      "loss": 0.839,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.019148189574480057,
+      "rewards/margins": 0.1822437047958374,
+      "rewards/rejected": -0.16309550404548645,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 26.25,
+      "grad_norm": 12.3125,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.9141011238098145,
-      "logits/rejected": -2.9099700450897217,
-      "logps/chosen": -29.81064796447754,
-      "logps/rejected": -28.773019790649414,
-      "loss": 0.7547,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.18803250789642334,
-      "rewards/margins": 0.3957784175872803,
-      "rewards/rejected": -0.20774583518505096,
+      "logits/chosen": -2.913604497909546,
+      "logits/rejected": -2.909559726715088,
+      "logps/chosen": -30.189212799072266,
+      "logps/rejected": -29.22299575805664,
+      "loss": 0.8485,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.000499642628710717,
+      "rewards/margins": 0.17259356379508972,
+      "rewards/rejected": -0.17309322953224182,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 27.625,
+      "grad_norm": 13.5625,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -2.9808688163757324,
-      "logits/rejected": -2.9687676429748535,
-      "logps/chosen": -32.98746871948242,
-      "logps/rejected": -30.666748046875,
-      "loss": 0.7163,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.3557746112346649,
-      "rewards/margins": 0.4721054136753082,
-      "rewards/rejected": -0.11633072048425674,
+      "logits/chosen": -2.9829163551330566,
+      "logits/rejected": -2.9701950550079346,
+      "logps/chosen": -33.297157287597656,
+      "logps/rejected": -31.153818130493164,
+      "loss": 0.8146,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.08037153631448746,
+      "rewards/margins": 0.22431731224060059,
+      "rewards/rejected": -0.14394578337669373,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.819563388824463,
-      "eval_logits/rejected": -2.817270278930664,
-      "eval_logps/chosen": -31.472963333129883,
-      "eval_logps/rejected": -35.05440902709961,
-      "eval_loss": 0.9220677018165588,
-      "eval_rewards/accuracies": 0.5514950156211853,
-      "eval_rewards/chosen": -0.09525591880083084,
-      "eval_rewards/margins": 0.0828702449798584,
-      "eval_rewards/rejected": -0.17812614142894745,
-      "eval_runtime": 113.0963,
-      "eval_samples_per_second": 3.033,
+      "eval_logits/chosen": -2.8232696056365967,
+      "eval_logits/rejected": -2.820622444152832,
+      "eval_logps/chosen": -31.65613555908203,
+      "eval_logps/rejected": -35.39246368408203,
+      "eval_loss": 0.9363206028938293,
+      "eval_rewards/accuracies": 0.6183555126190186,
+      "eval_rewards/chosen": -0.07473766803741455,
+      "eval_rewards/margins": 0.06412449479103088,
+      "eval_rewards/rejected": -0.13886214792728424,
+      "eval_runtime": 113.1194,
+      "eval_samples_per_second": 3.032,
       "eval_steps_per_second": 0.38,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 37.0,
+      "grad_norm": 22.0,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.9180774688720703,
-      "logits/rejected": -2.91807222366333,
-      "logps/chosen": -32.39319610595703,
-      "logps/rejected": -34.32632064819336,
-      "loss": 0.6144,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.3201075494289398,
-      "rewards/margins": 0.552533745765686,
-      "rewards/rejected": -0.23242616653442383,
+      "logits/chosen": -2.921879768371582,
+      "logits/rejected": -2.923065185546875,
+      "logps/chosen": -32.841976165771484,
+      "logps/rejected": -35.08049011230469,
+      "loss": 0.7569,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.03828667476773262,
+      "rewards/margins": 0.28209152817726135,
+      "rewards/rejected": -0.24380484223365784,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 30.25,
+      "grad_norm": 16.875,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -2.8982133865356445,
-      "logits/rejected": -2.9134175777435303,
-      "logps/chosen": -29.346981048583984,
-      "logps/rejected": -28.9754581451416,
-      "loss": 0.6305,
+      "logits/chosen": -2.905679225921631,
+      "logits/rejected": -2.9223124980926514,
+      "logps/chosen": -29.705821990966797,
+      "logps/rejected": -29.646404266357422,
+      "loss": 0.7313,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.415529727935791,
-      "rewards/margins": 0.6391232013702393,
-      "rewards/rejected": -0.22359342873096466,
+      "rewards/chosen": 0.09444301575422287,
+      "rewards/margins": 0.3180699944496155,
+      "rewards/rejected": -0.223626971244812,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 35.0,
+      "grad_norm": 17.5,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -2.9441702365875244,
-      "logits/rejected": -2.948596477508545,
-      "logps/chosen": -30.654144287109375,
-      "logps/rejected": -32.061561584472656,
-      "loss": 0.7505,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.2488628625869751,
-      "rewards/margins": 0.4005666673183441,
-      "rewards/rejected": -0.15170380473136902,
+      "logits/chosen": -2.9525959491729736,
+      "logits/rejected": -2.9570093154907227,
+      "logps/chosen": -30.988698959350586,
+      "logps/rejected": -32.67749786376953,
+      "loss": 0.8272,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.032633669674396515,
+      "rewards/margins": 0.21650373935699463,
+      "rewards/rejected": -0.1838700771331787,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 41.5,
+      "grad_norm": 17.875,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -2.9988656044006348,
-      "logits/rejected": -3.006716251373291,
-      "logps/chosen": -30.050405502319336,
-      "logps/rejected": -30.539688110351562,
-      "loss": 0.5427,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.42725998163223267,
-      "rewards/margins": 0.5794823169708252,
-      "rewards/rejected": -0.1522223949432373,
+      "logits/chosen": -3.010042667388916,
+      "logits/rejected": -3.0168490409851074,
+      "logps/chosen": -30.514179229736328,
+      "logps/rejected": -31.29245948791504,
+      "loss": 0.7261,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.07814909517765045,
+      "rewards/margins": 0.2895917594432831,
+      "rewards/rejected": -0.21144266426563263,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 39.75,
+      "grad_norm": 17.5,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.8314945697784424,
-      "logits/rejected": -2.8215582370758057,
-      "logps/chosen": -26.297893524169922,
-      "logps/rejected": -29.683481216430664,
-      "loss": 0.737,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.32816022634506226,
-      "rewards/margins": 0.4040789008140564,
-      "rewards/rejected": -0.07591867446899414,
+      "logits/chosen": -2.8428425788879395,
+      "logits/rejected": -2.8320090770721436,
+      "logps/chosen": -26.695632934570312,
+      "logps/rejected": -30.281723022460938,
+      "loss": 0.8345,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.051716603338718414,
+      "rewards/margins": 0.20173287391662598,
+      "rewards/rejected": -0.15001627802848816,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 24.875,
+      "grad_norm": 18.25,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -2.8124470710754395,
-      "logits/rejected": -2.83256196975708,
-      "logps/chosen": -28.941558837890625,
-      "logps/rejected": -34.62192916870117,
-      "loss": 0.5683,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.44420257210731506,
-      "rewards/margins": 0.6681241989135742,
-      "rewards/rejected": -0.22392161190509796,
+      "logits/chosen": -2.8217809200286865,
+      "logits/rejected": -2.8434066772460938,
+      "logps/chosen": -29.24324607849121,
+      "logps/rejected": -35.42533874511719,
+      "loss": 0.6631,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.11734296381473541,
+      "rewards/margins": 0.36759305000305176,
+      "rewards/rejected": -0.25025007128715515,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 44.25,
+      "grad_norm": 20.875,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -2.954413890838623,
-      "logits/rejected": -2.9607253074645996,
-      "logps/chosen": -30.094287872314453,
-      "logps/rejected": -30.396358489990234,
-      "loss": 0.5618,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.3435356020927429,
-      "rewards/margins": 0.6090848445892334,
-      "rewards/rejected": -0.2655491232872009,
+      "logits/chosen": -2.964048147201538,
+      "logits/rejected": -2.9707446098327637,
+      "logps/chosen": -30.70973777770996,
+      "logps/rejected": -31.365869522094727,
+      "loss": 0.6964,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.014324212446808815,
+      "rewards/margins": 0.3144455850124359,
+      "rewards/rejected": -0.3001214265823364,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 30.75,
+      "grad_norm": 17.375,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -2.9698939323425293,
-      "logits/rejected": -2.956036329269409,
-      "logps/chosen": -30.114521026611328,
-      "logps/rejected": -28.823684692382812,
-      "loss": 0.7337,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.30358150601387024,
-      "rewards/margins": 0.399749219417572,
-      "rewards/rejected": -0.09616778045892715,
+      "logits/chosen": -2.9795358180999756,
+      "logits/rejected": -2.9657275676727295,
+      "logps/chosen": -30.683574676513672,
+      "logps/rejected": -29.658838272094727,
+      "loss": 0.811,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.007621827535331249,
+      "rewards/margins": 0.21311911940574646,
+      "rewards/rejected": -0.20549730956554413,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 29.375,
+      "grad_norm": 17.875,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.8976500034332275,
-      "logits/rejected": -2.8795270919799805,
-      "logps/chosen": -31.536123275756836,
-      "logps/rejected": -31.159521102905273,
-      "loss": 0.4628,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.5500896573066711,
-      "rewards/margins": 0.8681961297988892,
-      "rewards/rejected": -0.31810641288757324,
+      "logits/chosen": -2.908496618270874,
+      "logits/rejected": -2.89042592048645,
+      "logps/chosen": -31.784709930419922,
+      "logps/rejected": -32.081748962402344,
+      "loss": 0.5814,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.17031878232955933,
+      "rewards/margins": 0.48200663924217224,
+      "rewards/rejected": -0.3116878867149353,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 30.25,
+      "grad_norm": 16.875,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -2.89839506149292,
-      "logits/rejected": -2.8952107429504395,
-      "logps/chosen": -31.317790985107422,
-      "logps/rejected": -27.9440860748291,
-      "loss": 0.6439,
+      "logits/chosen": -2.909313440322876,
+      "logits/rejected": -2.9058613777160645,
+      "logps/chosen": -31.75840187072754,
+      "logps/rejected": -28.829761505126953,
+      "loss": 0.7173,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.45242971181869507,
-      "rewards/margins": 0.6226793527603149,
-      "rewards/rejected": -0.17024962604045868,
+      "rewards/chosen": 0.09285064786672592,
+      "rewards/margins": 0.3380856215953827,
+      "rewards/rejected": -0.24523496627807617,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.8154501914978027,
-      "eval_logits/rejected": -2.8133084774017334,
-      "eval_logps/chosen": -31.455341339111328,
-      "eval_logps/rejected": -35.035438537597656,
-      "eval_loss": 0.9215981364250183,
-      "eval_rewards/accuracies": 0.5573089718818665,
-      "eval_rewards/chosen": -0.0864456370472908,
-      "eval_rewards/margins": 0.08219624310731888,
-      "eval_rewards/rejected": -0.16864188015460968,
-      "eval_runtime": 113.1056,
-      "eval_samples_per_second": 3.033,
-      "eval_steps_per_second": 0.38,
+      "eval_logits/chosen": -2.8267321586608887,
+      "eval_logits/rejected": -2.8247475624084473,
+      "eval_logps/chosen": -31.700992584228516,
+      "eval_logps/rejected": -35.481746673583984,
+      "eval_loss": 0.9279295802116394,
+      "eval_rewards/accuracies": 0.6125415563583374,
+      "eval_rewards/chosen": -0.08370788395404816,
+      "eval_rewards/margins": 0.07301054149866104,
+      "eval_rewards/rejected": -0.1567184180021286,
+      "eval_runtime": 112.8754,
+      "eval_samples_per_second": 3.039,
+      "eval_steps_per_second": 0.381,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 39.5,
+      "grad_norm": 18.875,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -2.779205560684204,
-      "logits/rejected": -2.7966835498809814,
-      "logps/chosen": -28.479604721069336,
-      "logps/rejected": -31.288341522216797,
-      "loss": 0.5317,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.37490805983543396,
-      "rewards/margins": 0.6530753374099731,
-      "rewards/rejected": -0.2781672477722168,
+      "logits/chosen": -2.7911877632141113,
+      "logits/rejected": -2.8082525730133057,
+      "logps/chosen": -29.013874053955078,
+      "logps/rejected": -32.23237228393555,
+      "loss": 0.6971,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.04310917109251022,
+      "rewards/margins": 0.34318238496780396,
+      "rewards/rejected": -0.3000732362270355,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 32.25,
+      "grad_norm": 17.875,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -3.0243513584136963,
-      "logits/rejected": -3.0100343227386475,
-      "logps/chosen": -29.37557601928711,
-      "logps/rejected": -28.844478607177734,
-      "loss": 0.5359,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.41800540685653687,
-      "rewards/margins": 0.6808558106422424,
-      "rewards/rejected": -0.262850284576416,
+      "logits/chosen": -3.034618616104126,
+      "logits/rejected": -3.0209412574768066,
+      "logps/chosen": -29.51166343688965,
+      "logps/rejected": -29.482135772705078,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.13998422026634216,
+      "rewards/margins": 0.3726558983325958,
+      "rewards/rejected": -0.23267166316509247,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 22.875,
+      "grad_norm": 16.375,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -2.9381539821624756,
-      "logits/rejected": -2.920642852783203,
-      "logps/chosen": -27.252010345458984,
-      "logps/rejected": -31.086780548095703,
-      "loss": 0.4505,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.5550299882888794,
-      "rewards/margins": 0.9025982022285461,
-      "rewards/rejected": -0.3475683331489563,
+      "logits/chosen": -2.9494917392730713,
+      "logits/rejected": -2.9318935871124268,
+      "logps/chosen": -27.68900489807129,
+      "logps/rejected": -31.994842529296875,
+      "loss": 0.6228,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.1346132755279541,
+      "rewards/margins": 0.4552524983882904,
+      "rewards/rejected": -0.3206392228603363,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 42.75,
+      "grad_norm": 20.25,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -3.1556878089904785,
-      "logits/rejected": -3.1616902351379395,
-      "logps/chosen": -30.897449493408203,
-      "logps/rejected": -33.362003326416016,
-      "loss": 0.532,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.3708776533603668,
-      "rewards/margins": 0.7038747072219849,
-      "rewards/rejected": -0.33299699425697327,
+      "logits/chosen": -3.168849468231201,
+      "logits/rejected": -3.174539089202881,
+      "logps/chosen": -31.354122161865234,
+      "logps/rejected": -34.371620178222656,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.057016681879758835,
+      "rewards/margins": 0.39213961362838745,
+      "rewards/rejected": -0.3351229131221771,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 36.25,
+      "grad_norm": 18.75,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -3.032553195953369,
-      "logits/rejected": -3.0359413623809814,
-      "logps/chosen": -29.842823028564453,
-      "logps/rejected": -31.924610137939453,
-      "loss": 0.5238,
+      "logits/chosen": -3.0446276664733887,
+      "logits/rejected": -3.0476937294006348,
+      "logps/chosen": -30.175695419311523,
+      "logps/rejected": -32.66219711303711,
+      "loss": 0.6878,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.4820283055305481,
-      "rewards/margins": 0.6876784563064575,
-      "rewards/rejected": -0.2056502103805542,
+      "rewards/chosen": 0.12623663246631622,
+      "rewards/margins": 0.35601407289505005,
+      "rewards/rejected": -0.22977742552757263,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 36.25,
+      "grad_norm": 20.25,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -2.871373414993286,
-      "logits/rejected": -2.8733484745025635,
-      "logps/chosen": -27.545650482177734,
-      "logps/rejected": -30.173965454101562,
-      "loss": 0.5016,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.5520386099815369,
-      "rewards/margins": 0.7235507965087891,
-      "rewards/rejected": -0.1715122014284134,
+      "logits/chosen": -2.8835089206695557,
+      "logits/rejected": -2.8853554725646973,
+      "logps/chosen": -27.79741859436035,
+      "logps/rejected": -30.80864906311035,
+      "loss": 0.6737,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.1704617589712143,
+      "rewards/margins": 0.3660039007663727,
+      "rewards/rejected": -0.19554206728935242,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 50.75,
+      "grad_norm": 24.375,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.948784589767456,
-      "logits/rejected": -2.9460253715515137,
-      "logps/chosen": -29.852313995361328,
-      "logps/rejected": -32.32441329956055,
-      "loss": 0.5601,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.46716976165771484,
-      "rewards/margins": 0.6653568148612976,
-      "rewards/rejected": -0.19818702340126038,
+      "logits/chosen": -2.960698127746582,
+      "logits/rejected": -2.958723306655884,
+      "logps/chosen": -30.233112335205078,
+      "logps/rejected": -33.06101608276367,
+      "loss": 0.7086,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.1107078567147255,
+      "rewards/margins": 0.33730247616767883,
+      "rewards/rejected": -0.22659464180469513,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 37.5,
+      "grad_norm": 18.25,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -2.9041707515716553,
-      "logits/rejected": -2.886718273162842,
-      "logps/chosen": -28.160015106201172,
-      "logps/rejected": -28.389995574951172,
-      "loss": 0.5382,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.3925102651119232,
-      "rewards/margins": 0.6426987648010254,
-      "rewards/rejected": -0.25018855929374695,
+      "logits/chosen": -2.9156675338745117,
+      "logits/rejected": -2.898934841156006,
+      "logps/chosen": -28.734481811523438,
+      "logps/rejected": -29.24869728088379,
+      "loss": 0.7185,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.04211098328232765,
+      "rewards/margins": 0.31392616033554077,
+      "rewards/rejected": -0.2718151807785034,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.694122297732861,
-      "train_runtime": 2717.9333,
-      "train_samples_per_second": 1.133,
-      "train_steps_per_second": 0.142
+      "train_loss": 0.8011291454364727,
+      "train_runtime": 2726.0249,
+      "train_samples_per_second": 1.129,
+      "train_steps_per_second": 0.141
     }
   ],
   "logging_steps": 10,