Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NorLLM-AI/NorMistral-7B
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: norllm-ai-normistral-7b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # norllm-ai-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/norllm-ai-normistral-7b-sft-qlora](https://huggingface.co/data/norllm-ai-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NorLLM-AI/NorMistral-7B](https://huggingface.co/NorLLM-AI/NorMistral-7B) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9105
-- Rewards/chosen: -0.0836
-- Rewards/rejected: -0.1783
-- Rewards/accuracies: 0.5515
-- Rewards/margins: 0.0947
-- Logps/rejected: -35.1438
-- Logps/chosen: -31.4914
-- Logits/rejected: -2.8211
-- Logits/chosen: -2.8232
+- Loss: 0.9559
+- Rewards/chosen: -0.0753
+- Rewards/rejected: -0.1195
+- Rewards/accuracies: 0.6005
+- Rewards/margins: 0.0441
+- Logps/rejected: -35.8927
+- Logps/chosen: -32.0356
+- Logits/rejected: -2.8365
+- Logits/chosen: -2.8392
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.8436        | 0.26  | 100  | 0.9278          | -0.0021        | -0.0777          | 0.5984             | 0.0756          | -34.8925       | -31.2877     | -2.8097         | -2.8124       |
-| 0.7459        | 0.52  | 200  | 0.9159          | -0.0693        | -0.1587          | 0.5835             | 0.0894          | -35.0949       | -31.4556     | -2.8227         | -2.8250       |
-| 0.665         | 0.78  | 300  | 0.9202          | -0.0777        | -0.1629          | 0.5656             | 0.0852          | -35.1054       | -31.4766     | -2.8221         | -2.8242       |
+| 0.9581        | 0.26  | 100  | 0.9819          | 0.0013         | -0.0170          | 0.5955             | 0.0182          | -34.8678       | -31.2699     | -2.8027         | -2.8053       |
+| 0.8824        | 0.52  | 200  | 0.9602          | -0.0546        | -0.0945          | 0.6092             | 0.0399          | -35.6429       | -31.8280     | -2.8288         | -2.8315       |
+| 0.8245        | 0.78  | 300  | 0.9559          | -0.0753        | -0.1195          | 0.6005             | 0.0441          | -35.8927       | -32.0356     | -2.8365         | -2.8392       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c849f2718f3c0470fc0716ff73825ce0a2bd725daff6159865f27d890e56ce76
+oid sha256:7a283772062be6a02cdb1fec51f20731bfc277bc3a6038cdf2bb5811ddde0a0f
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.8232436180114746,
-    "eval_logits/rejected": -2.8211076259613037,
-    "eval_logps/chosen": -31.491378784179688,
-    "eval_logps/rejected": -35.14382553100586,
-    "eval_loss": 0.9104546904563904,
-    "eval_rewards/accuracies": 0.5514950156211853,
-    "eval_rewards/chosen": -0.08357102423906326,
-    "eval_rewards/margins": 0.09469697624444962,
-    "eval_rewards/rejected": -0.17826800048351288,
-    "eval_runtime": 112.9595,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.036,
-    "eval_steps_per_second": 0.381,
-    "train_loss": 0.7203964704042906,
-    "train_runtime": 2720.5927,
+    "train_loss": 0.8805358874333369,
+    "train_runtime": 2723.8224,
     "train_samples": 3079,
-    "train_samples_per_second": 1.132,
-    "train_steps_per_second": 0.142
+    "train_samples_per_second": 1.13,
+    "train_steps_per_second": 0.141
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.7203964704042906,
-    "train_runtime": 2720.5927,
+    "train_loss": 0.8805358874333369,
+    "train_runtime": 2723.8224,
     "train_samples": 3079,
-    "train_samples_per_second": 1.132,
-    "train_steps_per_second": 0.142
+    "train_samples_per_second": 1.13,
+    "train_steps_per_second": 0.141
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 31.625,
+      "grad_norm": 7.90625,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": -2.7358343601226807,
       "logits/rejected": -2.7480404376983643,
@@ -25,630 +25,630 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 53.5,
+      "grad_norm": 13.4375,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -3.0091097354888916,
-      "logits/rejected": -2.998112678527832,
-      "logps/chosen": -33.19804763793945,
-      "logps/rejected": -31.97610092163086,
-      "loss": 1.002,
-      "rewards/accuracies": 0.4722222089767456,
-      "rewards/chosen": -0.006841929163783789,
-      "rewards/margins": -0.0019655984360724688,
-      "rewards/rejected": -0.004876333754509687,
+      "logits/chosen": -3.009408473968506,
+      "logits/rejected": -2.9977328777313232,
+      "logps/chosen": -33.20761489868164,
+      "logps/rejected": -31.960758209228516,
+      "loss": 1.003,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.0026669783983379602,
+      "rewards/margins": -0.002981698838993907,
+      "rewards/rejected": 0.00031472102273255587,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 36.75,
+      "grad_norm": 9.1875,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -2.899024248123169,
-      "logits/rejected": -2.8939948081970215,
-      "logps/chosen": -32.47606658935547,
-      "logps/rejected": -28.96236228942871,
-      "loss": 1.0082,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.0003868088242597878,
-      "rewards/margins": -0.008188480511307716,
-      "rewards/rejected": 0.007801672909408808,
+      "logits/chosen": -2.8990566730499268,
+      "logits/rejected": -2.894094944000244,
+      "logps/chosen": -32.50642013549805,
+      "logps/rejected": -28.963293075561523,
+      "loss": 1.005,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.0031326524913311005,
+      "rewards/margins": -0.00499005988240242,
+      "rewards/rejected": 0.0018574076239019632,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 35.25,
+      "grad_norm": 8.8125,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -3.097607135772705,
-      "logits/rejected": -3.109447956085205,
-      "logps/chosen": -32.788970947265625,
-      "logps/rejected": -30.134746551513672,
-      "loss": 0.9766,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.047886691987514496,
-      "rewards/margins": 0.023363908752799034,
-      "rewards/rejected": 0.02452278696000576,
+      "logits/chosen": -3.0970427989959717,
+      "logits/rejected": -3.108668804168701,
+      "logps/chosen": -32.80826187133789,
+      "logps/rejected": -30.164159774780273,
+      "loss": 0.9931,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.010042307898402214,
+      "rewards/margins": 0.006853123661130667,
+      "rewards/rejected": 0.0031891819089651108,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 33.0,
+      "grad_norm": 9.0625,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.8632471561431885,
-      "logits/rejected": -2.853929042816162,
-      "logps/chosen": -31.501651763916016,
-      "logps/rejected": -32.37067413330078,
-      "loss": 0.8747,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.1314682960510254,
-      "rewards/margins": 0.13675658404827118,
-      "rewards/rejected": -0.005288311745971441,
+      "logits/chosen": -2.8629202842712402,
+      "logits/rejected": -2.8535947799682617,
+      "logps/chosen": -31.525354385375977,
+      "logps/rejected": -32.399391174316406,
+      "loss": 0.9653,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.030497059226036072,
+      "rewards/margins": 0.034690458327531815,
+      "rewards/rejected": -0.004193395376205444,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 22.5,
+      "grad_norm": 8.1875,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -2.881110191345215,
-      "logits/rejected": -2.878807783126831,
-      "logps/chosen": -29.374563217163086,
-      "logps/rejected": -30.105758666992188,
-      "loss": 0.886,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.1462380588054657,
-      "rewards/margins": 0.15296456217765808,
-      "rewards/rejected": -0.006726491264998913,
+      "logits/chosen": -2.87955904006958,
+      "logits/rejected": -2.877352714538574,
+      "logps/chosen": -29.324417114257812,
+      "logps/rejected": -30.05527114868164,
+      "loss": 0.9618,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.04157434031367302,
+      "rewards/margins": 0.03820687159895897,
+      "rewards/rejected": 0.0033674687147140503,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 32.75,
+      "grad_norm": 8.0625,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -2.9093446731567383,
-      "logits/rejected": -2.9112706184387207,
-      "logps/chosen": -29.828176498413086,
-      "logps/rejected": -28.021154403686523,
-      "loss": 0.8701,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.12063708156347275,
-      "rewards/margins": 0.14682583510875702,
-      "rewards/rejected": -0.026188766583800316,
+      "logits/chosen": -2.9058096408843994,
+      "logits/rejected": -2.90769624710083,
+      "logps/chosen": -29.792831420898438,
+      "logps/rejected": -27.967548370361328,
+      "loss": 0.9651,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03369411081075668,
+      "rewards/margins": 0.03488066419959068,
+      "rewards/rejected": -0.0011865533888339996,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 43.5,
+      "grad_norm": 10.75,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.9932944774627686,
-      "logits/rejected": -2.999823808670044,
-      "logps/chosen": -29.177806854248047,
-      "logps/rejected": -30.96217918395996,
-      "loss": 0.9306,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.041265495121479034,
-      "rewards/margins": 0.06941740214824677,
-      "rewards/rejected": -0.028151914477348328,
+      "logits/chosen": -2.98918080329895,
+      "logits/rejected": -2.9952526092529297,
+      "logps/chosen": -29.09405517578125,
+      "logps/rejected": -30.891315460205078,
+      "loss": 0.9814,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.018691470846533775,
+      "rewards/margins": 0.018642673268914223,
+      "rewards/rejected": 4.879874177277088e-05,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 37.75,
+      "grad_norm": 9.375,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -2.8079781532287598,
-      "logits/rejected": -2.8230159282684326,
-      "logps/chosen": -29.30885887145996,
-      "logps/rejected": -29.878082275390625,
-      "loss": 0.8332,
+      "logits/chosen": -2.804455518722534,
+      "logits/rejected": -2.8193769454956055,
+      "logps/chosen": -29.27066993713379,
+      "logps/rejected": -29.863460540771484,
+      "loss": 0.9541,
       "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.09274748712778091,
-      "rewards/margins": 0.1742672175168991,
-      "rewards/rejected": -0.08151975274085999,
+      "rewards/chosen": 0.027005871757864952,
+      "rewards/margins": 0.04592365399003029,
+      "rewards/rejected": -0.018917780369520187,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 28.125,
+      "grad_norm": 9.5,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -2.897887945175171,
-      "logits/rejected": -2.8805594444274902,
-      "logps/chosen": -32.75906753540039,
-      "logps/rejected": -30.377796173095703,
-      "loss": 0.8621,
+      "logits/chosen": -2.8940837383270264,
+      "logits/rejected": -2.8764431476593018,
+      "logps/chosen": -32.690391540527344,
+      "logps/rejected": -30.344482421875,
+      "loss": 0.9531,
       "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.030946413055062294,
-      "rewards/margins": 0.17340537905693054,
-      "rewards/rejected": -0.1424589455127716,
+      "rewards/chosen": 0.014603940770030022,
+      "rewards/margins": 0.04688751697540283,
+      "rewards/rejected": -0.03228358179330826,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 27.625,
+      "grad_norm": 8.4375,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -3.0043857097625732,
-      "logits/rejected": -3.005366086959839,
-      "logps/chosen": -31.849411010742188,
-      "logps/rejected": -31.039392471313477,
-      "loss": 0.8436,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.0829862430691719,
-      "rewards/margins": 0.1722743958234787,
-      "rewards/rejected": -0.08928816020488739,
+      "logits/chosen": -2.999018669128418,
+      "logits/rejected": -2.9999501705169678,
+      "logps/chosen": -31.824331283569336,
+      "logps/rejected": -31.003093719482422,
+      "loss": 0.9581,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.023254599422216415,
+      "rewards/margins": 0.041946638375520706,
+      "rewards/rejected": -0.01869204081594944,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.812446117401123,
-      "eval_logits/rejected": -2.8096587657928467,
-      "eval_logps/chosen": -31.28770637512207,
-      "eval_logps/rejected": -34.89251708984375,
-      "eval_loss": 0.9278174042701721,
-      "eval_rewards/accuracies": 0.5984219312667847,
-      "eval_rewards/chosen": -0.0021020672284066677,
-      "eval_rewards/margins": 0.07564153522253036,
-      "eval_rewards/rejected": -0.07774360477924347,
-      "eval_runtime": 113.4025,
+      "eval_logits/chosen": -2.805298089981079,
+      "eval_logits/rejected": -2.802727460861206,
+      "eval_logps/chosen": -31.269874572753906,
+      "eval_logps/rejected": -34.867759704589844,
+      "eval_loss": 0.9819382429122925,
+      "eval_rewards/accuracies": 0.5955149531364441,
+      "eval_rewards/chosen": 0.0012576259905472398,
+      "eval_rewards/margins": 0.018218038603663445,
+      "eval_rewards/rejected": -0.01696041226387024,
+      "eval_runtime": 113.3967,
       "eval_samples_per_second": 3.025,
       "eval_steps_per_second": 0.379,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 42.0,
+      "grad_norm": 10.4375,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -2.961785078048706,
-      "logits/rejected": -2.9373226165771484,
-      "logps/chosen": -31.922677993774414,
-      "logps/rejected": -31.55402183532715,
-      "loss": 0.7753,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.11764738708734512,
-      "rewards/margins": 0.2757166028022766,
-      "rewards/rejected": -0.1580692082643509,
+      "logits/chosen": -2.9547231197357178,
+      "logits/rejected": -2.9307026863098145,
+      "logps/chosen": -31.882299423217773,
+      "logps/rejected": -31.55330467224121,
+      "loss": 0.9271,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03344918042421341,
+      "rewards/margins": 0.07289471477270126,
+      "rewards/rejected": -0.039445530623197556,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 35.5,
+      "grad_norm": 8.9375,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -3.0439839363098145,
-      "logits/rejected": -3.073005199432373,
-      "logps/chosen": -28.63894271850586,
-      "logps/rejected": -34.55387878417969,
-      "loss": 0.7242,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.20998422801494598,
-      "rewards/margins": 0.372516393661499,
-      "rewards/rejected": -0.16253218054771423,
+      "logits/chosen": -3.0366623401641846,
+      "logits/rejected": -3.0655264854431152,
+      "logps/chosen": -28.671463012695312,
+      "logps/rejected": -34.533443450927734,
+      "loss": 0.9122,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.049243707209825516,
+      "rewards/margins": 0.0878329798579216,
+      "rewards/rejected": -0.03858928382396698,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 33.75,
+      "grad_norm": 8.5625,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.7482008934020996,
-      "logits/rejected": -2.7447853088378906,
-      "logps/chosen": -28.648107528686523,
-      "logps/rejected": -30.5201358795166,
-      "loss": 0.7756,
+      "logits/chosen": -2.741682529449463,
+      "logits/rejected": -2.7387285232543945,
+      "logps/chosen": -28.681751251220703,
+      "logps/rejected": -30.781078338623047,
+      "loss": 0.9069,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.13062894344329834,
-      "rewards/margins": 0.2851604223251343,
-      "rewards/rejected": -0.15453146398067474,
+      "rewards/chosen": 0.029292941093444824,
+      "rewards/margins": 0.09402014315128326,
+      "rewards/rejected": -0.06472718715667725,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 26.25,
+      "grad_norm": 8.6875,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -3.0264618396759033,
-      "logits/rejected": -3.023954391479492,
-      "logps/chosen": -27.322336196899414,
-      "logps/rejected": -32.06721496582031,
-      "loss": 0.8404,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.06555580347776413,
-      "rewards/margins": 0.22973506152629852,
-      "rewards/rejected": -0.1641792207956314,
+      "logits/chosen": -3.0192549228668213,
+      "logits/rejected": -3.0171239376068115,
+      "logps/chosen": -27.48164939880371,
+      "logps/rejected": -32.18421936035156,
+      "loss": 0.9468,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.00045731887803412974,
+      "rewards/margins": 0.0532023087143898,
+      "rewards/rejected": -0.05274498462677002,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 27.625,
+      "grad_norm": 8.8125,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -2.823460817337036,
-      "logits/rejected": -2.818603038787842,
-      "logps/chosen": -27.322635650634766,
-      "logps/rejected": -31.723133087158203,
-      "loss": 0.711,
+      "logits/chosen": -2.8165481090545654,
+      "logits/rejected": -2.8123271465301514,
+      "logps/chosen": -27.479320526123047,
+      "logps/rejected": -32.02801513671875,
+      "loss": 0.8849,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.2011496126651764,
-      "rewards/margins": 0.4009442925453186,
-      "rewards/rejected": -0.1997946947813034,
+      "rewards/chosen": 0.034618668258190155,
+      "rewards/margins": 0.11505589634180069,
+      "rewards/rejected": -0.08043723553419113,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 40.25,
+      "grad_norm": 10.25,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -3.140930414199829,
-      "logits/rejected": -3.123213291168213,
-      "logps/chosen": -31.732044219970703,
-      "logps/rejected": -29.67775535583496,
-      "loss": 0.5884,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.3038697838783264,
-      "rewards/margins": 0.5830621123313904,
-      "rewards/rejected": -0.27919238805770874,
+      "logits/chosen": -3.137382745742798,
+      "logits/rejected": -3.1210262775421143,
+      "logps/chosen": -31.968801498413086,
+      "logps/rejected": -30.1705265045166,
+      "loss": 0.8343,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.05229174345731735,
+      "rewards/margins": 0.17136676609516144,
+      "rewards/rejected": -0.11907501518726349,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 33.5,
+      "grad_norm": 9.5625,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.9524331092834473,
-      "logits/rejected": -2.960047960281372,
-      "logps/chosen": -29.4462947845459,
-      "logps/rejected": -31.882787704467773,
-      "loss": 0.6552,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.1793280392885208,
-      "rewards/margins": 0.49264994263648987,
-      "rewards/rejected": -0.31332188844680786,
+      "logits/chosen": -2.9499218463897705,
+      "logits/rejected": -2.958042621612549,
+      "logps/chosen": -29.790802001953125,
+      "logps/rejected": -32.55553436279297,
+      "loss": 0.8471,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.010381256230175495,
+      "rewards/margins": 0.15598610043525696,
+      "rewards/rejected": -0.14560487866401672,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 28.375,
+      "grad_norm": 10.0625,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.8057327270507812,
-      "logits/rejected": -2.803337812423706,
-      "logps/chosen": -29.322677612304688,
-      "logps/rejected": -30.268985748291016,
-      "loss": 0.7292,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.1388693004846573,
-      "rewards/margins": 0.3391944468021393,
-      "rewards/rejected": -0.2003251612186432,
+      "logits/chosen": -2.810059070587158,
+      "logits/rejected": -2.8075289726257324,
+      "logps/chosen": -29.805904388427734,
+      "logps/rejected": -30.83272361755371,
+      "loss": 0.9106,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.013605661690235138,
+      "rewards/margins": 0.09284953773021698,
+      "rewards/rejected": -0.10645520687103271,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 23.625,
+      "grad_norm": 6.375,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.9184248447418213,
-      "logits/rejected": -2.9148693084716797,
-      "logps/chosen": -29.80621910095215,
-      "logps/rejected": -28.81686782836914,
-      "loss": 0.7753,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.1521972417831421,
-      "rewards/margins": 0.33593088388442993,
-      "rewards/rejected": -0.18373365700244904,
+      "logits/chosen": -2.920551300048828,
+      "logits/rejected": -2.917001247406006,
+      "logps/chosen": -30.380672454833984,
+      "logps/rejected": -29.473743438720703,
+      "loss": 0.9132,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.019395861774683,
+      "rewards/margins": 0.09222523123025894,
+      "rewards/rejected": -0.11162110418081284,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 28.125,
+      "grad_norm": 9.75,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -2.985816478729248,
-      "logits/rejected": -2.9742205142974854,
-      "logps/chosen": -32.966896057128906,
-      "logps/rejected": -30.653575897216797,
-      "loss": 0.7459,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.29284805059432983,
-      "rewards/margins": 0.38064295053482056,
-      "rewards/rejected": -0.08779484033584595,
+      "logits/chosen": -2.9904088973999023,
+      "logits/rejected": -2.9779367446899414,
+      "logps/chosen": -33.54608917236328,
+      "logps/rejected": -31.4923095703125,
+      "loss": 0.8824,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.015293078497052193,
+      "rewards/margins": 0.12111523002386093,
+      "rewards/rejected": -0.10582216084003448,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.825000762939453,
-      "eval_logits/rejected": -2.8226914405822754,
-      "eval_logps/chosen": -31.455589294433594,
-      "eval_logps/rejected": -35.09489822387695,
-      "eval_loss": 0.9159321784973145,
-      "eval_rewards/accuracies": 0.5834717750549316,
-      "eval_rewards/chosen": -0.06925402581691742,
-      "eval_rewards/margins": 0.0894429013133049,
-      "eval_rewards/rejected": -0.15869693458080292,
-      "eval_runtime": 113.1231,
-      "eval_samples_per_second": 3.032,
+      "eval_logits/chosen": -2.8315017223358154,
+      "eval_logits/rejected": -2.8287980556488037,
+      "eval_logps/chosen": -31.827983856201172,
+      "eval_logps/rejected": -35.64292526245117,
+      "eval_loss": 0.9602205753326416,
+      "eval_rewards/accuracies": 0.6092192530632019,
+      "eval_rewards/chosen": -0.05455298721790314,
+      "eval_rewards/margins": 0.039924219250679016,
+      "eval_rewards/rejected": -0.09447719901800156,
+      "eval_runtime": 113.1607,
+      "eval_samples_per_second": 3.031,
       "eval_steps_per_second": 0.38,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 34.0,
+      "grad_norm": 12.125,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.9240100383758545,
-      "logits/rejected": -2.9254937171936035,
-      "logps/chosen": -32.46898651123047,
-      "logps/rejected": -34.43748092651367,
-      "loss": 0.6514,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.22576837241649628,
-      "rewards/margins": 0.4561736583709717,
-      "rewards/rejected": -0.2304052859544754,
+      "logits/chosen": -2.9306328296661377,
+      "logits/rejected": -2.9321701526641846,
+      "logps/chosen": -33.18594741821289,
+      "logps/rejected": -35.648868560791016,
+      "loss": 0.8442,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.015253487043082714,
+      "rewards/margins": 0.16348664462566376,
+      "rewards/rejected": -0.17874012887477875,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 32.25,
+      "grad_norm": 10.375,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -2.9049904346466064,
-      "logits/rejected": -2.9205589294433594,
-      "logps/chosen": -29.369049072265625,
-      "logps/rejected": -29.028148651123047,
-      "loss": 0.6606,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.3235965967178345,
-      "rewards/margins": 0.523547351360321,
-      "rewards/rejected": -0.19995081424713135,
+      "logits/chosen": -2.9161877632141113,
+      "logits/rejected": -2.9325830936431885,
+      "logps/chosen": -29.98288345336914,
+      "logps/rejected": -30.195465087890625,
+      "loss": 0.818,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.019515613093972206,
+      "rewards/margins": 0.18623510003089905,
+      "rewards/rejected": -0.1667194664478302,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 28.75,
+      "grad_norm": 8.9375,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -2.950502395629883,
-      "logits/rejected": -2.9541735649108887,
-      "logps/chosen": -30.678882598876953,
-      "logps/rejected": -32.15251922607422,
-      "loss": 0.7668,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.18919439613819122,
-      "rewards/margins": 0.3469431698322296,
-      "rewards/rejected": -0.1577487289905548,
+      "logits/chosen": -2.9633114337921143,
+      "logits/rejected": -2.9677231311798096,
+      "logps/chosen": -31.465539932250977,
+      "logps/rejected": -33.331993103027344,
+      "loss": 0.8836,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03136688470840454,
+      "rewards/margins": 0.1260170042514801,
+      "rewards/rejected": -0.15738390386104584,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 34.0,
+      "grad_norm": 10.5625,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -3.00547456741333,
-      "logits/rejected": -3.013357400894165,
-      "logps/chosen": -30.189453125,
-      "logps/rejected": -30.650592803955078,
-      "loss": 0.6128,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.2861887216567993,
-      "rewards/margins": 0.4523293375968933,
-      "rewards/rejected": -0.1661405861377716,
+      "logits/chosen": -3.0217947959899902,
+      "logits/rejected": -3.029099702835083,
+      "logps/chosen": -31.11104965209961,
+      "logps/rejected": -31.991535186767578,
+      "loss": 0.845,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.020612526684999466,
+      "rewards/margins": 0.15501661598682404,
+      "rewards/rejected": -0.1756291538476944,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 33.0,
+      "grad_norm": 9.25,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.838496208190918,
-      "logits/rejected": -2.828334093093872,
-      "logps/chosen": -26.336994171142578,
-      "logps/rejected": -29.769458770751953,
-      "loss": 0.7551,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.24688871204853058,
-      "rewards/margins": 0.3420138955116272,
-      "rewards/rejected": -0.09512520581483841,
+      "logits/chosen": -2.8557331562042236,
+      "logits/rejected": -2.844738721847534,
+      "logps/chosen": -27.252483367919922,
+      "logps/rejected": -30.79751968383789,
+      "loss": 0.9072,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.029826903715729713,
+      "rewards/margins": 0.0967610701918602,
+      "rewards/rejected": -0.12658795714378357,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 29.625,
+      "grad_norm": 10.875,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -2.8198904991149902,
-      "logits/rejected": -2.8399436473846436,
-      "logps/chosen": -29.0089168548584,
-      "logps/rejected": -34.85820388793945,
-      "loss": 0.5813,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.32841864228248596,
-      "rewards/margins": 0.6020644903182983,
-      "rewards/rejected": -0.27364593744277954,
+      "logits/chosen": -2.8332571983337402,
+      "logits/rejected": -2.854872226715088,
+      "logps/chosen": -29.75779151916504,
+      "logps/rejected": -36.265663146972656,
+      "loss": 0.7843,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.007217098027467728,
+      "rewards/margins": 0.21637487411499023,
+      "rewards/rejected": -0.209157794713974,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 36.75,
+      "grad_norm": 11.6875,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -2.962282657623291,
-      "logits/rejected": -2.969045639038086,
-      "logps/chosen": -30.215042114257812,
-      "logps/rejected": -30.566543579101562,
-      "loss": 0.5913,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.22652491927146912,
-      "rewards/margins": 0.5070374608039856,
-      "rewards/rejected": -0.2805125117301941,
+      "logits/chosen": -2.975635051727295,
+      "logits/rejected": -2.9822394847869873,
+      "logps/chosen": -31.34609031677246,
+      "logps/rejected": -32.07633972167969,
+      "loss": 0.8354,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.05647352337837219,
+      "rewards/margins": 0.16463439166545868,
+      "rewards/rejected": -0.22110792994499207,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 29.5,
+      "grad_norm": 10.9375,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -2.9766674041748047,
-      "logits/rejected": -2.9624428749084473,
-      "logps/chosen": -30.22548484802246,
-      "logps/rejected": -28.989456176757812,
-      "loss": 0.7584,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.19847916066646576,
-      "rewards/margins": 0.34172323346138,
-      "rewards/rejected": -0.14324405789375305,
+      "logits/chosen": -2.9911038875579834,
+      "logits/rejected": -2.9766478538513184,
+      "logps/chosen": -31.343215942382812,
+      "logps/rejected": -30.425678253173828,
+      "loss": 0.8864,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.06215282529592514,
+      "rewards/margins": 0.1172800287604332,
+      "rewards/rejected": -0.17943285405635834,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 30.0,
+      "grad_norm": 13.5,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.9045450687408447,
-      "logits/rejected": -2.887028932571411,
-      "logps/chosen": -31.592443466186523,
-      "logps/rejected": -31.358936309814453,
-      "loss": 0.4878,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.4175439774990082,
-      "rewards/margins": 0.751793384552002,
-      "rewards/rejected": -0.3342495858669281,
+      "logits/chosen": -2.9202213287353516,
+      "logits/rejected": -2.9021944999694824,
+      "logps/chosen": -32.332275390625,
+      "logps/rejected": -32.9135627746582,
+      "loss": 0.7348,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.030402958393096924,
+      "rewards/margins": 0.26942819356918335,
+      "rewards/rejected": -0.23902523517608643,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 24.75,
+      "grad_norm": 9.0625,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -2.9053683280944824,
-      "logits/rejected": -2.9024596214294434,
-      "logps/chosen": -31.347707748413086,
-      "logps/rejected": -28.053564071655273,
-      "loss": 0.665,
+      "logits/chosen": -2.9209041595458984,
+      "logits/rejected": -2.9169058799743652,
+      "logps/chosen": -32.4577522277832,
+      "logps/rejected": -29.638641357421875,
+      "loss": 0.8245,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.34997984766960144,
-      "rewards/margins": 0.529971182346344,
-      "rewards/rejected": -0.17999130487442017,
+      "rewards/chosen": -0.02350962720811367,
+      "rewards/margins": 0.17999625205993652,
+      "rewards/rejected": -0.20350590348243713,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.8241891860961914,
-      "eval_logits/rejected": -2.8221466541290283,
-      "eval_logps/chosen": -31.476619720458984,
-      "eval_logps/rejected": -35.105430603027344,
-      "eval_loss": 0.9201633334159851,
-      "eval_rewards/accuracies": 0.565614640712738,
-      "eval_rewards/chosen": -0.07766681164503098,
-      "eval_rewards/margins": 0.08524217456579208,
-      "eval_rewards/rejected": -0.16290898621082306,
-      "eval_runtime": 113.1247,
-      "eval_samples_per_second": 3.032,
-      "eval_steps_per_second": 0.38,
+      "eval_logits/chosen": -2.8392350673675537,
+      "eval_logits/rejected": -2.836524486541748,
+      "eval_logps/chosen": -32.03562545776367,
+      "eval_logps/rejected": -35.89268112182617,
+      "eval_loss": 0.9559255838394165,
+      "eval_rewards/accuracies": 0.6004983186721802,
+      "eval_rewards/chosen": -0.07531756907701492,
+      "eval_rewards/margins": 0.044134579598903656,
+      "eval_rewards/rejected": -0.11945215612649918,
+      "eval_runtime": 112.8661,
+      "eval_samples_per_second": 3.039,
+      "eval_steps_per_second": 0.381,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 32.5,
+      "grad_norm": 10.1875,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -2.7875466346740723,
-      "logits/rejected": -2.8046317100524902,
-      "logps/chosen": -28.552978515625,
-      "logps/rejected": -31.452463150024414,
-      "loss": 0.5777,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.27057594060897827,
-      "rewards/margins": 0.5587579011917114,
-      "rewards/rejected": -0.28818196058273315,
+      "logits/chosen": -2.80441951751709,
+      "logits/rejected": -2.8214964866638184,
+      "logps/chosen": -29.70196533203125,
+      "logps/rejected": -33.18954849243164,
+      "loss": 0.8066,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.047254521399736404,
+      "rewards/margins": 0.19849951565265656,
+      "rewards/rejected": -0.24575403332710266,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 26.75,
+      "grad_norm": 9.5,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -3.0314865112304688,
-      "logits/rejected": -3.0174806118011475,
-      "logps/chosen": -29.399667739868164,
-      "logps/rejected": -28.924419403076172,
-      "loss": 0.5701,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.32476717233657837,
-      "rewards/margins": 0.567025363445282,
-      "rewards/rejected": -0.24225828051567078,
+      "logits/chosen": -3.0470097064971924,
+      "logits/rejected": -3.033452033996582,
+      "logps/chosen": -29.982385635375977,
+      "logps/rejected": -30.24712562561035,
+      "loss": 0.8017,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.022920172661542892,
+      "rewards/margins": 0.21575525403022766,
+      "rewards/rejected": -0.19283509254455566,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 18.25,
+      "grad_norm": 11.875,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -2.946519374847412,
-      "logits/rejected": -2.9290707111358643,
-      "logps/chosen": -27.334081649780273,
-      "logps/rejected": -31.206268310546875,
-      "loss": 0.4948,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.4111965298652649,
-      "rewards/margins": 0.7370456457138062,
-      "rewards/rejected": -0.32584911584854126,
+      "logits/chosen": -2.96113657951355,
+      "logits/rejected": -2.9435229301452637,
+      "logps/chosen": -28.380762100219727,
+      "logps/rejected": -32.93931579589844,
+      "loss": 0.7579,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.0018691420555114746,
+      "rewards/margins": 0.25289779901504517,
+      "rewards/rejected": -0.25476694107055664,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 35.0,
+      "grad_norm": 10.5625,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -3.16451096534729,
-      "logits/rejected": -3.1703600883483887,
-      "logps/chosen": -30.927570343017578,
-      "logps/rejected": -33.580810546875,
-      "loss": 0.5586,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.2846537232398987,
-      "rewards/margins": 0.6385756731033325,
-      "rewards/rejected": -0.35392194986343384,
+      "logits/chosen": -3.182105541229248,
+      "logits/rejected": -3.1877188682556152,
+      "logps/chosen": -31.98989486694336,
+      "logps/rejected": -35.27183532714844,
+      "loss": 0.7995,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.035068999975919724,
+      "rewards/margins": 0.22251394391059875,
+      "rewards/rejected": -0.2575829327106476,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 23.875,
+      "grad_norm": 11.75,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -3.040112018585205,
-      "logits/rejected": -3.0432353019714355,
-      "logps/chosen": -29.85060691833496,
-      "logps/rejected": -32.070159912109375,
-      "loss": 0.5751,
+      "logits/chosen": -3.057163953781128,
+      "logits/rejected": -3.0605666637420654,
+      "logps/chosen": -30.571701049804688,
+      "logps/rejected": -33.298683166503906,
+      "loss": 0.8066,
       "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.3825104236602783,
-      "rewards/margins": 0.6052514314651489,
-      "rewards/rejected": -0.22274093329906464,
+      "rewards/chosen": 0.023518025875091553,
+      "rewards/margins": 0.20205526053905487,
+      "rewards/rejected": -0.1785372495651245,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 35.0,
+      "grad_norm": 11.0,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -2.8801891803741455,
-      "logits/rejected": -2.8814785480499268,
-      "logps/chosen": -27.602718353271484,
-      "logps/rejected": -30.264202117919922,
-      "loss": 0.5555,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.41880473494529724,
-      "rewards/margins": 0.5921100974082947,
-      "rewards/rejected": -0.17330540716648102,
+      "logits/chosen": -2.8954837322235107,
+      "logits/rejected": -2.8972554206848145,
+      "logps/chosen": -28.211322784423828,
+      "logps/rejected": -31.518001556396484,
+      "loss": 0.7939,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.04384060949087143,
+      "rewards/margins": 0.21254661679267883,
+      "rewards/rejected": -0.1687059849500656,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 41.0,
+      "grad_norm": 10.3125,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.9562087059020996,
-      "logits/rejected": -2.9534800052642822,
-      "logps/chosen": -29.909082412719727,
-      "logps/rejected": -32.4835205078125,
-      "loss": 0.5884,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.3510281443595886,
-      "rewards/margins": 0.5732189416885376,
-      "rewards/rejected": -0.22219082713127136,
+      "logits/chosen": -2.973878860473633,
+      "logits/rejected": -2.9718644618988037,
+      "logps/chosen": -30.932153701782227,
+      "logps/rejected": -33.92184829711914,
+      "loss": 0.8202,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.014550352469086647,
+      "rewards/margins": 0.1848299205303192,
+      "rewards/rejected": -0.1993802934885025,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 30.75,
+      "grad_norm": 9.8125,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -2.9113378524780273,
-      "logits/rejected": -2.894592523574829,
-      "logps/chosen": -28.256174087524414,
-      "logps/rejected": -28.505691528320312,
-      "loss": 0.5793,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.275543749332428,
-      "rewards/margins": 0.521973192691803,
-      "rewards/rejected": -0.24642948806285858,
+      "logits/chosen": -2.927494764328003,
+      "logits/rejected": -2.9113755226135254,
+      "logps/chosen": -29.455524444580078,
+      "logps/rejected": -30.044204711914062,
+      "loss": 0.8384,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.05104842036962509,
+      "rewards/margins": 0.16441020369529724,
+      "rewards/rejected": -0.21545863151550293,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.7203964704042906,
-      "train_runtime": 2720.5927,
-      "train_samples_per_second": 1.132,
-      "train_steps_per_second": 0.142
+      "train_loss": 0.8805358874333369,
+      "train_runtime": 2723.8224,
+      "train_samples_per_second": 1.13,
+      "train_steps_per_second": 0.141
     }
   ],
   "logging_steps": 10,