Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NorLLM-AI/NorMistral-7B
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: norllm-ai-normistral-7b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # norllm-ai-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/norllm-ai-normistral-7b-sft-qlora](https://huggingface.co/data/norllm-ai-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NorLLM-AI/NorMistral-7B](https://huggingface.co/NorLLM-AI/NorMistral-7B) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4841
-- Rewards/chosen: 0.0714
-- Rewards/rejected: -0.0106
-- Rewards/accuracies: 0.5490
-- Rewards/margins: 0.0820
-- Logps/rejected: -34.7193
-- Logps/chosen: -31.1396
-- Logits/rejected: -2.8153
-- Logits/chosen: -2.8173
+- Loss: 0.4909
+- Rewards/chosen: 0.0420
+- Rewards/rejected: -0.0034
+- Rewards/accuracies: 0.6013
+- Rewards/margins: 0.0454
+- Logps/rejected: -34.7152
+- Logps/chosen: -31.0725
+- Logits/rejected: -2.8183
+- Logits/chosen: -2.8209
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4624        | 0.26  | 100  | 0.4860          | 0.0966         | 0.0340           | 0.5810             | 0.0626          | -34.6301       | -31.0892     | -2.8032         | -2.8064       |
-| 0.377         | 0.52  | 200  | 0.4831          | 0.0952         | 0.0100           | 0.5660             | 0.0852          | -34.6782       | -31.0920     | -2.8125         | -2.8151       |
-| 0.3692        | 0.78  | 300  | 0.4829          | 0.0838         | -0.0051          | 0.5839             | 0.0889          | -34.7084       | -31.1149     | -2.8154         | -2.8178       |
+| 0.4829        | 0.26  | 100  | 0.4945          | 0.0390         | 0.0134           | 0.5635             | 0.0256          | -34.6310       | -31.0874     | -2.8002         | -2.8030       |
+| 0.4376        | 0.52  | 200  | 0.4911          | 0.0473         | 0.0060           | 0.5577             | 0.0413          | -34.6682       | -31.0459     | -2.8137         | -2.8163       |
+| 0.4326        | 0.78  | 300  | 0.4909          | 0.0420         | -0.0034          | 0.6013             | 0.0454          | -34.7152       | -31.0725     | -2.8183         | -2.8209       |
 
 
 ### Framework versions

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
-    "v_proj",
-    "o_proj",
+    "gate_proj",
     "up_proj",
-    "q_proj",
     "k_proj",
-    "gate_proj"
+    "v_proj",
+    "q_proj",
+    "down_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0438a616b4de01a97b3c77b45f1f9b8e0d3ef99b2fb6acb180db55e592b2c1e0
+oid sha256:3c0fa1ebe16e46bb3fac3ad9917e31e381278f706148ebbf4289c7c9057f32a4
 size 671150064

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.8172521591186523,
-    "eval_logits/rejected": -2.8152647018432617,
-    "eval_logps/chosen": -31.13959503173828,
-    "eval_logps/rejected": -34.71931457519531,
-    "eval_loss": 0.48413917422294617,
-    "eval_rewards/accuracies": 0.5490033626556396,
-    "eval_rewards/chosen": 0.07142771780490875,
-    "eval_rewards/margins": 0.08200670033693314,
-    "eval_rewards/rejected": -0.010578980669379234,
-    "eval_runtime": 112.892,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.038,
-    "eval_steps_per_second": 0.381,
-    "train_loss": 0.4045382279854316,
-    "train_runtime": 2722.7053,
+    "train_loss": 0.4552104888024268,
+    "train_runtime": 2721.944,
     "train_samples": 3079,
     "train_samples_per_second": 1.131,
     "train_steps_per_second": 0.141

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.4045382279854316,
-    "train_runtime": 2722.7053,
+    "train_loss": 0.4552104888024268,
+    "train_runtime": 2721.944,
     "train_samples": 3079,
     "train_samples_per_second": 1.131,
     "train_steps_per_second": 0.141

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 9.875,
+      "grad_norm": 3.953125,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": -2.7358343601226807,
       "logits/rejected": -2.7480404376983643,
@@ -25,628 +25,628 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 9.375,
+      "grad_norm": 3.75,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -3.009662628173828,
-      "logits/rejected": -2.9985737800598145,
-      "logps/chosen": -33.19932174682617,
-      "logps/rejected": -31.99129867553711,
-      "loss": 0.4991,
+      "logits/chosen": -3.0089564323425293,
+      "logits/rejected": -2.998065710067749,
+      "logps/chosen": -33.19136428833008,
+      "logps/rejected": -31.97561264038086,
+      "loss": 0.4998,
       "rewards/accuracies": 0.4305555522441864,
-      "rewards/chosen": -0.009189460426568985,
-      "rewards/margins": 0.004507640842348337,
-      "rewards/rejected": -0.013697100803256035,
+      "rewards/chosen": -0.0020843851380050182,
+      "rewards/margins": 0.0002575106918811798,
+      "rewards/rejected": -0.002341895829886198,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 8.375,
+      "grad_norm": 3.359375,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -2.8997883796691895,
-      "logits/rejected": -2.8943028450012207,
-      "logps/chosen": -32.468807220458984,
-      "logps/rejected": -28.9471435546875,
-      "loss": 0.503,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.0031455964781343937,
-      "rewards/margins": -0.01421579159796238,
-      "rewards/rejected": 0.017361387610435486,
+      "logits/chosen": -2.899214267730713,
+      "logits/rejected": -2.8942105770111084,
+      "logps/chosen": -32.48638916015625,
+      "logps/rejected": -28.965356826782227,
+      "loss": 0.5009,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.0022583019454032183,
+      "rewards/margins": -0.0055608078837394714,
+      "rewards/rejected": 0.0033025064039975405,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 10.0625,
+      "grad_norm": 4.0625,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -3.0975382328033447,
-      "logits/rejected": -3.109549045562744,
-      "logps/chosen": -32.832576751708984,
-      "logps/rejected": -30.16579246520996,
-      "loss": 0.4947,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.03805363178253174,
-      "rewards/margins": 0.022922921925783157,
-      "rewards/rejected": 0.01513071171939373,
+      "logits/chosen": -3.096067190170288,
+      "logits/rejected": -3.1081583499908447,
+      "logps/chosen": -32.83055877685547,
+      "logps/rejected": -30.159122467041016,
+      "loss": 0.4979,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0156254880130291,
+      "rewards/margins": 0.008240064606070518,
+      "rewards/rejected": 0.007385422941297293,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 9.4375,
+      "grad_norm": 4.0625,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.8632423877716064,
-      "logits/rejected": -2.8540544509887695,
-      "logps/chosen": -31.514551162719727,
-      "logps/rejected": -32.35602569580078,
-      "loss": 0.4657,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.1578875333070755,
-      "rewards/margins": 0.15717332065105438,
-      "rewards/rejected": 0.0007142305257730186,
+      "logits/chosen": -2.8633229732513428,
+      "logits/rejected": -2.8539462089538574,
+      "logps/chosen": -31.576160430908203,
+      "logps/rejected": -32.43328857421875,
+      "loss": 0.4858,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.05083252862095833,
+      "rewards/margins": 0.06599889695644379,
+      "rewards/rejected": -0.01516636274755001,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 8.3125,
+      "grad_norm": 3.8125,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -2.8802037239074707,
-      "logits/rejected": -2.877748966217041,
-      "logps/chosen": -29.273544311523438,
-      "logps/rejected": -30.0130615234375,
-      "loss": 0.4574,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.23330914974212646,
-      "rewards/margins": 0.1953679919242859,
-      "rewards/rejected": 0.03794118016958237,
+      "logits/chosen": -2.880373954772949,
+      "logits/rejected": -2.87811541557312,
+      "logps/chosen": -29.325586318969727,
+      "logps/rejected": -30.077322006225586,
+      "loss": 0.4808,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.08291508257389069,
+      "rewards/margins": 0.08059060573577881,
+      "rewards/rejected": 0.002324472414329648,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 8.875,
+      "grad_norm": 3.59375,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -2.9084858894348145,
-      "logits/rejected": -2.910191059112549,
-      "logps/chosen": -29.779638290405273,
-      "logps/rejected": -27.947153091430664,
-      "loss": 0.4647,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.1750670224428177,
-      "rewards/margins": 0.17080120742321014,
-      "rewards/rejected": 0.004265817813575268,
+      "logits/chosen": -2.9070420265197754,
+      "logits/rejected": -2.908172369003296,
+      "logps/chosen": -29.754262924194336,
+      "logps/rejected": -27.948543548583984,
+      "loss": 0.4839,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.07510136812925339,
+      "rewards/margins": 0.0736737847328186,
+      "rewards/rejected": 0.0014275781577453017,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 8.875,
+      "grad_norm": 4.3125,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.992633104324341,
-      "logits/rejected": -2.9988465309143066,
-      "logps/chosen": -28.9884033203125,
-      "logps/rejected": -30.786937713623047,
-      "loss": 0.4794,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.14628295600414276,
-      "rewards/margins": 0.09385170042514801,
-      "rewards/rejected": 0.05243126302957535,
+      "logits/chosen": -2.9913430213928223,
+      "logits/rejected": -2.9976165294647217,
+      "logps/chosen": -29.0341796875,
+      "logps/rejected": -30.7557430267334,
+      "loss": 0.4938,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.04935770481824875,
+      "rewards/margins": 0.022146493196487427,
+      "rewards/rejected": 0.027211207896471024,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 10.375,
+      "grad_norm": 4.125,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -2.804734468460083,
-      "logits/rejected": -2.8200764656066895,
-      "logps/chosen": -29.139240264892578,
-      "logps/rejected": -29.688739776611328,
-      "loss": 0.4605,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.20074471831321716,
-      "rewards/margins": 0.20797061920166016,
-      "rewards/rejected": -0.007225893437862396,
+      "logits/chosen": -2.803948402404785,
+      "logits/rejected": -2.8194332122802734,
+      "logps/chosen": -29.147689819335938,
+      "logps/rejected": -29.726749420166016,
+      "loss": 0.4828,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.07860848307609558,
+      "rewards/margins": 0.08910086750984192,
+      "rewards/rejected": -0.010492382571101189,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 10.125,
+      "grad_norm": 4.4375,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -2.894127368927002,
-      "logits/rejected": -2.876558780670166,
-      "logps/chosen": -32.31249237060547,
-      "logps/rejected": -30.023962020874023,
-      "loss": 0.4408,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.2619698643684387,
-      "rewards/margins": 0.26312756538391113,
-      "rewards/rejected": -0.0011577218538150191,
+      "logits/chosen": -2.892725706100464,
+      "logits/rejected": -2.874434232711792,
+      "logps/chosen": -32.414127349853516,
+      "logps/rejected": -30.058170318603516,
+      "loss": 0.4789,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.08446021378040314,
+      "rewards/margins": 0.09176470339298248,
+      "rewards/rejected": -0.007304488681256771,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 8.375,
+      "grad_norm": 3.5,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.9993441104888916,
-      "logits/rejected": -3.00020170211792,
-      "logps/chosen": -31.58449935913086,
-      "logps/rejected": -30.727672576904297,
-      "loss": 0.4624,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.2361881285905838,
-      "rewards/margins": 0.19193710386753082,
-      "rewards/rejected": 0.04425101354718208,
+      "logits/chosen": -2.9959635734558105,
+      "logits/rejected": -2.9973392486572266,
+      "logps/chosen": -31.55927085876465,
+      "logps/rejected": -30.724218368530273,
+      "loss": 0.4829,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.0995207279920578,
+      "rewards/margins": 0.0811297670006752,
+      "rewards/rejected": 0.0183909572660923,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.80639910697937,
-      "eval_logits/rejected": -2.8032400608062744,
-      "eval_logps/chosen": -31.089191436767578,
-      "eval_logps/rejected": -34.630130767822266,
-      "eval_loss": 0.4859701097011566,
-      "eval_rewards/accuracies": 0.5809800624847412,
-      "eval_rewards/chosen": 0.09663014858961105,
-      "eval_rewards/margins": 0.06261748820543289,
-      "eval_rewards/rejected": 0.03401266410946846,
-      "eval_runtime": 113.3407,
-      "eval_samples_per_second": 3.026,
-      "eval_steps_per_second": 0.379,
+      "eval_logits/chosen": -2.802985906600952,
+      "eval_logits/rejected": -2.8001766204833984,
+      "eval_logps/chosen": -31.0873966217041,
+      "eval_logps/rejected": -34.63096618652344,
+      "eval_loss": 0.49451369047164917,
+      "eval_rewards/accuracies": 0.5635381937026978,
+      "eval_rewards/chosen": 0.03901098668575287,
+      "eval_rewards/margins": 0.02557329833507538,
+      "eval_rewards/rejected": 0.013437685556709766,
+      "eval_runtime": 113.153,
+      "eval_samples_per_second": 3.031,
+      "eval_steps_per_second": 0.38,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 9.625,
+      "grad_norm": 4.09375,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -2.9530532360076904,
-      "logits/rejected": -2.9286322593688965,
-      "logps/chosen": -31.438684463500977,
-      "logps/rejected": -31.172704696655273,
-      "loss": 0.4246,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.3890557289123535,
-      "rewards/margins": 0.39598459005355835,
-      "rewards/rejected": -0.006928825285285711,
+      "logits/chosen": -2.949113130569458,
+      "logits/rejected": -2.925306797027588,
+      "logps/chosen": -31.45932388305664,
+      "logps/rejected": -31.128564834594727,
+      "loss": 0.4692,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.1514938771724701,
+      "rewards/margins": 0.14543716609477997,
+      "rewards/rejected": 0.006056725047528744,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 9.875,
+      "grad_norm": 4.125,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -3.034228801727295,
-      "logits/rejected": -3.0629284381866455,
-      "logps/chosen": -28.331079483032227,
-      "logps/rejected": -34.00596618652344,
-      "loss": 0.429,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.4164126515388489,
-      "rewards/margins": 0.3456197679042816,
-      "rewards/rejected": 0.07079292088747025,
+      "logits/chosen": -3.031067371368408,
+      "logits/rejected": -3.060089588165283,
+      "logps/chosen": -28.312602996826172,
+      "logps/rejected": -34.01371765136719,
+      "loss": 0.4667,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.17025941610336304,
+      "rewards/margins": 0.14349231123924255,
+      "rewards/rejected": 0.026767095550894737,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 6.0,
+      "grad_norm": 2.515625,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.7350852489471436,
-      "logits/rejected": -2.7314858436584473,
-      "logps/chosen": -28.190832138061523,
-      "logps/rejected": -30.052515029907227,
-      "loss": 0.433,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.39192527532577515,
-      "rewards/margins": 0.3512800335884094,
-      "rewards/rejected": 0.040645234286785126,
+      "logits/chosen": -2.733306884765625,
+      "logits/rejected": -2.729750871658325,
+      "logps/chosen": -28.17226219177246,
+      "logps/rejected": -30.136898040771484,
+      "loss": 0.4656,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.1604831963777542,
+      "rewards/margins": 0.16110166907310486,
+      "rewards/rejected": -0.00061851367354393,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 8.0,
+      "grad_norm": 3.03125,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -3.010406494140625,
-      "logits/rejected": -3.0085320472717285,
-      "logps/chosen": -26.78188705444336,
-      "logps/rejected": -31.552181243896484,
-      "loss": 0.4424,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.35216885805130005,
-      "rewards/margins": 0.29987582564353943,
-      "rewards/rejected": 0.05229302495718002,
+      "logits/chosen": -3.0078999996185303,
+      "logits/rejected": -3.005507230758667,
+      "logps/chosen": -26.812694549560547,
+      "logps/rejected": -31.561386108398438,
+      "loss": 0.4732,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.13470572233200073,
+      "rewards/margins": 0.11562897264957428,
+      "rewards/rejected": 0.01907675340771675,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 8.0625,
+      "grad_norm": 3.671875,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -2.8064494132995605,
-      "logits/rejected": -2.801478862762451,
-      "logps/chosen": -26.937829971313477,
-      "logps/rejected": -31.226627349853516,
-      "loss": 0.4092,
+      "logits/chosen": -2.8041739463806152,
+      "logits/rejected": -2.7990527153015137,
+      "logps/chosen": -26.89498519897461,
+      "logps/rejected": -31.248632431030273,
+      "loss": 0.4572,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.4438399374485016,
-      "rewards/margins": 0.4453318119049072,
-      "rewards/rejected": -0.001491829752922058,
+      "rewards/chosen": 0.18610434234142303,
+      "rewards/margins": 0.1911022961139679,
+      "rewards/rejected": -0.004997962154448032,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 9.8125,
+      "grad_norm": 3.828125,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -3.125049114227295,
-      "logits/rejected": -3.107017993927002,
-      "logps/chosen": -31.143413543701172,
-      "logps/rejected": -29.148534774780273,
-      "loss": 0.3635,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.6741524338722229,
-      "rewards/margins": 0.7585334181785583,
-      "rewards/rejected": -0.08438104391098022,
+      "logits/chosen": -3.122868299484253,
+      "logits/rejected": -3.1052639484405518,
+      "logps/chosen": -31.118968963623047,
+      "logps/rejected": -29.14169692993164,
+      "loss": 0.4349,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.2745501399040222,
+      "rewards/margins": 0.30693456530570984,
+      "rewards/rejected": -0.0323844812810421,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 7.53125,
+      "grad_norm": 4.0,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.9369125366210938,
-      "logits/rejected": -2.9447195529937744,
-      "logps/chosen": -28.86051368713379,
-      "logps/rejected": -31.281164169311523,
-      "loss": 0.3882,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.5170499086380005,
-      "rewards/margins": 0.607892632484436,
-      "rewards/rejected": -0.09084277600049973,
+      "logits/chosen": -2.9357101917266846,
+      "logits/rejected": -2.9437410831451416,
+      "logps/chosen": -28.84354591369629,
+      "logps/rejected": -31.365503311157227,
+      "loss": 0.4446,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.21021375060081482,
+      "rewards/margins": 0.26341748237609863,
+      "rewards/rejected": -0.05320371314883232,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 6.21875,
+      "grad_norm": 2.9375,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.7863478660583496,
-      "logits/rejected": -2.78369140625,
-      "logps/chosen": -28.690837860107422,
-      "logps/rejected": -29.790157318115234,
-      "loss": 0.4032,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.4895057678222656,
-      "rewards/margins": 0.500499427318573,
-      "rewards/rejected": -0.010993694886565208,
+      "logits/chosen": -2.7853760719299316,
+      "logits/rejected": -2.782726287841797,
+      "logps/chosen": -28.669147491455078,
+      "logps/rejected": -29.730758666992188,
+      "loss": 0.4565,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.2001400738954544,
+      "rewards/margins": 0.19265684485435486,
+      "rewards/rejected": 0.00748323742300272,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 4.90625,
+      "grad_norm": 2.390625,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.9043054580688477,
-      "logits/rejected": -2.900820255279541,
-      "logps/chosen": -29.144983291625977,
-      "logps/rejected": -28.175140380859375,
-      "loss": 0.4158,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.520866870880127,
-      "rewards/margins": 0.4296707510948181,
-      "rewards/rejected": 0.09119615703821182,
+      "logits/chosen": -2.905327320098877,
+      "logits/rejected": -2.9015116691589355,
+      "logps/chosen": -29.04599380493164,
+      "logps/rejected": -28.085041046142578,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.2281440794467926,
+      "rewards/margins": 0.17364613711833954,
+      "rewards/rejected": 0.05449794605374336,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 5.4375,
+      "grad_norm": 2.84375,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -2.9771270751953125,
-      "logits/rejected": -2.964667320251465,
-      "logps/chosen": -31.960296630859375,
-      "logps/rejected": -30.09661865234375,
-      "loss": 0.377,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.8693603277206421,
-      "rewards/margins": 0.7006256580352783,
-      "rewards/rejected": 0.16873463988304138,
+      "logits/chosen": -2.9772303104400635,
+      "logits/rejected": -2.9655442237854004,
+      "logps/chosen": -31.91948890686035,
+      "logps/rejected": -30.082332611083984,
+      "loss": 0.4376,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.3559049963951111,
+      "rewards/margins": 0.28555411100387573,
+      "rewards/rejected": 0.07035084813833237,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.8150622844696045,
-      "eval_logits/rejected": -2.8125295639038086,
-      "eval_logps/chosen": -31.092016220092773,
-      "eval_logps/rejected": -34.67816162109375,
-      "eval_loss": 0.48307520151138306,
-      "eval_rewards/accuracies": 0.5660299062728882,
-      "eval_rewards/chosen": 0.09521817415952682,
-      "eval_rewards/margins": 0.08522038906812668,
-      "eval_rewards/rejected": 0.00999778974801302,
-      "eval_runtime": 113.0431,
-      "eval_samples_per_second": 3.034,
-      "eval_steps_per_second": 0.38,
+      "eval_logits/chosen": -2.8163414001464844,
+      "eval_logits/rejected": -2.8137245178222656,
+      "eval_logps/chosen": -31.0458927154541,
+      "eval_logps/rejected": -34.6682014465332,
+      "eval_loss": 0.49114447832107544,
+      "eval_rewards/accuracies": 0.5577242374420166,
+      "eval_rewards/chosen": 0.04731180891394615,
+      "eval_rewards/margins": 0.041320886462926865,
+      "eval_rewards/rejected": 0.005990919191390276,
+      "eval_runtime": 113.006,
+      "eval_samples_per_second": 3.035,
+      "eval_steps_per_second": 0.381,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 8.9375,
+      "grad_norm": 4.28125,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.913187026977539,
-      "logits/rejected": -2.915334939956665,
-      "logps/chosen": -31.631275177001953,
-      "logps/rejected": -33.838157653808594,
-      "loss": 0.3716,
+      "logits/chosen": -2.914222240447998,
+      "logits/rejected": -2.916249990463257,
+      "logps/chosen": -31.570764541625977,
+      "logps/rejected": -33.81729507446289,
+      "loss": 0.4391,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.7010680437088013,
-      "rewards/margins": 0.689413845539093,
-      "rewards/rejected": 0.011654138565063477,
+      "rewards/chosen": 0.2925291359424591,
+      "rewards/margins": 0.2836949825286865,
+      "rewards/rejected": 0.008834179490804672,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 6.46875,
+      "grad_norm": 3.1875,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -2.893833637237549,
-      "logits/rejected": -2.9098963737487793,
-      "logps/chosen": -28.740219116210938,
-      "logps/rejected": -28.43943214416504,
-      "loss": 0.368,
+      "logits/chosen": -2.8961727619171143,
+      "logits/rejected": -2.9133689403533936,
+      "logps/chosen": -28.757190704345703,
+      "logps/rejected": -28.49643325805664,
+      "loss": 0.4382,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.7189111113548279,
-      "rewards/margins": 0.6744931936264038,
-      "rewards/rejected": 0.04441786929965019,
+      "rewards/chosen": 0.28417009115219116,
+      "rewards/margins": 0.2778027653694153,
+      "rewards/rejected": 0.006367350462824106,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 8.25,
+      "grad_norm": 3.75,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -2.9437930583953857,
-      "logits/rejected": -2.949058771133423,
-      "logps/chosen": -30.01968765258789,
-      "logps/rejected": -31.47991371154785,
-      "loss": 0.4114,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.5660901069641113,
-      "rewards/margins": 0.42697352170944214,
-      "rewards/rejected": 0.13911649584770203,
+      "logits/chosen": -2.948033571243286,
+      "logits/rejected": -2.952341079711914,
+      "logps/chosen": -30.005146026611328,
+      "logps/rejected": -31.54683494567871,
+      "loss": 0.4579,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.22934475541114807,
+      "rewards/margins": 0.18708109855651855,
+      "rewards/rejected": 0.042263638228178024,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 8.5,
+      "grad_norm": 3.84375,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -2.9993679523468018,
-      "logits/rejected": -3.0062263011932373,
-      "logps/chosen": -29.48809814453125,
-      "logps/rejected": -30.120525360107422,
-      "loss": 0.3686,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.7084138989448547,
-      "rewards/margins": 0.6510564088821411,
-      "rewards/rejected": 0.05735747888684273,
+      "logits/chosen": -3.0030248165130615,
+      "logits/rejected": -3.009697675704956,
+      "logps/chosen": -29.56500816345215,
+      "logps/rejected": -30.203866958618164,
+      "loss": 0.4421,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.2679831385612488,
+      "rewards/margins": 0.2617082893848419,
+      "rewards/rejected": 0.00627488736063242,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 7.0625,
+      "grad_norm": 3.796875,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.827415943145752,
-      "logits/rejected": -2.8169562816619873,
-      "logps/chosen": -25.754283905029297,
-      "logps/rejected": -29.200170516967773,
-      "loss": 0.4103,
+      "logits/chosen": -2.8312201499938965,
+      "logits/rejected": -2.821024179458618,
+      "logps/chosen": -25.73629379272461,
+      "logps/rejected": -29.133464813232422,
+      "loss": 0.4609,
       "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.5999643206596375,
-      "rewards/margins": 0.4342280924320221,
-      "rewards/rejected": 0.16573622822761536,
+      "rewards/chosen": 0.24358424544334412,
+      "rewards/margins": 0.1639489382505417,
+      "rewards/rejected": 0.07963528484106064,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 6.40625,
+      "grad_norm": 3.625,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -2.810542583465576,
-      "logits/rejected": -2.8318700790405273,
-      "logps/chosen": -28.326793670654297,
-      "logps/rejected": -33.971923828125,
-      "loss": 0.3734,
+      "logits/chosen": -2.8129467964172363,
+      "logits/rejected": -2.8341376781463623,
+      "logps/chosen": -28.19793701171875,
+      "logps/rejected": -33.99031448364258,
+      "loss": 0.436,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.7515828013420105,
-      "rewards/margins": 0.6504999399185181,
-      "rewards/rejected": 0.10108289867639542,
+      "rewards/chosen": 0.3264053463935852,
+      "rewards/margins": 0.2896498441696167,
+      "rewards/rejected": 0.036755502223968506,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 7.21875,
+      "grad_norm": 3.484375,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -2.955075740814209,
-      "logits/rejected": -2.9621012210845947,
-      "logps/chosen": -29.6314640045166,
-      "logps/rejected": -29.941478729248047,
-      "loss": 0.372,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.5749462842941284,
-      "rewards/margins": 0.613055944442749,
-      "rewards/rejected": -0.0381096787750721,
+      "logits/chosen": -2.9577534198760986,
+      "logits/rejected": -2.9646975994110107,
+      "logps/chosen": -29.623525619506836,
+      "logps/rejected": -29.91342544555664,
+      "loss": 0.4449,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.23156669735908508,
+      "rewards/margins": 0.2411995679140091,
+      "rewards/rejected": -0.009632894769310951,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 7.375,
+      "grad_norm": 3.53125,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -2.9710047245025635,
-      "logits/rejected": -2.9573113918304443,
-      "logps/chosen": -29.810815811157227,
-      "logps/rejected": -28.53564453125,
-      "loss": 0.4019,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.45543354749679565,
-      "rewards/margins": 0.40757960081100464,
-      "rewards/rejected": 0.04785385727882385,
+      "logits/chosen": -2.973743200302124,
+      "logits/rejected": -2.960118532180786,
+      "logps/chosen": -29.57967185974121,
+      "logps/rejected": -28.558263778686523,
+      "loss": 0.4498,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.22840175032615662,
+      "rewards/margins": 0.2137848436832428,
+      "rewards/rejected": 0.014616942033171654,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 6.125,
+      "grad_norm": 2.9375,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.898607015609741,
-      "logits/rejected": -2.881021022796631,
-      "logps/chosen": -30.740427017211914,
-      "logps/rejected": -30.70845603942871,
-      "loss": 0.3058,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.9479381442070007,
-      "rewards/margins": 1.0405118465423584,
-      "rewards/rejected": -0.09257374703884125,
+      "logits/chosen": -2.901043653488159,
+      "logits/rejected": -2.883784294128418,
+      "logps/chosen": -30.675678253173828,
+      "logps/rejected": -30.71207618713379,
+      "loss": 0.4063,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.3921253979206085,
+      "rewards/margins": 0.4298785626888275,
+      "rewards/rejected": -0.03775321692228317,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 8.0625,
+      "grad_norm": 3.703125,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -2.901172161102295,
-      "logits/rejected": -2.8976128101348877,
-      "logps/chosen": -30.7450008392334,
-      "logps/rejected": -27.618209838867188,
-      "loss": 0.3692,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.738824725151062,
-      "rewards/margins": 0.7461366057395935,
-      "rewards/rejected": -0.007311972789466381,
+      "logits/chosen": -2.905961513519287,
+      "logits/rejected": -2.9014759063720703,
+      "logps/chosen": -30.50136947631836,
+      "logps/rejected": -27.470678329467773,
+      "loss": 0.4326,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.3442566692829132,
+      "rewards/margins": 0.3176761269569397,
+      "rewards/rejected": 0.02658059261739254,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.817830801010132,
-      "eval_logits/rejected": -2.8154354095458984,
-      "eval_logps/chosen": -31.114850997924805,
-      "eval_logps/rejected": -34.70843505859375,
-      "eval_loss": 0.4829135835170746,
-      "eval_rewards/accuracies": 0.5838870406150818,
-      "eval_rewards/chosen": 0.08379998803138733,
-      "eval_rewards/margins": 0.08894261717796326,
-      "eval_rewards/rejected": -0.005142624489963055,
-      "eval_runtime": 113.1025,
-      "eval_samples_per_second": 3.033,
-      "eval_steps_per_second": 0.38,
+      "eval_logits/chosen": -2.820904493331909,
+      "eval_logits/rejected": -2.8182966709136963,
+      "eval_logps/chosen": -31.072538375854492,
+      "eval_logps/rejected": -34.71522521972656,
+      "eval_loss": 0.49085888266563416,
+      "eval_rewards/accuracies": 0.6013289093971252,
+      "eval_rewards/chosen": 0.04198317229747772,
+      "eval_rewards/margins": 0.045396942645311356,
+      "eval_rewards/rejected": -0.0034137717448174953,
+      "eval_runtime": 112.9358,
+      "eval_samples_per_second": 3.037,
+      "eval_steps_per_second": 0.381,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 6.71875,
+      "grad_norm": 3.234375,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -2.777716636657715,
-      "logits/rejected": -2.796701192855835,
-      "logps/chosen": -28.045873641967773,
-      "logps/rejected": -30.80637550354004,
-      "loss": 0.3823,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.5917724370956421,
-      "rewards/margins": 0.6289572715759277,
-      "rewards/rejected": -0.03718482702970505,
+      "logits/chosen": -2.781768321990967,
+      "logits/rejected": -2.799870014190674,
+      "logps/chosen": -28.066579818725586,
+      "logps/rejected": -30.81374740600586,
+      "loss": 0.4473,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.23256821930408478,
+      "rewards/margins": 0.24891658127307892,
+      "rewards/rejected": -0.01634838618338108,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 7.34375,
+      "grad_norm": 3.5625,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -3.0270373821258545,
-      "logits/rejected": -3.0125222206115723,
-      "logps/chosen": -28.760854721069336,
-      "logps/rejected": -28.367849349975586,
-      "loss": 0.3548,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.7253671884536743,
-      "rewards/margins": 0.7499045729637146,
-      "rewards/rejected": -0.024537306278944016,
+      "logits/chosen": -3.0310263633728027,
+      "logits/rejected": -3.016596794128418,
+      "logps/chosen": -28.514297485351562,
+      "logps/rejected": -28.385051727294922,
+      "loss": 0.424,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.3394576907157898,
+      "rewards/margins": 0.35271310806274414,
+      "rewards/rejected": -0.013255435042083263,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 4.75,
+      "grad_norm": 3.234375,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -2.9413743019104004,
-      "logits/rejected": -2.9236152172088623,
-      "logps/chosen": -26.63749122619629,
-      "logps/rejected": -30.598424911499023,
-      "loss": 0.3132,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.8622905015945435,
-      "rewards/margins": 0.9656804800033569,
-      "rewards/rejected": -0.10339002311229706,
+      "logits/chosen": -2.94514799118042,
+      "logits/rejected": -2.927401065826416,
+      "logps/chosen": -26.721960067749023,
+      "logps/rejected": -30.596094131469727,
+      "loss": 0.4178,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.3280225396156311,
+      "rewards/margins": 0.3689119517803192,
+      "rewards/rejected": -0.0408894307911396,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 7.53125,
+      "grad_norm": 2.71875,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -3.1589274406433105,
-      "logits/rejected": -3.1642251014709473,
-      "logps/chosen": -30.21347999572754,
-      "logps/rejected": -33.01905822753906,
-      "loss": 0.3462,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.7128629088401794,
-      "rewards/margins": 0.8743909001350403,
-      "rewards/rejected": -0.16152794659137726,
+      "logits/chosen": -3.162492275238037,
+      "logits/rejected": -3.1681036949157715,
+      "logps/chosen": -30.14105796813965,
+      "logps/rejected": -32.993080139160156,
+      "loss": 0.4252,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.2996302545070648,
+      "rewards/margins": 0.3590452969074249,
+      "rewards/rejected": -0.05941504240036011,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 6.8125,
+      "grad_norm": 3.3125,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -3.03485107421875,
-      "logits/rejected": -3.0382072925567627,
-      "logps/chosen": -29.085901260375977,
-      "logps/rejected": -31.341243743896484,
-      "loss": 0.3558,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.8604887127876282,
-      "rewards/margins": 0.7744562029838562,
-      "rewards/rejected": 0.08603246510028839,
+      "logits/chosen": -3.0381181240081787,
+      "logits/rejected": -3.0421128273010254,
+      "logps/chosen": -29.018634796142578,
+      "logps/rejected": -31.29150390625,
+      "loss": 0.432,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.3576488494873047,
+      "rewards/margins": 0.3132873773574829,
+      "rewards/rejected": 0.044361427426338196,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 8.3125,
+      "grad_norm": 4.09375,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -2.8737285137176514,
-      "logits/rejected": -2.8753609657287598,
-      "logps/chosen": -26.806705474853516,
-      "logps/rejected": -29.70511245727539,
-      "loss": 0.3406,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.9215120077133179,
-      "rewards/margins": 0.8585984110832214,
-      "rewards/rejected": 0.06291364133358002,
+      "logits/chosen": -2.8777647018432617,
+      "logits/rejected": -2.879361152648926,
+      "logps/chosen": -26.779287338256836,
+      "logps/rejected": -29.69747543334961,
+      "loss": 0.4224,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.374088317155838,
+      "rewards/margins": 0.34739503264427185,
+      "rewards/rejected": 0.026693273335695267,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 12.0,
+      "grad_norm": 5.34375,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.9500927925109863,
-      "logits/rejected": -2.9481420516967773,
-      "logps/chosen": -29.373714447021484,
-      "logps/rejected": -31.941335678100586,
-      "loss": 0.368,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.7064663767814636,
-      "rewards/margins": 0.7131119966506958,
-      "rewards/rejected": -0.006645613815635443,
+      "logits/chosen": -2.953718900680542,
+      "logits/rejected": -2.9520201683044434,
+      "logps/chosen": -29.355998992919922,
+      "logps/rejected": -32.019020080566406,
+      "loss": 0.4352,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.28613001108169556,
+      "rewards/margins": 0.30432528257369995,
+      "rewards/rejected": -0.01819526217877865,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 7.0,
+      "grad_norm": 3.390625,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -2.9054651260375977,
-      "logits/rejected": -2.888453483581543,
-      "logps/chosen": -27.82125473022461,
-      "logps/rejected": -28.030620574951172,
-      "loss": 0.3791,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.5618898272514343,
-      "rewards/margins": 0.632390558719635,
-      "rewards/rejected": -0.0705007016658783,
+      "logits/chosen": -2.9095873832702637,
+      "logits/rejected": -2.892209053039551,
+      "logps/chosen": -27.843591690063477,
+      "logps/rejected": -28.02511215209961,
+      "loss": 0.4472,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.22028858959674835,
+      "rewards/margins": 0.24738721549510956,
+      "rewards/rejected": -0.02709861472249031,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.4045382279854316,
-      "train_runtime": 2722.7053,
+      "train_loss": 0.4552104888024268,
+      "train_runtime": 2721.944,
       "train_samples_per_second": 1.131,
       "train_steps_per_second": 0.141
     }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:428fd89ac4380aec582be16d3b03dc4062c9e559f287bd699a88e2ac0428dd59
+oid sha256:8cd55e98933069b2e4490dbcf1a439222449747889ee66fac7c164c4ab928e15
 size 5240