Model save

README.md

@@ -1,13 +1,11 @@
 ---
+license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: norallm/normistral-7b-warm
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: ap-normistral-7b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # ap-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/ap-normistral-7b-sft-qlora](https://huggingface.co/data/ap-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [norallm/normistral-7b-warm](https://huggingface.co/norallm/normistral-7b-warm) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4850
-- Rewards/chosen: 0.1654
-- Rewards/rejected: 0.0857
-- Rewards/accuracies: 0.5399
-- Rewards/margins: 0.0797
-- Logps/rejected: -35.8594
-- Logps/chosen: -32.2364
-- Logits/rejected: 98.4059
-- Logits/chosen: 98.4070
+- Loss: 0.4970
+- Rewards/chosen: 0.0579
+- Rewards/rejected: 0.0502
+- Rewards/accuracies: 0.5565
+- Rewards/margins: 0.0077
+- Logps/rejected: -35.8660
+- Logps/chosen: -32.3273
+- Logits/rejected: 98.3156
+- Logits/chosen: 98.3222
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4705        | 0.26  | 100  | 0.4947          | 0.1342         | 0.1226           | 0.4896             | 0.0116          | -35.8133       | -32.2754     | 98.6750         | 98.6871       |
-| 0.3097        | 0.52  | 200  | 0.4910          | 0.1604         | 0.1008           | 0.5482             | 0.0596          | -35.8405       | -32.2427     | 98.4702         | 98.4771       |
-| 0.3066        | 0.78  | 300  | 0.4855          | 0.1659         | 0.0986           | 0.5282             | 0.0673          | -35.8433       | -32.2358     | 98.4267         | 98.4293       |
+| 0.4665        | 0.26  | 100  | 0.4971          | 0.0771         | 0.0651           | 0.5187             | 0.0119          | -35.8363       | -32.2890     | 98.7124         | 98.7206       |
+| 0.3678        | 0.52  | 200  | 0.4983          | -0.0042        | -0.0160          | 0.5104             | 0.0118          | -35.9986       | -32.4516     | 98.3577         | 98.3655       |
+| 0.3546        | 0.78  | 300  | 0.4970          | 0.0579         | 0.0502           | 0.5565             | 0.0077          | -35.8660       | -32.3273     | 98.3156         | 98.3222       |
 
 
 ### Framework versions

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
     "down_proj",
+    "q_proj",
+    "k_proj",
     "gate_proj",
-    "up_proj",
     "o_proj",
-    "q_proj",
-    "v_proj"
+    "v_proj",
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf079650c439cdb1f37ec5806ba619b31a174f7b193b1d7b513c190631ec4ff1
+oid sha256:01ea61ca676bb5fb7626b3e65ac896e19efe9053567ca5104807f3f0f6b10fed
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 98.40702056884766,
-    "eval_logits/rejected": 98.40592193603516,
-    "eval_logps/chosen": -32.23640441894531,
-    "eval_logps/rejected": -35.8593864440918,
-    "eval_loss": 0.4850274920463562,
-    "eval_rewards/accuracies": 0.5398671627044678,
-    "eval_rewards/chosen": 0.1654178649187088,
-    "eval_rewards/margins": 0.07970692217350006,
-    "eval_rewards/rejected": 0.08571097254753113,
-    "eval_runtime": 103.9265,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.3,
-    "eval_steps_per_second": 0.414,
-    "train_loss": 0.37664644687206716,
-    "train_runtime": 2560.444,
+    "train_loss": 0.41032042689137643,
+    "train_runtime": 2557.4055,
     "train_samples": 3079,
-    "train_samples_per_second": 1.203,
-    "train_steps_per_second": 0.15
+    "train_samples_per_second": 1.204,
+    "train_steps_per_second": 0.151
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.37664644687206716,
-    "train_runtime": 2560.444,
+    "train_loss": 0.41032042689137643,
+    "train_runtime": 2557.4055,
     "train_samples": 3079,
-    "train_samples_per_second": 1.203,
-    "train_steps_per_second": 0.15
+    "train_samples_per_second": 1.204,
+    "train_steps_per_second": 0.151
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 26.125,
+      "grad_norm": 16.375,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": 88.18099975585938,
       "logits/rejected": 88.25153350830078,
@@ -25,630 +25,630 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 14.1875,
+      "grad_norm": 8.8125,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": 81.0709228515625,
-      "logits/rejected": 80.77515411376953,
-      "logps/chosen": -34.26983642578125,
-      "logps/rejected": -32.992767333984375,
-      "loss": 0.4962,
-      "rewards/accuracies": 0.4444444477558136,
-      "rewards/chosen": -0.022488469257950783,
-      "rewards/margins": 0.004999637603759766,
-      "rewards/rejected": -0.02748810313642025,
+      "logits/chosen": 81.07431030273438,
+      "logits/rejected": 80.78235626220703,
+      "logps/chosen": -34.18563461303711,
+      "logps/rejected": -33.02939987182617,
+      "loss": 0.4904,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": 0.028044726699590683,
+      "rewards/margins": 0.06354151666164398,
+      "rewards/rejected": -0.035496786236763,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 19.5,
+      "grad_norm": 12.1875,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": 80.6608657836914,
-      "logits/rejected": 80.54856872558594,
-      "logps/chosen": -33.62510299682617,
-      "logps/rejected": -30.784603118896484,
-      "loss": 0.4924,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.05184358358383179,
-      "rewards/margins": 0.057535119354724884,
-      "rewards/rejected": -0.00569154042750597,
+      "logits/chosen": 80.67153930664062,
+      "logits/rejected": 80.56294250488281,
+      "logps/chosen": -33.529541015625,
+      "logps/rejected": -30.742996215820312,
+      "loss": 0.4869,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.08018453419208527,
+      "rewards/margins": 0.06293892860412598,
+      "rewards/rejected": 0.017245600000023842,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 22.125,
+      "grad_norm": 14.625,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": 82.50384521484375,
-      "logits/rejected": 82.53746032714844,
-      "logps/chosen": -33.812801361083984,
-      "logps/rejected": -31.15009117126465,
-      "loss": 0.5123,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.16001427173614502,
-      "rewards/margins": -0.058978963643312454,
-      "rewards/rejected": 0.21899321675300598,
+      "logits/chosen": 82.5155029296875,
+      "logits/rejected": 82.54915618896484,
+      "logps/chosen": -33.7542610168457,
+      "logps/rejected": -31.115161895751953,
+      "loss": 0.5066,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.12928077578544617,
+      "rewards/margins": -0.02505389414727688,
+      "rewards/rejected": 0.15433469414710999,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 18.0,
+      "grad_norm": 12.5625,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": 81.07115173339844,
-      "logits/rejected": 81.06380462646484,
-      "logps/chosen": -32.71454620361328,
-      "logps/rejected": -33.17704772949219,
-      "loss": 0.4627,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.36761564016342163,
-      "rewards/margins": 0.21443231403827667,
-      "rewards/rejected": 0.15318328142166138,
+      "logits/chosen": 81.09931945800781,
+      "logits/rejected": 81.09571838378906,
+      "logps/chosen": -32.699241638183594,
+      "logps/rejected": -33.092124938964844,
+      "loss": 0.4793,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.23740839958190918,
+      "rewards/margins": 0.0992104634642601,
+      "rewards/rejected": 0.13819792866706848,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 16.75,
+      "grad_norm": 11.125,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": 78.77976989746094,
-      "logits/rejected": 78.78811645507812,
-      "logps/chosen": -30.60152816772461,
-      "logps/rejected": -30.760879516601562,
-      "loss": 0.4673,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.4829755425453186,
-      "rewards/margins": 0.2330504208803177,
-      "rewards/rejected": 0.24992509186267853,
+      "logits/chosen": 78.78851318359375,
+      "logits/rejected": 78.79679870605469,
+      "logps/chosen": -30.52480125427246,
+      "logps/rejected": -30.73565673828125,
+      "loss": 0.4662,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.34022435545921326,
+      "rewards/margins": 0.17140880227088928,
+      "rewards/rejected": 0.16881553828716278,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 15.125,
+      "grad_norm": 9.75,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": 83.38993072509766,
-      "logits/rejected": 83.4462890625,
-      "logps/chosen": -30.877517700195312,
-      "logps/rejected": -29.332889556884766,
-      "loss": 0.4954,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": 0.23811349272727966,
-      "rewards/margins": 0.03200835734605789,
-      "rewards/rejected": 0.20610514283180237,
+      "logits/chosen": 83.39762878417969,
+      "logits/rejected": 83.45609283447266,
+      "logps/chosen": -30.74808692932129,
+      "logps/rejected": -29.20343017578125,
+      "loss": 0.4975,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.21353764832019806,
+      "rewards/margins": 0.019991278648376465,
+      "rewards/rejected": 0.1935463845729828,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 15.6875,
+      "grad_norm": 10.9375,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": 84.03268432617188,
-      "logits/rejected": 84.06501007080078,
-      "logps/chosen": -30.23248291015625,
-      "logps/rejected": -32.74273681640625,
-      "loss": 0.4844,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.31723442673683167,
-      "rewards/margins": 0.08125714957714081,
-      "rewards/rejected": 0.23597724735736847,
+      "logits/chosen": 84.00616455078125,
+      "logits/rejected": 84.04234313964844,
+      "logps/chosen": -30.251235961914062,
+      "logps/rejected": -32.49794387817383,
+      "loss": 0.5173,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.18889428675174713,
+      "rewards/margins": -0.08099017292261124,
+      "rewards/rejected": 0.2698844373226166,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 17.625,
+      "grad_norm": 12.375,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": 81.7315673828125,
-      "logits/rejected": 81.71873474121094,
-      "logps/chosen": -30.963665008544922,
-      "logps/rejected": -30.75851058959961,
-      "loss": 0.4187,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.4330216348171234,
-      "rewards/margins": 0.4337124824523926,
-      "rewards/rejected": -0.0006908595678396523,
+      "logits/chosen": 81.75172424316406,
+      "logits/rejected": 81.72709655761719,
+      "logps/chosen": -31.1412353515625,
+      "logps/rejected": -30.653112411499023,
+      "loss": 0.4759,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.18185201287269592,
+      "rewards/margins": 0.12958452105522156,
+      "rewards/rejected": 0.05226749926805496,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 21.25,
+      "grad_norm": 13.9375,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": 78.5487060546875,
-      "logits/rejected": 78.52108001708984,
-      "logps/chosen": -32.06660461425781,
-      "logps/rejected": -30.661907196044922,
-      "loss": 0.4704,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.4572601318359375,
-      "rewards/margins": 0.22561049461364746,
-      "rewards/rejected": 0.23164963722229004,
+      "logits/chosen": 78.56499481201172,
+      "logits/rejected": 78.54066467285156,
+      "logps/chosen": -32.319190979003906,
+      "logps/rejected": -30.92232894897461,
+      "loss": 0.476,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.15949341654777527,
+      "rewards/margins": 0.14492300152778625,
+      "rewards/rejected": 0.014570409432053566,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 13.25,
+      "grad_norm": 11.875,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": 83.6882095336914,
-      "logits/rejected": 83.7122802734375,
-      "logps/chosen": -33.82537841796875,
-      "logps/rejected": -31.4283504486084,
-      "loss": 0.4705,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.38369041681289673,
-      "rewards/margins": 0.1259343922138214,
-      "rewards/rejected": 0.2577560245990753,
+      "logits/chosen": 83.73712158203125,
+      "logits/rejected": 83.75628662109375,
+      "logps/chosen": -33.78915023803711,
+      "logps/rejected": -31.565814971923828,
+      "loss": 0.4665,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.2579200267791748,
+      "rewards/margins": 0.16555462777614594,
+      "rewards/rejected": 0.09236541390419006,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": 98.68708801269531,
-      "eval_logits/rejected": 98.67501068115234,
-      "eval_logps/chosen": -32.27544021606445,
-      "eval_logps/rejected": -35.81333541870117,
-      "eval_loss": 0.4946621358394623,
-      "eval_rewards/accuracies": 0.4896179437637329,
-      "eval_rewards/chosen": 0.13418929278850555,
-      "eval_rewards/margins": 0.011633642949163914,
-      "eval_rewards/rejected": 0.12255565822124481,
-      "eval_runtime": 104.4117,
-      "eval_samples_per_second": 3.285,
-      "eval_steps_per_second": 0.412,
+      "eval_logits/chosen": 98.72062683105469,
+      "eval_logits/rejected": 98.71237182617188,
+      "eval_logps/chosen": -32.28904724121094,
+      "eval_logps/rejected": -35.83627700805664,
+      "eval_loss": 0.49707475304603577,
+      "eval_rewards/accuracies": 0.5186877250671387,
+      "eval_rewards/chosen": 0.07706477493047714,
+      "eval_rewards/margins": 0.011940184980630875,
+      "eval_rewards/rejected": 0.06512458622455597,
+      "eval_runtime": 104.0818,
+      "eval_samples_per_second": 3.295,
+      "eval_steps_per_second": 0.413,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 22.75,
+      "grad_norm": 14.8125,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": 83.8403091430664,
-      "logits/rejected": 83.71327209472656,
-      "logps/chosen": -31.912221908569336,
-      "logps/rejected": -32.58714294433594,
-      "loss": 0.3643,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.8884676694869995,
-      "rewards/margins": 0.8334689140319824,
-      "rewards/rejected": 0.054998766630887985,
+      "logits/chosen": 84.00208282470703,
+      "logits/rejected": 83.8756332397461,
+      "logps/chosen": -32.03178405761719,
+      "logps/rejected": -32.6201171875,
+      "loss": 0.4084,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.49551233649253845,
+      "rewards/margins": 0.47762519121170044,
+      "rewards/rejected": 0.01788714900612831,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 22.5,
+      "grad_norm": 13.5625,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": 83.93721008300781,
-      "logits/rejected": 84.06002044677734,
-      "logps/chosen": -27.933292388916016,
-      "logps/rejected": -35.20619201660156,
-      "loss": 0.4011,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.7970338463783264,
-      "rewards/margins": 0.594670295715332,
-      "rewards/rejected": 0.2023634910583496,
+      "logits/chosen": 84.0489273071289,
+      "logits/rejected": 84.1673355102539,
+      "logps/chosen": -27.966100692749023,
+      "logps/rejected": -35.121253967285156,
+      "loss": 0.441,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.48174038529396057,
+      "rewards/margins": 0.31279435753822327,
+      "rewards/rejected": 0.16894599795341492,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 15.25,
+      "grad_norm": 7.28125,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": 81.2428970336914,
-      "logits/rejected": 81.28059387207031,
-      "logps/chosen": -30.123279571533203,
-      "logps/rejected": -31.811786651611328,
-      "loss": 0.4158,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.6272625923156738,
-      "rewards/margins": 0.5313904881477356,
-      "rewards/rejected": 0.09587211161851883,
+      "logits/chosen": 81.32206726074219,
+      "logits/rejected": 81.3568115234375,
+      "logps/chosen": -30.228296279907227,
+      "logps/rejected": -31.748275756835938,
+      "loss": 0.4559,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.33953094482421875,
+      "rewards/margins": 0.24785713851451874,
+      "rewards/rejected": 0.09167381376028061,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 16.375,
+      "grad_norm": 8.625,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": 82.30902862548828,
-      "logits/rejected": 82.33167266845703,
-      "logps/chosen": -26.606557846069336,
-      "logps/rejected": -32.55984115600586,
-      "loss": 0.3859,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.6726120710372925,
-      "rewards/margins": 0.7817814946174622,
-      "rewards/rejected": -0.10916946828365326,
+      "logits/chosen": 82.2553939819336,
+      "logits/rejected": 82.27656555175781,
+      "logps/chosen": -26.800729751586914,
+      "logps/rejected": -32.687808990478516,
+      "loss": 0.423,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.32329756021499634,
+      "rewards/margins": 0.455514132976532,
+      "rewards/rejected": -0.13221657276153564,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 12.9375,
+      "grad_norm": 12.5625,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": 80.79722595214844,
-      "logits/rejected": 80.76870727539062,
-      "logps/chosen": -28.53302001953125,
-      "logps/rejected": -32.609378814697266,
-      "loss": 0.3794,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.6848722696304321,
-      "rewards/margins": 0.719173014163971,
-      "rewards/rejected": -0.03430069237947464,
+      "logits/chosen": 80.7685775756836,
+      "logits/rejected": 80.73258209228516,
+      "logps/chosen": -28.54288673400879,
+      "logps/rejected": -32.596839904785156,
+      "loss": 0.415,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.423112154006958,
+      "rewards/margins": 0.4382806718349457,
+      "rewards/rejected": -0.01516849733889103,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 19.75,
+      "grad_norm": 14.125,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": 82.5254135131836,
-      "logits/rejected": 82.54801940917969,
-      "logps/chosen": -33.080387115478516,
-      "logps/rejected": -30.11862564086914,
-      "loss": 0.3503,
+      "logits/chosen": 82.48309326171875,
+      "logits/rejected": 82.50714111328125,
+      "logps/chosen": -33.225772857666016,
+      "logps/rejected": -30.1079044342041,
+      "loss": 0.4023,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.8450896143913269,
-      "rewards/margins": 0.9779588580131531,
-      "rewards/rejected": -0.13286921381950378,
+      "rewards/chosen": 0.45548883080482483,
+      "rewards/margins": 0.533171534538269,
+      "rewards/rejected": -0.0776827409863472,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 10.5,
+      "grad_norm": 7.5625,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": 83.30370330810547,
-      "logits/rejected": 83.26591491699219,
-      "logps/chosen": -30.516061782836914,
-      "logps/rejected": -32.39433288574219,
-      "loss": 0.3746,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.5945821404457092,
-      "rewards/margins": 0.8940039873123169,
-      "rewards/rejected": -0.29942184686660767,
+      "logits/chosen": 83.15037536621094,
+      "logits/rejected": 83.10893249511719,
+      "logps/chosen": -30.452091217041016,
+      "logps/rejected": -32.47702407836914,
+      "loss": 0.3953,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.4035983681678772,
+      "rewards/margins": 0.6320796012878418,
+      "rewards/rejected": -0.2284812033176422,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 7.34375,
+      "grad_norm": 6.65625,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": 80.87037658691406,
-      "logits/rejected": 80.84497833251953,
-      "logps/chosen": -30.32219886779785,
-      "logps/rejected": -31.39385414123535,
-      "loss": 0.3678,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.6999613642692566,
-      "rewards/margins": 0.8892775774002075,
-      "rewards/rejected": -0.18931610882282257,
+      "logits/chosen": 80.6706314086914,
+      "logits/rejected": 80.64241027832031,
+      "logps/chosen": -30.28702163696289,
+      "logps/rejected": -31.376535415649414,
+      "loss": 0.404,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.45506635308265686,
+      "rewards/margins": 0.5647307634353638,
+      "rewards/rejected": -0.10966438055038452,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 9.25,
+      "grad_norm": 6.21875,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": 82.518310546875,
-      "logits/rejected": 82.48826599121094,
-      "logps/chosen": -29.71840476989746,
-      "logps/rejected": -30.17477798461914,
-      "loss": 0.4079,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.8384085893630981,
-      "rewards/margins": 0.6038115620613098,
-      "rewards/rejected": 0.23459705710411072,
+      "logits/chosen": 82.2298583984375,
+      "logits/rejected": 82.20623779296875,
+      "logps/chosen": -29.83859634399414,
+      "logps/rejected": -30.299697875976562,
+      "loss": 0.4314,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.4639095664024353,
+      "rewards/margins": 0.37974613904953003,
+      "rewards/rejected": 0.0841633602976799,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 9.0625,
+      "grad_norm": 6.875,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": 77.99578094482422,
-      "logits/rejected": 77.94181060791016,
-      "logps/chosen": -32.918636322021484,
-      "logps/rejected": -32.197086334228516,
-      "loss": 0.3097,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 1.4672811031341553,
-      "rewards/margins": 1.2302258014678955,
-      "rewards/rejected": 0.23705501854419708,
+      "logits/chosen": 77.64122009277344,
+      "logits/rejected": 77.57923889160156,
+      "logps/chosen": -33.089454650878906,
+      "logps/rejected": -32.29143524169922,
+      "loss": 0.3678,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.8316432237625122,
+      "rewards/margins": 0.7306608557701111,
+      "rewards/rejected": 0.10098233073949814,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": 98.47710418701172,
-      "eval_logits/rejected": 98.47020721435547,
-      "eval_logps/chosen": -32.242671966552734,
-      "eval_logps/rejected": -35.840518951416016,
-      "eval_loss": 0.49102693796157837,
-      "eval_rewards/accuracies": 0.5481727719306946,
-      "eval_rewards/chosen": 0.1604028344154358,
-      "eval_rewards/margins": 0.0595964714884758,
-      "eval_rewards/rejected": 0.1008063480257988,
-      "eval_runtime": 104.1775,
-      "eval_samples_per_second": 3.292,
+      "eval_logits/chosen": 98.36553955078125,
+      "eval_logits/rejected": 98.35772705078125,
+      "eval_logps/chosen": -32.45161056518555,
+      "eval_logps/rejected": -35.998619079589844,
+      "eval_loss": 0.49831458926200867,
+      "eval_rewards/accuracies": 0.5103820562362671,
+      "eval_rewards/chosen": -0.004216374363750219,
+      "eval_rewards/margins": 0.011831996031105518,
+      "eval_rewards/rejected": -0.01604837365448475,
+      "eval_runtime": 104.055,
+      "eval_samples_per_second": 3.296,
       "eval_steps_per_second": 0.413,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 20.25,
+      "grad_norm": 15.25,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": 80.51226806640625,
-      "logits/rejected": 80.43040466308594,
-      "logps/chosen": -32.629119873046875,
-      "logps/rejected": -34.76416778564453,
-      "loss": 0.3655,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 1.053282618522644,
-      "rewards/margins": 0.87574303150177,
-      "rewards/rejected": 0.1775396168231964,
+      "logits/chosen": 80.19715881347656,
+      "logits/rejected": 80.11329650878906,
+      "logps/chosen": -32.6320915222168,
+      "logps/rejected": -34.92211151123047,
+      "loss": 0.3902,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.656814455986023,
+      "rewards/margins": 0.624821662902832,
+      "rewards/rejected": 0.031992800533771515,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 14.5,
+      "grad_norm": 12.0,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": 82.59284973144531,
-      "logits/rejected": 82.66585540771484,
-      "logps/chosen": -30.357955932617188,
-      "logps/rejected": -30.737834930419922,
-      "loss": 0.2951,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 1.2727962732315063,
-      "rewards/margins": 1.2799025774002075,
-      "rewards/rejected": -0.007106387056410313,
+      "logits/chosen": 82.21825408935547,
+      "logits/rejected": 82.29025268554688,
+      "logps/chosen": -30.436092376708984,
+      "logps/rejected": -30.719776153564453,
+      "loss": 0.3564,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.7564297318458557,
+      "rewards/margins": 0.7518402934074402,
+      "rewards/rejected": 0.004589465446770191,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 12.0625,
+      "grad_norm": 9.625,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": 79.70103454589844,
-      "logits/rejected": 79.75657653808594,
-      "logps/chosen": -31.675893783569336,
-      "logps/rejected": -33.84308624267578,
-      "loss": 0.3884,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.9038804173469543,
-      "rewards/margins": 0.7686265707015991,
-      "rewards/rejected": 0.1352538764476776,
+      "logits/chosen": 79.31720733642578,
+      "logits/rejected": 79.36396789550781,
+      "logps/chosen": -31.852947235107422,
+      "logps/rejected": -34.1156120300293,
+      "loss": 0.4099,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.476398229598999,
+      "rewards/margins": 0.5281308889389038,
+      "rewards/rejected": -0.05173276737332344,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 20.875,
+      "grad_norm": 13.5,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": 82.11561584472656,
-      "logits/rejected": 82.39993286132812,
-      "logps/chosen": -30.073062896728516,
-      "logps/rejected": -31.364795684814453,
-      "loss": 0.3068,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 1.2920019626617432,
-      "rewards/margins": 1.158202886581421,
-      "rewards/rejected": 0.13379916548728943,
+      "logits/chosen": 81.70478820800781,
+      "logits/rejected": 81.98667907714844,
+      "logps/chosen": -30.028610229492188,
+      "logps/rejected": -31.500635147094727,
+      "loss": 0.3447,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.8297265768051147,
+      "rewards/margins": 0.8140215873718262,
+      "rewards/rejected": 0.015705054625868797,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 15.4375,
+      "grad_norm": 12.6875,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": 80.80778503417969,
-      "logits/rejected": 80.8788833618164,
-      "logps/chosen": -26.4283504486084,
-      "logps/rejected": -29.62093162536621,
-      "loss": 0.3934,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.934036135673523,
-      "rewards/margins": 0.7101906538009644,
-      "rewards/rejected": 0.22384557127952576,
+      "logits/chosen": 80.39967346191406,
+      "logits/rejected": 80.46524810791016,
+      "logps/chosen": -26.57110595703125,
+      "logps/rejected": -29.88582420349121,
+      "loss": 0.4136,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.5123947858810425,
+      "rewards/margins": 0.5049386024475098,
+      "rewards/rejected": 0.00745623093098402,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 13.1875,
+      "grad_norm": 9.8125,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": 78.03572082519531,
-      "logits/rejected": 78.22270202636719,
-      "logps/chosen": -29.80512046813965,
-      "logps/rejected": -35.99930953979492,
-      "loss": 0.297,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 1.4046982526779175,
-      "rewards/margins": 1.3228428363800049,
-      "rewards/rejected": 0.08185521513223648,
+      "logits/chosen": 77.56782531738281,
+      "logits/rejected": 77.75471496582031,
+      "logps/chosen": -29.75429916381836,
+      "logps/rejected": -36.2445068359375,
+      "loss": 0.3377,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.9033478498458862,
+      "rewards/margins": 0.9747893214225769,
+      "rewards/rejected": -0.07144142687320709,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 9.3125,
+      "grad_norm": 7.96875,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": 77.41798400878906,
-      "logits/rejected": 77.4520034790039,
-      "logps/chosen": -30.092838287353516,
-      "logps/rejected": -31.44430160522461,
-      "loss": 0.305,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 1.3488702774047852,
-      "rewards/margins": 1.2999744415283203,
-      "rewards/rejected": 0.04889560490846634,
+      "logits/chosen": 77.03749084472656,
+      "logits/rejected": 77.07095336914062,
+      "logps/chosen": -30.273632049560547,
+      "logps/rejected": -31.36687660217285,
+      "loss": 0.3778,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.7526459097862244,
+      "rewards/margins": 0.6833735704421997,
+      "rewards/rejected": 0.06927235424518585,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 20.125,
+      "grad_norm": 15.25,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": 80.09739685058594,
-      "logits/rejected": 79.87271881103516,
-      "logps/chosen": -30.528423309326172,
-      "logps/rejected": -29.228124618530273,
-      "loss": 0.382,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 1.0251753330230713,
-      "rewards/margins": 0.6740784049034119,
-      "rewards/rejected": 0.3510969281196594,
+      "logits/chosen": 79.75798797607422,
+      "logits/rejected": 79.53197479248047,
+      "logps/chosen": -30.511974334716797,
+      "logps/rejected": -29.195556640625,
+      "loss": 0.4169,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.6489585638046265,
+      "rewards/margins": 0.41323956847190857,
+      "rewards/rejected": 0.2357189655303955,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 14.25,
+      "grad_norm": 7.71875,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": 80.29197692871094,
-      "logits/rejected": 80.20320129394531,
-      "logps/chosen": -32.255340576171875,
-      "logps/rejected": -32.0814208984375,
-      "loss": 0.2756,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 1.4791247844696045,
-      "rewards/margins": 1.530770182609558,
-      "rewards/rejected": -0.05164545774459839,
+      "logits/chosen": 79.92450714111328,
+      "logits/rejected": 79.83580017089844,
+      "logps/chosen": -32.37202453613281,
+      "logps/rejected": -32.25506591796875,
+      "loss": 0.3223,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.8661117553710938,
+      "rewards/margins": 0.9852114915847778,
+      "rewards/rejected": -0.11909981071949005,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 14.625,
+      "grad_norm": 7.875,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": 75.86343383789062,
-      "logits/rejected": 75.9235610961914,
-      "logps/chosen": -31.682353973388672,
-      "logps/rejected": -28.961742401123047,
-      "loss": 0.3066,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 1.37868332862854,
-      "rewards/margins": 1.2664577960968018,
-      "rewards/rejected": 0.11222562938928604,
+      "logits/chosen": 75.48292541503906,
+      "logits/rejected": 75.55941009521484,
+      "logps/chosen": -31.580333709716797,
+      "logps/rejected": -28.897323608398438,
+      "loss": 0.3546,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.9126860499382019,
+      "rewards/margins": 0.8103362321853638,
+      "rewards/rejected": 0.10234987735748291,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": 98.42933654785156,
-      "eval_logits/rejected": 98.42666625976562,
-      "eval_logps/chosen": -32.235782623291016,
-      "eval_logps/rejected": -35.84326934814453,
-      "eval_loss": 0.48553377389907837,
-      "eval_rewards/accuracies": 0.5282392501831055,
-      "eval_rewards/chosen": 0.16591256856918335,
-      "eval_rewards/margins": 0.06730752438306808,
-      "eval_rewards/rejected": 0.09860503673553467,
-      "eval_runtime": 104.1659,
-      "eval_samples_per_second": 3.293,
+      "eval_logits/chosen": 98.32223510742188,
+      "eval_logits/rejected": 98.31561279296875,
+      "eval_logps/chosen": -32.32727813720703,
+      "eval_logps/rejected": -35.866024017333984,
+      "eval_loss": 0.49702879786491394,
+      "eval_rewards/accuracies": 0.5564784407615662,
+      "eval_rewards/chosen": 0.05794913321733475,
+      "eval_rewards/margins": 0.007699221838265657,
+      "eval_rewards/rejected": 0.05024990811944008,
+      "eval_runtime": 104.0255,
+      "eval_samples_per_second": 3.297,
       "eval_steps_per_second": 0.413,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 12.8125,
+      "grad_norm": 10.25,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": 82.97637176513672,
-      "logits/rejected": 83.01033020019531,
-      "logps/chosen": -29.268829345703125,
-      "logps/rejected": -32.1494026184082,
-      "loss": 0.3157,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 1.2757642269134521,
-      "rewards/margins": 1.2354792356491089,
-      "rewards/rejected": 0.04028487950563431,
+      "logits/chosen": 82.65113830566406,
+      "logits/rejected": 82.6902847290039,
+      "logps/chosen": -29.43534278869629,
+      "logps/rejected": -32.302738189697266,
+      "loss": 0.3663,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.714095413684845,
+      "rewards/margins": 0.7655847668647766,
+      "rewards/rejected": -0.05148933455348015,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 15.4375,
+      "grad_norm": 9.75,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": 80.3984603881836,
-      "logits/rejected": 80.40061950683594,
-      "logps/chosen": -29.979394912719727,
-      "logps/rejected": -28.756725311279297,
-      "loss": 0.3119,
+      "logits/chosen": 80.09264373779297,
+      "logits/rejected": 80.09661865234375,
+      "logps/chosen": -29.789846420288086,
+      "logps/rejected": -28.805282592773438,
+      "loss": 0.3454,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 1.2953336238861084,
-      "rewards/margins": 1.1651219129562378,
-      "rewards/rejected": 0.13021180033683777,
+      "rewards/chosen": 0.9043582081794739,
+      "rewards/margins": 0.8472534418106079,
+      "rewards/rejected": 0.05710482597351074,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 10.75,
+      "grad_norm": 10.5625,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": 77.65103912353516,
-      "logits/rejected": 77.68055725097656,
-      "logps/chosen": -28.304912567138672,
-      "logps/rejected": -32.50617218017578,
-      "loss": 0.254,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 1.6381542682647705,
-      "rewards/margins": 1.535199522972107,
-      "rewards/rejected": 0.10295436531305313,
+      "logits/chosen": 77.26579284667969,
+      "logits/rejected": 77.29621124267578,
+      "logps/chosen": -28.363937377929688,
+      "logps/rejected": -32.58740234375,
+      "loss": 0.3182,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.9943346977233887,
+      "rewards/margins": 0.9706042408943176,
+      "rewards/rejected": 0.02373054064810276,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 15.9375,
+      "grad_norm": 12.875,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": 81.86819458007812,
-      "logits/rejected": 81.89158630371094,
-      "logps/chosen": -31.62424659729004,
-      "logps/rejected": -33.315486907958984,
-      "loss": 0.3285,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 1.2703235149383545,
-      "rewards/margins": 1.2277603149414062,
-      "rewards/rejected": 0.0425633080303669,
+      "logits/chosen": 81.5138168334961,
+      "logits/rejected": 81.5446548461914,
+      "logps/chosen": -31.642475128173828,
+      "logps/rejected": -33.37983322143555,
+      "loss": 0.3675,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.7848353385925293,
+      "rewards/margins": 0.7904062271118164,
+      "rewards/rejected": -0.0055709658190608025,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 9.3125,
+      "grad_norm": 8.6875,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": 80.94864654541016,
-      "logits/rejected": 80.96649169921875,
-      "logps/chosen": -31.77408218383789,
-      "logps/rejected": -32.98318099975586,
-      "loss": 0.3071,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 1.4957287311553955,
-      "rewards/margins": 1.4088352918624878,
-      "rewards/rejected": 0.08689363300800323,
+      "logits/chosen": 80.62837982177734,
+      "logits/rejected": 80.64482116699219,
+      "logps/chosen": -31.80319595336914,
+      "logps/rejected": -32.94707107543945,
+      "loss": 0.3582,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.9202712774276733,
+      "rewards/margins": 0.8479059338569641,
+      "rewards/rejected": 0.07236528396606445,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 11.75,
+      "grad_norm": 8.9375,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": 82.43021392822266,
-      "logits/rejected": 82.48263549804688,
-      "logps/chosen": -27.799968719482422,
-      "logps/rejected": -31.40801429748535,
-      "loss": 0.3137,
+      "logits/chosen": 82.10346984863281,
+      "logits/rejected": 82.16381072998047,
+      "logps/chosen": -27.89191246032715,
+      "logps/rejected": -31.463871002197266,
+      "loss": 0.3663,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 1.4017945528030396,
-      "rewards/margins": 1.2083251476287842,
-      "rewards/rejected": 0.19346928596496582,
+      "rewards/chosen": 0.8301493525505066,
+      "rewards/margins": 0.7371596693992615,
+      "rewards/rejected": 0.0929897278547287,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 12.1875,
+      "grad_norm": 9.6875,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": 81.93843078613281,
-      "logits/rejected": 81.97197723388672,
-      "logps/chosen": -31.361141204833984,
-      "logps/rejected": -34.82567596435547,
-      "loss": 0.3594,
+      "logits/chosen": 81.55647277832031,
+      "logits/rejected": 81.5948715209961,
+      "logps/chosen": -31.506786346435547,
+      "logps/rejected": -34.95966339111328,
+      "loss": 0.3891,
       "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 1.2120168209075928,
-      "rewards/margins": 0.9850583076477051,
-      "rewards/rejected": 0.22695855796337128,
+      "rewards/chosen": 0.6846884489059448,
+      "rewards/margins": 0.6098321676254272,
+      "rewards/rejected": 0.07485616207122803,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 14.9375,
+      "grad_norm": 10.625,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": 75.7864761352539,
-      "logits/rejected": 75.65766143798828,
-      "logps/chosen": -29.02054786682129,
-      "logps/rejected": -27.994884490966797,
-      "loss": 0.3268,
+      "logits/chosen": 75.45634460449219,
+      "logits/rejected": 75.32949829101562,
+      "logps/chosen": -29.234609603881836,
+      "logps/rejected": -28.104198455810547,
+      "loss": 0.3841,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": 1.244463562965393,
-      "rewards/margins": 1.111914038658142,
-      "rewards/rejected": 0.13254934549331665,
+      "rewards/chosen": 0.6707575917243958,
+      "rewards/margins": 0.6425691246986389,
+      "rewards/rejected": 0.02818852663040161,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.37664644687206716,
-      "train_runtime": 2560.444,
-      "train_samples_per_second": 1.203,
-      "train_steps_per_second": 0.15
+      "train_loss": 0.41032042689137643,
+      "train_runtime": 2557.4055,
+      "train_samples_per_second": 1.204,
+      "train_steps_per_second": 0.151
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae8f524b3c0e5a2eac02b1edf0712ffd75f5307f8cacd4240f37a64d9e4f8f4f
+oid sha256:9d73c235d1f07b65a1136a51110e2d5076cf4515e88ec64273fbc7b1c48eb12f
 size 5176