Model save

README.md

@@ -1,13 +1,11 @@
 ---
+license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: norallm/normistral-7b-warm
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: ap-normistral-7b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # ap-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/ap-normistral-7b-sft-qlora](https://huggingface.co/data/ap-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [norallm/normistral-7b-warm](https://huggingface.co/norallm/normistral-7b-warm) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 8.0178
-- Rewards/chosen: 0.0011
-- Rewards/rejected: 0.0010
-- Rewards/accuracies: 0.5220
-- Rewards/margins: 0.0001
-- Logps/rejected: -35.9615
-- Logps/chosen: -32.4377
-- Logits/rejected: 98.9752
-- Logits/chosen: 98.9888
+- Loss: 1.0124
+- Rewards/chosen: -0.0636
+- Rewards/rejected: -0.1409
+- Rewards/accuracies: 0.5399
+- Rewards/margins: 0.0773
+- Logps/rejected: -36.1679
+- Logps/chosen: -32.5340
+- Logits/rejected: 98.7098
+- Logits/chosen: 98.7339
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 6.7218        | 0.26  | 100  | 7.5705          | -0.0028        | -0.0151          | 0.5083             | 0.0123          | -36.0421       | -32.4573     | 98.6549         | 98.6709       |
-| 6.2997        | 0.52  | 200  | 7.1457          | -0.0201        | -0.0563          | 0.5403             | 0.0361          | -36.2478       | -32.5438     | 98.9065         | 98.9208       |
-| 6.0651        | 0.78  | 300  | 7.6485          | 0.0015         | -0.0116          | 0.5390             | 0.0131          | -36.0246       | -32.4356     | 98.9867         | 99.0023       |
+| 0.9161        | 0.26  | 100  | 1.0662          | -0.0346        | -0.0243          | 0.5129             | -0.0103         | -36.0012       | -32.4926     | 98.7623         | 98.7729       |
+| 0.8398        | 0.52  | 200  | 1.0090          | -0.1355        | -0.2279          | 0.5141             | 0.0924          | -36.2922       | -32.6368     | 98.6916         | 98.7167       |
+| 0.733         | 0.78  | 300  | 1.0124          | -0.0636        | -0.1409          | 0.5399             | 0.0773          | -36.1679       | -32.5340     | 98.7098         | 98.7339       |
 
 
 ### Framework versions

adapter_config.json

@@ -21,12 +21,12 @@
   "revision": null,
   "target_modules": [
     "q_proj",
-    "v_proj",
-    "o_proj",
     "down_proj",
-    "up_proj",
+    "k_proj",
     "gate_proj",
-    "k_proj"
+    "o_proj",
+    "up_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e50138c59173f57650f9d0ad3b14bfcb03b10ed6b1305895e7aa0ced25a3fe7
+oid sha256:4e4bf485d9ede6e5833d2548a9848fdfe2eaa47e25ebcd62ded26e35d006d3b0
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 98.98882293701172,
-    "eval_logits/rejected": 98.9751968383789,
-    "eval_logps/chosen": -32.437740325927734,
-    "eval_logps/rejected": -35.96149826049805,
-    "eval_loss": 8.017751693725586,
-    "eval_rewards/accuracies": 0.5220099687576294,
-    "eval_rewards/chosen": 0.0010875342413783073,
-    "eval_rewards/margins": 8.220935706049204e-05,
-    "eval_rewards/rejected": 0.0010053252335637808,
-    "eval_runtime": 103.7392,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.306,
-    "eval_steps_per_second": 0.415,
-    "train_loss": 5.985443219271573,
-    "train_runtime": 2557.1257,
+    "train_loss": 0.7429099231571347,
+    "train_runtime": 2555.3187,
     "train_samples": 3079,
-    "train_samples_per_second": 1.204,
+    "train_samples_per_second": 1.205,
     "train_steps_per_second": 0.151
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 5.985443219271573,
-    "train_runtime": 2557.1257,
+    "train_loss": 0.7429099231571347,
+    "train_runtime": 2555.3187,
     "train_samples": 3079,
-    "train_samples_per_second": 1.204,
+    "train_samples_per_second": 1.205,
     "train_steps_per_second": 0.151
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 656.0,
+      "grad_norm": 91.5,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": 88.18099975585938,
       "logits/rejected": 88.25153350830078,
       "logps/chosen": -29.073104858398438,
       "logps/rejected": -26.25731658935547,
-      "loss": 6.25,
+      "loss": 1.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,629 +25,629 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 536.0,
+      "grad_norm": 76.5,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": 81.07795715332031,
-      "logits/rejected": 80.78132629394531,
-      "logps/chosen": -34.14677047729492,
-      "logps/rejected": -33.015296936035156,
-      "loss": 6.1804,
-      "rewards/accuracies": 0.5277777910232544,
-      "rewards/chosen": 0.0189919825643301,
-      "rewards/margins": 0.030369587242603302,
-      "rewards/rejected": -0.011377605609595776,
+      "logits/chosen": 81.08255004882812,
+      "logits/rejected": 80.78926086425781,
+      "logps/chosen": -34.20470428466797,
+      "logps/rejected": -33.038047790527344,
+      "loss": 0.9368,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.02591484785079956,
+      "rewards/margins": 0.0816662609577179,
+      "rewards/rejected": -0.055751409381628036,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 592.0,
+      "grad_norm": 60.25,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": 80.67132568359375,
-      "logits/rejected": 80.56291198730469,
-      "logps/chosen": -33.51841354370117,
-      "logps/rejected": -30.76877212524414,
-      "loss": 6.6549,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.034299030900001526,
-      "rewards/margins": 0.03255577012896538,
-      "rewards/rejected": 0.0017432533204555511,
+      "logits/chosen": 80.67174530029297,
+      "logits/rejected": 80.55998229980469,
+      "logps/chosen": -33.60923767089844,
+      "logps/rejected": -30.828128814697266,
+      "loss": 0.9439,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.05647152662277222,
+      "rewards/margins": 0.09191782772541046,
+      "rewards/rejected": -0.035446297377347946,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 448.0,
+      "grad_norm": 67.5,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": 82.46919250488281,
-      "logits/rejected": 82.5018539428711,
-      "logps/chosen": -33.83555603027344,
-      "logps/rejected": -31.259449005126953,
-      "loss": 7.4075,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.03545312210917473,
-      "rewards/margins": 0.0025764047168195248,
-      "rewards/rejected": 0.03287670761346817,
+      "logits/chosen": 82.49557495117188,
+      "logits/rejected": 82.5277099609375,
+      "logps/chosen": -33.90664291381836,
+      "logps/rejected": -31.188213348388672,
+      "loss": 1.0985,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.07432325184345245,
+      "rewards/margins": -0.0906088799238205,
+      "rewards/rejected": 0.16493213176727295,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 612.0,
+      "grad_norm": 75.5,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": 80.91734313964844,
-      "logits/rejected": 80.9122314453125,
-      "logps/chosen": -32.870399475097656,
-      "logps/rejected": -33.192291259765625,
-      "loss": 6.9708,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.06073303893208504,
-      "rewards/margins": 0.025485849007964134,
-      "rewards/rejected": 0.03524719178676605,
+      "logits/chosen": 81.06272888183594,
+      "logits/rejected": 81.05645751953125,
+      "logps/chosen": -32.722740173339844,
+      "logps/rejected": -33.15789031982422,
+      "loss": 0.8862,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.3159271776676178,
+      "rewards/margins": 0.1684812754392624,
+      "rewards/rejected": 0.1474459171295166,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 528.0,
+      "grad_norm": 48.75,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": 78.42672729492188,
-      "logits/rejected": 78.44120788574219,
-      "logps/chosen": -30.90542984008789,
-      "logps/rejected": -31.027896881103516,
-      "loss": 6.7099,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.059963710606098175,
-      "rewards/margins": 0.05088571831583977,
-      "rewards/rejected": 0.009077982977032661,
+      "logits/chosen": 78.66886901855469,
+      "logits/rejected": 78.68830871582031,
+      "logps/chosen": -30.57431411743164,
+      "logps/rejected": -30.798864364624023,
+      "loss": 0.9351,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.44165611267089844,
+      "rewards/margins": 0.24956238269805908,
+      "rewards/rejected": 0.19209368526935577,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 498.0,
+      "grad_norm": 72.5,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": 82.86418151855469,
-      "logits/rejected": 82.91536712646484,
-      "logps/chosen": -31.078943252563477,
-      "logps/rejected": -29.606725692749023,
-      "loss": 7.0829,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.0192436333745718,
-      "rewards/margins": 0.022484425455331802,
-      "rewards/rejected": -0.0032407897524535656,
+      "logits/chosen": 83.24879455566406,
+      "logits/rejected": 83.29969787597656,
+      "logps/chosen": -30.918521881103516,
+      "logps/rejected": -29.482006072998047,
+      "loss": 1.0359,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.17964980006217957,
+      "rewards/margins": 0.1036890521645546,
+      "rewards/rejected": 0.07596075534820557,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 960.0,
+      "grad_norm": 87.0,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": 83.54774475097656,
-      "logits/rejected": 83.57620239257812,
-      "logps/chosen": -30.6588077545166,
-      "logps/rejected": -33.22206115722656,
-      "loss": 6.976,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.0059564015828073025,
-      "rewards/margins": 0.0309152714908123,
-      "rewards/rejected": -0.03687167167663574,
+      "logits/chosen": 83.91014099121094,
+      "logits/rejected": 83.93647766113281,
+      "logps/chosen": -30.502422332763672,
+      "logps/rejected": -33.12609100341797,
+      "loss": 0.9567,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.08862228691577911,
+      "rewards/margins": 0.15049318969249725,
+      "rewards/rejected": -0.06187089532613754,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 556.0,
+      "grad_norm": 74.0,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": 81.12471771240234,
-      "logits/rejected": 81.11027526855469,
-      "logps/chosen": -31.388225555419922,
-      "logps/rejected": -30.88912010192871,
-      "loss": 6.8226,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.023343011736869812,
-      "rewards/margins": 0.049637455493211746,
-      "rewards/rejected": -0.026294440031051636,
+      "logits/chosen": 81.4886245727539,
+      "logits/rejected": 81.47552490234375,
+      "logps/chosen": -31.330123901367188,
+      "logps/rejected": -31.121978759765625,
+      "loss": 0.814,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.12237221002578735,
+      "rewards/margins": 0.3774043917655945,
+      "rewards/rejected": -0.25503218173980713,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 680.0,
+      "grad_norm": 83.0,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": 78.06856536865234,
-      "logits/rejected": 78.03953552246094,
-      "logps/chosen": -32.45366668701172,
-      "logps/rejected": -31.23443603515625,
-      "loss": 6.1663,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.03690256550908089,
-      "rewards/margins": 0.09349562227725983,
-      "rewards/rejected": -0.056593067944049835,
+      "logits/chosen": 78.25160217285156,
+      "logits/rejected": 78.21989440917969,
+      "logps/chosen": -32.49519729614258,
+      "logps/rejected": -31.26288414001465,
+      "loss": 0.8564,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.10008607059717178,
+      "rewards/margins": 0.31807559728622437,
+      "rewards/rejected": -0.21798951923847198,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 396.0,
+      "grad_norm": 82.5,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": 83.3121566772461,
-      "logits/rejected": 83.32379913330078,
-      "logps/chosen": -34.064453125,
-      "logps/rejected": -31.79021644592285,
-      "loss": 6.7218,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.04810718446969986,
-      "rewards/margins": 0.05604150891304016,
-      "rewards/rejected": -0.00793431792408228,
+      "logits/chosen": 83.49636840820312,
+      "logits/rejected": 83.53155517578125,
+      "logps/chosen": -34.187034606933594,
+      "logps/rejected": -31.907695770263672,
+      "loss": 0.9161,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0825684443116188,
+      "rewards/margins": 0.19257497787475586,
+      "rewards/rejected": -0.11000655591487885,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": 98.67085266113281,
-      "eval_logits/rejected": 98.65489959716797,
-      "eval_logps/chosen": -32.457313537597656,
-      "eval_logps/rejected": -36.04214859008789,
-      "eval_loss": 7.570494651794434,
-      "eval_rewards/accuracies": 0.5083056688308716,
-      "eval_rewards/chosen": -0.002827130490913987,
-      "eval_rewards/margins": 0.0122976154088974,
-      "eval_rewards/rejected": -0.015124746598303318,
-      "eval_runtime": 104.1006,
-      "eval_samples_per_second": 3.295,
-      "eval_steps_per_second": 0.413,
+      "eval_logits/chosen": 98.7728500366211,
+      "eval_logits/rejected": 98.76228332519531,
+      "eval_logps/chosen": -32.4925651550293,
+      "eval_logps/rejected": -36.00117492675781,
+      "eval_loss": 1.0662287473678589,
+      "eval_rewards/accuracies": 0.5128737688064575,
+      "eval_rewards/chosen": -0.03457321599125862,
+      "eval_rewards/margins": -0.010320436209440231,
+      "eval_rewards/rejected": -0.02425277978181839,
+      "eval_runtime": 104.2438,
+      "eval_samples_per_second": 3.29,
+      "eval_steps_per_second": 0.412,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 752.0,
+      "grad_norm": 91.5,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": 83.45378112792969,
-      "logits/rejected": 83.33932495117188,
-      "logps/chosen": -32.50124740600586,
-      "logps/rejected": -32.954078674316406,
-      "loss": 5.6892,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.104311004281044,
-      "rewards/margins": 0.16394877433776855,
-      "rewards/rejected": -0.05963778495788574,
+      "logits/chosen": 83.7912368774414,
+      "logits/rejected": 83.67459106445312,
+      "logps/chosen": -32.45995330810547,
+      "logps/rejected": -32.786006927490234,
+      "loss": 0.8008,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.39399290084838867,
+      "rewards/margins": 0.4850761294364929,
+      "rewards/rejected": -0.09108323603868484,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 620.0,
+      "grad_norm": 95.0,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": 83.64091491699219,
-      "logits/rejected": 83.74860382080078,
-      "logps/chosen": -28.32021141052246,
-      "logps/rejected": -35.66899871826172,
-      "loss": 5.3716,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.12187393009662628,
-      "rewards/margins": 0.16384465992450714,
-      "rewards/rejected": -0.04197072237730026,
+      "logits/chosen": 84.00267028808594,
+      "logits/rejected": 84.11933898925781,
+      "logps/chosen": -28.27166175842285,
+      "logps/rejected": -35.5056037902832,
+      "loss": 0.7011,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.4605434536933899,
+      "rewards/margins": 0.4930640757083893,
+      "rewards/rejected": -0.03252064064145088,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 484.0,
+      "grad_norm": 64.5,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": 80.95816040039062,
-      "logits/rejected": 80.97350311279297,
-      "logps/chosen": -30.554763793945312,
-      "logps/rejected": -32.3652458190918,
-      "loss": 6.2467,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.07051900774240494,
-      "rewards/margins": 0.15724298357963562,
-      "rewards/rejected": -0.08672399073839188,
+      "logits/chosen": 81.23250579833984,
+      "logits/rejected": 81.24813842773438,
+      "logps/chosen": -30.38728904724121,
+      "logps/rejected": -32.12664031982422,
+      "loss": 0.7344,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.3640448749065399,
+      "rewards/margins": 0.5005542039871216,
+      "rewards/rejected": -0.13650932908058167,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 428.0,
+      "grad_norm": 59.75,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": 82.39835357666016,
-      "logits/rejected": 82.39920806884766,
-      "logps/chosen": -27.08688735961914,
-      "logps/rejected": -33.00067901611328,
-      "loss": 5.3964,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.07208801805973053,
-      "rewards/margins": 0.18754826486110687,
-      "rewards/rejected": -0.11546025425195694,
+      "logits/chosen": 82.44822692871094,
+      "logits/rejected": 82.46932220458984,
+      "logps/chosen": -27.172740936279297,
+      "logps/rejected": -33.0168571472168,
+      "loss": 0.7509,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.19220882654190063,
+      "rewards/margins": 0.6076450347900391,
+      "rewards/rejected": -0.4154362082481384,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 482.0,
+      "grad_norm": 54.0,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": 81.19883728027344,
-      "logits/rejected": 81.17581939697266,
-      "logps/chosen": -28.9885311126709,
-      "logps/rejected": -32.907562255859375,
-      "loss": 5.3313,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.08011577278375626,
-      "rewards/margins": 0.14832790195941925,
-      "rewards/rejected": -0.06821214407682419,
+      "logits/chosen": 80.97503662109375,
+      "logits/rejected": 80.94606018066406,
+      "logps/chosen": -29.037616729736328,
+      "logps/rejected": -33.28493118286133,
+      "loss": 0.6493,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.2460467517375946,
+      "rewards/margins": 0.7489484548568726,
+      "rewards/rejected": -0.5029016733169556,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 1040.0,
+      "grad_norm": 63.0,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": 83.12278747558594,
-      "logits/rejected": 83.13607025146484,
-      "logps/chosen": -33.499732971191406,
-      "logps/rejected": -30.342330932617188,
-      "loss": 6.478,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.1274038851261139,
-      "rewards/margins": 0.2053629606962204,
-      "rewards/rejected": -0.07795912027359009,
+      "logits/chosen": 82.87214660644531,
+      "logits/rejected": 82.87894439697266,
+      "logps/chosen": -33.497344970703125,
+      "logps/rejected": -30.375295639038086,
+      "loss": 0.7389,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.4475820064544678,
+      "rewards/margins": 0.7435113191604614,
+      "rewards/rejected": -0.2959292531013489,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 684.0,
+      "grad_norm": 63.25,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": 84.02305603027344,
-      "logits/rejected": 83.95697784423828,
-      "logps/chosen": -30.76401710510254,
-      "logps/rejected": -32.3089599609375,
-      "loss": 5.8957,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.09905388206243515,
-      "rewards/margins": 0.1568334400653839,
-      "rewards/rejected": -0.057779550552368164,
+      "logits/chosen": 83.60514831542969,
+      "logits/rejected": 83.55717468261719,
+      "logps/chosen": -30.812519073486328,
+      "logps/rejected": -32.62251281738281,
+      "loss": 0.7275,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.31273749470710754,
+      "rewards/margins": 0.7344537377357483,
+      "rewards/rejected": -0.42171627283096313,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 498.0,
+      "grad_norm": 54.25,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": 81.92304992675781,
-      "logits/rejected": 81.90460205078125,
-      "logps/chosen": -30.485610961914062,
-      "logps/rejected": -31.443450927734375,
-      "loss": 5.0645,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.1423078328371048,
-      "rewards/margins": 0.19955609738826752,
-      "rewards/rejected": -0.05724828690290451,
+      "logits/chosen": 81.30427551269531,
+      "logits/rejected": 81.28076171875,
+      "logps/chosen": -30.593530654907227,
+      "logps/rejected": -31.66329574584961,
+      "loss": 0.58,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.4225357472896576,
+      "rewards/margins": 0.7767966985702515,
+      "rewards/rejected": -0.3542609214782715,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 502.0,
+      "grad_norm": 37.5,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": 83.73831176757812,
-      "logits/rejected": 83.71122741699219,
-      "logps/chosen": -30.49334716796875,
-      "logps/rejected": -30.74527931213379,
-      "loss": 7.3479,
+      "logits/chosen": 82.88330078125,
+      "logits/rejected": 82.88325500488281,
+      "logps/chosen": -30.3848934173584,
+      "logps/rejected": -30.6480770111084,
+      "loss": 0.8789,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.05461405590176582,
-      "rewards/margins": 0.11006426811218262,
-      "rewards/rejected": -0.055450212210416794,
+      "rewards/chosen": 0.2670655846595764,
+      "rewards/margins": 0.39310184121131897,
+      "rewards/rejected": -0.12603625655174255,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 294.0,
+      "grad_norm": 56.25,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": 79.29591369628906,
-      "logits/rejected": 79.25688171386719,
-      "logps/chosen": -34.00603485107422,
-      "logps/rejected": -32.55823516845703,
-      "loss": 6.2997,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.1493411809206009,
-      "rewards/margins": 0.16230711340904236,
-      "rewards/rejected": -0.012965935282409191,
+      "logits/chosen": 78.49826049804688,
+      "logits/rejected": 78.43463134765625,
+      "logps/chosen": -33.972293853759766,
+      "logps/rejected": -32.59092330932617,
+      "loss": 0.8398,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.5463098287582397,
+      "rewards/margins": 0.6145724058151245,
+      "rewards/rejected": -0.06826266646385193,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": 98.92082977294922,
-      "eval_logits/rejected": 98.90645599365234,
-      "eval_logps/chosen": -32.54380416870117,
-      "eval_logps/rejected": -36.24779510498047,
-      "eval_loss": 7.145691394805908,
-      "eval_rewards/accuracies": 0.5402824282646179,
-      "eval_rewards/chosen": -0.02012600749731064,
-      "eval_rewards/margins": 0.036127228289842606,
-      "eval_rewards/rejected": -0.056253232061862946,
-      "eval_runtime": 104.0138,
-      "eval_samples_per_second": 3.298,
+      "eval_logits/chosen": 98.71674346923828,
+      "eval_logits/rejected": 98.69157409667969,
+      "eval_logps/chosen": -32.63681411743164,
+      "eval_logps/rejected": -36.29216003417969,
+      "eval_loss": 1.0089657306671143,
+      "eval_rewards/accuracies": 0.5141196250915527,
+      "eval_rewards/chosen": -0.13554596900939941,
+      "eval_rewards/margins": 0.09239647537469864,
+      "eval_rewards/rejected": -0.22794245183467865,
+      "eval_runtime": 104.0982,
+      "eval_samples_per_second": 3.295,
       "eval_steps_per_second": 0.413,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 1192.0,
+      "grad_norm": 73.5,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": 81.78749084472656,
-      "logits/rejected": 81.68329620361328,
-      "logps/chosen": -33.50835037231445,
-      "logps/rejected": -35.296775817871094,
-      "loss": 5.7897,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.08747398853302002,
-      "rewards/margins": 0.14960959553718567,
-      "rewards/rejected": -0.062135614454746246,
+      "logits/chosen": 81.18994140625,
+      "logits/rejected": 81.10676574707031,
+      "logps/chosen": -33.214935302734375,
+      "logps/rejected": -35.187870025634766,
+      "loss": 0.6733,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.511551022529602,
+      "rewards/margins": 0.6527955532073975,
+      "rewards/rejected": -0.14124450087547302,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 382.0,
+      "grad_norm": 53.5,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": 83.9058609008789,
-      "logits/rejected": 83.9858627319336,
-      "logps/chosen": -31.16668128967285,
-      "logps/rejected": -30.917465209960938,
-      "loss": 6.2677,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.15645496547222137,
-      "rewards/margins": 0.1941571831703186,
-      "rewards/rejected": -0.03770221397280693,
+      "logits/chosen": 83.3000259399414,
+      "logits/rejected": 83.38983917236328,
+      "logps/chosen": -31.05356216430664,
+      "logps/rejected": -31.100086212158203,
+      "loss": 0.5573,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.6267757415771484,
+      "rewards/margins": 0.8865677714347839,
+      "rewards/rejected": -0.2597920000553131,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 688.0,
+      "grad_norm": 59.5,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": 81.14820098876953,
-      "logits/rejected": 81.19589233398438,
-      "logps/chosen": -32.21388626098633,
-      "logps/rejected": -34.05015182495117,
-      "loss": 6.1846,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.11837179958820343,
-      "rewards/margins": 0.12597264349460602,
-      "rewards/rejected": -0.007600849959999323,
+      "logits/chosen": 80.46002197265625,
+      "logits/rejected": 80.5117416381836,
+      "logps/chosen": -32.2435417175293,
+      "logps/rejected": -34.2344970703125,
+      "loss": 0.7635,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.393540620803833,
+      "rewards/margins": 0.5491863489151001,
+      "rewards/rejected": -0.15564575791358948,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 764.0,
+      "grad_norm": 72.0,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": 83.51747131347656,
-      "logits/rejected": 83.77994537353516,
-      "logps/chosen": -30.83194351196289,
-      "logps/rejected": -31.766277313232422,
-      "loss": 3.8985,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.17122383415699005,
-      "rewards/margins": 0.21807047724723816,
-      "rewards/rejected": -0.0468466654419899,
+      "logits/chosen": 82.81127166748047,
+      "logits/rejected": 83.10184478759766,
+      "logps/chosen": -30.6541748046875,
+      "logps/rejected": -31.797021865844727,
+      "loss": 0.5215,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.7237205505371094,
+      "rewards/margins": 0.9092057347297668,
+      "rewards/rejected": -0.18548506498336792,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 528.0,
+      "grad_norm": 76.5,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": 82.23722076416016,
-      "logits/rejected": 82.29248809814453,
-      "logps/chosen": -26.981786727905273,
-      "logps/rejected": -29.9254093170166,
-      "loss": 5.8549,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.122822105884552,
-      "rewards/margins": 0.12775607407093048,
-      "rewards/rejected": -0.0049339523538947105,
+      "logits/chosen": 81.55049133300781,
+      "logits/rejected": 81.6218032836914,
+      "logps/chosen": -26.869796752929688,
+      "logps/rejected": -30.16421890258789,
+      "loss": 0.7649,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.5082693696022034,
+      "rewards/margins": 0.6927058696746826,
+      "rewards/rejected": -0.18443644046783447,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 744.0,
+      "grad_norm": 52.5,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": 79.6116943359375,
-      "logits/rejected": 79.72075653076172,
-      "logps/chosen": -30.516536712646484,
-      "logps/rejected": -36.090003967285156,
-      "loss": 4.7707,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.20889195799827576,
-      "rewards/margins": 0.20656757056713104,
-      "rewards/rejected": 0.0023243515752255917,
+      "logits/chosen": 78.84456634521484,
+      "logits/rejected": 78.9933090209961,
+      "logps/chosen": -30.304040908813477,
+      "logps/rejected": -36.30915069580078,
+      "loss": 0.4765,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.8798693418502808,
+      "rewards/margins": 1.0251331329345703,
+      "rewards/rejected": -0.1452637016773224,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 348.0,
+      "grad_norm": 57.0,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": 79.08982849121094,
-      "logits/rejected": 79.111083984375,
-      "logps/chosen": -30.892492294311523,
-      "logps/rejected": -31.574138641357422,
-      "loss": 4.8727,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.1772860586643219,
-      "rewards/margins": 0.1910295933485031,
-      "rewards/rejected": -0.013743534684181213,
+      "logits/chosen": 78.18994903564453,
+      "logits/rejected": 78.21639251708984,
+      "logps/chosen": -30.825641632080078,
+      "logps/rejected": -31.800342559814453,
+      "loss": 0.6623,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.6672974228858948,
+      "rewards/margins": 0.8737428784370422,
+      "rewards/rejected": -0.20644548535346985,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 700.0,
+      "grad_norm": 76.0,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": 81.58210754394531,
-      "logits/rejected": 81.3693618774414,
-      "logps/chosen": -31.06771469116211,
-      "logps/rejected": -29.591297149658203,
-      "loss": 7.3884,
+      "logits/chosen": 80.78196716308594,
+      "logits/rejected": 80.56169128417969,
+      "logps/chosen": -30.946773529052734,
+      "logps/rejected": -29.854522705078125,
+      "loss": 0.6812,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.14843526482582092,
-      "rewards/margins": 0.13329505920410156,
-      "rewards/rejected": 0.015140200033783913,
+      "rewards/chosen": 0.6041832566261292,
+      "rewards/margins": 0.7354522943496704,
+      "rewards/rejected": -0.13126906752586365,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 428.0,
+      "grad_norm": 60.25,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": 81.82354736328125,
-      "logits/rejected": 81.73272705078125,
-      "logps/chosen": -33.02164077758789,
-      "logps/rejected": -32.17236328125,
-      "loss": 4.7078,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.21652066707611084,
-      "rewards/margins": 0.24762043356895447,
-      "rewards/rejected": -0.03109976276755333,
+      "logits/chosen": 80.99944305419922,
+      "logits/rejected": 80.91734313964844,
+      "logps/chosen": -32.94629669189453,
+      "logps/rejected": -32.49176025390625,
+      "loss": 0.5405,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.8105659484863281,
+      "rewards/margins": 1.1429924964904785,
+      "rewards/rejected": -0.3324265480041504,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 704.0,
+      "grad_norm": 72.5,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": 77.42338562011719,
-      "logits/rejected": 77.51200866699219,
-      "logps/chosen": -32.24098205566406,
-      "logps/rejected": -29.014667510986328,
-      "loss": 6.0651,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.2329459935426712,
-      "rewards/margins": 0.21547456085681915,
-      "rewards/rejected": 0.017471451312303543,
+      "logits/chosen": 76.61840057373047,
+      "logits/rejected": 76.69322204589844,
+      "logps/chosen": -32.09418487548828,
+      "logps/rejected": -29.156299591064453,
+      "loss": 0.733,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.9180667996406555,
+      "rewards/margins": 0.9560586214065552,
+      "rewards/rejected": -0.03799174353480339,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": 99.00227355957031,
-      "eval_logits/rejected": 98.98667907714844,
-      "eval_logps/chosen": -32.43557357788086,
-      "eval_logps/rejected": -36.024574279785156,
-      "eval_loss": 7.648494243621826,
-      "eval_rewards/accuracies": 0.5390365719795227,
-      "eval_rewards/chosen": 0.0015206302050501108,
-      "eval_rewards/margins": 0.01312990766018629,
-      "eval_rewards/rejected": -0.011609279550611973,
-      "eval_runtime": 104.0603,
+      "eval_logits/chosen": 98.73394012451172,
+      "eval_logits/rejected": 98.70979309082031,
+      "eval_logps/chosen": -32.53404235839844,
+      "eval_logps/rejected": -36.16787338256836,
+      "eval_loss": 1.0123846530914307,
+      "eval_rewards/accuracies": 0.5398671627044678,
+      "eval_rewards/chosen": -0.06360965222120285,
+      "eval_rewards/margins": 0.07733342051506042,
+      "eval_rewards/rejected": -0.14094306528568268,
+      "eval_runtime": 104.0616,
       "eval_samples_per_second": 3.296,
       "eval_steps_per_second": 0.413,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 1200.0,
+      "grad_norm": 63.0,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": 84.3228759765625,
-      "logits/rejected": 84.34573364257812,
-      "logps/chosen": -29.936237335205078,
-      "logps/rejected": -32.30895233154297,
-      "loss": 5.3743,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.18545907735824585,
-      "rewards/margins": 0.2072969228029251,
-      "rewards/rejected": -0.02183784916996956,
+      "logits/chosen": 83.6187515258789,
+      "logits/rejected": 83.65609741210938,
+      "logps/chosen": -30.124670028686523,
+      "logps/rejected": -32.455326080322266,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.5172038078308105,
+      "rewards/margins": 0.6960991024971008,
+      "rewards/rejected": -0.1788952797651291,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 406.0,
+      "grad_norm": 53.5,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": 81.96125030517578,
-      "logits/rejected": 81.95556640625,
-      "logps/chosen": -30.652542114257812,
-      "logps/rejected": -28.881006240844727,
-      "loss": 5.1557,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.18920376896858215,
-      "rewards/margins": 0.18150660395622253,
-      "rewards/rejected": 0.007697162684053183,
+      "logits/chosen": 81.21861267089844,
+      "logits/rejected": 81.21974182128906,
+      "logps/chosen": -30.413522720336914,
+      "logps/rejected": -29.050277709960938,
+      "loss": 0.5746,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.8295270204544067,
+      "rewards/margins": 0.921076774597168,
+      "rewards/rejected": -0.09154972434043884,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 576.0,
+      "grad_norm": 42.5,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": 79.27213287353516,
-      "logits/rejected": 79.31598663330078,
-      "logps/chosen": -29.210277557373047,
-      "logps/rejected": -32.73192596435547,
-      "loss": 5.7764,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.22846612334251404,
-      "rewards/margins": 0.2478782832622528,
-      "rewards/rejected": -0.019412150606513023,
+      "logits/chosen": 78.42388153076172,
+      "logits/rejected": 78.47276306152344,
+      "logps/chosen": -28.912973403930664,
+      "logps/rejected": -32.85163116455078,
+      "loss": 0.5118,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 1.007744550704956,
+      "rewards/margins": 1.1594822406768799,
+      "rewards/rejected": -0.151737779378891,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 708.0,
+      "grad_norm": 78.0,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": 83.35621643066406,
-      "logits/rejected": 83.36161041259766,
-      "logps/chosen": -32.25800323486328,
-      "logps/rejected": -33.48296356201172,
-      "loss": 5.3324,
+      "logits/chosen": 82.58137512207031,
+      "logits/rejected": 82.63166046142578,
+      "logps/chosen": -32.287044525146484,
+      "logps/rejected": -33.664268493652344,
+      "loss": 0.7276,
       "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.19082824885845184,
-      "rewards/margins": 0.213682621717453,
-      "rewards/rejected": -0.022854382172226906,
+      "rewards/chosen": 0.6475721597671509,
+      "rewards/margins": 0.8544757962226868,
+      "rewards/rejected": -0.2069036066532135,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 322.0,
+      "grad_norm": 56.75,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": 82.49974822998047,
-      "logits/rejected": 82.50485229492188,
-      "logps/chosen": -32.731300354003906,
-      "logps/rejected": -33.191688537597656,
-      "loss": 4.9862,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.18248799443244934,
-      "rewards/margins": 0.20246508717536926,
-      "rewards/rejected": -0.019977085292339325,
+      "logits/chosen": 81.68583679199219,
+      "logits/rejected": 81.69776916503906,
+      "logps/chosen": -32.62708282470703,
+      "logps/rejected": -33.35443115234375,
+      "loss": 0.626,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.7116604447364807,
+      "rewards/margins": 0.8955025672912598,
+      "rewards/rejected": -0.18384216725826263,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 520.0,
+      "grad_norm": 45.5,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": 83.8789291381836,
-      "logits/rejected": 83.89030456542969,
-      "logps/chosen": -28.570327758789062,
-      "logps/rejected": -31.54669761657715,
-      "loss": 5.4694,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.19637688994407654,
-      "rewards/margins": 0.17574702203273773,
-      "rewards/rejected": 0.02062985673546791,
+      "logits/chosen": 83.15383911132812,
+      "logits/rejected": 83.18663024902344,
+      "logps/chosen": -28.314571380615234,
+      "logps/rejected": -31.6760196685791,
+      "loss": 0.6231,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.8663473129272461,
+      "rewards/margins": 0.8846683502197266,
+      "rewards/rejected": -0.01832098886370659,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 370.0,
+      "grad_norm": 76.0,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": 83.33052062988281,
-      "logits/rejected": 83.347900390625,
-      "logps/chosen": -31.880367279052734,
-      "logps/rejected": -35.12144470214844,
-      "loss": 7.7502,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.1991589069366455,
-      "rewards/margins": 0.20157304406166077,
-      "rewards/rejected": -0.0024141333997249603,
+      "logits/chosen": 82.54973602294922,
+      "logits/rejected": 82.57438659667969,
+      "logps/chosen": -31.776432037353516,
+      "logps/rejected": -35.274253845214844,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.7698130011558533,
+      "rewards/margins": 0.8852267265319824,
+      "rewards/rejected": -0.11541371047496796,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 552.0,
+      "grad_norm": 72.0,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": 77.32600402832031,
-      "logits/rejected": 77.20854187011719,
-      "logps/chosen": -29.861562728881836,
-      "logps/rejected": -28.112417221069336,
-      "loss": 5.5564,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.14291271567344666,
-      "rewards/margins": 0.13328143954277039,
-      "rewards/rejected": 0.009631268680095673,
+      "logits/chosen": 76.53568267822266,
+      "logits/rejected": 76.40200805664062,
+      "logps/chosen": -29.737590789794922,
+      "logps/rejected": -28.21639060974121,
+      "loss": 0.7558,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.5869752168655396,
+      "rewards/margins": 0.6260467767715454,
+      "rewards/rejected": -0.03907149285078049,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 5.985443219271573,
-      "train_runtime": 2557.1257,
-      "train_samples_per_second": 1.204,
+      "train_loss": 0.7429099231571347,
+      "train_runtime": 2555.3187,
+      "train_samples_per_second": 1.205,
       "train_steps_per_second": 0.151
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:36caf2cdf572f31cfb78457f93ba1a0558bce2656d0098002c48658d566d6b26
+oid sha256:2b435ab8c55f09f04cca28fea6ed4f872528741b4c6183b32b876c674f200e19
 size 5176