Model save

README.md

@@ -1,13 +1,11 @@
 ---
+license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: norallm/normistral-7b-warm
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: ap-normistral-7b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # ap-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/ap-normistral-7b-sft-qlora](https://huggingface.co/data/ap-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [norallm/normistral-7b-warm](https://huggingface.co/norallm/normistral-7b-warm) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.9419
-- Rewards/chosen: 0.0337
-- Rewards/rejected: 0.0283
-- Rewards/accuracies: 0.5490
-- Rewards/margins: 0.0054
-- Logps/rejected: -35.8958
-- Logps/chosen: -32.3590
-- Logits/rejected: 99.0714
-- Logits/chosen: 99.0740
+- Loss: 26.1545
+- Rewards/chosen: -0.0226
+- Rewards/rejected: -0.0314
+- Rewards/accuracies: 0.5307
+- Rewards/margins: 0.0088
+- Logps/rejected: -36.2802
+- Logps/chosen: -32.6693
+- Logits/rejected: 98.6503
+- Logits/chosen: 98.6807
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 2.5238        | 0.26  | 100  | 2.6219          | 0.0095         | -0.0242          | 0.5482             | 0.0337          | -36.0271       | -32.4194     | 98.8165         | 98.8297       |
-| 3.7491        | 0.52  | 200  | 2.7965          | 0.0065         | -0.0331          | 0.5282             | 0.0396          | -36.0494       | -32.4270     | 99.0970         | 99.1017       |
-| 2.7187        | 0.78  | 300  | 2.8584          | 0.0540         | 0.0448           | 0.5341             | 0.0092          | -35.8546       | -32.3082     | 99.0749         | 99.0816       |
+| 23.2882       | 0.26  | 100  | 25.6712         | -0.0029        | -0.0100          | 0.5228             | 0.0071          | -36.0669       | -32.4722     | 98.7464         | 98.7592       |
+| 20.2659       | 0.52  | 200  | 24.8289         | -0.0134        | -0.0338          | 0.5341             | 0.0204          | -36.3046       | -32.5775     | 98.6263         | 98.6515       |
+| 20.0695       | 0.78  | 300  | 26.1545         | -0.0226        | -0.0314          | 0.5307             | 0.0088          | -36.2802       | -32.6693     | 98.6503         | 98.6807       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4e5e09f4d7989fa258240732649f6f79ff37bb3fab697a06ee338c84f365034d
+oid sha256:34c2040823b42ec1dd3552d2cd2d587db0a51970776de06c7992a0c67b9bfdba
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 99.07401275634766,
-    "eval_logits/rejected": 99.07140350341797,
-    "eval_logps/chosen": -32.35896301269531,
-    "eval_logps/rejected": -35.895816802978516,
-    "eval_loss": 2.941925048828125,
-    "eval_rewards/accuracies": 0.5490033626556396,
-    "eval_rewards/chosen": 0.03368505463004112,
-    "eval_rewards/margins": 0.005401855334639549,
-    "eval_rewards/rejected": 0.028283199295401573,
-    "eval_runtime": 103.7808,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.305,
-    "eval_steps_per_second": 0.414,
-    "train_loss": 2.828607769755574,
-    "train_runtime": 2555.3286,
+    "train_loss": 20.413429577319654,
+    "train_runtime": 2557.2601,
     "train_samples": 3079,
-    "train_samples_per_second": 1.205,
+    "train_samples_per_second": 1.204,
     "train_steps_per_second": 0.151
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 2.828607769755574,
-    "train_runtime": 2555.3286,
+    "train_loss": 20.413429577319654,
+    "train_runtime": 2557.2601,
     "train_samples": 3079,
-    "train_samples_per_second": 1.205,
+    "train_samples_per_second": 1.204,
     "train_steps_per_second": 0.151
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 328.0,
+      "grad_norm": 1312.0,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": 88.18099975585938,
       "logits/rejected": 88.25153350830078,
       "logps/chosen": -29.073104858398438,
       "logps/rejected": -26.25731658935547,
-      "loss": 1.5625,
+      "loss": 25.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,629 +25,629 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 250.0,
+      "grad_norm": 1168.0,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": 81.08969116210938,
-      "logits/rejected": 80.79109191894531,
-      "logps/chosen": -34.12071228027344,
-      "logps/rejected": -33.01869201660156,
-      "loss": 1.7496,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.04840615764260292,
-      "rewards/margins": 0.07252107560634613,
-      "rewards/rejected": -0.024114925414323807,
+      "logits/chosen": 81.08999633789062,
+      "logits/rejected": 80.79169464111328,
+      "logps/chosen": -34.18925094604492,
+      "logps/rejected": -33.03681945800781,
+      "loss": 24.3423,
+      "rewards/accuracies": 0.4722222089767456,
+      "rewards/chosen": 0.005247864406555891,
+      "rewards/margins": 0.0130887096747756,
+      "rewards/rejected": -0.007840845733880997,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 336.0,
+      "grad_norm": 1032.0,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": 80.68243408203125,
-      "logits/rejected": 80.568359375,
-      "logps/chosen": -33.56159210205078,
-      "logps/rejected": -30.76800537109375,
-      "loss": 2.2924,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.05132746696472168,
-      "rewards/margins": 0.04753490164875984,
-      "rewards/rejected": 0.003792577190324664,
+      "logits/chosen": 80.66268157958984,
+      "logits/rejected": 80.54837799072266,
+      "logps/chosen": -33.62754821777344,
+      "logps/rejected": -30.786510467529297,
+      "loss": 25.2801,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.006235760636627674,
+      "rewards/margins": 0.007138081826269627,
+      "rewards/rejected": -0.0009023217717185616,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 318.0,
+      "grad_norm": 1200.0,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": 82.52204895019531,
-      "logits/rejected": 82.54562377929688,
-      "logps/chosen": -33.78315353393555,
-      "logps/rejected": -31.24331283569336,
-      "loss": 2.7815,
+      "logits/chosen": 82.51216888427734,
+      "logits/rejected": 82.54035949707031,
+      "logps/chosen": -33.695411682128906,
+      "logps/rejected": -31.29660415649414,
+      "loss": 24.6437,
       "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.09186722338199615,
-      "rewards/margins": 0.019660871475934982,
-      "rewards/rejected": 0.07220635563135147,
+      "rewards/chosen": 0.03174129128456116,
+      "rewards/margins": 0.01901828870177269,
+      "rewards/rejected": 0.012723001651465893,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 426.0,
+      "grad_norm": 1144.0,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": 80.89669036865234,
-      "logits/rejected": 80.89623260498047,
-      "logps/chosen": -32.95878219604492,
-      "logps/rejected": -33.34799575805664,
-      "loss": 2.4607,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.08611182868480682,
-      "rewards/margins": 0.07789957523345947,
-      "rewards/rejected": 0.008212263695895672,
+      "logits/chosen": 81.04322814941406,
+      "logits/rejected": 81.03800964355469,
+      "logps/chosen": -32.80583572387695,
+      "logps/rejected": -33.2043571472168,
+      "loss": 24.7349,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.03682267293334007,
+      "rewards/margins": 0.020405994728207588,
+      "rewards/rejected": 0.016416678205132484,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 772.0,
+      "grad_norm": 948.0,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": 78.38099670410156,
-      "logits/rejected": 78.40118408203125,
-      "logps/chosen": -31.085933685302734,
-      "logps/rejected": -31.042804718017578,
-      "loss": 2.589,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.04772613197565079,
-      "rewards/margins": 0.03553418815135956,
-      "rewards/rejected": 0.01219194382429123,
+      "logits/chosen": 78.70301055908203,
+      "logits/rejected": 78.71062469482422,
+      "logps/chosen": -30.72637367248535,
+      "logps/rejected": -30.724206924438477,
+      "loss": 26.36,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.04788754880428314,
+      "rewards/margins": 0.012979410588741302,
+      "rewards/rejected": 0.03490813449025154,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 249.0,
+      "grad_norm": 968.0,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": 82.9079818725586,
-      "logits/rejected": 82.95909118652344,
-      "logps/chosen": -31.034317016601562,
-      "logps/rejected": -29.518524169921875,
-      "loss": 2.8739,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.05633721500635147,
-      "rewards/margins": 0.027538930997252464,
-      "rewards/rejected": 0.028798282146453857,
+      "logits/chosen": 83.19561767578125,
+      "logits/rejected": 83.24873352050781,
+      "logps/chosen": -30.971267700195312,
+      "logps/rejected": -29.551761627197266,
+      "loss": 25.0468,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.020389318466186523,
+      "rewards/margins": 0.016513368114829063,
+      "rewards/rejected": 0.0038759508170187473,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 752.0,
+      "grad_norm": 1552.0,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": 83.62979125976562,
-      "logits/rejected": 83.66517639160156,
-      "logps/chosen": -30.505615234375,
-      "logps/rejected": -32.857994079589844,
-      "loss": 2.9346,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.04936310276389122,
-      "rewards/margins": -0.02252076379954815,
-      "rewards/rejected": 0.07188385725021362,
+      "logits/chosen": 83.78221130371094,
+      "logits/rejected": 83.81100463867188,
+      "logps/chosen": -30.479488372802734,
+      "logps/rejected": -33.11530303955078,
+      "loss": 24.2453,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.01495352853089571,
+      "rewards/margins": 0.02271350473165512,
+      "rewards/rejected": -0.007759974803775549,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 370.0,
+      "grad_norm": 1104.0,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": 81.35310363769531,
-      "logits/rejected": 81.34590148925781,
-      "logps/chosen": -31.199981689453125,
-      "logps/rejected": -30.634307861328125,
-      "loss": 2.824,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.12198303639888763,
-      "rewards/margins": 0.07264719158411026,
-      "rewards/rejected": 0.049335844814777374,
+      "logits/chosen": 81.36155700683594,
+      "logits/rejected": 81.35380554199219,
+      "logps/chosen": -31.37007713317871,
+      "logps/rejected": -30.98931884765625,
+      "loss": 23.0324,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.013486603274941444,
+      "rewards/margins": 0.03665385767817497,
+      "rewards/rejected": -0.023167254403233528,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 524.0,
+      "grad_norm": 1120.0,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": 78.339111328125,
-      "logits/rejected": 78.30552673339844,
-      "logps/chosen": -32.22107696533203,
-      "logps/rejected": -31.029504776000977,
-      "loss": 2.6385,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.16684113442897797,
-      "rewards/margins": 0.19805487990379333,
-      "rewards/rejected": -0.03121376410126686,
+      "logits/chosen": 78.11141967773438,
+      "logits/rejected": 78.0789794921875,
+      "logps/chosen": -32.44235610961914,
+      "logps/rejected": -31.213552474975586,
+      "loss": 22.8826,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.01958216354250908,
+      "rewards/margins": 0.0457901768386364,
+      "rewards/rejected": -0.02620801329612732,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 225.0,
+      "grad_norm": 1056.0,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": 83.51721954345703,
-      "logits/rejected": 83.54518127441406,
-      "logps/chosen": -34.10065841674805,
-      "logps/rejected": -31.89223861694336,
-      "loss": 2.5238,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.08173254877328873,
-      "rewards/margins": 0.1384100764989853,
-      "rewards/rejected": -0.056677550077438354,
+      "logits/chosen": 83.38008117675781,
+      "logits/rejected": 83.412841796875,
+      "logps/chosen": -34.02827072143555,
+      "logps/rejected": -31.858572006225586,
+      "loss": 23.2882,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.027672046795487404,
+      "rewards/margins": 0.038474611937999725,
+      "rewards/rejected": -0.010802562348544598,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": 98.82966613769531,
-      "eval_logits/rejected": 98.81654357910156,
-      "eval_logps/chosen": -32.419429779052734,
-      "eval_logps/rejected": -36.027076721191406,
-      "eval_loss": 2.6218504905700684,
-      "eval_rewards/accuracies": 0.5481727719306946,
-      "eval_rewards/chosen": 0.009498294442892075,
-      "eval_rewards/margins": 0.03371964767575264,
-      "eval_rewards/rejected": -0.024221351370215416,
-      "eval_runtime": 104.1511,
-      "eval_samples_per_second": 3.293,
-      "eval_steps_per_second": 0.413,
+      "eval_logits/chosen": 98.75923156738281,
+      "eval_logits/rejected": 98.74642181396484,
+      "eval_logps/chosen": -32.47222137451172,
+      "eval_logps/rejected": -36.06691360473633,
+      "eval_loss": 25.671157836914062,
+      "eval_rewards/accuracies": 0.5228405594825745,
+      "eval_rewards/chosen": -0.0029044141992926598,
+      "eval_rewards/margins": 0.0071340943686664104,
+      "eval_rewards/rejected": -0.010038508102297783,
+      "eval_runtime": 104.2457,
+      "eval_samples_per_second": 3.29,
+      "eval_steps_per_second": 0.412,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 576.0,
+      "grad_norm": 1528.0,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": 83.7013931274414,
-      "logits/rejected": 83.58271789550781,
-      "logps/chosen": -32.618534088134766,
-      "logps/rejected": -32.656673431396484,
-      "loss": 2.9173,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.16170774400234222,
-      "rewards/margins": 0.16202135384082794,
-      "rewards/rejected": -0.00031360946013592184,
+      "logits/chosen": 83.55766296386719,
+      "logits/rejected": 83.4551010131836,
+      "logps/chosen": -32.478965759277344,
+      "logps/rejected": -32.80836486816406,
+      "loss": 21.9052,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.05438382551074028,
+      "rewards/margins": 0.06963126361370087,
+      "rewards/rejected": -0.015247439965605736,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 524.0,
+      "grad_norm": 1504.0,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": 83.95661926269531,
-      "logits/rejected": 84.07592010498047,
-      "logps/chosen": -28.62860107421875,
-      "logps/rejected": -35.55434036254883,
-      "loss": 3.1258,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.12039314210414886,
-      "rewards/margins": 0.1584714949131012,
-      "rewards/rejected": -0.03807835653424263,
+      "logits/chosen": 83.767822265625,
+      "logits/rejected": 83.86802673339844,
+      "logps/chosen": -28.277118682861328,
+      "logps/rejected": -35.63311004638672,
+      "loss": 20.22,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.06524648517370224,
+      "rewards/margins": 0.08264312148094177,
+      "rewards/rejected": -0.01739663816988468,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 216.0,
+      "grad_norm": 956.0,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": 81.45957946777344,
-      "logits/rejected": 81.48210144042969,
-      "logps/chosen": -30.638086318969727,
-      "logps/rejected": -32.03240203857422,
-      "loss": 2.331,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.10770728439092636,
-      "rewards/margins": 0.14801691472530365,
-      "rewards/rejected": -0.04030962288379669,
+      "logits/chosen": 80.87911224365234,
+      "logits/rejected": 80.89720153808594,
+      "logps/chosen": -30.450307846069336,
+      "logps/rejected": -32.20357131958008,
+      "loss": 20.9442,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.04570445418357849,
+      "rewards/margins": 0.07289845496416092,
+      "rewards/rejected": -0.02719399705529213,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 386.0,
+      "grad_norm": 740.0,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": 83.00135803222656,
-      "logits/rejected": 83.00582122802734,
-      "logps/chosen": -27.22810935974121,
-      "logps/rejected": -32.823734283447266,
-      "loss": 3.1239,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.08768650889396667,
-      "rewards/margins": 0.2478286772966385,
-      "rewards/rejected": -0.160142183303833,
+      "logits/chosen": 82.11528778076172,
+      "logits/rejected": 82.12232971191406,
+      "logps/chosen": -27.089996337890625,
+      "logps/rejected": -33.013877868652344,
+      "loss": 20.3465,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.035732947289943695,
+      "rewards/margins": 0.09478302299976349,
+      "rewards/rejected": -0.059050071984529495,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 444.0,
+      "grad_norm": 948.0,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": 81.87845611572266,
-      "logits/rejected": 81.85296630859375,
-      "logps/chosen": -28.920028686523438,
-      "logps/rejected": -32.752830505371094,
-      "loss": 2.5059,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.18763259053230286,
-      "rewards/margins": 0.26216596364974976,
-      "rewards/rejected": -0.07453340291976929,
+      "logits/chosen": 80.63235473632812,
+      "logits/rejected": 80.60447692871094,
+      "logps/chosen": -28.92165756225586,
+      "logps/rejected": -33.14097213745117,
+      "loss": 18.7455,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.04674536734819412,
+      "rewards/margins": 0.10419263690710068,
+      "rewards/rejected": -0.05744727700948715,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 604.0,
+      "grad_norm": 1664.0,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": 83.88510131835938,
-      "logits/rejected": 83.89151763916016,
-      "logps/chosen": -33.4796257019043,
-      "logps/rejected": -30.0944881439209,
-      "loss": 4.2076,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.26284781098365784,
-      "rewards/margins": 0.31962868571281433,
-      "rewards/rejected": -0.056780923157930374,
+      "logits/chosen": 82.47834777832031,
+      "logits/rejected": 82.48927307128906,
+      "logps/chosen": -33.57959747314453,
+      "logps/rejected": -30.44466209411621,
+      "loss": 21.1798,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.055715084075927734,
+      "rewards/margins": 0.10492750257253647,
+      "rewards/rejected": -0.049212418496608734,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 372.0,
+      "grad_norm": 1152.0,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": 84.69200134277344,
-      "logits/rejected": 84.63328552246094,
-      "logps/chosen": -30.8580322265625,
-      "logps/rejected": -32.00690460205078,
-      "loss": 2.4171,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.1605023592710495,
-      "rewards/margins": 0.15524213016033173,
-      "rewards/rejected": 0.005260218866169453,
+      "logits/chosen": 83.31620788574219,
+      "logits/rejected": 83.2592544555664,
+      "logps/chosen": -30.965845108032227,
+      "logps/rejected": -32.591552734375,
+      "loss": 21.522,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02934456244111061,
+      "rewards/margins": 0.08649395406246185,
+      "rewards/rejected": -0.057149387896060944,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 266.0,
+      "grad_norm": 896.0,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": 82.51325225830078,
-      "logits/rejected": 82.49774169921875,
-      "logps/chosen": -30.521617889404297,
-      "logps/rejected": -31.205219268798828,
-      "loss": 2.9555,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.27021360397338867,
-      "rewards/margins": 0.28941839933395386,
-      "rewards/rejected": -0.019204800948500633,
+      "logits/chosen": 80.9317626953125,
+      "logits/rejected": 80.91111755371094,
+      "logps/chosen": -30.60970687866211,
+      "logps/rejected": -31.63262939453125,
+      "loss": 18.9358,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.058744143694639206,
+      "rewards/margins": 0.10628656297922134,
+      "rewards/rejected": -0.04754243046045303,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 316.0,
+      "grad_norm": 620.0,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": 84.31837463378906,
-      "logits/rejected": 84.31514739990234,
-      "logps/chosen": -30.445659637451172,
-      "logps/rejected": -30.434545516967773,
-      "loss": 3.42,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.12830288708209991,
-      "rewards/margins": 0.11491189152002335,
-      "rewards/rejected": 0.013391007669270039,
+      "logits/chosen": 82.61732482910156,
+      "logits/rejected": 82.6050033569336,
+      "logps/chosen": -30.360843658447266,
+      "logps/rejected": -30.71734619140625,
+      "loss": 23.6279,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.04055742174386978,
+      "rewards/margins": 0.06548986583948135,
+      "rewards/rejected": -0.024932442232966423,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 372.0,
+      "grad_norm": 700.0,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": 79.8134765625,
-      "logits/rejected": 79.75944519042969,
-      "logps/chosen": -34.02518081665039,
-      "logps/rejected": -32.41716766357422,
-      "loss": 3.7491,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.2910219728946686,
-      "rewards/margins": 0.2605269253253937,
-      "rewards/rejected": 0.03049505315721035,
+      "logits/chosen": 78.06134033203125,
+      "logits/rejected": 78.00863647460938,
+      "logps/chosen": -33.79378890991211,
+      "logps/rejected": -32.66465377807617,
+      "loss": 20.2659,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.09589491784572601,
+      "rewards/margins": 0.1130196675658226,
+      "rewards/rejected": -0.017124753445386887,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": 99.1017074584961,
-      "eval_logits/rejected": 99.0970458984375,
-      "eval_logps/chosen": -32.42703628540039,
-      "eval_logps/rejected": -36.04937744140625,
-      "eval_loss": 2.7964677810668945,
-      "eval_rewards/accuracies": 0.5282392501831055,
-      "eval_rewards/chosen": 0.006455874536186457,
-      "eval_rewards/margins": 0.039595745503902435,
-      "eval_rewards/rejected": -0.033139873296022415,
-      "eval_runtime": 103.8428,
-      "eval_samples_per_second": 3.303,
-      "eval_steps_per_second": 0.414,
+      "eval_logits/chosen": 98.65145874023438,
+      "eval_logits/rejected": 98.6263427734375,
+      "eval_logps/chosen": -32.57748031616211,
+      "eval_logps/rejected": -36.30455017089844,
+      "eval_loss": 24.828882217407227,
+      "eval_rewards/accuracies": 0.5340532064437866,
+      "eval_rewards/chosen": -0.013430174440145493,
+      "eval_rewards/margins": 0.020371900871396065,
+      "eval_rewards/rejected": -0.03380206972360611,
+      "eval_runtime": 104.0575,
+      "eval_samples_per_second": 3.296,
+      "eval_steps_per_second": 0.413,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 588.0,
+      "grad_norm": 1320.0,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": 82.32527923583984,
-      "logits/rejected": 82.2273178100586,
-      "logps/chosen": -33.264827728271484,
-      "logps/rejected": -34.95970153808594,
-      "loss": 2.4885,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.2723545730113983,
-      "rewards/margins": 0.2617969214916229,
-      "rewards/rejected": 0.010557657107710838,
+      "logits/chosen": 80.6713638305664,
+      "logits/rejected": 80.57633972167969,
+      "logps/chosen": -33.16452407836914,
+      "logps/rejected": -35.397491455078125,
+      "loss": 18.2607,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.07811959832906723,
+      "rewards/margins": 0.11925957351922989,
+      "rewards/rejected": -0.04113996401429176,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 328.0,
+      "grad_norm": 652.0,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": 84.34782409667969,
-      "logits/rejected": 84.42643737792969,
-      "logps/chosen": -31.27395248413086,
-      "logps/rejected": -30.698680877685547,
-      "loss": 2.8757,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.27000054717063904,
-      "rewards/margins": 0.25789040327072144,
-      "rewards/rejected": 0.012110118754208088,
+      "logits/chosen": 82.79257202148438,
+      "logits/rejected": 82.88256072998047,
+      "logps/chosen": -30.9622859954834,
+      "logps/rejected": -31.26416015625,
+      "loss": 16.5714,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.09866676479578018,
+      "rewards/margins": 0.15218719840049744,
+      "rewards/rejected": -0.053520433604717255,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 274.0,
+      "grad_norm": 1144.0,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": 81.63847351074219,
-      "logits/rejected": 81.70894622802734,
-      "logps/chosen": -32.306007385253906,
-      "logps/rejected": -34.033103942871094,
-      "loss": 3.5257,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.19989387691020966,
-      "rewards/margins": 0.20827460289001465,
-      "rewards/rejected": -0.008380698040127754,
+      "logits/chosen": 79.95256042480469,
+      "logits/rejected": 80.00727844238281,
+      "logps/chosen": -32.26613235473633,
+      "logps/rejected": -34.29724884033203,
+      "loss": 21.5779,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.05396091938018799,
+      "rewards/margins": 0.08247107267379761,
+      "rewards/rejected": -0.028510143980383873,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 404.0,
+      "grad_norm": 856.0,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": 84.00340270996094,
-      "logits/rejected": 84.26656341552734,
-      "logps/chosen": -30.941274642944336,
-      "logps/rejected": -31.65194320678711,
-      "loss": 2.3257,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.29871702194213867,
-      "rewards/margins": 0.3466779887676239,
-      "rewards/rejected": -0.047960974276065826,
+      "logits/chosen": 82.29564666748047,
+      "logits/rejected": 82.59175109863281,
+      "logps/chosen": -30.715564727783203,
+      "logps/rejected": -31.892749786376953,
+      "loss": 16.3789,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.0972500815987587,
+      "rewards/margins": 0.13332059979438782,
+      "rewards/rejected": -0.03607049956917763,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 324.0,
+      "grad_norm": 1000.0,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": 82.73006439208984,
-      "logits/rejected": 82.80888366699219,
-      "logps/chosen": -27.05695152282715,
-      "logps/rejected": -29.929241180419922,
-      "loss": 2.6343,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.21557846665382385,
-      "rewards/margins": 0.226979061961174,
-      "rewards/rejected": -0.011400607414543629,
+      "logits/chosen": 80.96902465820312,
+      "logits/rejected": 81.0352554321289,
+      "logps/chosen": -26.926239013671875,
+      "logps/rejected": -30.266109466552734,
+      "loss": 20.1404,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.066965751349926,
+      "rewards/margins": 0.10350307077169418,
+      "rewards/rejected": -0.03653731197118759,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 516.0,
+      "grad_norm": 1064.0,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": 80.12226104736328,
-      "logits/rejected": 80.24698638916016,
-      "logps/chosen": -31.07294273376465,
-      "logps/rejected": -35.9581184387207,
-      "loss": 2.354,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.19522134959697723,
-      "rewards/margins": 0.13781873881816864,
-      "rewards/rejected": 0.057402610778808594,
+      "logits/chosen": 78.25106048583984,
+      "logits/rejected": 78.37794494628906,
+      "logps/chosen": -30.432825088500977,
+      "logps/rejected": -36.55792999267578,
+      "loss": 15.8988,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.11281673610210419,
+      "rewards/margins": 0.158446803689003,
+      "rewards/rejected": -0.0456300750374794,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 344.0,
+      "grad_norm": 708.0,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": 79.53446960449219,
-      "logits/rejected": 79.55906677246094,
-      "logps/chosen": -30.89791488647461,
-      "logps/rejected": -31.486038208007812,
-      "loss": 3.2552,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.3524041175842285,
-      "rewards/margins": 0.3446514904499054,
-      "rewards/rejected": 0.007752613630145788,
+      "logits/chosen": 77.56661224365234,
+      "logits/rejected": 77.58919525146484,
+      "logps/chosen": -30.973047256469727,
+      "logps/rejected": -31.90987777709961,
+      "loss": 18.1036,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.08058776706457138,
+      "rewards/margins": 0.1210336685180664,
+      "rewards/rejected": -0.040445905178785324,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 576.0,
+      "grad_norm": 1056.0,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": 81.95085144042969,
-      "logits/rejected": 81.75164031982422,
-      "logps/chosen": -31.267623901367188,
-      "logps/rejected": -29.534778594970703,
-      "loss": 3.0829,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.2169070690870285,
-      "rewards/margins": 0.16402050852775574,
-      "rewards/rejected": 0.05288654565811157,
+      "logits/chosen": 80.2997817993164,
+      "logits/rejected": 80.07062530517578,
+      "logps/chosen": -31.0832576751709,
+      "logps/rejected": -29.884586334228516,
+      "loss": 22.0775,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.07266353815793991,
+      "rewards/margins": 0.0944225937128067,
+      "rewards/rejected": -0.021759048104286194,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 294.0,
+      "grad_norm": 808.0,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": 82.23690795898438,
-      "logits/rejected": 82.1518325805664,
-      "logps/chosen": -33.551673889160156,
-      "logps/rejected": -31.901935577392578,
-      "loss": 2.7131,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.2210315465927124,
-      "rewards/margins": 0.17505864799022675,
-      "rewards/rejected": 0.04597286507487297,
+      "logits/chosen": 80.42820739746094,
+      "logits/rejected": 80.34810638427734,
+      "logps/chosen": -33.01291275024414,
+      "logps/rejected": -32.72394561767578,
+      "loss": 15.1146,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.10913344472646713,
+      "rewards/margins": 0.17984186112880707,
+      "rewards/rejected": -0.07070842385292053,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 414.0,
+      "grad_norm": 1072.0,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": 77.79912567138672,
-      "logits/rejected": 77.8694839477539,
-      "logps/chosen": -32.68059539794922,
-      "logps/rejected": -28.930795669555664,
-      "loss": 2.7187,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.29004615545272827,
-      "rewards/margins": 0.22155535221099854,
-      "rewards/rejected": 0.06849084049463272,
+      "logits/chosen": 76.04508209228516,
+      "logits/rejected": 76.13191986083984,
+      "logps/chosen": -32.2183952331543,
+      "logps/rejected": -29.19476890563965,
+      "loss": 20.0695,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.11873127520084381,
+      "rewards/margins": 0.12800584733486176,
+      "rewards/rejected": -0.009274585172533989,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": 99.0815658569336,
-      "eval_logits/rejected": 99.07491302490234,
-      "eval_logps/chosen": -32.308189392089844,
-      "eval_logps/rejected": -35.85464859008789,
-      "eval_loss": 2.8584375381469727,
-      "eval_rewards/accuracies": 0.5340532064437866,
-      "eval_rewards/chosen": 0.05399530008435249,
-      "eval_rewards/margins": 0.009244211949408054,
-      "eval_rewards/rejected": 0.044751089066267014,
-      "eval_runtime": 103.9675,
-      "eval_samples_per_second": 3.299,
+      "eval_logits/chosen": 98.68071746826172,
+      "eval_logits/rejected": 98.6502685546875,
+      "eval_logps/chosen": -32.66925811767578,
+      "eval_logps/rejected": -36.280174255371094,
+      "eval_loss": 26.15445899963379,
+      "eval_rewards/accuracies": 0.5307309031486511,
+      "eval_rewards/chosen": -0.022608023136854172,
+      "eval_rewards/margins": 0.008756463415920734,
+      "eval_rewards/rejected": -0.03136448562145233,
+      "eval_runtime": 103.8063,
+      "eval_samples_per_second": 3.304,
       "eval_steps_per_second": 0.414,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 340.0,
+      "grad_norm": 1136.0,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": 84.72640228271484,
-      "logits/rejected": 84.76838684082031,
-      "logps/chosen": -30.20013999938965,
-      "logps/rejected": -32.10908889770508,
-      "loss": 3.4161,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.26535817980766296,
-      "rewards/margins": 0.22908934950828552,
-      "rewards/rejected": 0.03626886010169983,
+      "logits/chosen": 83.16893005371094,
+      "logits/rejected": 83.19877624511719,
+      "logps/chosen": -30.013708114624023,
+      "logps/rejected": -32.592529296875,
+      "loss": 18.2229,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.08498243242502213,
+      "rewards/margins": 0.12425950914621353,
+      "rewards/rejected": -0.039277076721191406,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 466.0,
+      "grad_norm": 596.0,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": 82.32220458984375,
-      "logits/rejected": 82.32237243652344,
-      "logps/chosen": -30.7983341217041,
-      "logps/rejected": -28.84746742248535,
-      "loss": 4.1511,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.3200904428958893,
-      "rewards/margins": 0.2912808060646057,
-      "rewards/rejected": 0.028809573501348495,
+      "logits/chosen": 80.65386199951172,
+      "logits/rejected": 80.6548080444336,
+      "logps/chosen": -30.399967193603516,
+      "logps/rejected": -29.175945281982422,
+      "loss": 16.3256,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.11985927820205688,
+      "rewards/margins": 0.1455042064189911,
+      "rewards/rejected": -0.025644922628998756,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 340.0,
+      "grad_norm": 604.0,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": 79.7364501953125,
-      "logits/rejected": 79.77400207519531,
-      "logps/chosen": -29.450759887695312,
-      "logps/rejected": -32.553932189941406,
-      "loss": 2.6103,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.36073851585388184,
-      "rewards/margins": 0.3283671438694,
-      "rewards/rejected": 0.0323713943362236,
+      "logits/chosen": 77.81417846679688,
+      "logits/rejected": 77.8701171875,
+      "logps/chosen": -29.092737197875977,
+      "logps/rejected": -33.01492691040039,
+      "loss": 15.6194,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.12598693370819092,
+      "rewards/margins": 0.16399319469928741,
+      "rewards/rejected": -0.0380062535405159,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 382.0,
+      "grad_norm": 1384.0,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": 83.68680572509766,
-      "logits/rejected": 83.69349670410156,
-      "logps/chosen": -32.394351959228516,
-      "logps/rejected": -33.24668502807617,
-      "loss": 3.0897,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.32711952924728394,
-      "rewards/margins": 0.2783169746398926,
-      "rewards/rejected": 0.04880258068442345,
+      "logits/chosen": 82.1661376953125,
+      "logits/rejected": 82.20028686523438,
+      "logps/chosen": -32.25991439819336,
+      "logps/rejected": -33.82966995239258,
+      "loss": 18.1168,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.09522315859794617,
+      "rewards/margins": 0.1413211077451706,
+      "rewards/rejected": -0.04609795659780502,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 166.0,
+      "grad_norm": 736.0,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": 82.930419921875,
-      "logits/rejected": 82.9115219116211,
-      "logps/chosen": -32.881385803222656,
-      "logps/rejected": -33.06855010986328,
-      "loss": 2.0393,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.30494141578674316,
-      "rewards/margins": 0.29564136266708374,
-      "rewards/rejected": 0.00930009689182043,
+      "logits/chosen": 81.15172576904297,
+      "logits/rejected": 81.16615295410156,
+      "logps/chosen": -32.44929504394531,
+      "logps/rejected": -33.39020919799805,
+      "loss": 17.9927,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.11944446712732315,
+      "rewards/margins": 0.14928530156612396,
+      "rewards/rejected": -0.029840845614671707,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 179.0,
+      "grad_norm": 832.0,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": 84.22169494628906,
-      "logits/rejected": 84.2500991821289,
-      "logps/chosen": -28.731342315673828,
-      "logps/rejected": -31.580791473388672,
-      "loss": 2.2528,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.3283465504646301,
-      "rewards/margins": 0.30072346329689026,
-      "rewards/rejected": 0.027623046189546585,
+      "logits/chosen": 82.63230895996094,
+      "logits/rejected": 82.66060638427734,
+      "logps/chosen": -28.42384910583496,
+      "logps/rejected": -31.795475006103516,
+      "loss": 17.9341,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.11283614486455917,
+      "rewards/margins": 0.12739871442317963,
+      "rewards/rejected": -0.014562586322426796,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 472.0,
+      "grad_norm": 900.0,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": 83.67330932617188,
-      "logits/rejected": 83.69649505615234,
-      "logps/chosen": -32.3766975402832,
-      "logps/rejected": -35.0659065246582,
-      "loss": 3.2317,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.19978591799736023,
-      "rewards/margins": 0.18239986896514893,
-      "rewards/rejected": 0.017386028543114662,
+      "logits/chosen": 82.10162353515625,
+      "logits/rejected": 82.12476348876953,
+      "logps/chosen": -31.836299896240234,
+      "logps/rejected": -35.486595153808594,
+      "loss": 19.9538,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.10398608446121216,
+      "rewards/margins": 0.14170874655246735,
+      "rewards/rejected": -0.03772266209125519,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 324.0,
+      "grad_norm": 888.0,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": 77.76484680175781,
-      "logits/rejected": 77.6435546875,
-      "logps/chosen": -29.995555877685547,
-      "logps/rejected": -28.028635025024414,
-      "loss": 2.7989,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.23222878575325012,
-      "rewards/margins": 0.17945319414138794,
-      "rewards/rejected": 0.05277556926012039,
+      "logits/chosen": 76.01589965820312,
+      "logits/rejected": 75.89103698730469,
+      "logps/chosen": -29.72897720336914,
+      "logps/rejected": -28.481863021850586,
+      "loss": 18.4584,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.08471504598855972,
+      "rewards/margins": 0.11684386432170868,
+      "rewards/rejected": -0.03212881088256836,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 2.828607769755574,
-      "train_runtime": 2555.3286,
-      "train_samples_per_second": 1.205,
+      "train_loss": 20.413429577319654,
+      "train_runtime": 2557.2601,
+      "train_samples_per_second": 1.204,
       "train_steps_per_second": 0.151
     }
   ],