Model save

README.md

@@ -1,13 +1,11 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9991
-- Rewards/chosen: 0.0161
-- Rewards/rejected: 0.0150
-- Rewards/accuracies: 0.5365
-- Rewards/margins: 0.0010
-- Logps/rejected: -37.4999
-- Logps/chosen: -34.0167
-- Logits/rejected: -2.2391
-- Logits/chosen: -2.2439
+- Logits/chosen: -2.2443
+- Logits/rejected: -2.2394
+- Logps/chosen: -34.0255
+- Logps/rejected: -37.5088
+- Loss: 0.6931
+- Rewards/accuracies: 0.4958
+- Rewards/chosen: 0.0001
+- Rewards/margins: 0.0000
+- Rewards/rejected: 0.0001
 
 ## Model description
 
@@ -47,7 +45,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-07
+- learning_rate: 5e-06
 - train_batch_size: 4
 - eval_batch_size: 8
 - seed: 42
@@ -61,11 +59,11 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.9823        | 0.26  | 100  | 0.9982          | 0.0116         | 0.0097           | 0.5104             | 0.0019          | -37.5058       | -34.0216     | -2.2391         | -2.2440       |
-| 0.9696        | 0.52  | 200  | 1.0014          | 0.0116         | 0.0128           | 0.4693             | -0.0012         | -37.5024       | -34.0217     | -2.2390         | -2.2439       |
-| 0.9581        | 0.78  | 300  | 1.0120          | 0.0035         | 0.0155           | 0.4726             | -0.0120         | -37.4994       | -34.0306     | -2.2388         | -2.2437       |
+| Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
+|:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
+| 0.6932        | 0.26  | 100  | -2.2442       | -2.2393         | -34.0293     | -37.5038       | 0.6932          | 0.4605             | 0.0001         | -0.0001         | 0.0001           |
+| 0.693         | 0.52  | 200  | -2.2439       | -2.2390         | -34.0271     | -37.5001       | 0.6932          | 0.4992             | 0.0001         | -0.0001         | 0.0002           |
+| 0.693         | 0.78  | 300  | -2.2443       | -2.2394         | -34.0255     | -37.5088       | 0.6931          | 0.4958             | 0.0001         | 0.0000          | 0.0001           |
 
 
 ### Framework versions

adapter_config.json

@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
-    "up_proj",
-    "o_proj",
     "v_proj",
-    "k_proj",
+    "q_proj",
     "gate_proj",
-    "q_proj"
+    "k_proj",
+    "o_proj",
+    "down_proj",
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_rslora": false

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a1e7f79a4fe0164d309e418b64bfbaabd64069413317820121adc83c26c731cb
+oid sha256:81a12530c8d521b3699b0a5fe263f2b51bc1b52ef24c389c68c92ba8e08e4da3
 size 176183216

all_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2439348697662354,
-    "eval_logits/rejected": -2.2390594482421875,
-    "eval_logps/chosen": -34.016693115234375,
-    "eval_logps/rejected": -37.499900817871094,
-    "eval_loss": 0.9991074204444885,
-    "eval_rewards/accuracies": 0.5365448594093323,
-    "eval_rewards/chosen": 0.01607290655374527,
-    "eval_rewards/margins": 0.0010251295752823353,
-    "eval_rewards/rejected": 0.015047776512801647,
-    "eval_runtime": 145.7535,
+    "eval_logits/chosen": -2.2436320781707764,
+    "eval_logits/rejected": -2.2387471199035645,
+    "eval_logps/chosen": -34.02302932739258,
+    "eval_logps/rejected": -37.51007080078125,
+    "eval_loss": 0.6931225061416626,
+    "eval_rewards/accuracies": 0.5328072905540466,
+    "eval_rewards/chosen": 0.00011527122114785016,
+    "eval_rewards/margins": 4.97526052640751e-05,
+    "eval_rewards/rejected": 6.551862315973267e-05,
+    "eval_runtime": 145.6938,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.353,
+    "eval_samples_per_second": 2.354,
     "eval_steps_per_second": 0.295,
-    "train_loss": 0.9887851405453373,
-    "train_runtime": 3254.6365,
+    "train_loss": 0.1530164842481737,
+    "train_runtime": 627.5816,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
-    "train_steps_per_second": 0.118
+    "train_samples_per_second": 4.906,
+    "train_steps_per_second": 0.613
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2439348697662354,
-    "eval_logits/rejected": -2.2390594482421875,
-    "eval_logps/chosen": -34.016693115234375,
-    "eval_logps/rejected": -37.499900817871094,
-    "eval_loss": 0.9991074204444885,
-    "eval_rewards/accuracies": 0.5365448594093323,
-    "eval_rewards/chosen": 0.01607290655374527,
-    "eval_rewards/margins": 0.0010251295752823353,
-    "eval_rewards/rejected": 0.015047776512801647,
-    "eval_runtime": 145.7535,
+    "eval_logits/chosen": -2.2436320781707764,
+    "eval_logits/rejected": -2.2387471199035645,
+    "eval_logps/chosen": -34.02302932739258,
+    "eval_logps/rejected": -37.51007080078125,
+    "eval_loss": 0.6931225061416626,
+    "eval_rewards/accuracies": 0.5328072905540466,
+    "eval_rewards/chosen": 0.00011527122114785016,
+    "eval_rewards/margins": 4.97526052640751e-05,
+    "eval_rewards/rejected": 6.551862315973267e-05,
+    "eval_runtime": 145.6938,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.353,
+    "eval_samples_per_second": 2.354,
     "eval_steps_per_second": 0.295
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.9887851405453373,
-    "train_runtime": 3254.6365,
+    "train_loss": 0.1530164842481737,
+    "train_runtime": 627.5816,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
-    "train_steps_per_second": 0.118
+    "train_samples_per_second": 4.906,
+    "train_steps_per_second": 0.613
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 1.0,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,591 +25,591 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.8661272525787354,
-      "logits/rejected": -1.8704373836517334,
-      "logps/chosen": -36.993247985839844,
-      "logps/rejected": -33.67000961303711,
-      "loss": 0.952,
-      "rewards/accuracies": 0.5138888955116272,
-      "rewards/chosen": 0.012012440711259842,
-      "rewards/margins": 0.047955650836229324,
-      "rewards/rejected": -0.035943206399679184,
+      "logits/chosen": -1.86637282371521,
+      "logits/rejected": -1.8706889152526855,
+      "logps/chosen": -36.99605941772461,
+      "logps/rejected": -33.66212844848633,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": 0.00010540737275732681,
+      "rewards/margins": 0.00042594841215759516,
+      "rewards/rejected": -0.0003205410612281412,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.9975776672363281,
-      "logits/rejected": -2.000213146209717,
-      "logps/chosen": -29.647293090820312,
-      "logps/rejected": -29.056921005249023,
-      "loss": 1.0105,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.00459072832018137,
-      "rewards/margins": -0.010462107136845589,
-      "rewards/rejected": 0.005871377885341644,
+      "logits/chosen": -1.9977174997329712,
+      "logits/rejected": -2.0003600120544434,
+      "logps/chosen": -29.638538360595703,
+      "logps/rejected": -29.072032928466797,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 3.6563815228873864e-05,
+      "rewards/margins": 0.00012243764649610966,
+      "rewards/rejected": -8.587385673308745e-05,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.920013189315796,
-      "logits/rejected": -1.917337417602539,
-      "logps/chosen": -31.4055233001709,
-      "logps/rejected": -33.22685623168945,
-      "loss": 0.9829,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.009516814723610878,
-      "rewards/margins": 0.01707320474088192,
-      "rewards/rejected": -0.007556394673883915,
+      "logits/chosen": -1.91976797580719,
+      "logits/rejected": -1.917059302330017,
+      "logps/chosen": -31.427032470703125,
+      "logps/rejected": -33.21014404296875,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.00010937436309177428,
+      "rewards/margins": -0.000192504987353459,
+      "rewards/rejected": 8.313067519338802e-05,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.0167124271392822,
-      "logits/rejected": -2.0079712867736816,
-      "logps/chosen": -32.590545654296875,
-      "logps/rejected": -32.499671936035156,
-      "loss": 1.0241,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.01236567273736,
-      "rewards/margins": -0.024063030257821083,
-      "rewards/rejected": 0.011697360314428806,
+      "logits/chosen": -2.0164756774902344,
+      "logits/rejected": -2.0077216625213623,
+      "logps/chosen": -32.58618927001953,
+      "logps/rejected": -32.47720718383789,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -9.37797303777188e-05,
+      "rewards/margins": -0.00044843609794043005,
+      "rewards/rejected": 0.0003546563093550503,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8641105890274048,
-      "logits/rejected": -1.8533456325531006,
-      "logps/chosen": -33.56208038330078,
-      "logps/rejected": -35.437400817871094,
-      "loss": 1.0176,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.004139312542974949,
-      "rewards/margins": -0.017586929723620415,
-      "rewards/rejected": 0.013447612524032593,
+      "logits/chosen": -1.864516019821167,
+      "logits/rejected": -1.8537566661834717,
+      "logps/chosen": -33.572547912597656,
+      "logps/rejected": -35.45746994018555,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.00015066245396155864,
+      "rewards/margins": -9.940571908373386e-05,
+      "rewards/rejected": -5.125668758410029e-05,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9452192783355713,
-      "logits/rejected": -1.9471514225006104,
-      "logps/chosen": -32.587867736816406,
-      "logps/rejected": -33.20810317993164,
-      "loss": 0.9559,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.01108462829142809,
-      "rewards/margins": 0.04413585737347603,
-      "rewards/rejected": -0.033051230013370514,
+      "logits/chosen": -1.9447959661483765,
+      "logits/rejected": -1.946738839149475,
+      "logps/chosen": -32.576927185058594,
+      "logps/rejected": -33.18446350097656,
+      "loss": 0.693,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0002326248650206253,
+      "rewards/margins": 0.0003634841996245086,
+      "rewards/rejected": -0.00013085929094813764,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.0798697471618652,
-      "logits/rejected": -2.084836483001709,
-      "logps/chosen": -33.99953079223633,
-      "logps/rejected": -36.59130859375,
-      "loss": 0.9897,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.009940117597579956,
-      "rewards/margins": 0.010251833125948906,
-      "rewards/rejected": -0.020191945135593414,
+      "logits/chosen": -2.079601764678955,
+      "logits/rejected": -2.0845787525177,
+      "logps/chosen": -33.98583221435547,
+      "logps/rejected": -36.567405700683594,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 2.6532879928709008e-05,
+      "rewards/margins": 1.1868818546645343e-05,
+      "rewards/rejected": 1.4664057744084857e-05,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9422235488891602,
-      "logits/rejected": -1.9453849792480469,
-      "logps/chosen": -34.411861419677734,
-      "logps/rejected": -34.599830627441406,
-      "loss": 0.979,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.014353242702782154,
-      "rewards/margins": 0.02102992869913578,
-      "rewards/rejected": -0.006676685996353626,
+      "logits/chosen": -1.9421491622924805,
+      "logits/rejected": -1.9453084468841553,
+      "logps/chosen": -34.42374038696289,
+      "logps/rejected": -34.57770538330078,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 4.0700066165300086e-05,
+      "rewards/margins": -0.0001062863739207387,
+      "rewards/rejected": 0.0001469864509999752,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.950791358947754,
-      "logits/rejected": -1.9553003311157227,
-      "logps/chosen": -32.45661163330078,
-      "logps/rejected": -32.35947799682617,
-      "loss": 0.9898,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.009730497375130653,
-      "rewards/margins": 0.010243801400065422,
-      "rewards/rejected": -0.0005133040249347687,
+      "logits/chosen": -1.9507558345794678,
+      "logits/rejected": -1.9552711248397827,
+      "logps/chosen": -32.4448356628418,
+      "logps/rejected": -32.35200881958008,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.0002258731983602047,
+      "rewards/margins": 0.00015688061830587685,
+      "rewards/rejected": 6.899260188220069e-05,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.0489749908447266,
-      "logits/rejected": -2.046968460083008,
-      "logps/chosen": -32.233360290527344,
-      "logps/rejected": -31.278451919555664,
-      "loss": 0.9823,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.006729811429977417,
-      "rewards/margins": 0.01774778962135315,
-      "rewards/rejected": -0.011017980054020882,
+      "logits/chosen": -2.0490055084228516,
+      "logits/rejected": -2.0470049381256104,
+      "logps/chosen": -32.24321365356445,
+      "logps/rejected": -31.265995025634766,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -2.38026914303191e-05,
+      "rewards/margins": -2.5972036382881925e-05,
+      "rewards/rejected": 2.169329718526569e-06,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.243974447250366,
-      "eval_logits/rejected": -2.239091634750366,
-      "eval_logps/chosen": -34.021610260009766,
-      "eval_logps/rejected": -37.50578689575195,
-      "eval_loss": 0.9981797933578491,
-      "eval_rewards/accuracies": 0.5103820562362671,
-      "eval_rewards/chosen": 0.011647163890302181,
-      "eval_rewards/margins": 0.0018998426385223866,
-      "eval_rewards/rejected": 0.009747318923473358,
-      "eval_runtime": 146.0735,
-      "eval_samples_per_second": 2.348,
+      "eval_logits/chosen": -2.244168996810913,
+      "eval_logits/rejected": -2.23928165435791,
+      "eval_logps/chosen": -34.029258728027344,
+      "eval_logps/rejected": -37.50376510620117,
+      "eval_loss": 0.6931847929954529,
+      "eval_rewards/accuracies": 0.46054819226264954,
+      "eval_rewards/chosen": 5.296626841300167e-05,
+      "eval_rewards/margins": -7.559578079963103e-05,
+      "eval_rewards/rejected": 0.00012856203829869628,
+      "eval_runtime": 146.0358,
+      "eval_samples_per_second": 2.349,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.0054612159729004,
-      "logits/rejected": -2.003042221069336,
-      "logps/chosen": -33.2550048828125,
-      "logps/rejected": -34.02202224731445,
-      "loss": 1.0167,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.010068392381072044,
-      "rewards/margins": -0.016683781519532204,
-      "rewards/rejected": 0.006615391466766596,
+      "logits/chosen": -2.005403518676758,
+      "logits/rejected": -2.0029940605163574,
+      "logps/chosen": -33.23992156982422,
+      "logps/rejected": -34.01720428466797,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": 3.894489418598823e-05,
+      "rewards/margins": -8.27050898806192e-05,
+      "rewards/rejected": 0.00012164998042862862,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.016511917114258,
-      "logits/rejected": -2.008146047592163,
-      "logps/chosen": -32.45395278930664,
-      "logps/rejected": -32.17681121826172,
-      "loss": 0.9944,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.008141009137034416,
-      "rewards/margins": 0.00563241308555007,
-      "rewards/rejected": -0.013773423619568348,
+      "logits/chosen": -2.0166852474212646,
+      "logits/rejected": -2.008302688598633,
+      "logps/chosen": -32.434364318847656,
+      "logps/rejected": -32.170448303222656,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.00010542555537540466,
+      "rewards/margins": 0.00019486629753373563,
+      "rewards/rejected": -8.94407567102462e-05,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.0463106632232666,
-      "logits/rejected": -2.0382590293884277,
-      "logps/chosen": -30.476207733154297,
-      "logps/rejected": -32.020484924316406,
-      "loss": 1.0297,
-      "rewards/accuracies": 0.3499999940395355,
-      "rewards/chosen": -0.00044600124238058925,
-      "rewards/margins": -0.029714178293943405,
-      "rewards/rejected": 0.02926817536354065,
+      "logits/chosen": -2.0464212894439697,
+      "logits/rejected": -2.0383687019348145,
+      "logps/chosen": -30.498855590820312,
+      "logps/rejected": -32.038177490234375,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.0002313825534656644,
+      "rewards/margins": -0.0003796928212977946,
+      "rewards/rejected": 0.00014831028238404542,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.9767357110977173,
-      "logits/rejected": -1.986999273300171,
-      "logps/chosen": -31.381155014038086,
-      "logps/rejected": -32.54108428955078,
-      "loss": 0.9598,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.034655503928661346,
-      "rewards/margins": 0.04024083539843559,
-      "rewards/rejected": -0.005585335195064545,
+      "logits/chosen": -1.9772886037826538,
+      "logits/rejected": -1.987566590309143,
+      "logps/chosen": -31.40262794494629,
+      "logps/rejected": -32.533485412597656,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.00017036919598467648,
+      "rewards/margins": 0.00015645996609237045,
+      "rewards/rejected": 1.3909203516959678e-05,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8906068801879883,
-      "logits/rejected": -1.8916860818862915,
-      "logps/chosen": -34.20075225830078,
-      "logps/rejected": -34.7618522644043,
-      "loss": 1.0031,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.007252261973917484,
-      "rewards/margins": -0.0031175222247838974,
-      "rewards/rejected": -0.004134741611778736,
+      "logits/chosen": -1.8902900218963623,
+      "logits/rejected": -1.8913724422454834,
+      "logps/chosen": -34.174896240234375,
+      "logps/rejected": -34.764488220214844,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.00017796576139517128,
+      "rewards/margins": 0.00025028776144608855,
+      "rewards/rejected": -7.232203643070534e-05,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.942251443862915,
-      "logits/rejected": -1.9387743473052979,
-      "logps/chosen": -36.145790100097656,
-      "logps/rejected": -32.73196029663086,
-      "loss": 0.9684,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.023301539942622185,
-      "rewards/margins": 0.031586963683366776,
-      "rewards/rejected": -0.008285422809422016,
+      "logits/chosen": -1.9422016143798828,
+      "logits/rejected": -1.938711404800415,
+      "logps/chosen": -36.14549255371094,
+      "logps/rejected": -32.743194580078125,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.00026183511363342404,
+      "rewards/margins": 0.0004662507271859795,
+      "rewards/rejected": -0.00020441561355255544,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.041952610015869,
-      "logits/rejected": -2.0345559120178223,
-      "logps/chosen": -33.81856155395508,
-      "logps/rejected": -31.359323501586914,
-      "loss": 1.0119,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.018632039427757263,
-      "rewards/margins": -0.011905943043529987,
-      "rewards/rejected": -0.006726093590259552,
+      "logits/chosen": -2.041975498199463,
+      "logits/rejected": -2.034566879272461,
+      "logps/chosen": -33.78548049926758,
+      "logps/rejected": -31.344890594482422,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0001237928809132427,
+      "rewards/margins": 5.422577305580489e-05,
+      "rewards/rejected": 6.956708239158615e-05,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0473170280456543,
-      "logits/rejected": -2.0525729656219482,
-      "logps/chosen": -32.52764129638672,
-      "logps/rejected": -32.50192642211914,
-      "loss": 0.9695,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.015429094433784485,
-      "rewards/margins": 0.030534446239471436,
-      "rewards/rejected": -0.015105349011719227,
+      "logits/chosen": -2.047132968902588,
+      "logits/rejected": -2.0523972511291504,
+      "logps/chosen": -32.526214599609375,
+      "logps/rejected": -32.523216247558594,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.00018572784028947353,
+      "rewards/margins": 0.000566480855923146,
+      "rewards/rejected": -0.0003807530156336725,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.048530340194702,
-      "logits/rejected": -2.045767307281494,
-      "logps/chosen": -31.485219955444336,
-      "logps/rejected": -31.34262466430664,
-      "loss": 0.9747,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.003360496135428548,
-      "rewards/margins": 0.02533094584941864,
-      "rewards/rejected": -0.02197045087814331,
+      "logits/chosen": -2.048360824584961,
+      "logits/rejected": -2.0455949306488037,
+      "logps/chosen": -31.48598289489746,
+      "logps/rejected": -31.33578109741211,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 2.9682874810532667e-05,
+      "rewards/margins": 0.00020535592921078205,
+      "rewards/rejected": -0.00017567307804711163,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.918745756149292,
-      "logits/rejected": -1.9234097003936768,
-      "logps/chosen": -31.576425552368164,
-      "logps/rejected": -32.79480743408203,
-      "loss": 0.9696,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.026870254427194595,
-      "rewards/margins": 0.03035963699221611,
-      "rewards/rejected": -0.0034893869888037443,
+      "logits/chosen": -1.918714165687561,
+      "logits/rejected": -1.9233949184417725,
+      "logps/chosen": -31.575008392333984,
+      "logps/rejected": -32.78480911254883,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0003127219679299742,
+      "rewards/margins": 0.00025148707209154963,
+      "rewards/rejected": 6.123496132204309e-05,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.243912696838379,
-      "eval_logits/rejected": -2.2390317916870117,
-      "eval_logps/chosen": -34.021690368652344,
-      "eval_logps/rejected": -37.502418518066406,
-      "eval_loss": 1.001371145248413,
-      "eval_rewards/accuracies": 0.4692690968513489,
-      "eval_rewards/chosen": 0.011576101183891296,
-      "eval_rewards/margins": -0.0012021720176562667,
-      "eval_rewards/rejected": 0.012778270989656448,
-      "eval_runtime": 145.8493,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.2438974380493164,
+      "eval_logits/rejected": -2.2390170097351074,
+      "eval_logps/chosen": -34.02705764770508,
+      "eval_logps/rejected": -37.500091552734375,
+      "eval_loss": 0.6931942701339722,
+      "eval_rewards/accuracies": 0.4991694688796997,
+      "eval_rewards/chosen": 7.496162288589403e-05,
+      "eval_rewards/margins": -9.032993693836033e-05,
+      "eval_rewards/rejected": 0.0001652915816521272,
+      "eval_runtime": 145.6901,
+      "eval_samples_per_second": 2.354,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0318515300750732,
-      "logits/rejected": -2.0425400733947754,
-      "logps/chosen": -31.96026039123535,
-      "logps/rejected": -33.8691291809082,
-      "loss": 0.9843,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.009936656802892685,
-      "rewards/margins": 0.015721982344985008,
-      "rewards/rejected": -0.005785329267382622,
+      "logits/chosen": -2.0324437618255615,
+      "logits/rejected": -2.0431275367736816,
+      "logps/chosen": -31.958581924438477,
+      "logps/rejected": -33.874977111816406,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.00012724331463687122,
+      "rewards/margins": 0.00025004256167449057,
+      "rewards/rejected": -0.00012279924703761935,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9256188869476318,
-      "logits/rejected": -1.9404585361480713,
-      "logps/chosen": -30.080001831054688,
-      "logps/rejected": -31.53633689880371,
-      "loss": 0.9913,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.028384273871779442,
-      "rewards/margins": 0.00873211957514286,
-      "rewards/rejected": 0.01965215802192688,
+      "logits/chosen": -1.9262778759002686,
+      "logits/rejected": -1.9411571025848389,
+      "logps/chosen": -30.077739715576172,
+      "logps/rejected": -31.5812931060791,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0003380063863005489,
+      "rewards/margins": 0.0005691683036275208,
+      "rewards/rejected": -0.00023116180091165006,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9830671548843384,
-      "logits/rejected": -1.9870269298553467,
-      "logps/chosen": -33.386474609375,
-      "logps/rejected": -31.55719566345215,
-      "loss": 0.965,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.02789575420320034,
-      "rewards/margins": 0.03499894589185715,
-      "rewards/rejected": -0.007103185169398785,
+      "logits/chosen": -1.984165906906128,
+      "logits/rejected": -1.9881302118301392,
+      "logps/chosen": -33.39875030517578,
+      "logps/rejected": -31.574642181396484,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.00018722013919614255,
+      "rewards/margins": 0.0004406363586895168,
+      "rewards/rejected": -0.00025341621949337423,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9833042621612549,
-      "logits/rejected": -1.96132493019104,
-      "logps/chosen": -34.13536834716797,
-      "logps/rejected": -34.952049255371094,
-      "loss": 0.9823,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.016542484983801842,
-      "rewards/margins": 0.01767394319176674,
-      "rewards/rejected": -0.0011314575094729662,
+      "logits/chosen": -1.9830011129379272,
+      "logits/rejected": -1.9610159397125244,
+      "logps/chosen": -34.161964416503906,
+      "logps/rejected": -34.9656867980957,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -8.213297405745834e-05,
+      "rewards/margins": 6.679827492916957e-05,
+      "rewards/rejected": -0.0001489312417106703,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.0247886180877686,
-      "logits/rejected": -2.0214812755584717,
-      "logps/chosen": -32.913856506347656,
-      "logps/rejected": -36.23151397705078,
-      "loss": 0.984,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.010092089883983135,
-      "rewards/margins": 0.016022875905036926,
-      "rewards/rejected": -0.0059307836927473545,
+      "logits/chosen": -2.024148464202881,
+      "logits/rejected": -2.020843982696533,
+      "logps/chosen": -32.924686431884766,
+      "logps/rejected": -36.229591369628906,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 3.8046332520025317e-06,
+      "rewards/margins": 5.0471979193389416e-05,
+      "rewards/rejected": -4.666733366320841e-05,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8914467096328735,
-      "logits/rejected": -1.8890043497085571,
-      "logps/chosen": -34.18690872192383,
-      "logps/rejected": -35.524742126464844,
-      "loss": 0.9806,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.0013398710871115327,
-      "rewards/margins": 0.019445054233074188,
-      "rewards/rejected": -0.01810518093407154,
+      "logits/chosen": -1.8910725116729736,
+      "logits/rejected": -1.8886131048202515,
+      "logps/chosen": -34.17741012573242,
+      "logps/rejected": -35.50103759765625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.00010990710143232718,
+      "rewards/margins": 7.404615462291986e-05,
+      "rewards/rejected": 3.586096136132255e-05,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8761389255523682,
-      "logits/rejected": -1.8735923767089844,
-      "logps/chosen": -34.396785736083984,
-      "logps/rejected": -31.74139976501465,
-      "loss": 1.0262,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": 0.000734941684640944,
-      "rewards/margins": -0.026202131062746048,
-      "rewards/rejected": 0.02693706750869751,
+      "logits/chosen": -1.8763153553009033,
+      "logits/rejected": -1.873784065246582,
+      "logps/chosen": -34.39301300048828,
+      "logps/rejected": -31.753448486328125,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 4.5871554902987555e-05,
+      "rewards/margins": -0.00013292724906932563,
+      "rewards/rejected": 0.00017879880033433437,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9801757335662842,
-      "logits/rejected": -1.9695546627044678,
-      "logps/chosen": -35.34246063232422,
-      "logps/rejected": -31.823230743408203,
-      "loss": 1.0032,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.012763315811753273,
-      "rewards/margins": -0.003222113475203514,
-      "rewards/rejected": 0.015985429286956787,
+      "logits/chosen": -1.9792474508285522,
+      "logits/rejected": -1.9685996770858765,
+      "logps/chosen": -35.32111740112305,
+      "logps/rejected": -31.84047508239746,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.00035524697159416974,
+      "rewards/margins": 0.0003500745224300772,
+      "rewards/rejected": 5.172446435608435e-06,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.075316905975342,
-      "logits/rejected": -2.0602917671203613,
-      "logps/chosen": -30.897375106811523,
-      "logps/rejected": -32.64093780517578,
-      "loss": 0.9857,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.024072837084531784,
-      "rewards/margins": 0.014264687895774841,
-      "rewards/rejected": 0.009808152914047241,
+      "logits/chosen": -2.0755350589752197,
+      "logits/rejected": -2.0605063438415527,
+      "logps/chosen": -30.92108154296875,
+      "logps/rejected": -32.65219497680664,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 3.043042670469731e-05,
+      "rewards/margins": 3.4023898479063064e-05,
+      "rewards/rejected": -3.593444034777349e-06,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.946758508682251,
-      "logits/rejected": -1.9442226886749268,
-      "logps/chosen": -32.89502716064453,
-      "logps/rejected": -30.831274032592773,
-      "loss": 0.9581,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0288736280053854,
-      "rewards/margins": 0.04189068451523781,
-      "rewards/rejected": -0.013017055578529835,
+      "logits/chosen": -1.9464104175567627,
+      "logits/rejected": -1.9438741207122803,
+      "logps/chosen": -32.899200439453125,
+      "logps/rejected": -30.818578720092773,
+      "loss": 0.693,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.00027910774224437773,
+      "rewards/margins": 0.00029683284810744226,
+      "rewards/rejected": -1.772511677700095e-05,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2437047958374023,
-      "eval_logits/rejected": -2.2388339042663574,
-      "eval_logps/chosen": -34.03062438964844,
-      "eval_logps/rejected": -37.499412536621094,
-      "eval_loss": 1.0120128393173218,
-      "eval_rewards/accuracies": 0.472591370344162,
-      "eval_rewards/chosen": 0.003532935632392764,
-      "eval_rewards/margins": -0.011953731998801231,
-      "eval_rewards/rejected": 0.015486669726669788,
-      "eval_runtime": 145.7744,
-      "eval_samples_per_second": 2.353,
+      "eval_logits/chosen": -2.244286060333252,
+      "eval_logits/rejected": -2.2394051551818848,
+      "eval_logps/chosen": -34.025508880615234,
+      "eval_logps/rejected": -37.50877380371094,
+      "eval_loss": 0.6931424140930176,
+      "eval_rewards/accuracies": 0.4958471655845642,
+      "eval_rewards/chosen": 9.041713929036632e-05,
+      "eval_rewards/margins": 1.1897234799107537e-05,
+      "eval_rewards/rejected": 7.851990085327998e-05,
+      "eval_runtime": 145.6886,
+      "eval_samples_per_second": 2.354,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.928391695022583,
-      "logits/rejected": -1.925132393836975,
-      "logps/chosen": -31.573719024658203,
-      "logps/rejected": -33.733333587646484,
-      "loss": 0.9813,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.026669049635529518,
-      "rewards/margins": 0.01873381994664669,
-      "rewards/rejected": 0.007935228757560253,
+      "logits/chosen": -1.9285697937011719,
+      "logits/rejected": -1.9253038167953491,
+      "logps/chosen": -31.567317962646484,
+      "logps/rejected": -33.728389739990234,
+      "loss": 0.693,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.00036028382601216435,
+      "rewards/margins": 0.00022268423344939947,
+      "rewards/rejected": 0.00013759959256276488,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9808565378189087,
-      "logits/rejected": -1.9685465097427368,
-      "logps/chosen": -34.569026947021484,
-      "logps/rejected": -33.5385856628418,
-      "loss": 0.9759,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.02433854714035988,
-      "rewards/margins": 0.024108227342367172,
-      "rewards/rejected": 0.00023032091849017888,
+      "logits/chosen": -1.9806677103042603,
+      "logits/rejected": -1.9683544635772705,
+      "logps/chosen": -34.57938766479492,
+      "logps/rejected": -33.576698303222656,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0001669025223236531,
+      "rewards/margins": 0.0005454975180327892,
+      "rewards/rejected": -0.0003785950248129666,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.0160164833068848,
-      "logits/rejected": -2.0145697593688965,
-      "logps/chosen": -33.470794677734375,
-      "logps/rejected": -32.465301513671875,
-      "loss": 0.9994,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.013503548689186573,
-      "rewards/margins": 0.0006495133275166154,
-      "rewards/rejected": 0.012854036875069141,
+      "logits/chosen": -2.0156922340393066,
+      "logits/rejected": -2.014228105545044,
+      "logps/chosen": -33.48464584350586,
+      "logps/rejected": -32.47287368774414,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 1.1468652701296378e-05,
+      "rewards/margins": -5.562617297982797e-05,
+      "rewards/rejected": 6.709482113365084e-05,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.1030516624450684,
-      "logits/rejected": -2.087252140045166,
-      "logps/chosen": -34.154754638671875,
-      "logps/rejected": -33.087013244628906,
-      "loss": 1.0057,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.02217467688024044,
-      "rewards/margins": -0.005742859095335007,
-      "rewards/rejected": 0.027917543426156044,
+      "logits/chosen": -2.1031441688537598,
+      "logits/rejected": -2.0873398780822754,
+      "logps/chosen": -34.18089294433594,
+      "logps/rejected": -33.10489273071289,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -1.5041228834888898e-05,
+      "rewards/margins": -0.000146475198562257,
+      "rewards/rejected": 0.0001314339751843363,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9748103618621826,
-      "logits/rejected": -1.9738750457763672,
-      "logps/chosen": -33.23518371582031,
-      "logps/rejected": -32.46066665649414,
-      "loss": 0.9618,
+      "logits/chosen": -1.9747949838638306,
+      "logits/rejected": -1.9738445281982422,
+      "logps/chosen": -33.250999450683594,
+      "logps/rejected": -32.463138580322266,
+      "loss": 0.693,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.04222174361348152,
-      "rewards/margins": 0.03818847984075546,
-      "rewards/rejected": 0.004033264704048634,
+      "rewards/chosen": 0.0003109553945250809,
+      "rewards/margins": 0.00029085634741932154,
+      "rewards/rejected": 2.009904164879117e-05,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9303611516952515,
-      "logits/rejected": -1.940708875656128,
-      "logps/chosen": -32.200523376464844,
-      "logps/rejected": -35.27861785888672,
-      "loss": 1.0054,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": 0.0037514485884457827,
-      "rewards/margins": -0.005373493768274784,
-      "rewards/rejected": 0.009124943986535072,
+      "logits/chosen": -1.9307000637054443,
+      "logits/rejected": -1.9410593509674072,
+      "logps/chosen": -32.20253372192383,
+      "logps/rejected": -35.28581619262695,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 2.157145536330063e-05,
+      "rewards/margins": -7.86231794336345e-06,
+      "rewards/rejected": 2.9433791496558115e-05,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.069342613220215,
-      "logits/rejected": -2.062788486480713,
-      "logps/chosen": -33.65803527832031,
-      "logps/rejected": -29.199758529663086,
-      "loss": 1.0295,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.005459528882056475,
-      "rewards/margins": -0.029461050406098366,
-      "rewards/rejected": 0.02400151826441288,
+      "logits/chosen": -2.069913148880005,
+      "logits/rejected": -2.0633533000946045,
+      "logps/chosen": -33.65105438232422,
+      "logps/rejected": -29.224279403686523,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 9.127380508289207e-06,
+      "rewards/margins": -1.2333152881183196e-05,
+      "rewards/rejected": 2.1460538846440613e-05,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.9290788173675537,
-      "logits/rejected": -1.931239366531372,
-      "logps/chosen": -34.229915618896484,
-      "logps/rejected": -30.90093994140625,
-      "loss": 0.9663,
+      "logits/chosen": -1.9291718006134033,
+      "logits/rejected": -1.9313350915908813,
+      "logps/chosen": -34.24486541748047,
+      "logps/rejected": -30.897415161132812,
+      "loss": 0.6931,
       "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.00822196900844574,
-      "rewards/margins": 0.03370029479265213,
-      "rewards/rejected": -0.02547832392156124,
+      "rewards/chosen": -5.8152021665591747e-05,
+      "rewards/margins": 0.00018974675913341343,
+      "rewards/rejected": -0.000247898802626878,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.9887851405453373,
-      "train_runtime": 3254.6365,
-      "train_samples_per_second": 0.946,
-      "train_steps_per_second": 0.118
+      "train_loss": 0.1530164842481737,
+      "train_runtime": 627.5816,
+      "train_samples_per_second": 4.906,
+      "train_steps_per_second": 0.613
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac212fdd90cd2859bb63daeb4c4f5edfeadc70808cdfe6c0556e838c2837607e
+oid sha256:9f3046af59c1723a4e46073c3055c5eefec531e1771c8d3f6ef4da98df5fcbb1
 size 4984