Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9995
-- Rewards/chosen: 0.0012
-- Rewards/rejected: 0.0007
-- Rewards/accuracies: 0.5141
-- Rewards/margins: 0.0006
-- Logps/rejected: -37.5100
-- Logps/chosen: -34.0224
-- Logits/rejected: -2.2385
-- Logits/chosen: -2.2434
+- Loss: 0.9999
+- Rewards/chosen: 0.0034
+- Rewards/rejected: 0.0033
+- Rewards/accuracies: 0.4904
+- Rewards/margins: 0.0001
+- Logps/rejected: -37.5002
+- Logps/chosen: -34.0177
+- Logits/rejected: -2.2386
+- Logits/chosen: -2.2435
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 1.0003        | 0.26  | 100  | 1.0008          | 0.0011         | 0.0019           | 0.4983             | -0.0008         | -37.4972       | -34.0232     | -2.2384         | -2.2433       |
-| 0.999         | 0.52  | 200  | 0.9983          | 0.0016         | -0.0001          | 0.5328             | 0.0017          | -37.5175       | -34.0187     | -2.2389         | -2.2438       |
-| 0.9972        | 0.78  | 300  | 1.0000          | 0.0017         | 0.0016           | 0.4904             | 0.0000          | -37.5002       | -34.0177     | -2.2386         | -2.2435       |
+| 1.0006        | 0.26  | 100  | 1.0016          | 0.0023         | 0.0039           | 0.4983             | -0.0016         | -37.4972       | -34.0232     | -2.2384         | -2.2433       |
+| 0.9981        | 0.52  | 200  | 0.9966          | 0.0032         | -0.0002          | 0.5328             | 0.0033          | -37.5175       | -34.0187     | -2.2389         | -2.2438       |
+| 0.9944        | 0.78  | 300  | 0.9999          | 0.0034         | 0.0033           | 0.4904             | 0.0001          | -37.5002       | -34.0177     | -2.2386         | -2.2435       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5bdf74864526db2976a0432721d067682ebc5c68521aef8ad7742bc832e47156
+oid sha256:726700f20fcb28f7858cd1aa59e0cb2bd53a0fe4b7d2b662333531d42f0321bf
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2434139251708984,
-    "eval_logits/rejected": -2.2385354042053223,
-    "eval_logps/chosen": -34.0224494934082,
-    "eval_logps/rejected": -37.51002883911133,
-    "eval_loss": 0.9995055198669434,
-    "eval_rewards/accuracies": 0.5141196250915527,
-    "eval_rewards/chosen": 0.0012102578766644,
-    "eval_rewards/margins": 0.0005511684576049447,
-    "eval_rewards/rejected": 0.0006590897683054209,
-    "eval_runtime": 145.6584,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.355,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.9991211135666092,
-    "train_runtime": 3253.9902,
+    "train_loss": 0.998242255619594,
+    "train_runtime": 3253.3829,
     "train_samples": 3079,
     "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.9991211135666092,
-    "train_runtime": 3253.9902,
+    "train_loss": 0.998242255619594,
+    "train_runtime": 3253.3829,
     "train_samples": 3079,
     "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -29,11 +29,11 @@
       "logits/rejected": -1.8703795671463013,
       "logps/chosen": -36.988380432128906,
       "logps/rejected": -33.66728210449219,
-      "loss": 0.9945,
+      "loss": 0.9889,
       "rewards/accuracies": 0.5694444179534912,
-      "rewards/chosen": 0.0018219809280708432,
-      "rewards/margins": 0.00554280448704958,
-      "rewards/rejected": -0.0037208227440714836,
+      "rewards/chosen": 0.0036439618561416864,
+      "rewards/margins": 0.01108560897409916,
+      "rewards/rejected": -0.007441645488142967,
       "step": 10
     },
     {
@@ -43,11 +43,11 @@
       "logits/rejected": -1.999983549118042,
       "logps/chosen": -29.625896453857422,
       "logps/rejected": -29.035802841186523,
-      "loss": 1.0011,
+      "loss": 1.0023,
       "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.0016297042602673173,
-      "rewards/margins": -0.0011343134101480246,
-      "rewards/rejected": 0.00276401755400002,
+      "rewards/chosen": 0.0032594085205346346,
+      "rewards/margins": -0.002268626820296049,
+      "rewards/rejected": 0.00552803510800004,
       "step": 20
     },
     {
@@ -57,11 +57,11 @@
       "logits/rejected": -1.917249321937561,
       "logps/chosen": -31.421478271484375,
       "logps/rejected": -33.2115364074707,
-      "loss": 1.0012,
+      "loss": 1.0025,
       "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.0005385443801060319,
-      "rewards/margins": -0.0012309944722801447,
-      "rewards/rejected": 0.0006924502667970955,
+      "rewards/chosen": -0.0010770887602120638,
+      "rewards/margins": -0.0024619889445602894,
+      "rewards/rejected": 0.001384900533594191,
       "step": 30
     },
     {
@@ -71,11 +71,11 @@
       "logits/rejected": -2.008178949356079,
       "logps/chosen": -32.59435272216797,
       "logps/rejected": -32.49193572998047,
-      "loss": 1.0038,
+      "loss": 1.0077,
       "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.0017543068388476968,
-      "rewards/margins": -0.0038279418367892504,
-      "rewards/rejected": 0.0020736351143568754,
+      "rewards/chosen": -0.0035086136776953936,
+      "rewards/margins": -0.007655883673578501,
+      "rewards/rejected": 0.004147270228713751,
       "step": 40
     },
     {
@@ -85,11 +85,11 @@
       "logits/rejected": -1.8537908792495728,
       "logps/chosen": -33.56566619873047,
       "logps/rejected": -35.423240661621094,
-      "loss": 1.0037,
+      "loss": 1.0075,
       "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.0008188748615793884,
-      "rewards/margins": -0.003729355288669467,
-      "rewards/rejected": 0.0029104803688824177,
+      "rewards/chosen": -0.0016377497231587768,
+      "rewards/margins": -0.007458710577338934,
+      "rewards/rejected": 0.005820960737764835,
       "step": 50
     },
     {
@@ -99,11 +99,11 @@
       "logits/rejected": -1.9468472003936768,
       "logps/chosen": -32.59955596923828,
       "logps/rejected": -33.1828498840332,
-      "loss": 0.9988,
+      "loss": 0.9976,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 6.278457294683903e-05,
-      "rewards/margins": 0.0012094037374481559,
-      "rewards/rejected": -0.0011466194409877062,
+      "rewards/chosen": 0.00012556914589367807,
+      "rewards/margins": 0.0024188074748963118,
+      "rewards/rejected": -0.0022932388819754124,
       "step": 60
     },
     {
@@ -113,11 +113,11 @@
       "logits/rejected": -2.084862232208252,
       "logps/chosen": -33.98878860473633,
       "logps/rejected": -36.574462890625,
-      "loss": 0.9995,
+      "loss": 0.9989,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -3.0292372684925795e-05,
-      "rewards/margins": 0.0005289571708999574,
-      "rewards/rejected": -0.0005592495435848832,
+      "rewards/chosen": -6.058474536985159e-05,
+      "rewards/margins": 0.0010579143417999148,
+      "rewards/rejected": -0.0011184990871697664,
       "step": 70
     },
     {
@@ -127,11 +127,11 @@
       "logits/rejected": -1.9457191228866577,
       "logps/chosen": -34.40068054199219,
       "logps/rejected": -34.5762939453125,
-      "loss": 0.9989,
+      "loss": 0.9978,
       "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.002712726127356291,
-      "rewards/margins": 0.0011012849863618612,
-      "rewards/rejected": 0.0016114413738250732,
+      "rewards/chosen": 0.005425452254712582,
+      "rewards/margins": 0.0022025699727237225,
+      "rewards/rejected": 0.0032228827476501465,
       "step": 80
     },
     {
@@ -141,11 +141,11 @@
       "logits/rejected": -1.9552500247955322,
       "logps/chosen": -32.460357666015625,
       "logps/rejected": -32.354434967041016,
-      "loss": 0.9997,
+      "loss": 0.9995,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0007068099221214652,
-      "rewards/margins": 0.00025928454124368727,
-      "rewards/rejected": 0.0004475254681892693,
+      "rewards/chosen": 0.0014136198442429304,
+      "rewards/margins": 0.0005185690824873745,
+      "rewards/rejected": 0.0008950509363785386,
       "step": 90
     },
     {
@@ -155,11 +155,11 @@
       "logits/rejected": -2.0472888946533203,
       "logps/chosen": -32.23810958862305,
       "logps/rejected": -31.260278701782227,
-      "loss": 1.0003,
+      "loss": 1.0006,
       "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.00027285455144010484,
-      "rewards/margins": -0.00032049461151473224,
-      "rewards/rejected": 0.0005933489883318543,
+      "rewards/chosen": 0.0005457091028802097,
+      "rewards/margins": -0.0006409892230294645,
+      "rewards/rejected": 0.0011866979766637087,
       "step": 100
     },
     {
@@ -168,14 +168,14 @@
       "eval_logits/rejected": -2.238436222076416,
       "eval_logps/chosen": -34.023216247558594,
       "eval_logps/rejected": -37.49723434448242,
-      "eval_loss": 1.0008214712142944,
+      "eval_loss": 1.0016428232192993,
       "eval_rewards/accuracies": 0.49833887815475464,
-      "eval_rewards/chosen": 0.0011338649783283472,
-      "eval_rewards/margins": -0.0008048001327551901,
-      "eval_rewards/rejected": 0.0019386651692911983,
-      "eval_runtime": 146.1271,
-      "eval_samples_per_second": 2.347,
-      "eval_steps_per_second": 0.294,
+      "eval_rewards/chosen": 0.0022677299566566944,
+      "eval_rewards/margins": -0.0016096002655103803,
+      "eval_rewards/rejected": 0.0038773303385823965,
+      "eval_runtime": 145.8128,
+      "eval_samples_per_second": 2.352,
+      "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
@@ -185,11 +185,11 @@
       "logits/rejected": -2.0027499198913574,
       "logps/chosen": -33.2365837097168,
       "logps/rejected": -34.01953125,
-      "loss": 1.0003,
+      "loss": 1.0005,
       "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.0007230007322505116,
-      "rewards/margins": -0.00026112591149285436,
-      "rewards/rejected": 0.0009841264691203833,
+      "rewards/chosen": 0.0014460014645010233,
+      "rewards/margins": -0.0005222518229857087,
+      "rewards/rejected": 0.0019682529382407665,
       "step": 110
     },
     {
@@ -199,11 +199,11 @@
       "logits/rejected": -2.0083022117614746,
       "logps/chosen": -32.457847595214844,
       "logps/rejected": -32.18357467651367,
-      "loss": 0.9991,
+      "loss": 0.9982,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.0012942379107698798,
-      "rewards/margins": 0.0009121716138906777,
-      "rewards/rejected": -0.002206409815698862,
+      "rewards/chosen": -0.0025884758215397596,
+      "rewards/margins": 0.0018243432277813554,
+      "rewards/rejected": -0.004412819631397724,
       "step": 120
     },
     {
@@ -213,11 +213,11 @@
       "logits/rejected": -2.038203001022339,
       "logps/chosen": -30.475027084350586,
       "logps/rejected": -32.046302795410156,
-      "loss": 1.0006,
+      "loss": 1.0012,
       "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 6.882836896693334e-05,
-      "rewards/margins": -0.0006015264661982656,
-      "rewards/rejected": 0.0006703549297526479,
+      "rewards/chosen": 0.00013765673793386668,
+      "rewards/margins": -0.001203052932396531,
+      "rewards/rejected": 0.0013407098595052958,
       "step": 130
     },
     {
@@ -227,11 +227,11 @@
       "logits/rejected": -1.9872560501098633,
       "logps/chosen": -31.407278060913086,
       "logps/rejected": -32.543296813964844,
-      "loss": 0.9979,
+      "loss": 0.9958,
       "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.0012384230503812432,
-      "rewards/margins": 0.0020802102517336607,
-      "rewards/rejected": -0.0008417873759754002,
+      "rewards/chosen": 0.0024768461007624865,
+      "rewards/margins": 0.004160420503467321,
+      "rewards/rejected": -0.0016835747519508004,
       "step": 140
     },
     {
@@ -241,11 +241,11 @@
       "logits/rejected": -1.891632080078125,
       "logps/chosen": -34.20501708984375,
       "logps/rejected": -34.77235412597656,
-      "loss": 0.9997,
+      "loss": 0.9994,
       "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.0012326312717050314,
-      "rewards/margins": 0.00027702696388587356,
-      "rewards/rejected": -0.001509658177383244,
+      "rewards/chosen": -0.002465262543410063,
+      "rewards/margins": 0.0005540539277717471,
+      "rewards/rejected": -0.003019316354766488,
       "step": 150
     },
     {
@@ -255,11 +255,11 @@
       "logits/rejected": -1.939327597618103,
       "logps/chosen": -36.144107818603516,
       "logps/rejected": -32.72822570800781,
-      "loss": 0.9967,
+      "loss": 0.9934,
       "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.0027573786210268736,
-      "rewards/margins": 0.0033046435564756393,
-      "rewards/rejected": -0.0005472644697874784,
+      "rewards/chosen": 0.005514757242053747,
+      "rewards/margins": 0.006609287112951279,
+      "rewards/rejected": -0.001094528939574957,
       "step": 160
     },
     {
@@ -269,11 +269,11 @@
       "logits/rejected": -2.0346803665161133,
       "logps/chosen": -33.771934509277344,
       "logps/rejected": -31.371145248413086,
-      "loss": 0.9955,
+      "loss": 0.991,
       "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.002593145240098238,
-      "rewards/margins": 0.004522555507719517,
-      "rewards/rejected": -0.0019294100347906351,
+      "rewards/chosen": 0.005186290480196476,
+      "rewards/margins": 0.009045111015439034,
+      "rewards/rejected": -0.0038588200695812702,
       "step": 170
     },
     {
@@ -283,11 +283,11 @@
       "logits/rejected": -2.0528526306152344,
       "logps/chosen": -32.524593353271484,
       "logps/rejected": -32.510643005371094,
-      "loss": 0.9954,
+      "loss": 0.9909,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.0020192908123135567,
-      "rewards/margins": 0.004569421522319317,
-      "rewards/rejected": -0.002550130244344473,
+      "rewards/chosen": 0.004038581624627113,
+      "rewards/margins": 0.009138843044638634,
+      "rewards/rejected": -0.005100260488688946,
       "step": 180
     },
     {
@@ -297,11 +297,11 @@
       "logits/rejected": -2.0457024574279785,
       "logps/chosen": -31.492746353149414,
       "logps/rejected": -31.319293975830078,
-      "loss": 1.0003,
+      "loss": 1.0005,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.0003792368806898594,
-      "rewards/margins": -0.0002712804707698524,
-      "rewards/rejected": -0.0001079567227861844,
+      "rewards/chosen": -0.0007584737613797188,
+      "rewards/margins": -0.0005425609415397048,
+      "rewards/rejected": -0.0002159134455723688,
       "step": 190
     },
     {
@@ -311,11 +311,11 @@
       "logits/rejected": -1.9232347011566162,
       "logps/chosen": -31.5926513671875,
       "logps/rejected": -32.78697204589844,
-      "loss": 0.999,
+      "loss": 0.9981,
       "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.001363297225907445,
-      "rewards/margins": 0.0009675474721007049,
-      "rewards/rejected": 0.0003957498411182314,
+      "rewards/chosen": 0.00272659445181489,
+      "rewards/margins": 0.0019350949442014098,
+      "rewards/rejected": 0.0007914996822364628,
       "step": 200
     },
     {
@@ -324,13 +324,13 @@
       "eval_logits/rejected": -2.2389235496520996,
       "eval_logps/chosen": -34.018714904785156,
       "eval_logps/rejected": -37.517478942871094,
-      "eval_loss": 0.9983054399490356,
+      "eval_loss": 0.9966108798980713,
       "eval_rewards/accuracies": 0.5328072905540466,
-      "eval_rewards/chosen": 0.0015839905245229602,
-      "eval_rewards/margins": 0.0016701342537999153,
-      "eval_rewards/rejected": -8.614380931248888e-05,
-      "eval_runtime": 145.691,
-      "eval_samples_per_second": 2.354,
+      "eval_rewards/chosen": 0.0031679810490459204,
+      "eval_rewards/margins": 0.0033402685075998306,
+      "eval_rewards/rejected": -0.00017228761862497777,
+      "eval_runtime": 145.8204,
+      "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
@@ -341,11 +341,11 @@
       "logits/rejected": -2.042539596557617,
       "logps/chosen": -31.948400497436523,
       "logps/rejected": -33.86983871459961,
-      "loss": 0.997,
+      "loss": 0.994,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0022904814686626196,
-      "rewards/margins": 0.0030044156592339277,
-      "rewards/rejected": -0.0007139344816096127,
+      "rewards/chosen": 0.004580962937325239,
+      "rewards/margins": 0.0060088313184678555,
+      "rewards/rejected": -0.0014278689632192254,
       "step": 210
     },
     {
@@ -355,11 +355,11 @@
       "logits/rejected": -1.9399843215942383,
       "logps/chosen": -30.099853515625,
       "logps/rejected": -31.55409812927246,
-      "loss": 0.9992,
+      "loss": 0.9985,
       "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.0011685896897688508,
-      "rewards/margins": 0.000760918774176389,
-      "rewards/rejected": 0.00040767071186564863,
+      "rewards/chosen": 0.0023371793795377016,
+      "rewards/margins": 0.001521837548352778,
+      "rewards/rejected": 0.0008153414237312973,
       "step": 220
     },
     {
@@ -369,11 +369,11 @@
       "logits/rejected": -1.9875112771987915,
       "logps/chosen": -33.387638092041016,
       "logps/rejected": -31.554845809936523,
-      "loss": 0.9965,
+      "loss": 0.9929,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.0029833675362169743,
-      "rewards/margins": 0.003537458134815097,
-      "rewards/rejected": -0.0005540908314287663,
+      "rewards/chosen": 0.0059667350724339485,
+      "rewards/margins": 0.007074916269630194,
+      "rewards/rejected": -0.0011081816628575325,
       "step": 230
     },
     {
@@ -383,11 +383,11 @@
       "logits/rejected": -1.9607274532318115,
       "logps/chosen": -34.158443450927734,
       "logps/rejected": -34.963783264160156,
-      "loss": 0.9992,
+      "loss": 0.9983,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.00046964577632024884,
-      "rewards/margins": 0.0008290981058962643,
-      "rewards/rejected": -0.0012987437658011913,
+      "rewards/chosen": -0.0009392915526404977,
+      "rewards/margins": 0.0016581962117925286,
+      "rewards/rejected": -0.0025974875316023827,
       "step": 240
     },
     {
@@ -397,11 +397,11 @@
       "logits/rejected": -2.0210862159729004,
       "logps/chosen": -32.9254035949707,
       "logps/rejected": -36.251712799072266,
-      "loss": 0.9974,
+      "loss": 0.9947,
       "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -3.328696038806811e-05,
-      "rewards/margins": 0.0026453982573002577,
-      "rewards/rejected": -0.0026786853559315205,
+      "rewards/chosen": -6.657392077613622e-05,
+      "rewards/margins": 0.005290796514600515,
+      "rewards/rejected": -0.005357370711863041,
       "step": 250
     },
     {
@@ -411,11 +411,11 @@
       "logits/rejected": -1.8886839151382446,
       "logps/chosen": -34.194557189941406,
       "logps/rejected": -35.51445770263672,
-      "loss": 0.9996,
+      "loss": 0.9993,
       "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.0006158108590170741,
-      "rewards/margins": 0.0003673938917927444,
-      "rewards/rejected": -0.0009832048090174794,
+      "rewards/chosen": -0.0012316217180341482,
+      "rewards/margins": 0.0007347877835854888,
+      "rewards/rejected": -0.001966409618034959,
       "step": 260
     },
     {
@@ -425,11 +425,11 @@
       "logits/rejected": -1.8734045028686523,
       "logps/chosen": -34.40558624267578,
       "logps/rejected": -31.752349853515625,
-      "loss": 1.0027,
+      "loss": 1.0054,
       "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.0007984804688021541,
-      "rewards/margins": -0.0026967611629515886,
-      "rewards/rejected": 0.0018982805777341127,
+      "rewards/chosen": -0.0015969609376043081,
+      "rewards/margins": -0.005393522325903177,
+      "rewards/rejected": 0.0037965611554682255,
       "step": 270
     },
     {
@@ -439,11 +439,11 @@
       "logits/rejected": -1.9693737030029297,
       "logps/chosen": -35.33230209350586,
       "logps/rejected": -31.845691680908203,
-      "loss": 0.9971,
+      "loss": 0.9942,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.002433922840282321,
-      "rewards/margins": 0.0029038069769740105,
-      "rewards/rejected": -0.0004698836710304022,
+      "rewards/chosen": 0.004867845680564642,
+      "rewards/margins": 0.005807613953948021,
+      "rewards/rejected": -0.0009397673420608044,
       "step": 280
     },
     {
@@ -453,11 +453,11 @@
       "logits/rejected": -2.060606002807617,
       "logps/chosen": -30.907390594482422,
       "logps/rejected": -32.64055252075195,
-      "loss": 0.9995,
+      "loss": 0.9989,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.001673347083851695,
-      "rewards/margins": 0.0005450047319754958,
-      "rewards/rejected": 0.0011283422354608774,
+      "rewards/chosen": 0.00334669416770339,
+      "rewards/margins": 0.0010900094639509916,
+      "rewards/rejected": 0.002256684470921755,
       "step": 290
     },
     {
@@ -467,11 +467,11 @@
       "logits/rejected": -1.9440828561782837,
       "logps/chosen": -32.894561767578125,
       "logps/rejected": -30.812387466430664,
-      "loss": 0.9972,
+      "loss": 0.9944,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.003255209419876337,
-      "rewards/margins": 0.002812772523611784,
-      "rewards/rejected": 0.00044243651791475713,
+      "rewards/chosen": 0.006510418839752674,
+      "rewards/margins": 0.005625545047223568,
+      "rewards/rejected": 0.0008848730358295143,
       "step": 300
     },
     {
@@ -480,13 +480,13 @@
       "eval_logits/rejected": -2.2386035919189453,
       "eval_logps/chosen": -34.017669677734375,
       "eval_logps/rejected": -37.50018310546875,
-      "eval_loss": 0.9999711513519287,
+      "eval_loss": 0.9999422430992126,
       "eval_rewards/accuracies": 0.490448534488678,
-      "eval_rewards/chosen": 0.0016884810756891966,
-      "eval_rewards/margins": 4.470993735594675e-05,
-      "eval_rewards/rejected": 0.001643771305680275,
-      "eval_runtime": 145.8271,
-      "eval_samples_per_second": 2.352,
+      "eval_rewards/chosen": 0.003376962151378393,
+      "eval_rewards/margins": 8.94198747118935e-05,
+      "eval_rewards/rejected": 0.00328754261136055,
+      "eval_runtime": 145.8777,
+      "eval_samples_per_second": 2.351,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
@@ -497,11 +497,11 @@
       "logits/rejected": -1.925451636314392,
       "logps/chosen": -31.603496551513672,
       "logps/rejected": -33.734046936035156,
-      "loss": 1.0008,
+      "loss": 1.0017,
       "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.4918483429937623e-05,
-      "rewards/margins": -0.000825587019789964,
-      "rewards/rejected": 0.0008106685127131641,
+      "rewards/chosen": -2.9836966859875247e-05,
+      "rewards/margins": -0.001651174039579928,
+      "rewards/rejected": 0.0016213370254263282,
       "step": 310
     },
     {
@@ -511,11 +511,11 @@
       "logits/rejected": -1.9679291248321533,
       "logps/chosen": -34.585323333740234,
       "logps/rejected": -33.57084274291992,
-      "loss": 0.9957,
+      "loss": 0.9914,
       "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.00107504369225353,
-      "rewards/margins": 0.0042753927409648895,
-      "rewards/rejected": -0.0032003491651266813,
+      "rewards/chosen": 0.00215008738450706,
+      "rewards/margins": 0.008550785481929779,
+      "rewards/rejected": -0.006400698330253363,
       "step": 320
     },
     {
@@ -525,11 +525,11 @@
       "logits/rejected": -2.014427661895752,
       "logps/chosen": -33.49116516113281,
       "logps/rejected": -32.47978973388672,
-      "loss": 1.0005,
+      "loss": 1.001,
       "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.0005370675935409963,
-      "rewards/margins": -0.0005161463050171733,
-      "rewards/rejected": -2.0921288523823023e-05,
+      "rewards/chosen": -0.0010741351870819926,
+      "rewards/margins": -0.0010322926100343466,
+      "rewards/rejected": -4.1842577047646046e-05,
       "step": 330
     },
     {
@@ -539,11 +539,11 @@
       "logits/rejected": -2.0872654914855957,
       "logps/chosen": -34.18492889404297,
       "logps/rejected": -33.08319854736328,
-      "loss": 1.004,
+      "loss": 1.0081,
       "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.0005535687087103724,
-      "rewards/margins": -0.00403733691200614,
-      "rewards/rejected": 0.003483767854049802,
+      "rewards/chosen": -0.001107137417420745,
+      "rewards/margins": -0.00807467382401228,
+      "rewards/rejected": 0.006967535708099604,
       "step": 340
     },
     {
@@ -553,11 +553,11 @@
       "logits/rejected": -1.9736032485961914,
       "logps/chosen": -33.23271942138672,
       "logps/rejected": -32.4765510559082,
-      "loss": 0.9939,
+      "loss": 0.9878,
       "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.004937793128192425,
-      "rewards/margins": 0.006078074686229229,
-      "rewards/rejected": -0.001140281674452126,
+      "rewards/chosen": 0.00987558625638485,
+      "rewards/margins": 0.012156149372458458,
+      "rewards/rejected": -0.002280563348904252,
       "step": 350
     },
     {
@@ -567,11 +567,11 @@
       "logits/rejected": -1.940913438796997,
       "logps/chosen": -32.22040939331055,
       "logps/rejected": -35.28728103637695,
-      "loss": 1.0017,
+      "loss": 1.0034,
       "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.0015721082454547286,
-      "rewards/margins": -0.001719816355034709,
-      "rewards/rejected": 0.00014770813868381083,
+      "rewards/chosen": -0.003144216490909457,
+      "rewards/margins": -0.003439632710069418,
+      "rewards/rejected": 0.00029541627736762166,
       "step": 360
     },
     {
@@ -581,11 +581,11 @@
       "logits/rejected": -2.0630898475646973,
       "logps/chosen": -33.63695526123047,
       "logps/rejected": -29.226470947265625,
-      "loss": 0.9985,
+      "loss": 0.997,
       "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0015011827927082777,
-      "rewards/margins": 0.0015057541895657778,
-      "rewards/rejected": -4.571699719235767e-06,
+      "rewards/chosen": 0.0030023655854165554,
+      "rewards/margins": 0.0030115083791315556,
+      "rewards/rejected": -9.143399438471533e-06,
       "step": 370
     },
     {
@@ -595,19 +595,19 @@
       "logits/rejected": -1.9310123920440674,
       "logps/chosen": -34.243560791015625,
       "logps/rejected": -30.892742156982422,
-      "loss": 0.9984,
+      "loss": 0.9969,
       "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.0004508168494794518,
-      "rewards/margins": 0.0015608124667778611,
-      "rewards/rejected": -0.0020116292871534824,
+      "rewards/chosen": -0.0009016336989589036,
+      "rewards/margins": 0.0031216249335557222,
+      "rewards/rejected": -0.004023258574306965,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.9991211135666092,
-      "train_runtime": 3253.9902,
+      "train_loss": 0.998242255619594,
+      "train_runtime": 3253.3829,
       "train_samples_per_second": 0.946,
       "train_steps_per_second": 0.118
     }