Training in progress, step 700

README.md

@@ -1,11 +1,13 @@
 ---
 library_name: peft
 tags:
+- alignment-handbook
 - trl
 - dpo
-- alignment-handbook
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
+datasets:
+- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -16,17 +18,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
+This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.7673
-- Rewards/chosen: 0.0035
-- Rewards/rejected: -0.0188
-- Rewards/accuracies: 0.5723
-- Rewards/margins: 0.0223
-- Logps/rejected: -37.5792
-- Logps/chosen: -34.0229
-- Logits/rejected: -2.2269
-- Logits/chosen: -2.2318
+- Loss: 0.6926
+- Rewards/chosen: -0.0279
+- Rewards/rejected: -0.0401
+- Rewards/accuracies: 0.5212
+- Rewards/margins: 0.0122
+- Logps/rejected: -37.5969
+- Logps/chosen: -34.0903
+- Logits/rejected: -2.2228
+- Logits/chosen: -2.2276
 
 ## Model description
 
@@ -55,27 +57,15 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 4
+- num_epochs: 1
 
 ### Training results
 
-| Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
-|:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 2.59          | 0.26  | 100  | -2.2353       | -2.2305         | -34.0250     | -37.4993       | 2.8958          | 0.4730             | 0.0029         | -0.0024         | 0.0052           |
-| 2.2795        | 0.52  | 200  | -2.2339       | -2.2290         | -34.0545     | -37.5651       | 2.8012          | 0.5278             | -0.0060        | 0.0085          | -0.0145          |
-| 1.7902        | 0.78  | 300  | -2.2294       | -2.2245         | -34.0446     | -37.5724       | 2.7585          | 0.5748             | -0.0030        | 0.0137          | -0.0167          |
-| 1.1358        | 1.04  | 400  | 2.7145        | 0.0043          | -0.0169      | 0.5573         | 0.0212          | -37.5730           | -34.0204       | -2.2172         | -2.2221          |
-| 0.842         | 1.3   | 500  | 2.7488        | -0.0094         | -0.0331      | 0.5453         | 0.0237          | -37.6269           | -34.0658       | -2.2052         | -2.2100          |
-| 1.3074        | 1.56  | 600  | 2.7022        | -0.0190         | -0.0450      | 0.5718         | 0.0260          | -37.6668           | -34.0979       | -2.2232         | -2.2281          |
-| 1.2824        | 1.82  | 700  | 2.7508        | -0.0224         | -0.0454      | 0.5627         | 0.0230          | -37.6679           | -34.1092       | -2.2297         | -2.2346          |
-| 0.5946        | 2.08  | 800  | 2.8085        | -0.0013         | -0.0203      | 0.5573         | 0.0190          | -37.5843           | -34.0389       | -2.2324         | -2.2372          |
-| 0.3535        | 2.34  | 900  | 2.7256        | 0.0037          | -0.0203      | 0.5540         | 0.0240          | -37.5843           | -34.0222       | -2.2286         | -2.2334          |
-| 0.4205        | 2.6   | 1000 | 2.8066        | -0.0001         | -0.0187      | 0.5307         | 0.0186          | -37.5790           | -34.0348       | -2.2279         | -2.2328          |
-| 0.5121        | 2.86  | 1100 | 2.7946        | 0.0022          | -0.0186      | 0.5727         | 0.0208          | -37.5786           | -34.0271       | -2.2261         | -2.2310          |
-| 0.4011        | 3.12  | 1200 | 2.7715        | -0.0010         | -0.0223      | 0.5486         | 0.0213          | -37.5909           | -34.0378       | -2.2272         | -2.2321          |
-| 0.2672        | 3.38  | 1300 | 2.8251        | 0.0002          | -0.0172      | 0.5631         | 0.0174          | -37.5741           | -34.0340       | -2.2271         | -2.2320          |
-| 0.2484        | 3.64  | 1400 | 2.7913        | 0.0003          | -0.0200      | 0.5664         | 0.0203          | -37.5833           | -34.0337       | -2.2267         | -2.2316          |
-| 0.3309        | 3.9   | 1500 | 2.7673        | 0.0035          | -0.0188      | 0.5723         | 0.0223          | -37.5792           | -34.0229       | -2.2269         | -2.2318          |
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.6634        | 0.26  | 100  | 0.6931          | 0.0028         | -0.0041          | 0.5216             | 0.0069          | -37.5249       | -34.0290     | -2.2276         | -2.2324       |
+| 0.6329        | 0.52  | 200  | 0.6905          | -0.0127        | -0.0275          | 0.5274             | 0.0148          | -37.5716       | -34.0600     | -2.2255         | -2.2304       |
+| 0.5742        | 0.78  | 300  | 0.6920          | -0.0273        | -0.0406          | 0.5278             | 0.0133          | -37.5978       | -34.0891     | -2.2231         | -2.2279       |
 
 
 ### Framework versions

adapter_config.json

@@ -6,6 +6,7 @@
   "fan_in_fan_out": false,
   "inference_mode": true,
   "init_lora_weights": true,
+  "layer_replication": null,
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
@@ -19,14 +20,15 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
     "up_proj",
-    "o_proj",
     "v_proj",
+    "k_proj",
     "down_proj",
+    "gate_proj",
     "q_proj",
-    "gate_proj"
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM",
+  "use_dora": false,
   "use_rslora": false
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5be67a20fb9ec9e173f0c34b37f8efff061c1aa14ab63ad38093ea5ef1197b48
+oid sha256:32090b9ea7c556302f66e3f74d879385ba738e6848b5078c51bc8aa8f1f41db9
 size 176183216

all_results.json

@@ -1,21 +1,21 @@
 {
-    "epoch": 4.0,
-    "eval_logits/chosen": -2.2295007705688477,
-    "eval_logits/rejected": -2.2246556282043457,
-    "eval_logps/chosen": -34.054466247558594,
-    "eval_logps/rejected": -37.57353973388672,
-    "eval_loss": 2.790188789367676,
-    "eval_rewards/accuracies": 0.560215950012207,
-    "eval_rewards/chosen": -0.0059735761024057865,
-    "eval_rewards/margins": 0.011101600714027882,
-    "eval_rewards/rejected": -0.017075177282094955,
-    "eval_runtime": 145.7539,
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.22763729095459,
+    "eval_logits/rejected": -2.222792387008667,
+    "eval_logps/chosen": -34.090328216552734,
+    "eval_logps/rejected": -37.59687423706055,
+    "eval_loss": 0.6925650835037231,
+    "eval_rewards/accuracies": 0.5211793780326843,
+    "eval_rewards/chosen": -0.02788795717060566,
+    "eval_rewards/margins": 0.01224011555314064,
+    "eval_rewards/rejected": -0.04012807086110115,
+    "eval_runtime": 145.6508,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.353,
+    "eval_samples_per_second": 2.355,
     "eval_steps_per_second": 0.295,
-    "train_loss": 0.5763176552661053,
-    "train_runtime": 10799.143,
+    "train_loss": 0.64145151051608,
+    "train_runtime": 3249.8987,
     "train_samples": 3079,
-    "train_samples_per_second": 1.14,
-    "train_steps_per_second": 0.143
+    "train_samples_per_second": 0.947,
+    "train_steps_per_second": 0.118
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2295007705688477,
-    "eval_logits/rejected": -2.2246556282043457,
-    "eval_logps/chosen": -34.054466247558594,
-    "eval_logps/rejected": -37.57353973388672,
-    "eval_loss": 2.790188789367676,
-    "eval_rewards/accuracies": 0.560215950012207,
-    "eval_rewards/chosen": -0.0059735761024057865,
-    "eval_rewards/margins": 0.011101600714027882,
-    "eval_rewards/rejected": -0.017075177282094955,
-    "eval_runtime": 145.7539,
+    "eval_logits/chosen": -2.22763729095459,
+    "eval_logits/rejected": -2.222792387008667,
+    "eval_logps/chosen": -34.090328216552734,
+    "eval_logps/rejected": -37.59687423706055,
+    "eval_loss": 0.6925650835037231,
+    "eval_rewards/accuracies": 0.5211793780326843,
+    "eval_rewards/chosen": -0.02788795717060566,
+    "eval_rewards/margins": 0.01224011555314064,
+    "eval_rewards/rejected": -0.04012807086110115,
+    "eval_runtime": 145.6508,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.353,
+    "eval_samples_per_second": 2.355,
     "eval_steps_per_second": 0.295
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 4.0,
-    "train_loss": 0.5763176552661053,
-    "train_runtime": 10799.143,
+    "epoch": 1.0,
+    "train_loss": 0.64145151051608,
+    "train_runtime": 3249.8987,
     "train_samples": 3079,
-    "train_samples_per_second": 1.14,
-    "train_steps_per_second": 0.143
+    "train_samples_per_second": 0.947,
+    "train_steps_per_second": 0.118
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc3e8127207cbcb9bf61a05f5206c35cfece8824544474dcbb3c0ba691e632f8
-size 4984
+oid sha256:cd0bea9e6fb12ae12008f5d23f461ff25ae26d3baa3aa0559f7b01d5652bebcf
+size 5176