Model save

README.md

@@ -0,0 +1,80 @@
+---
+license: other
+base_model: deepseek-ai/deepseek-llm-7b-chat
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: ds_chat_sppo_hard_new_iter0_2024-09-17-14.35
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://ml.byteintl.net/experiment/tracking/detail?Id=project_20240915_20321b8f&selectedTrial=run_20240918_0b17b41a)
+# ds_chat_sppo_hard_new_iter0_2024-09-17-14.35
+
+This model is a fine-tuned version of [deepseek-ai/deepseek-llm-7b-chat](https://huggingface.co/deepseek-ai/deepseek-llm-7b-chat) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.4365
+- Rewards/chosen: 0.0466
+- Rewards/rejected: -0.0388
+- Rewards/accuracies: 0.5769
+- Rewards/margins: 0.0854
+- Logps/rejected: -287.4509
+- Logps/chosen: -250.6894
+- Logits/rejected: 1.4344
+- Logits/chosen: 1.2252
+- Debug/policy Chosen Logits: 1.2252
+- Debug/policy Rejected Logits: 1.4344
+- Debug/policy Chosen Logps: -250.6894
+- Debug/policy Rejected Logps: -287.4509
+- Debug/reference Chosen Logps: -255.3497
+- Debug/reference Rejected Logps: -283.5705
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-06
+- train_batch_size: 8
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: constant
+- lr_scheduler_warmup_ratio: 0.1
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 1.0
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Debug/policy Chosen Logits | Debug/policy Rejected Logits | Debug/policy Chosen Logps | Debug/policy Rejected Logps | Debug/reference Chosen Logps | Debug/reference Rejected Logps |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------------------------:|:----------------------------:|:-------------------------:|:---------------------------:|:----------------------------:|:------------------------------:|
+| 0.4429        | 0.6329 | 100  | 0.4365          | 0.0466         | -0.0388          | 0.5769             | 0.0854          | -287.4509      | -250.6894    | 1.4344          | 1.2252        | 1.2252                     | 1.4344                       | -250.6894                 | -287.4509                   | -255.3497                    | -283.5705                      |
+
+
+### Framework versions
+
+- Transformers 4.42.0
+- Pytorch 2.3.0+cu121
+- Datasets 2.14.6
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.44511839181562013,
+    "train_runtime": 1281.3009,
+    "train_samples": 10080,
+    "train_samples_per_second": 7.867,
+    "train_steps_per_second": 0.123
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100000,
+  "do_sample": true,
+  "eos_token_id": 100001,
+  "temperature": 0.7,
+  "top_p": 0.95,
+  "transformers_version": "4.42.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85aee1073c597a2c5b0d96189913c0b83a9de5f776174ccf4d6b47a4aea39eaf
+size 4987202208

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aea544cca944fa009885a7f30e2c26da87863becedfb53bf260211e9950b8a58
+size 4980945440

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c481388c3e80a319807f756ca6ae4976c4eab5f822b080053f55e44aa9005d1c
+size 3852615520

model.safetensors.index.json

@@ -0,0 +1,280 @@
+{
+  "metadata": {
+    "total_size": 13820731392
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.44511839181562013,
+    "train_runtime": 1281.3009,
+    "train_samples": 10080,
+    "train_samples_per_second": 7.867,
+    "train_steps_per_second": 0.123
+}

trainer_state.json

@@ -0,0 +1,736 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 158,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "debug/policy_chosen_logits": 1.5589828491210938,
+      "debug/policy_chosen_logps": -258.5330810546875,
+      "debug/policy_rejected_logits": 1.9977812767028809,
+      "debug/policy_rejected_logps": -304.0617980957031,
+      "debug/reference_chosen_logps": -258.5330810546875,
+      "debug/reference_rejected_logps": -304.0617980957031,
+      "epoch": 0.006329113924050633,
+      "grad_norm": 5.915865288930895,
+      "learning_rate": 1e-06,
+      "logits/chosen": 1.5589828491210938,
+      "logits/rejected": 1.9977812767028809,
+      "logps/chosen": -258.5330810546875,
+      "logps/rejected": -304.0617980957031,
+      "loss": 0.5,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "debug/policy_chosen_logits": 0.8514629602432251,
+      "debug/policy_chosen_logps": -222.75827026367188,
+      "debug/policy_rejected_logits": 1.458482027053833,
+      "debug/policy_rejected_logps": -292.2978210449219,
+      "debug/reference_chosen_logps": -222.56484985351562,
+      "debug/reference_rejected_logps": -288.334716796875,
+      "epoch": 0.03164556962025317,
+      "grad_norm": 6.379094662882782,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.8514629602432251,
+      "logits/rejected": 1.458482027053833,
+      "logps/chosen": -222.75827026367188,
+      "logps/rejected": -292.2978210449219,
+      "loss": 0.4816,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.0019342182204127312,
+      "rewards/margins": 0.03769642859697342,
+      "rewards/rejected": -0.039630644023418427,
+      "step": 5
+    },
+    {
+      "debug/policy_chosen_logits": 1.1418471336364746,
+      "debug/policy_chosen_logps": -261.1085510253906,
+      "debug/policy_rejected_logits": 1.316489338874817,
+      "debug/policy_rejected_logps": -285.4795837402344,
+      "debug/reference_chosen_logps": -260.5736999511719,
+      "debug/reference_rejected_logps": -280.2572937011719,
+      "epoch": 0.06329113924050633,
+      "grad_norm": 12.885197123935471,
+      "learning_rate": 1e-06,
+      "logits/chosen": 1.1418471336364746,
+      "logits/rejected": 1.316489338874817,
+      "logps/chosen": -261.1085510253906,
+      "logps/rejected": -285.4795837402344,
+      "loss": 0.4629,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.005348391830921173,
+      "rewards/margins": 0.04687455669045448,
+      "rewards/rejected": -0.05222295597195625,
+      "step": 10
+    },
+    {
+      "debug/policy_chosen_logits": 1.4202030897140503,
+      "debug/policy_chosen_logps": -305.30096435546875,
+      "debug/policy_rejected_logits": 1.608795404434204,
+      "debug/policy_rejected_logps": -339.3628845214844,
+      "debug/reference_chosen_logps": -305.89739990234375,
+      "debug/reference_rejected_logps": -336.0830078125,
+      "epoch": 0.0949367088607595,
+      "grad_norm": 6.031873391940916,
+      "learning_rate": 1e-06,
+      "logits/chosen": 1.4202030897140503,
+      "logits/rejected": 1.608795404434204,
+      "logps/chosen": -305.30096435546875,
+      "logps/rejected": -339.3628845214844,
+      "loss": 0.462,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.005964324809610844,
+      "rewards/margins": 0.038763098418712616,
+      "rewards/rejected": -0.032798778265714645,
+      "step": 15
+    },
+    {
+      "debug/policy_chosen_logits": 1.2072508335113525,
+      "debug/policy_chosen_logps": -259.9560546875,
+      "debug/policy_rejected_logits": 1.4596980810165405,
+      "debug/policy_rejected_logps": -266.99896240234375,
+      "debug/reference_chosen_logps": -262.2249450683594,
+      "debug/reference_rejected_logps": -262.94488525390625,
+      "epoch": 0.12658227848101267,
+      "grad_norm": 5.929430664241562,
+      "learning_rate": 1e-06,
+      "logits/chosen": 1.2072508335113525,
+      "logits/rejected": 1.4596980810165405,
+      "logps/chosen": -259.9560546875,
+      "logps/rejected": -266.99896240234375,
+      "loss": 0.4568,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.02268880605697632,
+      "rewards/margins": 0.06322960555553436,
+      "rewards/rejected": -0.04054080322384834,
+      "step": 20
+    },
+    {
+      "debug/policy_chosen_logits": 0.9236510992050171,
+      "debug/policy_chosen_logps": -263.56951904296875,
+      "debug/policy_rejected_logits": 1.2153400182724,
+      "debug/policy_rejected_logps": -276.596923828125,
+      "debug/reference_chosen_logps": -264.62982177734375,
+      "debug/reference_rejected_logps": -272.1346130371094,
+      "epoch": 0.15822784810126583,
+      "grad_norm": 6.795022163630081,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.9236510992050171,
+      "logits/rejected": 1.2153400182724,
+      "logps/chosen": -263.56951904296875,
+      "logps/rejected": -276.596923828125,
+      "loss": 0.4609,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.010603101924061775,
+      "rewards/margins": 0.05522637441754341,
+      "rewards/rejected": -0.04462327063083649,
+      "step": 25
+    },
+    {
+      "debug/policy_chosen_logits": 0.8845943212509155,
+      "debug/policy_chosen_logps": -232.0923309326172,
+      "debug/policy_rejected_logits": 1.284155011177063,
+      "debug/policy_rejected_logps": -287.80389404296875,
+      "debug/reference_chosen_logps": -233.78652954101562,
+      "debug/reference_rejected_logps": -284.5167236328125,
+      "epoch": 0.189873417721519,
+      "grad_norm": 6.4445556777608255,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.8845943212509155,
+      "logits/rejected": 1.284155011177063,
+      "logps/chosen": -232.0923309326172,
+      "logps/rejected": -287.80389404296875,
+      "loss": 0.4609,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.016941774636507034,
+      "rewards/margins": 0.04981378838419914,
+      "rewards/rejected": -0.03287201002240181,
+      "step": 30
+    },
+    {
+      "debug/policy_chosen_logits": 1.1807546615600586,
+      "debug/policy_chosen_logps": -263.7032165527344,
+      "debug/policy_rejected_logits": 1.3615357875823975,
+      "debug/policy_rejected_logps": -295.0924377441406,
+      "debug/reference_chosen_logps": -264.52520751953125,
+      "debug/reference_rejected_logps": -289.96612548828125,
+      "epoch": 0.22151898734177214,
+      "grad_norm": 6.39988158389298,
+      "learning_rate": 1e-06,
+      "logits/chosen": 1.1807546615600586,
+      "logits/rejected": 1.3615357875823975,
+      "logps/chosen": -263.7032165527344,
+      "logps/rejected": -295.0924377441406,
+      "loss": 0.4495,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.008219520561397076,
+      "rewards/margins": 0.05948234722018242,
+      "rewards/rejected": -0.05126282572746277,
+      "step": 35
+    },
+    {
+      "debug/policy_chosen_logits": 0.918303370475769,
+      "debug/policy_chosen_logps": -224.531982421875,
+      "debug/policy_rejected_logits": 1.2155705690383911,
+      "debug/policy_rejected_logps": -266.7242431640625,
+      "debug/reference_chosen_logps": -227.6628875732422,
+      "debug/reference_rejected_logps": -259.6141052246094,
+      "epoch": 0.25316455696202533,
+      "grad_norm": 8.66786179216246,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.918303370475769,
+      "logits/rejected": 1.2155705690383911,
+      "logps/chosen": -224.531982421875,
+      "logps/rejected": -266.7242431640625,
+      "loss": 0.4495,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.031309086829423904,
+      "rewards/margins": 0.10241049528121948,
+      "rewards/rejected": -0.07110141217708588,
+      "step": 40
+    },
+    {
+      "debug/policy_chosen_logits": 0.8259471654891968,
+      "debug/policy_chosen_logps": -230.60250854492188,
+      "debug/policy_rejected_logits": 1.2626183032989502,
+      "debug/policy_rejected_logps": -303.4950866699219,
+      "debug/reference_chosen_logps": -230.0920867919922,
+      "debug/reference_rejected_logps": -302.10784912109375,
+      "epoch": 0.2848101265822785,
+      "grad_norm": 6.143825464676947,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.8259471654891968,
+      "logits/rejected": 1.2626183032989502,
+      "logps/chosen": -230.60250854492188,
+      "logps/rejected": -303.4950866699219,
+      "loss": 0.4802,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0051041776314377785,
+      "rewards/margins": 0.008768384344875813,
+      "rewards/rejected": -0.013872561976313591,
+      "step": 45
+    },
+    {
+      "debug/policy_chosen_logits": 0.9409104585647583,
+      "debug/policy_chosen_logps": -241.2617950439453,
+      "debug/policy_rejected_logits": 1.2857184410095215,
+      "debug/policy_rejected_logps": -291.4665222167969,
+      "debug/reference_chosen_logps": -244.69577026367188,
+      "debug/reference_rejected_logps": -284.1947021484375,
+      "epoch": 0.31645569620253167,
+      "grad_norm": 8.46649937885156,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.9409104585647583,
+      "logits/rejected": 1.2857184410095215,
+      "logps/chosen": -241.2617950439453,
+      "logps/rejected": -291.4665222167969,
+      "loss": 0.4411,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.034339673817157745,
+      "rewards/margins": 0.10705772787332535,
+      "rewards/rejected": -0.0727180689573288,
+      "step": 50
+    },
+    {
+      "debug/policy_chosen_logits": 0.8741863369941711,
+      "debug/policy_chosen_logps": -250.87057495117188,
+      "debug/policy_rejected_logits": 1.258837103843689,
+      "debug/policy_rejected_logps": -289.27069091796875,
+      "debug/reference_chosen_logps": -255.7415771484375,
+      "debug/reference_rejected_logps": -283.4430847167969,
+      "epoch": 0.34810126582278483,
+      "grad_norm": 9.716442001601763,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.8741863369941711,
+      "logits/rejected": 1.258837103843689,
+      "logps/chosen": -250.87057495117188,
+      "logps/rejected": -289.27069091796875,
+      "loss": 0.4436,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.048710085451602936,
+      "rewards/margins": 0.10698604583740234,
+      "rewards/rejected": -0.05827596038579941,
+      "step": 55
+    },
+    {
+      "debug/policy_chosen_logits": 0.6640017628669739,
+      "debug/policy_chosen_logps": -269.62237548828125,
+      "debug/policy_rejected_logits": 0.8445190191268921,
+      "debug/policy_rejected_logps": -291.27325439453125,
+      "debug/reference_chosen_logps": -269.4212951660156,
+      "debug/reference_rejected_logps": -285.77349853515625,
+      "epoch": 0.379746835443038,
+      "grad_norm": 7.925495242886814,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.6640017628669739,
+      "logits/rejected": 0.8445190191268921,
+      "logps/chosen": -269.62237548828125,
+      "logps/rejected": -291.27325439453125,
+      "loss": 0.438,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.002010857220739126,
+      "rewards/margins": 0.052986472845077515,
+      "rewards/rejected": -0.05499732494354248,
+      "step": 60
+    },
+    {
+      "debug/policy_chosen_logits": 1.0082881450653076,
+      "debug/policy_chosen_logps": -241.1085662841797,
+      "debug/policy_rejected_logits": 1.5921090841293335,
+      "debug/policy_rejected_logps": -303.08465576171875,
+      "debug/reference_chosen_logps": -245.0981903076172,
+      "debug/reference_rejected_logps": -300.36328125,
+      "epoch": 0.41139240506329117,
+      "grad_norm": 7.096776814684128,
+      "learning_rate": 1e-06,
+      "logits/chosen": 1.0082881450653076,
+      "logits/rejected": 1.5921090841293335,
+      "logps/chosen": -241.1085662841797,
+      "logps/rejected": -303.08465576171875,
+      "loss": 0.4602,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03989603370428085,
+      "rewards/margins": 0.06710983067750931,
+      "rewards/rejected": -0.027213791385293007,
+      "step": 65
+    },
+    {
+      "debug/policy_chosen_logits": 0.7952272295951843,
+      "debug/policy_chosen_logps": -252.08798217773438,
+      "debug/policy_rejected_logits": 1.0696840286254883,
+      "debug/policy_rejected_logps": -287.27301025390625,
+      "debug/reference_chosen_logps": -253.79379272460938,
+      "debug/reference_rejected_logps": -279.5188903808594,
+      "epoch": 0.4430379746835443,
+      "grad_norm": 7.584678181203943,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.7952272295951843,
+      "logits/rejected": 1.0696840286254883,
+      "logps/chosen": -252.08798217773438,
+      "logps/rejected": -287.27301025390625,
+      "loss": 0.4335,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.017058206722140312,
+      "rewards/margins": 0.09459935128688812,
+      "rewards/rejected": -0.07754113525152206,
+      "step": 70
+    },
+    {
+      "debug/policy_chosen_logits": 0.9075101613998413,
+      "debug/policy_chosen_logps": -218.43185424804688,
+      "debug/policy_rejected_logits": 1.0321990251541138,
+      "debug/policy_rejected_logps": -245.87973022460938,
+      "debug/reference_chosen_logps": -221.93466186523438,
+      "debug/reference_rejected_logps": -243.0590057373047,
+      "epoch": 0.47468354430379744,
+      "grad_norm": 6.725884442562555,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.9075101613998413,
+      "logits/rejected": 1.0321990251541138,
+      "logps/chosen": -218.43185424804688,
+      "logps/rejected": -245.87973022460938,
+      "loss": 0.4441,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.03502799943089485,
+      "rewards/margins": 0.06323517113924026,
+      "rewards/rejected": -0.028207167983055115,
+      "step": 75
+    },
+    {
+      "debug/policy_chosen_logits": 0.6510931253433228,
+      "debug/policy_chosen_logps": -218.7671356201172,
+      "debug/policy_rejected_logits": 0.8215225338935852,
+      "debug/policy_rejected_logps": -276.33111572265625,
+      "debug/reference_chosen_logps": -222.28018188476562,
+      "debug/reference_rejected_logps": -267.1961364746094,
+      "epoch": 0.5063291139240507,
+      "grad_norm": 7.155350358859657,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.6510931253433228,
+      "logits/rejected": 0.8215225338935852,
+      "logps/chosen": -218.7671356201172,
+      "logps/rejected": -276.33111572265625,
+      "loss": 0.4348,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.03513062372803688,
+      "rewards/margins": 0.1264806091785431,
+      "rewards/rejected": -0.09134997427463531,
+      "step": 80
+    },
+    {
+      "debug/policy_chosen_logits": 0.9534305334091187,
+      "debug/policy_chosen_logps": -250.000244140625,
+      "debug/policy_rejected_logits": 1.0431879758834839,
+      "debug/policy_rejected_logps": -275.9551086425781,
+      "debug/reference_chosen_logps": -250.7502899169922,
+      "debug/reference_rejected_logps": -268.43548583984375,
+      "epoch": 0.5379746835443038,
+      "grad_norm": 26.837408837144096,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.9534305334091187,
+      "logits/rejected": 1.0431879758834839,
+      "logps/chosen": -250.000244140625,
+      "logps/rejected": -275.9551086425781,
+      "loss": 0.4926,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.007500249892473221,
+      "rewards/margins": 0.08269646763801575,
+      "rewards/rejected": -0.07519622147083282,
+      "step": 85
+    },
+    {
+      "debug/policy_chosen_logits": 1.1253650188446045,
+      "debug/policy_chosen_logps": -240.8356475830078,
+      "debug/policy_rejected_logits": 1.2428481578826904,
+      "debug/policy_rejected_logps": -265.67266845703125,
+      "debug/reference_chosen_logps": -245.643798828125,
+      "debug/reference_rejected_logps": -261.6888122558594,
+      "epoch": 0.569620253164557,
+      "grad_norm": 8.938690009286978,
+      "learning_rate": 1e-06,
+      "logits/chosen": 1.1253650188446045,
+      "logits/rejected": 1.2428481578826904,
+      "logps/chosen": -240.8356475830078,
+      "logps/rejected": -265.67266845703125,
+      "loss": 0.4314,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.04808169603347778,
+      "rewards/margins": 0.08791980892419815,
+      "rewards/rejected": -0.03983811289072037,
+      "step": 90
+    },
+    {
+      "debug/policy_chosen_logits": 0.9913564920425415,
+      "debug/policy_chosen_logps": -247.68453979492188,
+      "debug/policy_rejected_logits": 1.167474389076233,
+      "debug/policy_rejected_logps": -284.51300048828125,
+      "debug/reference_chosen_logps": -250.7725067138672,
+      "debug/reference_rejected_logps": -276.8506774902344,
+      "epoch": 0.6012658227848101,
+      "grad_norm": 7.214786092625251,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.9913564920425415,
+      "logits/rejected": 1.167474389076233,
+      "logps/chosen": -247.68453979492188,
+      "logps/rejected": -284.51300048828125,
+      "loss": 0.4481,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.030879342928528786,
+      "rewards/margins": 0.10750222206115723,
+      "rewards/rejected": -0.07662288844585419,
+      "step": 95
+    },
+    {
+      "debug/policy_chosen_logits": 1.548004388809204,
+      "debug/policy_chosen_logps": -286.9696350097656,
+      "debug/policy_rejected_logits": 1.2569023370742798,
+      "debug/policy_rejected_logps": -255.9474639892578,
+      "debug/reference_chosen_logps": -288.26263427734375,
+      "debug/reference_rejected_logps": -252.56982421875,
+      "epoch": 0.6329113924050633,
+      "grad_norm": 7.098617456221662,
+      "learning_rate": 1e-06,
+      "logits/chosen": 1.548004388809204,
+      "logits/rejected": 1.2569023370742798,
+      "logps/chosen": -286.9696350097656,
+      "logps/rejected": -255.9474639892578,
+      "loss": 0.4429,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.012930279597640038,
+      "rewards/margins": 0.04670674726366997,
+      "rewards/rejected": -0.03377646952867508,
+      "step": 100
+    },
+    {
+      "epoch": 0.6329113924050633,
+      "eval_debug/policy_chosen_logits": 1.2252188920974731,
+      "eval_debug/policy_chosen_logps": -250.68939208984375,
+      "eval_debug/policy_rejected_logits": 1.4343616962432861,
+      "eval_debug/policy_rejected_logps": -287.45086669921875,
+      "eval_debug/reference_chosen_logps": -255.34970092773438,
+      "eval_debug/reference_rejected_logps": -283.57049560546875,
+      "eval_logits/chosen": 1.2252188920974731,
+      "eval_logits/rejected": 1.4343616962432861,
+      "eval_logps/chosen": -250.68939208984375,
+      "eval_logps/rejected": -287.45086669921875,
+      "eval_loss": 0.43653252720832825,
+      "eval_rewards/accuracies": 0.5769230723381042,
+      "eval_rewards/chosen": 0.04660310223698616,
+      "eval_rewards/margins": 0.08540700376033783,
+      "eval_rewards/rejected": -0.03880389407277107,
+      "eval_runtime": 19.8549,
+      "eval_samples_per_second": 20.146,
+      "eval_steps_per_second": 0.655,
+      "step": 100
+    },
+    {
+      "debug/policy_chosen_logits": 1.011919617652893,
+      "debug/policy_chosen_logps": -279.73260498046875,
+      "debug/policy_rejected_logits": 1.211625337600708,
+      "debug/policy_rejected_logps": -298.412109375,
+      "debug/reference_chosen_logps": -281.5310974121094,
+      "debug/reference_rejected_logps": -292.20550537109375,
+      "epoch": 0.6645569620253164,
+      "grad_norm": 6.340425768293679,
+      "learning_rate": 1e-06,
+      "logits/chosen": 1.011919617652893,
+      "logits/rejected": 1.211625337600708,
+      "logps/chosen": -279.73260498046875,
+      "logps/rejected": -298.412109375,
+      "loss": 0.4362,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.017984820529818535,
+      "rewards/margins": 0.08005066215991974,
+      "rewards/rejected": -0.06206584721803665,
+      "step": 105
+    },
+    {
+      "debug/policy_chosen_logits": 1.0565037727355957,
+      "debug/policy_chosen_logps": -251.0978546142578,
+      "debug/policy_rejected_logits": 1.3947855234146118,
+      "debug/policy_rejected_logps": -316.4710998535156,
+      "debug/reference_chosen_logps": -253.4007110595703,
+      "debug/reference_rejected_logps": -309.9458923339844,
+      "epoch": 0.6962025316455697,
+      "grad_norm": 20.34165260676491,
+      "learning_rate": 1e-06,
+      "logits/chosen": 1.0565037727355957,
+      "logits/rejected": 1.3947855234146118,
+      "logps/chosen": -251.0978546142578,
+      "logps/rejected": -316.4710998535156,
+      "loss": 0.4383,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.023028511554002762,
+      "rewards/margins": 0.08828048408031464,
+      "rewards/rejected": -0.06525196880102158,
+      "step": 110
+    },
+    {
+      "debug/policy_chosen_logits": 0.8845629692077637,
+      "debug/policy_chosen_logps": -241.9716339111328,
+      "debug/policy_rejected_logits": 1.229775071144104,
+      "debug/policy_rejected_logps": -321.60186767578125,
+      "debug/reference_chosen_logps": -246.28433227539062,
+      "debug/reference_rejected_logps": -314.5198974609375,
+      "epoch": 0.7278481012658228,
+      "grad_norm": 7.789166803514712,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.8845629692077637,
+      "logits/rejected": 1.229775071144104,
+      "logps/chosen": -241.9716339111328,
+      "logps/rejected": -321.60186767578125,
+      "loss": 0.4426,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.043126728385686874,
+      "rewards/margins": 0.11394629627466202,
+      "rewards/rejected": -0.07081956416368484,
+      "step": 115
+    },
+    {
+      "debug/policy_chosen_logits": 0.6471331119537354,
+      "debug/policy_chosen_logps": -232.4429168701172,
+      "debug/policy_rejected_logits": 0.9131924510002136,
+      "debug/policy_rejected_logps": -279.41290283203125,
+      "debug/reference_chosen_logps": -237.39102172851562,
+      "debug/reference_rejected_logps": -273.61090087890625,
+      "epoch": 0.759493670886076,
+      "grad_norm": 7.468046301754059,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.6471331119537354,
+      "logits/rejected": 0.9131924510002136,
+      "logps/chosen": -232.4429168701172,
+      "logps/rejected": -279.41290283203125,
+      "loss": 0.4131,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.04948071017861366,
+      "rewards/margins": 0.1075005754828453,
+      "rewards/rejected": -0.05801987648010254,
+      "step": 120
+    },
+    {
+      "debug/policy_chosen_logits": 0.9338349103927612,
+      "debug/policy_chosen_logps": -260.35235595703125,
+      "debug/policy_rejected_logits": 1.0534359216690063,
+      "debug/policy_rejected_logps": -297.56683349609375,
+      "debug/reference_chosen_logps": -264.9391174316406,
+      "debug/reference_rejected_logps": -289.8217468261719,
+      "epoch": 0.7911392405063291,
+      "grad_norm": 8.935461685140815,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.9338349103927612,
+      "logits/rejected": 1.0534359216690063,
+      "logps/chosen": -260.35235595703125,
+      "logps/rejected": -297.56683349609375,
+      "loss": 0.4303,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.045867711305618286,
+      "rewards/margins": 0.12331867218017578,
+      "rewards/rejected": -0.0774509608745575,
+      "step": 125
+    },
+    {
+      "debug/policy_chosen_logits": 0.8780291676521301,
+      "debug/policy_chosen_logps": -284.29205322265625,
+      "debug/policy_rejected_logits": 0.8824840784072876,
+      "debug/policy_rejected_logps": -287.76690673828125,
+      "debug/reference_chosen_logps": -286.41943359375,
+      "debug/reference_rejected_logps": -283.56903076171875,
+      "epoch": 0.8227848101265823,
+      "grad_norm": 6.948216331668783,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.8780291676521301,
+      "logits/rejected": 0.8824840784072876,
+      "logps/chosen": -284.29205322265625,
+      "logps/rejected": -287.76690673828125,
+      "loss": 0.4375,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02127380482852459,
+      "rewards/margins": 0.06325232237577438,
+      "rewards/rejected": -0.041978511959314346,
+      "step": 130
+    },
+    {
+      "debug/policy_chosen_logits": 1.165907859802246,
+      "debug/policy_chosen_logps": -255.9198455810547,
+      "debug/policy_rejected_logits": 1.4020473957061768,
+      "debug/policy_rejected_logps": -301.6413879394531,
+      "debug/reference_chosen_logps": -260.84521484375,
+      "debug/reference_rejected_logps": -295.99700927734375,
+      "epoch": 0.8544303797468354,
+      "grad_norm": 6.0797186914906485,
+      "learning_rate": 1e-06,
+      "logits/chosen": 1.165907859802246,
+      "logits/rejected": 1.4020473957061768,
+      "logps/chosen": -255.9198455810547,
+      "logps/rejected": -301.6413879394531,
+      "loss": 0.4418,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.049253594130277634,
+      "rewards/margins": 0.10569741576910019,
+      "rewards/rejected": -0.056443821638822556,
+      "step": 135
+    },
+    {
+      "debug/policy_chosen_logits": 0.9684173464775085,
+      "debug/policy_chosen_logps": -240.7368927001953,
+      "debug/policy_rejected_logits": 1.522164225578308,
+      "debug/policy_rejected_logps": -300.8490295410156,
+      "debug/reference_chosen_logps": -244.41757202148438,
+      "debug/reference_rejected_logps": -289.0794372558594,
+      "epoch": 0.8860759493670886,
+      "grad_norm": 6.850074566718433,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.9684173464775085,
+      "logits/rejected": 1.522164225578308,
+      "logps/chosen": -240.7368927001953,
+      "logps/rejected": -300.8490295410156,
+      "loss": 0.43,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03680698946118355,
+      "rewards/margins": 0.1545029729604721,
+      "rewards/rejected": -0.11769597232341766,
+      "step": 140
+    },
+    {
+      "debug/policy_chosen_logits": 1.095474123954773,
+      "debug/policy_chosen_logps": -281.7500305175781,
+      "debug/policy_rejected_logits": 1.0368950366973877,
+      "debug/policy_rejected_logps": -281.8016052246094,
+      "debug/reference_chosen_logps": -285.4373474121094,
+      "debug/reference_rejected_logps": -278.67181396484375,
+      "epoch": 0.9177215189873418,
+      "grad_norm": 6.330596887372699,
+      "learning_rate": 1e-06,
+      "logits/chosen": 1.095474123954773,
+      "logits/rejected": 1.0368950366973877,
+      "logps/chosen": -281.7500305175781,
+      "logps/rejected": -281.8016052246094,
+      "loss": 0.4243,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.036873430013656616,
+      "rewards/margins": 0.06817178428173065,
+      "rewards/rejected": -0.03129836544394493,
+      "step": 145
+    },
+    {
+      "debug/policy_chosen_logits": 0.9509929418563843,
+      "debug/policy_chosen_logps": -247.018310546875,
+      "debug/policy_rejected_logits": 1.1111629009246826,
+      "debug/policy_rejected_logps": -272.07684326171875,
+      "debug/reference_chosen_logps": -250.40658569335938,
+      "debug/reference_rejected_logps": -265.6427001953125,
+      "epoch": 0.9493670886075949,
+      "grad_norm": 8.073046871358697,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.9509929418563843,
+      "logits/rejected": 1.1111629009246826,
+      "logps/chosen": -247.018310546875,
+      "logps/rejected": -272.07684326171875,
+      "loss": 0.4234,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.03388286381959915,
+      "rewards/margins": 0.09822405129671097,
+      "rewards/rejected": -0.06434118002653122,
+      "step": 150
+    },
+    {
+      "debug/policy_chosen_logits": 0.6622827053070068,
+      "debug/policy_chosen_logps": -237.2403106689453,
+      "debug/policy_rejected_logits": 0.8520939946174622,
+      "debug/policy_rejected_logps": -286.5059509277344,
+      "debug/reference_chosen_logps": -241.94467163085938,
+      "debug/reference_rejected_logps": -278.73272705078125,
+      "epoch": 0.9810126582278481,
+      "grad_norm": 7.904037537559287,
+      "learning_rate": 1e-06,
+      "logits/chosen": 0.6622827053070068,
+      "logits/rejected": 0.8520939946174622,
+      "logps/chosen": -237.2403106689453,
+      "logps/rejected": -286.5059509277344,
+      "loss": 0.423,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.04704369604587555,
+      "rewards/margins": 0.12477605044841766,
+      "rewards/rejected": -0.07773236930370331,
+      "step": 155
+    },
+    {
+      "epoch": 1.0,
+      "step": 158,
+      "total_flos": 0.0,
+      "train_loss": 0.44511839181562013,
+      "train_runtime": 1281.3009,
+      "train_samples_per_second": 7.867,
+      "train_steps_per_second": 0.123
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 158,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}