End of training

README.md

@@ -0,0 +1,79 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: mistral-orpo-mix-b0.05-l1024-pl512-lr5e-7-cosine
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mistral-orpo-mix-b0.05-l1024-pl512-lr5e-7-cosine
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.8648
+- Rewards/chosen: -0.0405
+- Rewards/rejected: -0.0502
+- Rewards/accuracies: 0.6458
+- Rewards/margins: 0.0097
+- Logps/rejected: -1.0036
+- Logps/chosen: -0.8096
+- Logits/rejected: -2.9146
+- Logits/chosen: -2.9040
+- Nll Loss: 0.8392
+- Log Odds Ratio: -0.6215
+- Log Odds Chosen: 0.3802
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Nll Loss | Log Odds Ratio | Log Odds Chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:--------:|:--------------:|:---------------:|
+| 0.9159        | 1.0   | 105  | 0.8794          | -0.0421        | -0.0499          | 0.6302             | 0.0078          | -0.9975        | -0.8413      | -2.8931         | -2.8875       | 0.8561   | -0.6429        | 0.3024          |
+| 0.8397        | 2.0   | 211  | 0.8612          | -0.0404        | -0.0495          | 0.6458             | 0.0092          | -0.9902        | -0.8071      | -2.8882         | -2.8794       | 0.8366   | -0.6257        | 0.3555          |
+| 0.7808        | 2.99  | 315  | 0.8648          | -0.0405        | -0.0502          | 0.6458             | 0.0097          | -1.0036        | -0.8096      | -2.9146         | -2.9040       | 0.8392   | -0.6215        | 0.3802          |
+
+
+### Framework versions
+
+- Transformers 4.39.0
+- Pytorch 2.1.1+cu121
+- Datasets 2.16.1
+- Tokenizers 0.15.2

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<|im_end|>": 32001,
+  "<|im_start|>": 32000
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.0",
+  "use_cache": true,
+  "vocab_size": 32002
+}

final_checkpoint/added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<|im_end|>": 32001,
+  "<|im_start|>": 32000
+}

final_checkpoint/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.0",
+  "use_cache": true,
+  "vocab_size": 32002
+}

final_checkpoint/special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

final_checkpoint/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

final_checkpoint/tokenizer_config.json

@@ -0,0 +1,62 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 32000,
+  "eos_token_id": 32001,
+  "pad_token_id": 32001,
+  "transformers_version": "4.39.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18d457cf6a5ca1f95cf59dce109e7594691ba8db02734c1d5e2c4e49c408a688
+size 4943178720

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6895049aa92d025dabc3002926fded24394c3b63868cf6d2a282274d7120acfd
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc45bdcd8a319de2b7b1abb2f44f8f051930de6bc9f13322a17faa3f9a3a8a9d
+size 4540532728

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483496960
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Mar26_04-45-16_alvarobartt-slimy-moth-1-0/events.out.tfevents.1711442779.alvarobartt-slimy-moth-1-0.445.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfa3e265f93cbeec1cc808b906966ce46404c01a77b2872489cfb05bb19359d8
+size 34575

special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,62 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "<|im_end|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

trainer_state.json

@@ -0,0 +1,645 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.985781990521327,
+  "eval_steps": 500,
+  "global_step": 315,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.09,
+      "grad_norm": 51.45086593171862,
+      "learning_rate": 5e-08,
+      "log_odds_chosen": 0.23327143490314484,
+      "log_odds_ratio": -0.7173982262611389,
+      "logits/chosen": -2.9019644260406494,
+      "logits/rejected": -2.911267042160034,
+      "logps/chosen": -1.2478530406951904,
+      "logps/rejected": -1.4603310823440552,
+      "loss": 1.1528,
+      "nll_loss": 1.110867977142334,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.06239265203475952,
+      "rewards/margins": 0.010623900219798088,
+      "rewards/rejected": -0.07301654666662216,
+      "step": 10
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 43.38597658388667,
+      "learning_rate": 1e-07,
+      "log_odds_chosen": 0.025622714310884476,
+      "log_odds_ratio": -0.803138256072998,
+      "logits/chosen": -2.957723617553711,
+      "logits/rejected": -2.9606709480285645,
+      "logps/chosen": -1.2851994037628174,
+      "logps/rejected": -1.2747653722763062,
+      "loss": 1.109,
+      "nll_loss": 1.0504928827285767,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.06425996124744415,
+      "rewards/margins": -0.0005216972786001861,
+      "rewards/rejected": -0.06373827159404755,
+      "step": 20
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 18.07276834258672,
+      "learning_rate": 1.5e-07,
+      "log_odds_chosen": 0.33495259284973145,
+      "log_odds_ratio": -0.6577204465866089,
+      "logits/chosen": -2.936415195465088,
+      "logits/rejected": -2.939385175704956,
+      "logps/chosen": -1.0753445625305176,
+      "logps/rejected": -1.3320289850234985,
+      "loss": 1.0256,
+      "nll_loss": 0.9573599696159363,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.053767234086990356,
+      "rewards/margins": 0.012834218330681324,
+      "rewards/rejected": -0.06660144031047821,
+      "step": 30
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 14.138346458866787,
+      "learning_rate": 2e-07,
+      "log_odds_chosen": 0.06818895787000656,
+      "log_odds_ratio": -0.7556289434432983,
+      "logits/chosen": -2.979851484298706,
+      "logits/rejected": -2.9792511463165283,
+      "logps/chosen": -1.0226948261260986,
+      "logps/rejected": -1.0867162942886353,
+      "loss": 0.9841,
+      "nll_loss": 0.9785293340682983,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.05113474652171135,
+      "rewards/margins": 0.003201070474460721,
+      "rewards/rejected": -0.0543358139693737,
+      "step": 40
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 19.142475208524697,
+      "learning_rate": 2.5e-07,
+      "log_odds_chosen": 0.2577429711818695,
+      "log_odds_ratio": -0.6650774478912354,
+      "logits/chosen": -3.0103330612182617,
+      "logits/rejected": -3.0095608234405518,
+      "logps/chosen": -0.9924309849739075,
+      "logps/rejected": -1.1665266752243042,
+      "loss": 0.9833,
+      "nll_loss": 0.965359091758728,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.04962155967950821,
+      "rewards/margins": 0.008704781532287598,
+      "rewards/rejected": -0.05832633376121521,
+      "step": 50
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 9.00103625601085,
+      "learning_rate": 3e-07,
+      "log_odds_chosen": 0.2601451277732849,
+      "log_odds_ratio": -0.6529109477996826,
+      "logits/chosen": -3.0405919551849365,
+      "logits/rejected": -3.0266852378845215,
+      "logps/chosen": -0.8860586881637573,
+      "logps/rejected": -1.0516645908355713,
+      "loss": 0.967,
+      "nll_loss": 0.8791006207466125,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.044302936643362045,
+      "rewards/margins": 0.008280294016003609,
+      "rewards/rejected": -0.0525832362473011,
+      "step": 60
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 8.208565662624949,
+      "learning_rate": 3.5e-07,
+      "log_odds_chosen": 0.24647900462150574,
+      "log_odds_ratio": -0.6602240800857544,
+      "logits/chosen": -2.9646213054656982,
+      "logits/rejected": -2.9692559242248535,
+      "logps/chosen": -0.8851203918457031,
+      "logps/rejected": -1.04824697971344,
+      "loss": 0.9521,
+      "nll_loss": 0.8758776783943176,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.044256024062633514,
+      "rewards/margins": 0.008156323805451393,
+      "rewards/rejected": -0.05241234228014946,
+      "step": 70
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 9.949021333520767,
+      "learning_rate": 4e-07,
+      "log_odds_chosen": 0.3642485439777374,
+      "log_odds_ratio": -0.6296299695968628,
+      "logits/chosen": -2.9997572898864746,
+      "logits/rejected": -2.9932103157043457,
+      "logps/chosen": -0.8595079183578491,
+      "logps/rejected": -1.0957872867584229,
+      "loss": 0.8992,
+      "nll_loss": 0.8687052726745605,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.042975399643182755,
+      "rewards/margins": 0.011813966557383537,
+      "rewards/rejected": -0.05478937178850174,
+      "step": 80
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 11.826771567950217,
+      "learning_rate": 4.5e-07,
+      "log_odds_chosen": 0.13775679469108582,
+      "log_odds_ratio": -0.7174943685531616,
+      "logits/chosen": -2.9466898441314697,
+      "logits/rejected": -2.9638590812683105,
+      "logps/chosen": -0.8956987261772156,
+      "logps/rejected": -0.9907738566398621,
+      "loss": 0.903,
+      "nll_loss": 0.8976006507873535,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.04478494077920914,
+      "rewards/margins": 0.0047537581995129585,
+      "rewards/rejected": -0.04953869432210922,
+      "step": 90
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 7.898257944088662,
+      "learning_rate": 5e-07,
+      "log_odds_chosen": 0.19350430369377136,
+      "log_odds_ratio": -0.6868109703063965,
+      "logits/chosen": -2.932610034942627,
+      "logits/rejected": -2.9326844215393066,
+      "logps/chosen": -0.8639435768127441,
+      "logps/rejected": -0.9883400797843933,
+      "loss": 0.9159,
+      "nll_loss": 0.8739891052246094,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.04319717735052109,
+      "rewards/margins": 0.0062198275700211525,
+      "rewards/rejected": -0.049417007714509964,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_log_odds_chosen": 0.3023655116558075,
+      "eval_log_odds_ratio": -0.6428868770599365,
+      "eval_logits/chosen": -2.887465238571167,
+      "eval_logits/rejected": -2.8930952548980713,
+      "eval_logps/chosen": -0.8412830829620361,
+      "eval_logps/rejected": -0.9974649548530579,
+      "eval_loss": 0.8794215321540833,
+      "eval_nll_loss": 0.8561177253723145,
+      "eval_rewards/accuracies": 0.6302083134651184,
+      "eval_rewards/chosen": -0.042064156383275986,
+      "eval_rewards/margins": 0.007809097412973642,
+      "eval_rewards/rejected": -0.04987325146794319,
+      "eval_runtime": 32.7887,
+      "eval_samples_per_second": 22.874,
+      "eval_steps_per_second": 0.732,
+      "step": 105
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 7.858898751829365,
+      "learning_rate": 4.973358420187775e-07,
+      "log_odds_chosen": 0.2784392237663269,
+      "log_odds_ratio": -0.6759603023529053,
+      "logits/chosen": -2.898139715194702,
+      "logits/rejected": -2.888554096221924,
+      "logps/chosen": -0.8769194483757019,
+      "logps/rejected": -1.0477098226547241,
+      "loss": 0.9267,
+      "nll_loss": 0.9084149599075317,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.043845973908901215,
+      "rewards/margins": 0.00853952020406723,
+      "rewards/rejected": -0.05238549783825874,
+      "step": 110
+    },
+    {
+      "epoch": 1.14,
+      "grad_norm": 8.696606134772816,
+      "learning_rate": 4.894001499771015e-07,
+      "log_odds_chosen": 0.408263623714447,
+      "log_odds_ratio": -0.5970031023025513,
+      "logits/chosen": -2.8863773345947266,
+      "logits/rejected": -2.8754923343658447,
+      "logps/chosen": -0.82969731092453,
+      "logps/rejected": -1.0684994459152222,
+      "loss": 0.8737,
+      "nll_loss": 0.8546596765518188,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.04148487001657486,
+      "rewards/margins": 0.011940106749534607,
+      "rewards/rejected": -0.05342497676610947,
+      "step": 120
+    },
+    {
+      "epoch": 1.23,
+      "grad_norm": 8.142229649165017,
+      "learning_rate": 4.7636205937328664e-07,
+      "log_odds_chosen": 0.24737975001335144,
+      "log_odds_ratio": -0.6806861162185669,
+      "logits/chosen": -2.8786978721618652,
+      "logits/rejected": -2.869795083999634,
+      "logps/chosen": -0.8045932054519653,
+      "logps/rejected": -0.9472286105155945,
+      "loss": 0.9147,
+      "nll_loss": 0.8494957089424133,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.04022965580224991,
+      "rewards/margins": 0.007131779100745916,
+      "rewards/rejected": -0.04736143350601196,
+      "step": 130
+    },
+    {
+      "epoch": 1.33,
+      "grad_norm": 8.40750837533949,
+      "learning_rate": 4.584994544724695e-07,
+      "log_odds_chosen": 0.2743343412876129,
+      "log_odds_ratio": -0.6686180830001831,
+      "logits/chosen": -2.866347312927246,
+      "logits/rejected": -2.8765628337860107,
+      "logps/chosen": -0.7923319935798645,
+      "logps/rejected": -0.9410168528556824,
+      "loss": 0.8631,
+      "nll_loss": 0.8559592366218567,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.039616603404283524,
+      "rewards/margins": 0.007434243801981211,
+      "rewards/rejected": -0.047050848603248596,
+      "step": 140
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 8.158461437554013,
+      "learning_rate": 4.3619304568594546e-07,
+      "log_odds_chosen": 0.3588339388370514,
+      "log_odds_ratio": -0.6238493919372559,
+      "logits/chosen": -2.869856119155884,
+      "logits/rejected": -2.8966941833496094,
+      "logps/chosen": -0.7852147817611694,
+      "logps/rejected": -0.9708358645439148,
+      "loss": 0.8549,
+      "nll_loss": 0.8177824020385742,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.03926074132323265,
+      "rewards/margins": 0.009281056001782417,
+      "rewards/rejected": -0.04854179546236992,
+      "step": 150
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 8.09400448571549,
+      "learning_rate": 4.099182553897228e-07,
+      "log_odds_chosen": 0.47473040223121643,
+      "log_odds_ratio": -0.5752421021461487,
+      "logits/chosen": -2.8670287132263184,
+      "logits/rejected": -2.849987506866455,
+      "logps/chosen": -0.7626146078109741,
+      "logps/rejected": -1.0768699645996094,
+      "loss": 0.8506,
+      "nll_loss": 0.7988383173942566,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.038130730390548706,
+      "rewards/margins": 0.01571277156472206,
+      "rewards/rejected": -0.05384349822998047,
+      "step": 160
+    },
+    {
+      "epoch": 1.61,
+      "grad_norm": 6.8406261430616535,
+      "learning_rate": 3.8023508512198257e-07,
+      "log_odds_chosen": 0.35588333010673523,
+      "log_odds_ratio": -0.6235536336898804,
+      "logits/chosen": -2.8909614086151123,
+      "logits/rejected": -2.8951175212860107,
+      "logps/chosen": -0.7602669596672058,
+      "logps/rejected": -0.9576327204704285,
+      "loss": 0.8503,
+      "nll_loss": 0.7980303764343262,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.03801335021853447,
+      "rewards/margins": 0.009868290275335312,
+      "rewards/rejected": -0.04788164049386978,
+      "step": 170
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 7.855569563727718,
+      "learning_rate": 3.4777618012253895e-07,
+      "log_odds_chosen": 0.243193581700325,
+      "log_odds_ratio": -0.6670368909835815,
+      "logits/chosen": -2.84794545173645,
+      "logits/rejected": -2.8654625415802,
+      "logps/chosen": -0.8232158422470093,
+      "logps/rejected": -0.9566510915756226,
+      "loss": 0.8633,
+      "nll_loss": 0.8674771189689636,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.041160788387060165,
+      "rewards/margins": 0.006671770475804806,
+      "rewards/rejected": -0.04783256724476814,
+      "step": 180
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 7.6860644181764375,
+      "learning_rate": 3.1323334559792015e-07,
+      "log_odds_chosen": 0.2766939699649811,
+      "log_odds_ratio": -0.6430349349975586,
+      "logits/chosen": -2.8580737113952637,
+      "logits/rejected": -2.8492133617401123,
+      "logps/chosen": -0.7749003171920776,
+      "logps/rejected": -0.9475242495536804,
+      "loss": 0.8497,
+      "nll_loss": 0.7961811423301697,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.03874502331018448,
+      "rewards/margins": 0.008631192147731781,
+      "rewards/rejected": -0.047376204282045364,
+      "step": 190
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 7.778871538887919,
+      "learning_rate": 2.7734280209446865e-07,
+      "log_odds_chosen": 0.2929554879665375,
+      "log_odds_ratio": -0.6454571485519409,
+      "logits/chosen": -2.874631881713867,
+      "logits/rejected": -2.872854232788086,
+      "logps/chosen": -0.8560446500778198,
+      "logps/rejected": -1.0225257873535156,
+      "loss": 0.8742,
+      "nll_loss": 0.8836696743965149,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04280223324894905,
+      "rewards/margins": 0.008324062451720238,
+      "rewards/rejected": -0.05112629011273384,
+      "step": 200
+    },
+    {
+      "epoch": 1.99,
+      "grad_norm": 8.608571431216085,
+      "learning_rate": 2.4086949423558525e-07,
+      "log_odds_chosen": 0.4544145464897156,
+      "log_odds_ratio": -0.5953847169876099,
+      "logits/chosen": -2.8644211292266846,
+      "logits/rejected": -2.8725366592407227,
+      "logps/chosen": -0.7150249481201172,
+      "logps/rejected": -0.9680275917053223,
+      "loss": 0.8397,
+      "nll_loss": 0.7700489163398743,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.03575124591588974,
+      "rewards/margins": 0.012650132179260254,
+      "rewards/rejected": -0.048401378095149994,
+      "step": 210
+    },
+    {
+      "epoch": 2.0,
+      "eval_log_odds_chosen": 0.3554897606372833,
+      "eval_log_odds_ratio": -0.6257309317588806,
+      "eval_logits/chosen": -2.8794047832489014,
+      "eval_logits/rejected": -2.888169527053833,
+      "eval_logps/chosen": -0.8071174621582031,
+      "eval_logps/rejected": -0.9902352690696716,
+      "eval_loss": 0.8611603379249573,
+      "eval_nll_loss": 0.8365828990936279,
+      "eval_rewards/accuracies": 0.6458333134651184,
+      "eval_rewards/chosen": -0.040355876088142395,
+      "eval_rewards/margins": 0.009155889973044395,
+      "eval_rewards/rejected": -0.04951176047325134,
+      "eval_runtime": 32.7226,
+      "eval_samples_per_second": 22.92,
+      "eval_steps_per_second": 0.733,
+      "step": 211
+    },
+    {
+      "epoch": 2.09,
+      "grad_norm": 6.957321522636136,
+      "learning_rate": 2.0459078725514089e-07,
+      "log_odds_chosen": 0.5733749866485596,
+      "log_odds_ratio": -0.5376938581466675,
+      "logits/chosen": -2.876321315765381,
+      "logits/rejected": -2.8998565673828125,
+      "logps/chosen": -0.6645466089248657,
+      "logps/rejected": -0.9755932092666626,
+      "loss": 0.7779,
+      "nll_loss": 0.7205012440681458,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.033227331936359406,
+      "rewards/margins": 0.015552327036857605,
+      "rewards/rejected": -0.04877965897321701,
+      "step": 220
+    },
+    {
+      "epoch": 2.18,
+      "grad_norm": 7.383557049949752,
+      "learning_rate": 1.692798988071385e-07,
+      "log_odds_chosen": 0.6264925003051758,
+      "log_odds_ratio": -0.5338762998580933,
+      "logits/chosen": -2.9063897132873535,
+      "logits/rejected": -2.9123330116271973,
+      "logps/chosen": -0.6798352003097534,
+      "logps/rejected": -1.0108721256256104,
+      "loss": 0.7879,
+      "nll_loss": 0.763096809387207,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.03399176150560379,
+      "rewards/margins": 0.016551844775676727,
+      "rewards/rejected": -0.05054359883069992,
+      "step": 230
+    },
+    {
+      "epoch": 2.27,
+      "grad_norm": 7.205471545860071,
+      "learning_rate": 1.3568941917384036e-07,
+      "log_odds_chosen": 0.6044402122497559,
+      "log_odds_ratio": -0.5256311297416687,
+      "logits/chosen": -2.929206609725952,
+      "logits/rejected": -2.9085187911987305,
+      "logps/chosen": -0.6959676742553711,
+      "logps/rejected": -1.0572305917739868,
+      "loss": 0.7677,
+      "nll_loss": 0.7402119636535645,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.034798379987478256,
+      "rewards/margins": 0.018063146620988846,
+      "rewards/rejected": -0.0528615303337574,
+      "step": 240
+    },
+    {
+      "epoch": 2.37,
+      "grad_norm": 7.342052939736115,
+      "learning_rate": 1.0453527111051183e-07,
+      "log_odds_chosen": 0.6251620054244995,
+      "log_odds_ratio": -0.5341116786003113,
+      "logits/chosen": -2.8869872093200684,
+      "logits/rejected": -2.912830352783203,
+      "logps/chosen": -0.6446677446365356,
+      "logps/rejected": -0.9978305101394653,
+      "loss": 0.7682,
+      "nll_loss": 0.6836553812026978,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.03223338723182678,
+      "rewards/margins": 0.017658134922385216,
+      "rewards/rejected": -0.04989152401685715,
+      "step": 250
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 7.710315387822491,
+      "learning_rate": 7.648145119484151e-08,
+      "log_odds_chosen": 0.582895815372467,
+      "log_odds_ratio": -0.5480377078056335,
+      "logits/chosen": -2.89570689201355,
+      "logits/rejected": -2.8959426879882812,
+      "logps/chosen": -0.7015441656112671,
+      "logps/rejected": -0.9980028867721558,
+      "loss": 0.7845,
+      "nll_loss": 0.7611762881278992,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.035077206790447235,
+      "rewards/margins": 0.014822937548160553,
+      "rewards/rejected": -0.04990014433860779,
+      "step": 260
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 7.850416556443741,
+      "learning_rate": 5.212587789268649e-08,
+      "log_odds_chosen": 0.5626873970031738,
+      "log_odds_ratio": -0.5825694799423218,
+      "logits/chosen": -2.898651599884033,
+      "logits/rejected": -2.9134531021118164,
+      "logps/chosen": -0.713202714920044,
+      "logps/rejected": -0.9749401211738586,
+      "loss": 0.7837,
+      "nll_loss": 0.7641984224319458,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.03566013649106026,
+      "rewards/margins": 0.013086864724755287,
+      "rewards/rejected": -0.04874700307846069,
+      "step": 270
+    },
+    {
+      "epoch": 2.65,
+      "grad_norm": 7.144333751483519,
+      "learning_rate": 3.198764796404807e-08,
+      "log_odds_chosen": 0.6033560037612915,
+      "log_odds_ratio": -0.5110877156257629,
+      "logits/chosen": -2.9342381954193115,
+      "logits/rejected": -2.9433302879333496,
+      "logps/chosen": -0.6664480566978455,
+      "logps/rejected": -0.9789284467697144,
+      "loss": 0.7707,
+      "nll_loss": 0.7404603362083435,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.03332240507006645,
+      "rewards/margins": 0.015624021179974079,
+      "rewards/rejected": -0.04894643276929855,
+      "step": 280
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 7.681448875266474,
+      "learning_rate": 1.649597281686302e-08,
+      "log_odds_chosen": 0.5207457542419434,
+      "log_odds_ratio": -0.5457995533943176,
+      "logits/chosen": -2.933040142059326,
+      "logits/rejected": -2.942648410797119,
+      "logps/chosen": -0.6956908106803894,
+      "logps/rejected": -0.970605194568634,
+      "loss": 0.7797,
+      "nll_loss": 0.7457938194274902,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.03478454053401947,
+      "rewards/margins": 0.013745710253715515,
+      "rewards/rejected": -0.048530250787734985,
+      "step": 290
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 7.3441185490833805,
+      "learning_rate": 5.981030611018234e-09,
+      "log_odds_chosen": 0.643921434879303,
+      "log_odds_ratio": -0.5143691897392273,
+      "logits/chosen": -2.923649549484253,
+      "logits/rejected": -2.937542676925659,
+      "logps/chosen": -0.7117995619773865,
+      "logps/rejected": -1.0296577215194702,
+      "loss": 0.7861,
+      "nll_loss": 0.7953752279281616,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.035589978098869324,
+      "rewards/margins": 0.015892909839749336,
+      "rewards/rejected": -0.05148288607597351,
+      "step": 300
+    },
+    {
+      "epoch": 2.94,
+      "grad_norm": 7.411020953558967,
+      "learning_rate": 6.66929084112089e-10,
+      "log_odds_chosen": 0.531533420085907,
+      "log_odds_ratio": -0.5645086765289307,
+      "logits/chosen": -2.9212653636932373,
+      "logits/rejected": -2.941697835922241,
+      "logps/chosen": -0.6984909772872925,
+      "logps/rejected": -0.9673376083374023,
+      "loss": 0.7808,
+      "nll_loss": 0.7526401877403259,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.034924548119306564,
+      "rewards/margins": 0.013442324474453926,
+      "rewards/rejected": -0.04836687445640564,
+      "step": 310
+    },
+    {
+      "epoch": 2.99,
+      "eval_log_odds_chosen": 0.3802085816860199,
+      "eval_log_odds_ratio": -0.6214679479598999,
+      "eval_logits/chosen": -2.9039885997772217,
+      "eval_logits/rejected": -2.914613723754883,
+      "eval_logps/chosen": -0.8096261024475098,
+      "eval_logps/rejected": -1.0036059617996216,
+      "eval_loss": 0.8648061752319336,
+      "eval_nll_loss": 0.8392227292060852,
+      "eval_rewards/accuracies": 0.6458333134651184,
+      "eval_rewards/chosen": -0.04048130661249161,
+      "eval_rewards/margins": 0.009698997251689434,
+      "eval_rewards/rejected": -0.05018030107021332,
+      "eval_runtime": 32.6828,
+      "eval_samples_per_second": 22.948,
+      "eval_steps_per_second": 0.734,
+      "step": 315
+    },
+    {
+      "epoch": 2.99,
+      "step": 315,
+      "total_flos": 0.0,
+      "train_loss": 0.877839538029262,
+      "train_runtime": 3399.396,
+      "train_samples_per_second": 5.957,
+      "train_steps_per_second": 0.093
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 315,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc8f78b338187df42575a50e4b266ef4e260ed66e8c7d74c9f9b3c79df212e82
+size 6456