Model save

README.md

@@ -0,0 +1,55 @@
+---
+license: apache-2.0
+base_model: EleutherAI/pythia-2.8b-deduped
+tags:
+- trl
+- sft
+- generated_from_trainer
+model-index:
+- name: pythia-2_8b_sft-gpt-turbo
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# pythia-2_8b_sft-gpt-turbo
+
+This model is a fine-tuned version of [EleutherAI/pythia-2.8b-deduped](https://huggingface.co/EleutherAI/pythia-2.8b-deduped) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 32
+- eval_batch_size: 4
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.41.1
+- Pytorch 2.1.2+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 7.533261039139123e+17,
+    "train_loss": 1.0007318160669976,
+    "train_runtime": 6142.5782,
+    "train_samples": 51577,
+    "train_samples_per_second": 8.397,
+    "train_steps_per_second": 0.262
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.41.1",
+  "use_cache": false
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e51ee42459117c226841d3f881d312d056ddd2d28c9ffacf26e8fe4049c85b27
+size 4978209248

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72dcf9a085323aee478eec6df877a9b49e719c2d24e57b5f087626f6e7495610
+size 572254856

model.safetensors.index.json

@@ -0,0 +1,395 @@
+{
+  "metadata": {
+    "total_size": 5550417920
+  },
+  "weight_map": {
+    "embed_out.weight": "model-00002-of-00002.safetensors",
+    "gpt_neox.embed_in.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.final_layer_norm.bias": "model-00002-of-00002.safetensors",
+    "gpt_neox.final_layer_norm.weight": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.0.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.0.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.0.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.0.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.0.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.0.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.0.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.0.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.0.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.0.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.1.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.1.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.1.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.1.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.1.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.1.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.1.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.1.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.1.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.1.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.10.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.10.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.10.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.10.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.10.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.10.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.10.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.10.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.10.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.10.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.11.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.11.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.11.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.11.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.11.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.11.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.11.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.11.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.11.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.11.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.12.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.12.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.12.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.12.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.12.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.12.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.12.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.12.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.12.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.12.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.13.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.13.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.13.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.13.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.13.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.13.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.13.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.13.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.13.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.13.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.14.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.14.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.14.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.14.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.14.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.14.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.14.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.14.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.14.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.14.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.15.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.15.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.15.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.15.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.15.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.15.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.15.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.15.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.15.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.15.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.16.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.16.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.16.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.16.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.16.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.16.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.16.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.16.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.16.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.16.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.17.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.17.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.17.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.17.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.17.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.17.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.17.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.17.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.17.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.17.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.18.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.18.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.18.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.18.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.18.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.18.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.18.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.18.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.18.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.18.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.19.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.19.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.19.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.19.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.19.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.19.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.19.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.19.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.19.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.19.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.2.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.2.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.2.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.2.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.2.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.2.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.2.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.2.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.2.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.2.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.20.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.20.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.20.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.20.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.20.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.20.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.20.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.20.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.20.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.20.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.21.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.21.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.21.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.21.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.21.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.21.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.21.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.21.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.21.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.21.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.22.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.22.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.22.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.22.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.22.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.22.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.22.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.22.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.22.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.22.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.23.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.23.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.23.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.23.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.23.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.23.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.23.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.23.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.23.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.23.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.24.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.24.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.24.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.24.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.24.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.24.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.24.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.24.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.24.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.24.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.24.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.24.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.25.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.25.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.25.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.25.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.25.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.25.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.25.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.25.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.25.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.25.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.25.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.25.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.26.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.26.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.26.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.26.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.26.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.26.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.26.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.26.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.26.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.26.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.26.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.26.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.27.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.27.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.27.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.27.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.27.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.27.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.27.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.27.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.27.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.27.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.27.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.27.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.28.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.28.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.28.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.28.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.28.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.28.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.28.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.28.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.28.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.28.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.28.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.28.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.29.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.29.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.29.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.29.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.29.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.29.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.29.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.29.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.29.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.29.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.29.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.29.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.3.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.3.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.3.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.3.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.3.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.3.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.3.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.3.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.3.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.3.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.30.attention.dense.bias": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.30.attention.dense.weight": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.30.attention.query_key_value.bias": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.30.attention.query_key_value.weight": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.30.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.30.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.30.mlp.dense_4h_to_h.bias": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.30.mlp.dense_4h_to_h.weight": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.30.mlp.dense_h_to_4h.bias": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.30.mlp.dense_h_to_4h.weight": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.30.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.30.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.31.attention.dense.bias": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.31.attention.dense.weight": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.31.attention.query_key_value.bias": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.31.attention.query_key_value.weight": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.31.input_layernorm.bias": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.31.mlp.dense_4h_to_h.bias": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.31.mlp.dense_4h_to_h.weight": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.31.mlp.dense_h_to_4h.bias": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.31.mlp.dense_h_to_4h.weight": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.31.post_attention_layernorm.bias": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "gpt_neox.layers.4.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.4.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.4.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.4.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.4.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.4.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.4.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.4.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.4.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.4.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.5.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.5.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.5.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.5.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.5.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.5.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.5.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.5.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.5.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.5.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.6.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.6.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.6.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.6.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.6.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.6.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.6.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.6.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.6.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.6.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.7.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.7.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.7.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.7.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.7.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.7.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.7.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.7.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.7.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.7.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.8.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.8.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.8.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.8.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.8.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.8.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.8.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.8.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.8.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.8.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.9.attention.dense.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.9.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.9.attention.query_key_value.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.9.attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.9.input_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.9.mlp.dense_4h_to_h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.9.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.9.mlp.dense_h_to_4h.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.9.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.9.post_attention_layernorm.bias": "model-00001-of-00002.safetensors",
+    "gpt_neox.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 7.533261039139123e+17,
+    "train_loss": 1.0007318160669976,
+    "train_runtime": 6142.5782,
+    "train_samples": 51577,
+    "train_samples_per_second": 8.397,
+    "train_steps_per_second": 0.262
+}

trainer_state.json

@@ -0,0 +1,1169 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 1612,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00620347394540943,
+      "grad_norm": 4.46875,
+      "learning_rate": 1.234567901234568e-06,
+      "loss": 1.4227,
+      "step": 10
+    },
+    {
+      "epoch": 0.01240694789081886,
+      "grad_norm": 5.84375,
+      "learning_rate": 2.469135802469136e-06,
+      "loss": 1.4156,
+      "step": 20
+    },
+    {
+      "epoch": 0.018610421836228287,
+      "grad_norm": 3.890625,
+      "learning_rate": 3.7037037037037037e-06,
+      "loss": 1.3094,
+      "step": 30
+    },
+    {
+      "epoch": 0.02481389578163772,
+      "grad_norm": 4.03125,
+      "learning_rate": 4.938271604938272e-06,
+      "loss": 1.2648,
+      "step": 40
+    },
+    {
+      "epoch": 0.031017369727047148,
+      "grad_norm": 2.921875,
+      "learning_rate": 6.17283950617284e-06,
+      "loss": 1.1773,
+      "step": 50
+    },
+    {
+      "epoch": 0.03722084367245657,
+      "grad_norm": 2.5625,
+      "learning_rate": 7.4074074074074075e-06,
+      "loss": 1.1281,
+      "step": 60
+    },
+    {
+      "epoch": 0.043424317617866005,
+      "grad_norm": 2.390625,
+      "learning_rate": 8.641975308641975e-06,
+      "loss": 1.1273,
+      "step": 70
+    },
+    {
+      "epoch": 0.04962779156327544,
+      "grad_norm": 2.234375,
+      "learning_rate": 9.876543209876543e-06,
+      "loss": 1.0973,
+      "step": 80
+    },
+    {
+      "epoch": 0.05583126550868486,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.1111111111111113e-05,
+      "loss": 1.084,
+      "step": 90
+    },
+    {
+      "epoch": 0.062034739454094295,
+      "grad_norm": 2.796875,
+      "learning_rate": 1.234567901234568e-05,
+      "loss": 1.0523,
+      "step": 100
+    },
+    {
+      "epoch": 0.06823821339950373,
+      "grad_norm": 2.375,
+      "learning_rate": 1.3580246913580248e-05,
+      "loss": 1.118,
+      "step": 110
+    },
+    {
+      "epoch": 0.07444168734491315,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.4814814814814815e-05,
+      "loss": 1.0496,
+      "step": 120
+    },
+    {
+      "epoch": 0.08064516129032258,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.6049382716049385e-05,
+      "loss": 1.0219,
+      "step": 130
+    },
+    {
+      "epoch": 0.08684863523573201,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.728395061728395e-05,
+      "loss": 1.0566,
+      "step": 140
+    },
+    {
+      "epoch": 0.09305210918114144,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.851851851851852e-05,
+      "loss": 1.0754,
+      "step": 150
+    },
+    {
+      "epoch": 0.09925558312655088,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.9753086419753087e-05,
+      "loss": 1.0496,
+      "step": 160
+    },
+    {
+      "epoch": 0.1054590570719603,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.9889655172413794e-05,
+      "loss": 1.0562,
+      "step": 170
+    },
+    {
+      "epoch": 0.11166253101736973,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.9751724137931037e-05,
+      "loss": 1.0219,
+      "step": 180
+    },
+    {
+      "epoch": 0.11786600496277916,
+      "grad_norm": 2.40625,
+      "learning_rate": 1.961379310344828e-05,
+      "loss": 1.0008,
+      "step": 190
+    },
+    {
+      "epoch": 0.12406947890818859,
+      "grad_norm": 2.25,
+      "learning_rate": 1.9475862068965518e-05,
+      "loss": 1.0355,
+      "step": 200
+    },
+    {
+      "epoch": 0.13027295285359802,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.933793103448276e-05,
+      "loss": 1.0281,
+      "step": 210
+    },
+    {
+      "epoch": 0.13647642679900746,
+      "grad_norm": 2.453125,
+      "learning_rate": 1.9200000000000003e-05,
+      "loss": 1.0484,
+      "step": 220
+    },
+    {
+      "epoch": 0.14267990074441686,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.906206896551724e-05,
+      "loss": 1.0082,
+      "step": 230
+    },
+    {
+      "epoch": 0.1488833746898263,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.8924137931034484e-05,
+      "loss": 1.0145,
+      "step": 240
+    },
+    {
+      "epoch": 0.15508684863523572,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.8786206896551726e-05,
+      "loss": 1.0531,
+      "step": 250
+    },
+    {
+      "epoch": 0.16129032258064516,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.864827586206897e-05,
+      "loss": 0.993,
+      "step": 260
+    },
+    {
+      "epoch": 0.1674937965260546,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.8510344827586207e-05,
+      "loss": 1.0422,
+      "step": 270
+    },
+    {
+      "epoch": 0.17369727047146402,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.837241379310345e-05,
+      "loss": 1.0027,
+      "step": 280
+    },
+    {
+      "epoch": 0.17990074441687345,
+      "grad_norm": 2.0,
+      "learning_rate": 1.8234482758620692e-05,
+      "loss": 0.9922,
+      "step": 290
+    },
+    {
+      "epoch": 0.18610421836228289,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.809655172413793e-05,
+      "loss": 1.0164,
+      "step": 300
+    },
+    {
+      "epoch": 0.19230769230769232,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.7958620689655173e-05,
+      "loss": 1.0242,
+      "step": 310
+    },
+    {
+      "epoch": 0.19851116625310175,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.7820689655172416e-05,
+      "loss": 1.0602,
+      "step": 320
+    },
+    {
+      "epoch": 0.20471464019851116,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.7682758620689658e-05,
+      "loss": 1.0082,
+      "step": 330
+    },
+    {
+      "epoch": 0.2109181141439206,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.7544827586206897e-05,
+      "loss": 1.0031,
+      "step": 340
+    },
+    {
+      "epoch": 0.21712158808933002,
+      "grad_norm": 2.4375,
+      "learning_rate": 1.740689655172414e-05,
+      "loss": 1.0031,
+      "step": 350
+    },
+    {
+      "epoch": 0.22332506203473945,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.7268965517241382e-05,
+      "loss": 0.9684,
+      "step": 360
+    },
+    {
+      "epoch": 0.22952853598014888,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.713103448275862e-05,
+      "loss": 0.984,
+      "step": 370
+    },
+    {
+      "epoch": 0.23573200992555832,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.6993103448275863e-05,
+      "loss": 0.9828,
+      "step": 380
+    },
+    {
+      "epoch": 0.24193548387096775,
+      "grad_norm": 2.5,
+      "learning_rate": 1.6855172413793106e-05,
+      "loss": 1.0297,
+      "step": 390
+    },
+    {
+      "epoch": 0.24813895781637718,
+      "grad_norm": 1.8828125,
+      "learning_rate": 1.6717241379310348e-05,
+      "loss": 0.9895,
+      "step": 400
+    },
+    {
+      "epoch": 0.2543424317617866,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.6579310344827587e-05,
+      "loss": 1.0195,
+      "step": 410
+    },
+    {
+      "epoch": 0.26054590570719605,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.644137931034483e-05,
+      "loss": 0.9945,
+      "step": 420
+    },
+    {
+      "epoch": 0.2667493796526055,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.630344827586207e-05,
+      "loss": 1.0043,
+      "step": 430
+    },
+    {
+      "epoch": 0.2729528535980149,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.616551724137931e-05,
+      "loss": 1.0293,
+      "step": 440
+    },
+    {
+      "epoch": 0.27915632754342434,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.6027586206896553e-05,
+      "loss": 1.0043,
+      "step": 450
+    },
+    {
+      "epoch": 0.2853598014888337,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.5889655172413795e-05,
+      "loss": 1.0004,
+      "step": 460
+    },
+    {
+      "epoch": 0.29156327543424315,
+      "grad_norm": 2.40625,
+      "learning_rate": 1.5751724137931034e-05,
+      "loss": 1.0094,
+      "step": 470
+    },
+    {
+      "epoch": 0.2977667493796526,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.5613793103448276e-05,
+      "loss": 0.9801,
+      "step": 480
+    },
+    {
+      "epoch": 0.303970223325062,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.547586206896552e-05,
+      "loss": 1.009,
+      "step": 490
+    },
+    {
+      "epoch": 0.31017369727047145,
+      "grad_norm": 2.375,
+      "learning_rate": 1.533793103448276e-05,
+      "loss": 1.0008,
+      "step": 500
+    },
+    {
+      "epoch": 0.3163771712158809,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.5200000000000002e-05,
+      "loss": 0.9852,
+      "step": 510
+    },
+    {
+      "epoch": 0.3225806451612903,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.5062068965517243e-05,
+      "loss": 0.9887,
+      "step": 520
+    },
+    {
+      "epoch": 0.32878411910669975,
+      "grad_norm": 2.3125,
+      "learning_rate": 1.4924137931034483e-05,
+      "loss": 1.0188,
+      "step": 530
+    },
+    {
+      "epoch": 0.3349875930521092,
+      "grad_norm": 2.4375,
+      "learning_rate": 1.4786206896551726e-05,
+      "loss": 0.9668,
+      "step": 540
+    },
+    {
+      "epoch": 0.3411910669975186,
+      "grad_norm": 2.125,
+      "learning_rate": 1.4648275862068966e-05,
+      "loss": 1.0008,
+      "step": 550
+    },
+    {
+      "epoch": 0.34739454094292804,
+      "grad_norm": 2.125,
+      "learning_rate": 1.4510344827586209e-05,
+      "loss": 0.9785,
+      "step": 560
+    },
+    {
+      "epoch": 0.3535980148883375,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.4372413793103451e-05,
+      "loss": 1.0262,
+      "step": 570
+    },
+    {
+      "epoch": 0.3598014888337469,
+      "grad_norm": 2.125,
+      "learning_rate": 1.4234482758620692e-05,
+      "loss": 0.9633,
+      "step": 580
+    },
+    {
+      "epoch": 0.36600496277915634,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.4096551724137932e-05,
+      "loss": 1.0066,
+      "step": 590
+    },
+    {
+      "epoch": 0.37220843672456577,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.3958620689655173e-05,
+      "loss": 0.9812,
+      "step": 600
+    },
+    {
+      "epoch": 0.3784119106699752,
+      "grad_norm": 1.9921875,
+      "learning_rate": 1.3820689655172415e-05,
+      "loss": 0.9266,
+      "step": 610
+    },
+    {
+      "epoch": 0.38461538461538464,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.3682758620689656e-05,
+      "loss": 1.0223,
+      "step": 620
+    },
+    {
+      "epoch": 0.39081885856079407,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.3544827586206896e-05,
+      "loss": 0.9871,
+      "step": 630
+    },
+    {
+      "epoch": 0.3970223325062035,
+      "grad_norm": 2.4375,
+      "learning_rate": 1.340689655172414e-05,
+      "loss": 1.0594,
+      "step": 640
+    },
+    {
+      "epoch": 0.4032258064516129,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.3268965517241381e-05,
+      "loss": 0.9625,
+      "step": 650
+    },
+    {
+      "epoch": 0.4094292803970223,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.3131034482758622e-05,
+      "loss": 1.0172,
+      "step": 660
+    },
+    {
+      "epoch": 0.41563275434243174,
+      "grad_norm": 2.5,
+      "learning_rate": 1.2993103448275863e-05,
+      "loss": 1.0176,
+      "step": 670
+    },
+    {
+      "epoch": 0.4218362282878412,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.2855172413793105e-05,
+      "loss": 0.9344,
+      "step": 680
+    },
+    {
+      "epoch": 0.4280397022332506,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.2717241379310346e-05,
+      "loss": 0.9543,
+      "step": 690
+    },
+    {
+      "epoch": 0.43424317617866004,
+      "grad_norm": 2.46875,
+      "learning_rate": 1.2579310344827586e-05,
+      "loss": 0.9949,
+      "step": 700
+    },
+    {
+      "epoch": 0.44044665012406947,
+      "grad_norm": 1.96875,
+      "learning_rate": 1.2441379310344829e-05,
+      "loss": 0.9859,
+      "step": 710
+    },
+    {
+      "epoch": 0.4466501240694789,
+      "grad_norm": 2.46875,
+      "learning_rate": 1.2303448275862071e-05,
+      "loss": 1.0063,
+      "step": 720
+    },
+    {
+      "epoch": 0.45285359801488834,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.2165517241379312e-05,
+      "loss": 0.959,
+      "step": 730
+    },
+    {
+      "epoch": 0.45905707196029777,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.2027586206896554e-05,
+      "loss": 0.9574,
+      "step": 740
+    },
+    {
+      "epoch": 0.4652605459057072,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.1889655172413795e-05,
+      "loss": 0.9863,
+      "step": 750
+    },
+    {
+      "epoch": 0.47146401985111663,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.1751724137931035e-05,
+      "loss": 0.9859,
+      "step": 760
+    },
+    {
+      "epoch": 0.47766749379652607,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.1613793103448276e-05,
+      "loss": 0.9984,
+      "step": 770
+    },
+    {
+      "epoch": 0.4838709677419355,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.1475862068965518e-05,
+      "loss": 0.9984,
+      "step": 780
+    },
+    {
+      "epoch": 0.49007444168734493,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.1337931034482759e-05,
+      "loss": 0.9402,
+      "step": 790
+    },
+    {
+      "epoch": 0.49627791563275436,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.1200000000000001e-05,
+      "loss": 0.9902,
+      "step": 800
+    },
+    {
+      "epoch": 0.5024813895781638,
+      "grad_norm": 2.46875,
+      "learning_rate": 1.1062068965517244e-05,
+      "loss": 1.0008,
+      "step": 810
+    },
+    {
+      "epoch": 0.5086848635235732,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.0924137931034484e-05,
+      "loss": 0.9727,
+      "step": 820
+    },
+    {
+      "epoch": 0.5148883374689827,
+      "grad_norm": 2.53125,
+      "learning_rate": 1.0786206896551725e-05,
+      "loss": 0.9672,
+      "step": 830
+    },
+    {
+      "epoch": 0.5210918114143921,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.0648275862068967e-05,
+      "loss": 0.9371,
+      "step": 840
+    },
+    {
+      "epoch": 0.5272952853598015,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.0510344827586208e-05,
+      "loss": 1.0145,
+      "step": 850
+    },
+    {
+      "epoch": 0.533498759305211,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.0372413793103449e-05,
+      "loss": 0.9672,
+      "step": 860
+    },
+    {
+      "epoch": 0.5397022332506204,
+      "grad_norm": 2.40625,
+      "learning_rate": 1.023448275862069e-05,
+      "loss": 0.9602,
+      "step": 870
+    },
+    {
+      "epoch": 0.5459057071960298,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.0096551724137933e-05,
+      "loss": 0.9871,
+      "step": 880
+    },
+    {
+      "epoch": 0.5521091811414393,
+      "grad_norm": 2.359375,
+      "learning_rate": 9.958620689655172e-06,
+      "loss": 0.9797,
+      "step": 890
+    },
+    {
+      "epoch": 0.5583126550868487,
+      "grad_norm": 2.390625,
+      "learning_rate": 9.820689655172415e-06,
+      "loss": 0.9848,
+      "step": 900
+    },
+    {
+      "epoch": 0.5645161290322581,
+      "grad_norm": 2.390625,
+      "learning_rate": 9.682758620689657e-06,
+      "loss": 0.9699,
+      "step": 910
+    },
+    {
+      "epoch": 0.5707196029776674,
+      "grad_norm": 2.234375,
+      "learning_rate": 9.544827586206898e-06,
+      "loss": 0.9641,
+      "step": 920
+    },
+    {
+      "epoch": 0.5769230769230769,
+      "grad_norm": 2.53125,
+      "learning_rate": 9.406896551724138e-06,
+      "loss": 1.0109,
+      "step": 930
+    },
+    {
+      "epoch": 0.5831265508684863,
+      "grad_norm": 1.9765625,
+      "learning_rate": 9.26896551724138e-06,
+      "loss": 0.9906,
+      "step": 940
+    },
+    {
+      "epoch": 0.5893300248138957,
+      "grad_norm": 2.34375,
+      "learning_rate": 9.131034482758621e-06,
+      "loss": 0.9938,
+      "step": 950
+    },
+    {
+      "epoch": 0.5955334987593052,
+      "grad_norm": 2.046875,
+      "learning_rate": 8.993103448275864e-06,
+      "loss": 0.923,
+      "step": 960
+    },
+    {
+      "epoch": 0.6017369727047146,
+      "grad_norm": 2.0625,
+      "learning_rate": 8.855172413793104e-06,
+      "loss": 0.9766,
+      "step": 970
+    },
+    {
+      "epoch": 0.607940446650124,
+      "grad_norm": 2.25,
+      "learning_rate": 8.717241379310347e-06,
+      "loss": 0.9832,
+      "step": 980
+    },
+    {
+      "epoch": 0.6141439205955335,
+      "grad_norm": 2.15625,
+      "learning_rate": 8.579310344827587e-06,
+      "loss": 0.9184,
+      "step": 990
+    },
+    {
+      "epoch": 0.6203473945409429,
+      "grad_norm": 2.359375,
+      "learning_rate": 8.441379310344828e-06,
+      "loss": 0.9367,
+      "step": 1000
+    },
+    {
+      "epoch": 0.6265508684863523,
+      "grad_norm": 2.40625,
+      "learning_rate": 8.30344827586207e-06,
+      "loss": 0.9969,
+      "step": 1010
+    },
+    {
+      "epoch": 0.6327543424317618,
+      "grad_norm": 2.4375,
+      "learning_rate": 8.165517241379311e-06,
+      "loss": 0.9941,
+      "step": 1020
+    },
+    {
+      "epoch": 0.6389578163771712,
+      "grad_norm": 2.0625,
+      "learning_rate": 8.027586206896553e-06,
+      "loss": 0.9363,
+      "step": 1030
+    },
+    {
+      "epoch": 0.6451612903225806,
+      "grad_norm": 2.3125,
+      "learning_rate": 7.889655172413794e-06,
+      "loss": 0.9961,
+      "step": 1040
+    },
+    {
+      "epoch": 0.6513647642679901,
+      "grad_norm": 1.9140625,
+      "learning_rate": 7.751724137931035e-06,
+      "loss": 0.9738,
+      "step": 1050
+    },
+    {
+      "epoch": 0.6575682382133995,
+      "grad_norm": 2.296875,
+      "learning_rate": 7.613793103448276e-06,
+      "loss": 0.9641,
+      "step": 1060
+    },
+    {
+      "epoch": 0.6637717121588089,
+      "grad_norm": 2.1875,
+      "learning_rate": 7.475862068965518e-06,
+      "loss": 0.9797,
+      "step": 1070
+    },
+    {
+      "epoch": 0.6699751861042184,
+      "grad_norm": 2.296875,
+      "learning_rate": 7.337931034482759e-06,
+      "loss": 0.9379,
+      "step": 1080
+    },
+    {
+      "epoch": 0.6761786600496278,
+      "grad_norm": 2.015625,
+      "learning_rate": 7.2000000000000005e-06,
+      "loss": 0.952,
+      "step": 1090
+    },
+    {
+      "epoch": 0.6823821339950372,
+      "grad_norm": 2.046875,
+      "learning_rate": 7.062068965517241e-06,
+      "loss": 0.9668,
+      "step": 1100
+    },
+    {
+      "epoch": 0.6885856079404467,
+      "grad_norm": 2.328125,
+      "learning_rate": 6.9241379310344835e-06,
+      "loss": 0.9824,
+      "step": 1110
+    },
+    {
+      "epoch": 0.6947890818858561,
+      "grad_norm": 2.296875,
+      "learning_rate": 6.786206896551725e-06,
+      "loss": 0.9926,
+      "step": 1120
+    },
+    {
+      "epoch": 0.7009925558312655,
+      "grad_norm": 1.78125,
+      "learning_rate": 6.648275862068966e-06,
+      "loss": 0.934,
+      "step": 1130
+    },
+    {
+      "epoch": 0.707196029776675,
+      "grad_norm": 2.265625,
+      "learning_rate": 6.510344827586207e-06,
+      "loss": 0.9621,
+      "step": 1140
+    },
+    {
+      "epoch": 0.7133995037220844,
+      "grad_norm": 1.9765625,
+      "learning_rate": 6.372413793103449e-06,
+      "loss": 0.982,
+      "step": 1150
+    },
+    {
+      "epoch": 0.7196029776674938,
+      "grad_norm": 2.09375,
+      "learning_rate": 6.23448275862069e-06,
+      "loss": 0.9883,
+      "step": 1160
+    },
+    {
+      "epoch": 0.7258064516129032,
+      "grad_norm": 2.40625,
+      "learning_rate": 6.096551724137932e-06,
+      "loss": 0.9762,
+      "step": 1170
+    },
+    {
+      "epoch": 0.7320099255583127,
+      "grad_norm": 2.15625,
+      "learning_rate": 5.958620689655172e-06,
+      "loss": 0.977,
+      "step": 1180
+    },
+    {
+      "epoch": 0.7382133995037221,
+      "grad_norm": 2.171875,
+      "learning_rate": 5.820689655172415e-06,
+      "loss": 1.0129,
+      "step": 1190
+    },
+    {
+      "epoch": 0.7444168734491315,
+      "grad_norm": 2.25,
+      "learning_rate": 5.682758620689655e-06,
+      "loss": 0.9887,
+      "step": 1200
+    },
+    {
+      "epoch": 0.750620347394541,
+      "grad_norm": 2.140625,
+      "learning_rate": 5.544827586206897e-06,
+      "loss": 0.9516,
+      "step": 1210
+    },
+    {
+      "epoch": 0.7568238213399504,
+      "grad_norm": 2.171875,
+      "learning_rate": 5.4068965517241375e-06,
+      "loss": 0.952,
+      "step": 1220
+    },
+    {
+      "epoch": 0.7630272952853598,
+      "grad_norm": 2.28125,
+      "learning_rate": 5.26896551724138e-06,
+      "loss": 0.9922,
+      "step": 1230
+    },
+    {
+      "epoch": 0.7692307692307693,
+      "grad_norm": 1.9375,
+      "learning_rate": 5.131034482758621e-06,
+      "loss": 0.9914,
+      "step": 1240
+    },
+    {
+      "epoch": 0.7754342431761787,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.993103448275862e-06,
+      "loss": 0.9449,
+      "step": 1250
+    },
+    {
+      "epoch": 0.7816377171215881,
+      "grad_norm": 2.359375,
+      "learning_rate": 4.8551724137931035e-06,
+      "loss": 1.0051,
+      "step": 1260
+    },
+    {
+      "epoch": 0.7878411910669976,
+      "grad_norm": 2.15625,
+      "learning_rate": 4.717241379310345e-06,
+      "loss": 0.9926,
+      "step": 1270
+    },
+    {
+      "epoch": 0.794044665012407,
+      "grad_norm": 2.109375,
+      "learning_rate": 4.5793103448275865e-06,
+      "loss": 0.9477,
+      "step": 1280
+    },
+    {
+      "epoch": 0.8002481389578163,
+      "grad_norm": 2.21875,
+      "learning_rate": 4.441379310344828e-06,
+      "loss": 0.9477,
+      "step": 1290
+    },
+    {
+      "epoch": 0.8064516129032258,
+      "grad_norm": 2.03125,
+      "learning_rate": 4.3034482758620695e-06,
+      "loss": 0.9543,
+      "step": 1300
+    },
+    {
+      "epoch": 0.8126550868486352,
+      "grad_norm": 2.3125,
+      "learning_rate": 4.16551724137931e-06,
+      "loss": 0.9492,
+      "step": 1310
+    },
+    {
+      "epoch": 0.8188585607940446,
+      "grad_norm": 2.421875,
+      "learning_rate": 4.027586206896552e-06,
+      "loss": 0.9809,
+      "step": 1320
+    },
+    {
+      "epoch": 0.825062034739454,
+      "grad_norm": 2.390625,
+      "learning_rate": 3.889655172413793e-06,
+      "loss": 0.9812,
+      "step": 1330
+    },
+    {
+      "epoch": 0.8312655086848635,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.7517241379310347e-06,
+      "loss": 0.9781,
+      "step": 1340
+    },
+    {
+      "epoch": 0.8374689826302729,
+      "grad_norm": 2.234375,
+      "learning_rate": 3.6137931034482758e-06,
+      "loss": 0.9754,
+      "step": 1350
+    },
+    {
+      "epoch": 0.8436724565756824,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.4758620689655177e-06,
+      "loss": 0.9641,
+      "step": 1360
+    },
+    {
+      "epoch": 0.8498759305210918,
+      "grad_norm": 2.671875,
+      "learning_rate": 3.3379310344827588e-06,
+      "loss": 0.9723,
+      "step": 1370
+    },
+    {
+      "epoch": 0.8560794044665012,
+      "grad_norm": 2.375,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 0.9938,
+      "step": 1380
+    },
+    {
+      "epoch": 0.8622828784119106,
+      "grad_norm": 2.359375,
+      "learning_rate": 3.0620689655172414e-06,
+      "loss": 0.9523,
+      "step": 1390
+    },
+    {
+      "epoch": 0.8684863523573201,
+      "grad_norm": 1.9609375,
+      "learning_rate": 2.924137931034483e-06,
+      "loss": 0.9508,
+      "step": 1400
+    },
+    {
+      "epoch": 0.8746898263027295,
+      "grad_norm": 2.28125,
+      "learning_rate": 2.7862068965517244e-06,
+      "loss": 0.9512,
+      "step": 1410
+    },
+    {
+      "epoch": 0.8808933002481389,
+      "grad_norm": 2.109375,
+      "learning_rate": 2.648275862068966e-06,
+      "loss": 0.9547,
+      "step": 1420
+    },
+    {
+      "epoch": 0.8870967741935484,
+      "grad_norm": 2.28125,
+      "learning_rate": 2.510344827586207e-06,
+      "loss": 0.9539,
+      "step": 1430
+    },
+    {
+      "epoch": 0.8933002481389578,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.3724137931034484e-06,
+      "loss": 0.9879,
+      "step": 1440
+    },
+    {
+      "epoch": 0.8995037220843672,
+      "grad_norm": 2.40625,
+      "learning_rate": 2.23448275862069e-06,
+      "loss": 0.9684,
+      "step": 1450
+    },
+    {
+      "epoch": 0.9057071960297767,
+      "grad_norm": 2.09375,
+      "learning_rate": 2.0965517241379315e-06,
+      "loss": 0.9348,
+      "step": 1460
+    },
+    {
+      "epoch": 0.9119106699751861,
+      "grad_norm": 2.25,
+      "learning_rate": 1.9586206896551725e-06,
+      "loss": 0.9711,
+      "step": 1470
+    },
+    {
+      "epoch": 0.9181141439205955,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.820689655172414e-06,
+      "loss": 0.9602,
+      "step": 1480
+    },
+    {
+      "epoch": 0.924317617866005,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.6827586206896553e-06,
+      "loss": 1.0289,
+      "step": 1490
+    },
+    {
+      "epoch": 0.9305210918114144,
+      "grad_norm": 2.359375,
+      "learning_rate": 1.5448275862068968e-06,
+      "loss": 0.9785,
+      "step": 1500
+    },
+    {
+      "epoch": 0.9367245657568238,
+      "grad_norm": 2.4375,
+      "learning_rate": 1.4068965517241381e-06,
+      "loss": 1.0277,
+      "step": 1510
+    },
+    {
+      "epoch": 0.9429280397022333,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.2689655172413794e-06,
+      "loss": 0.982,
+      "step": 1520
+    },
+    {
+      "epoch": 0.9491315136476427,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.1310344827586207e-06,
+      "loss": 0.9723,
+      "step": 1530
+    },
+    {
+      "epoch": 0.9553349875930521,
+      "grad_norm": 2.125,
+      "learning_rate": 9.931034482758622e-07,
+      "loss": 0.9535,
+      "step": 1540
+    },
+    {
+      "epoch": 0.9615384615384616,
+      "grad_norm": 2.21875,
+      "learning_rate": 8.551724137931035e-07,
+      "loss": 0.9652,
+      "step": 1550
+    },
+    {
+      "epoch": 0.967741935483871,
+      "grad_norm": 2.515625,
+      "learning_rate": 7.172413793103449e-07,
+      "loss": 0.9527,
+      "step": 1560
+    },
+    {
+      "epoch": 0.9739454094292804,
+      "grad_norm": 2.25,
+      "learning_rate": 5.793103448275863e-07,
+      "loss": 0.9809,
+      "step": 1570
+    },
+    {
+      "epoch": 0.9801488833746899,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.413793103448276e-07,
+      "loss": 0.9852,
+      "step": 1580
+    },
+    {
+      "epoch": 0.9863523573200993,
+      "grad_norm": 2.109375,
+      "learning_rate": 3.0344827586206897e-07,
+      "loss": 0.9164,
+      "step": 1590
+    },
+    {
+      "epoch": 0.9925558312655087,
+      "grad_norm": 2.40625,
+      "learning_rate": 1.6551724137931037e-07,
+      "loss": 0.9355,
+      "step": 1600
+    },
+    {
+      "epoch": 0.9987593052109182,
+      "grad_norm": 2.015625,
+      "learning_rate": 2.7586206896551723e-08,
+      "loss": 0.9949,
+      "step": 1610
+    },
+    {
+      "epoch": 1.0,
+      "step": 1612,
+      "total_flos": 7.533261039139123e+17,
+      "train_loss": 1.0007318160669976,
+      "train_runtime": 6142.5782,
+      "train_samples_per_second": 8.397,
+      "train_steps_per_second": 0.262
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1612,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.533261039139123e+17,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}