Model save

README.md

@@ -0,0 +1,69 @@
+---
+library_name: peft
+license: llama2
+base_model: meta-llama/Llama-2-13b-hf
+tags:
+- trl
+- sft
+- generated_from_trainer
+datasets:
+- generator
+model-index:
+- name: llama2-13b-pissa-alpaca-11-v1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# llama2-13b-pissa-alpaca-11-v1
+
+This model is a fine-tuned version of [meta-llama/Llama-2-13b-hf](https://huggingface.co/meta-llama/Llama-2-13b-hf) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.4016
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 256
+- total_eval_batch_size: 128
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 1.4059        | 0.9938 | 80   | 1.4016          |
+
+
+### Framework versions
+
+- PEFT 0.13.2
+- Transformers 4.46.2
+- Pytorch 2.5.1+cu124
+- Datasets 3.1.0
+- Tokenizers 0.20.3

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9937888198757764,
+    "total_flos": 8.089924109389005e+17,
+    "train_loss": 1.786448487639427,
+    "train_runtime": 830.0773,
+    "train_samples": 51241,
+    "train_samples_per_second": 24.8,
+    "train_steps_per_second": 0.096
+}

runs/Nov22_17-00-23_dull-time-lies-fin-02/events.out.tfevents.1732295119.dull-time-lies-fin-02.48810.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8a7057b623d18d32ab6c96687d2fb8b2be9aad1be426b5c4833cf5ec1d85f6a8
-size 9104
+oid sha256:60b5d24b6be2bd2be64285bf2fa4ad3392bb4b37d4b716d019c786169dc61506
+size 9718

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9937888198757764,
+    "total_flos": 8.089924109389005e+17,
+    "train_loss": 1.786448487639427,
+    "train_runtime": 830.0773,
+    "train_samples": 51241,
+    "train_samples_per_second": 24.8,
+    "train_steps_per_second": 0.096
+}

trainer_state.json

@@ -0,0 +1,169 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9937888198757764,
+  "eval_steps": 500,
+  "global_step": 80,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.012422360248447204,
+      "grad_norm": 21.184524536132812,
+      "learning_rate": 2.5e-05,
+      "loss": 3.7651,
+      "step": 1
+    },
+    {
+      "epoch": 0.062111801242236024,
+      "grad_norm": 22.220394134521484,
+      "learning_rate": 0.000125,
+      "loss": 3.7771,
+      "step": 5
+    },
+    {
+      "epoch": 0.12422360248447205,
+      "grad_norm": 12.797124862670898,
+      "learning_rate": 0.00019961946980917456,
+      "loss": 3.031,
+      "step": 10
+    },
+    {
+      "epoch": 0.18633540372670807,
+      "grad_norm": 5.477193832397461,
+      "learning_rate": 0.0001953716950748227,
+      "loss": 2.3288,
+      "step": 15
+    },
+    {
+      "epoch": 0.2484472049689441,
+      "grad_norm": 2.4697020053863525,
+      "learning_rate": 0.00018660254037844388,
+      "loss": 1.938,
+      "step": 20
+    },
+    {
+      "epoch": 0.3105590062111801,
+      "grad_norm": 1.60536789894104,
+      "learning_rate": 0.0001737277336810124,
+      "loss": 1.6815,
+      "step": 25
+    },
+    {
+      "epoch": 0.37267080745341613,
+      "grad_norm": 0.9478007555007935,
+      "learning_rate": 0.0001573576436351046,
+      "loss": 1.5625,
+      "step": 30
+    },
+    {
+      "epoch": 0.43478260869565216,
+      "grad_norm": 0.5295054912567139,
+      "learning_rate": 0.000138268343236509,
+      "loss": 1.4703,
+      "step": 35
+    },
+    {
+      "epoch": 0.4968944099378882,
+      "grad_norm": 0.42334404587745667,
+      "learning_rate": 0.00011736481776669306,
+      "loss": 1.4589,
+      "step": 40
+    },
+    {
+      "epoch": 0.5590062111801242,
+      "grad_norm": 0.3187739849090576,
+      "learning_rate": 9.563806126346642e-05,
+      "loss": 1.4376,
+      "step": 45
+    },
+    {
+      "epoch": 0.6211180124223602,
+      "grad_norm": 0.3597185015678406,
+      "learning_rate": 7.411809548974792e-05,
+      "loss": 1.4271,
+      "step": 50
+    },
+    {
+      "epoch": 0.6832298136645962,
+      "grad_norm": 0.2957977056503296,
+      "learning_rate": 5.382513867649663e-05,
+      "loss": 1.4297,
+      "step": 55
+    },
+    {
+      "epoch": 0.7453416149068323,
+      "grad_norm": 0.29805850982666016,
+      "learning_rate": 3.5721239031346066e-05,
+      "loss": 1.4191,
+      "step": 60
+    },
+    {
+      "epoch": 0.8074534161490683,
+      "grad_norm": 0.27294573187828064,
+      "learning_rate": 2.0664665970876496e-05,
+      "loss": 1.403,
+      "step": 65
+    },
+    {
+      "epoch": 0.8695652173913043,
+      "grad_norm": 0.33928054571151733,
+      "learning_rate": 9.369221296335006e-06,
+      "loss": 1.4174,
+      "step": 70
+    },
+    {
+      "epoch": 0.9316770186335404,
+      "grad_norm": 0.31092315912246704,
+      "learning_rate": 2.3703992880066638e-06,
+      "loss": 1.3976,
+      "step": 75
+    },
+    {
+      "epoch": 0.9937888198757764,
+      "grad_norm": 0.2693531811237335,
+      "learning_rate": 0.0,
+      "loss": 1.4059,
+      "step": 80
+    },
+    {
+      "epoch": 0.9937888198757764,
+      "eval_loss": 1.4015706777572632,
+      "eval_runtime": 2.2495,
+      "eval_samples_per_second": 91.13,
+      "eval_steps_per_second": 0.889,
+      "step": 80
+    },
+    {
+      "epoch": 0.9937888198757764,
+      "step": 80,
+      "total_flos": 8.089924109389005e+17,
+      "train_loss": 1.786448487639427,
+      "train_runtime": 830.0773,
+      "train_samples_per_second": 24.8,
+      "train_steps_per_second": 0.096
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 80,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.089924109389005e+17,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}