initial commit

README.md

@@ -1,3 +1,73 @@
 ---
 license: mit
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: 22_12_13_luther_blocks_larger_fp16_20ep
+  results: []
 ---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 22_12_13_luther_blocks_larger_fp16_20ep
+
+This model is a fine-tuned version of [stefan-it/german-gpt2-larger](https://huggingface.co/stefan-it/german-gpt2-larger) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5847
+- Accuracy: 0.3168
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 20.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Accuracy |
+|:-------------:|:-----:|:----:|:---------------:|:--------:|
+| No log        | 1.6   | 50   | 4.6218          | 0.2156   |
+| 8.1175        | 3.22  | 100  | 4.0404          | 0.2633   |
+| 8.1175        | 4.83  | 150  | 3.8120          | 0.2871   |
+| 3.734         | 6.44  | 200  | 3.7062          | 0.2997   |
+| 3.734         | 8.06  | 250  | 3.6382          | 0.3082   |
+| 3.3639        | 9.67  | 300  | 3.6108          | 0.3128   |
+| 3.3639        | 11.29 | 350  | 3.6012          | 0.3148   |
+| 3.1363        | 12.89 | 400  | 3.5847          | 0.3168   |
+| 3.1363        | 14.51 | 450  | 3.5914          | 0.3180   |
+| 2.9884        | 16.13 | 500  | 3.5954          | 0.3177   |
+| 2.9884        | 17.73 | 550  | 3.6001          | 0.3176   |
+| 2.8748        | 19.35 | 600  | 3.6048          | 0.3188   |
+
+
+### Framework versions
+
+- Transformers 4.26.0.dev0
+- Pytorch 1.13.0
+- Datasets 2.7.1
+- Tokenizers 0.12.1

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<|endoftext|>": 50265
+}

all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 19.98,
+    "eval_accuracy": 0.31680776111695164,
+    "eval_loss": 3.5847387313842773,
+    "eval_runtime": 1.1014,
+    "eval_samples": 53,
+    "eval_samples_per_second": 48.123,
+    "eval_steps_per_second": 6.356,
+    "perplexity": 36.04393933841306,
+    "train_loss": 3.996271416448778,
+    "train_runtime": 495.8533,
+    "train_samples": 500,
+    "train_samples_per_second": 20.167,
+    "train_steps_per_second": 1.25
+}

config.json

@@ -0,0 +1,40 @@
+{
+  "_name_or_path": "stefan-it/german-gpt2-larger",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.0,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 50256,
+  "gradient_checkpointing": false,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.0,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.26.0.dev0",
+  "use_cache": true,
+  "vocab_size": 50266
+}

eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 19.98,
+    "eval_accuracy": 0.31680776111695164,
+    "eval_loss": 3.5847387313842773,
+    "eval_runtime": 1.1014,
+    "eval_samples": 53,
+    "eval_samples_per_second": 48.123,
+    "eval_steps_per_second": 6.356,
+    "perplexity": 36.04393933841306
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32252ab5a3e3e702e7b04b061799e5f111fb9ecd1c6cf27240073fa0fa22b56e
+size 510425661

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,10 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "name_or_path": "stefan-it/german-gpt2-larger",
+  "special_tokens_map_file": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 19.98,
+    "train_loss": 3.996271416448778,
+    "train_runtime": 495.8533,
+    "train_samples": 500,
+    "train_samples_per_second": 20.167,
+    "train_steps_per_second": 1.25
+}

trainer_state.json

@@ -0,0 +1,169 @@
+{
+  "best_metric": 3.5847387313842773,
+  "best_model_checkpoint": "./models/22_12_13_luther_blocks_larger_fp16_20ep/checkpoint-400",
+  "epoch": 19.984126984126984,
+  "global_step": 620,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.6,
+      "eval_accuracy": 0.21560707501060514,
+      "eval_loss": 4.621785640716553,
+      "eval_runtime": 1.1004,
+      "eval_samples_per_second": 48.163,
+      "eval_steps_per_second": 6.361,
+      "step": 50
+    },
+    {
+      "epoch": 3.22,
+      "learning_rate": 4.2338709677419356e-05,
+      "loss": 8.1175,
+      "step": 100
+    },
+    {
+      "epoch": 3.22,
+      "eval_accuracy": 0.263284088603626,
+      "eval_loss": 4.0403876304626465,
+      "eval_runtime": 1.1008,
+      "eval_samples_per_second": 48.145,
+      "eval_steps_per_second": 6.359,
+      "step": 100
+    },
+    {
+      "epoch": 4.83,
+      "eval_accuracy": 0.28707648610265774,
+      "eval_loss": 3.8119544982910156,
+      "eval_runtime": 1.1015,
+      "eval_samples_per_second": 48.117,
+      "eval_steps_per_second": 6.355,
+      "step": 150
+    },
+    {
+      "epoch": 6.44,
+      "learning_rate": 3.427419354838709e-05,
+      "loss": 3.734,
+      "step": 200
+    },
+    {
+      "epoch": 6.44,
+      "eval_accuracy": 0.299655102454859,
+      "eval_loss": 3.7062137126922607,
+      "eval_runtime": 1.1015,
+      "eval_samples_per_second": 48.116,
+      "eval_steps_per_second": 6.355,
+      "step": 200
+    },
+    {
+      "epoch": 8.06,
+      "eval_accuracy": 0.3081576569099393,
+      "eval_loss": 3.638169765472412,
+      "eval_runtime": 1.1008,
+      "eval_samples_per_second": 48.148,
+      "eval_steps_per_second": 6.359,
+      "step": 250
+    },
+    {
+      "epoch": 9.67,
+      "learning_rate": 2.620967741935484e-05,
+      "loss": 3.3639,
+      "step": 300
+    },
+    {
+      "epoch": 9.67,
+      "eval_accuracy": 0.3127685866578137,
+      "eval_loss": 3.610761880874634,
+      "eval_runtime": 1.1014,
+      "eval_samples_per_second": 48.121,
+      "eval_steps_per_second": 6.356,
+      "step": 300
+    },
+    {
+      "epoch": 11.29,
+      "eval_accuracy": 0.31479739574687843,
+      "eval_loss": 3.6012349128723145,
+      "eval_runtime": 1.1022,
+      "eval_samples_per_second": 48.084,
+      "eval_steps_per_second": 6.351,
+      "step": 350
+    },
+    {
+      "epoch": 12.89,
+      "learning_rate": 1.8145161290322583e-05,
+      "loss": 3.1363,
+      "step": 400
+    },
+    {
+      "epoch": 12.89,
+      "eval_accuracy": 0.31680776111695164,
+      "eval_loss": 3.5847387313842773,
+      "eval_runtime": 1.1008,
+      "eval_samples_per_second": 48.147,
+      "eval_steps_per_second": 6.359,
+      "step": 400
+    },
+    {
+      "epoch": 14.51,
+      "eval_accuracy": 0.31802504657039044,
+      "eval_loss": 3.5913662910461426,
+      "eval_runtime": 1.0976,
+      "eval_samples_per_second": 48.286,
+      "eval_steps_per_second": 6.377,
+      "step": 450
+    },
+    {
+      "epoch": 16.13,
+      "learning_rate": 1.0080645161290323e-05,
+      "loss": 2.9884,
+      "step": 500
+    },
+    {
+      "epoch": 16.13,
+      "eval_accuracy": 0.317711503347535,
+      "eval_loss": 3.5954136848449707,
+      "eval_runtime": 1.0979,
+      "eval_samples_per_second": 48.272,
+      "eval_steps_per_second": 6.376,
+      "step": 500
+    },
+    {
+      "epoch": 17.73,
+      "eval_accuracy": 0.317563953595603,
+      "eval_loss": 3.600076675415039,
+      "eval_runtime": 1.1024,
+      "eval_samples_per_second": 48.075,
+      "eval_steps_per_second": 6.35,
+      "step": 550
+    },
+    {
+      "epoch": 19.35,
+      "learning_rate": 2.0161290322580646e-06,
+      "loss": 2.8748,
+      "step": 600
+    },
+    {
+      "epoch": 19.35,
+      "eval_accuracy": 0.31879968276803333,
+      "eval_loss": 3.604796886444092,
+      "eval_runtime": 1.1023,
+      "eval_samples_per_second": 48.08,
+      "eval_steps_per_second": 6.35,
+      "step": 600
+    },
+    {
+      "epoch": 19.98,
+      "step": 620,
+      "total_flos": 5223750303744000.0,
+      "train_loss": 3.996271416448778,
+      "train_runtime": 495.8533,
+      "train_samples_per_second": 20.167,
+      "train_steps_per_second": 1.25
+    }
+  ],
+  "max_steps": 620,
+  "num_train_epochs": 20,
+  "total_flos": 5223750303744000.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b821c5ff7f91fcefe5c21e76a17570d6fa1489b2c8bccb6113364cc7c3bbd39
+size 3515