huggingartists

README.md

@@ -14,7 +14,7 @@ widget:
 <div class="inline-flex flex-col" style="line-height: 1.5;">
     <div class="flex">
         <div
-			style="display:DISPLAY_1; margin-left: auto; margin-right: auto; width: 92px; height:92px; border-radius: 50%; background-size: cover; background-image: url(&#39;https://images.genius.com/5cad1fbda7135f40c49ea35ad60ba4f5.215x215x1.png&#39;)">
+			style="display:DISPLAY_1; margin-left: auto; margin-right: auto; width: 92px; height:92px; border-radius: 50%; background-size: cover; background-image: url(&#39;https://images.genius.com/e4051ba629f57622a2a35571e5676824.720x720x1.jpg&#39;)">
         </div>
     </div>
     <div style="text-align: center; margin-top: 3px; font-size: 16px; font-weight: 800">🤖 HuggingArtists Model 🤖</div>
@@ -45,15 +45,15 @@ from datasets import load_dataset
 dataset = load_dataset("huggingartists/madonna")
 ```
 
-[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/2eqc1vv3/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
+[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/1zgkgxrp/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
 
 ## Training procedure
 
 The model is based on a pre-trained [GPT-2](https://huggingface.co/gpt2) which is fine-tuned on Madonna's lyrics.
 
-Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/ppm2eyoj) for full transparency and reproducibility.
+Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/1nejy753) for full transparency and reproducibility.
 
-At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/ppm2eyoj/artifacts) is logged and versioned.
+At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/1nejy753/artifacts) is logged and versioned.
 
 ## How to use
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "gpt2",
+  "_name_or_path": "madonna",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"
@@ -18,7 +18,9 @@
   "n_inner": null,
   "n_layer": 12,
   "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
   "summary_first_dropout": 0.1,
@@ -35,7 +37,7 @@
     }
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.10.2",
+  "transformers_version": "4.15.0",
   "use_cache": true,
   "vocab_size": 50257
 }

evaluation.txt

@@ -1 +1 @@
-{"eval_loss": 3.1591076850891113, "eval_runtime": 3.3487, "eval_samples_per_second": 21.202, "eval_steps_per_second": 2.688, "epoch": 1.0}
+{"eval_loss": 1.5412778854370117, "eval_runtime": 17.7742, "eval_samples_per_second": 17.272, "eval_steps_per_second": 2.194, "epoch": 2.0}

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9b6a9dde4f77fe7e5286411ef04c90642d3d5632397b8526fcc446a7864b4f1
+oid sha256:25566d27aa0a638f5602254e69031264732abeaa33b25755bd5b7e9febdce8f7
 size 497764120

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ceae229ca904bed5d1bb6cd9932ed1f1ce4b1f959602365ba45f365e7ae157b8
-size 995603825
+oid sha256:dc55df2228257ecd9fbf03f2979f7659de46424b72ffa514ec925c65c4ceaf50
+size 995604017

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a28db54f1ddd728c41eb92a39efd92a5721e39626fd0317625976c4d82ef50fb
+oid sha256:cffe2698db26bffad1da8bea41d71775f16f1b35e022d24c8c80b1ca1297a533
 size 510403817

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d2957881bb5b55492f8fff01b392d863d09b0e66192bf730f50b9b2472170d6
+oid sha256:0706774df0979b928412046725f3584988207df65c21b575ba45213c1e46e201
 size 14503

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:33ebdf640e584fb96b06bd917ad6a00182e9ef4871b5e81f588509ccfb511410
+oid sha256:73d66e37400481ce71994edfed806089e04a5cb8f448f68f1ec267b591d1dbe6
 size 623

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "huggingartists/zemfira", "tokenizer_class": "GPT2Tokenizer"}
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "huggingartists/madonna", "tokenizer_class": "GPT2Tokenizer"}

trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 1.924851655960083,
-  "best_model_checkpoint": "output/madonna/checkpoint-222",
-  "epoch": 1.0,
-  "global_step": 222,
+  "best_metric": 1.5412778854370117,
+  "best_model_checkpoint": "output/madonna/checkpoint-448",
+  "epoch": 2.0,
+  "global_step": 448,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -278,11 +278,297 @@
       "eval_samples_per_second": 22.672,
       "eval_steps_per_second": 2.887,
       "step": 222
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 1.7440342903137207,
+      "eval_runtime": 17.6835,
+      "eval_samples_per_second": 17.361,
+      "eval_steps_per_second": 2.205,
+      "step": 224
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 6.746689293153874e-09,
+      "loss": 2.1578,
+      "step": 225
+    },
+    {
+      "epoch": 1.03,
+      "learning_rate": 2.427415035313232e-07,
+      "loss": 1.867,
+      "step": 230
+    },
+    {
+      "epoch": 1.05,
+      "learning_rate": 8.147449052190903e-07,
+      "loss": 1.7004,
+      "step": 235
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 1.7199452243268996e-06,
+      "loss": 1.8748,
+      "step": 240
+    },
+    {
+      "epoch": 1.09,
+      "learning_rate": 2.9538929687704672e-06,
+      "loss": 1.7175,
+      "step": 245
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 4.510522695790453e-06,
+      "loss": 1.9134,
+      "step": 250
+    },
+    {
+      "epoch": 1.14,
+      "learning_rate": 6.382182826501689e-06,
+      "loss": 1.8779,
+      "step": 255
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 8.559673257059505e-06,
+      "loss": 1.9084,
+      "step": 260
+    },
+    {
+      "epoch": 1.18,
+      "learning_rate": 1.1032290581565944e-05,
+      "loss": 1.7949,
+      "step": 265
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 1.3787880704424101e-05,
+      "loss": 1.7769,
+      "step": 270
+    },
+    {
+      "epoch": 1.23,
+      "learning_rate": 1.681289858352526e-05,
+      "loss": 1.724,
+      "step": 275
+    },
+    {
+      "epoch": 1.25,
+      "learning_rate": 2.009247481060283e-05,
+      "loss": 1.8311,
+      "step": 280
+    },
+    {
+      "epoch": 1.27,
+      "learning_rate": 2.3610488701478854e-05,
+      "loss": 1.7348,
+      "step": 285
+    },
+    {
+      "epoch": 1.29,
+      "learning_rate": 2.734964753692934e-05,
+      "loss": 1.768,
+      "step": 290
+    },
+    {
+      "epoch": 1.32,
+      "learning_rate": 3.129157156466198e-05,
+      "loss": 1.7488,
+      "step": 295
+    },
+    {
+      "epoch": 1.34,
+      "learning_rate": 3.541688434458052e-05,
+      "loss": 1.9316,
+      "step": 300
+    },
+    {
+      "epoch": 1.36,
+      "learning_rate": 3.970530799324656e-05,
+      "loss": 1.7607,
+      "step": 305
+    },
+    {
+      "epoch": 1.38,
+      "learning_rate": 4.413576285936533e-05,
+      "loss": 1.6681,
+      "step": 310
+    },
+    {
+      "epoch": 1.41,
+      "learning_rate": 4.8686471140343875e-05,
+      "loss": 2.065,
+      "step": 315
+    },
+    {
+      "epoch": 1.43,
+      "learning_rate": 5.333506393059682e-05,
+      "loss": 1.848,
+      "step": 320
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 5.805869117540521e-05,
+      "loss": 1.8802,
+      "step": 325
+    },
+    {
+      "epoch": 1.47,
+      "learning_rate": 6.283413398985365e-05,
+      "loss": 1.7551,
+      "step": 330
+    },
+    {
+      "epoch": 1.5,
+      "learning_rate": 6.763791879074358e-05,
+      "loss": 1.7544,
+      "step": 335
+    },
+    {
+      "epoch": 1.52,
+      "learning_rate": 7.244643268047132e-05,
+      "loss": 1.793,
+      "step": 340
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 7.723603951570171e-05,
+      "loss": 1.7917,
+      "step": 345
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 8.19831960903064e-05,
+      "loss": 1.63,
+      "step": 350
+    },
+    {
+      "epoch": 1.58,
+      "learning_rate": 8.666456786147141e-05,
+      "loss": 2.0457,
+      "step": 355
+    },
+    {
+      "epoch": 1.61,
+      "learning_rate": 9.125714365012444e-05,
+      "loss": 2.0322,
+      "step": 360
+    },
+    {
+      "epoch": 1.63,
+      "learning_rate": 9.5738348751875e-05,
+      "loss": 1.687,
+      "step": 365
+    },
+    {
+      "epoch": 1.65,
+      "learning_rate": 0.00010008615590247305,
+      "loss": 1.9025,
+      "step": 370
+    },
+    {
+      "epoch": 1.67,
+      "learning_rate": 0.00010427919355233763,
+      "loss": 1.7503,
+      "step": 375
+    },
+    {
+      "epoch": 1.7,
+      "learning_rate": 0.00010829685091793463,
+      "loss": 1.7749,
+      "step": 380
+    },
+    {
+      "epoch": 1.72,
+      "learning_rate": 0.00011211937929362609,
+      "loss": 1.6852,
+      "step": 385
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 0.00011572798912599424,
+      "loss": 1.8013,
+      "step": 390
+    },
+    {
+      "epoch": 1.76,
+      "learning_rate": 0.00011910494237347491,
+      "loss": 1.5069,
+      "step": 395
+    },
+    {
+      "epoch": 1.79,
+      "learning_rate": 0.00012223363969730684,
+      "loss": 1.5859,
+      "step": 400
+    },
+    {
+      "epoch": 1.81,
+      "learning_rate": 0.00012509870205521334,
+      "loss": 1.5894,
+      "step": 405
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 0.00012768604629674322,
+      "loss": 1.9085,
+      "step": 410
+    },
+    {
+      "epoch": 1.85,
+      "learning_rate": 0.00012998295438868449,
+      "loss": 1.8982,
+      "step": 415
+    },
+    {
+      "epoch": 1.88,
+      "learning_rate": 0.00013197813593027427,
+      "loss": 1.9507,
+      "step": 420
+    },
+    {
+      "epoch": 1.9,
+      "learning_rate": 0.00013366178365091335,
+      "loss": 1.7504,
+      "step": 425
+    },
+    {
+      "epoch": 1.92,
+      "learning_rate": 0.00013502562161758715,
+      "loss": 1.7843,
+      "step": 430
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 0.00013606294591503093,
+      "loss": 1.6083,
+      "step": 435
+    },
+    {
+      "epoch": 1.96,
+      "learning_rate": 0.00013676865759867644,
+      "loss": 1.5864,
+      "step": 440
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 0.00013713928775840084,
+      "loss": 1.8849,
+      "step": 445
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 1.5412778854370117,
+      "eval_runtime": 17.7637,
+      "eval_samples_per_second": 17.282,
+      "eval_steps_per_second": 2.195,
+      "step": 448
     }
   ],
-  "max_steps": 222,
-  "num_train_epochs": 1,
-  "total_flos": 232027324416000.0,
+  "max_steps": 448,
+  "num_train_epochs": 2,
+  "total_flos": 467712737280000.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83c9e6cfb63c365b4c49a065df2aef2c18c8bd82cedbf8864df569b4417ff289
-size 2671
+oid sha256:eaf00d98777c40d1ee6ae43b9a54114156e44aec0020e9e8e22c522be0398be6
+size 2991