End of training

all_results.json

@@ -1,19 +1,19 @@
 {
-    "epoch": 1.0,
-    "eval_bleu": 0.09842217214841122,
-    "eval_loss": 3.4630110263824463,
-    "eval_rouge1": 0.3093411104418377,
-    "eval_rouge2": 0.07177511381538038,
-    "eval_rougeL": 0.22957881226204224,
-    "eval_runtime": 27.2297,
+    "epoch": 5.0,
+    "eval_bleu": 0.16068905926811505,
+    "eval_loss": 2.6229476928710938,
+    "eval_rouge1": 0.4264320027787866,
+    "eval_rouge2": 0.1630682859845051,
+    "eval_rougeL": 0.367472815476786,
+    "eval_runtime": 27.1354,
     "eval_samples": 847,
-    "eval_samples_per_second": 31.106,
-    "eval_steps_per_second": 3.893,
-    "perplexity": 31.912922646039927,
-    "total_flos": 1000225898496000.0,
-    "train_loss": 0.0,
-    "train_runtime": 0.0037,
+    "eval_samples_per_second": 31.214,
+    "eval_steps_per_second": 3.906,
+    "perplexity": 13.77627201236392,
+    "total_flos": 5001129492480000.0,
+    "train_loss": 2.2704222102150275,
+    "train_runtime": 565.3472,
     "train_samples": 2552,
-    "train_samples_per_second": 699005.016,
-    "train_steps_per_second": 87375.627
+    "train_samples_per_second": 22.57,
+    "train_steps_per_second": 2.821
 }

egy_training_log.txt

@@ -440,3 +440,5 @@ INFO:absl:Using default tokenizer.
 INFO:root:Epoch 4.0: Train Loss = 2.9208, Eval Loss = 2.729825496673584
 INFO:absl:Using default tokenizer.
 INFO:root:Epoch 5.0: Train Loss = 2.619, Eval Loss = 2.6229476928710938
+INFO:__main__:*** Evaluate ***
+INFO:absl:Using default tokenizer.

eval_results.json

@@ -1,13 +1,13 @@
 {
-    "epoch": 1.0,
-    "eval_bleu": 0.09842217214841122,
-    "eval_loss": 3.4630110263824463,
-    "eval_rouge1": 0.3093411104418377,
-    "eval_rouge2": 0.07177511381538038,
-    "eval_rougeL": 0.22957881226204224,
-    "eval_runtime": 27.2297,
+    "epoch": 5.0,
+    "eval_bleu": 0.16068905926811505,
+    "eval_loss": 2.6229476928710938,
+    "eval_rouge1": 0.4264320027787866,
+    "eval_rouge2": 0.1630682859845051,
+    "eval_rougeL": 0.367472815476786,
+    "eval_runtime": 27.1354,
     "eval_samples": 847,
-    "eval_samples_per_second": 31.106,
-    "eval_steps_per_second": 3.893,
-    "perplexity": 31.912922646039927
+    "eval_samples_per_second": 31.214,
+    "eval_steps_per_second": 3.906,
+    "perplexity": 13.77627201236392
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 1.0,
-    "total_flos": 1000225898496000.0,
-    "train_loss": 0.0,
-    "train_runtime": 0.0037,
+    "epoch": 5.0,
+    "total_flos": 5001129492480000.0,
+    "train_loss": 2.2704222102150275,
+    "train_runtime": 565.3472,
     "train_samples": 2552,
-    "train_samples_per_second": 699005.016,
-    "train_steps_per_second": 87375.627
+    "train_samples_per_second": 22.57,
+    "train_steps_per_second": 2.821
 }

trainer_state.json

@@ -1,27 +1,84 @@
 {
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "best_metric": 2.6229476928710938,
+  "best_model_checkpoint": "/home/iais_marenpielka/Bouthaina/results/checkpoint-1500",
+  "epoch": 5.0,
   "eval_steps": 500,
-  "global_step": 319,
+  "global_step": 1595,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 1.0,
-      "step": 319,
-      "total_flos": 1000225898496000.0,
-      "train_loss": 0.0,
-      "train_runtime": 0.0037,
-      "train_samples_per_second": 699005.016,
-      "train_steps_per_second": 87375.627
+      "epoch": 1.567398119122257,
+      "grad_norm": 1.353641152381897,
+      "learning_rate": 5e-05,
+      "loss": 3.359,
+      "step": 500
+    },
+    {
+      "epoch": 1.567398119122257,
+      "eval_bleu": 0.11424038411303619,
+      "eval_loss": 3.128293514251709,
+      "eval_rouge1": 0.3297614987151056,
+      "eval_rouge2": 0.08429294540985294,
+      "eval_rougeL": 0.2561476738686219,
+      "eval_runtime": 26.8133,
+      "eval_samples_per_second": 31.589,
+      "eval_steps_per_second": 3.953,
+      "step": 500
+    },
+    {
+      "epoch": 3.134796238244514,
+      "grad_norm": 1.156111717224121,
+      "learning_rate": 2.71689497716895e-05,
+      "loss": 2.9208,
+      "step": 1000
+    },
+    {
+      "epoch": 3.134796238244514,
+      "eval_bleu": 0.1490666503828191,
+      "eval_loss": 2.729825496673584,
+      "eval_rouge1": 0.40409071928626966,
+      "eval_rouge2": 0.14297878002377568,
+      "eval_rougeL": 0.34083403761346187,
+      "eval_runtime": 27.234,
+      "eval_samples_per_second": 31.101,
+      "eval_steps_per_second": 3.892,
+      "step": 1000
+    },
+    {
+      "epoch": 4.702194357366771,
+      "grad_norm": 1.1165863275527954,
+      "learning_rate": 4.337899543378996e-06,
+      "loss": 2.619,
+      "step": 1500
+    },
+    {
+      "epoch": 4.702194357366771,
+      "eval_bleu": 0.16068905926811505,
+      "eval_loss": 2.6229476928710938,
+      "eval_rouge1": 0.4264320027787866,
+      "eval_rouge2": 0.1630682859845051,
+      "eval_rougeL": 0.367472815476786,
+      "eval_runtime": 27.3027,
+      "eval_samples_per_second": 31.023,
+      "eval_steps_per_second": 3.882,
+      "step": 1500
+    },
+    {
+      "epoch": 5.0,
+      "step": 1595,
+      "total_flos": 5001129492480000.0,
+      "train_loss": 2.2704222102150275,
+      "train_runtime": 565.3472,
+      "train_samples_per_second": 22.57,
+      "train_steps_per_second": 2.821
     }
   ],
   "logging_steps": 500,
-  "max_steps": 319,
+  "max_steps": 1595,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 5,
   "save_steps": 500,
   "stateful_callbacks": {
     "EarlyStoppingCallback": {
@@ -44,7 +101,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1000225898496000.0,
+  "total_flos": 5001129492480000.0,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null