End of training

README.md

@@ -18,11 +18,11 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [riotu-lab/ArabianGPT-01B](https://huggingface.co/riotu-lab/ArabianGPT-01B) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6607
-- Bleu: 0.3968
-- Rouge1: 0.6232
-- Rouge2: 0.3746
-- Rougel: 0.6212
+- Loss: 0.6246
+- Bleu: 0.3877
+- Rouge1: 0.5958
+- Rouge2: 0.3370
+- Rougel: 0.5935
 
 ## Model description
 

all_results.json

@@ -1,19 +1,19 @@
 {
-    "epoch": 15.0,
-    "eval_bleu": 0.2678870499231116,
-    "eval_loss": 0.6265950798988342,
-    "eval_rouge1": 0.5977012354572853,
-    "eval_rouge2": 0.34430833134800065,
-    "eval_rougeL": 0.5958973349618409,
-    "eval_runtime": 17.8984,
-    "eval_samples": 5405,
-    "eval_samples_per_second": 301.982,
-    "eval_steps_per_second": 37.769,
-    "perplexity": 1.8712283369394682,
-    "total_flos": 2.118621118464e+16,
-    "train_loss": 0.058961041791913305,
-    "train_runtime": 1668.9384,
-    "train_samples": 21622,
-    "train_samples_per_second": 259.111,
-    "train_steps_per_second": 32.392
+    "epoch": 8.0,
+    "eval_bleu": 0.3877191285680082,
+    "eval_loss": 0.6245766282081604,
+    "eval_rouge1": 0.5957940125562868,
+    "eval_rouge2": 0.3370143004573494,
+    "eval_rougeL": 0.5934967085426222,
+    "eval_runtime": 17.5031,
+    "eval_samples": 5380,
+    "eval_samples_per_second": 307.375,
+    "eval_steps_per_second": 38.45,
+    "perplexity": 1.8674551622056355,
+    "total_flos": 1.1198453907456e+16,
+    "train_loss": 0.5107668242644979,
+    "train_runtime": 1728.8387,
+    "train_samples": 21429,
+    "train_samples_per_second": 247.901,
+    "train_steps_per_second": 30.992
 }

egy_training_log.txt

@@ -156,3 +156,5 @@ INFO:root:Epoch 7.0: Train Loss = 0.328, Eval Loss = 0.6383510231971741
 INFO:absl:Using default tokenizer.
 INFO:root:Epoch 8.0: Train Loss = 0.2832, Eval Loss = 0.6490957736968994
 INFO:absl:Using default tokenizer.
+INFO:__main__:*** Evaluate ***
+INFO:absl:Using default tokenizer.

eval_results.json

@@ -1,13 +1,13 @@
 {
-    "epoch": 15.0,
-    "eval_bleu": 0.2678870499231116,
-    "eval_loss": 0.6265950798988342,
-    "eval_rouge1": 0.5977012354572853,
-    "eval_rouge2": 0.34430833134800065,
-    "eval_rougeL": 0.5958973349618409,
-    "eval_runtime": 17.8984,
-    "eval_samples": 5405,
-    "eval_samples_per_second": 301.982,
-    "eval_steps_per_second": 37.769,
-    "perplexity": 1.8712283369394682
+    "epoch": 8.0,
+    "eval_bleu": 0.3877191285680082,
+    "eval_loss": 0.6245766282081604,
+    "eval_rouge1": 0.5957940125562868,
+    "eval_rouge2": 0.3370143004573494,
+    "eval_rougeL": 0.5934967085426222,
+    "eval_runtime": 17.5031,
+    "eval_samples": 5380,
+    "eval_samples_per_second": 307.375,
+    "eval_steps_per_second": 38.45,
+    "perplexity": 1.8674551622056355
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 15.0,
-    "total_flos": 2.118621118464e+16,
-    "train_loss": 0.058961041791913305,
-    "train_runtime": 1668.9384,
-    "train_samples": 21622,
-    "train_samples_per_second": 259.111,
-    "train_steps_per_second": 32.392
+    "epoch": 8.0,
+    "total_flos": 1.1198453907456e+16,
+    "train_loss": 0.5107668242644979,
+    "train_runtime": 1728.8387,
+    "train_samples": 21429,
+    "train_samples_per_second": 247.901,
+    "train_steps_per_second": 30.992
 }

trainer_state.json

@@ -1,310 +1,177 @@
 {
-  "best_metric": 0.6265950798988342,
-  "best_model_checkpoint": "/home/iais_marenpielka/Bouthaina/res_nw_dj/checkpoint-13515",
-  "epoch": 15.0,
+  "best_metric": 0.6245766282081604,
+  "best_model_checkpoint": "/home/iais_marenpielka/Bouthaina/res_nw_dj/checkpoint-8037",
+  "epoch": 8.0,
   "eval_steps": 500,
-  "global_step": 40545,
+  "global_step": 21432,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 1.0,
-      "grad_norm": 1.3912957906723022,
-      "learning_rate": 3.552372190826653e-05,
-      "loss": 1.2513,
-      "step": 2703
+      "grad_norm": 1.1291303634643555,
+      "learning_rate": 4.7947437829691034e-05,
+      "loss": 1.2336,
+      "step": 2679
     },
     {
       "epoch": 1.0,
-      "eval_bleu": 0.22005527068442493,
-      "eval_loss": 0.7111806869506836,
-      "eval_rouge1": 0.5171822713118965,
-      "eval_rouge2": 0.253317924025756,
-      "eval_rougeL": 0.5148202154962768,
-      "eval_runtime": 21.8652,
-      "eval_samples_per_second": 247.197,
-      "eval_steps_per_second": 30.917,
-      "step": 2703
+      "eval_bleu": 0.3526371485696729,
+      "eval_loss": 0.7061845660209656,
+      "eval_rouge1": 0.5198447201202445,
+      "eval_rouge2": 0.25470674188424197,
+      "eval_rougeL": 0.516986428368343,
+      "eval_runtime": 17.6993,
+      "eval_samples_per_second": 303.967,
+      "eval_steps_per_second": 38.024,
+      "step": 2679
     },
     {
       "epoch": 2.0,
-      "grad_norm": 1.4323362112045288,
-      "learning_rate": 1.7761860954133264e-05,
-      "loss": 0.6462,
-      "step": 5406
+      "grad_norm": 1.4575163125991821,
+      "learning_rate": 4.542388847023361e-05,
+      "loss": 0.634,
+      "step": 5358
     },
     {
       "epoch": 2.0,
-      "eval_bleu": 0.24508343939723337,
-      "eval_loss": 0.6569304466247559,
-      "eval_rouge1": 0.557883823201639,
-      "eval_rouge2": 0.2973836473978787,
-      "eval_rougeL": 0.5553146565104614,
-      "eval_runtime": 150.8243,
-      "eval_samples_per_second": 35.836,
-      "eval_steps_per_second": 4.482,
-      "step": 5406
+      "eval_bleu": 0.37564617349393215,
+      "eval_loss": 0.6422649025917053,
+      "eval_rouge1": 0.5739420026060218,
+      "eval_rouge2": 0.3114152854574803,
+      "eval_rougeL": 0.571389088656927,
+      "eval_runtime": 17.5523,
+      "eval_samples_per_second": 306.513,
+      "eval_steps_per_second": 38.343,
+      "step": 5358
     },
     {
       "epoch": 3.0,
-      "grad_norm": 1.2555302381515503,
-      "learning_rate": 0.0,
-      "loss": 0.5673,
-      "step": 8109
+      "grad_norm": 1.237191915512085,
+      "learning_rate": 4.290033911077619e-05,
+      "loss": 0.5299,
+      "step": 8037
     },
     {
       "epoch": 3.0,
-      "eval_bleu": 0.2518224574721449,
-      "eval_loss": 0.6498554348945618,
-      "eval_rouge1": 0.57015304568835,
-      "eval_rouge2": 0.3118394078609683,
-      "eval_rougeL": 0.5676740384593599,
-      "eval_runtime": 172.9357,
-      "eval_samples_per_second": 31.254,
-      "eval_steps_per_second": 3.909,
-      "step": 8109
+      "eval_bleu": 0.3877191285680082,
+      "eval_loss": 0.6245766282081604,
+      "eval_rouge1": 0.5957940125562868,
+      "eval_rouge2": 0.3370143004573494,
+      "eval_rougeL": 0.5934967085426222,
+      "eval_runtime": 17.5587,
+      "eval_samples_per_second": 306.401,
+      "eval_steps_per_second": 38.329,
+      "step": 8037
     },
     {
       "epoch": 4.0,
-      "grad_norm": 1.4183156490325928,
-      "learning_rate": 4.037341299477222e-05,
-      "loss": 0.556,
-      "step": 10812
+      "grad_norm": 1.5628466606140137,
+      "learning_rate": 4.0376789751318766e-05,
+      "loss": 0.4492,
+      "step": 10716
     },
     {
       "epoch": 4.0,
-      "eval_bleu": 0.25983253831381714,
-      "eval_loss": 0.63295978307724,
-      "eval_rouge1": 0.5829310998998971,
-      "eval_rouge2": 0.326193735265751,
-      "eval_rougeL": 0.5807349807527458,
-      "eval_runtime": 40.5509,
-      "eval_samples_per_second": 133.289,
-      "eval_steps_per_second": 16.67,
-      "step": 10812
+      "eval_bleu": 0.39048560865149107,
+      "eval_loss": 0.6246171593666077,
+      "eval_rouge1": 0.6081397458304423,
+      "eval_rouge2": 0.3525706786064172,
+      "eval_rougeL": 0.6056655214414464,
+      "eval_runtime": 17.6128,
+      "eval_samples_per_second": 305.459,
+      "eval_steps_per_second": 38.211,
+      "step": 10716
     },
     {
       "epoch": 5.0,
-      "grad_norm": 1.186664342880249,
-      "learning_rate": 3.785007468259896e-05,
-      "loss": 0.4786,
-      "step": 13515
+      "grad_norm": 1.8470176458358765,
+      "learning_rate": 3.785324039186134e-05,
+      "loss": 0.3829,
+      "step": 13395
     },
     {
       "epoch": 5.0,
-      "eval_bleu": 0.2678870499231116,
-      "eval_loss": 0.6265950798988342,
-      "eval_rouge1": 0.5977012354572853,
-      "eval_rouge2": 0.34430833134800065,
-      "eval_rougeL": 0.5958973349618409,
-      "eval_runtime": 36.0053,
-      "eval_samples_per_second": 150.117,
-      "eval_steps_per_second": 18.775,
-      "step": 13515
+      "eval_bleu": 0.3963490407851369,
+      "eval_loss": 0.6300457715988159,
+      "eval_rouge1": 0.6145424292978614,
+      "eval_rouge2": 0.3620910031723723,
+      "eval_rougeL": 0.6124537005851034,
+      "eval_runtime": 17.4259,
+      "eval_samples_per_second": 308.735,
+      "eval_steps_per_second": 38.621,
+      "step": 13395
     },
     {
       "epoch": 6.0,
-      "grad_norm": 1.1597645282745361,
-      "learning_rate": 3.5326736370425696e-05,
-      "loss": 0.4123,
-      "step": 16218
+      "grad_norm": 1.5765687227249146,
+      "learning_rate": 3.532969103240392e-05,
+      "loss": 0.328,
+      "step": 16074
     },
     {
       "epoch": 6.0,
-      "eval_bleu": 0.2707063976336222,
-      "eval_loss": 0.630312979221344,
-      "eval_rouge1": 0.604436172840783,
-      "eval_rouge2": 0.35482389915380186,
-      "eval_rougeL": 0.6026792061487433,
-      "eval_runtime": 161.1219,
-      "eval_samples_per_second": 33.546,
-      "eval_steps_per_second": 4.196,
-      "step": 16218
+      "eval_bleu": 0.3960700684284105,
+      "eval_loss": 0.6383510231971741,
+      "eval_rouge1": 0.6213431858539703,
+      "eval_rouge2": 0.36997070576967905,
+      "eval_rougeL": 0.6194073222098655,
+      "eval_runtime": 17.7673,
+      "eval_samples_per_second": 302.803,
+      "eval_steps_per_second": 37.879,
+      "step": 16074
     },
     {
       "epoch": 7.0,
-      "grad_norm": 1.27555251121521,
-      "learning_rate": 3.280339805825243e-05,
-      "loss": 0.3573,
-      "step": 18921
+      "grad_norm": 1.960555076599121,
+      "learning_rate": 3.28061416729465e-05,
+      "loss": 0.2832,
+      "step": 18753
     },
     {
       "epoch": 7.0,
-      "eval_bleu": 0.27393786087649125,
-      "eval_loss": 0.6372247338294983,
-      "eval_rouge1": 0.6108088318784769,
-      "eval_rouge2": 0.36307215008060617,
-      "eval_rougeL": 0.6088126794948809,
-      "eval_runtime": 159.7397,
-      "eval_samples_per_second": 33.836,
-      "eval_steps_per_second": 4.232,
-      "step": 18921
+      "eval_bleu": 0.3998604582987598,
+      "eval_loss": 0.6490957736968994,
+      "eval_rouge1": 0.623202150680765,
+      "eval_rouge2": 0.3741196322885935,
+      "eval_rougeL": 0.6209434708040459,
+      "eval_runtime": 17.3486,
+      "eval_samples_per_second": 310.112,
+      "eval_steps_per_second": 38.793,
+      "step": 18753
     },
     {
       "epoch": 8.0,
-      "grad_norm": 1.6310491561889648,
-      "learning_rate": 3.0280059746079166e-05,
-      "loss": 0.3108,
-      "step": 21624
+      "grad_norm": 1.4162492752075195,
+      "learning_rate": 3.028259231348907e-05,
+      "loss": 0.2453,
+      "step": 21432
     },
     {
       "epoch": 8.0,
-      "eval_bleu": 0.2759919170184585,
-      "eval_loss": 0.646577000617981,
-      "eval_rouge1": 0.6130373826214599,
-      "eval_rouge2": 0.36776670773707465,
-      "eval_rougeL": 0.6111699095881062,
-      "eval_runtime": 150.1086,
-      "eval_samples_per_second": 36.007,
-      "eval_steps_per_second": 4.503,
-      "step": 21624
+      "eval_bleu": 0.3967586062831279,
+      "eval_loss": 0.660749614238739,
+      "eval_rouge1": 0.6231513583077875,
+      "eval_rouge2": 0.37461482860421447,
+      "eval_rougeL": 0.6211860899632824,
+      "eval_runtime": 17.8295,
+      "eval_samples_per_second": 301.747,
+      "eval_steps_per_second": 37.746,
+      "step": 21432
     },
     {
-      "epoch": 9.0,
-      "grad_norm": 1.6629250049591064,
-      "learning_rate": 2.77567214339059e-05,
-      "loss": 0.2719,
-      "step": 24327
-    },
-    {
-      "epoch": 9.0,
-      "eval_bleu": 0.2801810856751608,
-      "eval_loss": 0.6550981998443604,
-      "eval_rouge1": 0.6177880665801061,
-      "eval_rouge2": 0.3727206564119486,
-      "eval_rougeL": 0.6159303776582042,
-      "eval_runtime": 159.1145,
-      "eval_samples_per_second": 33.969,
-      "eval_steps_per_second": 4.249,
-      "step": 24327
-    },
-    {
-      "epoch": 10.0,
-      "grad_norm": 1.5184797048568726,
-      "learning_rate": 2.523338312173264e-05,
-      "loss": 0.2396,
-      "step": 27030
-    },
-    {
-      "epoch": 10.0,
-      "eval_bleu": 0.27697504925957206,
-      "eval_loss": 0.665830671787262,
-      "eval_rouge1": 0.6173182845826588,
-      "eval_rouge2": 0.37288754745117264,
-      "eval_rougeL": 0.6152835100872982,
-      "eval_runtime": 110.2964,
-      "eval_samples_per_second": 49.004,
-      "eval_steps_per_second": 6.129,
-      "step": 27030
-    },
-    {
-      "epoch": 11.0,
-      "grad_norm": 1.3458495140075684,
-      "learning_rate": 2.2710044809559374e-05,
-      "loss": 0.2129,
-      "step": 29733
-    },
-    {
-      "epoch": 11.0,
-      "eval_bleu": 0.27843197191410246,
-      "eval_loss": 0.6767598390579224,
-      "eval_rouge1": 0.6188286228800556,
-      "eval_rouge2": 0.3762109244532287,
-      "eval_rougeL": 0.617052574223907,
-      "eval_runtime": 24.0534,
-      "eval_samples_per_second": 224.709,
-      "eval_steps_per_second": 28.104,
-      "step": 29733
-    },
-    {
-      "epoch": 12.0,
-      "grad_norm": 1.636675477027893,
-      "learning_rate": 2.018670649738611e-05,
-      "loss": 0.191,
-      "step": 32436
-    },
-    {
-      "epoch": 12.0,
-      "eval_bleu": 0.2780108798800892,
-      "eval_loss": 0.6870447993278503,
-      "eval_rouge1": 0.6208121821010748,
-      "eval_rouge2": 0.37810190638421814,
-      "eval_rougeL": 0.6189436880506437,
-      "eval_runtime": 42.9158,
-      "eval_samples_per_second": 125.944,
-      "eval_steps_per_second": 15.752,
-      "step": 32436
-    },
-    {
-      "epoch": 13.0,
-      "grad_norm": 1.3978990316390991,
-      "learning_rate": 1.7663368185212848e-05,
-      "loss": 0.1733,
-      "step": 35139
-    },
-    {
-      "epoch": 13.0,
-      "eval_bleu": 0.2799527424248887,
-      "eval_loss": 0.696670651435852,
-      "eval_rouge1": 0.6213835516562576,
-      "eval_rouge2": 0.3799254363900967,
-      "eval_rougeL": 0.6195026516671875,
-      "eval_runtime": 17.6056,
-      "eval_samples_per_second": 307.005,
-      "eval_steps_per_second": 38.397,
-      "step": 35139
-    },
-    {
-      "epoch": 14.0,
-      "grad_norm": 1.386664628982544,
-      "learning_rate": 1.5140029873039583e-05,
-      "loss": 0.1593,
-      "step": 37842
-    },
-    {
-      "epoch": 14.0,
-      "eval_bleu": 0.2790160057234741,
-      "eval_loss": 0.7063180208206177,
-      "eval_rouge1": 0.6214879203521921,
-      "eval_rouge2": 0.379862056883408,
-      "eval_rougeL": 0.6194802915698101,
-      "eval_runtime": 17.8263,
-      "eval_samples_per_second": 303.204,
-      "eval_steps_per_second": 37.921,
-      "step": 37842
-    },
-    {
-      "epoch": 15.0,
-      "grad_norm": 1.5141432285308838,
-      "learning_rate": 1.261669156086632e-05,
-      "loss": 0.1478,
-      "step": 40545
-    },
-    {
-      "epoch": 15.0,
-      "eval_bleu": 0.2808668290315019,
-      "eval_loss": 0.7138631939888,
-      "eval_rouge1": 0.6216459883291815,
-      "eval_rouge2": 0.3804311054098596,
-      "eval_rougeL": 0.6196983257570402,
-      "eval_runtime": 17.6502,
-      "eval_samples_per_second": 306.229,
-      "eval_steps_per_second": 38.3,
-      "step": 40545
-    },
-    {
-      "epoch": 15.0,
-      "step": 40545,
-      "total_flos": 2.118621118464e+16,
-      "train_loss": 0.058961041791913305,
-      "train_runtime": 1668.9384,
-      "train_samples_per_second": 259.111,
-      "train_steps_per_second": 32.392
+      "epoch": 8.0,
+      "step": 21432,
+      "total_flos": 1.1198453907456e+16,
+      "train_loss": 0.5107668242644979,
+      "train_runtime": 1728.8387,
+      "train_samples_per_second": 247.901,
+      "train_steps_per_second": 30.992
     }
   ],
   "logging_steps": 500,
-  "max_steps": 54060,
+  "max_steps": 53580,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 20,
   "save_steps": 500,
@@ -329,7 +196,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.118621118464e+16,
+  "total_flos": 1.1198453907456e+16,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null