Training in progress, step 100, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94b8fc1c83955f38bddb966bb4385e12e4c06895b7796ebbb9b15fe43d32b466
+oid sha256:0c6e7ba7c860279e242c09bc79c672c890d885ea0dc4618e98239227fa4f688a
 size 579417298

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7734e7ec07842d84820832b42a77f3b72983981916b0f5a1484cab769858655e
+oid sha256:a165fbb58b2baccb6b76266343ab0a210716f7518ee69cdf68f5d4dd67a12076
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f23e2214bcafb439ebc7528dcc283ef6218d509a276c0baff0743503ecbe3d92
+oid sha256:49d60a69e2379be2053e816cbaff31e6c931b5922dd86c71c9eaf473299cbf62
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.004857355655581101,
+  "epoch": 0.006476474207441469,
   "eval_steps": 9,
-  "global_step": 75,
+  "global_step": 100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -604,6 +604,205 @@
       "learning_rate": 1.7860619515673033e-05,
       "loss": 0.0,
       "step": 75
+    },
+    {
+      "epoch": 0.0049221203976555165,
+      "grad_norm": NaN,
+      "learning_rate": 1.6543469682057106e-05,
+      "loss": 0.0,
+      "step": 76
+    },
+    {
+      "epoch": 0.004986885139729931,
+      "grad_norm": NaN,
+      "learning_rate": 1.526708147705013e-05,
+      "loss": 0.0,
+      "step": 77
+    },
+    {
+      "epoch": 0.005051649881804346,
+      "grad_norm": NaN,
+      "learning_rate": 1.4033009983067452e-05,
+      "loss": 0.0,
+      "step": 78
+    },
+    {
+      "epoch": 0.00511641462387876,
+      "grad_norm": NaN,
+      "learning_rate": 1.2842758726130283e-05,
+      "loss": 0.0,
+      "step": 79
+    },
+    {
+      "epoch": 0.005181179365953175,
+      "grad_norm": NaN,
+      "learning_rate": 1.1697777844051105e-05,
+      "loss": 0.0,
+      "step": 80
+    },
+    {
+      "epoch": 0.00524594410802759,
+      "grad_norm": NaN,
+      "learning_rate": 1.0599462319663905e-05,
+      "loss": 0.0,
+      "step": 81
+    },
+    {
+      "epoch": 0.00524594410802759,
+      "eval_loss": NaN,
+      "eval_runtime": 2478.6796,
+      "eval_samples_per_second": 5.246,
+      "eval_steps_per_second": 0.656,
+      "step": 81
+    },
+    {
+      "epoch": 0.005310708850102004,
+      "grad_norm": NaN,
+      "learning_rate": 9.549150281252633e-06,
+      "loss": 0.0,
+      "step": 82
+    },
+    {
+      "epoch": 0.005375473592176419,
+      "grad_norm": NaN,
+      "learning_rate": 8.548121372247918e-06,
+      "loss": 0.0,
+      "step": 83
+    },
+    {
+      "epoch": 0.005440238334250834,
+      "grad_norm": NaN,
+      "learning_rate": 7.597595192178702e-06,
+      "loss": 0.0,
+      "step": 84
+    },
+    {
+      "epoch": 0.005505003076325249,
+      "grad_norm": NaN,
+      "learning_rate": 6.698729810778065e-06,
+      "loss": 0.0,
+      "step": 85
+    },
+    {
+      "epoch": 0.005569767818399663,
+      "grad_norm": NaN,
+      "learning_rate": 5.852620357053651e-06,
+      "loss": 0.0,
+      "step": 86
+    },
+    {
+      "epoch": 0.005634532560474078,
+      "grad_norm": NaN,
+      "learning_rate": 5.060297685041659e-06,
+      "loss": 0.0,
+      "step": 87
+    },
+    {
+      "epoch": 0.005699297302548492,
+      "grad_norm": NaN,
+      "learning_rate": 4.322727117869951e-06,
+      "loss": 0.0,
+      "step": 88
+    },
+    {
+      "epoch": 0.005764062044622907,
+      "grad_norm": NaN,
+      "learning_rate": 3.6408072716606346e-06,
+      "loss": 0.0,
+      "step": 89
+    },
+    {
+      "epoch": 0.005828826786697322,
+      "grad_norm": NaN,
+      "learning_rate": 3.0153689607045845e-06,
+      "loss": 0.0,
+      "step": 90
+    },
+    {
+      "epoch": 0.005828826786697322,
+      "eval_loss": NaN,
+      "eval_runtime": 2477.0861,
+      "eval_samples_per_second": 5.249,
+      "eval_steps_per_second": 0.656,
+      "step": 90
+    },
+    {
+      "epoch": 0.005893591528771737,
+      "grad_norm": NaN,
+      "learning_rate": 2.4471741852423237e-06,
+      "loss": 0.0,
+      "step": 91
+    },
+    {
+      "epoch": 0.005958356270846152,
+      "grad_norm": NaN,
+      "learning_rate": 1.9369152030840556e-06,
+      "loss": 0.0,
+      "step": 92
+    },
+    {
+      "epoch": 0.006023121012920566,
+      "grad_norm": NaN,
+      "learning_rate": 1.4852136862001764e-06,
+      "loss": 0.0,
+      "step": 93
+    },
+    {
+      "epoch": 0.006087885754994981,
+      "grad_norm": NaN,
+      "learning_rate": 1.0926199633097157e-06,
+      "loss": 0.0,
+      "step": 94
+    },
+    {
+      "epoch": 0.006152650497069395,
+      "grad_norm": NaN,
+      "learning_rate": 7.596123493895991e-07,
+      "loss": 0.0,
+      "step": 95
+    },
+    {
+      "epoch": 0.00621741523914381,
+      "grad_norm": NaN,
+      "learning_rate": 4.865965629214819e-07,
+      "loss": 0.0,
+      "step": 96
+    },
+    {
+      "epoch": 0.0062821799812182244,
+      "grad_norm": NaN,
+      "learning_rate": 2.7390523158633554e-07,
+      "loss": 0.0,
+      "step": 97
+    },
+    {
+      "epoch": 0.0063469447232926395,
+      "grad_norm": NaN,
+      "learning_rate": 1.2179748700879012e-07,
+      "loss": 0.0,
+      "step": 98
+    },
+    {
+      "epoch": 0.006411709465367055,
+      "grad_norm": NaN,
+      "learning_rate": 3.04586490452119e-08,
+      "loss": 0.0,
+      "step": 99
+    },
+    {
+      "epoch": 0.006411709465367055,
+      "eval_loss": NaN,
+      "eval_runtime": 2475.2134,
+      "eval_samples_per_second": 5.253,
+      "eval_steps_per_second": 0.657,
+      "step": 99
+    },
+    {
+      "epoch": 0.006476474207441469,
+      "grad_norm": NaN,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "step": 100
     }
   ],
   "logging_steps": 1,
@@ -618,12 +817,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 9.7429223374848e+16,
+  "total_flos": 1.29905631166464e+17,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null