Training in progress, step 50, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9acf4f3d26f7b4fd54998a9515beea83b2d039657976d2c69d01f9b1d9b8c205
+oid sha256:71713e17474dda25d53af38ac36b303d61cb779be838011ad46a2ba2ecffc24b
 size 640009682

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9ce67f7c69421c2d00d3c1db1e7d3417c96c46b3b9153606662a24c19ecc914
+oid sha256:a6a8c3ef83e435343108acd5bb1960963c4432ee4ee3e72258a0ac4800f5d794
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4097a9f2d88f3c47734eb83bf0951871b26242c422687abefe02f5d3b95cf86b
+oid sha256:6c65173b3f64f021b2c16dcc2bf76c0722fad39c34f53e4d1de23db6b75754be
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9bd8a8a97716817b266fedf98936a149169e7d5cee48034394464057072acc4
+oid sha256:ff3a014be90a86f46d173d0d9cfe7a414eea131535a6c2d1e76884197b987b7f
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9a23d230180b49b170f11362cb69fc1771dab554c0071682174d692b1faa3909
+oid sha256:3cef2eaa60f500d63daccbe25adcec3b6fc23114912db1c3e624737f047ccaf9
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5761536cc8e255d80fe4c4ee0d2510adf38c0eae13f989d5464f4214794de9bd
+oid sha256:d973d18023ad83a6f250eae4b517afb43e4909da13d79036a60c34b2f251a741
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": NaN,
   "best_model_checkpoint": "miner_id_24/checkpoint-25",
-  "epoch": 1.2818471337579618,
+  "epoch": 2.5636942675159236,
   "eval_steps": 25,
-  "global_step": 25,
+  "global_step": 50,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -198,6 +198,189 @@
       "eval_samples_per_second": 33.812,
       "eval_steps_per_second": 8.791,
       "step": 25
+    },
+    {
+      "epoch": 1.3328025477707006,
+      "grad_norm": NaN,
+      "learning_rate": 6.604684692133597e-05,
+      "loss": 0.0,
+      "step": 26
+    },
+    {
+      "epoch": 1.3837579617834395,
+      "grad_norm": NaN,
+      "learning_rate": 6.362697843656823e-05,
+      "loss": 0.0,
+      "step": 27
+    },
+    {
+      "epoch": 1.4347133757961783,
+      "grad_norm": NaN,
+      "learning_rate": 6.118091010182837e-05,
+      "loss": 0.0,
+      "step": 28
+    },
+    {
+      "epoch": 1.4856687898089171,
+      "grad_norm": NaN,
+      "learning_rate": 5.8716070546254966e-05,
+      "loss": 0.0,
+      "step": 29
+    },
+    {
+      "epoch": 1.5366242038216562,
+      "grad_norm": NaN,
+      "learning_rate": 5.62399454065673e-05,
+      "loss": 0.0,
+      "step": 30
+    },
+    {
+      "epoch": 1.587579617834395,
+      "grad_norm": NaN,
+      "learning_rate": 5.376005459343272e-05,
+      "loss": 0.0,
+      "step": 31
+    },
+    {
+      "epoch": 1.638535031847134,
+      "grad_norm": NaN,
+      "learning_rate": 5.128392945374505e-05,
+      "loss": 0.0,
+      "step": 32
+    },
+    {
+      "epoch": 1.6894904458598727,
+      "grad_norm": NaN,
+      "learning_rate": 4.881908989817163e-05,
+      "loss": 0.0,
+      "step": 33
+    },
+    {
+      "epoch": 1.7404458598726116,
+      "grad_norm": NaN,
+      "learning_rate": 4.6373021563431784e-05,
+      "loss": 0.0,
+      "step": 34
+    },
+    {
+      "epoch": 1.7914012738853504,
+      "grad_norm": NaN,
+      "learning_rate": 4.395315307866405e-05,
+      "loss": 0.0,
+      "step": 35
+    },
+    {
+      "epoch": 1.8423566878980893,
+      "grad_norm": NaN,
+      "learning_rate": 4.156683350492376e-05,
+      "loss": 0.0,
+      "step": 36
+    },
+    {
+      "epoch": 1.893312101910828,
+      "grad_norm": NaN,
+      "learning_rate": 3.922131001632606e-05,
+      "loss": 0.0,
+      "step": 37
+    },
+    {
+      "epoch": 1.944267515923567,
+      "grad_norm": NaN,
+      "learning_rate": 3.692370589061639e-05,
+      "loss": 0.0,
+      "step": 38
+    },
+    {
+      "epoch": 2.0031847133757963,
+      "grad_norm": NaN,
+      "learning_rate": 3.468099887600999e-05,
+      "loss": 0.0,
+      "step": 39
+    },
+    {
+      "epoch": 2.054140127388535,
+      "grad_norm": NaN,
+      "learning_rate": 3.250000000000001e-05,
+      "loss": 0.0,
+      "step": 40
+    },
+    {
+      "epoch": 2.105095541401274,
+      "grad_norm": NaN,
+      "learning_rate": 3.0387332884490805e-05,
+      "loss": 0.0,
+      "step": 41
+    },
+    {
+      "epoch": 2.156050955414013,
+      "grad_norm": NaN,
+      "learning_rate": 2.8349413630075906e-05,
+      "loss": 0.0,
+      "step": 42
+    },
+    {
+      "epoch": 2.2070063694267517,
+      "grad_norm": NaN,
+      "learning_rate": 2.639243133055145e-05,
+      "loss": 0.0,
+      "step": 43
+    },
+    {
+      "epoch": 2.2579617834394905,
+      "grad_norm": NaN,
+      "learning_rate": 2.4522329276841663e-05,
+      "loss": 0.0,
+      "step": 44
+    },
+    {
+      "epoch": 2.3089171974522293,
+      "grad_norm": NaN,
+      "learning_rate": 2.2744786907419703e-05,
+      "loss": 0.0,
+      "step": 45
+    },
+    {
+      "epoch": 2.359872611464968,
+      "grad_norm": NaN,
+      "learning_rate": 2.1065202560039677e-05,
+      "loss": 0.0,
+      "step": 46
+    },
+    {
+      "epoch": 2.410828025477707,
+      "grad_norm": NaN,
+      "learning_rate": 1.9488677077162295e-05,
+      "loss": 0.0,
+      "step": 47
+    },
+    {
+      "epoch": 2.461783439490446,
+      "grad_norm": NaN,
+      "learning_rate": 1.8019998314863974e-05,
+      "loss": 0.0,
+      "step": 48
+    },
+    {
+      "epoch": 2.5127388535031847,
+      "grad_norm": NaN,
+      "learning_rate": 1.6663626602275288e-05,
+      "loss": 0.0,
+      "step": 49
+    },
+    {
+      "epoch": 2.5636942675159236,
+      "grad_norm": NaN,
+      "learning_rate": 1.5423681195707997e-05,
+      "loss": 0.0,
+      "step": 50
+    },
+    {
+      "epoch": 2.5636942675159236,
+      "eval_loss": NaN,
+      "eval_runtime": 1.4786,
+      "eval_samples_per_second": 33.816,
+      "eval_steps_per_second": 8.792,
+      "step": 50
     }
   ],
   "logging_steps": 1,
@@ -212,7 +395,7 @@
         "early_stopping_threshold": 0.0
       },
       "attributes": {
-        "early_stopping_patience_counter": 0
+        "early_stopping_patience_counter": 1
       }
     },
     "TrainerControl": {
@@ -221,12 +404,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 2.629578157719552e+17,
+  "total_flos": 5.259156315439104e+17,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null