Saving weights and logs of step 480000

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94aa2f6944862d84e1adaa5f09047077eab0a59d439901fb829aef44c233a1ef
+oid sha256:4f38acdc66d0d9dd7e6f7f9e1af3e281fe54a25be30002b509e5df27a5855267
 size 891548548

run_t5.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+python run_t5_mlm_flax.py \
+    --output_dir="${MODEL_PATH}" \
+    --model_type="t5" \
+    --config_name="${MODEL_PATH}" \
+    --tokenizer_name="${MODEL_PATH}" \
+    --preprocessing_num_workers="96" \
+    --do_train --do_eval \
+    --dataset_name="${DATASET}" \
+    --dataset_config_name="${DATASET_CONFIG}" \
+    --max_seq_length="512" \
+    --per_device_train_batch_size="16" \
+    --per_device_eval_batch_size="16" \
+    --adafactor \
+    --learning_rate="0.005" \
+    --overwrite_output_dir \
+    --num_train_epochs="1" \
+    --logging_steps="500" \
+    --save_steps="80000" \
+    --eval_steps="2500" \
+    --weight_decay="0.01" \
+    --warmup_steps="10000" \
+    --validation_split_count="15000" \
+    --push_to_hub \
+#    --adam_beta1="0.9" \
+#    --adam_beta2="0.98" \
+#    --resume_from_checkpoint="${MODEL_DIR}" \ # Uncomment to resume from ckpt
+#    --max_train_samples 100000 \
+#    --max_eval_samples 1000 \
+#    --adafactor \
+#    --save_steps="80000" \
+
+
+# Instead of adafactor: adamw
+

runs/Dec12_13-32-15_t1v-n-aa1c2160-w-0/events.out.tfevents.1639325974.t1v-n-aa1c2160-w-0.15396.0.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41e90507985444eec7879012ba52eee14f9f44420707271b4e9358d22b514b88
-size 59643155
+oid sha256:56a1019d63fc5417fea18e1ffa05a7d16acda66f09c60cb2d1f78c8a27044cca
+size 71578452