| mode: pt | |
| device: gpu | |
| precision: bf16 | |
| eval_only: false | |
| predict_only: false | |
| seed: 34534 | |
| model: | |
| klass: hf_t5 | |
| name: pszemraj/tFINE-900m-e16-d32 | |
| overwrite: | |
| dropout_rate: 0.0 | |
| checkpoint_path: '' | |
| random_init: false | |
| compile: true | |
| tokenizer: | |
| name: BEE-spoke-data/slimpajama_tok-48128-BPE-forT5 | |
| data: | |
| input_length: 1024 | |
| mlm_probability: 0.15 | |
| mean_noise_span_length: 3.0 | |
| num_workers: 16 | |
| optim: | |
| name: adamwscale | |
| base_lr: 0.01 | |
| batch_size: 128 | |
| total_steps: 20000 | |
| epochs: -1 | |
| warmup_steps: 5000 | |
| lr_scheduler: cosine | |
| weight_decay: 0.0001 | |
| grad_clip: 1.0 | |
| grad_acc: 8 | |
| final_cosine: 2.0e-05 | |
| eval: | |
| every_steps: 1000000000 | |
| steps: 500 | |
| checkpoint: | |
| every_steps: 2500 | |
| logging: | |
| use_wandb: true | |
| wandb_config: | |
| project: nanoT5 | |
| entity: pszemraj | |
| tags: | |
| - 900m | |
| - '1024' | |
| mode: online | |
| every_steps: 25 | |
| grad_l2: true | |
| weights_l2: true | |