|
encoder: |
|
num_layers: 8 |
|
hidden_size: 512 |
|
ffn_hidden_size: 1024 |
|
num_attention_heads: 6 |
|
init_method_std: 0.02 |
|
hidden_dropout: 0.0 |
|
attention_dropout: 0.0 |
|
ffn_dropout: 0.0 |
|
position_embedding_type: relative |
|
relative_attention_num_buckets: 32 |
|
relative_attention_max_distance: 128 |
|
relative_position_bias_self_attention_only: true |
|
kv_channels: 64 |
|
apply_query_key_layer_scaling: false |
|
layernorm_epsilon: 1.0e-06 |
|
persist_layer_norm: true |
|
bias_activation_fusion: false |
|
grad_div_ar_fusion: true |
|
masked_softmax_fusion: false |
|
bias_dropout_add_fusion: false |
|
bias: false |
|
normalization: rmsnorm |
|
arch: transformer |
|
activation: geglu |
|
headscale: false |
|
transformer_block_type: pre_ln |
|
hidden_steps: 32 |
|
num_self_attention_per_cross_attention: 1 |
|
openai_gelu: true |
|
onnx_safe: false |
|
fp32_residual_connection: false |
|
activations_checkpoint_method: null |
|
activations_checkpoint_num_layers: 1 |
|
megatron_legacy: true |
|
normalize_attention_scores: false |
|
decoder: |
|
num_layers: 8 |
|
hidden_size: 512 |
|
ffn_hidden_size: 1024 |
|
num_attention_heads: 6 |
|
init_method_std: 0.02 |
|
hidden_dropout: 0.0 |
|
attention_dropout: 0.0 |
|
ffn_dropout: 0.0 |
|
position_embedding_type: relative |
|
relative_attention_num_buckets: 32 |
|
relative_attention_max_distance: 128 |
|
relative_position_bias_self_attention_only: true |
|
kv_channels: 64 |
|
apply_query_key_layer_scaling: false |
|
layernorm_epsilon: 1.0e-06 |
|
persist_layer_norm: true |
|
bias_activation_fusion: false |
|
grad_div_ar_fusion: true |
|
masked_softmax_fusion: false |
|
bias_dropout_add_fusion: false |
|
bias: false |
|
normalization: rmsnorm |
|
arch: transformer |
|
activation: geglu |
|
headscale: false |
|
transformer_block_type: pre_ln |
|
hidden_steps: 32 |
|
num_self_attention_per_cross_attention: 1 |
|
openai_gelu: true |
|
onnx_safe: false |
|
fp32_residual_connection: false |
|
activations_checkpoint_method: null |
|
activations_checkpoint_num_layers: 1 |
|
megatron_legacy: true |
|
normalize_attention_scores: false |
|
micro_batch_size: 4 |
|
global_batch_size: 8 |
|
tensor_model_parallel_size: 1 |
|
pipeline_model_parallel_size: 1 |
|
resume_from_checkpoint: null |
|
pipeline_model_parallel_split_rank: 0 |
|
make_vocab_size_divisible_by: 128 |
|
megatron_amp_O2: false |
|
grad_allreduce_chunk_size_mb: 125 |
|
grad_div_ar_fusion: true |
|
gradient_as_bucket_view: true |
|
seq_length: 512 |
|
max_position_embeddings: 512 |
|
tokenizer: |
|
library: sentencepiece |
|
type: null |
|
model: nemo:ce65b6d8f4fb4975955e935db699cba3_t5_small_tokenizer.model |
|
vocab_file: null |
|
merge_file: null |
|
num_sentinel_tokens: 100 |
|
sentencepiece_legacy: true |
|
add_sentinel_tokens_in_reverse_order: true |
|
add_sentinel_tokens_first: true |
|
embedding_init_method_std: 0.02 |
|
embedding_dropout: 0.1 |
|
share_token_embeddings: true |
|
share_decoder_tokens_head_embeddings: false |
|
tokens_head_bias: false |
|
native_amp_init_scale: 4294967296 |
|
native_amp_growth_interval: 1000 |
|
fp16_lm_cross_entropy: false |
|
seed: 1234 |
|
use_cpu_initialization: false |
|
apex_transformer_log_level: 30 |
|
data: |
|
data_prefix: null |
|
index_mapping_dir: null |
|
data_impl: mmap |
|
splits_string: 949,45,5 |
|
seq_length: 512 |
|
seq_length_dec: 128 |
|
skip_warmup: true |
|
num_workers: 0 |
|
dataloader_type: single |
|
masked_lm_prob: 0.15 |
|
dataset_type: t5 |
|
short_seq_prob: 0.0 |
|
max_ngram_size: 10 |
|
mean_ngram_size: null |
|
geometric_dist: true |
|
permutation: false |
|
whole_word_masking: false |
|
favor_longer_ngrams: false |
|
respect_document_boundaries: true |
|
optim: |
|
name: fused_adam |
|
lr: 0.0001 |
|
betas: |
|
- 0.9 |
|
- 0.999 |
|
eps: 1.0e-08 |
|
weight_decay: 0.01 |
|
sched: |
|
name: WarmupAnnealing |
|
min_lr: 1.0e-05 |
|
last_epoch: -1 |
|
warmup_ratio: 0.01 |
|
precision: bf16 |
|
target: nemo.collections.nlp.models.language_modeling.megatron_t5_model.MegatronT5Model |
|
nemo_version: 1.11.0rc0 |
|
library: huggingface-t5v1_1 |