| { | |
| "device": "cuda:0", | |
| "seed": 42, | |
| "dtype": "torch.bfloat16", | |
| "hook_point_in": "blocks.4.hook_attn_out", | |
| "hook_point_out": "blocks.4.hook_attn_out", | |
| "use_decoder_bias": true, | |
| "apply_decoder_bias_to_pre_encoder": false, | |
| "expansion_factor": 8, | |
| "d_model": 4096, | |
| "d_sae": 32768, | |
| "bias_init_method": "all_zero", | |
| "act_fn": "jumprelu", | |
| "jump_relu_threshold": 0.0546875, | |
| "norm_activation": "dataset-wise", | |
| "dataset_average_activation_norm": { | |
| "in": 1.3984375, | |
| "out": 1.3984375 | |
| }, | |
| "decoder_exactly_fixed_norm": false, | |
| "sparsity_include_decoder_norm": true, | |
| "use_glu_encoder": false, | |
| "init_decoder_norm": 0.5, | |
| "init_encoder_norm": null, | |
| "init_encoder_with_decoder_transpose": true, | |
| "lp": 1, | |
| "l1_coefficient": 8e-05, | |
| "l1_coefficient_warmup_steps": 39062, | |
| "top_k": 50, | |
| "k_warmup_steps": 39062, | |
| "use_batch_norm_mse": true, | |
| "use_ghost_grads": false, | |
| "tp_size": 1, | |
| "ddp_size": 1 | |
| } |