Hzfinfdu commited on
Commit
c00e776
1 Parent(s): 5ed0de9

Upload folder using huggingface_hub

Browse files
Llama3_1Base-L1A-8x/checkpoints/final.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ed7c0f43da5d056232d2c8e8c4af46929beb7a402b7263b93131f0dd72812223
3
+ size 537076272
Llama3_1Base-L1A-8x/hyperparams.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "hook_point_in": "blocks.1.hook_attn_out",
3
+ "hook_point_out": "blocks.1.hook_attn_out",
4
+ "use_decoder_bias": true,
5
+ "apply_decoder_bias_to_pre_encoder": false,
6
+ "expansion_factor": 8,
7
+ "d_model": 4096,
8
+ "d_sae": 32768,
9
+ "norm_activation": "token-wise",
10
+ "dataset_average_activation_norm": null,
11
+ "decoder_exactly_fixed_norm": false,
12
+ "sparsity_include_decoder_norm": true,
13
+ "use_glu_encoder": false,
14
+ "init_decoder_norm": null,
15
+ "init_encoder_norm": null,
16
+ "init_encoder_with_decoder_transpose": true,
17
+ "l1_coefficient": 1.6e-05,
18
+ "l1_coefficient_warmup_steps": 14648,
19
+ "lp": 1,
20
+ "use_ghost_grads": false,
21
+ "tp_size": 1,
22
+ "ddp_size": 1
23
+ }
Llama3_1Base-L1A-8x/lm_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_name": "meta-llama/Meta-Llama-3.1-8B",
3
+ "use_flash_attn": false,
4
+ "cache_dir": null,
5
+ "d_model": 4096,
6
+ "local_files_only": false
7
+ }