jeffreygo commited on
Commit
1f3730c
1 Parent(s): f166c1f

Init cpm-bee-10b

Browse files
Files changed (2) hide show
  1. cpm-bee-10b.json +14 -0
  2. vocab.txt +0 -0
cpm-bee-10b.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 86583,
3
+ "dim_model": 4096,
4
+ "dim_ff" : 10240,
5
+ "num_layers" : 48,
6
+ "num_heads": 32,
7
+ "dim_head" : 128,
8
+ "dropout_p" : 0.0,
9
+ "position_bias_num_buckets" : 256,
10
+ "position_bias_num_segment_buckets": 256,
11
+ "position_bias_max_distance" : 2048,
12
+ "eps" : 1e-6,
13
+ "half" : true
14
+ }
vocab.txt ADDED
The diff for this file is too large to render. See raw diff