diff --git "a/ndarray-cache.json" "b/ndarray-cache.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache.json" @@ -0,0 +1,5857 @@ +{ + "metadata": { + "ParamSize": 518, + "ParamBytes": 1738526720.0, + "BitsPerParam": 5.010408347335939 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 64552960, + "records": [ + { + "name": "gpt_neox.embed_in.q_weight", + "shape": [ + 50432, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 64552960, + "byteOffset": 0 + } + ], + "md5sum": "e65e8796e5e8cce8ac7ea3dfbf8a288a" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 32604160, + "records": [ + { + "name": "gpt_neox.embed_in.q_scale", + "shape": [ + 50432, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 16138240, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.0.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16138240 + }, + { + "name": "gpt_neox.layers.0.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16148480 + }, + { + "name": "gpt_neox.layers.0.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16158720 + }, + { + "name": "gpt_neox.layers.0.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16168960 + }, + { + "name": "gpt_neox.layers.0.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16179200 + }, + { + "name": "gpt_neox.layers.0.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26009600 + }, + { + "name": "gpt_neox.layers.0.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28467200 + }, + { + "name": "gpt_neox.layers.0.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28497920 + }, + { + "name": "gpt_neox.layers.0.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 31774720 + }, + { + "name": "gpt_neox.layers.0.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32593920 + } + ], + "md5sum": "506e9427406c74b0bf5b4b24fb00cbae" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.1.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.1.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.1.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.1.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "6185ffeb1b33da59d73ae2967c6bb442" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.1.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.1.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.1.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.1.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.1.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.1.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "8f749f91ddbab931dd212fa1b4df0d51" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.2.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.2.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.2.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.2.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.2.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.2.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.2.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.2.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.2.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.2.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "6e3e620ee925777df20f56f5b88e2b35" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.3.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.3.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.3.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.3.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "6799407e1a1eb4c5f3417c9c5709f642" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.3.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.3.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.3.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.3.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.3.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.3.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "d6fe5397b8cc5e0ad3a66b7c7a3536bf" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.4.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.4.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.4.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.4.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.4.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.4.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.4.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.4.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.4.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.4.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "b47ab02c5b6de722a6ea74fab12f2137" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.5.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.5.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.5.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.5.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "9100970503defeb117f58f11f03ea8de" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.5.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.5.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.5.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.5.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.5.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.5.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "dab7777ac759ba43db020d9c61ca24e9" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.6.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.6.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.6.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.6.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.6.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.6.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.6.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.6.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.6.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.6.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "5e9b5beac432bd7587c2f9abc49b4b3b" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.7.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.7.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.7.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.7.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "a71f63374fe69d389a9cf465ee611aca" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.7.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.7.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.7.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.7.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.7.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.7.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "e85556f704c2f3bb85c69e8ce71c2b2e" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.8.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.8.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.8.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.8.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.8.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.8.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.8.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.8.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.8.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.8.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "f22b0bd4f27faf2427f206899591181d" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.9.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.9.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.9.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.9.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "2438429194fbbac9c078460ca03c507c" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.9.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.9.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.9.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.9.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.9.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.9.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "ca6811a361c3c1067ca595cc5d6ce65f" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.10.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.10.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.10.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.10.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.10.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.10.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.10.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.10.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.10.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.10.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "6b5b31948d57cbf3553792e07dab0ad2" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.11.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.11.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.11.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.11.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "820ad1a5ec9ec66e960a27f157f80648" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.11.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.11.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.11.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.11.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.11.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.11.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "00acc9917321b899ff99ae3750804355" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.12.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.12.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.12.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.12.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.12.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.12.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.12.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.12.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.12.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.12.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "af475bbe569923eb2a2356049bf8ca83" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.13.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.13.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.13.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.13.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "c368499847e95159e151f2fb82ee34ca" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.13.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.13.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.13.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.13.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.13.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.13.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "d9655a3d9c6e7ea22f16eba46828c3ba" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.14.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.14.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.14.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.14.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.14.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.14.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.14.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.14.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.14.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.14.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "c1c1c1dca1a0fd1d23ac2743c94df013" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.15.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.15.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.15.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.15.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "5a574a2d682d4459daec407348aa7a43" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.15.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.15.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.15.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.15.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.15.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.15.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "754acaa5c6b0dc8316a437e14ea2f430" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.16.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.16.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.16.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.16.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.16.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.16.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.16.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.16.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.16.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.16.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "b26084a5f4defe2db8644ac3c8dc1d44" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.16.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.16.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.17.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.17.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.17.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.17.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "7525b918517a80e5d548f614d2fcd2e0" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.17.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.17.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.17.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.17.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.17.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.17.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.17.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "1ae1c39414fc4a973c739c12433a6443" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.17.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.18.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.18.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.18.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.18.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.18.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.18.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.18.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.18.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.18.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.18.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "dd741f555678b27d5707a01d9be859a2" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.18.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.18.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.19.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.19.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.19.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.19.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "88fbad4e8f625461904603962dd4e3df" + }, + { + "dataPath": "params_shard_30.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.19.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.19.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.19.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.19.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.19.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.19.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.19.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "347678378af964ef64328217a72063bf" + }, + { + "dataPath": "params_shard_31.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.19.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.20.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.20.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.20.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.20.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.20.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.20.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.20.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.20.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.20.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.20.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "e3d095771bb04ff0ec6a2ebed2d72652" + }, + { + "dataPath": "params_shard_32.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.20.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.20.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.21.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.21.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.21.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.21.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "a08359c8c11cda410d439aec53a7c687" + }, + { + "dataPath": "params_shard_33.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.21.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.21.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.21.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.21.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.21.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.21.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.21.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "4db4063ca67dd7b6d3437df6d0598228" + }, + { + "dataPath": "params_shard_34.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.21.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.22.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.22.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.22.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.22.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.22.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.22.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.22.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.22.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.22.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.22.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "eba53144a64724705d075065a345a4f9" + }, + { + "dataPath": "params_shard_35.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.22.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.22.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.23.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.23.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.23.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.23.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "fd7850af14a6a8d87cf20d105acfa0a3" + }, + { + "dataPath": "params_shard_36.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.23.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.23.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.23.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.23.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.23.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.23.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.23.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "2508be19561d35b5e2c2ff8a53e9a269" + }, + { + "dataPath": "params_shard_37.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.23.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.24.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.24.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.24.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.24.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.24.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.24.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.24.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.24.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.24.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.24.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "aa9d9236ec75307d77a3b57db9bc72ba" + }, + { + "dataPath": "params_shard_38.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.24.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.24.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.24.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.24.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.24.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.24.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.25.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.25.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.25.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.25.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "b250bb0f6165f4317a256902e5093657" + }, + { + "dataPath": "params_shard_39.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.25.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.25.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.25.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.25.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.25.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.25.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.25.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.25.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.25.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "fbe1041cc8d2b2e1829bb9d6dba62028" + }, + { + "dataPath": "params_shard_40.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.25.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.25.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.25.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.26.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.26.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.26.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.26.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.26.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.26.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.26.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.26.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.26.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.26.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "8f928520e3d689e7726d18548206366d" + }, + { + "dataPath": "params_shard_41.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.26.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.26.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.26.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.26.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.26.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.26.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.27.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.27.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.27.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.27.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "409b12f1645ef4fdd9a61bd0da8e83fe" + }, + { + "dataPath": "params_shard_42.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.27.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.27.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.27.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.27.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.27.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.27.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.27.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.27.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.27.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "33c3d76b713c519d7fe90bfd80e93def" + }, + { + "dataPath": "params_shard_43.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.27.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.27.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.27.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.28.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.28.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.28.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.28.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.28.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.28.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.28.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.28.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.28.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.28.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "87cf7e48f7d6cc520be0fae00115ff57" + }, + { + "dataPath": "params_shard_44.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.28.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.28.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.28.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.28.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.28.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.28.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.29.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.29.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.29.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.29.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "1667fb8df1a66ee054283caea6113da9" + }, + { + "dataPath": "params_shard_45.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.29.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.29.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.29.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.29.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.29.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.29.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.29.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.29.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.29.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "f448efc3ae9e949d9343581e78fa64ef" + }, + { + "dataPath": "params_shard_46.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.29.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.29.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.29.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.30.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.layers.30.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "gpt_neox.layers.30.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.30.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.30.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 16435200 + }, + { + "name": "gpt_neox.layers.30.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 26265600 + }, + { + "name": "gpt_neox.layers.30.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 28723200 + }, + { + "name": "gpt_neox.layers.30.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 28753920 + }, + { + "name": "gpt_neox.layers.30.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 32030720 + }, + { + "name": "gpt_neox.layers.30.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "37b201b978536e7abf8694075a364488" + }, + { + "dataPath": "params_shard_47.bin", + "format": "raw-shard", + "nbytes": 32860160, + "records": [ + { + "name": "gpt_neox.layers.30.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.30.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.30.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.layers.30.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.30.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.30.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32808960 + }, + { + "name": "gpt_neox.layers.31.input_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32819200 + }, + { + "name": "gpt_neox.layers.31.input_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32829440 + }, + { + "name": "gpt_neox.layers.31.post_attention_layernorm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32839680 + }, + { + "name": "gpt_neox.layers.31.post_attention_layernorm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 32849920 + } + ], + "md5sum": "77bf1ed46cfaf71cf88bbf05609cb3fc" + }, + { + "dataPath": "params_shard_48.bin", + "format": "raw-shard", + "nbytes": 32849920, + "records": [ + { + "name": "gpt_neox.layers.31.attention.query_key_value.q_weight", + "shape": [ + 7680, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 9830400, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.31.attention.query_key_value.q_scale", + "shape": [ + 7680, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 2457600, + "byteOffset": 9830400 + }, + { + "name": "gpt_neox.layers.31.attention.query_key_value.bias", + "shape": [ + 7680 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 30720, + "byteOffset": 12288000 + }, + { + "name": "gpt_neox.layers.31.attention.dense.q_weight", + "shape": [ + 2560, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 12318720 + }, + { + "name": "gpt_neox.layers.31.attention.dense.q_scale", + "shape": [ + 2560, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 819200, + "byteOffset": 15595520 + }, + { + "name": "gpt_neox.layers.31.attention.dense.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16414720 + }, + { + "name": "gpt_neox.layers.31.mlp.dense_h_to_4h.q_weight", + "shape": [ + 10240, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 16424960 + }, + { + "name": "gpt_neox.layers.31.mlp.dense_h_to_4h.q_scale", + "shape": [ + 10240, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 29532160 + }, + { + "name": "gpt_neox.layers.31.mlp.dense_h_to_4h.bias", + "shape": [ + 10240 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 40960, + "byteOffset": 32808960 + } + ], + "md5sum": "3b34c10fdd5704db70771615e5e5ab6d" + }, + { + "dataPath": "params_shard_49.bin", + "format": "raw-shard", + "nbytes": 64552960, + "records": [ + { + "name": "embed_out.q_weight", + "shape": [ + 50432, + 320 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 64552960, + "byteOffset": 0 + } + ], + "md5sum": "25ea3e71fa498e95a5f1849d36ab9f50" + }, + { + "dataPath": "params_shard_50.bin", + "format": "raw-shard", + "nbytes": 32552960, + "records": [ + { + "name": "gpt_neox.layers.31.mlp.dense_4h_to_h.q_weight", + "shape": [ + 2560, + 1280 + ], + "dtype": "uint32", + "format": "raw", + "nbytes": 13107200, + "byteOffset": 0 + }, + { + "name": "gpt_neox.layers.31.mlp.dense_4h_to_h.q_scale", + "shape": [ + 2560, + 320 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 3276800, + "byteOffset": 13107200 + }, + { + "name": "gpt_neox.layers.31.mlp.dense_4h_to_h.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16384000 + }, + { + "name": "gpt_neox.final_layer_norm.weight", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16394240 + }, + { + "name": "gpt_neox.final_layer_norm.bias", + "shape": [ + 2560 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 10240, + "byteOffset": 16404480 + }, + { + "name": "embed_out.q_scale", + "shape": [ + 50432, + 80 + ], + "dtype": "float32", + "format": "raw", + "nbytes": 16138240, + "byteOffset": 16414720 + } + ], + "md5sum": "3d49453ac0ed0334cf213967b00a0e7b" + } + ] +} \ No newline at end of file