layer_norms / unnatural-instructions-7b_norms.json
cfierro's picture
Add norms of modules per model
e5fca79
{"layer":{"0":-1,"1":"0","2":"0","3":-1,"4":"1","5":"1","6":"2","7":"2","8":"3","9":"3","10":"4","11":"4","12":"5","13":"5","14":"6","15":"6","16":"7","17":"7","18":"8","19":"8","20":"9","21":"9","22":"10","23":"10","24":"11","25":"11","26":"12","27":"12","28":"13","29":"13","30":"14","31":"14","32":"15","33":"15","34":"16","35":"16","36":"17","37":"17","38":"18","39":"18","40":"19","41":"19","42":"20","43":"20","44":"21","45":"21","46":"22","47":"22","48":"23","49":"23","50":"24","51":"24","52":"25","53":"25","54":"26","55":"26","56":"27","57":"27","58":"28","59":"28","60":"29","61":"29","62":"30","63":"30","64":"31","65":"31"},"module":{"0":"lm_head.weight","1":"self_attn","2":"mlp","3":"model.norm.weight","4":"self_attn","5":"mlp","6":"self_attn","7":"mlp","8":"self_attn","9":"mlp","10":"self_attn","11":"mlp","12":"self_attn","13":"mlp","14":"self_attn","15":"mlp","16":"self_attn","17":"mlp","18":"self_attn","19":"mlp","20":"self_attn","21":"mlp","22":"self_attn","23":"mlp","24":"self_attn","25":"mlp","26":"self_attn","27":"mlp","28":"self_attn","29":"mlp","30":"self_attn","31":"mlp","32":"self_attn","33":"mlp","34":"self_attn","35":"mlp","36":"self_attn","37":"mlp","38":"self_attn","39":"mlp","40":"self_attn","41":"mlp","42":"self_attn","43":"mlp","44":"self_attn","45":"mlp","46":"self_attn","47":"mlp","48":"self_attn","49":"mlp","50":"self_attn","51":"mlp","52":"self_attn","53":"mlp","54":"self_attn","55":"mlp","56":"self_attn","57":"mlp","58":"self_attn","59":"mlp","60":"self_attn","61":"mlp","62":"self_attn","63":"mlp","64":"self_attn","65":"mlp"},"1":{"0":4108834.625,"1":1037805.578125,"2":1917219.625,"3":6554.6635742188,"4":1093750.328125,"5":2250960.875,"6":1298427.59375,"7":2305591.8125,"8":1217475.859375,"9":2333152.0,"10":1216313.296875,"11":2341646.125,"12":1189155.84375,"13":2347659.6875,"14":1209365.140625,"15":2351540.375,"16":1198787.734375,"17":2347490.125,"18":1176704.546875,"19":2348014.75,"20":1150951.1875,"21":2345026.5625,"22":1176655.359375,"23":2357082.8125,"24":1230146.46875,"25":2361555.9375,"26":1177097.171875,"27":2374714.25,"28":1180853.453125,"29":2388351.5625,"30":1189216.5,"31":2394018.9375,"32":1191254.390625,"33":2397221.375,"34":1224863.09375,"35":2398859.875,"36":1211835.5625,"37":2409442.5,"38":1196211.84375,"39":2414074.625,"40":1210366.4375,"41":2419089.375,"42":1242301.3125,"43":2426402.3125,"44":1217459.78125,"45":2431144.25,"46":1231955.15625,"47":2440389.125,"48":1220771.875,"49":2446514.0,"50":1234953.78125,"51":2456063.375,"52":1264583.9375,"53":2462903.75,"54":1275041.71875,"55":2472496.125,"56":1285420.5,"57":2477001.3125,"58":1268850.09375,"59":2478591.5,"60":1288573.09375,"61":2481559.6875,"62":1289779.90625,"63":2474667.8125,"64":1227339.21875,"65":2554257.125},"2":{"0":452.420211792,"1":358.4527015686,"2":359.3196868896,"3":102.6377868652,"4":370.3111000061,"5":420.6345825195,"6":418.0511932373,"7":430.6473846436,"8":385.766784668,"9":435.5082397461,"10":385.7497177124,"11":437.4872131348,"12":375.099029541,"13":438.8388214111,"14":380.6313476562,"15":439.607635498,"16":378.3453979492,"17":439.253692627,"18":372.5401229858,"19":439.7696685791,"20":364.779586792,"21":439.35105896,"22":370.9337234497,"23":441.4199981689,"24":385.3360748291,"25":442.8640136719,"26":370.9208374023,"27":445.0789031982,"28":371.6686019897,"29":447.2158203125,"30":372.9982070923,"31":448.2364501953,"32":374.4995727539,"33":448.5243988037,"34":382.0094909668,"35":448.9906768799,"36":378.3959503174,"37":450.5345458984,"38":374.8459320068,"39":451.1598815918,"40":378.4437179565,"41":451.9407806396,"42":386.404258728,"43":453.1228637695,"44":380.427947998,"45":453.7487945557,"46":383.2412948608,"47":455.282333374,"48":382.3883514404,"49":456.3740539551,"50":385.2509155273,"51":458.1691894531,"52":392.4166412354,"53":459.5590362549,"54":394.8032150269,"55":461.4518432617,"56":398.4821166992,"57":462.6396789551,"58":395.6588821411,"59":463.2229309082,"60":400.2480621338,"61":464.3321838379,"62":400.2441101074,"63":466.7425689697,"64":384.7885131836,"65":480.1660614014},"inf":{"0":0.693359375,"1":2.5908203125,"2":2.30078125,"3":2.625,"4":1.8959960938,"5":1.935546875,"6":2.029296875,"7":3.17578125,"8":1.8291015625,"9":1.849609375,"10":1.80859375,"11":2.5546875,"12":2.1826171875,"13":1.748046875,"14":2.015625,"15":1.96875,"16":1.849609375,"17":1.779296875,"18":1.6806640625,"19":1.470703125,"20":1.69140625,"21":2.265625,"22":1.619140625,"23":1.4765625,"24":1.8203125,"25":1.8203125,"26":1.5400390625,"27":1.87109375,"28":1.603515625,"29":1.197265625,"30":1.61328125,"31":1.794921875,"32":1.4765625,"33":1.748046875,"34":1.908203125,"35":1.84765625,"36":1.873046875,"37":1.685546875,"38":1.6474609375,"39":1.701171875,"40":1.818359375,"41":1.607421875,"42":2.064453125,"43":1.923828125,"44":2.076171875,"45":1.2294921875,"46":2.0302734375,"47":1.666015625,"48":1.9990234375,"49":1.75390625,"50":1.826171875,"51":1.76953125,"52":1.9189453125,"53":1.318359375,"54":1.701171875,"55":2.1015625,"56":2.1875,"57":2.28125,"58":1.693359375,"59":2.17578125,"60":1.8212890625,"61":2.611328125,"62":2.0791015625,"63":3.20703125,"64":2.669921875,"65":4.24609375}}