analysis.json

{
    "aveage_var_degree": 3.892243738108603e-09,
    "total_diff_num": 11385304937,
    "total_param_num": 14498703360,
    "diff_rate": 78.5263664915757,
    "layer_to_params": {
        "0": 1.988666535246705e-08,
        "1": 2.6983774006922925e-08,
        "2": 1.319228530894409e-08,
        "3": 6.557201376281705e-08,
        "4": 4.205192939414228e-08,
        "5": 1.3925403408610472e-08,
        "6": 1.2824742500792588e-08,
        "7": 1.1327664732443272e-08,
        "8": 1.3955765562481565e-08,
        "9": 2.148611166390114e-08,
        "10": 1.7289768219057282e-08,
        "11": 1.329685737538368e-08,
        "12": 1.8387197199242386e-08,
        "13": 1.573349900733471e-08,
        "14": 1.8524112318538675e-08,
        "15": 1.8548633124556574e-08,
        "16": 1.590110241443387e-08,
        "17": 1.5135453732038522e-08,
        "18": 1.1342735059537811e-08,
        "19": 1.0097682785134383e-08,
        "20": 1.248061909883821e-08,
        "21": 1.415265751110913e-08,
        "22": 1.490716693811472e-08,
        "23": 1.6227143244326544e-08,
        "24": 1.0399099193610557e-08,
        "25": 1.4272791352000781e-08,
        "26": 1.2074979475566277e-08,
        "27": 1.75786421745232e-08,
        "28": 1.7063607155750844e-08,
        "29": 1.5309838493256697e-08,
        "30": 2.0628235593752226e-08,
        "31": 2.4866034351936353e-08,
        "32": 2.5866199138135922e-08,
        "33": 3.406494921406588e-08,
        "34": 3.5020059196237226e-08,
        "35": 4.820790008388855e-08,
        "36": 4.640676214287311e-08,
        "37": 3.5461280324019145e-08,
        "38": 3.979688822634368e-08,
        "39": 6.100158140990667e-08
    },
    "key_to_params": {
        "model.embed_tokens.weight": 2.917434234714228e-19,
        "model.layers.0.self_attn.q_proj.weight": 1.187384697232119e-09,
        "model.layers.0.self_attn.k_proj.weight": 1.1349608541877388e-09,
        "model.layers.0.self_attn.v_proj.weight": 3.0414742013817886e-09,
        "model.layers.0.self_attn.o_proj.weight": 2.556667766384635e-09,
        "model.layers.0.mlp.gate_proj.weight": 8.003662514965982e-09,
        "model.layers.0.mlp.up_proj.weight": 2.3990156705622212e-09,
        "model.layers.0.mlp.down_proj.weight": 1.5634996467876287e-09,
        "model.layers.0.input_layernorm.weight": 0.0,
        "model.layers.0.input_layernorm.bias": 2.710505431213761e-19,
        "model.layers.0.post_attention_layernorm.weight": 0.0,
        "model.layers.0.post_attention_layernorm.bias": 6.938893903907229e-19,
        "model.layers.1.self_attn.q_proj.weight": 2.0518091048415955e-09,
        "model.layers.1.self_attn.k_proj.weight": 1.6146566395036643e-09,
        "model.layers.1.self_attn.v_proj.weight": 1.4650281627837103e-08,
        "model.layers.1.self_attn.o_proj.weight": 4.025015698516654e-09,
        "model.layers.1.mlp.gate_proj.weight": 1.7941124497156125e-09,
        "model.layers.1.mlp.up_proj.weight": 2.191004947841672e-09,
        "model.layers.1.mlp.down_proj.weight": 6.568935380831438e-10,
        "model.layers.1.input_layernorm.weight": 0.0,
        "model.layers.1.input_layernorm.bias": 3.9916427639358903e-19,
        "model.layers.1.post_attention_layernorm.weight": 0.0,
        "model.layers.1.post_attention_layernorm.bias": 1.8431436932253575e-19,
        "model.layers.2.self_attn.q_proj.weight": 8.317828559256668e-10,
        "model.layers.2.self_attn.k_proj.weight": 1.0357459956367166e-09,
        "model.layers.2.self_attn.v_proj.weight": 4.803724777957541e-09,
        "model.layers.2.self_attn.o_proj.weight": 2.938668899332697e-09,
        "model.layers.2.mlp.gate_proj.weight": 1.4420785040177482e-09,
        "model.layers.2.mlp.up_proj.weight": 1.6792421320133145e-09,
        "model.layers.2.mlp.down_proj.weight": 4.6104214372159426e-10,
        "model.layers.2.input_layernorm.weight": 0.0,
        "model.layers.2.input_layernorm.bias": 2.9544509200229997e-19,
        "model.layers.2.post_attention_layernorm.weight": 0.0,
        "model.layers.2.post_attention_layernorm.bias": 4.336808689942018e-20,
        "model.layers.3.self_attn.q_proj.weight": 8.837429277264163e-10,
        "model.layers.3.self_attn.k_proj.weight": 1.0162795405221914e-09,
        "model.layers.3.self_attn.v_proj.weight": 3.0072123990976253e-09,
        "model.layers.3.self_attn.o_proj.weight": 4.018537538286182e-09,
        "model.layers.3.mlp.gate_proj.weight": 1.4122169507876e-09,
        "model.layers.3.mlp.up_proj.weight": 1.5453091843179814e-09,
        "model.layers.3.mlp.down_proj.weight": 5.368871522174838e-08,
        "model.layers.3.input_layernorm.weight": 0.0,
        "model.layers.3.input_layernorm.bias": 1.328147661294743e-19,
        "model.layers.3.post_attention_layernorm.weight": 0.0,
        "model.layers.3.post_attention_layernorm.bias": 1.9786689647860455e-19,
        "model.layers.4.self_attn.q_proj.weight": 8.681465857307558e-10,
        "model.layers.4.self_attn.k_proj.weight": 1.7090624737647886e-09,
        "model.layers.4.self_attn.v_proj.weight": 2.7331961405252512e-09,
        "model.layers.4.self_attn.o_proj.weight": 1.8018978664713358e-09,
        "model.layers.4.mlp.gate_proj.weight": 1.7898181188987412e-09,
        "model.layers.4.mlp.up_proj.weight": 1.897217695538226e-09,
        "model.layers.4.mlp.down_proj.weight": 3.12525905125464e-08,
        "model.layers.4.input_layernorm.weight": 0.0,
        "model.layers.4.input_layernorm.bias": 4.797594613248357e-19,
        "model.layers.4.post_attention_layernorm.weight": 0.0,
        "model.layers.4.post_attention_layernorm.bias": 1.8702487475374952e-19,
        "model.layers.5.self_attn.q_proj.weight": 9.632125852476747e-10,
        "model.layers.5.self_attn.k_proj.weight": 7.343454910824221e-10,
        "model.layers.5.self_attn.v_proj.weight": 3.0922541327527143e-09,
        "model.layers.5.self_attn.o_proj.weight": 1.6237197542068315e-09,
        "model.layers.5.mlp.gate_proj.weight": 1.7408918514168667e-09,
        "model.layers.5.mlp.up_proj.weight": 1.8568865319442314e-09,
        "model.layers.5.mlp.down_proj.weight": 3.914093061515208e-09,
        "model.layers.5.input_layernorm.weight": 0.0,
        "model.layers.5.input_layernorm.bias": 0.0,
        "model.layers.5.post_attention_layernorm.weight": 0.0,
        "model.layers.5.post_attention_layernorm.bias": 4.445228907190568e-19,
        "model.layers.6.self_attn.q_proj.weight": 4.4907185525744355e-10,
        "model.layers.6.self_attn.k_proj.weight": 6.927900386699548e-10,
        "model.layers.6.self_attn.v_proj.weight": 2.1942298644717086e-09,
        "model.layers.6.self_attn.o_proj.weight": 2.0572406356222928e-09,
        "model.layers.6.mlp.gate_proj.weight": 3.026063382094435e-09,
        "model.layers.6.mlp.up_proj.weight": 3.793664973272826e-09,
        "model.layers.6.mlp.down_proj.weight": 6.116817511762444e-10,
        "model.layers.6.input_layernorm.weight": 0.0,
        "model.layers.6.input_layernorm.bias": 0.0,
        "model.layers.6.post_attention_layernorm.weight": 0.0,
        "model.layers.6.post_attention_layernorm.bias": 2.2768245622195594e-19,
        "model.layers.7.self_attn.q_proj.weight": 5.192902108319686e-10,
        "model.layers.7.self_attn.k_proj.weight": 4.908492456934255e-10,
        "model.layers.7.self_attn.v_proj.weight": 3.6511215739665203e-09,
        "model.layers.7.self_attn.o_proj.weight": 1.6526492174762098e-09,
        "model.layers.7.mlp.gate_proj.weight": 1.8395443627620504e-09,
        "model.layers.7.mlp.up_proj.weight": 2.646799543981615e-09,
        "model.layers.7.mlp.down_proj.weight": 5.274105774333293e-10,
        "model.layers.7.input_layernorm.weight": 0.0,
        "model.layers.7.input_layernorm.bias": 1.7618285302889447e-19,
        "model.layers.7.post_attention_layernorm.weight": 0.0,
        "model.layers.7.post_attention_layernorm.bias": 1.2197274440461925e-19,
        "model.layers.8.self_attn.q_proj.weight": 6.803252006193361e-10,
        "model.layers.8.self_attn.k_proj.weight": 1.0779841375097021e-09,
        "model.layers.8.self_attn.v_proj.weight": 2.402063898898632e-09,
        "model.layers.8.self_attn.o_proj.weight": 2.236740499483858e-09,
        "model.layers.8.mlp.gate_proj.weight": 4.132192543693236e-09,
        "model.layers.8.mlp.up_proj.weight": 2.8210202647945454e-09,
        "model.layers.8.mlp.down_proj.weight": 6.054390174388875e-10,
        "model.layers.8.input_layernorm.weight": 0.0,
        "model.layers.8.input_layernorm.bias": 4.336808689942018e-20,
        "model.layers.8.post_attention_layernorm.weight": 0.0,
        "model.layers.8.post_attention_layernorm.bias": 0.0,
        "model.layers.9.self_attn.q_proj.weight": 6.007289243825653e-10,
        "model.layers.9.self_attn.k_proj.weight": 6.41353707919734e-10,
        "model.layers.9.self_attn.v_proj.weight": 2.956998343961459e-09,
        "model.layers.9.self_attn.o_proj.weight": 2.4913035190365917e-09,
        "model.layers.9.mlp.gate_proj.weight": 7.570661182398908e-09,
        "model.layers.9.mlp.up_proj.weight": 6.353935380805827e-09,
        "model.layers.9.mlp.down_proj.weight": 8.711306046886118e-10,
        "model.layers.9.input_layernorm.weight": 0.0,
        "model.layers.9.input_layernorm.bias": 2.168404344971009e-19,
        "model.layers.9.post_attention_layernorm.weight": 0.0,
        "model.layers.9.post_attention_layernorm.bias": 4.906014830496908e-19,
        "model.layers.10.self_attn.q_proj.weight": 5.71690890183163e-10,
        "model.layers.10.self_attn.k_proj.weight": 7.170714866333583e-10,
        "model.layers.10.self_attn.v_proj.weight": 5.3668185273636486e-09,
        "model.layers.10.self_attn.o_proj.weight": 1.271927203561063e-09,
        "model.layers.10.mlp.gate_proj.weight": 5.268815736296043e-09,
        "model.layers.10.mlp.up_proj.weight": 3.586305865610484e-09,
        "model.layers.10.mlp.down_proj.weight": 5.071385089649993e-10,
        "model.layers.10.input_layernorm.weight": 0.0,
        "model.layers.10.input_layernorm.bias": 0.0,
        "model.layers.10.post_attention_layernorm.weight": 0.0,
        "model.layers.10.post_attention_layernorm.bias": 4.445228907190568e-19,
        "model.layers.11.self_attn.q_proj.weight": 7.266194046451346e-10,
        "model.layers.11.self_attn.k_proj.weight": 6.184255596508592e-10,
        "model.layers.11.self_attn.v_proj.weight": 2.661776079548872e-09,
        "model.layers.11.self_attn.o_proj.weight": 1.118204693284497e-09,
        "model.layers.11.mlp.gate_proj.weight": 4.1432076614000836e-09,
        "model.layers.11.mlp.up_proj.weight": 3.5073454303831873e-09,
        "model.layers.11.mlp.down_proj.weight": 5.212785462542039e-10,
        "model.layers.11.input_layernorm.weight": 0.0,
        "model.layers.11.input_layernorm.bias": 0.0,
        "model.layers.11.post_attention_layernorm.weight": 0.0,
        "model.layers.11.post_attention_layernorm.bias": 2.168404344971009e-19,
        "model.layers.12.self_attn.q_proj.weight": 5.733744146141361e-10,
        "model.layers.12.self_attn.k_proj.weight": 5.487667209536085e-10,
        "model.layers.12.self_attn.v_proj.weight": 4.031819287320104e-09,
        "model.layers.12.self_attn.o_proj.weight": 1.7893317760808713e-09,
        "model.layers.12.mlp.gate_proj.weight": 6.7238200548066134e-09,
        "model.layers.12.mlp.up_proj.weight": 4.333610756172371e-09,
        "model.layers.12.mlp.down_proj.weight": 3.8647418894773487e-10,
        "model.layers.12.input_layernorm.weight": 0.0,
        "model.layers.12.input_layernorm.bias": 1.7347234759768072e-19,
        "model.layers.12.post_attention_layernorm.weight": 0.0,
        "model.layers.12.post_attention_layernorm.bias": 1.7347234759768072e-19,
        "model.layers.13.self_attn.q_proj.weight": 3.6206333930977054e-10,
        "model.layers.13.self_attn.k_proj.weight": 3.510607982093461e-10,
        "model.layers.13.self_attn.v_proj.weight": 3.862275548272009e-09,
        "model.layers.13.self_attn.o_proj.weight": 8.800259365671081e-10,
        "model.layers.13.mlp.gate_proj.weight": 5.46204698063472e-09,
        "model.layers.13.mlp.up_proj.weight": 4.405306223513131e-09,
        "model.layers.13.mlp.down_proj.weight": 4.107201808286239e-10,
        "model.layers.13.input_layernorm.weight": 0.0,
        "model.layers.13.input_layernorm.bias": 0.0,
        "model.layers.13.post_attention_layernorm.weight": 0.0,
        "model.layers.13.post_attention_layernorm.bias": 0.0,
        "model.layers.14.self_attn.q_proj.weight": 3.8464165186269383e-10,
        "model.layers.14.self_attn.k_proj.weight": 2.8596257806157154e-10,
        "model.layers.14.self_attn.v_proj.weight": 3.1012530143925687e-09,
        "model.layers.14.self_attn.o_proj.weight": 1.4942531834094552e-09,
        "model.layers.14.mlp.gate_proj.weight": 6.4883253495888005e-09,
        "model.layers.14.mlp.up_proj.weight": 6.394475349225104e-09,
        "model.layers.14.mlp.down_proj.weight": 3.752011915973223e-10,
        "model.layers.14.input_layernorm.weight": 0.0,
        "model.layers.14.input_layernorm.bias": 0.0,
        "model.layers.14.post_attention_layernorm.weight": 0.0,
        "model.layers.14.post_attention_layernorm.bias": 4.0115480381963666e-19,
        "model.layers.15.self_attn.q_proj.weight": 4.776023843078292e-10,
        "model.layers.15.self_attn.k_proj.weight": 4.016206744950068e-10,
        "model.layers.15.self_attn.v_proj.weight": 4.840656515625596e-09,
        "model.layers.15.self_attn.o_proj.weight": 1.2165354235094129e-09,
        "model.layers.15.mlp.gate_proj.weight": 6.1591943752622074e-09,
        "model.layers.15.mlp.up_proj.weight": 5.179849722480867e-09,
        "model.layers.15.mlp.down_proj.weight": 2.731740285829195e-10,
        "model.layers.15.input_layernorm.weight": 0.0,
        "model.layers.15.input_layernorm.bias": 9.75781955236954e-20,
        "model.layers.15.post_attention_layernorm.weight": 0.0,
        "model.layers.15.post_attention_layernorm.bias": 1.951563910473908e-19,
        "model.layers.16.self_attn.q_proj.weight": 2.842091184618312e-10,
        "model.layers.16.self_attn.k_proj.weight": 5.433628658124689e-10,
        "model.layers.16.self_attn.v_proj.weight": 4.038230372316321e-09,
        "model.layers.16.self_attn.o_proj.weight": 8.410473384401484e-10,
        "model.layers.16.mlp.gate_proj.weight": 5.810029506392311e-09,
        "model.layers.16.mlp.up_proj.weight": 4.040571184305008e-09,
        "model.layers.16.mlp.down_proj.weight": 3.436520283154702e-10,
        "model.layers.16.input_layernorm.weight": 0.0,
        "model.layers.16.input_layernorm.bias": 4.336808689942018e-20,
        "model.layers.16.post_attention_layernorm.weight": 0.0,
        "model.layers.16.post_attention_layernorm.bias": 3.4694469519536144e-19,
        "model.layers.17.self_attn.q_proj.weight": 5.428772098525769e-10,
        "model.layers.17.self_attn.k_proj.weight": 4.891210991786466e-10,
        "model.layers.17.self_attn.v_proj.weight": 5.218405476625776e-09,
        "model.layers.17.self_attn.o_proj.weight": 1.6160302607204358e-09,
        "model.layers.17.mlp.gate_proj.weight": 4.256941489681291e-09,
        "model.layers.17.mlp.up_proj.weight": 2.7025096945484016e-09,
        "model.layers.17.mlp.down_proj.weight": 3.0956850120370897e-10,
        "model.layers.17.input_layernorm.weight": 0.0,
        "model.layers.17.input_layernorm.bias": 0.0,
        "model.layers.17.post_attention_layernorm.weight": 0.0,
        "model.layers.17.post_attention_layernorm.bias": 2.2768245622195594e-19,
        "model.layers.18.self_attn.q_proj.weight": 9.754786134408278e-10,
        "model.layers.18.self_attn.k_proj.weight": 1.0905268510441602e-09,
        "model.layers.18.self_attn.v_proj.weight": 1.7681153963167162e-09,
        "model.layers.18.self_attn.o_proj.weight": 6.709172595265045e-10,
        "model.layers.18.mlp.gate_proj.weight": 2.5347956504144045e-09,
        "model.layers.18.mlp.up_proj.weight": 3.966439786988e-09,
        "model.layers.18.mlp.down_proj.weight": 3.36461501622883e-10,
        "model.layers.18.input_layernorm.weight": 0.0,
        "model.layers.18.input_layernorm.bias": 1.7347234759768072e-19,
        "model.layers.18.post_attention_layernorm.weight": 0.0,
        "model.layers.18.post_attention_layernorm.bias": 1.0842021724855045e-20,
        "model.layers.19.self_attn.q_proj.weight": 5.481194875756046e-10,
        "model.layers.19.self_attn.k_proj.weight": 7.619642872214172e-10,
        "model.layers.19.self_attn.v_proj.weight": 2.641356218191504e-09,
        "model.layers.19.self_attn.o_proj.weight": 9.716190874087261e-10,
        "model.layers.19.mlp.gate_proj.weight": 2.508716079319129e-09,
        "model.layers.19.mlp.up_proj.weight": 2.292312615281844e-09,
        "model.layers.19.mlp.down_proj.weight": 3.7359500974313657e-10,
        "model.layers.19.input_layernorm.weight": 0.0,
        "model.layers.19.input_layernorm.bias": 4.336808689942018e-20,
        "model.layers.19.post_attention_layernorm.weight": 0.0,
        "model.layers.19.post_attention_layernorm.bias": 3.496552006265752e-19,
        "model.layers.20.self_attn.q_proj.weight": 4.460271441075747e-10,
        "model.layers.20.self_attn.k_proj.weight": 5.93622040412356e-10,
        "model.layers.20.self_attn.v_proj.weight": 2.9228209541543036e-09,
        "model.layers.20.self_attn.o_proj.weight": 9.93879041288892e-10,
        "model.layers.20.mlp.gate_proj.weight": 4.262360941235481e-09,
        "model.layers.20.mlp.up_proj.weight": 2.8738960130188692e-09,
        "model.layers.20.mlp.down_proj.weight": 3.880129639810548e-10,
        "model.layers.20.input_layernorm.weight": 0.0,
        "model.layers.20.input_layernorm.bias": 2.710505431213761e-19,
        "model.layers.20.post_attention_layernorm.weight": 0.0,
        "model.layers.20.post_attention_layernorm.bias": 3.686287386450715e-19,
        "model.layers.21.self_attn.q_proj.weight": 7.124691592252929e-10,
        "model.layers.21.self_attn.k_proj.weight": 5.755662257911353e-10,
        "model.layers.21.self_attn.v_proj.weight": 1.891308869517161e-09,
        "model.layers.21.self_attn.o_proj.weight": 1.4094457867486198e-09,
        "model.layers.21.mlp.gate_proj.weight": 4.1152960269149236e-09,
        "model.layers.21.mlp.up_proj.weight": 4.749576116106861e-09,
        "model.layers.21.mlp.down_proj.weight": 6.98995326805137e-10,
        "model.layers.21.input_layernorm.weight": 0.0,
        "model.layers.21.input_layernorm.bias": 0.0,
        "model.layers.21.post_attention_layernorm.weight": 0.0,
        "model.layers.21.post_attention_layernorm.bias": 0.0,
        "model.layers.22.self_attn.q_proj.weight": 9.83956525146823e-10,
        "model.layers.22.self_attn.k_proj.weight": 7.087214726197998e-10,
        "model.layers.22.self_attn.v_proj.weight": 2.4996680281219597e-09,
        "model.layers.22.self_attn.o_proj.weight": 8.651306160345484e-10,
        "model.layers.22.mlp.gate_proj.weight": 4.86394924337219e-09,
        "model.layers.22.mlp.up_proj.weight": 4.5905771154745404e-09,
        "model.layers.22.mlp.down_proj.weight": 3.95163937128018e-10,
        "model.layers.22.input_layernorm.weight": 0.0,
        "model.layers.22.input_layernorm.bias": 1.7347234759768072e-19,
        "model.layers.22.post_attention_layernorm.weight": 0.0,
        "model.layers.22.post_attention_layernorm.bias": 4.336808689942018e-20,
        "model.layers.23.self_attn.q_proj.weight": 4.918045704016549e-10,
        "model.layers.23.self_attn.k_proj.weight": 8.33777420439219e-10,
        "model.layers.23.self_attn.v_proj.weight": 2.41253161448185e-09,
        "model.layers.23.self_attn.o_proj.weight": 1.0251370241576296e-09,
        "model.layers.23.mlp.gate_proj.weight": 7.800711803914358e-09,
        "model.layers.23.mlp.up_proj.weight": 3.2208852947708994e-09,
        "model.layers.23.mlp.down_proj.weight": 4.422955157679098e-10,
        "model.layers.23.input_layernorm.weight": 0.0,
        "model.layers.23.input_layernorm.bias": 3.9302328752599536e-19,
        "model.layers.23.post_attention_layernorm.weight": 0.0,
        "model.layers.23.post_attention_layernorm.bias": 0.0,
        "model.layers.24.self_attn.q_proj.weight": 5.472487174529306e-10,
        "model.layers.24.self_attn.k_proj.weight": 4.2753729445621503e-10,
        "model.layers.24.self_attn.v_proj.weight": 1.443581112425818e-09,
        "model.layers.24.self_attn.o_proj.weight": 8.208573376577987e-10,
        "model.layers.24.mlp.gate_proj.weight": 2.998762624883966e-09,
        "model.layers.24.mlp.up_proj.weight": 3.6555445603880797e-09,
        "model.layers.24.mlp.down_proj.weight": 5.055675463457495e-10,
        "model.layers.24.input_layernorm.weight": 0.0,
        "model.layers.24.input_layernorm.bias": 0.0,
        "model.layers.24.post_attention_layernorm.weight": 0.0,
        "model.layers.24.post_attention_layernorm.bias": 0.0,
        "model.layers.25.self_attn.q_proj.weight": 1.166278238429186e-09,
        "model.layers.25.self_attn.k_proj.weight": 7.62426850542397e-10,
        "model.layers.25.self_attn.v_proj.weight": 1.1932593224628363e-09,
        "model.layers.25.self_attn.o_proj.weight": 8.748911994871378e-10,
        "model.layers.25.mlp.gate_proj.weight": 5.748820512962994e-09,
        "model.layers.25.mlp.up_proj.weight": 3.9719869467565635e-09,
        "model.layers.25.mlp.down_proj.weight": 5.551282811211422e-10,
        "model.layers.25.input_layernorm.weight": 0.0,
        "model.layers.25.input_layernorm.bias": 4.336808689942018e-20,
        "model.layers.25.post_attention_layernorm.weight": 0.0,
        "model.layers.25.post_attention_layernorm.bias": 1.951563910473908e-19,
        "model.layers.26.self_attn.q_proj.weight": 1.0714719422821872e-09,
        "model.layers.26.self_attn.k_proj.weight": 6.015845599449676e-10,
        "model.layers.26.self_attn.v_proj.weight": 8.382126992501071e-10,
        "model.layers.26.self_attn.o_proj.weight": 6.916177852644978e-10,
        "model.layers.26.mlp.gate_proj.weight": 5.836964949897568e-09,
        "model.layers.26.mlp.up_proj.weight": 2.5183671444513798e-09,
        "model.layers.26.mlp.down_proj.weight": 5.167603944755683e-10,
        "model.layers.26.input_layernorm.weight": 0.0,
        "model.layers.26.input_layernorm.bias": 0.0,
        "model.layers.26.post_attention_layernorm.weight": 0.0,
        "model.layers.26.post_attention_layernorm.bias": 0.0,
        "model.layers.27.self_attn.q_proj.weight": 1.2824335726691061e-09,
        "model.layers.27.self_attn.k_proj.weight": 5.505033939812165e-10,
        "model.layers.27.self_attn.v_proj.weight": 1.0342314027411703e-09,
        "model.layers.27.self_attn.o_proj.weight": 8.070220758327195e-10,
        "model.layers.27.mlp.gate_proj.weight": 9.268433132092468e-09,
        "model.layers.27.mlp.up_proj.weight": 4.086552962689893e-09,
        "model.layers.27.mlp.down_proj.weight": 5.494656344732599e-10,
        "model.layers.27.input_layernorm.weight": 0.0,
        "model.layers.27.input_layernorm.bias": 0.0,
        "model.layers.27.post_attention_layernorm.weight": 0.0,
        "model.layers.27.post_attention_layernorm.bias": 4.336808689942018e-20,
        "model.layers.28.self_attn.q_proj.weight": 1.4427759253976547e-09,
        "model.layers.28.self_attn.k_proj.weight": 8.499876003043027e-10,
        "model.layers.28.self_attn.v_proj.weight": 2.8016114583806485e-09,
        "model.layers.28.self_attn.o_proj.weight": 8.107200244467094e-10,
        "model.layers.28.mlp.gate_proj.weight": 5.27831199785093e-09,
        "model.layers.28.mlp.up_proj.weight": 5.316902994915533e-09,
        "model.layers.28.mlp.down_proj.weight": 5.632971541065975e-10,
        "model.layers.28.input_layernorm.weight": 0.0,
        "model.layers.28.input_layernorm.bias": 3.4846935450041914e-19,
        "model.layers.28.post_attention_layernorm.weight": 0.0,
        "model.layers.28.post_attention_layernorm.bias": 0.0,
        "model.layers.29.self_attn.q_proj.weight": 9.260426736545924e-10,
        "model.layers.29.self_attn.k_proj.weight": 6.835472277089139e-10,
        "model.layers.29.self_attn.v_proj.weight": 9.57920534006007e-10,
        "model.layers.29.self_attn.o_proj.weight": 9.050408777966367e-10,
        "model.layers.29.mlp.gate_proj.weight": 7.60078516274613e-09,
        "model.layers.29.mlp.up_proj.weight": 3.756985430906449e-09,
        "model.layers.29.mlp.down_proj.weight": 4.79516586437967e-10,
        "model.layers.29.input_layernorm.weight": 0.0,
        "model.layers.29.input_layernorm.bias": 0.0,
        "model.layers.29.post_attention_layernorm.weight": 0.0,
        "model.layers.29.post_attention_layernorm.bias": 0.0,
        "model.layers.30.self_attn.q_proj.weight": 1.3636689288887282e-09,
        "model.layers.30.self_attn.k_proj.weight": 1.5714329038019058e-09,
        "model.layers.30.self_attn.v_proj.weight": 1.1758381646131966e-09,
        "model.layers.30.self_attn.o_proj.weight": 9.795578392868266e-10,
        "model.layers.30.mlp.gate_proj.weight": 9.374549942246327e-09,
        "model.layers.30.mlp.up_proj.weight": 5.656590928992955e-09,
        "model.layers.30.mlp.down_proj.weight": 5.065968859222873e-10,
        "model.layers.30.input_layernorm.weight": 0.0,
        "model.layers.30.input_layernorm.bias": 0.0,
        "model.layers.30.post_attention_layernorm.weight": 0.0,
        "model.layers.30.post_attention_layernorm.bias": 0.0,
        "model.layers.31.self_attn.q_proj.weight": 2.28110124567138e-09,
        "model.layers.31.self_attn.k_proj.weight": 1.3092864037389517e-09,
        "model.layers.31.self_attn.v_proj.weight": 1.0963311325440373e-09,
        "model.layers.31.self_attn.o_proj.weight": 9.979353166045257e-10,
        "model.layers.31.mlp.gate_proj.weight": 1.401771745198251e-08,
        "model.layers.31.mlp.up_proj.weight": 4.416380458375594e-09,
        "model.layers.31.mlp.down_proj.weight": 7.472823426724062e-10,
        "model.layers.31.input_layernorm.weight": 0.0,
        "model.layers.31.input_layernorm.bias": 1.7347234759768072e-19,
        "model.layers.31.post_attention_layernorm.weight": 0.0,
        "model.layers.31.post_attention_layernorm.bias": 1.7347234759768072e-19,
        "model.layers.32.self_attn.q_proj.weight": 1.4159746797304251e-09,
        "model.layers.32.self_attn.k_proj.weight": 1.0075619627514242e-09,
        "model.layers.32.self_attn.v_proj.weight": 2.012204163293063e-09,
        "model.layers.32.self_attn.o_proj.weight": 1.2717424624497653e-09,
        "model.layers.32.mlp.gate_proj.weight": 1.0099245173478266e-08,
        "model.layers.32.mlp.up_proj.weight": 9.207220349101893e-09,
        "model.layers.32.mlp.down_proj.weight": 8.522503473310887e-10,
        "model.layers.32.input_layernorm.weight": 0.0,
        "model.layers.32.input_layernorm.bias": 0.0,
        "model.layers.32.post_attention_layernorm.weight": 0.0,
        "model.layers.32.post_attention_layernorm.bias": 0.0,
        "model.layers.33.self_attn.q_proj.weight": 1.7309234578988253e-09,
        "model.layers.33.self_attn.k_proj.weight": 1.6029864013944461e-09,
        "model.layers.33.self_attn.v_proj.weight": 4.697651831975236e-09,
        "model.layers.33.self_attn.o_proj.weight": 1.531523423636827e-09,
        "model.layers.33.mlp.gate_proj.weight": 1.4452407413045876e-08,
        "model.layers.33.mlp.up_proj.weight": 9.082494519437507e-09,
        "model.layers.33.mlp.down_proj.weight": 9.669621666337964e-10,
        "model.layers.33.input_layernorm.weight": 0.0,
        "model.layers.33.input_layernorm.bias": 4.336808689942018e-20,
        "model.layers.33.post_attention_layernorm.weight": 0.0,
        "model.layers.33.post_attention_layernorm.bias": 0.0,
        "model.layers.34.self_attn.q_proj.weight": 2.4314635993505364e-09,
        "model.layers.34.self_attn.k_proj.weight": 1.703990619716933e-09,
        "model.layers.34.self_attn.v_proj.weight": 2.379507009209192e-09,
        "model.layers.34.self_attn.o_proj.weight": 1.3013656996463397e-09,
        "model.layers.34.mlp.gate_proj.weight": 1.7839614277666746e-08,
        "model.layers.34.mlp.up_proj.weight": 8.645561896022022e-09,
        "model.layers.34.mlp.down_proj.weight": 7.185560946254554e-10,
        "model.layers.34.input_layernorm.weight": 0.0,
        "model.layers.34.input_layernorm.bias": 0.0,
        "model.layers.34.post_attention_layernorm.weight": 0.0,
        "model.layers.34.post_attention_layernorm.bias": 0.0,
        "model.layers.35.self_attn.q_proj.weight": 2.9998821560184297e-09,
        "model.layers.35.self_attn.k_proj.weight": 9.999101990842974e-10,
        "model.layers.35.self_attn.v_proj.weight": 3.91424379131422e-09,
        "model.layers.35.self_attn.o_proj.weight": 1.8113006206021964e-09,
        "model.layers.35.mlp.gate_proj.weight": 2.1299209341426224e-08,
        "model.layers.35.mlp.up_proj.weight": 1.5169396950417043e-08,
        "model.layers.35.mlp.down_proj.weight": 2.0139570248526678e-09,
        "model.layers.35.input_layernorm.weight": 0.0,
        "model.layers.35.input_layernorm.bias": 1.7347234759768072e-19,
        "model.layers.35.post_attention_layernorm.weight": 0.0,
        "model.layers.35.post_attention_layernorm.bias": 0.0,
        "model.layers.36.self_attn.q_proj.weight": 2.6555719045973093e-09,
        "model.layers.36.self_attn.k_proj.weight": 1.5296728861358134e-09,
        "model.layers.36.self_attn.v_proj.weight": 6.2084450291877144e-09,
        "model.layers.36.self_attn.o_proj.weight": 1.9505691284393834e-09,
        "model.layers.36.mlp.gate_proj.weight": 1.6187862759882893e-08,
        "model.layers.36.mlp.up_proj.weight": 1.652175493897327e-08,
        "model.layers.36.mlp.down_proj.weight": 1.3528854954832544e-09,
        "model.layers.36.input_layernorm.weight": 0.0,
        "model.layers.36.input_layernorm.bias": 0.0,
        "model.layers.36.post_attention_layernorm.weight": 0.0,
        "model.layers.36.post_attention_layernorm.bias": 1.7347234759768072e-19,
        "model.layers.37.self_attn.q_proj.weight": 2.406661110399e-09,
        "model.layers.37.self_attn.k_proj.weight": 1.7653792383498513e-09,
        "model.layers.37.self_attn.v_proj.weight": 7.7608228821191e-09,
        "model.layers.37.self_attn.o_proj.weight": 2.089813762040649e-09,
        "model.layers.37.mlp.gate_proj.weight": 1.0708132928508955e-08,
        "model.layers.37.mlp.up_proj.weight": 9.600955763744423e-09,
        "model.layers.37.mlp.down_proj.weight": 1.129514638857169e-09,
        "model.layers.37.input_layernorm.weight": 0.0,
        "model.layers.37.input_layernorm.bias": 0.0,
        "model.layers.37.post_attention_layernorm.weight": 0.0,
        "model.layers.37.post_attention_layernorm.bias": 0.0,
        "model.layers.38.self_attn.q_proj.weight": 2.8534219609355205e-09,
        "model.layers.38.self_attn.k_proj.weight": 2.46495147848691e-09,
        "model.layers.38.self_attn.v_proj.weight": 5.143008934282989e-09,
        "model.layers.38.self_attn.o_proj.weight": 2.608180125207582e-09,
        "model.layers.38.mlp.gate_proj.weight": 1.637199299390583e-08,
        "model.layers.38.mlp.up_proj.weight": 7.963470428270133e-09,
        "model.layers.38.mlp.down_proj.weight": 2.3918623052547144e-09,
        "model.layers.38.input_layernorm.weight": 0.0,
        "model.layers.38.input_layernorm.bias": 0.0,
        "model.layers.38.post_attention_layernorm.weight": 0.0,
        "model.layers.38.post_attention_layernorm.bias": 0.0,
        "model.layers.39.self_attn.q_proj.weight": 1.945519016999242e-09,
        "model.layers.39.self_attn.k_proj.weight": 1.2198944432384451e-09,
        "model.layers.39.self_attn.v_proj.weight": 5.185719373912434e-09,
        "model.layers.39.self_attn.o_proj.weight": 8.505105597578222e-09,
        "model.layers.39.mlp.gate_proj.weight": 2.532518919906579e-08,
        "model.layers.39.mlp.up_proj.weight": 1.6030430742830504e-08,
        "model.layers.39.mlp.down_proj.weight": 2.789723036282036e-09,
        "model.layers.39.input_layernorm.weight": 0.0,
        "model.layers.39.input_layernorm.bias": 0.0,
        "model.layers.39.post_attention_layernorm.weight": 0.0,
        "model.layers.39.post_attention_layernorm.bias": 0.0,
        "model.norm.weight": 0.0,
        "model.norm.bias": 0.0,
        "lm_head.weight": 1.5311139267753199e-19
    },
    "module_to_params": {
        "model.embed_tokens.weight": 2.917434234714228e-19,
        "model.layers.0.self_attn": 7.920487519186281e-09,
        "model.layers.0.mlp": 1.1966177832315832e-08,
        "model.layers.0.input_layernorm": 2.710505431213761e-19,
        "model.layers.0.post_attention_layernorm": 6.938893903907229e-19,
        "model.layers.1.self_attn": 2.2341763070699017e-08,
        "model.layers.1.mlp": 4.642010935640428e-09,
        "model.layers.1.input_layernorm": 3.9916427639358903e-19,
        "model.layers.1.post_attention_layernorm": 1.8431436932253575e-19,
        "model.layers.2.self_attn": 9.60992252885262e-09,
        "model.layers.2.mlp": 3.582362779752657e-09,
        "model.layers.2.input_layernorm": 2.9544509200229997e-19,
        "model.layers.2.post_attention_layernorm": 4.336808689942018e-20,
        "model.layers.3.self_attn": 8.925772405632415e-09,
        "model.layers.3.mlp": 5.664624135685396e-08,
        "model.layers.3.input_layernorm": 1.328147661294743e-19,
        "model.layers.3.post_attention_layernorm": 1.9786689647860455e-19,
        "model.layers.4.self_attn": 7.112303066492131e-09,
        "model.layers.4.mlp": 3.493962632698337e-08,
        "model.layers.4.input_layernorm": 4.797594613248357e-19,
        "model.layers.4.post_attention_layernorm": 1.8702487475374952e-19,
        "model.layers.5.self_attn": 6.413531963289643e-09,
        "model.layers.5.mlp": 7.511871444876305e-09,
        "model.layers.5.input_layernorm": 0.0,
        "model.layers.5.post_attention_layernorm": 4.445228907190568e-19,
        "model.layers.6.self_attn": 5.3933323940214e-09,
        "model.layers.6.mlp": 7.4314101065435046e-09,
        "model.layers.6.input_layernorm": 0.0,
        "model.layers.6.post_attention_layernorm": 2.2768245622195594e-19,
        "model.layers.7.self_attn": 6.313910247968123e-09,
        "model.layers.7.mlp": 5.013754484176995e-09,
        "model.layers.7.input_layernorm": 1.7618285302889447e-19,
        "model.layers.7.post_attention_layernorm": 1.2197274440461925e-19,
        "model.layers.8.self_attn": 6.397113736511528e-09,
        "model.layers.8.mlp": 7.55865182592667e-09,
        "model.layers.8.input_layernorm": 4.336808689942018e-20,
        "model.layers.8.post_attention_layernorm": 0.0,
        "model.layers.9.self_attn": 6.690384495300349e-09,
        "model.layers.9.mlp": 1.4795727167893346e-08,
        "model.layers.9.input_layernorm": 2.168404344971009e-19,
        "model.layers.9.post_attention_layernorm": 4.906014830496908e-19,
        "model.layers.10.self_attn": 7.927508107741233e-09,
        "model.layers.10.mlp": 9.362260110871526e-09,
        "model.layers.10.input_layernorm": 0.0,
        "model.layers.10.post_attention_layernorm": 4.445228907190568e-19,
        "model.layers.11.self_attn": 5.125025737129363e-09,
        "model.layers.11.mlp": 8.171831638037475e-09,
        "model.layers.11.input_layernorm": 0.0,
        "model.layers.11.post_attention_layernorm": 2.168404344971009e-19,
        "model.layers.12.self_attn": 6.943292198968721e-09,
        "model.layers.12.mlp": 1.1443904999926721e-08,
        "model.layers.12.input_layernorm": 1.7347234759768072e-19,
        "model.layers.12.post_attention_layernorm": 1.7347234759768072e-19,
        "model.layers.13.self_attn": 5.4554256223582344e-09,
        "model.layers.13.mlp": 1.0278073384976474e-08,
        "model.layers.13.input_layernorm": 0.0,
        "model.layers.13.post_attention_layernorm": 0.0,
        "model.layers.14.self_attn": 5.2661104277262895e-09,
        "model.layers.14.mlp": 1.3258001890411226e-08,
        "model.layers.14.input_layernorm": 0.0,
        "model.layers.14.post_attention_layernorm": 4.0115480381963666e-19,
        "model.layers.15.self_attn": 6.936414997937845e-09,
        "model.layers.15.mlp": 1.1612218126325994e-08,
        "model.layers.15.input_layernorm": 9.75781955236954e-20,
        "model.layers.15.post_attention_layernorm": 1.951563910473908e-19,
        "model.layers.16.self_attn": 5.706849695030769e-09,
        "model.layers.16.mlp": 1.0194252719012789e-08,
        "model.layers.16.input_layernorm": 4.336808689942018e-20,
        "model.layers.16.post_attention_layernorm": 3.4694469519536144e-19,
        "model.layers.17.self_attn": 7.866434046377435e-09,
        "model.layers.17.mlp": 7.269019685433401e-09,
        "model.layers.17.input_layernorm": 0.0,
        "model.layers.17.post_attention_layernorm": 2.2768245622195594e-19,
        "model.layers.18.self_attn": 4.505038120328209e-09,
        "model.layers.18.mlp": 6.837696939025288e-09,
        "model.layers.18.input_layernorm": 1.7347234759768072e-19,
        "model.layers.18.post_attention_layernorm": 1.0842021724855045e-20,
        "model.layers.19.self_attn": 4.923059080397252e-09,
        "model.layers.19.mlp": 5.174623704344109e-09,
        "model.layers.19.input_layernorm": 4.336808689942018e-20,
        "model.layers.19.post_attention_layernorm": 3.496552006265752e-19,
        "model.layers.20.self_attn": 4.9563491799631266e-09,
        "model.layers.20.mlp": 7.524269918235405e-09,
        "model.layers.20.input_layernorm": 2.710505431213761e-19,
        "model.layers.20.post_attention_layernorm": 3.686287386450715e-19,
        "model.layers.21.self_attn": 4.5887900412822085e-09,
        "model.layers.21.mlp": 9.56386746982692e-09,
        "model.layers.21.input_layernorm": 0.0,
        "model.layers.21.post_attention_layernorm": 0.0,
        "model.layers.22.self_attn": 5.057476641923131e-09,
        "model.layers.22.mlp": 9.849690295974748e-09,
        "model.layers.22.input_layernorm": 1.7347234759768072e-19,
        "model.layers.22.post_attention_layernorm": 4.336808689942018e-20,
        "model.layers.23.self_attn": 4.763250629480353e-09,
        "model.layers.23.mlp": 1.1463892614453168e-08,
        "model.layers.23.input_layernorm": 3.9302328752599536e-19,
        "model.layers.23.post_attention_layernorm": 0.0,
        "model.layers.24.self_attn": 3.239224461992763e-09,
        "model.layers.24.mlp": 7.159874731617795e-09,
        "model.layers.24.input_layernorm": 0.0,
        "model.layers.24.post_attention_layernorm": 0.0,
        "model.layers.25.self_attn": 3.996855610921557e-09,
        "model.layers.25.mlp": 1.0275935740840701e-08,
        "model.layers.25.input_layernorm": 4.336808689942018e-20,
        "model.layers.25.post_attention_layernorm": 1.951563910473908e-19,
        "model.layers.26.self_attn": 3.2028869867417595e-09,
        "model.layers.26.mlp": 8.872092488824517e-09,
        "model.layers.26.input_layernorm": 0.0,
        "model.layers.26.post_attention_layernorm": 0.0,
        "model.layers.27.self_attn": 3.6741904452242126e-09,
        "model.layers.27.mlp": 1.3904451729255621e-08,
        "model.layers.27.input_layernorm": 0.0,
        "model.layers.27.post_attention_layernorm": 4.336808689942018e-20,
        "model.layers.28.self_attn": 5.905095008529315e-09,
        "model.layers.28.mlp": 1.115851214687306e-08,
        "model.layers.28.input_layernorm": 3.4846935450041914e-19,
        "model.layers.28.post_attention_layernorm": 0.0,
        "model.layers.29.self_attn": 3.47255131316615e-09,
        "model.layers.29.mlp": 1.1837287180090546e-08,
        "model.layers.29.input_layernorm": 0.0,
        "model.layers.29.post_attention_layernorm": 0.0,
        "model.layers.30.self_attn": 5.090497836590657e-09,
        "model.layers.30.mlp": 1.553773775716157e-08,
        "model.layers.30.input_layernorm": 0.0,
        "model.layers.30.post_attention_layernorm": 0.0,
        "model.layers.31.self_attn": 5.684654098558895e-09,
        "model.layers.31.mlp": 1.9181380253030513e-08,
        "model.layers.31.input_layernorm": 1.7347234759768072e-19,
        "model.layers.31.post_attention_layernorm": 1.7347234759768072e-19,
        "model.layers.32.self_attn": 5.707483268224678e-09,
        "model.layers.32.mlp": 2.0158715869911248e-08,
        "model.layers.32.input_layernorm": 0.0,
        "model.layers.32.post_attention_layernorm": 0.0,
        "model.layers.33.self_attn": 9.563085114905333e-09,
        "model.layers.33.mlp": 2.4501864099117178e-08,
        "model.layers.33.input_layernorm": 4.336808689942018e-20,
        "model.layers.33.post_attention_layernorm": 0.0,
        "model.layers.34.self_attn": 7.816326927923e-09,
        "model.layers.34.mlp": 2.7203732268314224e-08,
        "model.layers.34.input_layernorm": 0.0,
        "model.layers.34.post_attention_layernorm": 0.0,
        "model.layers.35.self_attn": 9.725336767019142e-09,
        "model.layers.35.mlp": 3.848256331669593e-08,
        "model.layers.35.input_layernorm": 1.7347234759768072e-19,
        "model.layers.35.post_attention_layernorm": 0.0,
        "model.layers.36.self_attn": 1.234425894836022e-08,
        "model.layers.36.mlp": 3.406250319433942e-08,
        "model.layers.36.input_layernorm": 0.0,
        "model.layers.36.post_attention_layernorm": 1.7347234759768072e-19,
        "model.layers.37.self_attn": 1.40226769929086e-08,
        "model.layers.37.mlp": 2.1438603331110546e-08,
        "model.layers.37.input_layernorm": 0.0,
        "model.layers.37.post_attention_layernorm": 0.0,
        "model.layers.38.self_attn": 1.3069562498913002e-08,
        "model.layers.38.mlp": 2.6727325727430677e-08,
        "model.layers.38.input_layernorm": 0.0,
        "model.layers.38.post_attention_layernorm": 0.0,
        "model.layers.39.self_attn": 1.6856238431728344e-08,
        "model.layers.39.mlp": 4.414534297817833e-08,
        "model.layers.39.input_layernorm": 0.0,
        "model.layers.39.post_attention_layernorm": 0.0,
        "model.norm.weight": 0.0,
        "model.norm.bias": 0.0,
        "lm_head.weight": 1.5311139267753199e-19
    }
}