layer,module,loss,samples,damp,time
0,self_attn.qkv_proj,0.0000013210,0.05000,1.033
0,self_attn.o_proj,0.0000006942,0.05000,0.924
0,mlp.gate.wg,0.0000193136,0.05000,0.914
0,mlp.shared_mlp.gate_and_up_proj,0.0001328699,0.05000,0.931
0,mlp.shared_mlp.down_proj,0.0000001367,0.05000,0.692
0,mlp.experts.0.gate_and_up_proj,0.0001083258,0.05000,2.264
0,mlp.experts.1.gate_and_up_proj,0.0000749003,0.05000,2.778
0,mlp.experts.2.gate_and_up_proj,0.0000104977,0.05000,2.885
0,mlp.experts.3.gate_and_up_proj,0.0000411588,0.05000,2.428
0,mlp.experts.4.gate_and_up_proj,0.0000592379,0.05000,2.481
0,mlp.experts.5.gate_and_up_proj,0.0001800526,0.05000,2.717
0,mlp.experts.6.gate_and_up_proj,0.0001108092,0.05000,3.171
0,mlp.experts.7.gate_and_up_proj,0.0000827744,0.05000,3.086
0,mlp.experts.8.gate_and_up_proj,0.0001965260,0.05000,3.178
0,mlp.experts.9.gate_and_up_proj,0.0000166107,0.05000,3.508
0,mlp.experts.10.gate_and_up_proj,0.0001167371,0.05000,3.220
0,mlp.experts.11.gate_and_up_proj,0.0000304897,0.05000,3.227
0,mlp.experts.12.gate_and_up_proj,0.0000222237,0.05000,2.897
0,mlp.experts.13.gate_and_up_proj,0.0000692534,0.05000,2.833
0,mlp.experts.14.gate_and_up_proj,0.0000037838,0.05000,2.745
0,mlp.experts.16.gate_and_up_proj,0.0003199826,0.05000,3.317
0,mlp.experts.15.gate_and_up_proj,0.0001222956,0.05000,3.515
0,mlp.experts.17.gate_and_up_proj,0.0000407326,0.05000,3.356
0,mlp.experts.18.gate_and_up_proj,0.0000474245,0.05000,3.552
0,mlp.experts.19.gate_and_up_proj,0.0000854146,0.05000,3.717
0,mlp.experts.20.gate_and_up_proj,0.0000192879,0.05000,3.398
0,mlp.experts.21.gate_and_up_proj,0.0000697057,0.05000,2.820
0,mlp.experts.22.gate_and_up_proj,0.0000886431,0.05000,2.839
0,mlp.experts.23.gate_and_up_proj,0.0001855115,0.05000,2.838
0,mlp.experts.24.gate_and_up_proj,0.0001141496,0.05000,2.478
0,mlp.experts.25.gate_and_up_proj,0.0000218530,0.05000,2.544
0,mlp.experts.26.gate_and_up_proj,0.0002639470,0.05000,2.539
0,mlp.experts.27.gate_and_up_proj,0.0001882138,0.05000,2.318
0,mlp.experts.29.gate_and_up_proj,0.0000658584,0.05000,2.340
0,mlp.experts.28.gate_and_up_proj,0.0003008131,0.05000,2.434
0,mlp.experts.30.gate_and_up_proj,0.0000321412,0.05000,2.661
0,mlp.experts.31.gate_and_up_proj,0.0001295475,0.05000,2.565
0,mlp.experts.32.gate_and_up_proj,0.0001085272,0.05000,2.619
0,mlp.experts.33.gate_and_up_proj,0.0000453778,0.05000,2.497
0,mlp.experts.34.gate_and_up_proj,0.0001735315,0.05000,2.502
0,mlp.experts.35.gate_and_up_proj,0.0000925053,0.05000,2.512
0,mlp.experts.36.gate_and_up_proj,0.0000101716,0.05000,2.387
0,mlp.experts.37.gate_and_up_proj,0.0001855513,0.05000,2.417
0,mlp.experts.38.gate_and_up_proj,0.0000428233,0.05000,2.342
0,mlp.experts.39.gate_and_up_proj,0.0000852429,0.05000,2.284
0,mlp.experts.40.gate_and_up_proj,0.0002690668,0.05000,2.308
0,mlp.experts.41.gate_and_up_proj,0.0001590157,0.05000,2.390
0,mlp.experts.42.gate_and_up_proj,0.0006843657,0.05000,3.075
0,mlp.experts.43.gate_and_up_proj,0.0001207427,0.05000,3.392
0,mlp.experts.44.gate_and_up_proj,0.0003054813,0.05000,3.260
0,mlp.experts.45.gate_and_up_proj,0.0000951896,0.05000,3.121
0,mlp.experts.47.gate_and_up_proj,0.0003395518,0.05000,3.299
0,mlp.experts.46.gate_and_up_proj,0.0000743718,0.05000,3.379
0,mlp.experts.48.gate_and_up_proj,0.0000556324,0.05000,3.682
0,mlp.experts.50.gate_and_up_proj,0.0003046558,0.05000,3.245
0,mlp.experts.49.gate_and_up_proj,0.0002387028,0.05000,3.553
0,mlp.experts.51.gate_and_up_proj,0.0001513623,0.05000,2.659
0,mlp.experts.52.gate_and_up_proj,0.0003419575,0.05000,2.437
0,mlp.experts.53.gate_and_up_proj,0.0000827590,0.05000,2.366
0,mlp.experts.54.gate_and_up_proj,0.0001931512,0.05000,2.316
0,mlp.experts.55.gate_and_up_proj,0.0001948214,0.05000,2.369
0,mlp.experts.56.gate_and_up_proj,0.0001129530,0.05000,2.374
0,mlp.experts.57.gate_and_up_proj,0.0002457041,0.05000,2.384
0,mlp.experts.58.gate_and_up_proj,0.0001165774,0.05000,2.425
0,mlp.experts.59.gate_and_up_proj,0.0001189524,0.05000,2.467
0,mlp.experts.60.gate_and_up_proj,0.0000665488,0.05000,2.420
0,mlp.experts.61.gate_and_up_proj,0.0000237103,0.05000,2.474
0,mlp.experts.62.gate_and_up_proj,0.0002694456,0.05000,2.405
0,mlp.experts.63.gate_and_up_proj,0.0000988195,0.05000,1.204
0,mlp.experts.0.down_proj,0.0000011447,0.05000,2.012
0,mlp.experts.2.down_proj,0.0000000496,0.05000,2.026
0,mlp.experts.1.down_proj,0.0000010520,0.05000,2.068
0,mlp.experts.4.down_proj,0.0000006606,0.05000,1.853
0,mlp.experts.5.down_proj,0.0000164182,0.05000,1.822
0,mlp.experts.3.down_proj,0.0000003314,0.05000,1.909
0,mlp.experts.6.down_proj,0.0000017066,0.05000,1.742
0,mlp.experts.7.down_proj,0.0000007149,0.05000,1.757
0,mlp.experts.8.down_proj,0.0000015472,0.05000,1.761
0,mlp.experts.9.down_proj,0.0000000214,0.05000,1.690
0,mlp.experts.10.down_proj,0.0000009596,0.05000,1.711
0,mlp.experts.11.down_proj,0.0000004934,0.05000,1.731
0,mlp.experts.12.down_proj,0.0000003657,0.05000,1.751
0,mlp.experts.13.down_proj,0.0000006149,0.05000,1.749
0,mlp.experts.14.down_proj,0.0000000178,0.05000,1.781
0,mlp.experts.15.down_proj,0.0000005773,0.05000,1.837
0,mlp.experts.16.down_proj,0.0000123838,0.05000,1.843
0,mlp.experts.17.down_proj,0.0000005552,0.05000,1.844
0,mlp.experts.18.down_proj,0.0000011380,0.05000,1.815
0,mlp.experts.19.down_proj,0.0000004018,0.05000,1.780
0,mlp.experts.20.down_proj,0.0000001321,0.05000,1.775
0,mlp.experts.22.down_proj,0.0000085246,0.05000,1.771
0,mlp.experts.21.down_proj,0.0000041879,0.05000,1.786
0,mlp.experts.23.down_proj,0.0000046157,0.05000,1.758
0,mlp.experts.24.down_proj,0.0000052220,0.05000,1.815
0,mlp.experts.25.down_proj,0.0000000768,0.05000,1.923
0,mlp.experts.26.down_proj,0.0000023747,0.05000,1.863
0,mlp.experts.27.down_proj,0.0000027811,0.05000,1.856
0,mlp.experts.28.down_proj,0.0000037516,0.05000,1.883
0,mlp.experts.29.down_proj,0.0000012548,0.05000,1.894
0,mlp.experts.30.down_proj,0.0000003933,0.05000,2.245
0,mlp.experts.31.down_proj,0.0000015117,0.05000,2.234
0,mlp.experts.32.down_proj,0.0000012075,0.05000,2.533
0,mlp.experts.33.down_proj,0.0000003703,0.05000,2.377
0,mlp.experts.34.down_proj,0.0000017191,0.05000,2.436
0,mlp.experts.35.down_proj,0.0000016773,0.05000,2.736
0,mlp.experts.36.down_proj,0.0000000722,0.05000,2.322
0,mlp.experts.37.down_proj,0.0000035722,0.05000,2.290
0,mlp.experts.38.down_proj,0.0000007036,0.05000,2.293
0,mlp.experts.39.down_proj,0.0000006147,0.05000,2.230
0,mlp.experts.40.down_proj,0.0000023884,0.05000,2.221
0,mlp.experts.41.down_proj,0.0000083730,0.05000,2.273
0,mlp.experts.42.down_proj,0.0000132523,0.05000,2.205
0,mlp.experts.43.down_proj,0.0000014082,0.05000,2.221
0,mlp.experts.44.down_proj,0.0000034865,0.05000,2.243
0,mlp.experts.45.down_proj,0.0000009402,0.05000,2.255
0,mlp.experts.46.down_proj,0.0000047778,0.05000,2.260
0,mlp.experts.47.down_proj,0.0000027072,0.05000,2.369
0,mlp.experts.48.down_proj,0.0000032031,0.05000,2.173
0,mlp.experts.49.down_proj,0.0000028569,0.05000,2.245
0,mlp.experts.50.down_proj,0.0000040882,0.05000,2.242
0,mlp.experts.51.down_proj,0.0000112064,0.05000,2.142
0,mlp.experts.52.down_proj,0.0000033166,0.05000,2.164
0,mlp.experts.53.down_proj,0.0000005051,0.05000,2.186
0,mlp.experts.54.down_proj,0.0000046332,0.05000,2.211
0,mlp.experts.55.down_proj,0.0000127368,0.05000,2.175
0,mlp.experts.56.down_proj,0.0000007082,0.05000,2.242
0,mlp.experts.57.down_proj,0.0000017157,0.05000,2.140
0,mlp.experts.58.down_proj,0.0000021482,0.05000,2.169
0,mlp.experts.59.down_proj,0.0000016031,0.05000,2.189
0,mlp.experts.60.down_proj,0.0000010528,0.05000,2.112
0,mlp.experts.61.down_proj,0.0000002663,0.05000,2.098
0,mlp.experts.62.down_proj,0.0000054056,0.05000,1.710
0,mlp.experts.63.down_proj,0.0000030174,0.05000,1.032
1,self_attn.qkv_proj,0.0000051038,0.05000,0.926
1,self_attn.o_proj,0.0000033380,0.05000,0.922
1,mlp.gate.wg,0.0000532403,0.05000,0.937
1,mlp.shared_mlp.gate_and_up_proj,0.0002997211,0.05000,0.925
1,mlp.shared_mlp.down_proj,0.0000070259,0.05000,0.722
1,mlp.experts.0.gate_and_up_proj,0.0004859935,0.05000,2.475
1,mlp.experts.1.gate_and_up_proj,0.0001303600,0.05000,2.488
1,mlp.experts.2.gate_and_up_proj,0.0000632412,0.05000,2.494
1,mlp.experts.3.gate_and_up_proj,0.0000755052,0.05000,3.048
1,mlp.experts.4.gate_and_up_proj,0.0000483369,0.05000,3.272
1,mlp.experts.5.gate_and_up_proj,0.0000821033,0.05000,3.355
1,mlp.experts.6.gate_and_up_proj,0.0000332146,0.05000,3.317
1,mlp.experts.7.gate_and_up_proj,0.0002003007,0.05000,3.328
1,mlp.experts.8.gate_and_up_proj,0.0000394988,0.05000,3.597
1,mlp.experts.9.gate_and_up_proj,0.0001302042,0.05000,2.841
1,mlp.experts.10.gate_and_up_proj,0.0000528522,0.05000,2.859
1,mlp.experts.11.gate_and_up_proj,0.0000304049,0.05000,2.751
1,mlp.experts.12.gate_and_up_proj,0.0001239488,0.05000,2.544
1,mlp.experts.13.gate_and_up_proj,0.0000800479,0.05000,2.755
1,mlp.experts.14.gate_and_up_proj,0.0000353804,0.05000,2.591
1,mlp.experts.15.gate_and_up_proj,0.0000679141,0.05000,2.394
1,mlp.experts.17.gate_and_up_proj,0.0002310623,0.05000,2.364
1,mlp.experts.16.gate_and_up_proj,0.0001350448,0.05000,2.585
1,mlp.experts.18.gate_and_up_proj,0.0000135923,0.05000,2.403
1,mlp.experts.19.gate_and_up_proj,0.0000729392,0.05000,2.422
1,mlp.experts.20.gate_and_up_proj,0.0000977872,0.05000,2.547
1,mlp.experts.21.gate_and_up_proj,0.0000821548,0.05000,2.284
1,mlp.experts.22.gate_and_up_proj,0.0000745188,0.05000,2.307
1,mlp.experts.23.gate_and_up_proj,0.0001941427,0.05000,2.361
1,mlp.experts.24.gate_and_up_proj,0.0000375462,0.05000,2.231
1,mlp.experts.25.gate_and_up_proj,0.0000695331,0.05000,2.269
1,mlp.experts.26.gate_and_up_proj,0.0000456613,0.05000,2.414
1,mlp.experts.27.gate_and_up_proj,0.0000433122,0.05000,2.251
1,mlp.experts.28.gate_and_up_proj,0.0000605339,0.05000,2.318
1,mlp.experts.29.gate_and_up_proj,0.0000418677,0.05000,2.468
1,mlp.experts.30.gate_and_up_proj,0.0000863213,0.05000,2.247
1,mlp.experts.31.gate_and_up_proj,0.0000913844,0.05000,2.298
1,mlp.experts.32.gate_and_up_proj,0.0000505317,0.05000,2.301
1,mlp.experts.33.gate_and_up_proj,0.0002106223,0.05000,2.236
1,mlp.experts.34.gate_and_up_proj,0.0000967719,0.05000,2.270
1,mlp.experts.35.gate_and_up_proj,0.0001383943,0.05000,2.465
1,mlp.experts.36.gate_and_up_proj,0.0000479545,0.05000,2.240
1,mlp.experts.37.gate_and_up_proj,0.0003458288,0.05000,2.310
1,mlp.experts.38.gate_and_up_proj,0.0000409501,0.05000,2.335
1,mlp.experts.39.gate_and_up_proj,0.0000605247,0.05000,2.277
1,mlp.experts.40.gate_and_up_proj,0.0000497956,0.05000,2.355
1,mlp.experts.41.gate_and_up_proj,0.0000543358,0.05000,2.377
1,mlp.experts.42.gate_and_up_proj,0.0001414728,0.05000,2.278
1,mlp.experts.43.gate_and_up_proj,0.0000311661,0.05000,2.377
1,mlp.experts.44.gate_and_up_proj,0.0000655543,0.05000,2.476
1,mlp.experts.45.gate_and_up_proj,0.0002325718,0.05000,2.342
1,mlp.experts.46.gate_and_up_proj,0.0000258416,0.05000,2.345
1,mlp.experts.47.gate_and_up_proj,0.0002130999,0.05000,2.388
1,mlp.experts.48.gate_and_up_proj,0.0002983082,0.05000,2.299
1,mlp.experts.49.gate_and_up_proj,0.0000661580,0.05000,2.404
1,mlp.experts.50.gate_and_up_proj,0.0000203128,0.05000,2.562
1,mlp.experts.51.gate_and_up_proj,0.0002288541,0.05000,2.337
1,mlp.experts.52.gate_and_up_proj,0.0000377569,0.05000,2.414
1,mlp.experts.53.gate_and_up_proj,0.0000778524,0.05000,2.320
1,mlp.experts.54.gate_and_up_proj,0.0001542183,0.05000,2.287
1,mlp.experts.55.gate_and_up_proj,0.0000924724,0.05000,2.343
1,mlp.experts.57.gate_and_up_proj,0.0000758918,0.05000,2.266
1,mlp.experts.56.gate_and_up_proj,0.0000787396,0.05000,2.366
1,mlp.experts.58.gate_and_up_proj,0.0001080521,0.05000,2.674
1,mlp.experts.59.gate_and_up_proj,0.0001143290,0.05000,2.386
1,mlp.experts.60.gate_and_up_proj,0.0000757194,0.05000,2.534
1,mlp.experts.61.gate_and_up_proj,0.0001201123,0.05000,2.380
1,mlp.experts.62.gate_and_up_proj,0.0000128760,0.05000,2.066
1,mlp.experts.63.gate_and_up_proj,0.0000245553,0.05000,1.939
1,mlp.experts.0.down_proj,0.0000052948,0.05000,1.772
1,mlp.experts.2.down_proj,0.0000008504,0.05000,1.779
1,mlp.experts.1.down_proj,0.0000016991,0.05000,1.831
1,mlp.experts.3.down_proj,0.0000008634,0.05000,1.818
1,mlp.experts.5.down_proj,0.0000011558,0.05000,1.854
1,mlp.experts.4.down_proj,0.0000005744,0.05000,1.980
1,mlp.experts.6.down_proj,0.0000004622,0.05000,1.959
1,mlp.experts.7.down_proj,0.0000041182,0.05000,1.926
1,mlp.experts.8.down_proj,0.0000005999,0.05000,1.934
1,mlp.experts.9.down_proj,0.0000019476,0.05000,1.979
1,mlp.experts.10.down_proj,0.0000006341,0.05000,2.095
1,mlp.experts.11.down_proj,0.0000006483,0.05000,2.032
1,mlp.experts.12.down_proj,0.0000016017,0.05000,2.502
1,mlp.experts.14.down_proj,0.0000004133,0.05000,2.495
1,mlp.experts.13.down_proj,0.0000025243,0.05000,2.828
1,mlp.experts.15.down_proj,0.0000011742,0.05000,2.302
1,mlp.experts.16.down_proj,0.0000024735,0.05000,2.229
1,mlp.experts.17.down_proj,0.0000041530,0.05000,2.184
1,mlp.experts.18.down_proj,0.0000001728,0.05000,2.062
1,mlp.experts.19.down_proj,0.0000010190,0.05000,2.034
1,mlp.experts.20.down_proj,0.0000013823,0.05000,2.037
1,mlp.experts.21.down_proj,0.0000009915,0.05000,2.084
1,mlp.experts.22.down_proj,0.0000007424,0.05000,2.081
1,mlp.experts.23.down_proj,0.0000035476,0.05000,2.116
1,mlp.experts.24.down_proj,0.0000004274,0.05000,2.107
1,mlp.experts.25.down_proj,0.0000009268,0.05000,2.101
1,mlp.experts.26.down_proj,0.0000005603,0.05000,2.157
1,mlp.experts.27.down_proj,0.0000007367,0.05000,2.078
1,mlp.experts.28.down_proj,0.0000008526,0.05000,2.073
1,mlp.experts.29.down_proj,0.0000005933,0.05000,2.113
1,mlp.experts.30.down_proj,0.0000032272,0.05000,2.135
1,mlp.experts.31.down_proj,0.0000011718,0.05000,2.104
1,mlp.experts.32.down_proj,0.0000005809,0.05000,2.126
1,mlp.experts.34.down_proj,0.0000012400,0.05000,2.006
1,mlp.experts.33.down_proj,0.0000043716,0.05000,2.107
1,mlp.experts.35.down_proj,0.0000012818,0.05000,2.029
1,mlp.experts.36.down_proj,0.0000004736,0.05000,2.108
1,mlp.experts.37.down_proj,0.0000042844,0.05000,2.125
1,mlp.experts.38.down_proj,0.0000003652,0.05000,2.133
1,mlp.experts.39.down_proj,0.0000007157,0.05000,2.126
1,mlp.experts.40.down_proj,0.0000011941,0.05000,2.138
1,mlp.experts.41.down_proj,0.0000010989,0.05000,2.163
1,mlp.experts.42.down_proj,0.0000016609,0.05000,2.098
1,mlp.experts.43.down_proj,0.0000007219,0.05000,2.053
1,mlp.experts.44.down_proj,0.0000015204,0.05000,2.049
1,mlp.experts.45.down_proj,0.0000042061,0.05000,2.006
1,mlp.experts.46.down_proj,0.0000001727,0.05000,1.992
1,mlp.experts.47.down_proj,0.0000034376,0.05000,2.057
1,mlp.experts.48.down_proj,0.0000061126,0.05000,2.046
1,mlp.experts.49.down_proj,0.0000010329,0.05000,2.030
1,mlp.experts.50.down_proj,0.0000004718,0.05000,2.095
1,mlp.experts.51.down_proj,0.0000024939,0.05000,2.026
1,mlp.experts.52.down_proj,0.0000004751,0.05000,2.030
1,mlp.experts.53.down_proj,0.0000012257,0.05000,2.078
1,mlp.experts.54.down_proj,0.0000021556,0.05000,1.991
1,mlp.experts.55.down_proj,0.0000030018,0.05000,1.989
1,mlp.experts.56.down_proj,0.0000017715,0.05000,2.009
1,mlp.experts.57.down_proj,0.0000012353,0.05000,1.958
1,mlp.experts.58.down_proj,0.0000014017,0.05000,1.965
1,mlp.experts.59.down_proj,0.0000029494,0.05000,2.024
1,mlp.experts.60.down_proj,0.0000021181,0.05000,2.003
1,mlp.experts.61.down_proj,0.0000033379,0.05000,2.021
1,mlp.experts.62.down_proj,0.0000001454,0.05000,1.735
1,mlp.experts.63.down_proj,0.0000003748,0.05000,0.855
2,self_attn.qkv_proj,0.0000103872,0.05000,0.949
2,self_attn.o_proj,0.0000078095,0.05000,0.929
2,mlp.gate.wg,0.0000617241,0.05000,0.909
2,mlp.shared_mlp.gate_and_up_proj,0.0005148675,0.05000,0.938
2,mlp.shared_mlp.down_proj,0.0000050205,0.05000,0.699
2,mlp.experts.1.gate_and_up_proj,0.0001621934,0.05000,2.469
2,mlp.experts.2.gate_and_up_proj,0.0003320299,0.05000,2.512
2,mlp.experts.0.gate_and_up_proj,0.0007006008,0.05000,2.514
2,mlp.experts.3.gate_and_up_proj,0.0001415542,0.05000,2.384
2,mlp.experts.5.gate_and_up_proj,0.0003562053,0.05000,2.419
2,mlp.experts.4.gate_and_up_proj,0.0001719892,0.05000,2.572
2,mlp.experts.6.gate_and_up_proj,0.0000946573,0.05000,2.520
2,mlp.experts.8.gate_and_up_proj,0.0002075517,0.05000,2.436
2,mlp.experts.7.gate_and_up_proj,0.0005817800,0.05000,2.621
2,mlp.experts.9.gate_and_up_proj,0.0004988082,0.05000,2.353
2,mlp.experts.10.gate_and_up_proj,0.0003871990,0.05000,2.350
2,mlp.experts.11.gate_and_up_proj,0.0002681158,0.05000,2.549
2,mlp.experts.12.gate_and_up_proj,0.0013603967,0.05000,2.249
2,mlp.experts.13.gate_and_up_proj,0.0000536586,0.05000,2.309
2,mlp.experts.14.gate_and_up_proj,0.0001387926,0.05000,2.351
2,mlp.experts.15.gate_and_up_proj,0.0002084635,0.05000,2.330
2,mlp.experts.16.gate_and_up_proj,0.0008837416,0.05000,2.344
2,mlp.experts.17.gate_and_up_proj,0.0005417382,0.05000,2.334
2,mlp.experts.18.gate_and_up_proj,0.0002603145,0.05000,2.262
2,mlp.experts.19.gate_and_up_proj,0.0001245929,0.05000,2.342
2,mlp.experts.20.gate_and_up_proj,0.0002012397,0.05000,2.331
2,mlp.experts.21.gate_and_up_proj,0.0012311347,0.05000,2.413
2,mlp.experts.22.gate_and_up_proj,0.0000333929,0.05000,2.469
2,mlp.experts.23.gate_and_up_proj,0.0004357920,0.05000,2.453
2,mlp.experts.24.gate_and_up_proj,0.0000389934,0.05000,2.344
2,mlp.experts.25.gate_and_up_proj,0.0004384932,0.05000,2.398
2,mlp.experts.26.gate_and_up_proj,0.0000977125,0.05000,2.406
2,mlp.experts.27.gate_and_up_proj,0.0003809474,0.05000,2.485
2,mlp.experts.28.gate_and_up_proj,0.0002557123,0.05000,2.428
2,mlp.experts.29.gate_and_up_proj,0.0002050602,0.05000,2.409
2,mlp.experts.30.gate_and_up_proj,0.0003593747,0.05000,2.370
2,mlp.experts.31.gate_and_up_proj,0.0004748369,0.05000,2.431
2,mlp.experts.32.gate_and_up_proj,0.0008089837,0.05000,2.439
2,mlp.experts.33.gate_and_up_proj,0.0000902284,0.05000,2.360
2,mlp.experts.34.gate_and_up_proj,0.0004181758,0.05000,2.394
2,mlp.experts.35.gate_and_up_proj,0.0009428974,0.05000,2.350
2,mlp.experts.36.gate_and_up_proj,0.0005297594,0.05000,2.265
2,mlp.experts.37.gate_and_up_proj,0.0009020506,0.05000,2.273
2,mlp.experts.38.gate_and_up_proj,0.0004017656,0.05000,2.291
2,mlp.experts.39.gate_and_up_proj,0.0000774820,0.05000,2.270
2,mlp.experts.40.gate_and_up_proj,0.0001415429,0.05000,2.344
2,mlp.experts.41.gate_and_up_proj,0.0003512282,0.05000,2.318
2,mlp.experts.42.gate_and_up_proj,0.0003935216,0.05000,2.331
2,mlp.experts.43.gate_and_up_proj,0.0000931241,0.05000,2.360
2,mlp.experts.44.gate_and_up_proj,0.0003538199,0.05000,2.340
2,mlp.experts.45.gate_and_up_proj,0.0007784167,0.05000,2.342
2,mlp.experts.46.gate_and_up_proj,0.0000804878,0.05000,2.388
2,mlp.experts.47.gate_and_up_proj,0.0004007432,0.05000,2.343
2,mlp.experts.48.gate_and_up_proj,0.0002518621,0.05000,2.289
2,mlp.experts.49.gate_and_up_proj,0.0001606662,0.05000,2.311
2,mlp.experts.50.gate_and_up_proj,0.0002054233,0.05000,2.322
2,mlp.experts.51.gate_and_up_proj,0.0007126565,0.05000,2.328
2,mlp.experts.52.gate_and_up_proj,0.0000335569,0.05000,2.383
2,mlp.experts.53.gate_and_up_proj,0.0001502224,0.05000,2.391
2,mlp.experts.54.gate_and_up_proj,0.0004366240,0.05000,2.397
2,mlp.experts.55.gate_and_up_proj,0.0001701881,0.05000,2.355
2,mlp.experts.56.gate_and_up_proj,0.0002416170,0.05000,2.362
2,mlp.experts.57.gate_and_up_proj,0.0002579805,0.05000,2.291
2,mlp.experts.58.gate_and_up_proj,0.0002856220,0.05000,2.307
2,mlp.experts.59.gate_and_up_proj,0.0002829524,0.05000,2.328
2,mlp.experts.60.gate_and_up_proj,0.0003191924,0.05000,2.399
2,mlp.experts.61.gate_and_up_proj,0.0004869445,0.05000,2.396
2,mlp.experts.62.gate_and_up_proj,0.0000394865,0.05000,2.341
2,mlp.experts.63.gate_and_up_proj,0.0004412975,0.05000,1.369
2,mlp.experts.0.down_proj,0.0000197287,0.05000,1.805
2,mlp.experts.1.down_proj,0.0000018587,0.05000,1.814
2,mlp.experts.2.down_proj,0.0000122083,0.05000,1.828
2,mlp.experts.3.down_proj,0.0000022098,0.05000,1.764
2,mlp.experts.4.down_proj,0.0000032527,0.05000,1.761
2,mlp.experts.5.down_proj,0.0000072299,0.05000,1.753
2,mlp.experts.6.down_proj,0.0000010931,0.05000,1.768
2,mlp.experts.7.down_proj,0.0000120920,0.05000,1.773
2,mlp.experts.8.down_proj,0.0000031073,0.05000,1.771
2,mlp.experts.11.down_proj,0.0000042748,0.05000,1.738
2,mlp.experts.10.down_proj,0.0000106505,0.05000,1.751
2,mlp.experts.9.down_proj,0.0000063531,0.05000,1.785
2,mlp.experts.12.down_proj,0.0000240966,0.05000,1.705
2,mlp.experts.13.down_proj,0.0000006020,0.05000,1.744
2,mlp.experts.14.down_proj,0.0000020894,0.05000,1.730
2,mlp.experts.15.down_proj,0.0000019487,0.05000,1.689
2,mlp.experts.16.down_proj,0.0000094785,0.05000,1.720
2,mlp.experts.17.down_proj,0.0000072256,0.05000,1.723
2,mlp.experts.18.down_proj,0.0000044087,0.05000,1.744
2,mlp.experts.20.down_proj,0.0000016324,0.05000,1.772
2,mlp.experts.19.down_proj,0.0000024296,0.05000,1.810
2,mlp.experts.22.down_proj,0.0000023658,0.05000,1.852
2,mlp.experts.23.down_proj,0.0000078071,0.05000,1.851
2,mlp.experts.21.down_proj,0.0000227952,0.05000,2.001
2,mlp.experts.24.down_proj,0.0000004567,0.05000,1.946
2,mlp.experts.26.down_proj,0.0000015703,0.05000,1.930
2,mlp.experts.25.down_proj,0.0000045322,0.05000,1.978
2,mlp.experts.27.down_proj,0.0000071155,0.05000,1.885
2,mlp.experts.28.down_proj,0.0000056981,0.05000,1.906
2,mlp.experts.29.down_proj,0.0000045279,0.05000,1.922
2,mlp.experts.30.down_proj,0.0000061232,0.05000,1.912
2,mlp.experts.31.down_proj,0.0000224934,0.05000,1.926
2,mlp.experts.32.down_proj,0.0000118367,0.05000,1.937
2,mlp.experts.33.down_proj,0.0000012052,0.05000,1.877
2,mlp.experts.34.down_proj,0.0000064879,0.05000,1.902
2,mlp.experts.35.down_proj,0.0000217368,0.05000,1.872
2,mlp.experts.36.down_proj,0.0000083102,0.05000,1.833
2,mlp.experts.37.down_proj,0.0000147714,0.05000,1.834
2,mlp.experts.38.down_proj,0.0000117363,0.05000,1.853
2,mlp.experts.39.down_proj,0.0000009739,0.05000,1.807
2,mlp.experts.40.down_proj,0.0000020659,0.05000,1.816
2,mlp.experts.41.down_proj,0.0000031616,0.05000,1.839
2,mlp.experts.42.down_proj,0.0000053344,0.05000,1.919
2,mlp.experts.43.down_proj,0.0000017160,0.05000,1.928
2,mlp.experts.44.down_proj,0.0000053511,0.05000,2.042
2,mlp.experts.45.down_proj,0.0000147297,0.05000,1.970
2,mlp.experts.46.down_proj,0.0000010533,0.05000,1.937
2,mlp.experts.47.down_proj,0.0000071224,0.05000,1.971
2,mlp.experts.48.down_proj,0.0000026844,0.05000,1.899
2,mlp.experts.49.down_proj,0.0000041947,0.05000,1.917
2,mlp.experts.50.down_proj,0.0000063391,0.05000,1.979
2,mlp.experts.51.down_proj,0.0000062980,0.05000,1.917
2,mlp.experts.52.down_proj,0.0000004383,0.05000,1.959
2,mlp.experts.53.down_proj,0.0000018784,0.05000,2.027
2,mlp.experts.54.down_proj,0.0000046238,0.05000,2.022
2,mlp.experts.55.down_proj,0.0000017995,0.05000,2.074
2,mlp.experts.56.down_proj,0.0000024869,0.05000,2.065
2,mlp.experts.57.down_proj,0.0000038402,0.05000,1.956
2,mlp.experts.58.down_proj,0.0000030344,0.05000,1.980
2,mlp.experts.59.down_proj,0.0000050439,0.05000,1.982
2,mlp.experts.60.down_proj,0.0000253331,0.05000,2.002
2,mlp.experts.61.down_proj,0.0000073452,0.05000,2.039
2,mlp.experts.62.down_proj,0.0000005698,0.05000,1.893
2,mlp.experts.63.down_proj,0.0000049538,0.05000,0.919
3,self_attn.qkv_proj,0.0000154700,0.05000,0.928
3,self_attn.o_proj,0.0000068831,0.05000,0.925
3,mlp.gate.wg,0.0000659766,0.05000,0.903
3,mlp.shared_mlp.gate_and_up_proj,0.0007304466,0.05000,0.928
3,mlp.shared_mlp.down_proj,0.0000050786,0.05000,0.687
3,mlp.experts.0.gate_and_up_proj,0.0005771862,0.05000,2.640
3,mlp.experts.2.gate_and_up_proj,0.0003089199,0.05000,2.658
3,mlp.experts.1.gate_and_up_proj,0.0009652832,0.05000,2.710
3,mlp.experts.3.gate_and_up_proj,0.0004372892,0.05000,2.500
3,mlp.experts.5.gate_and_up_proj,0.0002442221,0.05000,2.530
3,mlp.experts.4.gate_and_up_proj,0.0000882386,0.05000,2.640
3,mlp.experts.7.gate_and_up_proj,0.0003305741,0.05000,2.427
3,mlp.experts.6.gate_and_up_proj,0.0003847881,0.05000,2.596
3,mlp.experts.8.gate_and_up_proj,0.0007599284,0.05000,2.468
3,mlp.experts.9.gate_and_up_proj,0.0002253932,0.05000,2.422
3,mlp.experts.10.gate_and_up_proj,0.0014525202,0.05000,2.432
3,mlp.experts.11.gate_and_up_proj,0.0003177464,0.05000,2.463
3,mlp.experts.12.gate_and_up_proj,0.0001630315,0.05000,2.366
3,mlp.experts.13.gate_and_up_proj,0.0002928832,0.05000,2.365
3,mlp.experts.14.gate_and_up_proj,0.0004809555,0.05000,2.387
3,mlp.experts.15.gate_and_up_proj,0.0010670723,0.05000,2.392
3,mlp.experts.16.gate_and_up_proj,0.0007886532,0.05000,2.422
3,mlp.experts.17.gate_and_up_proj,0.0008787753,0.05000,2.411
3,mlp.experts.18.gate_and_up_proj,0.0004593807,0.05000,2.359
3,mlp.experts.19.gate_and_up_proj,0.0001546541,0.05000,2.380
3,mlp.experts.20.gate_and_up_proj,0.0019238065,0.05000,2.374
3,mlp.experts.21.gate_and_up_proj,0.0002251233,0.05000,2.378
3,mlp.experts.22.gate_and_up_proj,0.0002664150,0.05000,2.353
3,mlp.experts.23.gate_and_up_proj,0.0011745466,0.05000,2.373
3,mlp.experts.24.gate_and_up_proj,0.0003079128,0.05000,2.318
3,mlp.experts.25.gate_and_up_proj,0.0002061325,0.05000,2.353
3,mlp.experts.26.gate_and_up_proj,0.0009507070,0.05000,2.350
3,mlp.experts.27.gate_and_up_proj,0.0001137671,0.05000,2.382
3,mlp.experts.28.gate_and_up_proj,0.0003996223,0.05000,2.385
3,mlp.experts.29.gate_and_up_proj,0.0002241898,0.05000,2.413
3,mlp.experts.30.gate_and_up_proj,0.0003978349,0.05000,2.328
3,mlp.experts.31.gate_and_up_proj,0.0002702382,0.05000,2.319
3,mlp.experts.32.gate_and_up_proj,0.0007113594,0.05000,2.327
3,mlp.experts.33.gate_and_up_proj,0.0015900676,0.05000,2.361
3,mlp.experts.34.gate_and_up_proj,0.0004719087,0.05000,2.403
3,mlp.experts.35.gate_and_up_proj,0.0003154924,0.05000,2.421
3,mlp.experts.36.gate_and_up_proj,0.0002810370,0.05000,3.115
3,mlp.experts.37.gate_and_up_proj,0.0003499081,0.05000,3.227
3,mlp.experts.38.gate_and_up_proj,0.0000734476,0.05000,3.481
3,mlp.experts.39.gate_and_up_proj,0.0001481123,0.05000,3.388
3,mlp.experts.40.gate_and_up_proj,0.0014046374,0.05000,3.462
3,mlp.experts.41.gate_and_up_proj,0.0002040659,0.05000,3.527
3,mlp.experts.42.gate_and_up_proj,0.0002590493,0.05000,3.131
3,mlp.experts.43.gate_and_up_proj,0.0002258345,0.05000,3.164
3,mlp.experts.44.gate_and_up_proj,0.0002787547,0.05000,3.485
3,mlp.experts.45.gate_and_up_proj,0.0001023415,0.05000,3.293
3,mlp.experts.46.gate_and_up_proj,0.0002448353,0.05000,3.316
3,mlp.experts.47.gate_and_up_proj,0.0003413458,0.05000,3.815
3,mlp.experts.48.gate_and_up_proj,0.0001792709,0.05000,3.552
3,mlp.experts.49.gate_and_up_proj,0.0003866614,0.05000,3.566
3,mlp.experts.50.gate_and_up_proj,0.0008814492,0.05000,3.710
3,mlp.experts.51.gate_and_up_proj,0.0002413148,0.05000,3.398
3,mlp.experts.52.gate_and_up_proj,0.0002303226,0.05000,3.368
3,mlp.experts.53.gate_and_up_proj,0.0002776605,0.05000,3.578
3,mlp.experts.54.gate_and_up_proj,0.0004768515,0.05000,3.428
3,mlp.experts.55.gate_and_up_proj,0.0007387286,0.05000,3.362
3,mlp.experts.56.gate_and_up_proj,0.0008578828,0.05000,3.513
3,mlp.experts.57.gate_and_up_proj,0.0005701633,0.05000,3.030
3,mlp.experts.58.gate_and_up_proj,0.0000634721,0.05000,2.964
3,mlp.experts.59.gate_and_up_proj,0.0007308037,0.05000,2.567
3,mlp.experts.60.gate_and_up_proj,0.0005372818,0.05000,2.498
3,mlp.experts.61.gate_and_up_proj,0.0004780397,0.05000,2.542
3,mlp.experts.62.gate_and_up_proj,0.0000380607,0.05000,1.954
3,mlp.experts.63.gate_and_up_proj,0.0004240115,0.05000,1.401
3,mlp.experts.1.down_proj,0.0000115157,0.05000,1.950
3,mlp.experts.2.down_proj,0.0000049123,0.05000,1.973
3,mlp.experts.0.down_proj,0.0000088195,0.05000,1.980
3,mlp.experts.3.down_proj,0.0000154417,0.05000,1.821
3,mlp.experts.5.down_proj,0.0000036730,0.05000,1.832
3,mlp.experts.4.down_proj,0.0000011132,0.05000,1.914
3,mlp.experts.7.down_proj,0.0000168119,0.05000,1.786
3,mlp.experts.6.down_proj,0.0000042029,0.05000,1.890
3,mlp.experts.8.down_proj,0.0000185681,0.05000,1.796
3,mlp.experts.9.down_proj,0.0000027856,0.05000,1.774
3,mlp.experts.10.down_proj,0.0000241057,0.05000,1.779
3,mlp.experts.11.down_proj,0.0000036393,0.05000,1.808
3,mlp.experts.12.down_proj,0.0000019487,0.05000,1.795
3,mlp.experts.13.down_proj,0.0000049771,0.05000,1.785
3,mlp.experts.14.down_proj,0.0000097288,0.05000,1.822
3,mlp.experts.15.down_proj,0.0000242422,0.05000,1.798
3,mlp.experts.16.down_proj,0.0000146894,0.05000,1.838
3,mlp.experts.17.down_proj,0.0000128379,0.05000,1.769
3,mlp.experts.18.down_proj,0.0000069434,0.05000,1.809
3,mlp.experts.20.down_proj,0.0000305532,0.05000,1.782
3,mlp.experts.19.down_proj,0.0000021122,0.05000,1.929
3,mlp.experts.22.down_proj,0.0000050053,0.05000,1.855
3,mlp.experts.21.down_proj,0.0000022669,0.05000,2.002
3,mlp.experts.23.down_proj,0.0000286127,0.05000,1.934
3,mlp.experts.24.down_proj,0.0000055334,0.05000,1.850
3,mlp.experts.25.down_proj,0.0000028253,0.05000,1.911
3,mlp.experts.26.down_proj,0.0000158690,0.05000,1.993
3,mlp.experts.27.down_proj,0.0000019726,0.05000,1.893
3,mlp.experts.28.down_proj,0.0000046845,0.05000,1.869
3,mlp.experts.29.down_proj,0.0000029383,0.05000,1.952
3,mlp.experts.30.down_proj,0.0000071303,0.05000,1.833
3,mlp.experts.31.down_proj,0.0000049335,0.05000,1.873
3,mlp.experts.32.down_proj,0.0000155761,0.05000,1.956
3,mlp.experts.33.down_proj,0.0000238052,0.05000,1.846
3,mlp.experts.34.down_proj,0.0000122062,0.05000,1.914
3,mlp.experts.35.down_proj,0.0000033638,0.05000,1.951
3,mlp.experts.36.down_proj,0.0000043585,0.05000,1.840
3,mlp.experts.37.down_proj,0.0000059507,0.05000,1.894
3,mlp.experts.38.down_proj,0.0000011668,0.05000,1.952
3,mlp.experts.39.down_proj,0.0000017111,0.05000,1.882
3,mlp.experts.40.down_proj,0.0000157641,0.05000,1.919
3,mlp.experts.41.down_proj,0.0000029070,0.05000,1.985
3,mlp.experts.42.down_proj,0.0000041368,0.05000,1.911
3,mlp.experts.43.down_proj,0.0000030972,0.05000,1.918
3,mlp.experts.44.down_proj,0.0000048269,0.05000,1.903
3,mlp.experts.45.down_proj,0.0000015866,0.05000,1.894
3,mlp.experts.46.down_proj,0.0000040738,0.05000,1.902
3,mlp.experts.47.down_proj,0.0000049989,0.05000,1.902
3,mlp.experts.48.down_proj,0.0000028828,0.05000,1.966
3,mlp.experts.49.down_proj,0.0000037362,0.05000,1.993
3,mlp.experts.50.down_proj,0.0000142973,0.05000,2.022
3,mlp.experts.51.down_proj,0.0000030592,0.05000,1.925
3,mlp.experts.52.down_proj,0.0000037323,0.05000,1.922
3,mlp.experts.53.down_proj,0.0000048471,0.05000,1.935
3,mlp.experts.54.down_proj,0.0000104952,0.05000,1.890
3,mlp.experts.55.down_proj,0.0000240150,0.05000,1.953
3,mlp.experts.56.down_proj,0.0000132714,0.05000,1.944
3,mlp.experts.57.down_proj,0.0000055173,0.05000,1.927
3,mlp.experts.58.down_proj,0.0000010950,0.05000,1.931
3,mlp.experts.59.down_proj,0.0000108160,0.05000,1.892
3,mlp.experts.60.down_proj,0.0000058385,0.05000,1.853
3,mlp.experts.61.down_proj,0.0000044316,0.05000,1.877
3,mlp.experts.62.down_proj,0.0000004538,0.05000,1.660
3,mlp.experts.63.down_proj,0.0000070272,0.05000,1.070
4,self_attn.qkv_proj,0.0000301769,0.05000,0.938
4,self_attn.o_proj,0.0000140644,0.05000,0.933
4,mlp.gate.wg,0.0000786354,0.05000,0.905
4,mlp.shared_mlp.gate_and_up_proj,0.0007913395,0.05000,0.928
4,mlp.shared_mlp.down_proj,0.0000046881,0.05000,0.689
4,mlp.experts.0.gate_and_up_proj,0.0000633311,0.05000,2.361
4,mlp.experts.1.gate_and_up_proj,0.0005695894,0.05000,2.366
4,mlp.experts.2.gate_and_up_proj,0.0002216710,0.05000,2.390
4,mlp.experts.3.gate_and_up_proj,0.0005949466,0.05000,2.372
4,mlp.experts.4.gate_and_up_proj,0.0004892177,0.05000,2.399
4,mlp.experts.5.gate_and_up_proj,0.0013857502,0.05000,2.396
4,mlp.experts.6.gate_and_up_proj,0.0010582455,0.05000,2.294
4,mlp.experts.7.gate_and_up_proj,0.0007308254,0.05000,2.345
4,mlp.experts.8.gate_and_up_proj,0.0007657146,0.05000,2.346
4,mlp.experts.9.gate_and_up_proj,0.0006521622,0.05000,2.348
4,mlp.experts.10.gate_and_up_proj,0.0005768567,0.05000,2.367
4,mlp.experts.11.gate_and_up_proj,0.0002493473,0.05000,2.380
4,mlp.experts.12.gate_and_up_proj,0.0004681131,0.05000,2.423
4,mlp.experts.13.gate_and_up_proj,0.0001925650,0.05000,2.486
4,mlp.experts.14.gate_and_up_proj,0.0001714302,0.05000,2.469
4,mlp.experts.15.gate_and_up_proj,0.0013421421,0.05000,2.335
4,mlp.experts.16.gate_and_up_proj,0.0003300517,0.05000,2.372
4,mlp.experts.17.gate_and_up_proj,0.0000978303,0.05000,2.401
4,mlp.experts.18.gate_and_up_proj,0.0002724330,0.05000,2.330
4,mlp.experts.19.gate_and_up_proj,0.0000983987,0.05000,2.370
4,mlp.experts.20.gate_and_up_proj,0.0020833329,0.05000,2.363
4,mlp.experts.21.gate_and_up_proj,0.0019028871,0.05000,2.323
4,mlp.experts.22.gate_and_up_proj,0.0000365361,0.05000,2.347
4,mlp.experts.23.gate_and_up_proj,0.0001741835,0.05000,2.377
4,mlp.experts.24.gate_and_up_proj,0.0002935405,0.05000,2.340
4,mlp.experts.25.gate_and_up_proj,0.0011758897,0.05000,2.415
4,mlp.experts.26.gate_and_up_proj,0.0006851344,0.05000,2.377
4,mlp.experts.27.gate_and_up_proj,0.0001920436,0.05000,2.408
4,mlp.experts.28.gate_and_up_proj,0.0003794669,0.05000,2.459
4,mlp.experts.29.gate_and_up_proj,0.0001464180,0.05000,2.440
4,mlp.experts.30.gate_and_up_proj,0.0004932969,0.05000,2.302
4,mlp.experts.31.gate_and_up_proj,0.0006446654,0.05000,2.333
4,mlp.experts.32.gate_and_up_proj,0.0005141438,0.05000,2.345
4,mlp.experts.33.gate_and_up_proj,0.0007323213,0.05000,2.393
4,mlp.experts.34.gate_and_up_proj,0.0000000000,0.05000,2.442
4,mlp.experts.35.gate_and_up_proj,0.0002583642,0.05000,2.420
4,mlp.experts.36.gate_and_up_proj,0.0003060734,0.05000,2.618
4,mlp.experts.37.gate_and_up_proj,0.0005405806,0.05000,2.753
4,mlp.experts.38.gate_and_up_proj,0.0009231812,0.05000,2.768
4,mlp.experts.39.gate_and_up_proj,0.0003218774,0.05000,2.733
4,mlp.experts.40.gate_and_up_proj,0.0002610776,0.05000,2.859
4,mlp.experts.41.gate_and_up_proj,0.0001165838,0.05000,2.955
4,mlp.experts.42.gate_and_up_proj,0.0006525074,0.05000,2.691
4,mlp.experts.43.gate_and_up_proj,0.0008995159,0.05000,2.619
4,mlp.experts.44.gate_and_up_proj,0.0018266537,0.05000,2.616
4,mlp.experts.45.gate_and_up_proj,0.0008559718,0.05000,2.617
4,mlp.experts.46.gate_and_up_proj,0.0008535720,0.05000,2.544
4,mlp.experts.47.gate_and_up_proj,0.0000000000,0.05000,2.585
4,mlp.experts.48.gate_and_up_proj,0.0012932079,0.05000,2.504
4,mlp.experts.49.gate_and_up_proj,0.0005613387,0.05000,2.518
4,mlp.experts.50.gate_and_up_proj,0.0001132603,0.05000,2.500
4,mlp.experts.51.gate_and_up_proj,0.0008756125,0.05000,2.507
4,mlp.experts.52.gate_and_up_proj,0.0006704857,0.05000,2.537
4,mlp.experts.53.gate_and_up_proj,0.0017854000,0.05000,2.507
4,mlp.experts.54.gate_and_up_proj,0.0002753286,0.05000,2.366
4,mlp.experts.55.gate_and_up_proj,0.0001667004,0.05000,2.326
4,mlp.experts.56.gate_and_up_proj,0.0023113203,0.05000,2.339
4,mlp.experts.57.gate_and_up_proj,0.0003912649,0.05000,2.352
4,mlp.experts.58.gate_and_up_proj,0.0005056146,0.05000,2.418
4,mlp.experts.59.gate_and_up_proj,0.0001579174,0.05000,2.428
4,mlp.experts.60.gate_and_up_proj,0.0000433404,0.05000,2.346
4,mlp.experts.61.gate_and_up_proj,0.0003610321,0.05000,2.324
4,mlp.experts.62.gate_and_up_proj,0.0000606549,0.05000,2.290
4,mlp.experts.63.gate_and_up_proj,0.0005929987,0.05000,1.347
4,mlp.experts.2.down_proj,0.0000026506,0.05000,1.845
4,mlp.experts.1.down_proj,0.0000093772,0.05000,1.853
4,mlp.experts.0.down_proj,0.0000009308,0.05000,1.928
4,mlp.experts.3.down_proj,0.0000084597,0.05000,1.772
4,mlp.experts.4.down_proj,0.0000063267,0.05000,1.798
4,mlp.experts.5.down_proj,0.0000147824,0.05000,1.840
4,mlp.experts.6.down_proj,0.0000182854,0.05000,1.723
4,mlp.experts.7.down_proj,0.0000204587,0.05000,1.746
4,mlp.experts.8.down_proj,0.0000111714,0.05000,1.750
4,mlp.experts.9.down_proj,0.0000102231,0.05000,1.797
4,mlp.experts.10.down_proj,0.0000159987,0.05000,1.814
4,mlp.experts.11.down_proj,0.0000039617,0.05000,1.820
4,mlp.experts.12.down_proj,0.0000047432,0.05000,1.799
4,mlp.experts.13.down_proj,0.0000027690,0.05000,1.817
4,mlp.experts.14.down_proj,0.0000025511,0.05000,1.860
4,mlp.experts.15.down_proj,0.0000285642,0.05000,1.854
4,mlp.experts.16.down_proj,0.0000058419,0.05000,1.881
4,mlp.experts.17.down_proj,0.0000011413,0.05000,1.890
4,mlp.experts.18.down_proj,0.0000044242,0.05000,1.850
4,mlp.experts.19.down_proj,0.0000011941,0.05000,1.889
4,mlp.experts.20.down_proj,0.0000298956,0.05000,1.904
4,mlp.experts.21.down_proj,0.0000358372,0.05000,1.871
4,mlp.experts.22.down_proj,0.0000006168,0.05000,1.866
4,mlp.experts.23.down_proj,0.0000036150,0.05000,1.872
4,mlp.experts.24.down_proj,0.0000041368,0.05000,1.844
4,mlp.experts.25.down_proj,0.0000225312,0.05000,1.888
4,mlp.experts.26.down_proj,0.0000105610,0.05000,1.897
4,mlp.experts.27.down_proj,0.0000040217,0.05000,1.805
4,mlp.experts.28.down_proj,0.0000082108,0.05000,1.832
4,mlp.experts.29.down_proj,0.0000016776,0.05000,1.851
4,mlp.experts.30.down_proj,0.0000090251,0.05000,1.770
4,mlp.experts.31.down_proj,0.0000133385,0.05000,1.749
4,mlp.experts.32.down_proj,0.0000080110,0.05000,1.761
4,mlp.experts.33.down_proj,0.0000110125,0.05000,1.733
4,mlp.experts.34.down_proj,0.0000000000,0.05000,1.758
4,mlp.experts.35.down_proj,0.0000036176,0.05000,1.742
4,mlp.experts.36.down_proj,0.0000098646,0.05000,1.753
4,mlp.experts.37.down_proj,0.0000111321,0.05000,1.731
4,mlp.experts.38.down_proj,0.0000150637,0.05000,1.767
4,mlp.experts.39.down_proj,0.0000071451,0.05000,1.688
4,mlp.experts.40.down_proj,0.0000060385,0.05000,1.698
4,mlp.experts.41.down_proj,0.0000013576,0.05000,1.719
4,mlp.experts.42.down_proj,0.0000105779,0.05000,1.737
4,mlp.experts.43.down_proj,0.0000139774,0.05000,1.758
4,mlp.experts.44.down_proj,0.0000502069,0.05000,1.770
4,mlp.experts.45.down_proj,0.0000112244,0.05000,1.790
4,mlp.experts.46.down_proj,0.0000119340,0.05000,1.765
4,mlp.experts.47.down_proj,0.0000000000,0.05000,1.795
4,mlp.experts.48.down_proj,0.0000199187,0.05000,1.712
4,mlp.experts.49.down_proj,0.0000095142,0.05000,1.749
4,mlp.experts.50.down_proj,0.0000015885,0.05000,1.752
4,mlp.experts.51.down_proj,0.0000255984,0.05000,1.692
4,mlp.experts.52.down_proj,0.0000239536,0.05000,1.709
4,mlp.experts.53.down_proj,0.0000257170,0.05000,1.719
4,mlp.experts.54.down_proj,0.0000036428,0.05000,1.740
4,mlp.experts.55.down_proj,0.0000020791,0.05000,1.784
4,mlp.experts.56.down_proj,0.0000519420,0.05000,1.817
4,mlp.experts.57.down_proj,0.0000071189,0.05000,1.776
4,mlp.experts.58.down_proj,0.0000092119,0.05000,1.820
4,mlp.experts.59.down_proj,0.0000045035,0.05000,1.807
4,mlp.experts.60.down_proj,0.0000005746,0.05000,1.782
4,mlp.experts.61.down_proj,0.0000074471,0.05000,1.831
4,mlp.experts.62.down_proj,0.0000010234,0.05000,1.690
4,mlp.experts.63.down_proj,0.0000071537,0.05000,1.012
5,self_attn.qkv_proj,0.0000339299,0.05000,0.946
5,self_attn.o_proj,0.0000194908,0.05000,0.928
5,mlp.gate.wg,0.0000701408,0.05000,0.907
5,mlp.shared_mlp.gate_and_up_proj,0.0008309802,0.05000,0.928
5,mlp.shared_mlp.down_proj,0.0000062843,0.05000,0.692
5,mlp.experts.0.gate_and_up_proj,0.0016558976,0.05000,2.340
5,mlp.experts.2.gate_and_up_proj,0.0007155802,0.05000,2.362
5,mlp.experts.1.gate_and_up_proj,0.0010226792,0.05000,2.416
5,mlp.experts.4.gate_and_up_proj,0.0013637444,0.05000,2.393
5,mlp.experts.5.gate_and_up_proj,0.0016788932,0.05000,2.396
5,mlp.experts.3.gate_and_up_proj,0.0000000000,0.05000,2.684
5,mlp.experts.6.gate_and_up_proj,0.0001279778,0.05000,2.503
5,mlp.experts.7.gate_and_up_proj,0.0000831259,0.05000,2.516
5,mlp.experts.8.gate_and_up_proj,0.0012767184,0.05000,2.490
5,mlp.experts.9.gate_and_up_proj,0.0021380522,0.05000,2.429
5,mlp.experts.10.gate_and_up_proj,0.0005826047,0.05000,2.409
5,mlp.experts.11.gate_and_up_proj,0.0008627099,0.05000,2.457
5,mlp.experts.12.gate_and_up_proj,0.0009059262,0.05000,2.460
5,mlp.experts.13.gate_and_up_proj,0.0003225892,0.05000,2.439
5,mlp.experts.14.gate_and_up_proj,0.0008762801,0.05000,2.465
5,mlp.experts.15.gate_and_up_proj,0.0006005790,0.05000,2.283
5,mlp.experts.16.gate_and_up_proj,0.0000543865,0.05000,2.322
5,mlp.experts.17.gate_and_up_proj,0.0000301079,0.05000,2.339
5,mlp.experts.18.gate_and_up_proj,0.0004472028,0.05000,2.535
5,mlp.experts.19.gate_and_up_proj,0.0001190159,0.05000,2.497
5,mlp.experts.20.gate_and_up_proj,0.0005505677,0.05000,2.478
5,mlp.experts.21.gate_and_up_proj,0.0013919875,0.05000,2.308
5,mlp.experts.22.gate_and_up_proj,0.0007868408,0.05000,2.316
5,mlp.experts.23.gate_and_up_proj,0.0009733346,0.05000,2.308
5,mlp.experts.24.gate_and_up_proj,0.0009844867,0.05000,2.259
5,mlp.experts.25.gate_and_up_proj,0.0000519903,0.05000,2.287
5,mlp.experts.26.gate_and_up_proj,0.0005235073,0.05000,2.292
5,mlp.experts.27.gate_and_up_proj,0.0002260867,0.05000,2.262
5,mlp.experts.28.gate_and_up_proj,0.0011604565,0.05000,2.278
5,mlp.experts.29.gate_and_up_proj,0.0011706481,0.05000,2.288
5,mlp.experts.30.gate_and_up_proj,0.0006235637,0.05000,2.351
5,mlp.experts.31.gate_and_up_proj,0.0023259533,0.05000,2.383
5,mlp.experts.32.gate_and_up_proj,0.0001604771,0.05000,2.423
5,mlp.experts.33.gate_and_up_proj,0.0003571386,0.05000,2.392
5,mlp.experts.34.gate_and_up_proj,0.0008208544,0.05000,2.389
5,mlp.experts.35.gate_and_up_proj,0.0005484211,0.05000,2.394
5,mlp.experts.36.gate_and_up_proj,0.0002877291,0.05000,2.447
5,mlp.experts.37.gate_and_up_proj,0.0013202729,0.05000,2.469
5,mlp.experts.38.gate_and_up_proj,0.0009200066,0.05000,2.493
5,mlp.experts.39.gate_and_up_proj,0.0007083330,0.05000,2.433
5,mlp.experts.40.gate_and_up_proj,0.0000572520,0.05000,2.372
5,mlp.experts.41.gate_and_up_proj,0.0009431707,0.05000,2.343
5,mlp.experts.42.gate_and_up_proj,0.0002506860,0.05000,2.268
5,mlp.experts.43.gate_and_up_proj,0.0002922854,0.05000,2.278
5,mlp.experts.44.gate_and_up_proj,0.0004601795,0.05000,2.288
5,mlp.experts.45.gate_and_up_proj,0.0007337837,0.05000,2.339
5,mlp.experts.46.gate_and_up_proj,0.0001016574,0.05000,2.361
5,mlp.experts.47.gate_and_up_proj,0.0006770692,0.05000,2.391
5,mlp.experts.48.gate_and_up_proj,0.0008937640,0.05000,2.288
5,mlp.experts.49.gate_and_up_proj,0.0019516029,0.05000,2.282
5,mlp.experts.50.gate_and_up_proj,0.0012038606,0.05000,2.287
5,mlp.experts.51.gate_and_up_proj,0.0002618958,0.05000,2.406
5,mlp.experts.52.gate_and_up_proj,0.0007774519,0.05000,2.402
5,mlp.experts.53.gate_and_up_proj,0.0005929093,0.05000,2.381
5,mlp.experts.54.gate_and_up_proj,0.0009195153,0.05000,2.427
5,mlp.experts.55.gate_and_up_proj,0.0002040308,0.05000,2.482
5,mlp.experts.56.gate_and_up_proj,0.0004079019,0.05000,2.474
5,mlp.experts.57.gate_and_up_proj,0.0002719256,0.05000,2.379
5,mlp.experts.58.gate_and_up_proj,0.0004285893,0.05000,2.433
5,mlp.experts.59.gate_and_up_proj,0.0003346382,0.05000,2.411
5,mlp.experts.60.gate_and_up_proj,0.0003029622,0.05000,2.531
5,mlp.experts.61.gate_and_up_proj,0.0000515497,0.05000,2.519
5,mlp.experts.62.gate_and_up_proj,0.0012539249,0.05000,2.429
5,mlp.experts.63.gate_and_up_proj,0.0009250852,0.05000,1.135
5,mlp.experts.0.down_proj,0.0000221666,0.05000,1.843
5,mlp.experts.2.down_proj,0.0000342795,0.05000,1.865
5,mlp.experts.1.down_proj,0.0000155638,0.05000,1.888
5,mlp.experts.3.down_proj,0.0000000000,0.05000,1.981
5,mlp.experts.4.down_proj,0.0000376333,0.05000,1.972
5,mlp.experts.5.down_proj,0.0000200980,0.05000,1.996
5,mlp.experts.6.down_proj,0.0000024054,0.05000,1.919
5,mlp.experts.7.down_proj,0.0000017166,0.05000,1.967
5,mlp.experts.8.down_proj,0.0000185187,0.05000,1.969
5,mlp.experts.9.down_proj,0.0000346094,0.05000,1.931
5,mlp.experts.10.down_proj,0.0000098821,0.05000,1.996
5,mlp.experts.11.down_proj,0.0000205020,0.05000,1.986
5,mlp.experts.12.down_proj,0.0000178005,0.05000,1.949
5,mlp.experts.13.down_proj,0.0000064313,0.05000,1.946
5,mlp.experts.14.down_proj,0.0000259974,0.05000,1.948
5,mlp.experts.15.down_proj,0.0000097271,0.05000,1.908
5,mlp.experts.16.down_proj,0.0000009655,0.05000,1.930
5,mlp.experts.17.down_proj,0.0000003876,0.05000,1.907
5,mlp.experts.18.down_proj,0.0000082660,0.05000,1.970
5,mlp.experts.19.down_proj,0.0000027978,0.05000,1.920
5,mlp.experts.20.down_proj,0.0000175921,0.05000,1.991
5,mlp.experts.21.down_proj,0.0000214605,0.05000,1.840
5,mlp.experts.22.down_proj,0.0000118996,0.05000,1.851
5,mlp.experts.23.down_proj,0.0000134385,0.05000,1.886
5,mlp.experts.24.down_proj,0.0000241954,0.05000,1.841
5,mlp.experts.25.down_proj,0.0000009187,0.05000,1.819
5,mlp.experts.26.down_proj,0.0000079886,0.05000,1.825
5,mlp.experts.27.down_proj,0.0000024216,0.05000,1.805
5,mlp.experts.28.down_proj,0.0000233321,0.05000,1.771
5,mlp.experts.29.down_proj,0.0000393838,0.05000,1.800
5,mlp.experts.30.down_proj,0.0000110855,0.05000,1.882
5,mlp.experts.31.down_proj,0.0000377987,0.05000,1.880
5,mlp.experts.32.down_proj,0.0000037766,0.05000,1.881
5,mlp.experts.33.down_proj,0.0000068133,0.05000,1.885
5,mlp.experts.34.down_proj,0.0000164171,0.05000,1.905
5,mlp.experts.35.down_proj,0.0000094407,0.05000,1.888
5,mlp.experts.36.down_proj,0.0000040280,0.05000,1.897
5,mlp.experts.37.down_proj,0.0000358635,0.05000,1.913
5,mlp.experts.38.down_proj,0.0000134833,0.05000,1.933
5,mlp.experts.39.down_proj,0.0000116183,0.05000,1.824
5,mlp.experts.40.down_proj,0.0000015052,0.05000,1.822
5,mlp.experts.41.down_proj,0.0000162093,0.05000,1.851
5,mlp.experts.42.down_proj,0.0000057077,0.05000,1.877
5,mlp.experts.43.down_proj,0.0000070302,0.05000,1.870
5,mlp.experts.44.down_proj,0.0000072527,0.05000,1.872
5,mlp.experts.45.down_proj,0.0000188337,0.05000,1.897
5,mlp.experts.46.down_proj,0.0000020137,0.05000,1.894
5,mlp.experts.47.down_proj,0.0000139084,0.05000,1.923
5,mlp.experts.48.down_proj,0.0000449407,0.05000,1.880
5,mlp.experts.49.down_proj,0.0000376439,0.05000,1.892
5,mlp.experts.50.down_proj,0.0000295072,0.05000,1.900
5,mlp.experts.51.down_proj,0.0000043904,0.05000,1.885
5,mlp.experts.52.down_proj,0.0000135393,0.05000,1.887
5,mlp.experts.53.down_proj,0.0000130182,0.05000,1.900
5,mlp.experts.54.down_proj,0.0000162184,0.05000,1.834
5,mlp.experts.55.down_proj,0.0000045465,0.05000,1.817
5,mlp.experts.56.down_proj,0.0000082472,0.05000,1.821
5,mlp.experts.57.down_proj,0.0000047948,0.05000,1.872
5,mlp.experts.58.down_proj,0.0000045915,0.05000,1.808
5,mlp.experts.59.down_proj,0.0000090866,0.05000,1.810
5,mlp.experts.61.down_proj,0.0000008574,0.05000,1.785
5,mlp.experts.60.down_proj,0.0000037708,0.05000,1.819
5,mlp.experts.62.down_proj,0.0000161710,0.05000,1.715
5,mlp.experts.63.down_proj,0.0000159727,0.05000,0.757
6,self_attn.qkv_proj,0.0000386181,0.05000,0.931
6,self_attn.o_proj,0.0000343008,0.05000,0.930
6,mlp.gate.wg,0.0000693923,0.05000,0.906
6,mlp.shared_mlp.gate_and_up_proj,0.0009947273,0.05000,0.933
6,mlp.shared_mlp.down_proj,0.0000098132,0.05000,0.697
6,mlp.experts.1.gate_and_up_proj,0.0002812018,0.05000,2.518
6,mlp.experts.2.gate_and_up_proj,0.0002547949,0.05000,2.529
6,mlp.experts.0.gate_and_up_proj,0.0007158677,0.05000,2.566
6,mlp.experts.3.gate_and_up_proj,0.0005566926,0.05000,2.335
6,mlp.experts.5.gate_and_up_proj,0.0009592548,0.05000,2.402
6,mlp.experts.4.gate_and_up_proj,0.0003450062,0.05000,2.492
6,mlp.experts.6.gate_and_up_proj,0.0017346159,0.05000,2.561
6,mlp.experts.7.gate_and_up_proj,0.0002218732,0.05000,2.521
6,mlp.experts.8.gate_and_up_proj,0.0011904260,0.05000,2.524
6,mlp.experts.9.gate_and_up_proj,0.0000601310,0.05000,2.411
6,mlp.experts.10.gate_and_up_proj,0.0005697663,0.05000,2.465
6,mlp.experts.11.gate_and_up_proj,0.0008470413,0.05000,2.439
6,mlp.experts.12.gate_and_up_proj,0.0013729770,0.05000,2.353
6,mlp.experts.14.gate_and_up_proj,0.0011331394,0.05000,2.380
6,mlp.experts.13.gate_and_up_proj,0.0005751011,0.05000,2.447
6,mlp.experts.15.gate_and_up_proj,0.0006457743,0.05000,2.526
6,mlp.experts.16.gate_and_up_proj,0.0000645172,0.05000,2.474
6,mlp.experts.17.gate_and_up_proj,0.0001258373,0.05000,2.466
6,mlp.experts.18.gate_and_up_proj,0.0026503488,0.05000,2.415
6,mlp.experts.19.gate_and_up_proj,0.0007056361,0.05000,2.433
6,mlp.experts.20.gate_and_up_proj,0.0026484601,0.05000,2.427
6,mlp.experts.21.gate_and_up_proj,0.0006163722,0.05000,2.362
6,mlp.experts.22.gate_and_up_proj,0.0003962448,0.05000,2.386
6,mlp.experts.23.gate_and_up_proj,0.0003031699,0.05000,2.412
6,mlp.experts.24.gate_and_up_proj,0.0002095009,0.05000,2.408
6,mlp.experts.25.gate_and_up_proj,0.0005098392,0.05000,2.433
6,mlp.experts.26.gate_and_up_proj,0.0013974478,0.05000,2.441
6,mlp.experts.27.gate_and_up_proj,0.0012470153,0.05000,2.358
6,mlp.experts.28.gate_and_up_proj,0.0001514022,0.05000,2.376
6,mlp.experts.29.gate_and_up_proj,0.0014915605,0.05000,2.395
6,mlp.experts.30.gate_and_up_proj,0.0013576243,0.05000,2.327
6,mlp.experts.31.gate_and_up_proj,0.0001485240,0.05000,2.338
6,mlp.experts.32.gate_and_up_proj,0.0001161271,0.05000,2.340
6,mlp.experts.33.gate_and_up_proj,0.0004597216,0.05000,2.402
6,mlp.experts.34.gate_and_up_proj,0.0011034395,0.05000,2.448
6,mlp.experts.35.gate_and_up_proj,0.0008495371,0.05000,2.477
6,mlp.experts.36.gate_and_up_proj,0.0004986625,0.05000,2.478
6,mlp.experts.37.gate_and_up_proj,0.0010492365,0.05000,2.508
6,mlp.experts.38.gate_and_up_proj,0.0002875507,0.05000,2.491
6,mlp.experts.39.gate_and_up_proj,0.0006061339,0.05000,2.400
6,mlp.experts.40.gate_and_up_proj,0.0006016141,0.05000,2.448
6,mlp.experts.41.gate_and_up_proj,0.0022590407,0.05000,2.461
6,mlp.experts.42.gate_and_up_proj,0.0011465965,0.05000,2.450
6,mlp.experts.43.gate_and_up_proj,0.0021890281,0.05000,2.465
6,mlp.experts.44.gate_and_up_proj,0.0015993782,0.05000,2.458
6,mlp.experts.45.gate_and_up_proj,0.0001424384,0.05000,2.349
6,mlp.experts.46.gate_and_up_proj,0.0003925639,0.05000,2.348
6,mlp.experts.47.gate_and_up_proj,0.0003689335,0.05000,2.334
6,mlp.experts.48.gate_and_up_proj,0.0001783039,0.05000,2.375
6,mlp.experts.49.gate_and_up_proj,0.0001835738,0.05000,2.409
6,mlp.experts.50.gate_and_up_proj,0.0002191585,0.05000,2.429
6,mlp.experts.51.gate_and_up_proj,0.0006951007,0.05000,2.387
6,mlp.experts.52.gate_and_up_proj,0.0008238406,0.05000,2.462
6,mlp.experts.53.gate_and_up_proj,0.0015532615,0.05000,2.473
6,mlp.experts.54.gate_and_up_proj,0.0028155705,0.05000,2.393
6,mlp.experts.55.gate_and_up_proj,0.0002250525,0.05000,2.388
6,mlp.experts.56.gate_and_up_proj,0.0009945815,0.05000,2.411
6,mlp.experts.57.gate_and_up_proj,0.0006395403,0.05000,2.452
6,mlp.experts.58.gate_and_up_proj,0.0006714028,0.05000,2.499
6,mlp.experts.59.gate_and_up_proj,0.0008011599,0.05000,2.457
6,mlp.experts.60.gate_and_up_proj,0.0003505417,0.05000,2.513
6,mlp.experts.61.gate_and_up_proj,0.0006463615,0.05000,2.414
6,mlp.experts.62.gate_and_up_proj,0.0016554567,0.05000,2.380
6,mlp.experts.63.gate_and_up_proj,0.0001196463,0.05000,1.205
6,mlp.experts.0.down_proj,0.0000526592,0.05000,1.824
6,mlp.experts.2.down_proj,0.0000028428,0.05000,1.825
6,mlp.experts.1.down_proj,0.0000043112,0.05000,1.848
6,mlp.experts.4.down_proj,0.0000069912,0.05000,1.918
6,mlp.experts.3.down_proj,0.0000069892,0.05000,1.966
6,mlp.experts.5.down_proj,0.0000653430,0.05000,1.943
6,mlp.experts.6.down_proj,0.0000216134,0.05000,1.787
6,mlp.experts.8.down_proj,0.0000208645,0.05000,1.837
6,mlp.experts.7.down_proj,0.0000037426,0.05000,1.865
6,mlp.experts.9.down_proj,0.0000015238,0.05000,1.813
6,mlp.experts.10.down_proj,0.0000177904,0.05000,1.789
6,mlp.experts.11.down_proj,0.0000120047,0.05000,1.776
6,mlp.experts.12.down_proj,0.0000269401,0.05000,1.796
6,mlp.experts.13.down_proj,0.0000101445,0.05000,1.813
6,mlp.experts.14.down_proj,0.0000291756,0.05000,1.809
6,mlp.experts.15.down_proj,0.0000103434,0.05000,1.842
6,mlp.experts.16.down_proj,0.0000012962,0.05000,1.831
6,mlp.experts.17.down_proj,0.0000015831,0.05000,1.867
6,mlp.experts.18.down_proj,0.0000407081,0.05000,1.843
6,mlp.experts.19.down_proj,0.0000106618,0.05000,1.843
6,mlp.experts.20.down_proj,0.0000399464,0.05000,1.873
6,mlp.experts.21.down_proj,0.0000077352,0.05000,1.910
6,mlp.experts.22.down_proj,0.0000062565,0.05000,1.952
6,mlp.experts.23.down_proj,0.0000088561,0.05000,1.986
6,mlp.experts.24.down_proj,0.0000029871,0.05000,1.977
6,mlp.experts.25.down_proj,0.0000084745,0.05000,2.039
6,mlp.experts.26.down_proj,0.0000243918,0.05000,2.003
6,mlp.experts.27.down_proj,0.0000386178,0.05000,1.897
6,mlp.experts.28.down_proj,0.0000033061,0.05000,1.954
6,mlp.experts.29.down_proj,0.0000680452,0.05000,1.931
6,mlp.experts.30.down_proj,0.0000258418,0.05000,1.880
6,mlp.experts.31.down_proj,0.0000017328,0.05000,1.915
6,mlp.experts.32.down_proj,0.0000020734,0.05000,1.920
6,mlp.experts.33.down_proj,0.0000140741,0.05000,1.958
6,mlp.experts.34.down_proj,0.0000184172,0.05000,1.983
6,mlp.experts.35.down_proj,0.0000099643,0.05000,1.991
6,mlp.experts.36.down_proj,0.0000105092,0.05000,2.075
6,mlp.experts.37.down_proj,0.0000196226,0.05000,2.086
6,mlp.experts.38.down_proj,0.0000043830,0.05000,2.082
6,mlp.experts.39.down_proj,0.0000082962,0.05000,2.175
6,mlp.experts.40.down_proj,0.0000152305,0.05000,2.159
6,mlp.experts.41.down_proj,0.0000327799,0.05000,2.178
6,mlp.experts.42.down_proj,0.0000258040,0.05000,2.120
6,mlp.experts.43.down_proj,0.0000268850,0.05000,2.103
6,mlp.experts.44.down_proj,0.0000278036,0.05000,2.118
6,mlp.experts.45.down_proj,0.0000021379,0.05000,2.149
6,mlp.experts.46.down_proj,0.0000074717,0.05000,2.118
6,mlp.experts.47.down_proj,0.0000041745,0.05000,2.131
6,mlp.experts.48.down_proj,0.0000023344,0.05000,2.137
6,mlp.experts.49.down_proj,0.0000028371,0.05000,2.121
6,mlp.experts.50.down_proj,0.0000024589,0.05000,2.137
6,mlp.experts.51.down_proj,0.0000112670,0.05000,2.172
6,mlp.experts.52.down_proj,0.0000111054,0.05000,2.109
6,mlp.experts.53.down_proj,0.0000474322,0.05000,2.158
6,mlp.experts.54.down_proj,0.0000406613,0.05000,2.215
6,mlp.experts.55.down_proj,0.0000083325,0.05000,2.182
6,mlp.experts.56.down_proj,0.0000193871,0.05000,2.154
6,mlp.experts.57.down_proj,0.0000103816,0.05000,2.209
6,mlp.experts.58.down_proj,0.0000260109,0.05000,2.149
6,mlp.experts.59.down_proj,0.0000268433,0.05000,2.167
6,mlp.experts.61.down_proj,0.0000126818,0.05000,2.214
6,mlp.experts.60.down_proj,0.0000081706,0.05000,2.294
6,mlp.experts.62.down_proj,0.0000280768,0.05000,2.173
6,mlp.experts.63.down_proj,0.0000016466,0.05000,0.747
7,self_attn.qkv_proj,0.0000656151,0.05000,0.933
7,self_attn.o_proj,0.0000417691,0.05000,0.929
7,mlp.gate.wg,0.0000641342,0.05000,0.914
7,mlp.shared_mlp.gate_and_up_proj,0.0010774122,0.05000,0.934
7,mlp.shared_mlp.down_proj,0.0000137913,0.05000,0.695
7,mlp.experts.0.gate_and_up_proj,0.0002770774,0.05000,2.468
7,mlp.experts.2.gate_and_up_proj,0.0001689403,0.05000,2.501
7,mlp.experts.1.gate_and_up_proj,0.0000211195,0.05000,2.550
7,mlp.experts.4.gate_and_up_proj,0.0003559590,0.05000,2.527
7,mlp.experts.5.gate_and_up_proj,0.0012131861,0.05000,2.517
7,mlp.experts.3.gate_and_up_proj,0.0000208427,0.05000,2.630
7,mlp.experts.6.gate_and_up_proj,0.0002737211,0.05000,2.372
7,mlp.experts.7.gate_and_up_proj,0.0001876114,0.05000,2.379
7,mlp.experts.8.gate_and_up_proj,0.0001348996,0.05000,2.396
7,mlp.experts.9.gate_and_up_proj,0.0005977988,0.05000,2.292
7,mlp.experts.10.gate_and_up_proj,0.0006650880,0.05000,2.341
7,mlp.experts.11.gate_and_up_proj,0.0000692728,0.05000,2.341
7,mlp.experts.12.gate_and_up_proj,0.0001947779,0.05000,2.335
7,mlp.experts.13.gate_and_up_proj,0.0004143649,0.05000,2.383
7,mlp.experts.14.gate_and_up_proj,0.0008212740,0.05000,2.374
7,mlp.experts.15.gate_and_up_proj,0.0004367815,0.05000,2.271
7,mlp.experts.16.gate_and_up_proj,0.0002483736,0.05000,2.306
7,mlp.experts.17.gate_and_up_proj,0.0000289972,0.05000,2.295
7,mlp.experts.18.gate_and_up_proj,0.0001035122,0.05000,2.245
7,mlp.experts.19.gate_and_up_proj,0.0000000000,0.05000,2.293
7,mlp.experts.20.gate_and_up_proj,0.0007464611,0.05000,2.289
7,mlp.experts.21.gate_and_up_proj,0.0004388647,0.05000,2.289
7,mlp.experts.22.gate_and_up_proj,0.0000315725,0.05000,2.338
7,mlp.experts.23.gate_and_up_proj,0.0000191945,0.05000,2.392
7,mlp.experts.24.gate_and_up_proj,0.0002818679,0.05000,2.425
7,mlp.experts.25.gate_and_up_proj,0.0000635254,0.05000,2.368
7,mlp.experts.26.gate_and_up_proj,0.0002480095,0.05000,2.338
7,mlp.experts.27.gate_and_up_proj,0.0000225336,0.05000,2.305
7,mlp.experts.28.gate_and_up_proj,0.0000698364,0.05000,2.354
7,mlp.experts.29.gate_and_up_proj,0.0000000000,0.05000,2.360
7,mlp.experts.30.gate_and_up_proj,0.0002522901,0.05000,2.335
7,mlp.experts.31.gate_and_up_proj,0.0003566682,0.05000,2.425
7,mlp.experts.32.gate_and_up_proj,0.0002157183,0.05000,2.392
7,mlp.experts.33.gate_and_up_proj,0.0019009910,0.05000,2.362
7,mlp.experts.34.gate_and_up_proj,0.0001110357,0.05000,2.398
7,mlp.experts.35.gate_and_up_proj,0.0000240485,0.05000,2.397
7,mlp.experts.36.gate_and_up_proj,0.0009679256,0.05000,2.291
7,mlp.experts.37.gate_and_up_proj,0.0001536679,0.05000,2.350
7,mlp.experts.38.gate_and_up_proj,0.0000369968,0.05000,2.354
7,mlp.experts.39.gate_and_up_proj,0.0001939577,0.05000,2.322
7,mlp.experts.40.gate_and_up_proj,0.0003508570,0.05000,2.376
7,mlp.experts.41.gate_and_up_proj,0.0000000000,0.05000,2.399
7,mlp.experts.42.gate_and_up_proj,0.0000626270,0.05000,2.422
7,mlp.experts.43.gate_and_up_proj,0.0008572081,0.05000,2.361
7,mlp.experts.44.gate_and_up_proj,0.0002244365,0.05000,2.359
7,mlp.experts.45.gate_and_up_proj,0.0001288067,0.05000,2.402
7,mlp.experts.46.gate_and_up_proj,0.0006841289,0.05000,2.438
7,mlp.experts.47.gate_and_up_proj,0.0000861019,0.05000,2.437
7,mlp.experts.48.gate_and_up_proj,0.0004997562,0.05000,2.464
7,mlp.experts.49.gate_and_up_proj,0.0007201608,0.05000,2.481
7,mlp.experts.50.gate_and_up_proj,0.0000239669,0.05000,2.473
7,mlp.experts.51.gate_and_up_proj,0.0001529482,0.05000,2.377
7,mlp.experts.52.gate_and_up_proj,0.0002296535,0.05000,2.411
7,mlp.experts.53.gate_and_up_proj,0.0005230482,0.05000,2.398
7,mlp.experts.54.gate_and_up_proj,0.0022928801,0.05000,2.308
7,mlp.experts.55.gate_and_up_proj,0.0000498583,0.05000,2.373
7,mlp.experts.56.gate_and_up_proj,0.0003851070,0.05000,2.372
7,mlp.experts.57.gate_and_up_proj,0.0003153158,0.05000,2.346
7,mlp.experts.58.gate_and_up_proj,0.0000000000,0.05000,2.386
7,mlp.experts.59.gate_and_up_proj,0.0002364212,0.05000,2.387
7,mlp.experts.60.gate_and_up_proj,0.0000454471,0.05000,2.359
7,mlp.experts.61.gate_and_up_proj,0.0003296313,0.05000,2.378
7,mlp.experts.62.gate_and_up_proj,0.0002013174,0.05000,2.370
7,mlp.experts.63.gate_and_up_proj,0.0000179955,0.05000,1.335
7,mlp.experts.0.down_proj,0.0000042691,0.05000,1.915
7,mlp.experts.2.down_proj,0.0000049006,0.05000,1.918
7,mlp.experts.1.down_proj,0.0000002835,0.05000,1.957
7,mlp.experts.4.down_proj,0.0000065484,0.05000,1.873
7,mlp.experts.5.down_proj,0.0000258543,0.05000,1.879
7,mlp.experts.3.down_proj,0.0000003681,0.05000,1.927
7,mlp.experts.6.down_proj,0.0000039072,0.05000,1.822
7,mlp.experts.7.down_proj,0.0000050241,0.05000,1.831
7,mlp.experts.8.down_proj,0.0000026090,0.05000,1.831
7,mlp.experts.9.down_proj,0.0000096458,0.05000,1.824
7,mlp.experts.11.down_proj,0.0000012378,0.05000,1.858
7,mlp.experts.10.down_proj,0.0000105492,0.05000,1.889
7,mlp.experts.13.down_proj,0.0000079084,0.05000,1.829
7,mlp.experts.14.down_proj,0.0000167631,0.05000,1.838
7,mlp.experts.12.down_proj,0.0000031433,0.05000,1.963
7,mlp.experts.15.down_proj,0.0000153968,0.05000,2.015
7,mlp.experts.16.down_proj,0.0000063951,0.05000,1.999
7,mlp.experts.17.down_proj,0.0000003749,0.05000,2.001
7,mlp.experts.19.down_proj,0.0000000000,0.05000,2.119
7,mlp.experts.20.down_proj,0.0000134921,0.05000,2.116
7,mlp.experts.18.down_proj,0.0000017735,0.05000,2.204
7,mlp.experts.22.down_proj,0.0000002829,0.05000,2.051
7,mlp.experts.21.down_proj,0.0000064813,0.05000,2.121
7,mlp.experts.23.down_proj,0.0000001519,0.05000,2.121
7,mlp.experts.24.down_proj,0.0000064898,0.05000,2.024
7,mlp.experts.25.down_proj,0.0000010470,0.05000,2.012
7,mlp.experts.26.down_proj,0.0000100150,0.05000,2.103
7,mlp.experts.27.down_proj,0.0000002029,0.05000,1.992
7,mlp.experts.28.down_proj,0.0000011455,0.05000,1.989
7,mlp.experts.29.down_proj,0.0000000000,0.05000,2.019
7,mlp.experts.30.down_proj,0.0000049141,0.05000,1.920
7,mlp.experts.31.down_proj,0.0000052194,0.05000,1.941
7,mlp.experts.32.down_proj,0.0000041416,0.05000,1.926
7,mlp.experts.33.down_proj,0.0000277648,0.05000,1.899
7,mlp.experts.34.down_proj,0.0000018273,0.05000,1.958
7,mlp.experts.35.down_proj,0.0000012299,0.05000,1.982
7,mlp.experts.36.down_proj,0.0000111426,0.05000,1.955
7,mlp.experts.37.down_proj,0.0000031573,0.05000,1.997
7,mlp.experts.38.down_proj,0.0000006118,0.05000,2.074
7,mlp.experts.39.down_proj,0.0000030715,0.05000,2.138
7,mlp.experts.40.down_proj,0.0000073570,0.05000,2.118
7,mlp.experts.41.down_proj,0.0000000000,0.05000,2.133
7,mlp.experts.42.down_proj,0.0000017418,0.05000,2.091
7,mlp.experts.43.down_proj,0.0000138698,0.05000,2.127
7,mlp.experts.44.down_proj,0.0000048365,0.05000,2.157
7,mlp.experts.45.down_proj,0.0000020525,0.05000,2.199
7,mlp.experts.46.down_proj,0.0000123810,0.05000,2.264
7,mlp.experts.47.down_proj,0.0000037739,0.05000,2.306
7,mlp.experts.48.down_proj,0.0000108385,0.05000,2.195
7,mlp.experts.49.down_proj,0.0000160019,0.05000,2.235
7,mlp.experts.50.down_proj,0.0000005549,0.05000,2.242
7,mlp.experts.51.down_proj,0.0000038359,0.05000,2.226
7,mlp.experts.52.down_proj,0.0000033303,0.05000,2.244
7,mlp.experts.53.down_proj,0.0000096380,0.05000,2.224
7,mlp.experts.54.down_proj,0.0000363027,0.05000,2.171
7,mlp.experts.55.down_proj,0.0000010178,0.05000,2.208
7,mlp.experts.56.down_proj,0.0000058954,0.05000,2.086
7,mlp.experts.57.down_proj,0.0000055813,0.05000,1.963
7,mlp.experts.58.down_proj,0.0000000000,0.05000,2.026
7,mlp.experts.59.down_proj,0.0000133237,0.05000,1.993
7,mlp.experts.60.down_proj,0.0000007452,0.05000,1.924
7,mlp.experts.61.down_proj,0.0000086262,0.05000,1.962
7,mlp.experts.62.down_proj,0.0000158807,0.05000,1.877
7,mlp.experts.63.down_proj,0.0000001971,0.05000,1.029
8,self_attn.qkv_proj,0.0000854751,0.05000,0.944
8,self_attn.o_proj,0.0000449481,0.05000,0.930
8,mlp.gate.wg,0.0000900276,0.05000,0.906
8,mlp.shared_mlp.gate_and_up_proj,0.0011659878,0.05000,0.943
8,mlp.shared_mlp.down_proj,0.0000189327,0.05000,0.731
8,mlp.experts.0.gate_and_up_proj,0.0000635570,0.05000,2.438
8,mlp.experts.1.gate_and_up_proj,0.0001761785,0.05000,2.466
8,mlp.experts.2.gate_and_up_proj,0.0000535422,0.05000,2.470
8,mlp.experts.3.gate_and_up_proj,0.0000903918,0.05000,2.292
8,mlp.experts.4.gate_and_up_proj,0.0025391795,0.05000,2.342
8,mlp.experts.5.gate_and_up_proj,0.0007556261,0.05000,2.354
8,mlp.experts.6.gate_and_up_proj,0.0000987130,0.05000,2.317
8,mlp.experts.7.gate_and_up_proj,0.0002892929,0.05000,2.336
8,mlp.experts.8.gate_and_up_proj,0.0001143521,0.05000,2.409
8,mlp.experts.9.gate_and_up_proj,0.0000000000,0.05000,2.381
8,mlp.experts.10.gate_and_up_proj,0.0000846838,0.05000,2.349
8,mlp.experts.11.gate_and_up_proj,0.0011527190,0.05000,2.394
8,mlp.experts.12.gate_and_up_proj,0.0003392749,0.05000,2.332
8,mlp.experts.13.gate_and_up_proj,0.0006438498,0.05000,2.369
8,mlp.experts.14.gate_and_up_proj,0.0006418798,0.05000,2.385
8,mlp.experts.15.gate_and_up_proj,0.0003689602,0.05000,2.393
8,mlp.experts.16.gate_and_up_proj,0.0000599272,0.05000,2.407
8,mlp.experts.17.gate_and_up_proj,0.0000914935,0.05000,2.438
8,mlp.experts.18.gate_and_up_proj,0.0001763202,0.05000,2.387
8,mlp.experts.19.gate_and_up_proj,0.0000390064,0.05000,2.423
8,mlp.experts.20.gate_and_up_proj,0.0003076322,0.05000,2.452
8,mlp.experts.21.gate_and_up_proj,0.0023931308,0.05000,2.402
8,mlp.experts.22.gate_and_up_proj,0.0001993300,0.05000,2.428
8,mlp.experts.23.gate_and_up_proj,0.0015227080,0.05000,2.450
8,mlp.experts.24.gate_and_up_proj,0.0002421820,0.05000,2.319
8,mlp.experts.25.gate_and_up_proj,0.0006611865,0.05000,2.312
8,mlp.experts.26.gate_and_up_proj,0.0009000387,0.05000,2.359
8,mlp.experts.27.gate_and_up_proj,0.0008348411,0.05000,2.299
8,mlp.experts.28.gate_and_up_proj,0.0004973774,0.05000,2.298
8,mlp.experts.29.gate_and_up_proj,0.0006331378,0.05000,2.337
8,mlp.experts.30.gate_and_up_proj,0.0002641287,0.05000,2.324
8,mlp.experts.31.gate_and_up_proj,0.0001314641,0.05000,2.359
8,mlp.experts.32.gate_and_up_proj,0.0011260035,0.05000,2.414
8,mlp.experts.33.gate_and_up_proj,0.0002404862,0.05000,2.399
8,mlp.experts.34.gate_and_up_proj,0.0005196907,0.05000,2.451
8,mlp.experts.35.gate_and_up_proj,0.0000850918,0.05000,2.499
8,mlp.experts.36.gate_and_up_proj,0.0000617843,0.05000,2.332
8,mlp.experts.37.gate_and_up_proj,0.0002411080,0.05000,2.373
8,mlp.experts.38.gate_and_up_proj,0.0002531969,0.05000,2.388
8,mlp.experts.39.gate_and_up_proj,0.0001591526,0.05000,2.321
8,mlp.experts.40.gate_and_up_proj,0.0000366917,0.05000,2.346
8,mlp.experts.41.gate_and_up_proj,0.0002517100,0.05000,2.353
8,mlp.experts.42.gate_and_up_proj,0.0005233424,0.05000,2.324
8,mlp.experts.43.gate_and_up_proj,0.0003368497,0.05000,2.329
8,mlp.experts.44.gate_and_up_proj,0.0001367910,0.05000,2.348
8,mlp.experts.45.gate_and_up_proj,0.0001789615,0.05000,2.287
8,mlp.experts.46.gate_and_up_proj,0.0023088051,0.05000,2.305
8,mlp.experts.47.gate_and_up_proj,0.0000692877,0.05000,2.359
8,mlp.experts.48.gate_and_up_proj,0.0003678329,0.05000,2.374
8,mlp.experts.49.gate_and_up_proj,0.0004163110,0.05000,2.379
8,mlp.experts.50.gate_and_up_proj,0.0008340181,0.05000,2.426
8,mlp.experts.51.gate_and_up_proj,0.0002334263,0.05000,2.374
8,mlp.experts.52.gate_and_up_proj,0.0001733421,0.05000,2.379
8,mlp.experts.53.gate_and_up_proj,0.0002565231,0.05000,2.354
8,mlp.experts.54.gate_and_up_proj,0.0000914480,0.05000,2.292
8,mlp.experts.55.gate_and_up_proj,0.0013141142,0.05000,2.356
8,mlp.experts.56.gate_and_up_proj,0.0000583369,0.05000,2.358
8,mlp.experts.57.gate_and_up_proj,0.0026354141,0.05000,2.327
8,mlp.experts.58.gate_and_up_proj,0.0005873107,0.05000,2.319
8,mlp.experts.59.gate_and_up_proj,0.0000264560,0.05000,2.334
8,mlp.experts.60.gate_and_up_proj,0.0006559095,0.05000,2.332
8,mlp.experts.61.gate_and_up_proj,0.0000444331,0.05000,2.345
8,mlp.experts.62.gate_and_up_proj,0.0001759887,0.05000,2.170
8,mlp.experts.63.gate_and_up_proj,0.0022640790,0.05000,1.332
8,mlp.experts.1.down_proj,0.0000028284,0.05000,1.845
8,mlp.experts.2.down_proj,0.0000029936,0.05000,1.850
8,mlp.experts.0.down_proj,0.0000015250,0.05000,1.971
8,mlp.experts.3.down_proj,0.0000024825,0.05000,2.005
8,mlp.experts.4.down_proj,0.0000476869,0.05000,2.035
8,mlp.experts.5.down_proj,0.0000193548,0.05000,1.954
8,mlp.experts.6.down_proj,0.0000044044,0.05000,1.895
8,mlp.experts.8.down_proj,0.0000019948,0.05000,1.876
8,mlp.experts.7.down_proj,0.0000046959,0.05000,1.952
8,mlp.experts.9.down_proj,0.0000000000,0.05000,1.959
8,mlp.experts.10.down_proj,0.0000013062,0.05000,1.911
8,mlp.experts.11.down_proj,0.0000397062,0.05000,1.905
8,mlp.experts.13.down_proj,0.0000145109,0.05000,1.960
8,mlp.experts.14.down_proj,0.0000129436,0.05000,1.987
8,mlp.experts.12.down_proj,0.0000076783,0.05000,2.034
8,mlp.experts.15.down_proj,0.0000070854,0.05000,1.994
8,mlp.experts.17.down_proj,0.0000019300,0.05000,2.061
8,mlp.experts.16.down_proj,0.0000028652,0.05000,2.097
8,mlp.experts.18.down_proj,0.0000030398,0.05000,2.020
8,mlp.experts.19.down_proj,0.0000004742,0.05000,2.036
8,mlp.experts.20.down_proj,0.0000053240,0.05000,2.025
8,mlp.experts.21.down_proj,0.0000443854,0.05000,1.959
8,mlp.experts.23.down_proj,0.0000251512,0.05000,2.007
8,mlp.experts.22.down_proj,0.0000036754,0.05000,2.024
8,mlp.experts.24.down_proj,0.0000041562,0.05000,1.904
8,mlp.experts.26.down_proj,0.0000140479,0.05000,1.930
8,mlp.experts.25.down_proj,0.0000134165,0.05000,1.986
8,mlp.experts.27.down_proj,0.0000165492,0.05000,1.945
8,mlp.experts.28.down_proj,0.0000237375,0.05000,1.957
8,mlp.experts.29.down_proj,0.0000200035,0.05000,1.904
8,mlp.experts.30.down_proj,0.0000234319,0.05000,1.815
8,mlp.experts.32.down_proj,0.0000213412,0.05000,1.859
8,mlp.experts.31.down_proj,0.0000027702,0.05000,1.890
8,mlp.experts.33.down_proj,0.0000058608,0.05000,2.074
8,mlp.experts.34.down_proj,0.0000080712,0.05000,1.882
8,mlp.experts.35.down_proj,0.0000014817,0.05000,1.882
8,mlp.experts.36.down_proj,0.0000012601,0.05000,1.885
8,mlp.experts.37.down_proj,0.0000042726,0.05000,1.845
8,mlp.experts.38.down_proj,0.0000036291,0.05000,1.838
8,mlp.experts.39.down_proj,0.0000039905,0.05000,1.897
8,mlp.experts.40.down_proj,0.0000025328,0.05000,1.837
8,mlp.experts.41.down_proj,0.0000063962,0.05000,1.834
8,mlp.experts.42.down_proj,0.0000098573,0.05000,1.907
8,mlp.experts.44.down_proj,0.0000018593,0.05000,1.900
8,mlp.experts.43.down_proj,0.0000079384,0.05000,1.919
8,mlp.experts.45.down_proj,0.0000030973,0.05000,1.801
8,mlp.experts.47.down_proj,0.0000018519,0.05000,1.828
8,mlp.experts.46.down_proj,0.0000463490,0.05000,1.852
8,mlp.experts.49.down_proj,0.0000184709,0.05000,1.786
8,mlp.experts.48.down_proj,0.0000102778,0.05000,1.877
8,mlp.experts.50.down_proj,0.0000173471,0.05000,1.821
8,mlp.experts.51.down_proj,0.0000137644,0.05000,1.793
8,mlp.experts.52.down_proj,0.0000049235,0.05000,1.868
8,mlp.experts.53.down_proj,0.0000085276,0.05000,1.866
8,mlp.experts.54.down_proj,0.0000014730,0.05000,1.846
8,mlp.experts.56.down_proj,0.0000014666,0.05000,1.890
8,mlp.experts.55.down_proj,0.0000288641,0.05000,1.921
8,mlp.experts.57.down_proj,0.0000577771,0.05000,1.873
8,mlp.experts.58.down_proj,0.0000094031,0.05000,1.819
8,mlp.experts.59.down_proj,0.0000003475,0.05000,1.825
8,mlp.experts.60.down_proj,0.0000122889,0.05000,1.819
8,mlp.experts.61.down_proj,0.0000010776,0.05000,1.831
8,mlp.experts.62.down_proj,0.0000031790,0.05000,1.834
8,mlp.experts.63.down_proj,0.0000437788,0.05000,0.787
9,self_attn.qkv_proj,0.0001174726,0.05000,0.943
9,self_attn.o_proj,0.0000586711,0.05000,0.938
9,mlp.gate.wg,0.0000780638,0.05000,0.912
9,mlp.shared_mlp.gate_and_up_proj,0.0013428223,0.05000,0.939
9,mlp.shared_mlp.down_proj,0.0000107899,0.05000,0.694
9,mlp.experts.0.gate_and_up_proj,0.0000574008,0.05000,2.542
9,mlp.experts.2.gate_and_up_proj,0.0008856338,0.05000,2.612
9,mlp.experts.1.gate_and_up_proj,0.0008451167,0.05000,2.700
9,mlp.experts.4.gate_and_up_proj,0.0004617797,0.05000,2.472
9,mlp.experts.5.gate_and_up_proj,0.0004583536,0.05000,2.477
9,mlp.experts.3.gate_and_up_proj,0.0003358944,0.05000,2.770
9,mlp.experts.6.gate_and_up_proj,0.0000664202,0.05000,2.491
9,mlp.experts.7.gate_and_up_proj,0.0000728563,0.05000,2.477
9,mlp.experts.8.gate_and_up_proj,0.0009817751,0.05000,2.509
9,mlp.experts.9.gate_and_up_proj,0.0000499779,0.05000,2.380
9,mlp.experts.10.gate_and_up_proj,0.0000858949,0.05000,2.414
9,mlp.experts.11.gate_and_up_proj,0.0020416298,0.05000,2.400
9,mlp.experts.12.gate_and_up_proj,0.0001512334,0.05000,2.328
9,mlp.experts.13.gate_and_up_proj,0.0000000000,0.05000,2.365
9,mlp.experts.14.gate_and_up_proj,0.0028869977,0.05000,2.344
9,mlp.experts.15.gate_and_up_proj,0.0043960173,0.05000,2.295
9,mlp.experts.16.gate_and_up_proj,0.0000704285,0.05000,2.311
9,mlp.experts.17.gate_and_up_proj,0.0004315435,0.05000,2.300
9,mlp.experts.18.gate_and_up_proj,0.0019504093,0.05000,2.348
9,mlp.experts.19.gate_and_up_proj,0.0006862356,0.05000,2.369
9,mlp.experts.20.gate_and_up_proj,0.0000769618,0.05000,2.359
9,mlp.experts.21.gate_and_up_proj,0.0001937229,0.05000,2.315
9,mlp.experts.22.gate_and_up_proj,0.0020683324,0.05000,2.306
9,mlp.experts.23.gate_and_up_proj,0.0004523403,0.05000,2.292
9,mlp.experts.24.gate_and_up_proj,0.0005194434,0.05000,2.441
9,mlp.experts.25.gate_and_up_proj,0.0007193728,0.05000,2.449
9,mlp.experts.26.gate_and_up_proj,0.0000640706,0.05000,2.467
9,mlp.experts.27.gate_and_up_proj,0.0002628762,0.05000,2.398
9,mlp.experts.28.gate_and_up_proj,0.0015139456,0.05000,2.385
9,mlp.experts.29.gate_and_up_proj,0.0000000000,0.05000,2.378
9,mlp.experts.30.gate_and_up_proj,0.0043021526,0.05000,2.338
9,mlp.experts.31.gate_and_up_proj,0.0001743929,0.05000,2.366
9,mlp.experts.32.gate_and_up_proj,0.0017619648,0.05000,2.356
9,mlp.experts.33.gate_and_up_proj,0.0000308149,0.05000,2.370
9,mlp.experts.34.gate_and_up_proj,0.0013184191,0.05000,2.392
9,mlp.experts.35.gate_and_up_proj,0.0000000000,0.05000,2.375
9,mlp.experts.36.gate_and_up_proj,0.0003493678,0.05000,2.317
9,mlp.experts.37.gate_and_up_proj,0.0028522651,0.05000,2.317
9,mlp.experts.38.gate_and_up_proj,0.0009573867,0.05000,2.308
9,mlp.experts.39.gate_and_up_proj,0.0000880494,0.05000,2.325
9,mlp.experts.40.gate_and_up_proj,0.0012118439,0.05000,2.345
9,mlp.experts.41.gate_and_up_proj,0.0002912105,0.05000,2.387
9,mlp.experts.42.gate_and_up_proj,0.0003915490,0.05000,2.360
9,mlp.experts.43.gate_and_up_proj,0.0008496158,0.05000,2.374
9,mlp.experts.44.gate_and_up_proj,0.0014933031,0.05000,2.378
9,mlp.experts.45.gate_and_up_proj,0.0010695452,0.05000,2.397
9,mlp.experts.46.gate_and_up_proj,0.0033779990,0.05000,2.384
9,mlp.experts.47.gate_and_up_proj,0.0013715473,0.05000,2.393
9,mlp.experts.48.gate_and_up_proj,0.0014968647,0.05000,2.380
9,mlp.experts.49.gate_and_up_proj,0.0001600170,0.05000,2.342
9,mlp.experts.50.gate_and_up_proj,0.0002582255,0.05000,2.348
9,mlp.experts.51.gate_and_up_proj,0.0035912781,0.05000,2.287
9,mlp.experts.52.gate_and_up_proj,0.0002726622,0.05000,2.310
9,mlp.experts.53.gate_and_up_proj,0.0003120050,0.05000,2.321
9,mlp.experts.54.gate_and_up_proj,0.0005154769,0.05000,2.408
9,mlp.experts.55.gate_and_up_proj,0.0001393655,0.05000,2.429
9,mlp.experts.56.gate_and_up_proj,0.0005527516,0.05000,2.444
9,mlp.experts.57.gate_and_up_proj,0.0012982166,0.05000,2.519
9,mlp.experts.58.gate_and_up_proj,0.0000623193,0.05000,2.518
9,mlp.experts.59.gate_and_up_proj,0.0001911563,0.05000,2.458
9,mlp.experts.60.gate_and_up_proj,0.0002864945,0.05000,2.446
9,mlp.experts.61.gate_and_up_proj,0.0037010166,0.05000,2.476
9,mlp.experts.62.gate_and_up_proj,0.0012736525,0.05000,2.472
9,mlp.experts.63.gate_and_up_proj,0.0000910503,0.05000,1.148
9,mlp.experts.2.down_proj,0.0000250686,0.05000,1.815
9,mlp.experts.0.down_proj,0.0000005899,0.05000,1.852
9,mlp.experts.1.down_proj,0.0000209528,0.05000,1.883
9,mlp.experts.3.down_proj,0.0000178215,0.05000,1.857
9,mlp.experts.4.down_proj,0.0000120946,0.05000,1.897
9,mlp.experts.5.down_proj,0.0000098388,0.05000,1.921
9,mlp.experts.6.down_proj,0.0000048492,0.05000,1.970
9,mlp.experts.7.down_proj,0.0000033747,0.05000,2.001
9,mlp.experts.8.down_proj,0.0000224898,0.05000,2.029
9,mlp.experts.9.down_proj,0.0000010525,0.05000,1.915
9,mlp.experts.10.down_proj,0.0000015617,0.05000,1.921
9,mlp.experts.11.down_proj,0.0000465685,0.05000,1.942
9,mlp.experts.12.down_proj,0.0000041910,0.05000,1.848
9,mlp.experts.13.down_proj,0.0000000000,0.05000,1.868
9,mlp.experts.14.down_proj,0.0000770220,0.05000,1.886
9,mlp.experts.15.down_proj,0.0001420068,0.05000,1.904
9,mlp.experts.16.down_proj,0.0000017252,0.05000,1.891
9,mlp.experts.17.down_proj,0.0000096701,0.05000,1.889
9,mlp.experts.18.down_proj,0.0000382568,0.05000,1.869
9,mlp.experts.19.down_proj,0.0000132736,0.05000,1.869
9,mlp.experts.20.down_proj,0.0000018349,0.05000,1.904
9,mlp.experts.21.down_proj,0.0000039414,0.05000,1.878
9,mlp.experts.22.down_proj,0.0000464786,0.05000,1.926
9,mlp.experts.23.down_proj,0.0000238634,0.05000,1.946
9,mlp.experts.24.down_proj,0.0000430512,0.05000,2.124
9,mlp.experts.25.down_proj,0.0000336122,0.05000,2.132
9,mlp.experts.26.down_proj,0.0000019903,0.05000,2.113
9,mlp.experts.27.down_proj,0.0000058330,0.05000,2.119
9,mlp.experts.28.down_proj,0.0000301310,0.05000,2.102
9,mlp.experts.29.down_proj,0.0000000000,0.05000,2.152
9,mlp.experts.30.down_proj,0.0001051276,0.05000,2.119
9,mlp.experts.31.down_proj,0.0000103803,0.05000,2.112
9,mlp.experts.32.down_proj,0.0000470300,0.05000,2.032
9,mlp.experts.33.down_proj,0.0000018973,0.05000,2.084
9,mlp.experts.34.down_proj,0.0000435007,0.05000,2.047
9,mlp.experts.35.down_proj,0.0000000000,0.05000,1.998
9,mlp.experts.36.down_proj,0.0000073768,0.05000,2.085
9,mlp.experts.37.down_proj,0.0000659542,0.05000,2.030
9,mlp.experts.38.down_proj,0.0000240474,0.05000,2.033
9,mlp.experts.39.down_proj,0.0000044260,0.05000,2.086
9,mlp.experts.40.down_proj,0.0000308416,0.05000,2.044
9,mlp.experts.41.down_proj,0.0000065440,0.05000,2.033
9,mlp.experts.43.down_proj,0.0000492502,0.05000,2.039
9,mlp.experts.44.down_proj,0.0000248306,0.05000,2.012
9,mlp.experts.42.down_proj,0.0000083795,0.05000,2.154
9,mlp.experts.45.down_proj,0.0000269005,0.05000,1.959
9,mlp.experts.46.down_proj,0.0000759470,0.05000,1.976
9,mlp.experts.47.down_proj,0.0000336370,0.05000,2.032
9,mlp.experts.48.down_proj,0.0000711859,0.05000,1.901
9,mlp.experts.49.down_proj,0.0000056491,0.05000,1.921
9,mlp.experts.50.down_proj,0.0000097519,0.05000,2.037
9,mlp.experts.51.down_proj,0.0000902623,0.05000,2.013
9,mlp.experts.52.down_proj,0.0000061295,0.05000,1.979
9,mlp.experts.53.down_proj,0.0000076895,0.05000,2.010
9,mlp.experts.54.down_proj,0.0000174235,0.05000,1.930
9,mlp.experts.55.down_proj,0.0000084796,0.05000,1.945
9,mlp.experts.56.down_proj,0.0000130944,0.05000,1.936
9,mlp.experts.57.down_proj,0.0000394520,0.05000,1.901
9,mlp.experts.58.down_proj,0.0000015880,0.05000,1.913
9,mlp.experts.59.down_proj,0.0000103403,0.05000,1.881
9,mlp.experts.60.down_proj,0.0000089017,0.05000,1.957
9,mlp.experts.61.down_proj,0.0001168808,0.05000,1.927
9,mlp.experts.62.down_proj,0.0000312364,0.05000,1.857
9,mlp.experts.63.down_proj,0.0000021233,0.05000,0.780
10,self_attn.qkv_proj,0.0001264720,0.05000,0.932
10,self_attn.o_proj,0.0000578116,0.05000,0.937
10,mlp.gate.wg,0.0000901669,0.05000,0.908
10,mlp.shared_mlp.gate_and_up_proj,0.0013750951,0.05000,0.935
10,mlp.shared_mlp.down_proj,0.0000121113,0.05000,0.695
10,mlp.experts.0.gate_and_up_proj,0.0004989565,0.05000,2.462
10,mlp.experts.2.gate_and_up_proj,0.0015565886,0.05000,2.487
10,mlp.experts.1.gate_and_up_proj,0.0000390902,0.05000,2.585
10,mlp.experts.4.gate_and_up_proj,0.0008168410,0.05000,2.366
10,mlp.experts.3.gate_and_up_proj,0.0023138353,0.05000,2.394
10,mlp.experts.5.gate_and_up_proj,0.0002590454,0.05000,2.412
10,mlp.experts.7.gate_and_up_proj,0.0012371157,0.05000,2.312
10,mlp.experts.6.gate_and_up_proj,0.0001983553,0.05000,2.386
10,mlp.experts.8.gate_and_up_proj,0.0012285755,0.05000,2.384
10,mlp.experts.9.gate_and_up_proj,0.0005174377,0.05000,2.424
10,mlp.experts.10.gate_and_up_proj,0.0035901321,0.05000,2.467
10,mlp.experts.11.gate_and_up_proj,0.0006157486,0.05000,2.461
10,mlp.experts.12.gate_and_up_proj,0.0003628990,0.05000,2.445
10,mlp.experts.13.gate_and_up_proj,0.0010372973,0.05000,2.510
10,mlp.experts.14.gate_and_up_proj,0.0000000000,0.05000,2.486
10,mlp.experts.15.gate_and_up_proj,0.0003760677,0.05000,2.324
10,mlp.experts.16.gate_and_up_proj,0.0023027397,0.05000,2.392
10,mlp.experts.17.gate_and_up_proj,0.0000775356,0.05000,2.354
10,mlp.experts.18.gate_and_up_proj,0.0001339843,0.05000,2.444
10,mlp.experts.19.gate_and_up_proj,0.0039844671,0.05000,2.523
10,mlp.experts.20.gate_and_up_proj,0.0003415548,0.05000,2.544
10,mlp.experts.21.gate_and_up_proj,0.0038770176,0.05000,2.460
10,mlp.experts.22.gate_and_up_proj,0.0003813441,0.05000,2.580
10,mlp.experts.23.gate_and_up_proj,0.0008177588,0.05000,2.500
10,mlp.experts.24.gate_and_up_proj,0.0004365585,0.05000,2.483
10,mlp.experts.25.gate_and_up_proj,0.0002569025,0.05000,2.383
10,mlp.experts.26.gate_and_up_proj,0.0000646536,0.05000,2.388
10,mlp.experts.27.gate_and_up_proj,0.0001714800,0.05000,2.426
10,mlp.experts.29.gate_and_up_proj,0.0012784553,0.05000,2.431
10,mlp.experts.28.gate_and_up_proj,0.0001949333,0.05000,2.698
10,mlp.experts.30.gate_and_up_proj,0.0012313536,0.05000,2.435
10,mlp.experts.31.gate_and_up_proj,0.0011619748,0.05000,2.487
10,mlp.experts.32.gate_and_up_proj,0.0012023281,0.05000,2.481
10,mlp.experts.33.gate_and_up_proj,0.0015404068,0.05000,2.457
10,mlp.experts.34.gate_and_up_proj,0.0005174841,0.05000,2.491
10,mlp.experts.35.gate_and_up_proj,0.0008092610,0.05000,2.605
10,mlp.experts.36.gate_and_up_proj,0.0000000000,0.05000,2.670
10,mlp.experts.37.gate_and_up_proj,0.0004253276,0.05000,2.538
10,mlp.experts.38.gate_and_up_proj,0.0001032968,0.05000,2.553
10,mlp.experts.39.gate_and_up_proj,0.0032159445,0.05000,2.397
10,mlp.experts.40.gate_and_up_proj,0.0027542553,0.05000,2.411
10,mlp.experts.41.gate_and_up_proj,0.0004416156,0.05000,2.382
10,mlp.experts.42.gate_and_up_proj,0.0009465787,0.05000,2.311
10,mlp.experts.43.gate_and_up_proj,0.0000688053,0.05000,2.332
10,mlp.experts.44.gate_and_up_proj,0.0003201085,0.05000,2.407
10,mlp.experts.45.gate_and_up_proj,0.0000000000,0.05000,2.295
10,mlp.experts.46.gate_and_up_proj,0.0019097092,0.05000,2.315
10,mlp.experts.47.gate_and_up_proj,0.0012569833,0.05000,2.335
10,mlp.experts.48.gate_and_up_proj,0.0023224034,0.05000,2.415
10,mlp.experts.49.gate_and_up_proj,0.0001379998,0.05000,2.575
10,mlp.experts.50.gate_and_up_proj,0.0011718015,0.05000,2.630
10,mlp.experts.51.gate_and_up_proj,0.0013678101,0.05000,2.600
10,mlp.experts.52.gate_and_up_proj,0.0004455770,0.05000,2.656
10,mlp.experts.53.gate_and_up_proj,0.0002653734,0.05000,2.630
10,mlp.experts.54.gate_and_up_proj,0.0000362171,0.05000,2.559
10,mlp.experts.55.gate_and_up_proj,0.0000435790,0.05000,2.521
10,mlp.experts.56.gate_and_up_proj,0.0000000000,0.05000,2.479
10,mlp.experts.57.gate_and_up_proj,0.0011344176,0.05000,2.557
10,mlp.experts.58.gate_and_up_proj,0.0000798748,0.05000,2.544
10,mlp.experts.59.gate_and_up_proj,0.0005198136,0.05000,2.662
10,mlp.experts.60.gate_and_up_proj,0.0003894578,0.05000,2.475
10,mlp.experts.61.gate_and_up_proj,0.0002581698,0.05000,2.463
10,mlp.experts.62.gate_and_up_proj,0.0008449160,0.05000,2.235
10,mlp.experts.63.gate_and_up_proj,0.0009295560,0.05000,1.381
10,mlp.experts.2.down_proj,0.0000464779,0.05000,1.898
10,mlp.experts.1.down_proj,0.0000005689,0.05000,1.924
10,mlp.experts.0.down_proj,0.0000115312,0.05000,2.017
10,mlp.experts.3.down_proj,0.0000524688,0.05000,1.956
10,mlp.experts.4.down_proj,0.0000440249,0.05000,1.946
10,mlp.experts.5.down_proj,0.0000071944,0.05000,1.926
10,mlp.experts.6.down_proj,0.0000043383,0.05000,2.018
10,mlp.experts.7.down_proj,0.0000361040,0.05000,2.029
10,mlp.experts.8.down_proj,0.0000279078,0.05000,2.052
10,mlp.experts.9.down_proj,0.0000137866,0.05000,2.144
10,mlp.experts.10.down_proj,0.0000495736,0.05000,2.178
10,mlp.experts.11.down_proj,0.0000143327,0.05000,2.149
10,mlp.experts.12.down_proj,0.0000059300,0.05000,1.996
10,mlp.experts.13.down_proj,0.0000222223,0.05000,1.994
10,mlp.experts.14.down_proj,0.0000000000,0.05000,1.986
10,mlp.experts.15.down_proj,0.0000059129,0.05000,1.883
10,mlp.experts.16.down_proj,0.0000521681,0.05000,1.931
10,mlp.experts.17.down_proj,0.0000016164,0.05000,1.906
10,mlp.experts.18.down_proj,0.0000042283,0.05000,1.928
10,mlp.experts.20.down_proj,0.0000071737,0.05000,1.944
10,mlp.experts.19.down_proj,0.0001130890,0.05000,2.048
10,mlp.experts.21.down_proj,0.0001176746,0.05000,2.044
10,mlp.experts.22.down_proj,0.0000083114,0.05000,1.998
10,mlp.experts.23.down_proj,0.0000171237,0.05000,2.006
10,mlp.experts.24.down_proj,0.0000150540,0.05000,2.064
10,mlp.experts.25.down_proj,0.0000112474,0.05000,2.052
10,mlp.experts.26.down_proj,0.0000055554,0.05000,2.015
10,mlp.experts.27.down_proj,0.0000044496,0.05000,1.890
10,mlp.experts.28.down_proj,0.0000065126,0.05000,1.883
10,mlp.experts.29.down_proj,0.0000455004,0.05000,1.856
10,mlp.experts.30.down_proj,0.0000243746,0.05000,1.952
10,mlp.experts.31.down_proj,0.0000442169,0.05000,1.928
10,mlp.experts.32.down_proj,0.0000246815,0.05000,1.926
10,mlp.experts.33.down_proj,0.0000416813,0.05000,1.904
10,mlp.experts.34.down_proj,0.0000101051,0.05000,1.893
10,mlp.experts.35.down_proj,0.0000250772,0.05000,1.889
10,mlp.experts.36.down_proj,0.0000000000,0.05000,1.813
10,mlp.experts.37.down_proj,0.0000088446,0.05000,1.858
10,mlp.experts.38.down_proj,0.0000020998,0.05000,1.864
10,mlp.experts.39.down_proj,0.0000910958,0.05000,1.945
10,mlp.experts.40.down_proj,0.0000709907,0.05000,1.927
10,mlp.experts.41.down_proj,0.0000137788,0.05000,1.932
10,mlp.experts.42.down_proj,0.0000230336,0.05000,2.062
10,mlp.experts.43.down_proj,0.0000013077,0.05000,2.065
10,mlp.experts.44.down_proj,0.0000090036,0.05000,2.057
10,mlp.experts.45.down_proj,0.0000000000,0.05000,2.026
10,mlp.experts.47.down_proj,0.0000294473,0.05000,1.966
10,mlp.experts.46.down_proj,0.0000399257,0.05000,2.000
10,mlp.experts.50.down_proj,0.0000309132,0.05000,1.993
10,mlp.experts.49.down_proj,0.0000032182,0.05000,2.021
10,mlp.experts.48.down_proj,0.0000590153,0.05000,2.175
10,mlp.experts.51.down_proj,0.0000447398,0.05000,2.121
10,mlp.experts.52.down_proj,0.0000095098,0.05000,2.143
10,mlp.experts.53.down_proj,0.0000078879,0.05000,2.111
10,mlp.experts.54.down_proj,0.0000007457,0.05000,2.040
10,mlp.experts.55.down_proj,0.0000009998,0.05000,2.083
10,mlp.experts.56.down_proj,0.0000000000,0.05000,2.123
10,mlp.experts.57.down_proj,0.0000293768,0.05000,1.985
10,mlp.experts.58.down_proj,0.0000020952,0.05000,1.995
10,mlp.experts.59.down_proj,0.0000128769,0.05000,2.022
10,mlp.experts.60.down_proj,0.0000094809,0.05000,1.918
10,mlp.experts.61.down_proj,0.0000068299,0.05000,1.927
10,mlp.experts.62.down_proj,0.0000126836,0.05000,1.879
10,mlp.experts.63.down_proj,0.0000214054,0.05000,0.793
11,self_attn.qkv_proj,0.0001312802,0.05000,0.943
11,self_attn.o_proj,0.0000656495,0.05000,0.936
11,mlp.gate.wg,0.0000959795,0.05000,0.914
11,mlp.shared_mlp.gate_and_up_proj,0.0015470892,0.05000,0.941
11,mlp.shared_mlp.down_proj,0.0000124434,0.05000,0.696
11,mlp.experts.2.gate_and_up_proj,0.0002038187,0.05000,2.576
11,mlp.experts.1.gate_and_up_proj,0.0001403459,0.05000,2.586
11,mlp.experts.0.gate_and_up_proj,0.0000389906,0.05000,2.701
11,mlp.experts.3.gate_and_up_proj,0.0004477693,0.05000,2.364
11,mlp.experts.4.gate_and_up_proj,0.0005727747,0.05000,2.396
11,mlp.experts.5.gate_and_up_proj,0.0034226617,0.05000,2.370
11,mlp.experts.6.gate_and_up_proj,0.0008090909,0.05000,2.459
11,mlp.experts.7.gate_and_up_proj,0.0013796968,0.05000,2.445
11,mlp.experts.8.gate_and_up_proj,0.0005496189,0.05000,2.424
11,mlp.experts.9.gate_and_up_proj,0.0009694313,0.05000,2.396
11,mlp.experts.10.gate_and_up_proj,0.0002827694,0.05000,2.415
11,mlp.experts.11.gate_and_up_proj,0.0000000000,0.05000,2.397
11,mlp.experts.12.gate_and_up_proj,0.0014356062,0.05000,2.383
11,mlp.experts.13.gate_and_up_proj,0.0001249970,0.05000,2.464
11,mlp.experts.14.gate_and_up_proj,0.0001451414,0.05000,2.430
11,mlp.experts.15.gate_and_up_proj,0.0003312115,0.05000,2.310
11,mlp.experts.17.gate_and_up_proj,0.0000000000,0.05000,2.326
11,mlp.experts.16.gate_and_up_proj,0.0015118701,0.05000,2.381
11,mlp.experts.19.gate_and_up_proj,0.0001035626,0.05000,2.503
11,mlp.experts.20.gate_and_up_proj,0.0000000000,0.05000,2.500
11,mlp.experts.18.gate_and_up_proj,0.0010350290,0.05000,2.801
11,mlp.experts.21.gate_and_up_proj,0.0003448003,0.05000,2.442
11,mlp.experts.22.gate_and_up_proj,0.0016119539,0.05000,2.528
11,mlp.experts.23.gate_and_up_proj,0.0016518637,0.05000,2.504
11,mlp.experts.24.gate_and_up_proj,0.0007043405,0.05000,2.284
11,mlp.experts.25.gate_and_up_proj,0.0002414270,0.05000,2.334
11,mlp.experts.26.gate_and_up_proj,0.0017503916,0.05000,2.300
11,mlp.experts.27.gate_and_up_proj,0.0038018938,0.05000,2.286
11,mlp.experts.28.gate_and_up_proj,0.0004179648,0.05000,2.353
11,mlp.experts.29.gate_and_up_proj,0.0005088458,0.05000,2.347
11,mlp.experts.30.gate_and_up_proj,0.0003488203,0.05000,2.421
11,mlp.experts.31.gate_and_up_proj,0.0005010090,0.05000,2.458
11,mlp.experts.32.gate_and_up_proj,0.0001154348,0.05000,2.451
11,mlp.experts.33.gate_and_up_proj,0.0006668223,0.05000,2.356
11,mlp.experts.34.gate_and_up_proj,0.0018670428,0.05000,2.379
11,mlp.experts.35.gate_and_up_proj,0.0006601059,0.05000,2.382
11,mlp.experts.36.gate_and_up_proj,0.0000356097,0.05000,2.453
11,mlp.experts.37.gate_and_up_proj,0.0032607548,0.05000,2.452
11,mlp.experts.38.gate_and_up_proj,0.0009601314,0.05000,2.520
11,mlp.experts.39.gate_and_up_proj,0.0012776506,0.05000,2.303
11,mlp.experts.40.gate_and_up_proj,0.0037122523,0.05000,2.311
11,mlp.experts.41.gate_and_up_proj,0.0026324021,0.05000,2.344
11,mlp.experts.42.gate_and_up_proj,0.0003831250,0.05000,2.269
11,mlp.experts.43.gate_and_up_proj,0.0002017173,0.05000,2.300
11,mlp.experts.44.gate_and_up_proj,0.0014219021,0.05000,2.312
11,mlp.experts.45.gate_and_up_proj,0.0008682093,0.05000,2.298
11,mlp.experts.46.gate_and_up_proj,0.0003553499,0.05000,2.362
11,mlp.experts.47.gate_and_up_proj,0.0002595275,0.05000,2.388
11,mlp.experts.48.gate_and_up_proj,0.0000914702,0.05000,2.296
11,mlp.experts.49.gate_and_up_proj,0.0034205846,0.05000,2.347
11,mlp.experts.50.gate_and_up_proj,0.0007918844,0.05000,2.342
11,mlp.experts.51.gate_and_up_proj,0.0003561900,0.05000,2.343
11,mlp.experts.52.gate_and_up_proj,0.0000000000,0.05000,2.428
11,mlp.experts.53.gate_and_up_proj,0.0003359804,0.05000,2.470
11,mlp.experts.54.gate_and_up_proj,0.0020373586,0.05000,2.520
11,mlp.experts.55.gate_and_up_proj,0.0007781330,0.05000,2.564
11,mlp.experts.56.gate_and_up_proj,0.0005575765,0.05000,2.545
11,mlp.experts.57.gate_and_up_proj,0.0003040938,0.05000,2.535
11,mlp.experts.58.gate_and_up_proj,0.0000777337,0.05000,2.566
11,mlp.experts.59.gate_and_up_proj,0.0018356344,0.05000,2.591
11,mlp.experts.60.gate_and_up_proj,0.0008778119,0.05000,2.540
11,mlp.experts.61.gate_and_up_proj,0.0012986575,0.05000,2.513
11,mlp.experts.62.gate_and_up_proj,0.0000746175,0.05000,2.449
11,mlp.experts.63.gate_and_up_proj,0.0028270584,0.05000,1.359
11,mlp.experts.0.down_proj,0.0000006175,0.05000,1.720
11,mlp.experts.1.down_proj,0.0000031416,0.05000,1.739
11,mlp.experts.2.down_proj,0.0000065099,0.05000,1.749
11,mlp.experts.3.down_proj,0.0000115480,0.05000,1.714
11,mlp.experts.4.down_proj,0.0000144367,0.05000,1.761
11,mlp.experts.5.down_proj,0.0001055708,0.05000,1.757
11,mlp.experts.6.down_proj,0.0000193570,0.05000,1.743
11,mlp.experts.8.down_proj,0.0000169565,0.05000,1.771
11,mlp.experts.7.down_proj,0.0000499772,0.05000,1.836
11,mlp.experts.9.down_proj,0.0000225687,0.05000,1.848
11,mlp.experts.10.down_proj,0.0000074869,0.05000,1.774
11,mlp.experts.11.down_proj,0.0000000000,0.05000,1.854
11,mlp.experts.12.down_proj,0.0000343458,0.05000,1.772
11,mlp.experts.13.down_proj,0.0000054740,0.05000,1.781
11,mlp.experts.14.down_proj,0.0000032243,0.05000,1.791
11,mlp.experts.15.down_proj,0.0000077358,0.05000,1.782
11,mlp.experts.16.down_proj,0.0000300145,0.05000,1.787
11,mlp.experts.17.down_proj,0.0000000000,0.05000,1.800
11,mlp.experts.18.down_proj,0.0000257923,0.05000,1.767
11,mlp.experts.19.down_proj,0.0000020385,0.05000,1.751
11,mlp.experts.20.down_proj,0.0000000000,0.05000,1.764
11,mlp.experts.21.down_proj,0.0000088984,0.05000,1.845
11,mlp.experts.22.down_proj,0.0000473265,0.05000,1.832
11,mlp.experts.23.down_proj,0.0000524026,0.05000,1.822
11,mlp.experts.24.down_proj,0.0000242601,0.05000,1.796
11,mlp.experts.25.down_proj,0.0000092104,0.05000,1.800
11,mlp.experts.26.down_proj,0.0000386751,0.05000,1.806
11,mlp.experts.27.down_proj,0.0001130329,0.05000,1.788
11,mlp.experts.28.down_proj,0.0000090672,0.05000,1.801
11,mlp.experts.29.down_proj,0.0000128894,0.05000,1.844
11,mlp.experts.31.down_proj,0.0000233440,0.05000,1.869
11,mlp.experts.30.down_proj,0.0000092206,0.05000,1.917
11,mlp.experts.32.down_proj,0.0000031530,0.05000,1.925
11,mlp.experts.33.down_proj,0.0000193785,0.05000,1.842
11,mlp.experts.34.down_proj,0.0000473996,0.05000,1.857
11,mlp.experts.35.down_proj,0.0000170910,0.05000,1.911
11,mlp.experts.36.down_proj,0.0000011149,0.05000,1.941
11,mlp.experts.37.down_proj,0.0000931621,0.05000,1.950
11,mlp.experts.38.down_proj,0.0000345134,0.05000,1.976
11,mlp.experts.39.down_proj,0.0000283187,0.05000,1.995
11,mlp.experts.40.down_proj,0.0000699229,0.05000,1.952
11,mlp.experts.41.down_proj,0.0000589668,0.05000,1.974
11,mlp.experts.43.down_proj,0.0000046922,0.05000,1.941
11,mlp.experts.42.down_proj,0.0000083608,0.05000,1.996
11,mlp.experts.44.down_proj,0.0000298158,0.05000,1.936
11,mlp.experts.45.down_proj,0.0000242020,0.05000,1.812
11,mlp.experts.46.down_proj,0.0000909976,0.05000,1.830
11,mlp.experts.47.down_proj,0.0000046091,0.05000,1.856
11,mlp.experts.48.down_proj,0.0000020064,0.05000,1.972
11,mlp.experts.49.down_proj,0.0000682446,0.05000,1.976
11,mlp.experts.50.down_proj,0.0000193576,0.05000,1.958
11,mlp.experts.51.down_proj,0.0000066958,0.05000,1.913
11,mlp.experts.52.down_proj,0.0000000000,0.05000,1.912
11,mlp.experts.53.down_proj,0.0000173712,0.05000,1.930
11,mlp.experts.54.down_proj,0.0000568936,0.05000,1.855
11,mlp.experts.55.down_proj,0.0000241917,0.05000,1.865
11,mlp.experts.56.down_proj,0.0000134551,0.05000,1.882
11,mlp.experts.57.down_proj,0.0000076078,0.05000,1.790
11,mlp.experts.58.down_proj,0.0000015525,0.05000,1.829
11,mlp.experts.59.down_proj,0.0000452005,0.05000,1.846
11,mlp.experts.60.down_proj,0.0000238530,0.05000,1.802
11,mlp.experts.61.down_proj,0.0000358783,0.05000,1.837
11,mlp.experts.62.down_proj,0.0000013930,0.05000,1.702
11,mlp.experts.63.down_proj,0.0000664949,0.05000,0.860
12,self_attn.qkv_proj,0.0001512585,0.05000,0.944
12,self_attn.o_proj,0.0000567460,0.05000,0.936
12,mlp.gate.wg,0.0000984876,0.05000,0.912
12,mlp.shared_mlp.gate_and_up_proj,0.0015328547,0.05000,0.941
12,mlp.shared_mlp.down_proj,0.0000090021,0.05000,0.692
12,mlp.experts.0.gate_and_up_proj,0.0005714005,0.05000,2.385
12,mlp.experts.1.gate_and_up_proj,0.0000899810,0.05000,2.408
12,mlp.experts.2.gate_and_up_proj,0.0009528996,0.05000,2.513
12,mlp.experts.3.gate_and_up_proj,0.0036783784,0.05000,2.309
12,mlp.experts.4.gate_and_up_proj,0.0008841285,0.05000,2.317
12,mlp.experts.5.gate_and_up_proj,0.0009592619,0.05000,2.303
12,mlp.experts.6.gate_and_up_proj,0.0015376650,0.05000,2.321
12,mlp.experts.7.gate_and_up_proj,0.0006321520,0.05000,2.392
12,mlp.experts.8.gate_and_up_proj,0.0003510726,0.05000,2.357
12,mlp.experts.9.gate_and_up_proj,0.0008688873,0.05000,2.329
12,mlp.experts.10.gate_and_up_proj,0.0035032460,0.05000,2.372
12,mlp.experts.11.gate_and_up_proj,0.0006526443,0.05000,2.370
12,mlp.experts.12.gate_and_up_proj,0.0010697483,0.05000,2.367
12,mlp.experts.13.gate_and_up_proj,0.0000000000,0.05000,2.380
12,mlp.experts.14.gate_and_up_proj,0.0002409073,0.05000,2.387
12,mlp.experts.15.gate_and_up_proj,0.0009264025,0.05000,2.353
12,mlp.experts.17.gate_and_up_proj,0.0006324762,0.05000,2.387
12,mlp.experts.16.gate_and_up_proj,0.0009030334,0.05000,2.561
12,mlp.experts.18.gate_and_up_proj,0.0013206578,0.05000,2.506
12,mlp.experts.19.gate_and_up_proj,0.0008377703,0.05000,2.455
12,mlp.experts.20.gate_and_up_proj,0.0006028143,0.05000,2.710
12,mlp.experts.21.gate_and_up_proj,0.0001778515,0.05000,2.421
12,mlp.experts.22.gate_and_up_proj,0.0014545426,0.05000,2.423
12,mlp.experts.23.gate_and_up_proj,0.0010316398,0.05000,2.525
12,mlp.experts.24.gate_and_up_proj,0.0007679197,0.05000,2.395
12,mlp.experts.25.gate_and_up_proj,0.0000897295,0.05000,2.412
12,mlp.experts.26.gate_and_up_proj,0.0000771388,0.05000,2.458
12,mlp.experts.27.gate_and_up_proj,0.0000968518,0.05000,2.295
12,mlp.experts.28.gate_and_up_proj,0.0009396153,0.05000,2.338
12,mlp.experts.29.gate_and_up_proj,0.0014164258,0.05000,2.380
12,mlp.experts.30.gate_and_up_proj,0.0031135971,0.05000,2.320
12,mlp.experts.31.gate_and_up_proj,0.0008066410,0.05000,2.370
12,mlp.experts.32.gate_and_up_proj,0.0000378134,0.05000,2.458
12,mlp.experts.33.gate_and_up_proj,0.0003740928,0.05000,2.469
12,mlp.experts.34.gate_and_up_proj,0.0001931240,0.05000,2.483
12,mlp.experts.35.gate_and_up_proj,0.0000859622,0.05000,2.602
12,mlp.experts.36.gate_and_up_proj,0.0001327523,0.05000,2.464
12,mlp.experts.37.gate_and_up_proj,0.0006281426,0.05000,2.486
12,mlp.experts.38.gate_and_up_proj,0.0004302500,0.05000,2.539
12,mlp.experts.39.gate_and_up_proj,0.0040146056,0.05000,2.576
12,mlp.experts.40.gate_and_up_proj,0.0000715504,0.05000,2.582
12,mlp.experts.41.gate_and_up_proj,0.0016489678,0.05000,2.627
12,mlp.experts.42.gate_and_up_proj,0.0009209153,0.05000,2.480
12,mlp.experts.43.gate_and_up_proj,0.0000436344,0.05000,2.440
12,mlp.experts.44.gate_and_up_proj,0.0001153330,0.05000,2.391
12,mlp.experts.45.gate_and_up_proj,0.0015954817,0.05000,2.394
12,mlp.experts.46.gate_and_up_proj,0.0030837167,0.05000,2.404
12,mlp.experts.47.gate_and_up_proj,0.0011986951,0.05000,2.433
12,mlp.experts.48.gate_and_up_proj,0.0001640934,0.05000,2.423
12,mlp.experts.49.gate_and_up_proj,0.0042976421,0.05000,2.476
12,mlp.experts.50.gate_and_up_proj,0.0000713225,0.05000,2.546
12,mlp.experts.51.gate_and_up_proj,0.0000658275,0.05000,2.429
12,mlp.experts.52.gate_and_up_proj,0.0021228903,0.05000,2.394
12,mlp.experts.53.gate_and_up_proj,0.0023864950,0.05000,2.381
12,mlp.experts.54.gate_and_up_proj,0.0034704989,0.05000,2.438
12,mlp.experts.55.gate_and_up_proj,0.0000659735,0.05000,2.507
12,mlp.experts.56.gate_and_up_proj,0.0010491708,0.05000,2.590
12,mlp.experts.57.gate_and_up_proj,0.0025216030,0.05000,2.617
12,mlp.experts.58.gate_and_up_proj,0.0002411097,0.05000,2.615
12,mlp.experts.59.gate_and_up_proj,0.0009253346,0.05000,2.748
12,mlp.experts.60.gate_and_up_proj,0.0012031641,0.05000,2.625
12,mlp.experts.61.gate_and_up_proj,0.0000000000,0.05000,2.633
12,mlp.experts.62.gate_and_up_proj,0.0002117402,0.05000,2.137
12,mlp.experts.63.gate_and_up_proj,0.0000000000,0.05000,1.406
12,mlp.experts.0.down_proj,0.0000235661,0.05000,1.801
12,mlp.experts.2.down_proj,0.0000341092,0.05000,1.827
12,mlp.experts.1.down_proj,0.0000024312,0.05000,1.947
12,mlp.experts.3.down_proj,0.0000898847,0.05000,1.859
12,mlp.experts.4.down_proj,0.0000192417,0.05000,1.866
12,mlp.experts.5.down_proj,0.0000260248,0.05000,1.905
12,mlp.experts.6.down_proj,0.0000357203,0.05000,1.890
12,mlp.experts.7.down_proj,0.0000148560,0.05000,1.890
12,mlp.experts.8.down_proj,0.0000104889,0.05000,1.899
12,mlp.experts.9.down_proj,0.0000175784,0.05000,1.853
12,mlp.experts.10.down_proj,0.0000754927,0.05000,1.842
12,mlp.experts.11.down_proj,0.0000213333,0.05000,1.891
12,mlp.experts.12.down_proj,0.0000278566,0.05000,1.814
12,mlp.experts.13.down_proj,0.0000000000,0.05000,1.807
12,mlp.experts.14.down_proj,0.0000052605,0.05000,1.850
12,mlp.experts.15.down_proj,0.0000308348,0.05000,1.881
12,mlp.experts.16.down_proj,0.0000185751,0.05000,1.901
12,mlp.experts.17.down_proj,0.0000154737,0.05000,1.894
12,mlp.experts.18.down_proj,0.0000330718,0.05000,1.862
12,mlp.experts.19.down_proj,0.0000151325,0.05000,1.853
12,mlp.experts.20.down_proj,0.0000173117,0.05000,1.878
12,mlp.experts.22.down_proj,0.0000348497,0.05000,1.903
12,mlp.experts.21.down_proj,0.0000054772,0.05000,1.972
12,mlp.experts.23.down_proj,0.0000219568,0.05000,1.950
12,mlp.experts.24.down_proj,0.0000162479,0.05000,1.826
12,mlp.experts.25.down_proj,0.0000023788,0.05000,1.876
12,mlp.experts.26.down_proj,0.0000023760,0.05000,1.885
12,mlp.experts.27.down_proj,0.0000018444,0.05000,1.811
12,mlp.experts.28.down_proj,0.0000210004,0.05000,1.875
12,mlp.experts.29.down_proj,0.0000439464,0.05000,1.903
12,mlp.experts.30.down_proj,0.0000675503,0.05000,1.798
12,mlp.experts.31.down_proj,0.0000178836,0.05000,1.848
12,mlp.experts.32.down_proj,0.0000005657,0.05000,1.823
12,mlp.experts.33.down_proj,0.0000083541,0.05000,1.775
12,mlp.experts.34.down_proj,0.0000042282,0.05000,1.814
12,mlp.experts.35.down_proj,0.0000029173,0.05000,1.873
12,mlp.experts.36.down_proj,0.0000031905,0.05000,1.821
12,mlp.experts.37.down_proj,0.0000128098,0.05000,1.874
12,mlp.experts.38.down_proj,0.0000124531,0.05000,1.959
12,mlp.experts.39.down_proj,0.0001003325,0.05000,2.028
12,mlp.experts.40.down_proj,0.0000016188,0.05000,2.018
12,mlp.experts.41.down_proj,0.0000413095,0.05000,2.088
12,mlp.experts.42.down_proj,0.0000212370,0.05000,2.067
12,mlp.experts.43.down_proj,0.0000018706,0.05000,2.102
12,mlp.experts.44.down_proj,0.0000056543,0.05000,2.103
12,mlp.experts.45.down_proj,0.0000386052,0.05000,2.100
12,mlp.experts.46.down_proj,0.0000719616,0.05000,2.062
12,mlp.experts.47.down_proj,0.0000339936,0.05000,2.024
12,mlp.experts.48.down_proj,0.0000038675,0.05000,2.044
12,mlp.experts.49.down_proj,0.0000989775,0.05000,1.985
12,mlp.experts.50.down_proj,0.0000017296,0.05000,2.042
12,mlp.experts.51.down_proj,0.0000011213,0.05000,1.953
12,mlp.experts.52.down_proj,0.0000533431,0.05000,1.945
12,mlp.experts.53.down_proj,0.0000570668,0.05000,1.984
12,mlp.experts.54.down_proj,0.0000681424,0.05000,2.047
12,mlp.experts.55.down_proj,0.0000037168,0.05000,1.999
12,mlp.experts.56.down_proj,0.0000342357,0.05000,2.016
12,mlp.experts.57.down_proj,0.0000644543,0.05000,2.037
12,mlp.experts.58.down_proj,0.0000043929,0.05000,2.051
12,mlp.experts.59.down_proj,0.0000172305,0.05000,2.023
12,mlp.experts.60.down_proj,0.0000346667,0.05000,1.927
12,mlp.experts.61.down_proj,0.0000000000,0.05000,1.954
12,mlp.experts.62.down_proj,0.0000049318,0.05000,1.741
12,mlp.experts.63.down_proj,0.0000000000,0.05000,0.952
13,self_attn.qkv_proj,0.0001786940,0.05000,0.941
13,self_attn.o_proj,0.0000702613,0.05000,0.951
13,mlp.gate.wg,0.0001192095,0.05000,0.915
13,mlp.shared_mlp.gate_and_up_proj,0.0014438717,0.05000,0.950
13,mlp.shared_mlp.down_proj,0.0000096936,0.05000,0.705
13,mlp.experts.0.gate_and_up_proj,0.0002141100,0.05000,2.473
13,mlp.experts.1.gate_and_up_proj,0.0016708128,0.05000,2.491
13,mlp.experts.2.gate_and_up_proj,0.0012532014,0.05000,2.500
13,mlp.experts.3.gate_and_up_proj,0.0008592445,0.05000,2.404
13,mlp.experts.5.gate_and_up_proj,0.0000367021,0.05000,2.451
13,mlp.experts.4.gate_and_up_proj,0.0005894250,0.05000,2.501
13,mlp.experts.7.gate_and_up_proj,0.0005369104,0.05000,2.464
13,mlp.experts.6.gate_and_up_proj,0.0002835659,0.05000,2.593
13,mlp.experts.8.gate_and_up_proj,0.0001801619,0.05000,2.495
13,mlp.experts.9.gate_and_up_proj,0.0004900518,0.05000,2.353
13,mlp.experts.10.gate_and_up_proj,0.0006656292,0.05000,2.404
13,mlp.experts.11.gate_and_up_proj,0.0010396993,0.05000,2.420
13,mlp.experts.12.gate_and_up_proj,0.0000633978,0.05000,2.362
13,mlp.experts.13.gate_and_up_proj,0.0025723268,0.05000,2.406
13,mlp.experts.14.gate_and_up_proj,0.0017060048,0.05000,2.426
13,mlp.experts.15.gate_and_up_proj,0.0001531782,0.05000,2.356
13,mlp.experts.16.gate_and_up_proj,0.0006020995,0.05000,2.426
13,mlp.experts.17.gate_and_up_proj,0.0008219193,0.05000,2.388
13,mlp.experts.18.gate_and_up_proj,0.0008069384,0.05000,2.431
13,mlp.experts.19.gate_and_up_proj,0.0010367091,0.05000,2.508
13,mlp.experts.20.gate_and_up_proj,0.0000753231,0.05000,2.508
13,mlp.experts.21.gate_and_up_proj,0.0001412321,0.05000,2.421
13,mlp.experts.22.gate_and_up_proj,0.0001953006,0.05000,2.423
13,mlp.experts.23.gate_and_up_proj,0.0006438847,0.05000,2.416
13,mlp.experts.24.gate_and_up_proj,0.0002293067,0.05000,2.279
13,mlp.experts.26.gate_and_up_proj,0.0026443939,0.05000,2.334
13,mlp.experts.25.gate_and_up_proj,0.0003635033,0.05000,2.377
13,mlp.experts.27.gate_and_up_proj,0.0005666108,0.05000,2.822
13,mlp.experts.28.gate_and_up_proj,0.0001088698,0.05000,2.584
13,mlp.experts.29.gate_and_up_proj,0.0007775591,0.05000,2.705
13,mlp.experts.30.gate_and_up_proj,0.0010656189,0.05000,2.634
13,mlp.experts.31.gate_and_up_proj,0.0000532649,0.05000,2.584
13,mlp.experts.32.gate_and_up_proj,0.0001149042,0.05000,2.568
13,mlp.experts.33.gate_and_up_proj,0.0001062159,0.05000,2.399
13,mlp.experts.34.gate_and_up_proj,0.0006488973,0.05000,2.407
13,mlp.experts.35.gate_and_up_proj,0.0012268485,0.05000,2.402
13,mlp.experts.36.gate_and_up_proj,0.0027644409,0.05000,2.445
13,mlp.experts.37.gate_and_up_proj,0.0025324241,0.05000,2.472
13,mlp.experts.38.gate_and_up_proj,0.0014871877,0.05000,2.473
13,mlp.experts.39.gate_and_up_proj,0.0003472805,0.05000,2.532
13,mlp.experts.40.gate_and_up_proj,0.0045520744,0.05000,2.564
13,mlp.experts.41.gate_and_up_proj,0.0000341845,0.05000,2.528
13,mlp.experts.42.gate_and_up_proj,0.0013074343,0.05000,2.464
13,mlp.experts.43.gate_and_up_proj,0.0000677092,0.05000,2.438
13,mlp.experts.44.gate_and_up_proj,0.0015413034,0.05000,2.466
13,mlp.experts.45.gate_and_up_proj,0.0016484086,0.05000,2.507
13,mlp.experts.46.gate_and_up_proj,0.0003378061,0.05000,2.488
13,mlp.experts.47.gate_and_up_proj,0.0004137825,0.05000,2.488
13,mlp.experts.48.gate_and_up_proj,0.0019323732,0.05000,2.412
13,mlp.experts.49.gate_and_up_proj,0.0005531559,0.05000,2.420
13,mlp.experts.50.gate_and_up_proj,0.0010875794,0.05000,2.418
13,mlp.experts.51.gate_and_up_proj,0.0010487007,0.05000,2.378
13,mlp.experts.52.gate_and_up_proj,0.0002625411,0.05000,2.423
13,mlp.experts.53.gate_and_up_proj,0.0020620295,0.05000,2.479
13,mlp.experts.54.gate_and_up_proj,0.0022118784,0.05000,2.359
13,mlp.experts.55.gate_and_up_proj,0.0013294930,0.05000,2.369
13,mlp.experts.56.gate_and_up_proj,0.0000537108,0.05000,2.373
13,mlp.experts.57.gate_and_up_proj,0.0000471971,0.05000,2.426
13,mlp.experts.58.gate_and_up_proj,0.0000723677,0.05000,2.387
13,mlp.experts.59.gate_and_up_proj,0.0024828598,0.05000,2.403
13,mlp.experts.60.gate_and_up_proj,0.0002512150,0.05000,2.453
13,mlp.experts.61.gate_and_up_proj,0.0000772830,0.05000,2.492
13,mlp.experts.62.gate_and_up_proj,0.0022887588,0.05000,2.449
13,mlp.experts.63.gate_and_up_proj,0.0001591399,0.05000,1.098
13,mlp.experts.1.down_proj,0.0000460428,0.05000,1.882
13,mlp.experts.2.down_proj,0.0000320122,0.05000,1.889
13,mlp.experts.0.down_proj,0.0000059720,0.05000,1.961
13,mlp.experts.3.down_proj,0.0000184843,0.05000,1.851
13,mlp.experts.4.down_proj,0.0000105840,0.05000,1.952
13,mlp.experts.5.down_proj,0.0000007068,0.05000,1.888
13,mlp.experts.6.down_proj,0.0000058130,0.05000,2.015
13,mlp.experts.7.down_proj,0.0000134921,0.05000,1.966
13,mlp.experts.8.down_proj,0.0000037148,0.05000,2.039
13,mlp.experts.9.down_proj,0.0000092823,0.05000,1.925
13,mlp.experts.10.down_proj,0.0000157621,0.05000,1.924
13,mlp.experts.11.down_proj,0.0000200828,0.05000,1.914
13,mlp.experts.12.down_proj,0.0000006511,0.05000,1.911
13,mlp.experts.13.down_proj,0.0000646977,0.05000,1.897
13,mlp.experts.14.down_proj,0.0000327079,0.05000,2.006
13,mlp.experts.15.down_proj,0.0000035873,0.05000,1.871
13,mlp.experts.16.down_proj,0.0000095834,0.05000,1.915
13,mlp.experts.17.down_proj,0.0000156190,0.05000,1.926
13,mlp.experts.18.down_proj,0.0000169897,0.05000,1.962
13,mlp.experts.19.down_proj,0.0000225873,0.05000,2.009
13,mlp.experts.20.down_proj,0.0000031785,0.05000,2.036
13,mlp.experts.21.down_proj,0.0000042172,0.05000,1.967
13,mlp.experts.22.down_proj,0.0000059227,0.05000,2.012
13,mlp.experts.23.down_proj,0.0000157384,0.05000,2.042
13,mlp.experts.24.down_proj,0.0000114267,0.05000,2.078
13,mlp.experts.25.down_proj,0.0000085760,0.05000,1.960
13,mlp.experts.26.down_proj,0.0000641136,0.05000,2.131
13,mlp.experts.27.down_proj,0.0000082999,0.05000,1.899
13,mlp.experts.28.down_proj,0.0000019129,0.05000,1.934
13,mlp.experts.29.down_proj,0.0000174075,0.05000,1.989
13,mlp.experts.30.down_proj,0.0000288441,0.05000,1.955
13,mlp.experts.31.down_proj,0.0000008060,0.05000,1.982
13,mlp.experts.32.down_proj,0.0000032491,0.05000,2.133
13,mlp.experts.33.down_proj,0.0000033599,0.05000,2.092
13,mlp.experts.34.down_proj,0.0000126042,0.05000,2.080
13,mlp.experts.35.down_proj,0.0000327871,0.05000,2.164
13,mlp.experts.36.down_proj,0.0000693673,0.05000,2.145
13,mlp.experts.37.down_proj,0.0000555933,0.05000,2.149
13,mlp.experts.38.down_proj,0.0000285410,0.05000,2.211
13,mlp.experts.39.down_proj,0.0000080313,0.05000,1.915
13,mlp.experts.40.down_proj,0.0001066195,0.05000,1.881
13,mlp.experts.41.down_proj,0.0000004864,0.05000,1.755
13,mlp.experts.42.down_proj,0.0000362212,0.05000,1.843
13,mlp.experts.43.down_proj,0.0000009050,0.05000,1.803
13,mlp.experts.44.down_proj,0.0000451690,0.05000,1.838
13,mlp.experts.45.down_proj,0.0000369221,0.05000,1.791
13,mlp.experts.46.down_proj,0.0000056776,0.05000,1.787
13,mlp.experts.47.down_proj,0.0000146217,0.05000,1.754
13,mlp.experts.48.down_proj,0.0000440811,0.05000,1.713
13,mlp.experts.49.down_proj,0.0000171819,0.05000,1.762
13,mlp.experts.50.down_proj,0.0000316847,0.05000,1.751
13,mlp.experts.51.down_proj,0.0000374385,0.05000,1.738
13,mlp.experts.52.down_proj,0.0000048051,0.05000,1.725
13,mlp.experts.53.down_proj,0.0000399238,0.05000,1.739
13,mlp.experts.54.down_proj,0.0000513730,0.05000,1.761
13,mlp.experts.55.down_proj,0.0000280794,0.05000,1.789
13,mlp.experts.56.down_proj,0.0000009639,0.05000,1.817
13,mlp.experts.57.down_proj,0.0000013082,0.05000,1.788
13,mlp.experts.58.down_proj,0.0000014680,0.05000,1.809
13,mlp.experts.59.down_proj,0.0000770267,0.05000,1.847
13,mlp.experts.60.down_proj,0.0000051015,0.05000,1.840
13,mlp.experts.61.down_proj,0.0000023237,0.05000,1.884
13,mlp.experts.62.down_proj,0.0000483097,0.05000,1.582
13,mlp.experts.63.down_proj,0.0000044765,0.05000,0.942
14,self_attn.qkv_proj,0.0001783586,0.05000,0.947
14,self_attn.o_proj,0.0000856143,0.05000,0.938
14,mlp.gate.wg,0.0001179270,0.05000,0.913
14,mlp.shared_mlp.gate_and_up_proj,0.0015275759,0.05000,0.941
14,mlp.shared_mlp.down_proj,0.0000110572,0.05000,0.701
14,mlp.experts.1.gate_and_up_proj,0.0027798832,0.05000,2.458
14,mlp.experts.2.gate_and_up_proj,0.0001392747,0.05000,2.495
14,mlp.experts.0.gate_and_up_proj,0.0000000000,0.05000,2.577
14,mlp.experts.3.gate_and_up_proj,0.0003625774,0.05000,2.347
14,mlp.experts.4.gate_and_up_proj,0.0000783097,0.05000,2.371
14,mlp.experts.5.gate_and_up_proj,0.0020181172,0.05000,2.340
14,mlp.experts.6.gate_and_up_proj,0.0000657734,0.05000,2.443
14,mlp.experts.8.gate_and_up_proj,0.0001025054,0.05000,2.379
14,mlp.experts.7.gate_and_up_proj,0.0004048237,0.05000,2.514
14,mlp.experts.9.gate_and_up_proj,0.0005353124,0.05000,2.483
14,mlp.experts.10.gate_and_up_proj,0.0001041104,0.05000,2.449
14,mlp.experts.11.gate_and_up_proj,0.0002082974,0.05000,2.439
14,mlp.experts.12.gate_and_up_proj,0.0001871714,0.05000,2.395
14,mlp.experts.13.gate_and_up_proj,0.0013825510,0.05000,2.389
14,mlp.experts.14.gate_and_up_proj,0.0013812209,0.05000,2.363
14,mlp.experts.15.gate_and_up_proj,0.0002897651,0.05000,2.325
14,mlp.experts.16.gate_and_up_proj,0.0002113377,0.05000,2.336
14,mlp.experts.17.gate_and_up_proj,0.0003007843,0.05000,2.346
14,mlp.experts.18.gate_and_up_proj,0.0009722569,0.05000,2.277
14,mlp.experts.19.gate_and_up_proj,0.0009515473,0.05000,2.316
14,mlp.experts.20.gate_and_up_proj,0.0008682778,0.05000,2.350
14,mlp.experts.21.gate_and_up_proj,0.0008351535,0.05000,2.299
14,mlp.experts.22.gate_and_up_proj,0.0009643452,0.05000,2.341
14,mlp.experts.23.gate_and_up_proj,0.0003686910,0.05000,2.341
14,mlp.experts.24.gate_and_up_proj,0.0010935287,0.05000,2.416
14,mlp.experts.25.gate_and_up_proj,0.0044406021,0.05000,2.471
14,mlp.experts.26.gate_and_up_proj,0.0004218804,0.05000,2.477
14,mlp.experts.27.gate_and_up_proj,0.0000662265,0.05000,2.351
14,mlp.experts.28.gate_and_up_proj,0.0003291908,0.05000,2.374
14,mlp.experts.29.gate_and_up_proj,0.0004826360,0.05000,2.424
14,mlp.experts.30.gate_and_up_proj,0.0014169975,0.05000,2.290
14,mlp.experts.31.gate_and_up_proj,0.0030288801,0.05000,2.294
14,mlp.experts.32.gate_and_up_proj,0.0023640602,0.05000,2.318
14,mlp.experts.33.gate_and_up_proj,0.0020845436,0.05000,2.341
14,mlp.experts.34.gate_and_up_proj,0.0022067347,0.05000,2.418
14,mlp.experts.35.gate_and_up_proj,0.0015911887,0.05000,2.446
14,mlp.experts.36.gate_and_up_proj,0.0003135391,0.05000,2.328
14,mlp.experts.37.gate_and_up_proj,0.0018325615,0.05000,2.366
14,mlp.experts.38.gate_and_up_proj,0.0026716981,0.05000,2.348
14,mlp.experts.39.gate_and_up_proj,0.0001938056,0.05000,2.264
14,mlp.experts.40.gate_and_up_proj,0.0008580844,0.05000,2.308
14,mlp.experts.41.gate_and_up_proj,0.0000530201,0.05000,2.303
14,mlp.experts.42.gate_and_up_proj,0.0000521679,0.05000,2.287
14,mlp.experts.43.gate_and_up_proj,0.0009281131,0.05000,2.312
14,mlp.experts.44.gate_and_up_proj,0.0014888562,0.05000,2.336
14,mlp.experts.45.gate_and_up_proj,0.0001800180,0.05000,2.312
14,mlp.experts.46.gate_and_up_proj,0.0002471139,0.05000,2.343
14,mlp.experts.47.gate_and_up_proj,0.0001634097,0.05000,2.416
14,mlp.experts.48.gate_and_up_proj,0.0000000000,0.05000,2.428
14,mlp.experts.49.gate_and_up_proj,0.0035809814,0.05000,2.415
14,mlp.experts.50.gate_and_up_proj,0.0007062023,0.05000,2.382
14,mlp.experts.51.gate_and_up_proj,0.0005211995,0.05000,2.356
14,mlp.experts.52.gate_and_up_proj,0.0005514002,0.05000,2.497
14,mlp.experts.53.gate_and_up_proj,0.0005487058,0.05000,2.452
14,mlp.experts.54.gate_and_up_proj,0.0004484240,0.05000,2.523
14,mlp.experts.55.gate_and_up_proj,0.0000552080,0.05000,2.454
14,mlp.experts.56.gate_and_up_proj,0.0001847390,0.05000,2.458
14,mlp.experts.57.gate_and_up_proj,0.0005347470,0.05000,2.377
14,mlp.experts.58.gate_and_up_proj,0.0000767496,0.05000,2.394
14,mlp.experts.59.gate_and_up_proj,0.0012879534,0.05000,2.363
14,mlp.experts.60.gate_and_up_proj,0.0010141196,0.05000,2.348
14,mlp.experts.61.gate_and_up_proj,0.0029325330,0.05000,2.459
14,mlp.experts.62.gate_and_up_proj,0.0024318515,0.05000,2.419
14,mlp.experts.63.gate_and_up_proj,0.0000641324,0.05000,1.342
14,mlp.experts.0.down_proj,0.0000000000,0.05000,1.905
14,mlp.experts.1.down_proj,0.0000519932,0.05000,1.931
14,mlp.experts.2.down_proj,0.0000025807,0.05000,2.063
14,mlp.experts.4.down_proj,0.0000010005,0.05000,1.956
14,mlp.experts.3.down_proj,0.0000082399,0.05000,2.058
14,mlp.experts.5.down_proj,0.0000484622,0.05000,1.955
14,mlp.experts.6.down_proj,0.0000016478,0.05000,1.969
14,mlp.experts.7.down_proj,0.0000141777,0.05000,1.993
14,mlp.experts.8.down_proj,0.0000036695,0.05000,1.958
14,mlp.experts.9.down_proj,0.0000101513,0.05000,1.933
14,mlp.experts.11.down_proj,0.0000082317,0.05000,1.924
14,mlp.experts.10.down_proj,0.0000020051,0.05000,1.948
14,mlp.experts.12.down_proj,0.0000054935,0.05000,1.982
14,mlp.experts.13.down_proj,0.0000426617,0.05000,2.026
14,mlp.experts.14.down_proj,0.0000523443,0.05000,2.031
14,mlp.experts.15.down_proj,0.0000051209,0.05000,2.075
14,mlp.experts.17.down_proj,0.0000152175,0.05000,2.118
14,mlp.experts.16.down_proj,0.0000065665,0.05000,2.152
14,mlp.experts.19.down_proj,0.0000208517,0.05000,2.061
14,mlp.experts.20.down_proj,0.0000288547,0.05000,2.063
14,mlp.experts.18.down_proj,0.0000198928,0.05000,2.310
14,mlp.experts.21.down_proj,0.0000201547,0.05000,2.062
14,mlp.experts.22.down_proj,0.0000185141,0.05000,2.056
14,mlp.experts.23.down_proj,0.0000109465,0.05000,2.100
14,mlp.experts.24.down_proj,0.0000330394,0.05000,2.047
14,mlp.experts.25.down_proj,0.0000956753,0.05000,2.040
14,mlp.experts.26.down_proj,0.0000102139,0.05000,2.045
14,mlp.experts.27.down_proj,0.0000015438,0.05000,2.017
14,mlp.experts.28.down_proj,0.0000136637,0.05000,2.040
14,mlp.experts.29.down_proj,0.0000135479,0.05000,2.076
14,mlp.experts.30.down_proj,0.0000319989,0.05000,1.950
14,mlp.experts.31.down_proj,0.0000872453,0.05000,1.965
14,mlp.experts.32.down_proj,0.0000482427,0.05000,1.970
14,mlp.experts.33.down_proj,0.0000580560,0.05000,1.901
14,mlp.experts.34.down_proj,0.0000520238,0.05000,1.930
14,mlp.experts.35.down_proj,0.0000323012,0.05000,1.927
14,mlp.experts.36.down_proj,0.0000063395,0.05000,1.948
14,mlp.experts.37.down_proj,0.0000484409,0.05000,1.923
14,mlp.experts.38.down_proj,0.0000712449,0.05000,1.946
14,mlp.experts.39.down_proj,0.0000038638,0.05000,2.017
14,mlp.experts.40.down_proj,0.0000130143,0.05000,2.022
14,mlp.experts.41.down_proj,0.0000009286,0.05000,2.044
14,mlp.experts.42.down_proj,0.0000007434,0.05000,2.121
14,mlp.experts.43.down_proj,0.0000343039,0.05000,2.134
14,mlp.experts.44.down_proj,0.0000414076,0.05000,2.159
14,mlp.experts.45.down_proj,0.0000054715,0.05000,2.049
14,mlp.experts.46.down_proj,0.0000063727,0.05000,2.084
14,mlp.experts.47.down_proj,0.0000032167,0.05000,2.073
14,mlp.experts.48.down_proj,0.0000000000,0.05000,2.021
14,mlp.experts.49.down_proj,0.0000909533,0.05000,2.005
14,mlp.experts.50.down_proj,0.0000236284,0.05000,2.054
14,mlp.experts.51.down_proj,0.0000119573,0.05000,2.009
14,mlp.experts.52.down_proj,0.0000140922,0.05000,1.950
14,mlp.experts.53.down_proj,0.0000136740,0.05000,1.971
14,mlp.experts.54.down_proj,0.0000123663,0.05000,2.048
14,mlp.experts.55.down_proj,0.0000018793,0.05000,2.015
14,mlp.experts.56.down_proj,0.0000062896,0.05000,2.021
14,mlp.experts.58.down_proj,0.0000020997,0.05000,2.043
14,mlp.experts.57.down_proj,0.0000169064,0.05000,2.126
14,mlp.experts.59.down_proj,0.0000373025,0.05000,2.103
14,mlp.experts.60.down_proj,0.0000303450,0.05000,1.988
14,mlp.experts.61.down_proj,0.0000655096,0.05000,2.047
14,mlp.experts.62.down_proj,0.0000508876,0.05000,1.982
14,mlp.experts.63.down_proj,0.0000025279,0.05000,0.846
15,self_attn.qkv_proj,0.0001718323,0.05000,0.942
15,self_attn.o_proj,0.0000741788,0.05000,0.942
15,mlp.gate.wg,0.0001521649,0.05000,0.914
15,mlp.shared_mlp.gate_and_up_proj,0.0015813817,0.05000,0.939
15,mlp.shared_mlp.down_proj,0.0000124514,0.05000,0.810
15,mlp.experts.2.gate_and_up_proj,0.0018669661,0.05000,2.412
15,mlp.experts.1.gate_and_up_proj,0.0012882023,0.05000,2.457
15,mlp.experts.0.gate_and_up_proj,0.0012786376,0.05000,2.459
15,mlp.experts.3.gate_and_up_proj,0.0038435575,0.05000,2.439
15,mlp.experts.5.gate_and_up_proj,0.0006749658,0.05000,2.470
15,mlp.experts.4.gate_and_up_proj,0.0010170259,0.05000,2.612
15,mlp.experts.7.gate_and_up_proj,0.0001264816,0.05000,2.555
15,mlp.experts.8.gate_and_up_proj,0.0000464312,0.05000,2.603
15,mlp.experts.6.gate_and_up_proj,0.0004064441,0.05000,2.863
15,mlp.experts.9.gate_and_up_proj,0.0000408579,0.05000,2.543
15,mlp.experts.11.gate_and_up_proj,0.0008907572,0.05000,2.552
15,mlp.experts.10.gate_and_up_proj,0.0017488949,0.05000,2.751
15,mlp.experts.12.gate_and_up_proj,0.0004321679,0.05000,2.741
15,mlp.experts.13.gate_and_up_proj,0.0002362878,0.05000,2.562
15,mlp.experts.14.gate_and_up_proj,0.0016849989,0.05000,2.586
15,mlp.experts.15.gate_and_up_proj,0.0007294257,0.05000,2.491
15,mlp.experts.16.gate_and_up_proj,0.0000000000,0.05000,2.473
15,mlp.experts.17.gate_and_up_proj,0.0003655461,0.05000,2.485
15,mlp.experts.19.gate_and_up_proj,0.0001270140,0.05000,2.370
15,mlp.experts.18.gate_and_up_proj,0.0007546528,0.05000,2.473
15,mlp.experts.20.gate_and_up_proj,0.0001889565,0.05000,2.406
15,mlp.experts.21.gate_and_up_proj,0.0011539563,0.05000,2.323
15,mlp.experts.23.gate_and_up_proj,0.0008271369,0.05000,2.362
15,mlp.experts.22.gate_and_up_proj,0.0008590171,0.05000,2.595
15,mlp.experts.24.gate_and_up_proj,0.0009165408,0.05000,2.473
15,mlp.experts.25.gate_and_up_proj,0.0028710841,0.05000,2.392
15,mlp.experts.26.gate_and_up_proj,0.0001625832,0.05000,2.417
15,mlp.experts.27.gate_and_up_proj,0.0006027341,0.05000,2.270
15,mlp.experts.28.gate_and_up_proj,0.0013944074,0.05000,2.309
15,mlp.experts.29.gate_and_up_proj,0.0004631020,0.05000,2.322
15,mlp.experts.30.gate_and_up_proj,0.0014193697,0.05000,2.377
15,mlp.experts.31.gate_and_up_proj,0.0010658990,0.05000,2.423
15,mlp.experts.32.gate_and_up_proj,0.0000000000,0.05000,2.425
15,mlp.experts.33.gate_and_up_proj,0.0019412328,0.05000,2.314
15,mlp.experts.34.gate_and_up_proj,0.0017150991,0.05000,2.370
15,mlp.experts.35.gate_and_up_proj,0.0011525697,0.05000,2.343
15,mlp.experts.36.gate_and_up_proj,0.0007310865,0.05000,2.304
15,mlp.experts.37.gate_and_up_proj,0.0007423277,0.05000,2.339
15,mlp.experts.38.gate_and_up_proj,0.0002821018,0.05000,2.349
15,mlp.experts.39.gate_and_up_proj,0.0024994744,0.05000,2.257
15,mlp.experts.40.gate_and_up_proj,0.0014133381,0.05000,2.306
15,mlp.experts.41.gate_and_up_proj,0.0025615711,0.05000,2.293
15,mlp.experts.42.gate_and_up_proj,0.0001506715,0.05000,2.312
15,mlp.experts.43.gate_and_up_proj,0.0004146010,0.05000,2.336
15,mlp.experts.44.gate_and_up_proj,0.0003612974,0.05000,2.364
15,mlp.experts.45.gate_and_up_proj,0.0013025121,0.05000,2.307
15,mlp.experts.46.gate_and_up_proj,0.0003682369,0.05000,2.373
15,mlp.experts.47.gate_and_up_proj,0.0003763745,0.05000,2.387
15,mlp.experts.48.gate_and_up_proj,0.0000640550,0.05000,2.336
15,mlp.experts.49.gate_and_up_proj,0.0002995552,0.05000,2.375
15,mlp.experts.50.gate_and_up_proj,0.0004029088,0.05000,2.350
15,mlp.experts.51.gate_and_up_proj,0.0000667910,0.05000,2.371
15,mlp.experts.52.gate_and_up_proj,0.0001488929,0.05000,2.467
15,mlp.experts.53.gate_and_up_proj,0.0025842550,0.05000,2.432
15,mlp.experts.54.gate_and_up_proj,0.0010958143,0.05000,2.394
15,mlp.experts.55.gate_and_up_proj,0.0004721767,0.05000,2.377
15,mlp.experts.56.gate_and_up_proj,0.0000000000,0.05000,2.350
15,mlp.experts.57.gate_and_up_proj,0.0010105770,0.05000,2.340
15,mlp.experts.58.gate_and_up_proj,0.0001525840,0.05000,2.382
15,mlp.experts.59.gate_and_up_proj,0.0004575757,0.05000,2.368
15,mlp.experts.60.gate_and_up_proj,0.0004537153,0.05000,2.329
15,mlp.experts.62.gate_and_up_proj,0.0007401044,0.05000,2.360
15,mlp.experts.61.gate_and_up_proj,0.0009531864,0.05000,2.381
15,mlp.experts.63.gate_and_up_proj,0.0002612641,0.05000,1.358
15,mlp.experts.0.down_proj,0.0000295559,0.05000,2.090
15,mlp.experts.1.down_proj,0.0000449271,0.05000,2.096
15,mlp.experts.2.down_proj,0.0000445077,0.05000,2.124
15,mlp.experts.4.down_proj,0.0000215302,0.05000,2.010
15,mlp.experts.3.down_proj,0.0000994695,0.05000,2.048
15,mlp.experts.5.down_proj,0.0000170398,0.05000,2.027
15,mlp.experts.6.down_proj,0.0000080144,0.05000,2.102
15,mlp.experts.7.down_proj,0.0000044862,0.05000,2.103
15,mlp.experts.8.down_proj,0.0000009165,0.05000,2.140
15,mlp.experts.10.down_proj,0.0000519362,0.05000,1.998
15,mlp.experts.11.down_proj,0.0000305843,0.05000,1.990
15,mlp.experts.9.down_proj,0.0000006075,0.05000,2.109
15,mlp.experts.12.down_proj,0.0000124445,0.05000,2.027
15,mlp.experts.13.down_proj,0.0000044983,0.05000,2.060
15,mlp.experts.14.down_proj,0.0000378408,0.05000,2.113
15,mlp.experts.15.down_proj,0.0000159994,0.05000,2.019
15,mlp.experts.16.down_proj,0.0000000000,0.05000,2.037
15,mlp.experts.17.down_proj,0.0000098150,0.05000,2.052
15,mlp.experts.18.down_proj,0.0000192899,0.05000,2.020
15,mlp.experts.19.down_proj,0.0000030277,0.05000,2.002
15,mlp.experts.20.down_proj,0.0000034477,0.05000,2.035
15,mlp.experts.21.down_proj,0.0000391961,0.05000,2.053
15,mlp.experts.22.down_proj,0.0000184075,0.05000,2.131
15,mlp.experts.23.down_proj,0.0000210100,0.05000,2.091
15,mlp.experts.24.down_proj,0.0000275316,0.05000,2.102
15,mlp.experts.25.down_proj,0.0000620071,0.05000,2.138
15,mlp.experts.26.down_proj,0.0000039011,0.05000,2.160
15,mlp.experts.27.down_proj,0.0000151807,0.05000,2.014
15,mlp.experts.28.down_proj,0.0000281054,0.05000,2.071
15,mlp.experts.29.down_proj,0.0000126668,0.05000,2.158
15,mlp.experts.30.down_proj,0.0000285643,0.05000,1.864
15,mlp.experts.31.down_proj,0.0000252325,0.05000,1.929
15,mlp.experts.32.down_proj,0.0000000000,0.05000,1.944
15,mlp.experts.33.down_proj,0.0000382293,0.05000,1.881
15,mlp.experts.34.down_proj,0.0000355399,0.05000,1.948
15,mlp.experts.35.down_proj,0.0000231749,0.05000,2.001
15,mlp.experts.36.down_proj,0.0000177420,0.05000,1.935
15,mlp.experts.37.down_proj,0.0000151975,0.05000,2.019
15,mlp.experts.38.down_proj,0.0000061859,0.05000,2.032
15,mlp.experts.39.down_proj,0.0000542070,0.05000,1.982
15,mlp.experts.40.down_proj,0.0000319497,0.05000,2.019
15,mlp.experts.41.down_proj,0.0000746559,0.05000,2.056
15,mlp.experts.42.down_proj,0.0000050992,0.05000,1.969
15,mlp.experts.43.down_proj,0.0000086190,0.05000,2.011
15,mlp.experts.44.down_proj,0.0000066598,0.05000,2.048
15,mlp.experts.45.down_proj,0.0000328010,0.05000,2.070
15,mlp.experts.46.down_proj,0.0000085187,0.05000,2.141
15,mlp.experts.47.down_proj,0.0000100688,0.05000,2.214
15,mlp.experts.48.down_proj,0.0000013235,0.05000,2.145
15,mlp.experts.49.down_proj,0.0000067329,0.05000,2.102
15,mlp.experts.50.down_proj,0.0000134156,0.05000,2.115
15,mlp.experts.51.down_proj,0.0000021312,0.05000,2.101
15,mlp.experts.52.down_proj,0.0000046706,0.05000,2.034
15,mlp.experts.53.down_proj,0.0000594839,0.05000,2.072
15,mlp.experts.54.down_proj,0.0000437504,0.05000,1.972
15,mlp.experts.55.down_proj,0.0000173268,0.05000,1.960
15,mlp.experts.56.down_proj,0.0000000000,0.05000,1.982
15,mlp.experts.57.down_proj,0.0000266936,0.05000,1.931
15,mlp.experts.58.down_proj,0.0000040567,0.05000,1.944
15,mlp.experts.59.down_proj,0.0000136500,0.05000,1.921
15,mlp.experts.60.down_proj,0.0000085541,0.05000,1.906
15,mlp.experts.61.down_proj,0.0000271266,0.05000,1.911
15,mlp.experts.62.down_proj,0.0000180402,0.05000,1.729
15,mlp.experts.63.down_proj,0.0000074317,0.05000,1.136
16,self_attn.qkv_proj,0.0001994896,0.05000,0.950
16,self_attn.o_proj,0.0000905584,0.05000,0.942
16,mlp.gate.wg,0.0002088493,0.05000,0.915
16,mlp.shared_mlp.gate_and_up_proj,0.0017210955,0.05000,0.942
16,mlp.shared_mlp.down_proj,0.0000130929,0.05000,0.701
16,mlp.experts.0.gate_and_up_proj,0.0008156672,0.05000,2.330
16,mlp.experts.2.gate_and_up_proj,0.0009455098,0.05000,2.351
16,mlp.experts.1.gate_and_up_proj,0.0022668666,0.05000,2.395
16,mlp.experts.3.gate_and_up_proj,0.0012285445,0.05000,2.280
16,mlp.experts.4.gate_and_up_proj,0.0017044542,0.05000,2.311
16,mlp.experts.5.gate_and_up_proj,0.0003692364,0.05000,2.305
16,mlp.experts.6.gate_and_up_proj,0.0013001651,0.05000,2.348
16,mlp.experts.7.gate_and_up_proj,0.0005611926,0.05000,2.373
16,mlp.experts.8.gate_and_up_proj,0.0002236904,0.05000,2.386
16,mlp.experts.9.gate_and_up_proj,0.0000565074,0.05000,2.416
16,mlp.experts.10.gate_and_up_proj,0.0000648386,0.05000,2.474
16,mlp.experts.11.gate_and_up_proj,0.0008382057,0.05000,2.494
16,mlp.experts.12.gate_and_up_proj,0.0024213820,0.05000,2.418
16,mlp.experts.13.gate_and_up_proj,0.0002448086,0.05000,2.413
16,mlp.experts.14.gate_and_up_proj,0.0010230369,0.05000,2.472
16,mlp.experts.15.gate_and_up_proj,0.0001059533,0.05000,2.284
16,mlp.experts.16.gate_and_up_proj,0.0013957669,0.05000,2.310
16,mlp.experts.17.gate_and_up_proj,0.0003179434,0.05000,2.315
16,mlp.experts.18.gate_and_up_proj,0.0041406571,0.05000,2.245
16,mlp.experts.19.gate_and_up_proj,0.0016304313,0.05000,2.269
16,mlp.experts.20.gate_and_up_proj,0.0003709218,0.05000,2.269
16,mlp.experts.21.gate_and_up_proj,0.0010241558,0.05000,2.278
16,mlp.experts.22.gate_and_up_proj,0.0002113502,0.05000,2.282
16,mlp.experts.23.gate_and_up_proj,0.0023545826,0.05000,2.288
16,mlp.experts.24.gate_and_up_proj,0.0012116641,0.05000,2.342
16,mlp.experts.25.gate_and_up_proj,0.0008146450,0.05000,2.375
16,mlp.experts.26.gate_and_up_proj,0.0000569239,0.05000,2.366
16,mlp.experts.27.gate_and_up_proj,0.0008861498,0.05000,2.374
16,mlp.experts.28.gate_and_up_proj,0.0015621371,0.05000,2.366
16,mlp.experts.29.gate_and_up_proj,0.0001859265,0.05000,2.362
16,mlp.experts.30.gate_and_up_proj,0.0005016728,0.05000,2.405
16,mlp.experts.31.gate_and_up_proj,0.0004336531,0.05000,2.389
16,mlp.experts.32.gate_and_up_proj,0.0005801763,0.05000,2.406
16,mlp.experts.33.gate_and_up_proj,0.0010961962,0.05000,2.331
16,mlp.experts.34.gate_and_up_proj,0.0034416870,0.05000,2.327
16,mlp.experts.35.gate_and_up_proj,0.0018021114,0.05000,2.320
16,mlp.experts.36.gate_and_up_proj,0.0002457206,0.05000,2.327
16,mlp.experts.37.gate_and_up_proj,0.0003417832,0.05000,2.341
16,mlp.experts.38.gate_and_up_proj,0.0016304376,0.05000,2.349
16,mlp.experts.39.gate_and_up_proj,0.0014978795,0.05000,2.371
16,mlp.experts.40.gate_and_up_proj,0.0013805114,0.05000,2.419
16,mlp.experts.41.gate_and_up_proj,0.0006695312,0.05000,2.423
16,mlp.experts.42.gate_and_up_proj,0.0002941849,0.05000,2.403
16,mlp.experts.43.gate_and_up_proj,0.0004225806,0.05000,2.404
16,mlp.experts.44.gate_and_up_proj,0.0016343952,0.05000,2.421
16,mlp.experts.45.gate_and_up_proj,0.0012709084,0.05000,2.310
16,mlp.experts.46.gate_and_up_proj,0.0011078661,0.05000,2.344
16,mlp.experts.47.gate_and_up_proj,0.0013315314,0.05000,2.326
16,mlp.experts.48.gate_and_up_proj,0.0012203694,0.05000,2.297
16,mlp.experts.49.gate_and_up_proj,0.0002443741,0.05000,2.341
16,mlp.experts.50.gate_and_up_proj,0.0010332407,0.05000,2.356
16,mlp.experts.51.gate_and_up_proj,0.0000506062,0.05000,2.438
16,mlp.experts.52.gate_and_up_proj,0.0001320506,0.05000,2.436
16,mlp.experts.53.gate_and_up_proj,0.0020202769,0.05000,2.423
16,mlp.experts.54.gate_and_up_proj,0.0006533891,0.05000,2.344
16,mlp.experts.55.gate_and_up_proj,0.0003889941,0.05000,2.344
16,mlp.experts.56.gate_and_up_proj,0.0003710396,0.05000,2.339
16,mlp.experts.57.gate_and_up_proj,0.0001597187,0.05000,2.355
16,mlp.experts.58.gate_and_up_proj,0.0001938180,0.05000,2.406
16,mlp.experts.59.gate_and_up_proj,0.0000787446,0.05000,2.437
16,mlp.experts.60.gate_and_up_proj,0.0006467251,0.05000,2.380
16,mlp.experts.61.gate_and_up_proj,0.0009957540,0.05000,2.426
16,mlp.experts.62.gate_and_up_proj,0.0002499544,0.05000,2.364
16,mlp.experts.63.gate_and_up_proj,0.0004561371,0.05000,1.242
16,mlp.experts.0.down_proj,0.0000187255,0.05000,1.764
16,mlp.experts.1.down_proj,0.0000628335,0.05000,1.780
16,mlp.experts.2.down_proj,0.0000204307,0.05000,1.792
16,mlp.experts.3.down_proj,0.0000307799,0.05000,1.730
16,mlp.experts.4.down_proj,0.0000408110,0.05000,1.756
16,mlp.experts.5.down_proj,0.0000158332,0.05000,1.778
16,mlp.experts.6.down_proj,0.0000274823,0.05000,1.838
16,mlp.experts.7.down_proj,0.0000193121,0.05000,1.845
16,mlp.experts.8.down_proj,0.0000059584,0.05000,1.849
16,mlp.experts.9.down_proj,0.0000010516,0.05000,1.914
16,mlp.experts.10.down_proj,0.0000007463,0.05000,1.913
16,mlp.experts.11.down_proj,0.0000228475,0.05000,1.889
16,mlp.experts.12.down_proj,0.0000704367,0.05000,1.991
16,mlp.experts.13.down_proj,0.0000053724,0.05000,1.999
16,mlp.experts.14.down_proj,0.0000282396,0.05000,1.995
16,mlp.experts.15.down_proj,0.0000027347,0.05000,1.916
16,mlp.experts.17.down_proj,0.0000084747,0.05000,1.902
16,mlp.experts.16.down_proj,0.0000327350,0.05000,1.984
16,mlp.experts.18.down_proj,0.0001408115,0.05000,1.730
16,mlp.experts.19.down_proj,0.0000462518,0.05000,1.770
16,mlp.experts.20.down_proj,0.0000078987,0.05000,1.778
16,mlp.experts.21.down_proj,0.0000241504,0.05000,1.773
16,mlp.experts.22.down_proj,0.0000050480,0.05000,1.839
16,mlp.experts.23.down_proj,0.0000665196,0.05000,1.825
16,mlp.experts.24.down_proj,0.0000338430,0.05000,1.734
16,mlp.experts.25.down_proj,0.0000251628,0.05000,1.787
16,mlp.experts.26.down_proj,0.0000008562,0.05000,1.779
16,mlp.experts.27.down_proj,0.0000190284,0.05000,1.741
16,mlp.experts.28.down_proj,0.0000349575,0.05000,1.787
16,mlp.experts.29.down_proj,0.0000032633,0.05000,1.773
16,mlp.experts.30.down_proj,0.0000115065,0.05000,1.769
16,mlp.experts.31.down_proj,0.0000131175,0.05000,1.791
16,mlp.experts.32.down_proj,0.0000091460,0.05000,1.768
16,mlp.experts.33.down_proj,0.0000323949,0.05000,1.751
16,mlp.experts.34.down_proj,0.0000780977,0.05000,1.778
16,mlp.experts.35.down_proj,0.0000377503,0.05000,1.775
16,mlp.experts.36.down_proj,0.0000062121,0.05000,1.804
16,mlp.experts.37.down_proj,0.0000076686,0.05000,1.782
16,mlp.experts.38.down_proj,0.0000415637,0.05000,1.796
16,mlp.experts.39.down_proj,0.0000403037,0.05000,1.832
16,mlp.experts.40.down_proj,0.0000277452,0.05000,1.829
16,mlp.experts.41.down_proj,0.0000261905,0.05000,1.851
16,mlp.experts.42.down_proj,0.0000072903,0.05000,1.858
16,mlp.experts.43.down_proj,0.0000104469,0.05000,1.853
16,mlp.experts.44.down_proj,0.0000339970,0.05000,1.881
16,mlp.experts.45.down_proj,0.0000306130,0.05000,1.863
16,mlp.experts.46.down_proj,0.0000266848,0.05000,1.915
16,mlp.experts.47.down_proj,0.0000384594,0.05000,1.947
16,mlp.experts.48.down_proj,0.0000323085,0.05000,1.833
16,mlp.experts.49.down_proj,0.0000065238,0.05000,1.835
16,mlp.experts.50.down_proj,0.0000301887,0.05000,1.850
16,mlp.experts.51.down_proj,0.0000011350,0.05000,1.946
16,mlp.experts.52.down_proj,0.0000031132,0.05000,1.919
16,mlp.experts.53.down_proj,0.0000423055,0.05000,1.938
16,mlp.experts.54.down_proj,0.0000240842,0.05000,1.975
16,mlp.experts.55.down_proj,0.0000105813,0.05000,1.912
16,mlp.experts.56.down_proj,0.0000099822,0.05000,1.942
16,mlp.experts.57.down_proj,0.0000049372,0.05000,1.933
16,mlp.experts.58.down_proj,0.0000039758,0.05000,1.951
16,mlp.experts.59.down_proj,0.0000021547,0.05000,1.994
16,mlp.experts.60.down_proj,0.0000168522,0.05000,2.000
16,mlp.experts.61.down_proj,0.0000234592,0.05000,2.056
16,mlp.experts.62.down_proj,0.0000056695,0.05000,1.971
16,mlp.experts.63.down_proj,0.0000122707,0.05000,0.950
17,self_attn.qkv_proj,0.0002347542,0.05000,0.948
17,self_attn.o_proj,0.0001217481,0.05000,0.941
17,mlp.gate.wg,0.0002069253,0.05000,0.916
17,mlp.shared_mlp.gate_and_up_proj,0.0019828295,0.05000,0.943
17,mlp.shared_mlp.down_proj,0.0000150038,0.05000,0.699
17,mlp.experts.2.gate_and_up_proj,0.0000000000,0.05000,2.365
17,mlp.experts.1.gate_and_up_proj,0.0005666827,0.05000,2.366
17,mlp.experts.0.gate_and_up_proj,0.0000000000,0.05000,2.414
17,mlp.experts.3.gate_and_up_proj,0.0026040057,0.05000,2.485
17,mlp.experts.4.gate_and_up_proj,0.0025439014,0.05000,2.494
17,mlp.experts.5.gate_and_up_proj,0.0005273848,0.05000,2.480
17,mlp.experts.6.gate_and_up_proj,0.0001201873,0.05000,2.311
17,mlp.experts.7.gate_and_up_proj,0.0008249778,0.05000,2.353
17,mlp.experts.8.gate_and_up_proj,0.0002254830,0.05000,2.372
17,mlp.experts.9.gate_and_up_proj,0.0029362527,0.05000,2.361
17,mlp.experts.10.gate_and_up_proj,0.0017601494,0.05000,2.348
17,mlp.experts.11.gate_and_up_proj,0.0005983572,0.05000,2.355
17,mlp.experts.12.gate_and_up_proj,0.0021177353,0.05000,2.320
17,mlp.experts.13.gate_and_up_proj,0.0015511525,0.05000,2.312
17,mlp.experts.14.gate_and_up_proj,0.0007611383,0.05000,2.334
17,mlp.experts.15.gate_and_up_proj,0.0009772885,0.05000,2.374
17,mlp.experts.16.gate_and_up_proj,0.0001947559,0.05000,2.372
17,mlp.experts.17.gate_and_up_proj,0.0050632780,0.05000,2.444
17,mlp.experts.18.gate_and_up_proj,0.0022002671,0.05000,2.342
17,mlp.experts.19.gate_and_up_proj,0.0008588292,0.05000,2.371
17,mlp.experts.20.gate_and_up_proj,0.0030633931,0.05000,2.382
17,mlp.experts.21.gate_and_up_proj,0.0033927610,0.05000,2.404
17,mlp.experts.22.gate_and_up_proj,0.0016854539,0.05000,2.404
17,mlp.experts.23.gate_and_up_proj,0.0024880913,0.05000,2.480
17,mlp.experts.24.gate_and_up_proj,0.0008399839,0.05000,2.450
17,mlp.experts.25.gate_and_up_proj,0.0035386973,0.05000,2.440
17,mlp.experts.26.gate_and_up_proj,0.0044872453,0.05000,2.444
17,mlp.experts.27.gate_and_up_proj,0.0028900672,0.05000,2.409
17,mlp.experts.28.gate_and_up_proj,0.0012484087,0.05000,2.386
17,mlp.experts.29.gate_and_up_proj,0.0000575156,0.05000,2.381
17,mlp.experts.30.gate_and_up_proj,0.0038654488,0.05000,2.394
17,mlp.experts.31.gate_and_up_proj,0.0031829830,0.05000,2.418
17,mlp.experts.32.gate_and_up_proj,0.0008650340,0.05000,2.463
17,mlp.experts.33.gate_and_up_proj,0.0035292829,0.05000,2.274
17,mlp.experts.34.gate_and_up_proj,0.0003422446,0.05000,2.275
17,mlp.experts.35.gate_and_up_proj,0.0014925489,0.05000,2.289
17,mlp.experts.36.gate_and_up_proj,0.0015468334,0.05000,2.318
17,mlp.experts.37.gate_and_up_proj,0.0023090430,0.05000,2.308
17,mlp.experts.38.gate_and_up_proj,0.0006445058,0.05000,2.355
17,mlp.experts.39.gate_and_up_proj,0.0013081434,0.05000,2.304
17,mlp.experts.40.gate_and_up_proj,0.0026046143,0.05000,2.315
17,mlp.experts.41.gate_and_up_proj,0.0009467782,0.05000,2.336
17,mlp.experts.42.gate_and_up_proj,0.0016534397,0.05000,2.319
17,mlp.experts.43.gate_and_up_proj,0.0017406018,0.05000,2.359
17,mlp.experts.44.gate_and_up_proj,0.0008898455,0.05000,2.389
17,mlp.experts.45.gate_and_up_proj,0.0005475412,0.05000,2.436
17,mlp.experts.46.gate_and_up_proj,0.0020276927,0.05000,2.450
17,mlp.experts.47.gate_and_up_proj,0.0002326481,0.05000,2.507
17,mlp.experts.48.gate_and_up_proj,0.0056357707,0.05000,2.345
17,mlp.experts.49.gate_and_up_proj,0.0004772251,0.05000,2.330
17,mlp.experts.50.gate_and_up_proj,0.0003942648,0.05000,2.359
17,mlp.experts.51.gate_and_up_proj,0.0003574089,0.05000,2.321
17,mlp.experts.52.gate_and_up_proj,0.0005023569,0.05000,2.341
17,mlp.experts.53.gate_and_up_proj,0.0018749431,0.05000,2.360
17,mlp.experts.54.gate_and_up_proj,0.0001715486,0.05000,2.430
17,mlp.experts.55.gate_and_up_proj,0.0014579752,0.05000,2.389
17,mlp.experts.56.gate_and_up_proj,0.0000815771,0.05000,2.425
17,mlp.experts.57.gate_and_up_proj,0.0017914091,0.05000,2.374
17,mlp.experts.58.gate_and_up_proj,0.0030063738,0.05000,2.428
17,mlp.experts.59.gate_and_up_proj,0.0000870045,0.05000,2.453
17,mlp.experts.60.gate_and_up_proj,0.0016801531,0.05000,2.329
17,mlp.experts.61.gate_and_up_proj,0.0003154327,0.05000,2.357
17,mlp.experts.62.gate_and_up_proj,0.0008217269,0.05000,2.140
17,mlp.experts.63.gate_and_up_proj,0.0027530240,0.05000,1.146
17,mlp.experts.1.down_proj,0.0000213224,0.05000,1.851
17,mlp.experts.0.down_proj,0.0000000000,0.05000,1.887
17,mlp.experts.2.down_proj,0.0000000000,0.05000,1.895
17,mlp.experts.3.down_proj,0.0000570142,0.05000,1.858
17,mlp.experts.4.down_proj,0.0000617442,0.05000,1.900
17,mlp.experts.5.down_proj,0.0000166312,0.05000,1.924
17,mlp.experts.6.down_proj,0.0000041038,0.05000,1.867
17,mlp.experts.7.down_proj,0.0000187554,0.05000,1.917
17,mlp.experts.8.down_proj,0.0000084032,0.05000,1.902
17,mlp.experts.9.down_proj,0.0001282072,0.05000,1.822
17,mlp.experts.10.down_proj,0.0000638489,0.05000,1.872
17,mlp.experts.11.down_proj,0.0000225990,0.05000,1.858
17,mlp.experts.12.down_proj,0.0001170046,0.05000,1.756
17,mlp.experts.14.down_proj,0.0000207644,0.05000,1.788
17,mlp.experts.13.down_proj,0.0000378846,0.05000,1.830
17,mlp.experts.15.down_proj,0.0000271762,0.05000,1.721
17,mlp.experts.16.down_proj,0.0000056990,0.05000,1.724
17,mlp.experts.17.down_proj,0.0001177105,0.05000,1.739
17,mlp.experts.18.down_proj,0.0000489025,0.05000,1.751
17,mlp.experts.19.down_proj,0.0000217771,0.05000,1.767
17,mlp.experts.20.down_proj,0.0000715381,0.05000,1.768
17,mlp.experts.21.down_proj,0.0000934189,0.05000,1.782
17,mlp.experts.22.down_proj,0.0000600477,0.05000,1.813
17,mlp.experts.23.down_proj,0.0000902840,0.05000,1.847
17,mlp.experts.24.down_proj,0.0000198461,0.05000,1.795
17,mlp.experts.25.down_proj,0.0000865483,0.05000,1.791
17,mlp.experts.26.down_proj,0.0001303752,0.05000,1.830
17,mlp.experts.27.down_proj,0.0000710669,0.05000,1.853
17,mlp.experts.28.down_proj,0.0000476120,0.05000,1.837
17,mlp.experts.29.down_proj,0.0000009850,0.05000,1.845
17,mlp.experts.30.down_proj,0.0000972084,0.05000,1.909
17,mlp.experts.31.down_proj,0.0000711713,0.05000,1.904
17,mlp.experts.32.down_proj,0.0000245559,0.05000,1.943
17,mlp.experts.33.down_proj,0.0000978176,0.05000,1.891
17,mlp.experts.34.down_proj,0.0000102754,0.05000,1.888
17,mlp.experts.35.down_proj,0.0000310218,0.05000,1.910
17,mlp.experts.36.down_proj,0.0000299097,0.05000,1.874
17,mlp.experts.37.down_proj,0.0000463205,0.05000,1.913
17,mlp.experts.38.down_proj,0.0000167228,0.05000,1.936
17,mlp.experts.39.down_proj,0.0000279076,0.05000,1.958
17,mlp.experts.40.down_proj,0.0000746813,0.05000,1.937
17,mlp.experts.41.down_proj,0.0000237732,0.05000,1.994
17,mlp.experts.42.down_proj,0.0000553050,0.05000,1.942
17,mlp.experts.43.down_proj,0.0000510792,0.05000,1.921
17,mlp.experts.44.down_proj,0.0000231764,0.05000,2.073
17,mlp.experts.45.down_proj,0.0000210748,0.05000,1.862
17,mlp.experts.46.down_proj,0.0000508136,0.05000,1.898
17,mlp.experts.47.down_proj,0.0000057098,0.05000,1.913
17,mlp.experts.48.down_proj,0.0001884088,0.05000,1.827
17,mlp.experts.49.down_proj,0.0000121853,0.05000,1.835
17,mlp.experts.50.down_proj,0.0000102243,0.05000,1.892
17,mlp.experts.51.down_proj,0.0000098073,0.05000,1.952
17,mlp.experts.52.down_proj,0.0000107660,0.05000,2.010
17,mlp.experts.53.down_proj,0.0000500164,0.05000,1.973
17,mlp.experts.54.down_proj,0.0000058402,0.05000,2.028
17,mlp.experts.55.down_proj,0.0000435028,0.05000,2.081
17,mlp.experts.56.down_proj,0.0000029668,0.05000,2.028
17,mlp.experts.57.down_proj,0.0000421190,0.05000,2.112
17,mlp.experts.58.down_proj,0.0000682893,0.05000,2.044
17,mlp.experts.59.down_proj,0.0000009900,0.05000,2.075
17,mlp.experts.60.down_proj,0.0000516686,0.05000,2.152
17,mlp.experts.61.down_proj,0.0000095934,0.05000,2.051
17,mlp.experts.62.down_proj,0.0000190909,0.05000,1.862
17,mlp.experts.63.down_proj,0.0000639456,0.05000,0.939
18,self_attn.qkv_proj,0.0002156963,0.05000,0.945
18,self_attn.o_proj,0.0000973792,0.05000,0.934
18,mlp.gate.wg,0.0002350022,0.05000,0.918
18,mlp.shared_mlp.gate_and_up_proj,0.0020655804,0.05000,0.935
18,mlp.shared_mlp.down_proj,0.0000145117,0.05000,0.702
18,mlp.experts.0.gate_and_up_proj,0.0018699934,0.05000,2.657
18,mlp.experts.1.gate_and_up_proj,0.0032295366,0.05000,2.666
18,mlp.experts.2.gate_and_up_proj,0.0006385839,0.05000,2.728
18,mlp.experts.3.gate_and_up_proj,0.0006043276,0.05000,2.360
18,mlp.experts.4.gate_and_up_proj,0.0005409194,0.05000,2.402
18,mlp.experts.5.gate_and_up_proj,0.0013508818,0.05000,2.391
18,mlp.experts.6.gate_and_up_proj,0.0027332975,0.05000,2.313
18,mlp.experts.7.gate_and_up_proj,0.0000694347,0.05000,2.347
18,mlp.experts.8.gate_and_up_proj,0.0008598546,0.05000,2.358
18,mlp.experts.9.gate_and_up_proj,0.0004601068,0.05000,2.378
18,mlp.experts.10.gate_and_up_proj,0.0000000000,0.05000,2.412
18,mlp.experts.11.gate_and_up_proj,0.0048353128,0.05000,2.433
18,mlp.experts.12.gate_and_up_proj,0.0004023768,0.05000,2.398
18,mlp.experts.13.gate_and_up_proj,0.0005886604,0.05000,2.386
18,mlp.experts.14.gate_and_up_proj,0.0038188136,0.05000,2.404
18,mlp.experts.15.gate_and_up_proj,0.0036027626,0.05000,2.310
18,mlp.experts.16.gate_and_up_proj,0.0036544970,0.05000,2.328
18,mlp.experts.17.gate_and_up_proj,0.0033196868,0.05000,2.375
18,mlp.experts.18.gate_and_up_proj,0.0007002375,0.05000,2.284
18,mlp.experts.19.gate_and_up_proj,0.0005730390,0.05000,2.316
18,mlp.experts.20.gate_and_up_proj,0.0031862484,0.05000,2.372
18,mlp.experts.21.gate_and_up_proj,0.0051564903,0.05000,2.381
18,mlp.experts.22.gate_and_up_proj,0.0009147909,0.05000,2.408
18,mlp.experts.23.gate_and_up_proj,0.0017109042,0.05000,2.427
18,mlp.experts.24.gate_and_up_proj,0.0010099453,0.05000,2.273
18,mlp.experts.25.gate_and_up_proj,0.0008213506,0.05000,2.263
18,mlp.experts.26.gate_and_up_proj,0.0002959919,0.05000,2.277
18,mlp.experts.27.gate_and_up_proj,0.0000000000,0.05000,2.328
18,mlp.experts.28.gate_and_up_proj,0.0006582266,0.05000,2.362
18,mlp.experts.29.gate_and_up_proj,0.0005979361,0.05000,2.398
18,mlp.experts.30.gate_and_up_proj,0.0032520985,0.05000,2.306
18,mlp.experts.31.gate_and_up_proj,0.0024549674,0.05000,2.313
18,mlp.experts.32.gate_and_up_proj,0.0024156782,0.05000,2.333
18,mlp.experts.33.gate_and_up_proj,0.0005206530,0.05000,2.267
18,mlp.experts.34.gate_and_up_proj,0.0011243501,0.05000,2.303
18,mlp.experts.35.gate_and_up_proj,0.0001901942,0.05000,2.319
18,mlp.experts.36.gate_and_up_proj,0.0009257636,0.05000,2.285
18,mlp.experts.37.gate_and_up_proj,0.0040333897,0.05000,2.328
18,mlp.experts.38.gate_and_up_proj,0.0006385236,0.05000,2.354
18,mlp.experts.39.gate_and_up_proj,0.0008173830,0.05000,2.341
18,mlp.experts.40.gate_and_up_proj,0.0008622863,0.05000,2.338
18,mlp.experts.41.gate_and_up_proj,0.0005495075,0.05000,2.330
18,mlp.experts.42.gate_and_up_proj,0.0015892280,0.05000,2.290
18,mlp.experts.43.gate_and_up_proj,0.0002860064,0.05000,2.300
18,mlp.experts.44.gate_and_up_proj,0.0052256925,0.05000,2.306
18,mlp.experts.45.gate_and_up_proj,0.0013316824,0.05000,2.293
18,mlp.experts.46.gate_and_up_proj,0.0024396750,0.05000,2.324
18,mlp.experts.47.gate_and_up_proj,0.0016401857,0.05000,2.344
18,mlp.experts.48.gate_and_up_proj,0.0034828001,0.05000,2.318
18,mlp.experts.49.gate_and_up_proj,0.0003536004,0.05000,2.360
18,mlp.experts.50.gate_and_up_proj,0.0000000000,0.05000,2.377
18,mlp.experts.51.gate_and_up_proj,0.0022919069,0.05000,2.317
18,mlp.experts.52.gate_and_up_proj,0.0017039347,0.05000,2.336
18,mlp.experts.53.gate_and_up_proj,0.0002306443,0.05000,2.343
18,mlp.experts.54.gate_and_up_proj,0.0035389622,0.05000,2.308
18,mlp.experts.55.gate_and_up_proj,0.0008841889,0.05000,2.354
18,mlp.experts.56.gate_and_up_proj,0.0005350368,0.05000,2.384
18,mlp.experts.57.gate_and_up_proj,0.0034550687,0.05000,2.308
18,mlp.experts.58.gate_and_up_proj,0.0005739153,0.05000,2.315
18,mlp.experts.59.gate_and_up_proj,0.0021624886,0.05000,2.312
18,mlp.experts.60.gate_and_up_proj,0.0004123806,0.05000,2.273
18,mlp.experts.61.gate_and_up_proj,0.0011808629,0.05000,2.295
18,mlp.experts.62.gate_and_up_proj,0.0002070472,0.05000,2.168
18,mlp.experts.63.gate_and_up_proj,0.0003365382,0.05000,1.314
18,mlp.experts.0.down_proj,0.0000460956,0.05000,1.823
18,mlp.experts.2.down_proj,0.0000158751,0.05000,1.848
18,mlp.experts.1.down_proj,0.0001146402,0.05000,1.861
18,mlp.experts.3.down_proj,0.0000177249,0.05000,1.981
18,mlp.experts.5.down_proj,0.0000647910,0.05000,1.953
18,mlp.experts.4.down_proj,0.0000151972,0.05000,2.009
18,mlp.experts.7.down_proj,0.0000008824,0.05000,1.862
18,mlp.experts.6.down_proj,0.0000941314,0.05000,1.929
18,mlp.experts.8.down_proj,0.0000235967,0.05000,1.926
18,mlp.experts.9.down_proj,0.0000134685,0.05000,1.968
18,mlp.experts.10.down_proj,0.0000000000,0.05000,2.010
18,mlp.experts.11.down_proj,0.0001352133,0.05000,2.050
18,mlp.experts.12.down_proj,0.0000107684,0.05000,2.209
18,mlp.experts.13.down_proj,0.0000136196,0.05000,2.268
18,mlp.experts.14.down_proj,0.0001044561,0.05000,2.225
18,mlp.experts.15.down_proj,0.0000884256,0.05000,2.260
18,mlp.experts.16.down_proj,0.0001410790,0.05000,2.273
18,mlp.experts.17.down_proj,0.0000911118,0.05000,2.236
18,mlp.experts.18.down_proj,0.0000171574,0.05000,2.222
18,mlp.experts.19.down_proj,0.0000148284,0.05000,2.190
18,mlp.experts.20.down_proj,0.0000842886,0.05000,2.187
18,mlp.experts.21.down_proj,0.0001559986,0.05000,2.259
18,mlp.experts.22.down_proj,0.0000319449,0.05000,2.276
18,mlp.experts.23.down_proj,0.0000561157,0.05000,2.283
18,mlp.experts.24.down_proj,0.0000224053,0.05000,2.331
18,mlp.experts.26.down_proj,0.0000093793,0.05000,2.255
18,mlp.experts.25.down_proj,0.0000157804,0.05000,2.296
18,mlp.experts.27.down_proj,0.0000000000,0.05000,2.192
18,mlp.experts.29.down_proj,0.0000154599,0.05000,2.177
18,mlp.experts.28.down_proj,0.0000175278,0.05000,2.239
18,mlp.experts.31.down_proj,0.0000544697,0.05000,2.087
18,mlp.experts.30.down_proj,0.0000943457,0.05000,2.209
18,mlp.experts.32.down_proj,0.0000792363,0.05000,2.078
18,mlp.experts.33.down_proj,0.0000104675,0.05000,1.934
18,mlp.experts.35.down_proj,0.0000051407,0.05000,1.999
18,mlp.experts.34.down_proj,0.0000285673,0.05000,2.073
18,mlp.experts.36.down_proj,0.0000283144,0.05000,1.919
18,mlp.experts.37.down_proj,0.0000975323,0.05000,1.927
18,mlp.experts.38.down_proj,0.0000214082,0.05000,1.925
18,mlp.experts.39.down_proj,0.0000186118,0.05000,1.919
18,mlp.experts.40.down_proj,0.0000294260,0.05000,1.965
18,mlp.experts.41.down_proj,0.0000179981,0.05000,1.981
18,mlp.experts.42.down_proj,0.0000504977,0.05000,1.874
18,mlp.experts.43.down_proj,0.0000119079,0.05000,1.882
18,mlp.experts.44.down_proj,0.0001855050,0.05000,1.912
18,mlp.experts.45.down_proj,0.0000318015,0.05000,1.940
18,mlp.experts.46.down_proj,0.0000796176,0.05000,1.908
18,mlp.experts.47.down_proj,0.0000405907,0.05000,1.902
18,mlp.experts.48.down_proj,0.0000854175,0.05000,1.884
18,mlp.experts.49.down_proj,0.0000095984,0.05000,1.846
18,mlp.experts.50.down_proj,0.0000000000,0.05000,1.828
18,mlp.experts.51.down_proj,0.0000599851,0.05000,1.769
18,mlp.experts.52.down_proj,0.0000502436,0.05000,1.810
18,mlp.experts.53.down_proj,0.0000050143,0.05000,1.781
18,mlp.experts.54.down_proj,0.0000731338,0.05000,1.819
18,mlp.experts.55.down_proj,0.0000271328,0.05000,1.820
18,mlp.experts.56.down_proj,0.0000235470,0.05000,1.793
18,mlp.experts.57.down_proj,0.0000899838,0.05000,1.782
18,mlp.experts.58.down_proj,0.0000215895,0.05000,1.780
18,mlp.experts.59.down_proj,0.0000497401,0.05000,1.790
18,mlp.experts.60.down_proj,0.0000107828,0.05000,1.899
18,mlp.experts.61.down_proj,0.0000380054,0.05000,1.929
18,mlp.experts.62.down_proj,0.0000080573,0.05000,1.900
18,mlp.experts.63.down_proj,0.0000143093,0.05000,0.817
19,self_attn.qkv_proj,0.0002958569,0.05000,0.947
19,self_attn.o_proj,0.0001620796,0.05000,0.936
19,mlp.gate.wg,0.0002702847,0.05000,0.915
19,mlp.shared_mlp.gate_and_up_proj,0.0021554424,0.05000,0.941
19,mlp.shared_mlp.down_proj,0.0000154709,0.05000,0.695
19,mlp.experts.0.gate_and_up_proj,0.0001974247,0.05000,2.520
19,mlp.experts.1.gate_and_up_proj,0.0017625093,0.05000,2.530
19,mlp.experts.2.gate_and_up_proj,0.0000294623,0.05000,2.573
19,mlp.experts.3.gate_and_up_proj,0.0002088703,0.05000,2.297
19,mlp.experts.5.gate_and_up_proj,0.0000405285,0.05000,2.361
19,mlp.experts.4.gate_and_up_proj,0.0006992413,0.05000,2.479
19,mlp.experts.6.gate_and_up_proj,0.0006165093,0.05000,2.432
19,mlp.experts.7.gate_and_up_proj,0.0023615605,0.05000,2.430
19,mlp.experts.8.gate_and_up_proj,0.0009750421,0.05000,2.428
19,mlp.experts.9.gate_and_up_proj,0.0003624929,0.05000,2.436
19,mlp.experts.10.gate_and_up_proj,0.0010295082,0.05000,2.472
19,mlp.experts.11.gate_and_up_proj,0.0000375636,0.05000,2.487
19,mlp.experts.12.gate_and_up_proj,0.0003549917,0.05000,2.604
19,mlp.experts.13.gate_and_up_proj,0.0006780073,0.05000,2.585
19,mlp.experts.14.gate_and_up_proj,0.0002992398,0.05000,2.572
19,mlp.experts.15.gate_and_up_proj,0.0003121146,0.05000,2.421
19,mlp.experts.16.gate_and_up_proj,0.0012705591,0.05000,2.474
19,mlp.experts.17.gate_and_up_proj,0.0000918438,0.05000,2.433
19,mlp.experts.18.gate_and_up_proj,0.0007330064,0.05000,2.287
19,mlp.experts.19.gate_and_up_proj,0.0016791143,0.05000,2.283
19,mlp.experts.20.gate_and_up_proj,0.0001942673,0.05000,2.294
19,mlp.experts.21.gate_and_up_proj,0.0002021988,0.05000,2.326
19,mlp.experts.22.gate_and_up_proj,0.0005499804,0.05000,2.357
19,mlp.experts.23.gate_and_up_proj,0.0005271939,0.05000,2.387
19,mlp.experts.24.gate_and_up_proj,0.0011459656,0.05000,2.340
19,mlp.experts.25.gate_and_up_proj,0.0004553685,0.05000,2.378
19,mlp.experts.26.gate_and_up_proj,0.0002722395,0.05000,2.359
19,mlp.experts.27.gate_and_up_proj,0.0001382571,0.05000,2.326
19,mlp.experts.28.gate_and_up_proj,0.0009697525,0.05000,2.385
19,mlp.experts.29.gate_and_up_proj,0.0004409787,0.05000,2.366
19,mlp.experts.30.gate_and_up_proj,0.0011826112,0.05000,2.340
19,mlp.experts.31.gate_and_up_proj,0.0009234806,0.05000,2.388
19,mlp.experts.32.gate_and_up_proj,0.0003637864,0.05000,2.417
19,mlp.experts.33.gate_and_up_proj,0.0009896990,0.05000,2.293
19,mlp.experts.34.gate_and_up_proj,0.0000000000,0.05000,2.349
19,mlp.experts.35.gate_and_up_proj,0.0002791072,0.05000,2.332
19,mlp.experts.36.gate_and_up_proj,0.0000000000,0.05000,2.498
19,mlp.experts.37.gate_and_up_proj,0.0001462519,0.05000,2.499
19,mlp.experts.38.gate_and_up_proj,0.0000380860,0.05000,2.484
19,mlp.experts.39.gate_and_up_proj,0.0009077857,0.05000,2.379
19,mlp.experts.40.gate_and_up_proj,0.0016910288,0.05000,2.307
19,mlp.experts.41.gate_and_up_proj,0.0012603516,0.05000,2.304
19,mlp.experts.42.gate_and_up_proj,0.0004328465,0.05000,2.394
19,mlp.experts.43.gate_and_up_proj,0.0001499577,0.05000,2.470
19,mlp.experts.44.gate_and_up_proj,0.0000755947,0.05000,2.458
19,mlp.experts.45.gate_and_up_proj,0.0029550807,0.05000,2.298
19,mlp.experts.46.gate_and_up_proj,0.0043825079,0.05000,2.349
19,mlp.experts.47.gate_and_up_proj,0.0000551650,0.05000,2.346
19,mlp.experts.48.gate_and_up_proj,0.0014561717,0.05000,2.463
19,mlp.experts.49.gate_and_up_proj,0.0003647699,0.05000,2.476
19,mlp.experts.50.gate_and_up_proj,0.0005210945,0.05000,2.475
19,mlp.experts.51.gate_and_up_proj,0.0000546165,0.05000,2.392
19,mlp.experts.52.gate_and_up_proj,0.0011624663,0.05000,2.395
19,mlp.experts.53.gate_and_up_proj,0.0010349996,0.05000,2.389
19,mlp.experts.54.gate_and_up_proj,0.0002702701,0.05000,2.317
19,mlp.experts.56.gate_and_up_proj,0.0021166691,0.05000,2.372
19,mlp.experts.55.gate_and_up_proj,0.0008876597,0.05000,2.428
19,mlp.experts.57.gate_and_up_proj,0.0000000000,0.05000,2.398
19,mlp.experts.58.gate_and_up_proj,0.0011375390,0.05000,2.403
19,mlp.experts.59.gate_and_up_proj,0.0002954962,0.05000,2.472
19,mlp.experts.60.gate_and_up_proj,0.0000000000,0.05000,2.420
19,mlp.experts.61.gate_and_up_proj,0.0004079298,0.05000,2.414
19,mlp.experts.62.gate_and_up_proj,0.0009015962,0.05000,2.391
19,mlp.experts.63.gate_and_up_proj,0.0001220120,0.05000,1.303
19,mlp.experts.0.down_proj,0.0000079466,0.05000,1.782
19,mlp.experts.1.down_proj,0.0000561612,0.05000,1.819
19,mlp.experts.2.down_proj,0.0000004641,0.05000,1.895
19,mlp.experts.3.down_proj,0.0000054612,0.05000,1.742
19,mlp.experts.4.down_proj,0.0000182688,0.05000,1.780
19,mlp.experts.5.down_proj,0.0000010664,0.05000,1.769
19,mlp.experts.6.down_proj,0.0000170174,0.05000,1.690
19,mlp.experts.7.down_proj,0.0000647302,0.05000,1.726
19,mlp.experts.8.down_proj,0.0000244187,0.05000,1.708
19,mlp.experts.9.down_proj,0.0000072069,0.05000,1.798
19,mlp.experts.10.down_proj,0.0000259802,0.05000,1.852
19,mlp.experts.11.down_proj,0.0000003390,0.05000,1.841
19,mlp.experts.12.down_proj,0.0000221860,0.05000,1.866
19,mlp.experts.13.down_proj,0.0000224774,0.05000,1.900
19,mlp.experts.14.down_proj,0.0000087078,0.05000,1.869
19,mlp.experts.15.down_proj,0.0000081347,0.05000,1.873
19,mlp.experts.17.down_proj,0.0000040673,0.05000,1.897
19,mlp.experts.16.down_proj,0.0000353857,0.05000,1.982
19,mlp.experts.18.down_proj,0.0000226527,0.05000,2.095
19,mlp.experts.19.down_proj,0.0000502172,0.05000,1.951
19,mlp.experts.20.down_proj,0.0000056531,0.05000,1.959
19,mlp.experts.21.down_proj,0.0000078658,0.05000,2.045
19,mlp.experts.22.down_proj,0.0000153729,0.05000,1.995
19,mlp.experts.23.down_proj,0.0000116685,0.05000,2.019
19,mlp.experts.24.down_proj,0.0000444241,0.05000,2.117
19,mlp.experts.25.down_proj,0.0000100198,0.05000,2.093
19,mlp.experts.26.down_proj,0.0000080841,0.05000,2.124
19,mlp.experts.28.down_proj,0.0000265070,0.05000,2.033
19,mlp.experts.27.down_proj,0.0000051518,0.05000,2.108
19,mlp.experts.29.down_proj,0.0000138388,0.05000,2.077
19,mlp.experts.30.down_proj,0.0000281044,0.05000,2.115
19,mlp.experts.31.down_proj,0.0000339493,0.05000,2.093
19,mlp.experts.32.down_proj,0.0000133091,0.05000,2.141
19,mlp.experts.33.down_proj,0.0000335150,0.05000,1.994
19,mlp.experts.34.down_proj,0.0000000000,0.05000,1.976
19,mlp.experts.35.down_proj,0.0000095504,0.05000,2.000
19,mlp.experts.36.down_proj,0.0000000000,0.05000,1.794
19,mlp.experts.37.down_proj,0.0000034017,0.05000,1.824
19,mlp.experts.38.down_proj,0.0000010932,0.05000,1.865
19,mlp.experts.39.down_proj,0.0000188260,0.05000,1.857
19,mlp.experts.40.down_proj,0.0000436089,0.05000,1.855
19,mlp.experts.41.down_proj,0.0000571544,0.05000,1.894
19,mlp.experts.42.down_proj,0.0000111750,0.05000,1.858
19,mlp.experts.43.down_proj,0.0000037263,0.05000,1.904
19,mlp.experts.44.down_proj,0.0000016500,0.05000,1.945
19,mlp.experts.45.down_proj,0.0000806475,0.05000,1.976
19,mlp.experts.46.down_proj,0.0001710004,0.05000,1.958
19,mlp.experts.47.down_proj,0.0000019241,0.05000,1.994
19,mlp.experts.48.down_proj,0.0000413402,0.05000,1.967
19,mlp.experts.49.down_proj,0.0000119073,0.05000,1.968
19,mlp.experts.50.down_proj,0.0000132773,0.05000,1.982
19,mlp.experts.51.down_proj,0.0000020620,0.05000,1.990
19,mlp.experts.52.down_proj,0.0000297007,0.05000,2.013
19,mlp.experts.53.down_proj,0.0000304240,0.05000,2.065
19,mlp.experts.54.down_proj,0.0000094328,0.05000,1.998
19,mlp.experts.55.down_proj,0.0000276861,0.05000,2.024
19,mlp.experts.56.down_proj,0.0000433730,0.05000,2.014
19,mlp.experts.57.down_proj,0.0000000000,0.05000,1.895
19,mlp.experts.58.down_proj,0.0000611859,0.05000,1.939
19,mlp.experts.59.down_proj,0.0000094846,0.05000,1.940
19,mlp.experts.60.down_proj,0.0000000000,0.05000,1.927
19,mlp.experts.61.down_proj,0.0000098512,0.05000,1.989
19,mlp.experts.62.down_proj,0.0000177983,0.05000,1.804
19,mlp.experts.63.down_proj,0.0000051651,0.05000,0.928
20,self_attn.qkv_proj,0.0002630927,0.05000,0.948
20,self_attn.o_proj,0.0001599194,0.05000,0.941
20,mlp.gate.wg,0.0002965637,0.05000,0.909
20,mlp.shared_mlp.gate_and_up_proj,0.0024753873,0.05000,0.936
20,mlp.shared_mlp.down_proj,0.0000195342,0.05000,0.707
20,mlp.experts.0.gate_and_up_proj,0.0002107774,0.05000,2.607
20,mlp.experts.1.gate_and_up_proj,0.0018273022,0.05000,2.609
20,mlp.experts.2.gate_and_up_proj,0.0002618846,0.05000,2.646
20,mlp.experts.3.gate_and_up_proj,0.0022699374,0.05000,2.396
20,mlp.experts.4.gate_and_up_proj,0.0003814900,0.05000,2.429
20,mlp.experts.5.gate_and_up_proj,0.0049254951,0.05000,2.437
20,mlp.experts.6.gate_and_up_proj,0.0004395008,0.05000,2.412
20,mlp.experts.7.gate_and_up_proj,0.0011327702,0.05000,2.398
20,mlp.experts.8.gate_and_up_proj,0.0007672835,0.05000,2.387
20,mlp.experts.9.gate_and_up_proj,0.0002395231,0.05000,2.331
20,mlp.experts.10.gate_and_up_proj,0.0000000000,0.05000,2.365
20,mlp.experts.11.gate_and_up_proj,0.0015441354,0.05000,2.380
20,mlp.experts.12.gate_and_up_proj,0.0025059817,0.05000,2.424
20,mlp.experts.13.gate_and_up_proj,0.0008302722,0.05000,2.452
20,mlp.experts.14.gate_and_up_proj,0.0007938261,0.05000,2.434
20,mlp.experts.15.gate_and_up_proj,0.0011090117,0.05000,2.324
20,mlp.experts.16.gate_and_up_proj,0.0012352381,0.05000,2.326
20,mlp.experts.17.gate_and_up_proj,0.0000615648,0.05000,2.315
20,mlp.experts.18.gate_and_up_proj,0.0008432422,0.05000,2.340
20,mlp.experts.19.gate_and_up_proj,0.0011025826,0.05000,2.392
20,mlp.experts.20.gate_and_up_proj,0.0010670186,0.05000,2.392
20,mlp.experts.21.gate_and_up_proj,0.0011150004,0.05000,2.395
20,mlp.experts.22.gate_and_up_proj,0.0015067912,0.05000,2.426
20,mlp.experts.23.gate_and_up_proj,0.0019195002,0.05000,2.438
20,mlp.experts.24.gate_and_up_proj,0.0026746137,0.05000,2.332
20,mlp.experts.25.gate_and_up_proj,0.0017467989,0.05000,2.366
20,mlp.experts.26.gate_and_up_proj,0.0038726609,0.05000,2.363
20,mlp.experts.27.gate_and_up_proj,0.0000000000,0.05000,2.346
20,mlp.experts.28.gate_and_up_proj,0.0011131938,0.05000,2.425
20,mlp.experts.29.gate_and_up_proj,0.0003041763,0.05000,2.415
20,mlp.experts.30.gate_and_up_proj,0.0002316974,0.05000,2.402
20,mlp.experts.32.gate_and_up_proj,0.0003221512,0.05000,2.405
20,mlp.experts.31.gate_and_up_proj,0.0016607099,0.05000,2.495
20,mlp.experts.33.gate_and_up_proj,0.0000000000,0.05000,2.642
20,mlp.experts.34.gate_and_up_proj,0.0010819220,0.05000,2.524
20,mlp.experts.35.gate_and_up_proj,0.0038295573,0.05000,2.529
20,mlp.experts.36.gate_and_up_proj,0.0006954106,0.05000,2.367
20,mlp.experts.37.gate_and_up_proj,0.0011229646,0.05000,2.406
20,mlp.experts.38.gate_and_up_proj,0.0008506535,0.05000,2.404
20,mlp.experts.39.gate_and_up_proj,0.0017541413,0.05000,2.378
20,mlp.experts.40.gate_and_up_proj,0.0002372422,0.05000,2.402
20,mlp.experts.41.gate_and_up_proj,0.0003797045,0.05000,2.385
20,mlp.experts.42.gate_and_up_proj,0.0006248397,0.05000,2.362
20,mlp.experts.43.gate_and_up_proj,0.0000000000,0.05000,2.402
20,mlp.experts.44.gate_and_up_proj,0.0020638018,0.05000,2.453
20,mlp.experts.45.gate_and_up_proj,0.0002826415,0.05000,2.344
20,mlp.experts.46.gate_and_up_proj,0.0007323027,0.05000,2.389
20,mlp.experts.47.gate_and_up_proj,0.0001865941,0.05000,2.399
20,mlp.experts.48.gate_and_up_proj,0.0018480027,0.05000,2.463
20,mlp.experts.49.gate_and_up_proj,0.0000722150,0.05000,2.483
20,mlp.experts.50.gate_and_up_proj,0.0012920837,0.05000,2.506
20,mlp.experts.51.gate_and_up_proj,0.0001254884,0.05000,2.540
20,mlp.experts.52.gate_and_up_proj,0.0005826270,0.05000,2.557
20,mlp.experts.53.gate_and_up_proj,0.0005451938,0.05000,2.571
20,mlp.experts.54.gate_and_up_proj,0.0010492501,0.05000,2.617
20,mlp.experts.55.gate_and_up_proj,0.0008054360,0.05000,2.595
20,mlp.experts.56.gate_and_up_proj,0.0009757953,0.05000,2.607
20,mlp.experts.57.gate_and_up_proj,0.0004533680,0.05000,2.556
20,mlp.experts.58.gate_and_up_proj,0.0001388595,0.05000,2.574
20,mlp.experts.59.gate_and_up_proj,0.0000000000,0.05000,2.571
20,mlp.experts.60.gate_and_up_proj,0.0001205988,0.05000,2.383
20,mlp.experts.61.gate_and_up_proj,0.0016294048,0.05000,2.377
20,mlp.experts.62.gate_and_up_proj,0.0003902758,0.05000,2.354
20,mlp.experts.63.gate_and_up_proj,0.0000870508,0.05000,1.205
20,mlp.experts.0.down_proj,0.0000100730,0.05000,2.045
20,mlp.experts.1.down_proj,0.0000824897,0.05000,2.047
20,mlp.experts.2.down_proj,0.0000114125,0.05000,2.060
20,mlp.experts.5.down_proj,0.0001479975,0.05000,1.939
20,mlp.experts.4.down_proj,0.0000193983,0.05000,1.972
20,mlp.experts.3.down_proj,0.0000810907,0.05000,1.989
20,mlp.experts.6.down_proj,0.0000157113,0.05000,1.887
20,mlp.experts.7.down_proj,0.0000392272,0.05000,1.980
20,mlp.experts.8.down_proj,0.0000347802,0.05000,1.983
20,mlp.experts.9.down_proj,0.0000097243,0.05000,1.792
20,mlp.experts.10.down_proj,0.0000000000,0.05000,1.797
20,mlp.experts.11.down_proj,0.0000424847,0.05000,1.813
20,mlp.experts.12.down_proj,0.0001303566,0.05000,1.796
20,mlp.experts.13.down_proj,0.0000284270,0.05000,1.819
20,mlp.experts.14.down_proj,0.0000332942,0.05000,1.820
20,mlp.experts.15.down_proj,0.0000375252,0.05000,1.794
20,mlp.experts.16.down_proj,0.0000299737,0.05000,1.831
20,mlp.experts.17.down_proj,0.0000015834,0.05000,1.811
20,mlp.experts.18.down_proj,0.0000271751,0.05000,1.844
20,mlp.experts.19.down_proj,0.0000350605,0.05000,1.836
20,mlp.experts.20.down_proj,0.0000459012,0.05000,1.832
20,mlp.experts.21.down_proj,0.0000365035,0.05000,1.872
20,mlp.experts.22.down_proj,0.0000610798,0.05000,1.865
20,mlp.experts.23.down_proj,0.0000619437,0.05000,1.863
20,mlp.experts.24.down_proj,0.0001149511,0.05000,1.909
20,mlp.experts.25.down_proj,0.0000562732,0.05000,1.882
20,mlp.experts.26.down_proj,0.0001100376,0.05000,1.917
20,mlp.experts.27.down_proj,0.0000000000,0.05000,1.832
20,mlp.experts.28.down_proj,0.0000285396,0.05000,1.792
20,mlp.experts.29.down_proj,0.0000119982,0.05000,1.810
20,mlp.experts.30.down_proj,0.0000084367,0.05000,1.750
20,mlp.experts.31.down_proj,0.0000481831,0.05000,1.763
20,mlp.experts.32.down_proj,0.0000140981,0.05000,1.783
20,mlp.experts.33.down_proj,0.0000000000,0.05000,1.782
20,mlp.experts.34.down_proj,0.0000461353,0.05000,1.805
20,mlp.experts.35.down_proj,0.0001197239,0.05000,1.762
20,mlp.experts.36.down_proj,0.0000208951,0.05000,1.794
20,mlp.experts.37.down_proj,0.0000384988,0.05000,1.837
20,mlp.experts.38.down_proj,0.0000193118,0.05000,1.810
20,mlp.experts.39.down_proj,0.0000783483,0.05000,1.803
20,mlp.experts.40.down_proj,0.0000103486,0.05000,1.793
20,mlp.experts.41.down_proj,0.0000161498,0.05000,1.790
20,mlp.experts.42.down_proj,0.0000247371,0.05000,1.811
20,mlp.experts.43.down_proj,0.0000000000,0.05000,1.795
20,mlp.experts.44.down_proj,0.0000675469,0.05000,1.805
20,mlp.experts.45.down_proj,0.0000067604,0.05000,1.868
20,mlp.experts.46.down_proj,0.0000281294,0.05000,1.829
20,mlp.experts.47.down_proj,0.0000078109,0.05000,1.833
20,mlp.experts.48.down_proj,0.0000503994,0.05000,1.886
20,mlp.experts.49.down_proj,0.0000021543,0.05000,1.871
20,mlp.experts.50.down_proj,0.0000386236,0.05000,1.882
20,mlp.experts.51.down_proj,0.0000063380,0.05000,1.806
20,mlp.experts.52.down_proj,0.0000255327,0.05000,1.803
20,mlp.experts.53.down_proj,0.0000150078,0.05000,1.818
20,mlp.experts.54.down_proj,0.0000361879,0.05000,1.743
20,mlp.experts.55.down_proj,0.0000306046,0.05000,1.761
20,mlp.experts.56.down_proj,0.0000385452,0.05000,1.767
20,mlp.experts.57.down_proj,0.0000153712,0.05000,1.815
20,mlp.experts.58.down_proj,0.0000040477,0.05000,1.813
20,mlp.experts.59.down_proj,0.0000000000,0.05000,1.825
20,mlp.experts.60.down_proj,0.0000051488,0.05000,1.807
20,mlp.experts.61.down_proj,0.0000422726,0.05000,1.788
20,mlp.experts.62.down_proj,0.0000148385,0.05000,1.779
20,mlp.experts.63.down_proj,0.0000040634,0.05000,0.770
21,self_attn.qkv_proj,0.0003397182,0.05000,0.951
21,self_attn.o_proj,0.0001392894,0.05000,0.947
21,mlp.gate.wg,0.0003122866,0.05000,0.920
21,mlp.shared_mlp.gate_and_up_proj,0.0026162649,0.05000,0.951
21,mlp.shared_mlp.down_proj,0.0000215860,0.05000,0.701
21,mlp.experts.0.gate_and_up_proj,0.0011336483,0.05000,2.451
21,mlp.experts.2.gate_and_up_proj,0.0006547542,0.05000,2.463
21,mlp.experts.1.gate_and_up_proj,0.0005186678,0.05000,2.531
21,mlp.experts.3.gate_and_up_proj,0.0001835598,0.05000,2.471
21,mlp.experts.4.gate_and_up_proj,0.0016305729,0.05000,2.476
21,mlp.experts.5.gate_and_up_proj,0.0005682438,0.05000,2.444
21,mlp.experts.6.gate_and_up_proj,0.0006708072,0.05000,2.400
21,mlp.experts.7.gate_and_up_proj,0.0002310676,0.05000,2.391
21,mlp.experts.8.gate_and_up_proj,0.0001622348,0.05000,2.404
21,mlp.experts.9.gate_and_up_proj,0.0012324121,0.05000,2.394
21,mlp.experts.10.gate_and_up_proj,0.0004973143,0.05000,2.439
21,mlp.experts.11.gate_and_up_proj,0.0047982620,0.05000,2.423
21,mlp.experts.12.gate_and_up_proj,0.0014538754,0.05000,2.403
21,mlp.experts.13.gate_and_up_proj,0.0007978164,0.05000,2.418
21,mlp.experts.14.gate_and_up_proj,0.0023327505,0.05000,2.415
21,mlp.experts.15.gate_and_up_proj,0.0009887369,0.05000,2.426
21,mlp.experts.16.gate_and_up_proj,0.0005840996,0.05000,2.446
21,mlp.experts.17.gate_and_up_proj,0.0005064691,0.05000,2.435
21,mlp.experts.18.gate_and_up_proj,0.0000889130,0.05000,2.319
21,mlp.experts.19.gate_and_up_proj,0.0009086558,0.05000,2.367
21,mlp.experts.20.gate_and_up_proj,0.0015517047,0.05000,2.353
21,mlp.experts.21.gate_and_up_proj,0.0009368463,0.05000,2.502
21,mlp.experts.22.gate_and_up_proj,0.0003169700,0.05000,2.471
21,mlp.experts.23.gate_and_up_proj,0.0005464950,0.05000,2.474
21,mlp.experts.24.gate_and_up_proj,0.0005156358,0.05000,2.521
21,mlp.experts.25.gate_and_up_proj,0.0047188268,0.05000,2.543
21,mlp.experts.26.gate_and_up_proj,0.0000556681,0.05000,2.558
21,mlp.experts.27.gate_and_up_proj,0.0002910625,0.05000,2.453
21,mlp.experts.28.gate_and_up_proj,0.0009509087,0.05000,2.434
21,mlp.experts.29.gate_and_up_proj,0.0008384725,0.05000,2.438
21,mlp.experts.30.gate_and_up_proj,0.0005896384,0.05000,2.448
21,mlp.experts.31.gate_and_up_proj,0.0021642171,0.05000,2.392
21,mlp.experts.32.gate_and_up_proj,0.0007548356,0.05000,2.397
21,mlp.experts.33.gate_and_up_proj,0.0007933583,0.05000,2.254
21,mlp.experts.34.gate_and_up_proj,0.0000420437,0.05000,2.266
21,mlp.experts.35.gate_and_up_proj,0.0011465356,0.05000,2.289
21,mlp.experts.36.gate_and_up_proj,0.0003054959,0.05000,2.383
21,mlp.experts.37.gate_and_up_proj,0.0001602992,0.05000,2.396
21,mlp.experts.38.gate_and_up_proj,0.0008047551,0.05000,2.401
21,mlp.experts.39.gate_and_up_proj,0.0038031682,0.05000,2.456
21,mlp.experts.40.gate_and_up_proj,0.0011505721,0.05000,2.438
21,mlp.experts.41.gate_and_up_proj,0.0001865717,0.05000,2.440
21,mlp.experts.42.gate_and_up_proj,0.0012952047,0.05000,2.309
21,mlp.experts.43.gate_and_up_proj,0.0002641381,0.05000,2.318
21,mlp.experts.44.gate_and_up_proj,0.0013862640,0.05000,2.331
21,mlp.experts.45.gate_and_up_proj,0.0000733306,0.05000,2.337
21,mlp.experts.46.gate_and_up_proj,0.0003372354,0.05000,2.320
21,mlp.experts.47.gate_and_up_proj,0.0003040431,0.05000,2.329
21,mlp.experts.48.gate_and_up_proj,0.0006757573,0.05000,2.385
21,mlp.experts.49.gate_and_up_proj,0.0023203786,0.05000,2.399
21,mlp.experts.50.gate_and_up_proj,0.0003655330,0.05000,2.399
21,mlp.experts.51.gate_and_up_proj,0.0023668217,0.05000,2.468
21,mlp.experts.52.gate_and_up_proj,0.0008745384,0.05000,2.431
21,mlp.experts.53.gate_and_up_proj,0.0004693126,0.05000,2.406
21,mlp.experts.54.gate_and_up_proj,0.0005019003,0.05000,2.420
21,mlp.experts.55.gate_and_up_proj,0.0028516530,0.05000,2.417
21,mlp.experts.56.gate_and_up_proj,0.0004127673,0.05000,2.402
21,mlp.experts.57.gate_and_up_proj,0.0001078674,0.05000,2.498
21,mlp.experts.58.gate_and_up_proj,0.0006468746,0.05000,2.579
21,mlp.experts.59.gate_and_up_proj,0.0028694944,0.05000,2.578
21,mlp.experts.60.gate_and_up_proj,0.0000979700,0.05000,2.478
21,mlp.experts.61.gate_and_up_proj,0.0003554552,0.05000,2.578
21,mlp.experts.62.gate_and_up_proj,0.0011568768,0.05000,2.566
21,mlp.experts.63.gate_and_up_proj,0.0000000000,0.05000,1.082
21,mlp.experts.0.down_proj,0.0000470243,0.05000,1.769
21,mlp.experts.2.down_proj,0.0000181356,0.05000,1.773
21,mlp.experts.1.down_proj,0.0000207059,0.05000,1.825
21,mlp.experts.3.down_proj,0.0000072884,0.05000,1.814
21,mlp.experts.4.down_proj,0.0000561982,0.05000,1.816
21,mlp.experts.5.down_proj,0.0000232811,0.05000,1.789
21,mlp.experts.6.down_proj,0.0000354896,0.05000,1.730
21,mlp.experts.7.down_proj,0.0000075502,0.05000,1.751
21,mlp.experts.8.down_proj,0.0000041627,0.05000,1.774
21,mlp.experts.9.down_proj,0.0000574280,0.05000,1.752
21,mlp.experts.10.down_proj,0.0000179899,0.05000,1.751
21,mlp.experts.11.down_proj,0.0001416957,0.05000,1.787
21,mlp.experts.12.down_proj,0.0000619188,0.05000,1.798
21,mlp.experts.13.down_proj,0.0000299530,0.05000,1.803
21,mlp.experts.14.down_proj,0.0000833479,0.05000,1.788
21,mlp.experts.15.down_proj,0.0000306294,0.05000,1.828
21,mlp.experts.16.down_proj,0.0000283078,0.05000,1.888
21,mlp.experts.17.down_proj,0.0000218044,0.05000,1.892
21,mlp.experts.18.down_proj,0.0000041620,0.05000,1.818
21,mlp.experts.19.down_proj,0.0000360726,0.05000,1.791
21,mlp.experts.20.down_proj,0.0000504469,0.05000,1.824
21,mlp.experts.21.down_proj,0.0000399660,0.05000,1.828
21,mlp.experts.22.down_proj,0.0000113461,0.05000,1.803
21,mlp.experts.23.down_proj,0.0000191348,0.05000,1.832
21,mlp.experts.25.down_proj,0.0001445734,0.05000,1.860
21,mlp.experts.24.down_proj,0.0000147822,0.05000,1.928
21,mlp.experts.26.down_proj,0.0000018153,0.05000,1.924
21,mlp.experts.27.down_proj,0.0000127604,0.05000,1.900
21,mlp.experts.28.down_proj,0.0000338900,0.05000,1.989
21,mlp.experts.29.down_proj,0.0000233910,0.05000,1.987
21,mlp.experts.30.down_proj,0.0000167034,0.05000,1.885
21,mlp.experts.31.down_proj,0.0000709987,0.05000,1.933
21,mlp.experts.32.down_proj,0.0000231959,0.05000,1.935
21,mlp.experts.33.down_proj,0.0000245788,0.05000,1.970
21,mlp.experts.34.down_proj,0.0000008906,0.05000,1.992
21,mlp.experts.35.down_proj,0.0000526484,0.05000,1.959
21,mlp.experts.36.down_proj,0.0000139174,0.05000,1.971
21,mlp.experts.37.down_proj,0.0000043970,0.05000,2.028
21,mlp.experts.38.down_proj,0.0000301378,0.05000,1.966
21,mlp.experts.39.down_proj,0.0001131004,0.05000,1.941
21,mlp.experts.41.down_proj,0.0000083296,0.05000,1.950
21,mlp.experts.40.down_proj,0.0000384498,0.05000,2.062
21,mlp.experts.42.down_proj,0.0000388453,0.05000,1.945
21,mlp.experts.43.down_proj,0.0000125145,0.05000,1.929
21,mlp.experts.44.down_proj,0.0000474377,0.05000,1.946
21,mlp.experts.45.down_proj,0.0000026366,0.05000,1.873
21,mlp.experts.46.down_proj,0.0000117527,0.05000,1.895
21,mlp.experts.47.down_proj,0.0000176364,0.05000,1.874
21,mlp.experts.48.down_proj,0.0000193019,0.05000,1.870
21,mlp.experts.49.down_proj,0.0000789139,0.05000,1.899
21,mlp.experts.50.down_proj,0.0000122502,0.05000,1.899
21,mlp.experts.51.down_proj,0.0000793861,0.05000,1.860
21,mlp.experts.52.down_proj,0.0000335337,0.05000,1.888
21,mlp.experts.53.down_proj,0.0000164829,0.05000,1.899
21,mlp.experts.54.down_proj,0.0000168829,0.05000,1.901
21,mlp.experts.55.down_proj,0.0000819046,0.05000,1.935
21,mlp.experts.56.down_proj,0.0000137027,0.05000,1.952
21,mlp.experts.57.down_proj,0.0000040826,0.05000,1.937
21,mlp.experts.58.down_proj,0.0000223028,0.05000,1.905
21,mlp.experts.59.down_proj,0.0000849881,0.05000,1.902
21,mlp.experts.60.down_proj,0.0000033822,0.05000,1.865
21,mlp.experts.61.down_proj,0.0000129169,0.05000,1.838
21,mlp.experts.62.down_proj,0.0000624021,0.05000,1.819
21,mlp.experts.63.down_proj,0.0000000000,0.05000,0.943
22,self_attn.qkv_proj,0.0003164143,0.05000,0.944
22,self_attn.o_proj,0.0001571469,0.05000,0.931
22,mlp.gate.wg,0.0002774460,0.05000,0.911
22,mlp.shared_mlp.gate_and_up_proj,0.0030229390,0.05000,0.938
22,mlp.shared_mlp.down_proj,0.0000227828,0.05000,0.697
22,mlp.experts.0.gate_and_up_proj,0.0007763853,0.05000,2.445
22,mlp.experts.1.gate_and_up_proj,0.0000528846,0.05000,2.458
22,mlp.experts.2.gate_and_up_proj,0.0007794119,0.05000,2.490
22,mlp.experts.3.gate_and_up_proj,0.0003071410,0.05000,2.280
22,mlp.experts.4.gate_and_up_proj,0.0003107414,0.05000,2.371
22,mlp.experts.5.gate_and_up_proj,0.0001872564,0.05000,2.359
22,mlp.experts.6.gate_and_up_proj,0.0002885021,0.05000,2.298
22,mlp.experts.7.gate_and_up_proj,0.0003815353,0.05000,2.371
22,mlp.experts.8.gate_and_up_proj,0.0005084157,0.05000,2.362
22,mlp.experts.9.gate_and_up_proj,0.0006235185,0.05000,2.319
22,mlp.experts.10.gate_and_up_proj,0.0008659247,0.05000,2.433
22,mlp.experts.11.gate_and_up_proj,0.0011160007,0.05000,2.427
22,mlp.experts.12.gate_and_up_proj,0.0011463077,0.05000,2.338
22,mlp.experts.13.gate_and_up_proj,0.0000000000,0.05000,2.391
22,mlp.experts.14.gate_and_up_proj,0.0012683127,0.05000,2.404
22,mlp.experts.15.gate_and_up_proj,0.0008496035,0.05000,2.407
22,mlp.experts.16.gate_and_up_proj,0.0001132668,0.05000,2.371
22,mlp.experts.17.gate_and_up_proj,0.0012195560,0.05000,2.383
22,mlp.experts.18.gate_and_up_proj,0.0000626066,0.05000,2.356
22,mlp.experts.19.gate_and_up_proj,0.0008781412,0.05000,2.312
22,mlp.experts.20.gate_and_up_proj,0.0001708000,0.05000,2.335
22,mlp.experts.21.gate_and_up_proj,0.0004910130,0.05000,2.379
22,mlp.experts.22.gate_and_up_proj,0.0003961192,0.05000,2.301
22,mlp.experts.23.gate_and_up_proj,0.0017042737,0.05000,2.324
22,mlp.experts.24.gate_and_up_proj,0.0002441741,0.05000,2.325
22,mlp.experts.25.gate_and_up_proj,0.0001899652,0.05000,2.363
22,mlp.experts.26.gate_and_up_proj,0.0001752150,0.05000,2.362
22,mlp.experts.27.gate_and_up_proj,0.0001293565,0.05000,2.263
22,mlp.experts.28.gate_and_up_proj,0.0000000000,0.05000,2.305
22,mlp.experts.29.gate_and_up_proj,0.0026966839,0.05000,2.317
22,mlp.experts.30.gate_and_up_proj,0.0019369527,0.05000,2.262
22,mlp.experts.31.gate_and_up_proj,0.0056213279,0.05000,2.329
22,mlp.experts.32.gate_and_up_proj,0.0012303286,0.05000,2.326
22,mlp.experts.33.gate_and_up_proj,0.0042271821,0.05000,2.326
22,mlp.experts.34.gate_and_up_proj,0.0011072524,0.05000,2.303
22,mlp.experts.35.gate_and_up_proj,0.0007598845,0.05000,2.336
22,mlp.experts.36.gate_and_up_proj,0.0019422286,0.05000,2.331
22,mlp.experts.37.gate_and_up_proj,0.0000975399,0.05000,2.337
22,mlp.experts.38.gate_and_up_proj,0.0001591803,0.05000,2.422
22,mlp.experts.39.gate_and_up_proj,0.0009329102,0.05000,2.363
22,mlp.experts.40.gate_and_up_proj,0.0000535510,0.05000,2.362
22,mlp.experts.41.gate_and_up_proj,0.0049499476,0.05000,2.364
22,mlp.experts.42.gate_and_up_proj,0.0005994740,0.05000,2.299
22,mlp.experts.43.gate_and_up_proj,0.0011482080,0.05000,2.345
22,mlp.experts.44.gate_and_up_proj,0.0008005268,0.05000,2.410
22,mlp.experts.45.gate_and_up_proj,0.0009087006,0.05000,2.475
22,mlp.experts.46.gate_and_up_proj,0.0011860296,0.05000,2.429
22,mlp.experts.47.gate_and_up_proj,0.0000534018,0.05000,2.431
22,mlp.experts.48.gate_and_up_proj,0.0004373004,0.05000,2.341
22,mlp.experts.49.gate_and_up_proj,0.0000000000,0.05000,2.319
22,mlp.experts.50.gate_and_up_proj,0.0022750581,0.05000,2.325
22,mlp.experts.51.gate_and_up_proj,0.0011597689,0.05000,2.299
22,mlp.experts.52.gate_and_up_proj,0.0051297593,0.05000,2.293
22,mlp.experts.53.gate_and_up_proj,0.0001173313,0.05000,2.311
22,mlp.experts.54.gate_and_up_proj,0.0005303500,0.05000,2.314
22,mlp.experts.55.gate_and_up_proj,0.0000394948,0.05000,2.371
22,mlp.experts.56.gate_and_up_proj,0.0007291957,0.05000,2.390
22,mlp.experts.57.gate_and_up_proj,0.0001901486,0.05000,2.273
22,mlp.experts.58.gate_and_up_proj,0.0028075589,0.05000,2.292
22,mlp.experts.59.gate_and_up_proj,0.0015658921,0.05000,2.299
22,mlp.experts.60.gate_and_up_proj,0.0002111601,0.05000,2.448
22,mlp.experts.61.gate_and_up_proj,0.0006212089,0.05000,2.476
22,mlp.experts.62.gate_and_up_proj,0.0022009140,0.05000,2.359
22,mlp.experts.63.gate_and_up_proj,0.0024528944,0.05000,1.257
22,mlp.experts.0.down_proj,0.0000292299,0.05000,1.767
22,mlp.experts.1.down_proj,0.0000014821,0.05000,1.788
22,mlp.experts.2.down_proj,0.0000266586,0.05000,1.802
22,mlp.experts.3.down_proj,0.0000097295,0.05000,1.846
22,mlp.experts.4.down_proj,0.0000132921,0.05000,1.869
22,mlp.experts.5.down_proj,0.0000078541,0.05000,1.889
22,mlp.experts.6.down_proj,0.0000112614,0.05000,1.928
22,mlp.experts.7.down_proj,0.0000183291,0.05000,1.959
22,mlp.experts.8.down_proj,0.0000160569,0.05000,1.959
22,mlp.experts.9.down_proj,0.0000240714,0.05000,1.921
22,mlp.experts.10.down_proj,0.0000387761,0.05000,1.946
22,mlp.experts.11.down_proj,0.0000383791,0.05000,1.950
22,mlp.experts.12.down_proj,0.0000404190,0.05000,2.044
22,mlp.experts.13.down_proj,0.0000000000,0.05000,1.990
22,mlp.experts.14.down_proj,0.0000459387,0.05000,2.043
22,mlp.experts.15.down_proj,0.0000377658,0.05000,2.102
22,mlp.experts.16.down_proj,0.0000039802,0.05000,2.103
22,mlp.experts.17.down_proj,0.0000335127,0.05000,2.145
22,mlp.experts.18.down_proj,0.0000013804,0.05000,2.124
22,mlp.experts.19.down_proj,0.0000381129,0.05000,2.181
22,mlp.experts.20.down_proj,0.0000044441,0.05000,2.120
22,mlp.experts.21.down_proj,0.0000230455,0.05000,2.117
22,mlp.experts.22.down_proj,0.0000164245,0.05000,2.106
22,mlp.experts.23.down_proj,0.0000524576,0.05000,2.128
22,mlp.experts.24.down_proj,0.0000144159,0.05000,2.110
22,mlp.experts.25.down_proj,0.0000076800,0.05000,2.114
22,mlp.experts.26.down_proj,0.0000097961,0.05000,2.120
22,mlp.experts.27.down_proj,0.0000051739,0.05000,2.011
22,mlp.experts.28.down_proj,0.0000000000,0.05000,1.975
22,mlp.experts.29.down_proj,0.0001723459,0.05000,2.023
22,mlp.experts.30.down_proj,0.0000660770,0.05000,2.008
22,mlp.experts.31.down_proj,0.0002317127,0.05000,2.004
22,mlp.experts.32.down_proj,0.0000406046,0.05000,2.031
22,mlp.experts.33.down_proj,0.0001230521,0.05000,1.887
22,mlp.experts.34.down_proj,0.0000505037,0.05000,1.893
22,mlp.experts.35.down_proj,0.0000221075,0.05000,1.945
22,mlp.experts.36.down_proj,0.0000681315,0.05000,2.015
22,mlp.experts.37.down_proj,0.0000046786,0.05000,1.967
22,mlp.experts.38.down_proj,0.0000032171,0.05000,2.013
22,mlp.experts.40.down_proj,0.0000021596,0.05000,1.977
22,mlp.experts.39.down_proj,0.0000334978,0.05000,2.015
22,mlp.experts.41.down_proj,0.0001705986,0.05000,2.059
22,mlp.experts.42.down_proj,0.0000168235,0.05000,2.029
22,mlp.experts.43.down_proj,0.0000373238,0.05000,2.030
22,mlp.experts.44.down_proj,0.0000282002,0.05000,2.075
22,mlp.experts.45.down_proj,0.0000329900,0.05000,2.116
22,mlp.experts.46.down_proj,0.0000402715,0.05000,2.093
22,mlp.experts.47.down_proj,0.0000012624,0.05000,2.088
22,mlp.experts.49.down_proj,0.0000000000,0.05000,1.976
22,mlp.experts.48.down_proj,0.0000181955,0.05000,2.081
22,mlp.experts.50.down_proj,0.0000825230,0.05000,2.076
22,mlp.experts.51.down_proj,0.0000451943,0.05000,1.986
22,mlp.experts.52.down_proj,0.0001781030,0.05000,1.974
22,mlp.experts.53.down_proj,0.0000038620,0.05000,2.050
22,mlp.experts.54.down_proj,0.0000148976,0.05000,2.002
22,mlp.experts.55.down_proj,0.0000008025,0.05000,1.958
22,mlp.experts.56.down_proj,0.0000271396,0.05000,2.010
22,mlp.experts.58.down_proj,0.0000987747,0.05000,1.902
22,mlp.experts.57.down_proj,0.0000045669,0.05000,1.971
22,mlp.experts.59.down_proj,0.0000493731,0.05000,2.054
22,mlp.experts.60.down_proj,0.0000068546,0.05000,1.975
22,mlp.experts.61.down_proj,0.0000218315,0.05000,2.017
22,mlp.experts.62.down_proj,0.0000641675,0.05000,1.783
22,mlp.experts.63.down_proj,0.0000834630,0.05000,0.880
23,self_attn.qkv_proj,0.0004121403,0.05000,0.951
23,self_attn.o_proj,0.0002102360,0.05000,0.941
23,mlp.gate.wg,0.0002609166,0.05000,0.911
23,mlp.shared_mlp.gate_and_up_proj,0.0035788214,0.05000,0.945
23,mlp.shared_mlp.down_proj,0.0000261631,0.05000,0.696
23,mlp.experts.0.gate_and_up_proj,0.0023462599,0.05000,2.391
23,mlp.experts.2.gate_and_up_proj,0.0008180052,0.05000,2.414
23,mlp.experts.1.gate_and_up_proj,0.0072014900,0.05000,2.472
23,mlp.experts.3.gate_and_up_proj,0.0042285428,0.05000,2.541
23,mlp.experts.4.gate_and_up_proj,0.0006734584,0.05000,2.573
23,mlp.experts.5.gate_and_up_proj,0.0050268753,0.05000,2.577
23,mlp.experts.6.gate_and_up_proj,0.0028843024,0.05000,2.275
23,mlp.experts.7.gate_and_up_proj,0.0005919799,0.05000,2.346
23,mlp.experts.8.gate_and_up_proj,0.0056763954,0.05000,2.304
23,mlp.experts.9.gate_and_up_proj,0.0026284550,0.05000,2.310
23,mlp.experts.10.gate_and_up_proj,0.0007859659,0.05000,2.351
23,mlp.experts.11.gate_and_up_proj,0.0001283827,0.05000,2.353
23,mlp.experts.12.gate_and_up_proj,0.0009831139,0.05000,2.305
23,mlp.experts.13.gate_and_up_proj,0.0045183102,0.05000,2.347
23,mlp.experts.14.gate_and_up_proj,0.0016187383,0.05000,2.381
23,mlp.experts.15.gate_and_up_proj,0.0008443060,0.05000,2.388
23,mlp.experts.16.gate_and_up_proj,0.0019766281,0.05000,2.401
23,mlp.experts.17.gate_and_up_proj,0.0016266825,0.05000,2.434
23,mlp.experts.18.gate_and_up_proj,0.0024583431,0.05000,2.254
23,mlp.experts.19.gate_and_up_proj,0.0000000000,0.05000,2.280
23,mlp.experts.20.gate_and_up_proj,0.0010083176,0.05000,2.295
23,mlp.experts.21.gate_and_up_proj,0.0000754753,0.05000,2.294
23,mlp.experts.22.gate_and_up_proj,0.0002803202,0.05000,2.363
23,mlp.experts.23.gate_and_up_proj,0.0045129704,0.05000,2.330
23,mlp.experts.24.gate_and_up_proj,0.0002825824,0.05000,2.264
23,mlp.experts.25.gate_and_up_proj,0.0006226216,0.05000,2.273
23,mlp.experts.26.gate_and_up_proj,0.0042121250,0.05000,2.287
23,mlp.experts.27.gate_and_up_proj,0.0007393374,0.05000,2.333
23,mlp.experts.28.gate_and_up_proj,0.0005845128,0.05000,2.354
23,mlp.experts.29.gate_and_up_proj,0.0043509655,0.05000,2.348
23,mlp.experts.30.gate_and_up_proj,0.0048141833,0.05000,2.310
23,mlp.experts.31.gate_and_up_proj,0.0060601891,0.05000,2.339
23,mlp.experts.32.gate_and_up_proj,0.0053016085,0.05000,2.343
23,mlp.experts.33.gate_and_up_proj,0.0002682839,0.05000,2.350
23,mlp.experts.34.gate_and_up_proj,0.0003332628,0.05000,2.434
23,mlp.experts.35.gate_and_up_proj,0.0004265709,0.05000,2.464
23,mlp.experts.36.gate_and_up_proj,0.0016433064,0.05000,2.278
23,mlp.experts.37.gate_and_up_proj,0.0014915139,0.05000,2.279
23,mlp.experts.38.gate_and_up_proj,0.0000000000,0.05000,2.278
23,mlp.experts.39.gate_and_up_proj,0.0060018908,0.05000,2.305
23,mlp.experts.40.gate_and_up_proj,0.0010714390,0.05000,2.353
23,mlp.experts.41.gate_and_up_proj,0.0005288684,0.05000,2.357
23,mlp.experts.42.gate_and_up_proj,0.0010211880,0.05000,2.301
23,mlp.experts.43.gate_and_up_proj,0.0052084823,0.05000,2.345
23,mlp.experts.44.gate_and_up_proj,0.0010370541,0.05000,2.353
23,mlp.experts.45.gate_and_up_proj,0.0000000000,0.05000,2.365
23,mlp.experts.46.gate_and_up_proj,0.0038216958,0.05000,2.366
23,mlp.experts.47.gate_and_up_proj,0.0005064969,0.05000,2.406
23,mlp.experts.48.gate_and_up_proj,0.0006214858,0.05000,2.317
23,mlp.experts.49.gate_and_up_proj,0.0026172979,0.05000,2.347
23,mlp.experts.50.gate_and_up_proj,0.0002052354,0.05000,2.367
23,mlp.experts.51.gate_and_up_proj,0.0004356425,0.05000,2.303
23,mlp.experts.52.gate_and_up_proj,0.0030645736,0.05000,2.306
23,mlp.experts.53.gate_and_up_proj,0.0030471658,0.05000,2.316
23,mlp.experts.54.gate_and_up_proj,0.0009933620,0.05000,2.305
23,mlp.experts.55.gate_and_up_proj,0.0017161732,0.05000,2.300
23,mlp.experts.56.gate_and_up_proj,0.0005670963,0.05000,2.291
23,mlp.experts.57.gate_and_up_proj,0.0006170877,0.05000,2.309
23,mlp.experts.58.gate_and_up_proj,0.0000000000,0.05000,2.316
23,mlp.experts.59.gate_and_up_proj,0.0026578702,0.05000,2.304
23,mlp.experts.60.gate_and_up_proj,0.0002743079,0.05000,2.264
23,mlp.experts.61.gate_and_up_proj,0.0008341822,0.05000,2.281
23,mlp.experts.62.gate_and_up_proj,0.0001650572,0.05000,2.228
23,mlp.experts.63.gate_and_up_proj,0.0008536169,0.05000,1.334
23,mlp.experts.0.down_proj,0.0000881054,0.05000,1.756
23,mlp.experts.2.down_proj,0.0000289469,0.05000,1.767
23,mlp.experts.1.down_proj,0.0003039110,0.05000,1.779
23,mlp.experts.3.down_proj,0.0001573143,0.05000,1.767
23,mlp.experts.4.down_proj,0.0000236668,0.05000,1.775
23,mlp.experts.5.down_proj,0.0001922391,0.05000,1.781
23,mlp.experts.6.down_proj,0.0001330935,0.05000,1.748
23,mlp.experts.7.down_proj,0.0000308898,0.05000,1.749
23,mlp.experts.8.down_proj,0.0002921313,0.05000,1.764
23,mlp.experts.9.down_proj,0.0000958593,0.05000,1.783
23,mlp.experts.10.down_proj,0.0000325412,0.05000,1.787
23,mlp.experts.11.down_proj,0.0000041830,0.05000,1.783
23,mlp.experts.12.down_proj,0.0000471860,0.05000,1.826
23,mlp.experts.13.down_proj,0.0001183377,0.05000,1.808
23,mlp.experts.14.down_proj,0.0000684327,0.05000,1.825
23,mlp.experts.15.down_proj,0.0000438108,0.05000,1.829
23,mlp.experts.16.down_proj,0.0000992164,0.05000,1.857
23,mlp.experts.17.down_proj,0.0000597471,0.05000,1.825
23,mlp.experts.18.down_proj,0.0000797137,0.05000,1.825
23,mlp.experts.19.down_proj,0.0000000000,0.05000,1.806
23,mlp.experts.20.down_proj,0.0000333991,0.05000,1.805
23,mlp.experts.21.down_proj,0.0000026588,0.05000,1.931
23,mlp.experts.22.down_proj,0.0000111978,0.05000,1.936
23,mlp.experts.23.down_proj,0.0001544453,0.05000,1.961
23,mlp.experts.24.down_proj,0.0000083349,0.05000,1.941
23,mlp.experts.25.down_proj,0.0000229690,0.05000,1.938
23,mlp.experts.26.down_proj,0.0001673011,0.05000,1.935
23,mlp.experts.27.down_proj,0.0000311725,0.05000,1.896
23,mlp.experts.28.down_proj,0.0000189885,0.05000,1.887
23,mlp.experts.29.down_proj,0.0001561339,0.05000,1.896
23,mlp.experts.31.down_proj,0.0002074997,0.05000,1.821
23,mlp.experts.30.down_proj,0.0001613018,0.05000,1.868
23,mlp.experts.32.down_proj,0.0001924599,0.05000,1.827
23,mlp.experts.33.down_proj,0.0000113565,0.05000,1.881
23,mlp.experts.34.down_proj,0.0000110057,0.05000,1.916
23,mlp.experts.35.down_proj,0.0000136344,0.05000,1.922
23,mlp.experts.36.down_proj,0.0000421882,0.05000,1.942
23,mlp.experts.37.down_proj,0.0000418717,0.05000,1.936
23,mlp.experts.38.down_proj,0.0000000000,0.05000,1.939
23,mlp.experts.39.down_proj,0.0001758685,0.05000,2.057
23,mlp.experts.40.down_proj,0.0000527451,0.05000,1.994
23,mlp.experts.41.down_proj,0.0000198223,0.05000,1.991
23,mlp.experts.43.down_proj,0.0001841501,0.05000,1.975
23,mlp.experts.42.down_proj,0.0000304391,0.05000,2.007
23,mlp.experts.44.down_proj,0.0000380371,0.05000,1.997
23,mlp.experts.45.down_proj,0.0000000000,0.05000,2.006
23,mlp.experts.46.down_proj,0.0001125904,0.05000,2.002
23,mlp.experts.47.down_proj,0.0000122201,0.05000,2.065
23,mlp.experts.49.down_proj,0.0001077158,0.05000,1.905
23,mlp.experts.48.down_proj,0.0000231722,0.05000,1.949
23,mlp.experts.50.down_proj,0.0000064802,0.05000,1.962
23,mlp.experts.51.down_proj,0.0000119809,0.05000,1.957
23,mlp.experts.52.down_proj,0.0001315819,0.05000,1.980
23,mlp.experts.53.down_proj,0.0001180423,0.05000,2.030
23,mlp.experts.54.down_proj,0.0000557580,0.05000,1.954
23,mlp.experts.55.down_proj,0.0000966195,0.05000,2.069
23,mlp.experts.56.down_proj,0.0000223310,0.05000,2.005
23,mlp.experts.57.down_proj,0.0000316505,0.05000,1.960
23,mlp.experts.58.down_proj,0.0000000000,0.05000,1.954
23,mlp.experts.59.down_proj,0.0000872111,0.05000,1.970
23,mlp.experts.60.down_proj,0.0000108873,0.05000,1.909
23,mlp.experts.61.down_proj,0.0000452071,0.05000,1.914
23,mlp.experts.62.down_proj,0.0000067732,0.05000,1.890
23,mlp.experts.63.down_proj,0.0000258677,0.05000,0.838
24,self_attn.qkv_proj,0.0004830592,0.05000,0.942
24,self_attn.o_proj,0.0002045739,0.05000,0.938
24,mlp.gate.wg,0.0002633689,0.05000,0.913
24,mlp.shared_mlp.gate_and_up_proj,0.0052503359,0.05000,0.945
24,mlp.shared_mlp.down_proj,0.0000338993,0.05000,0.699
24,mlp.experts.0.gate_and_up_proj,0.0012614000,0.05000,2.390
24,mlp.experts.2.gate_and_up_proj,0.0030247545,0.05000,2.401
24,mlp.experts.1.gate_and_up_proj,0.0002219189,0.05000,2.485
24,mlp.experts.3.gate_and_up_proj,0.0010247049,0.05000,2.332
24,mlp.experts.4.gate_and_up_proj,0.0056839870,0.05000,2.347
24,mlp.experts.5.gate_and_up_proj,0.0007158587,0.05000,2.306
24,mlp.experts.6.gate_and_up_proj,0.0001104248,0.05000,2.355
24,mlp.experts.7.gate_and_up_proj,0.0004417555,0.05000,2.367
24,mlp.experts.8.gate_and_up_proj,0.0037506491,0.05000,2.356
24,mlp.experts.9.gate_and_up_proj,0.0004565283,0.05000,2.288
24,mlp.experts.10.gate_and_up_proj,0.0006112552,0.05000,2.338
24,mlp.experts.11.gate_and_up_proj,0.0003434428,0.05000,2.344
24,mlp.experts.12.gate_and_up_proj,0.0021388315,0.05000,2.279
24,mlp.experts.13.gate_and_up_proj,0.0005286782,0.05000,2.329
24,mlp.experts.14.gate_and_up_proj,0.0070165167,0.05000,2.320
24,mlp.experts.15.gate_and_up_proj,0.0046193600,0.05000,2.306
24,mlp.experts.17.gate_and_up_proj,0.0025118270,0.05000,2.341
24,mlp.experts.16.gate_and_up_proj,0.0006303221,0.05000,2.403
24,mlp.experts.19.gate_and_up_proj,0.0005615460,0.05000,2.461
24,mlp.experts.20.gate_and_up_proj,0.0010097897,0.05000,2.475
24,mlp.experts.18.gate_and_up_proj,0.0000000000,0.05000,2.758
24,mlp.experts.21.gate_and_up_proj,0.0013893195,0.05000,2.344
24,mlp.experts.22.gate_and_up_proj,0.0006384163,0.05000,2.400
24,mlp.experts.23.gate_and_up_proj,0.0002047954,0.05000,2.370
24,mlp.experts.24.gate_and_up_proj,0.0036056454,0.05000,2.334
24,mlp.experts.25.gate_and_up_proj,0.0003878197,0.05000,2.370
24,mlp.experts.26.gate_and_up_proj,0.0029362833,0.05000,2.399
24,mlp.experts.27.gate_and_up_proj,0.0009236841,0.05000,2.303
24,mlp.experts.28.gate_and_up_proj,0.0004431140,0.05000,2.368
24,mlp.experts.29.gate_and_up_proj,0.0011870493,0.05000,2.361
24,mlp.experts.30.gate_and_up_proj,0.0011626453,0.05000,2.307
24,mlp.experts.31.gate_and_up_proj,0.0022802478,0.05000,2.351
24,mlp.experts.32.gate_and_up_proj,0.0004025259,0.05000,2.342
24,mlp.experts.33.gate_and_up_proj,0.0020680026,0.05000,2.359
24,mlp.experts.34.gate_and_up_proj,0.0016417650,0.05000,2.446
24,mlp.experts.35.gate_and_up_proj,0.0015295855,0.05000,2.445
24,mlp.experts.36.gate_and_up_proj,0.0004739184,0.05000,2.332
24,mlp.experts.37.gate_and_up_proj,0.0011601902,0.05000,2.393
24,mlp.experts.38.gate_and_up_proj,0.0021941761,0.05000,2.377
24,mlp.experts.39.gate_and_up_proj,0.0007791511,0.05000,2.287
24,mlp.experts.41.gate_and_up_proj,0.0009773790,0.05000,2.329
24,mlp.experts.40.gate_and_up_proj,0.0000000000,0.05000,2.402
24,mlp.experts.42.gate_and_up_proj,0.0002914884,0.05000,2.533
24,mlp.experts.43.gate_and_up_proj,0.0026176766,0.05000,2.442
24,mlp.experts.44.gate_and_up_proj,0.0028462876,0.05000,2.406
24,mlp.experts.45.gate_and_up_proj,0.0000501352,0.05000,2.352
24,mlp.experts.46.gate_and_up_proj,0.0025961211,0.05000,2.389
24,mlp.experts.47.gate_and_up_proj,0.0022623228,0.05000,2.421
24,mlp.experts.48.gate_and_up_proj,0.0006586852,0.05000,2.391
24,mlp.experts.49.gate_and_up_proj,0.0030750237,0.05000,2.416
24,mlp.experts.50.gate_and_up_proj,0.0004476917,0.05000,2.443
24,mlp.experts.51.gate_and_up_proj,0.0000000000,0.05000,2.372
24,mlp.experts.52.gate_and_up_proj,0.0009311163,0.05000,2.393
24,mlp.experts.53.gate_and_up_proj,0.0008345579,0.05000,2.379
24,mlp.experts.54.gate_and_up_proj,0.0000000000,0.05000,2.283
24,mlp.experts.55.gate_and_up_proj,0.0009752210,0.05000,2.295
24,mlp.experts.56.gate_and_up_proj,0.0013333289,0.05000,2.323
24,mlp.experts.57.gate_and_up_proj,0.0004916478,0.05000,2.262
24,mlp.experts.58.gate_and_up_proj,0.0002592907,0.05000,2.292
24,mlp.experts.59.gate_and_up_proj,0.0008904334,0.05000,2.323
24,mlp.experts.60.gate_and_up_proj,0.0016754727,0.05000,2.322
24,mlp.experts.61.gate_and_up_proj,0.0007721978,0.05000,2.378
24,mlp.experts.62.gate_and_up_proj,0.0003998702,0.05000,2.344
24,mlp.experts.63.gate_and_up_proj,0.0000000000,0.05000,1.296
24,mlp.experts.2.down_proj,0.0001421001,0.05000,1.922
24,mlp.experts.1.down_proj,0.0000085607,0.05000,1.924
24,mlp.experts.0.down_proj,0.0000381935,0.05000,1.996
24,mlp.experts.4.down_proj,0.0002225705,0.05000,1.838
24,mlp.experts.3.down_proj,0.0000368265,0.05000,1.840
24,mlp.experts.5.down_proj,0.0000295329,0.05000,1.841
24,mlp.experts.6.down_proj,0.0000053269,0.05000,1.854
24,mlp.experts.7.down_proj,0.0000169752,0.05000,1.924
24,mlp.experts.8.down_proj,0.0001505464,0.05000,1.889
24,mlp.experts.9.down_proj,0.0000208534,0.05000,1.934
24,mlp.experts.10.down_proj,0.0000290061,0.05000,1.923
24,mlp.experts.11.down_proj,0.0000116278,0.05000,1.972
24,mlp.experts.12.down_proj,0.0000686163,0.05000,2.038
24,mlp.experts.13.down_proj,0.0000220966,0.05000,2.077
24,mlp.experts.14.down_proj,0.0002670182,0.05000,2.071
24,mlp.experts.15.down_proj,0.0001532756,0.05000,2.073
24,mlp.experts.16.down_proj,0.0000324838,0.05000,2.112
24,mlp.experts.17.down_proj,0.0000897142,0.05000,2.127
24,mlp.experts.18.down_proj,0.0000000000,0.05000,2.108
24,mlp.experts.19.down_proj,0.0000293863,0.05000,2.071
24,mlp.experts.20.down_proj,0.0000382543,0.05000,2.108
24,mlp.experts.21.down_proj,0.0000651883,0.05000,2.146
24,mlp.experts.22.down_proj,0.0000254341,0.05000,2.096
24,mlp.experts.23.down_proj,0.0000075195,0.05000,2.100
24,mlp.experts.24.down_proj,0.0001437344,0.05000,1.974
24,mlp.experts.25.down_proj,0.0000144334,0.05000,1.958
24,mlp.experts.26.down_proj,0.0001297002,0.05000,1.968
24,mlp.experts.27.down_proj,0.0000520033,0.05000,1.939
24,mlp.experts.28.down_proj,0.0000163351,0.05000,1.912
24,mlp.experts.29.down_proj,0.0000448248,0.05000,1.911
24,mlp.experts.31.down_proj,0.0000905058,0.05000,1.825
24,mlp.experts.30.down_proj,0.0000393106,0.05000,1.916
24,mlp.experts.32.down_proj,0.0000141114,0.05000,1.806
24,mlp.experts.33.down_proj,0.0000680814,0.05000,1.845
24,mlp.experts.34.down_proj,0.0000854009,0.05000,1.858
24,mlp.experts.35.down_proj,0.0000743372,0.05000,1.896
24,mlp.experts.36.down_proj,0.0000178382,0.05000,1.958
24,mlp.experts.37.down_proj,0.0000564692,0.05000,1.965
24,mlp.experts.38.down_proj,0.0000826937,0.05000,2.034
24,mlp.experts.39.down_proj,0.0000387259,0.05000,2.037
24,mlp.experts.40.down_proj,0.0000000000,0.05000,2.081
24,mlp.experts.41.down_proj,0.0000335843,0.05000,2.025
24,mlp.experts.42.down_proj,0.0000096221,0.05000,1.959
24,mlp.experts.43.down_proj,0.0000999773,0.05000,1.992
24,mlp.experts.44.down_proj,0.0001246197,0.05000,1.997
24,mlp.experts.45.down_proj,0.0000015913,0.05000,2.003
24,mlp.experts.46.down_proj,0.0001131731,0.05000,1.989
24,mlp.experts.47.down_proj,0.0000917202,0.05000,1.995
24,mlp.experts.48.down_proj,0.0000296255,0.05000,1.965
24,mlp.experts.49.down_proj,0.0001015533,0.05000,1.962
24,mlp.experts.50.down_proj,0.0000123700,0.05000,1.958
24,mlp.experts.51.down_proj,0.0000000000,0.05000,2.006
24,mlp.experts.52.down_proj,0.0000276387,0.05000,1.967
24,mlp.experts.53.down_proj,0.0000377483,0.05000,1.970
24,mlp.experts.54.down_proj,0.0000000000,0.05000,1.946
24,mlp.experts.55.down_proj,0.0000587515,0.05000,1.960
24,mlp.experts.56.down_proj,0.0000495248,0.05000,1.958
24,mlp.experts.57.down_proj,0.0000159603,0.05000,1.928
24,mlp.experts.58.down_proj,0.0000119209,0.05000,1.935
24,mlp.experts.59.down_proj,0.0000280847,0.05000,1.939
24,mlp.experts.60.down_proj,0.0000584160,0.05000,1.914
24,mlp.experts.61.down_proj,0.0000219544,0.05000,1.923
24,mlp.experts.62.down_proj,0.0000123241,0.05000,1.894
24,mlp.experts.63.down_proj,0.0000000000,0.05000,0.832
25,self_attn.qkv_proj,0.0007440102,0.05000,0.949
25,self_attn.o_proj,0.0002621907,0.05000,0.936
25,mlp.gate.wg,0.0002194071,0.05000,0.910
25,mlp.shared_mlp.gate_and_up_proj,0.0082288089,0.05000,0.945
25,mlp.shared_mlp.down_proj,0.0000419154,0.05000,0.698
25,mlp.experts.0.gate_and_up_proj,0.0089589793,0.05000,2.433
25,mlp.experts.1.gate_and_up_proj,0.0038227650,0.05000,2.444
25,mlp.experts.2.gate_and_up_proj,0.0077907590,0.05000,2.543
25,mlp.experts.3.gate_and_up_proj,0.0073310124,0.05000,2.308
25,mlp.experts.4.gate_and_up_proj,0.0016895807,0.05000,2.389
25,mlp.experts.5.gate_and_up_proj,0.0031381935,0.05000,2.370
25,mlp.experts.6.gate_and_up_proj,0.0016555786,0.05000,2.312
25,mlp.experts.7.gate_and_up_proj,0.0019888516,0.05000,2.338
25,mlp.experts.8.gate_and_up_proj,0.0018931211,0.05000,2.353
25,mlp.experts.9.gate_and_up_proj,0.0032099594,0.05000,2.270
25,mlp.experts.10.gate_and_up_proj,0.0042850513,0.05000,2.296
25,mlp.experts.11.gate_and_up_proj,0.0014199576,0.05000,2.310
25,mlp.experts.12.gate_and_up_proj,0.0030000835,0.05000,2.366
25,mlp.experts.13.gate_and_up_proj,0.0078792393,0.05000,2.423
25,mlp.experts.14.gate_and_up_proj,0.0002583595,0.05000,2.448
25,mlp.experts.15.gate_and_up_proj,0.0015144351,0.05000,2.372
25,mlp.experts.16.gate_and_up_proj,0.0039098628,0.05000,2.381
25,mlp.experts.17.gate_and_up_proj,0.0038795522,0.05000,2.397
25,mlp.experts.18.gate_and_up_proj,0.0008106817,0.05000,2.381
25,mlp.experts.19.gate_and_up_proj,0.0047674576,0.05000,2.484
25,mlp.experts.20.gate_and_up_proj,0.0037408404,0.05000,2.490
25,mlp.experts.21.gate_and_up_proj,0.0121686548,0.05000,2.405
25,mlp.experts.22.gate_and_up_proj,0.0010296774,0.05000,2.438
25,mlp.experts.23.gate_and_up_proj,0.0003651264,0.05000,2.516
25,mlp.experts.24.gate_and_up_proj,0.0046205508,0.05000,2.493
25,mlp.experts.25.gate_and_up_proj,0.0053917612,0.05000,2.461
25,mlp.experts.26.gate_and_up_proj,0.0076799066,0.05000,2.410
25,mlp.experts.27.gate_and_up_proj,0.0019135299,0.05000,2.452
25,mlp.experts.28.gate_and_up_proj,0.0007045152,0.05000,2.523
25,mlp.experts.29.gate_and_up_proj,0.0022234257,0.05000,2.531
25,mlp.experts.30.gate_and_up_proj,0.0057459095,0.05000,2.338
25,mlp.experts.31.gate_and_up_proj,0.0095983609,0.05000,2.370
25,mlp.experts.32.gate_and_up_proj,0.0024711108,0.05000,2.391
25,mlp.experts.33.gate_and_up_proj,0.0008811863,0.05000,2.354
25,mlp.experts.34.gate_and_up_proj,0.0008660423,0.05000,2.390
25,mlp.experts.35.gate_and_up_proj,0.0013600545,0.05000,2.414
25,mlp.experts.36.gate_and_up_proj,0.0006621134,0.05000,2.331
25,mlp.experts.37.gate_and_up_proj,0.0057441586,0.05000,2.337
25,mlp.experts.38.gate_and_up_proj,0.0005681467,0.05000,2.335
25,mlp.experts.39.gate_and_up_proj,0.0101924712,0.05000,2.346
25,mlp.experts.40.gate_and_up_proj,0.0018995646,0.05000,2.374
25,mlp.experts.41.gate_and_up_proj,0.0119896345,0.05000,2.356
25,mlp.experts.42.gate_and_up_proj,0.0065851545,0.05000,2.291
25,mlp.experts.43.gate_and_up_proj,0.0000000000,0.05000,2.315
25,mlp.experts.44.gate_and_up_proj,0.0008416483,0.05000,2.310
25,mlp.experts.45.gate_and_up_proj,0.0035612433,0.05000,2.341
25,mlp.experts.46.gate_and_up_proj,0.0001455051,0.05000,2.465
25,mlp.experts.47.gate_and_up_proj,0.0019518924,0.05000,2.430
25,mlp.experts.48.gate_and_up_proj,0.0002205626,0.05000,2.486
25,mlp.experts.49.gate_and_up_proj,0.0033098267,0.05000,2.463
25,mlp.experts.50.gate_and_up_proj,0.0047477714,0.05000,2.422
25,mlp.experts.51.gate_and_up_proj,0.0000000000,0.05000,2.370
25,mlp.experts.52.gate_and_up_proj,0.0025062383,0.05000,2.370
25,mlp.experts.53.gate_and_up_proj,0.0048853108,0.05000,2.418
25,mlp.experts.54.gate_and_up_proj,0.0008674177,0.05000,2.361
25,mlp.experts.55.gate_and_up_proj,0.0091571654,0.05000,2.366
25,mlp.experts.56.gate_and_up_proj,0.0011477946,0.05000,2.372
25,mlp.experts.57.gate_and_up_proj,0.0002503084,0.05000,2.311
25,mlp.experts.58.gate_and_up_proj,0.0025999945,0.05000,2.377
25,mlp.experts.59.gate_and_up_proj,0.0014075485,0.05000,2.393
25,mlp.experts.60.gate_and_up_proj,0.0084448450,0.05000,2.337
25,mlp.experts.61.gate_and_up_proj,0.0024460147,0.05000,2.363
25,mlp.experts.62.gate_and_up_proj,0.0001524903,0.05000,2.303
25,mlp.experts.63.gate_and_up_proj,0.0050277306,0.05000,1.420
25,mlp.experts.2.down_proj,0.0003258351,0.05000,2.012
25,mlp.experts.0.down_proj,0.0003864281,0.05000,2.076
25,mlp.experts.1.down_proj,0.0001150532,0.05000,2.132
25,mlp.experts.3.down_proj,0.0002251573,0.05000,2.091
25,mlp.experts.4.down_proj,0.0000729743,0.05000,2.086
25,mlp.experts.5.down_proj,0.0001224839,0.05000,2.210
25,mlp.experts.6.down_proj,0.0000577122,0.05000,2.052
25,mlp.experts.7.down_proj,0.0001139942,0.05000,2.065
25,mlp.experts.8.down_proj,0.0000778445,0.05000,2.061
25,mlp.experts.9.down_proj,0.0001372570,0.05000,2.111
25,mlp.experts.10.down_proj,0.0001156415,0.05000,2.081
25,mlp.experts.11.down_proj,0.0000576878,0.05000,2.077
25,mlp.experts.13.down_proj,0.0004466855,0.05000,1.972
25,mlp.experts.12.down_proj,0.0000949253,0.05000,2.081
25,mlp.experts.14.down_proj,0.0000100403,0.05000,2.006
25,mlp.experts.15.down_proj,0.0000703938,0.05000,1.990
25,mlp.experts.16.down_proj,0.0001586903,0.05000,2.010
25,mlp.experts.17.down_proj,0.0001819724,0.05000,2.020
25,mlp.experts.18.down_proj,0.0000434372,0.05000,2.122
25,mlp.experts.19.down_proj,0.0002131300,0.05000,2.092
25,mlp.experts.20.down_proj,0.0001915701,0.05000,2.079
25,mlp.experts.21.down_proj,0.0005155988,0.05000,2.015
25,mlp.experts.22.down_proj,0.0000401063,0.05000,1.994
25,mlp.experts.23.down_proj,0.0000130424,0.05000,1.996
25,mlp.experts.24.down_proj,0.0001439538,0.05000,1.900
25,mlp.experts.25.down_proj,0.0001767129,0.05000,1.925
25,mlp.experts.26.down_proj,0.0003397168,0.05000,1.920
25,mlp.experts.27.down_proj,0.0000787619,0.05000,1.873
25,mlp.experts.28.down_proj,0.0000310054,0.05000,1.878
25,mlp.experts.29.down_proj,0.0000964556,0.05000,1.934
25,mlp.experts.30.down_proj,0.0002420114,0.05000,2.022
25,mlp.experts.31.down_proj,0.0003719919,0.05000,2.046
25,mlp.experts.32.down_proj,0.0000895149,0.05000,2.021
25,mlp.experts.33.down_proj,0.0000368302,0.05000,1.971
25,mlp.experts.34.down_proj,0.0000400696,0.05000,2.021
25,mlp.experts.35.down_proj,0.0000735025,0.05000,1.990
25,mlp.experts.36.down_proj,0.0000345181,0.05000,1.983
25,mlp.experts.37.down_proj,0.0001959447,0.05000,2.018
25,mlp.experts.38.down_proj,0.0000242283,0.05000,2.029
25,mlp.experts.39.down_proj,0.0004969059,0.05000,1.955
25,mlp.experts.40.down_proj,0.0000653624,0.05000,2.023
25,mlp.experts.41.down_proj,0.0004707858,0.05000,1.960
25,mlp.experts.42.down_proj,0.0003025359,0.05000,1.945
25,mlp.experts.43.down_proj,0.0000000000,0.05000,1.979
25,mlp.experts.44.down_proj,0.0000310647,0.05000,1.986
25,mlp.experts.45.down_proj,0.0002108614,0.05000,2.031
25,mlp.experts.46.down_proj,0.0000030474,0.05000,1.978
25,mlp.experts.47.down_proj,0.0000788132,0.05000,1.978
25,mlp.experts.48.down_proj,0.0000077621,0.05000,1.915
25,mlp.experts.49.down_proj,0.0001327477,0.05000,1.900
25,mlp.experts.50.down_proj,0.0002227326,0.05000,1.896
25,mlp.experts.51.down_proj,0.0000000000,0.05000,1.831
25,mlp.experts.52.down_proj,0.0001305447,0.05000,1.864
25,mlp.experts.53.down_proj,0.0002059580,0.05000,1.838
25,mlp.experts.54.down_proj,0.0000450773,0.05000,1.870
25,mlp.experts.55.down_proj,0.0003796624,0.05000,1.861
25,mlp.experts.56.down_proj,0.0000452287,0.05000,1.911
25,mlp.experts.57.down_proj,0.0000070497,0.05000,1.789
25,mlp.experts.58.down_proj,0.0001096159,0.05000,1.787
25,mlp.experts.59.down_proj,0.0000734882,0.05000,1.884
25,mlp.experts.60.down_proj,0.0003019583,0.05000,1.845
25,mlp.experts.61.down_proj,0.0001027185,0.05000,1.868
25,mlp.experts.62.down_proj,0.0000061436,0.05000,1.777
25,mlp.experts.63.down_proj,0.0001657193,0.05000,0.890
26,self_attn.qkv_proj,0.0007108346,0.05000,0.940
26,self_attn.o_proj,0.0003730854,0.05000,0.935
26,mlp.gate.wg,0.0001707770,0.05000,0.912
26,mlp.shared_mlp.gate_and_up_proj,0.0110801030,0.05000,0.941
26,mlp.shared_mlp.down_proj,0.0000531215,0.05000,0.699
26,mlp.experts.2.gate_and_up_proj,0.0017367663,0.05000,2.467
26,mlp.experts.1.gate_and_up_proj,0.0015875648,0.05000,2.477
26,mlp.experts.0.gate_and_up_proj,0.0044922237,0.05000,2.552
26,mlp.experts.3.gate_and_up_proj,0.0004142061,0.05000,2.358
26,mlp.experts.4.gate_and_up_proj,0.0045354142,0.05000,2.350
26,mlp.experts.5.gate_and_up_proj,0.0004106489,0.05000,2.348
26,mlp.experts.6.gate_and_up_proj,0.0014642127,0.05000,2.427
26,mlp.experts.7.gate_and_up_proj,0.0006396484,0.05000,2.444
26,mlp.experts.8.gate_and_up_proj,0.0008177393,0.05000,2.449
26,mlp.experts.9.gate_and_up_proj,0.0021067723,0.05000,2.451
26,mlp.experts.10.gate_and_up_proj,0.0009286613,0.05000,2.495
26,mlp.experts.11.gate_and_up_proj,0.0001746493,0.05000,2.474
26,mlp.experts.12.gate_and_up_proj,0.0019161072,0.05000,2.399
26,mlp.experts.13.gate_and_up_proj,0.0017767764,0.05000,2.445
26,mlp.experts.14.gate_and_up_proj,0.0013143964,0.05000,2.464
26,mlp.experts.15.gate_and_up_proj,0.0026563263,0.05000,2.314
26,mlp.experts.16.gate_and_up_proj,0.0011884548,0.05000,2.355
26,mlp.experts.17.gate_and_up_proj,0.0073327507,0.05000,2.354
26,mlp.experts.18.gate_and_up_proj,0.0026643068,0.05000,2.430
26,mlp.experts.19.gate_and_up_proj,0.0003865314,0.05000,2.451
26,mlp.experts.20.gate_and_up_proj,0.0022642267,0.05000,2.482
26,mlp.experts.21.gate_and_up_proj,0.0011364852,0.05000,2.377
26,mlp.experts.22.gate_and_up_proj,0.0025906115,0.05000,2.362
26,mlp.experts.23.gate_and_up_proj,0.0025021925,0.05000,2.332
26,mlp.experts.24.gate_and_up_proj,0.0017084563,0.05000,2.366
26,mlp.experts.25.gate_and_up_proj,0.0009134685,0.05000,2.434
26,mlp.experts.26.gate_and_up_proj,0.0013186979,0.05000,2.426
26,mlp.experts.27.gate_and_up_proj,0.0032603920,0.05000,2.360
26,mlp.experts.28.gate_and_up_proj,0.0026110675,0.05000,2.371
26,mlp.experts.29.gate_and_up_proj,0.0016900484,0.05000,2.358
26,mlp.experts.30.gate_and_up_proj,0.0003897410,0.05000,2.356
26,mlp.experts.31.gate_and_up_proj,0.0017012004,0.05000,2.372
26,mlp.experts.32.gate_and_up_proj,0.0022493032,0.05000,2.380
26,mlp.experts.33.gate_and_up_proj,0.0002741942,0.05000,2.326
26,mlp.experts.34.gate_and_up_proj,0.0028674553,0.05000,2.370
26,mlp.experts.35.gate_and_up_proj,0.0019013012,0.05000,2.356
26,mlp.experts.36.gate_and_up_proj,0.0012452936,0.05000,2.377
26,mlp.experts.37.gate_and_up_proj,0.0071542657,0.05000,2.397
26,mlp.experts.38.gate_and_up_proj,0.0058010265,0.05000,2.382
26,mlp.experts.39.gate_and_up_proj,0.0105307999,0.05000,2.386
26,mlp.experts.40.gate_and_up_proj,0.0005443502,0.05000,2.472
26,mlp.experts.41.gate_and_up_proj,0.0002509011,0.05000,2.503
26,mlp.experts.42.gate_and_up_proj,0.0039784326,0.05000,2.447
26,mlp.experts.43.gate_and_up_proj,0.0104965993,0.05000,2.455
26,mlp.experts.44.gate_and_up_proj,0.0008769159,0.05000,2.452
26,mlp.experts.45.gate_and_up_proj,0.0017264815,0.05000,2.376
26,mlp.experts.46.gate_and_up_proj,0.0000000000,0.05000,2.368
26,mlp.experts.47.gate_and_up_proj,0.0044417982,0.05000,2.396
26,mlp.experts.48.gate_and_up_proj,0.0013096929,0.05000,2.296
26,mlp.experts.49.gate_and_up_proj,0.0043527757,0.05000,2.391
26,mlp.experts.50.gate_and_up_proj,0.0046489409,0.05000,2.392
26,mlp.experts.51.gate_and_up_proj,0.0018735332,0.05000,2.300
26,mlp.experts.52.gate_and_up_proj,0.0011334908,0.05000,2.336
26,mlp.experts.53.gate_and_up_proj,0.0001799551,0.05000,2.344
26,mlp.experts.54.gate_and_up_proj,0.0009248071,0.05000,2.320
26,mlp.experts.55.gate_and_up_proj,0.0005993006,0.05000,2.407
26,mlp.experts.56.gate_and_up_proj,0.0010086708,0.05000,2.389
26,mlp.experts.57.gate_and_up_proj,0.0006139967,0.05000,2.533
26,mlp.experts.58.gate_and_up_proj,0.0006948993,0.05000,2.627
26,mlp.experts.59.gate_and_up_proj,0.0001803908,0.05000,2.611
26,mlp.experts.60.gate_and_up_proj,0.0019144492,0.05000,2.531
26,mlp.experts.61.gate_and_up_proj,0.0009199020,0.05000,2.517
26,mlp.experts.62.gate_and_up_proj,0.0050535359,0.05000,2.498
26,mlp.experts.63.gate_and_up_proj,0.0001061752,0.05000,1.386
26,mlp.experts.1.down_proj,0.0000756020,0.05000,1.955
26,mlp.experts.2.down_proj,0.0000520386,0.05000,1.961
26,mlp.experts.0.down_proj,0.0001854951,0.05000,1.983
26,mlp.experts.5.down_proj,0.0000161551,0.05000,1.982
26,mlp.experts.3.down_proj,0.0000171809,0.05000,2.055
26,mlp.experts.4.down_proj,0.0002028814,0.05000,2.053
26,mlp.experts.6.down_proj,0.0000630956,0.05000,2.135
26,mlp.experts.7.down_proj,0.0000242430,0.05000,2.187
26,mlp.experts.8.down_proj,0.0000336486,0.05000,2.271
26,mlp.experts.9.down_proj,0.0000875224,0.05000,2.113
26,mlp.experts.10.down_proj,0.0000401318,0.05000,2.194
26,mlp.experts.11.down_proj,0.0000080868,0.05000,2.165
26,mlp.experts.12.down_proj,0.0000857219,0.05000,2.047
26,mlp.experts.13.down_proj,0.0000816065,0.05000,2.174
26,mlp.experts.14.down_proj,0.0000471967,0.05000,2.116
26,mlp.experts.15.down_proj,0.0001305621,0.05000,2.054
26,mlp.experts.17.down_proj,0.0002542199,0.05000,2.090
26,mlp.experts.16.down_proj,0.0000493281,0.05000,2.133
26,mlp.experts.18.down_proj,0.0001091843,0.05000,2.362
26,mlp.experts.19.down_proj,0.0000191159,0.05000,2.286
26,mlp.experts.20.down_proj,0.0000846702,0.05000,2.303
26,mlp.experts.21.down_proj,0.0000429505,0.05000,2.186
26,mlp.experts.22.down_proj,0.0001015003,0.05000,2.339
26,mlp.experts.23.down_proj,0.0000950215,0.05000,2.411
26,mlp.experts.24.down_proj,0.0000693494,0.05000,2.080
26,mlp.experts.25.down_proj,0.0000306850,0.05000,2.240
26,mlp.experts.26.down_proj,0.0000389360,0.05000,2.177
26,mlp.experts.27.down_proj,0.0001209833,0.05000,2.121
26,mlp.experts.28.down_proj,0.0001236184,0.05000,2.130
26,mlp.experts.29.down_proj,0.0001406608,0.05000,2.143
26,mlp.experts.30.down_proj,0.0000156590,0.05000,2.147
26,mlp.experts.31.down_proj,0.0000885612,0.05000,2.127
26,mlp.experts.32.down_proj,0.0000880658,0.05000,2.130
26,mlp.experts.33.down_proj,0.0000171469,0.05000,2.137
26,mlp.experts.34.down_proj,0.0000997798,0.05000,2.104
26,mlp.experts.35.down_proj,0.0000885210,0.05000,2.062
26,mlp.experts.36.down_proj,0.0000506622,0.05000,2.017
26,mlp.experts.37.down_proj,0.0002730402,0.05000,2.028
26,mlp.experts.38.down_proj,0.0002996192,0.05000,1.998
26,mlp.experts.39.down_proj,0.0003245047,0.05000,1.979
26,mlp.experts.40.down_proj,0.0000211450,0.05000,2.057
26,mlp.experts.41.down_proj,0.0000092159,0.05000,2.062
26,mlp.experts.42.down_proj,0.0001830373,0.05000,1.898
26,mlp.experts.43.down_proj,0.0004009262,0.05000,1.884
26,mlp.experts.44.down_proj,0.0000326461,0.05000,1.876
26,mlp.experts.45.down_proj,0.0000702003,0.05000,1.782
26,mlp.experts.47.down_proj,0.0001621034,0.05000,1.796
26,mlp.experts.46.down_proj,0.0000000000,0.05000,1.827
26,mlp.experts.48.down_proj,0.0000477664,0.05000,1.849
26,mlp.experts.49.down_proj,0.0001902513,0.05000,1.766
26,mlp.experts.50.down_proj,0.0001711454,0.05000,1.762
26,mlp.experts.51.down_proj,0.0000648754,0.05000,1.822
26,mlp.experts.52.down_proj,0.0000559263,0.05000,1.816
26,mlp.experts.53.down_proj,0.0000076692,0.05000,1.816
26,mlp.experts.54.down_proj,0.0000354056,0.05000,1.856
26,mlp.experts.56.down_proj,0.0000463033,0.05000,1.870
26,mlp.experts.55.down_proj,0.0000321981,0.05000,1.890
26,mlp.experts.57.down_proj,0.0000259130,0.05000,1.875
26,mlp.experts.58.down_proj,0.0000246672,0.05000,1.922
26,mlp.experts.59.down_proj,0.0000068095,0.05000,1.937
26,mlp.experts.60.down_proj,0.0000780044,0.05000,2.022
26,mlp.experts.61.down_proj,0.0000276940,0.05000,1.925
26,mlp.experts.62.down_proj,0.0001640504,0.05000,1.914
26,mlp.experts.63.down_proj,0.0000052359,0.05000,1.043
27,self_attn.qkv_proj,0.0009811966,0.05000,0.948
27,self_attn.o_proj,0.0004089459,0.05000,0.946
27,mlp.gate.wg,0.0001528152,0.05000,0.918
27,mlp.shared_mlp.gate_and_up_proj,0.0132744590,0.05000,0.947
27,mlp.shared_mlp.down_proj,0.0000639861,0.05000,0.701
27,mlp.experts.1.gate_and_up_proj,0.0019645234,0.05000,2.529
27,mlp.experts.0.gate_and_up_proj,0.0001946715,0.05000,2.570
27,mlp.experts.2.gate_and_up_proj,0.0080545743,0.05000,2.685
27,mlp.experts.3.gate_and_up_proj,0.0000000000,0.05000,2.390
27,mlp.experts.4.gate_and_up_proj,0.0023897797,0.05000,2.444
27,mlp.experts.5.gate_and_up_proj,0.0033279667,0.05000,2.480
27,mlp.experts.6.gate_and_up_proj,0.0011100165,0.05000,2.515
27,mlp.experts.7.gate_and_up_proj,0.0006368954,0.05000,2.510
27,mlp.experts.8.gate_and_up_proj,0.0039359609,0.05000,2.521
27,mlp.experts.9.gate_and_up_proj,0.0017177836,0.05000,2.434
27,mlp.experts.10.gate_and_up_proj,0.0051632329,0.05000,2.433
27,mlp.experts.11.gate_and_up_proj,0.0017538301,0.05000,2.432
27,mlp.experts.12.gate_and_up_proj,0.0065106765,0.05000,2.360
27,mlp.experts.13.gate_and_up_proj,0.0012203495,0.05000,2.345
27,mlp.experts.14.gate_and_up_proj,0.0009658460,0.05000,2.359
27,mlp.experts.15.gate_and_up_proj,0.0034498207,0.05000,2.445
27,mlp.experts.16.gate_and_up_proj,0.0013605027,0.05000,2.482
27,mlp.experts.17.gate_and_up_proj,0.0001134079,0.05000,2.557
27,mlp.experts.18.gate_and_up_proj,0.0079062351,0.05000,2.420
27,mlp.experts.19.gate_and_up_proj,0.0011473997,0.05000,2.486
27,mlp.experts.20.gate_and_up_proj,0.0008280562,0.05000,2.470
27,mlp.experts.21.gate_and_up_proj,0.0005379352,0.05000,2.412
27,mlp.experts.22.gate_and_up_proj,0.0023665184,0.05000,2.464
27,mlp.experts.23.gate_and_up_proj,0.0000000000,0.05000,2.492
27,mlp.experts.24.gate_and_up_proj,0.0001544582,0.05000,2.430
27,mlp.experts.25.gate_and_up_proj,0.0006032670,0.05000,2.422
27,mlp.experts.26.gate_and_up_proj,0.0030922892,0.05000,2.455
27,mlp.experts.27.gate_and_up_proj,0.0037595816,0.05000,2.410
27,mlp.experts.28.gate_and_up_proj,0.0012693185,0.05000,2.419
27,mlp.experts.29.gate_and_up_proj,0.0010857996,0.05000,2.468
27,mlp.experts.30.gate_and_up_proj,0.0013317684,0.05000,2.381
27,mlp.experts.31.gate_and_up_proj,0.0006293195,0.05000,2.460
27,mlp.experts.32.gate_and_up_proj,0.0022479622,0.05000,2.466
27,mlp.experts.33.gate_and_up_proj,0.0005519822,0.05000,2.524
27,mlp.experts.34.gate_and_up_proj,0.0000000000,0.05000,2.465
27,mlp.experts.35.gate_and_up_proj,0.0027302051,0.05000,2.497
27,mlp.experts.36.gate_and_up_proj,0.0010201049,0.05000,2.416
27,mlp.experts.37.gate_and_up_proj,0.0044991227,0.05000,2.465
27,mlp.experts.38.gate_and_up_proj,0.0058954485,0.05000,2.548
27,mlp.experts.39.gate_and_up_proj,0.0000896484,0.05000,2.364
27,mlp.experts.40.gate_and_up_proj,0.0012064797,0.05000,2.388
27,mlp.experts.41.gate_and_up_proj,0.0005045280,0.05000,2.414
27,mlp.experts.42.gate_and_up_proj,0.0009259428,0.05000,2.437
27,mlp.experts.43.gate_and_up_proj,0.0088791108,0.05000,2.445
27,mlp.experts.44.gate_and_up_proj,0.0011758415,0.05000,2.408
27,mlp.experts.45.gate_and_up_proj,0.0119475285,0.05000,2.332
27,mlp.experts.46.gate_and_up_proj,0.0000000000,0.05000,2.383
27,mlp.experts.47.gate_and_up_proj,0.0018677278,0.05000,2.391
27,mlp.experts.48.gate_and_up_proj,0.0144777266,0.05000,2.405
27,mlp.experts.49.gate_and_up_proj,0.0002891570,0.05000,2.414
27,mlp.experts.50.gate_and_up_proj,0.0019917200,0.05000,2.363
27,mlp.experts.51.gate_and_up_proj,0.0000000000,0.05000,2.317
27,mlp.experts.52.gate_and_up_proj,0.0021574632,0.05000,2.357
27,mlp.experts.53.gate_and_up_proj,0.0000000000,0.05000,2.359
27,mlp.experts.54.gate_and_up_proj,0.0003895766,0.05000,2.377
27,mlp.experts.55.gate_and_up_proj,0.0000000000,0.05000,2.436
27,mlp.experts.56.gate_and_up_proj,0.0029493678,0.05000,2.459
27,mlp.experts.57.gate_and_up_proj,0.0023258124,0.05000,2.369
27,mlp.experts.58.gate_and_up_proj,0.0007584227,0.05000,2.406
27,mlp.experts.59.gate_and_up_proj,0.0037699922,0.05000,2.439
27,mlp.experts.60.gate_and_up_proj,0.0016250082,0.05000,2.499
27,mlp.experts.61.gate_and_up_proj,0.0015419904,0.05000,2.444
27,mlp.experts.62.gate_and_up_proj,0.0033522765,0.05000,2.263
27,mlp.experts.63.gate_and_up_proj,0.0010228121,0.05000,1.328
27,mlp.experts.0.down_proj,0.0000076509,0.05000,1.850
27,mlp.experts.1.down_proj,0.0001104436,0.05000,1.858
27,mlp.experts.2.down_proj,0.0003218549,0.05000,1.903
27,mlp.experts.3.down_proj,0.0000000000,0.05000,1.851
27,mlp.experts.4.down_proj,0.0001500026,0.05000,1.848
27,mlp.experts.5.down_proj,0.0001039121,0.05000,1.848
27,mlp.experts.7.down_proj,0.0000279850,0.05000,1.795
27,mlp.experts.8.down_proj,0.0001120524,0.05000,1.822
27,mlp.experts.6.down_proj,0.0000376239,0.05000,1.885
27,mlp.experts.9.down_proj,0.0000644217,0.05000,1.866
27,mlp.experts.11.down_proj,0.0000578527,0.05000,1.890
27,mlp.experts.10.down_proj,0.0002044570,0.05000,1.928
27,mlp.experts.12.down_proj,0.0002272200,0.05000,2.042
27,mlp.experts.13.down_proj,0.0000325957,0.05000,1.962
27,mlp.experts.14.down_proj,0.0000248872,0.05000,1.962
27,mlp.experts.15.down_proj,0.0001426362,0.05000,2.023
27,mlp.experts.17.down_proj,0.0000029240,0.05000,2.017
27,mlp.experts.16.down_proj,0.0000430077,0.05000,2.043
27,mlp.experts.18.down_proj,0.0002640053,0.05000,2.036
27,mlp.experts.19.down_proj,0.0000458791,0.05000,2.085
27,mlp.experts.20.down_proj,0.0000235977,0.05000,2.094
27,mlp.experts.21.down_proj,0.0000258951,0.05000,2.092
27,mlp.experts.22.down_proj,0.0000832973,0.05000,2.058
27,mlp.experts.23.down_proj,0.0000000000,0.05000,2.124
27,mlp.experts.25.down_proj,0.0000282836,0.05000,2.020
27,mlp.experts.24.down_proj,0.0000037794,0.05000,2.115
27,mlp.experts.26.down_proj,0.0000970072,0.05000,2.051
27,mlp.experts.27.down_proj,0.0001301949,0.05000,1.950
27,mlp.experts.28.down_proj,0.0000494015,0.05000,1.966
27,mlp.experts.29.down_proj,0.0000415679,0.05000,1.982
27,mlp.experts.30.down_proj,0.0000603115,0.05000,1.912
27,mlp.experts.31.down_proj,0.0000279028,0.05000,1.972
27,mlp.experts.32.down_proj,0.0000721708,0.05000,1.913
27,mlp.experts.33.down_proj,0.0000256883,0.05000,1.935
27,mlp.experts.34.down_proj,0.0000000000,0.05000,1.985
27,mlp.experts.35.down_proj,0.0000874275,0.05000,2.003
27,mlp.experts.36.down_proj,0.0000402229,0.05000,1.920
27,mlp.experts.37.down_proj,0.0001678768,0.05000,1.973
27,mlp.experts.38.down_proj,0.0001919980,0.05000,2.028
27,mlp.experts.39.down_proj,0.0000038592,0.05000,2.034
27,mlp.experts.40.down_proj,0.0000383793,0.05000,2.064
27,mlp.experts.41.down_proj,0.0000165981,0.05000,2.067
27,mlp.experts.42.down_proj,0.0000264748,0.05000,2.099
27,mlp.experts.43.down_proj,0.0002336149,0.05000,2.044
27,mlp.experts.44.down_proj,0.0000377417,0.05000,2.067
27,mlp.experts.45.down_proj,0.0004522958,0.05000,2.088
27,mlp.experts.46.down_proj,0.0000000000,0.05000,2.052
27,mlp.experts.47.down_proj,0.0000361508,0.05000,2.082
27,mlp.experts.48.down_proj,0.0006406646,0.05000,2.041
27,mlp.experts.49.down_proj,0.0000091518,0.05000,2.058
27,mlp.experts.50.down_proj,0.0000530164,0.05000,2.143
27,mlp.experts.51.down_proj,0.0000000000,0.05000,2.102
27,mlp.experts.52.down_proj,0.0000783393,0.05000,2.087
27,mlp.experts.53.down_proj,0.0000000000,0.05000,2.073
27,mlp.experts.54.down_proj,0.0000118866,0.05000,1.980
27,mlp.experts.55.down_proj,0.0000000000,0.05000,2.019
27,mlp.experts.56.down_proj,0.0001014596,0.05000,2.001
27,mlp.experts.57.down_proj,0.0000911027,0.05000,2.071
27,mlp.experts.58.down_proj,0.0000237111,0.05000,2.100
27,mlp.experts.59.down_proj,0.0001265966,0.05000,2.088
27,mlp.experts.60.down_proj,0.0000551706,0.05000,2.075
27,mlp.experts.61.down_proj,0.0000643961,0.05000,2.046
27,mlp.experts.62.down_proj,0.0000998207,0.05000,1.960
27,mlp.experts.63.down_proj,0.0000279668,0.05000,0.859
28,self_attn.qkv_proj,0.0007668782,0.05000,0.943
28,self_attn.o_proj,0.0004640478,0.05000,0.945
28,mlp.gate.wg,0.0001169508,0.05000,0.920
28,mlp.shared_mlp.gate_and_up_proj,0.0127999597,0.05000,0.944
28,mlp.shared_mlp.down_proj,0.0000794361,0.05000,0.699
28,mlp.experts.0.gate_and_up_proj,0.0186686042,0.05000,2.774
28,mlp.experts.1.gate_and_up_proj,0.0003103882,0.05000,2.785
28,mlp.experts.2.gate_and_up_proj,0.0012873227,0.05000,2.795
28,mlp.experts.3.gate_and_up_proj,0.0001523224,0.05000,2.405
28,mlp.experts.5.gate_and_up_proj,0.0080586184,0.05000,2.481
28,mlp.experts.4.gate_and_up_proj,0.0011710537,0.05000,2.525
28,mlp.experts.6.gate_and_up_proj,0.0000000000,0.05000,2.350
28,mlp.experts.7.gate_and_up_proj,0.0011837992,0.05000,2.358
28,mlp.experts.8.gate_and_up_proj,0.0004991470,0.05000,2.352
28,mlp.experts.9.gate_and_up_proj,0.0000578945,0.05000,2.307
28,mlp.experts.10.gate_and_up_proj,0.0035622015,0.05000,2.333
28,mlp.experts.11.gate_and_up_proj,0.0029637584,0.05000,2.399
28,mlp.experts.12.gate_and_up_proj,0.0015046590,0.05000,2.348
28,mlp.experts.13.gate_and_up_proj,0.0004369193,0.05000,2.370
28,mlp.experts.14.gate_and_up_proj,0.0034511835,0.05000,2.371
28,mlp.experts.15.gate_and_up_proj,0.0016536587,0.05000,2.328
28,mlp.experts.16.gate_and_up_proj,0.0010707378,0.05000,2.394
28,mlp.experts.17.gate_and_up_proj,0.0002611744,0.05000,2.401
28,mlp.experts.18.gate_and_up_proj,0.0000851950,0.05000,2.386
28,mlp.experts.19.gate_and_up_proj,0.0006214485,0.05000,2.371
28,mlp.experts.20.gate_and_up_proj,0.0008747780,0.05000,2.389
28,mlp.experts.21.gate_and_up_proj,0.0000999213,0.05000,2.469
28,mlp.experts.22.gate_and_up_proj,0.0051265969,0.05000,2.472
28,mlp.experts.23.gate_and_up_proj,0.0001060205,0.05000,2.479
28,mlp.experts.24.gate_and_up_proj,0.0041421230,0.05000,2.449
28,mlp.experts.25.gate_and_up_proj,0.0006826238,0.05000,2.451
28,mlp.experts.26.gate_and_up_proj,0.0012981807,0.05000,2.474
28,mlp.experts.27.gate_and_up_proj,0.0023571530,0.05000,2.522
28,mlp.experts.28.gate_and_up_proj,0.0000934885,0.05000,2.492
28,mlp.experts.29.gate_and_up_proj,0.0045578926,0.05000,2.473
28,mlp.experts.30.gate_and_up_proj,0.0008099146,0.05000,2.394
28,mlp.experts.31.gate_and_up_proj,0.0051710431,0.05000,2.465
28,mlp.experts.32.gate_and_up_proj,0.0046052250,0.05000,2.441
28,mlp.experts.33.gate_and_up_proj,0.0115134521,0.05000,2.458
28,mlp.experts.34.gate_and_up_proj,0.0008746363,0.05000,2.413
28,mlp.experts.35.gate_and_up_proj,0.0014161884,0.05000,2.416
28,mlp.experts.36.gate_and_up_proj,0.0020165452,0.05000,2.469
28,mlp.experts.37.gate_and_up_proj,0.0004910472,0.05000,2.465
28,mlp.experts.38.gate_and_up_proj,0.0001935192,0.05000,2.503
28,mlp.experts.39.gate_and_up_proj,0.0000000000,0.05000,2.457
28,mlp.experts.40.gate_and_up_proj,0.0013002120,0.05000,2.470
28,mlp.experts.41.gate_and_up_proj,0.0005197279,0.05000,2.457
28,mlp.experts.42.gate_and_up_proj,0.0005992932,0.05000,2.300
28,mlp.experts.43.gate_and_up_proj,0.0017023268,0.05000,2.352
28,mlp.experts.44.gate_and_up_proj,0.0007952243,0.05000,2.375
28,mlp.experts.45.gate_and_up_proj,0.0000000000,0.05000,2.419
28,mlp.experts.46.gate_and_up_proj,0.0007757984,0.05000,2.482
28,mlp.experts.47.gate_and_up_proj,0.0048891776,0.05000,2.465
28,mlp.experts.48.gate_and_up_proj,0.0002373920,0.05000,2.408
28,mlp.experts.49.gate_and_up_proj,0.0000846770,0.05000,2.457
28,mlp.experts.50.gate_and_up_proj,0.0018150313,0.05000,2.458
28,mlp.experts.51.gate_and_up_proj,0.0000000000,0.05000,2.506
28,mlp.experts.52.gate_and_up_proj,0.0008341759,0.05000,2.497
28,mlp.experts.53.gate_and_up_proj,0.0031315813,0.05000,2.506
28,mlp.experts.54.gate_and_up_proj,0.0003140087,0.05000,2.368
28,mlp.experts.55.gate_and_up_proj,0.0047624007,0.05000,2.376
28,mlp.experts.56.gate_and_up_proj,0.0042430246,0.05000,2.363
28,mlp.experts.57.gate_and_up_proj,0.0003422715,0.05000,2.280
28,mlp.experts.58.gate_and_up_proj,0.0023776577,0.05000,2.335
28,mlp.experts.59.gate_and_up_proj,0.0002565186,0.05000,2.344
28,mlp.experts.60.gate_and_up_proj,0.0029069044,0.05000,2.263
28,mlp.experts.61.gate_and_up_proj,0.0000000000,0.05000,2.309
28,mlp.experts.62.gate_and_up_proj,0.0011483347,0.05000,2.261
28,mlp.experts.63.gate_and_up_proj,0.0047299496,0.05000,1.249
28,mlp.experts.2.down_proj,0.0000642157,0.05000,1.893
28,mlp.experts.0.down_proj,0.0001911910,0.05000,1.916
28,mlp.experts.1.down_proj,0.0000097532,0.05000,1.990
28,mlp.experts.3.down_proj,0.0000058462,0.05000,1.923
28,mlp.experts.4.down_proj,0.0000305407,0.05000,1.953
28,mlp.experts.5.down_proj,0.0002529165,0.05000,1.964
28,mlp.experts.6.down_proj,0.0000000000,0.05000,2.010
28,mlp.experts.7.down_proj,0.0000332867,0.05000,2.007
28,mlp.experts.8.down_proj,0.0000264523,0.05000,2.003
28,mlp.experts.9.down_proj,0.0000015967,0.05000,2.061
28,mlp.experts.10.down_proj,0.0001401235,0.05000,2.086
28,mlp.experts.11.down_proj,0.0001348003,0.05000,2.057
28,mlp.experts.12.down_proj,0.0000423399,0.05000,2.074
28,mlp.experts.14.down_proj,0.0001464454,0.05000,2.082
28,mlp.experts.13.down_proj,0.0000149431,0.05000,2.176
28,mlp.experts.15.down_proj,0.0001436478,0.05000,2.347
28,mlp.experts.16.down_proj,0.0000690099,0.05000,2.300
28,mlp.experts.17.down_proj,0.0000097300,0.05000,2.304
28,mlp.experts.18.down_proj,0.0000028264,0.05000,2.266
28,mlp.experts.19.down_proj,0.0000150166,0.05000,2.241
28,mlp.experts.20.down_proj,0.0000267112,0.05000,2.262
28,mlp.experts.21.down_proj,0.0000039728,0.05000,1.984
28,mlp.experts.22.down_proj,0.0002132964,0.05000,2.124
28,mlp.experts.23.down_proj,0.0000036139,0.05000,2.065
28,mlp.experts.24.down_proj,0.0001248566,0.05000,1.966
28,mlp.experts.25.down_proj,0.0000311546,0.05000,2.003
28,mlp.experts.26.down_proj,0.0000323892,0.05000,2.015
28,mlp.experts.27.down_proj,0.0000804138,0.05000,1.974
28,mlp.experts.28.down_proj,0.0000031807,0.05000,2.032
28,mlp.experts.29.down_proj,0.0001689071,0.05000,2.043
28,mlp.experts.30.down_proj,0.0000281216,0.05000,2.037
28,mlp.experts.32.down_proj,0.0001704014,0.05000,2.091
28,mlp.experts.31.down_proj,0.0001809321,0.05000,2.121
28,mlp.experts.33.down_proj,0.0002558329,0.05000,2.028
28,mlp.experts.35.down_proj,0.0000574138,0.05000,2.148
28,mlp.experts.34.down_proj,0.0000313857,0.05000,2.203
28,mlp.experts.36.down_proj,0.0000807165,0.05000,2.151
28,mlp.experts.37.down_proj,0.0000205454,0.05000,2.149
28,mlp.experts.38.down_proj,0.0000076296,0.05000,2.139
28,mlp.experts.39.down_proj,0.0000000000,0.05000,2.136
28,mlp.experts.40.down_proj,0.0000357057,0.05000,2.183
28,mlp.experts.41.down_proj,0.0000163239,0.05000,2.167
28,mlp.experts.42.down_proj,0.0000229221,0.05000,2.102
28,mlp.experts.43.down_proj,0.0000754891,0.05000,2.158
28,mlp.experts.44.down_proj,0.0000397810,0.05000,2.158
28,mlp.experts.45.down_proj,0.0000000000,0.05000,2.338
28,mlp.experts.47.down_proj,0.0002104919,0.05000,2.214
28,mlp.experts.46.down_proj,0.0000240571,0.05000,2.272
28,mlp.experts.48.down_proj,0.0000074460,0.05000,2.213
28,mlp.experts.49.down_proj,0.0000037220,0.05000,2.156
28,mlp.experts.50.down_proj,0.0000555444,0.05000,2.174
28,mlp.experts.51.down_proj,0.0000000000,0.05000,1.992
28,mlp.experts.52.down_proj,0.0000224612,0.05000,2.025
28,mlp.experts.53.down_proj,0.0001104856,0.05000,2.022
28,mlp.experts.54.down_proj,0.0000129728,0.05000,2.004
28,mlp.experts.55.down_proj,0.0001772461,0.05000,1.992
28,mlp.experts.56.down_proj,0.0001447104,0.05000,2.057
28,mlp.experts.57.down_proj,0.0000115786,0.05000,2.096
28,mlp.experts.58.down_proj,0.0000756690,0.05000,2.032
28,mlp.experts.59.down_proj,0.0000100717,0.05000,2.026
28,mlp.experts.60.down_proj,0.0001621820,0.05000,1.978
28,mlp.experts.61.down_proj,0.0000000000,0.05000,1.955
28,mlp.experts.62.down_proj,0.0000431231,0.05000,1.925
28,mlp.experts.63.down_proj,0.0001231506,0.05000,0.972
29,self_attn.qkv_proj,0.0008296597,0.05000,0.938
29,self_attn.o_proj,0.0007943168,0.05000,0.939
29,mlp.gate.wg,0.0000960905,0.05000,0.919
29,mlp.shared_mlp.gate_and_up_proj,0.0126098811,0.05000,0.942
29,mlp.shared_mlp.down_proj,0.0000977641,0.05000,0.699
29,mlp.experts.2.gate_and_up_proj,0.0063293584,0.05000,2.599
29,mlp.experts.0.gate_and_up_proj,0.0010193051,0.05000,2.660
29,mlp.experts.1.gate_and_up_proj,0.0010070975,0.05000,2.662
29,mlp.experts.3.gate_and_up_proj,0.0042959937,0.05000,2.426
29,mlp.experts.4.gate_and_up_proj,0.0002282754,0.05000,2.459
29,mlp.experts.5.gate_and_up_proj,0.0000759159,0.05000,2.554
29,mlp.experts.6.gate_and_up_proj,0.0005146875,0.05000,2.353
29,mlp.experts.7.gate_and_up_proj,0.0006277599,0.05000,2.372
29,mlp.experts.8.gate_and_up_proj,0.0000000000,0.05000,2.371
29,mlp.experts.9.gate_and_up_proj,0.0004034345,0.05000,2.315
29,mlp.experts.10.gate_and_up_proj,0.0002151498,0.05000,2.350
29,mlp.experts.11.gate_and_up_proj,0.0027984129,0.05000,2.347
29,mlp.experts.12.gate_and_up_proj,0.0005139545,0.05000,2.347
29,mlp.experts.13.gate_and_up_proj,0.0021182302,0.05000,2.359
29,mlp.experts.14.gate_and_up_proj,0.0018512195,0.05000,2.364
29,mlp.experts.15.gate_and_up_proj,0.0107083850,0.05000,2.494
29,mlp.experts.16.gate_and_up_proj,0.0027544460,0.05000,2.537
29,mlp.experts.17.gate_and_up_proj,0.0041760585,0.05000,2.547
29,mlp.experts.18.gate_and_up_proj,0.0056646746,0.05000,2.541
29,mlp.experts.19.gate_and_up_proj,0.0078580519,0.05000,2.480
29,mlp.experts.20.gate_and_up_proj,0.0002718535,0.05000,2.471
29,mlp.experts.21.gate_and_up_proj,0.0016225643,0.05000,2.348
29,mlp.experts.22.gate_and_up_proj,0.0002455614,0.05000,2.368
29,mlp.experts.23.gate_and_up_proj,0.0020647292,0.05000,2.356
29,mlp.experts.24.gate_and_up_proj,0.0030600922,0.05000,2.462
29,mlp.experts.25.gate_and_up_proj,0.0005630270,0.05000,2.499
29,mlp.experts.26.gate_and_up_proj,0.0001427025,0.05000,2.493
29,mlp.experts.27.gate_and_up_proj,0.0026908283,0.05000,2.440
29,mlp.experts.28.gate_and_up_proj,0.0009849187,0.05000,2.378
29,mlp.experts.29.gate_and_up_proj,0.0014381703,0.05000,2.360
29,mlp.experts.30.gate_and_up_proj,0.0112558522,0.05000,2.347
29,mlp.experts.31.gate_and_up_proj,0.0000000000,0.05000,2.398
29,mlp.experts.32.gate_and_up_proj,0.0003868708,0.05000,2.413
29,mlp.experts.33.gate_and_up_proj,0.0003071660,0.05000,2.339
29,mlp.experts.34.gate_and_up_proj,0.0000000000,0.05000,2.352
29,mlp.experts.35.gate_and_up_proj,0.0000000000,0.05000,2.342
29,mlp.experts.36.gate_and_up_proj,0.0020512519,0.05000,2.329
29,mlp.experts.37.gate_and_up_proj,0.0003503519,0.05000,2.394
29,mlp.experts.38.gate_and_up_proj,0.0050293704,0.05000,2.366
29,mlp.experts.39.gate_and_up_proj,0.0007205820,0.05000,2.348
29,mlp.experts.40.gate_and_up_proj,0.0070271014,0.05000,2.371
29,mlp.experts.41.gate_and_up_proj,0.0076881986,0.05000,2.353
29,mlp.experts.42.gate_and_up_proj,0.0012471401,0.05000,2.320
29,mlp.experts.43.gate_and_up_proj,0.0034641572,0.05000,2.380
29,mlp.experts.44.gate_and_up_proj,0.0019756563,0.05000,2.376
29,mlp.experts.45.gate_and_up_proj,0.0026369787,0.05000,2.302
29,mlp.experts.46.gate_and_up_proj,0.0032763290,0.05000,2.330
29,mlp.experts.47.gate_and_up_proj,0.0001395199,0.05000,2.328
29,mlp.experts.48.gate_and_up_proj,0.0056735873,0.05000,2.329
29,mlp.experts.49.gate_and_up_proj,0.0148918011,0.05000,2.315
29,mlp.experts.50.gate_and_up_proj,0.0017588444,0.05000,2.312
29,mlp.experts.51.gate_and_up_proj,0.0001670036,0.05000,2.440
29,mlp.experts.52.gate_and_up_proj,0.0005357169,0.05000,2.526
29,mlp.experts.53.gate_and_up_proj,0.0047543969,0.05000,2.519
29,mlp.experts.54.gate_and_up_proj,0.0000000000,0.05000,2.494
29,mlp.experts.55.gate_and_up_proj,0.0019085255,0.05000,2.572
29,mlp.experts.56.gate_and_up_proj,0.0009892121,0.05000,2.577
29,mlp.experts.57.gate_and_up_proj,0.0066072087,0.05000,2.513
29,mlp.experts.58.gate_and_up_proj,0.0001082726,0.05000,2.480
29,mlp.experts.59.gate_and_up_proj,0.0033884409,0.05000,2.474
29,mlp.experts.60.gate_and_up_proj,0.0008514689,0.05000,2.290
29,mlp.experts.61.gate_and_up_proj,0.0000000000,0.05000,2.316
29,mlp.experts.62.gate_and_up_proj,0.0000000000,0.05000,2.319
29,mlp.experts.63.gate_and_up_proj,0.0025423855,0.05000,1.245
29,mlp.experts.0.down_proj,0.0000295039,0.05000,1.858
29,mlp.experts.1.down_proj,0.0000582282,0.05000,1.873
29,mlp.experts.2.down_proj,0.0002031288,0.05000,1.961
29,mlp.experts.3.down_proj,0.0001558253,0.05000,1.856
29,mlp.experts.4.down_proj,0.0000062370,0.05000,1.854
29,mlp.experts.5.down_proj,0.0000014171,0.05000,1.867
29,mlp.experts.6.down_proj,0.0000147400,0.05000,1.827
29,mlp.experts.7.down_proj,0.0000201579,0.05000,1.855
29,mlp.experts.8.down_proj,0.0000000000,0.05000,1.869
29,mlp.experts.9.down_proj,0.0000331239,0.05000,1.816
29,mlp.experts.10.down_proj,0.0000089233,0.05000,1.804
29,mlp.experts.11.down_proj,0.0000983600,0.05000,1.876
29,mlp.experts.12.down_proj,0.0000209180,0.05000,1.803
29,mlp.experts.13.down_proj,0.0000627129,0.05000,1.810
29,mlp.experts.14.down_proj,0.0000502969,0.05000,1.864
29,mlp.experts.15.down_proj,0.0004355083,0.05000,1.917
29,mlp.experts.16.down_proj,0.0000818335,0.05000,1.923
29,mlp.experts.17.down_proj,0.0001227797,0.05000,1.966
29,mlp.experts.18.down_proj,0.0001757610,0.05000,1.917
29,mlp.experts.19.down_proj,0.0002872587,0.05000,1.903
29,mlp.experts.20.down_proj,0.0000056542,0.05000,1.926
29,mlp.experts.21.down_proj,0.0000818249,0.05000,1.954
29,mlp.experts.22.down_proj,0.0000130961,0.05000,1.944
29,mlp.experts.23.down_proj,0.0000769234,0.05000,1.993
29,mlp.experts.24.down_proj,0.0001625281,0.05000,1.935
29,mlp.experts.25.down_proj,0.0000574946,0.05000,1.930
29,mlp.experts.26.down_proj,0.0000093632,0.05000,1.949
29,mlp.experts.27.down_proj,0.0001384969,0.05000,1.866
29,mlp.experts.28.down_proj,0.0000306510,0.05000,1.868
29,mlp.experts.29.down_proj,0.0000458544,0.05000,1.860
29,mlp.experts.30.down_proj,0.0005350269,0.05000,1.903
29,mlp.experts.31.down_proj,0.0000000000,0.05000,1.898
29,mlp.experts.32.down_proj,0.0000105287,0.05000,1.918
29,mlp.experts.34.down_proj,0.0000000000,0.05000,1.936
29,mlp.experts.33.down_proj,0.0000087426,0.05000,1.984
29,mlp.experts.35.down_proj,0.0000000000,0.05000,1.990
29,mlp.experts.36.down_proj,0.0000875181,0.05000,2.000
29,mlp.experts.37.down_proj,0.0000081599,0.05000,2.090
29,mlp.experts.38.down_proj,0.0003096768,0.05000,2.014
29,mlp.experts.39.down_proj,0.0000272731,0.05000,2.098
29,mlp.experts.40.down_proj,0.0001388829,0.05000,2.133
29,mlp.experts.41.down_proj,0.0002636044,0.05000,2.121
29,mlp.experts.42.down_proj,0.0000647750,0.05000,2.194
29,mlp.experts.43.down_proj,0.0001088429,0.05000,2.128
29,mlp.experts.44.down_proj,0.0000548729,0.05000,2.182
29,mlp.experts.45.down_proj,0.0000657609,0.05000,2.089
29,mlp.experts.46.down_proj,0.0000866902,0.05000,2.134
29,mlp.experts.47.down_proj,0.0000053203,0.05000,2.120
29,mlp.experts.48.down_proj,0.0002138957,0.05000,2.088
29,mlp.experts.49.down_proj,0.0005180061,0.05000,2.119
29,mlp.experts.50.down_proj,0.0000438149,0.05000,2.173
29,mlp.experts.51.down_proj,0.0000073946,0.05000,2.182
29,mlp.experts.52.down_proj,0.0000158747,0.05000,2.103
29,mlp.experts.53.down_proj,0.0001359519,0.05000,2.124
29,mlp.experts.54.down_proj,0.0000000000,0.05000,2.067
29,mlp.experts.55.down_proj,0.0000557813,0.05000,2.065
29,mlp.experts.56.down_proj,0.0000267468,0.05000,2.058
29,mlp.experts.57.down_proj,0.0002748417,0.05000,2.036
29,mlp.experts.58.down_proj,0.0000036242,0.05000,1.989
29,mlp.experts.59.down_proj,0.0001421792,0.05000,2.035
29,mlp.experts.60.down_proj,0.0000343349,0.05000,2.009
29,mlp.experts.61.down_proj,0.0000000000,0.05000,1.981
29,mlp.experts.62.down_proj,0.0000000000,0.05000,1.805
29,mlp.experts.63.down_proj,0.0000664252,0.05000,0.811
30,self_attn.qkv_proj,0.0004906960,0.05000,0.962
30,self_attn.o_proj,0.0004607741,0.05000,0.956
30,mlp.gate.wg,0.0000794312,0.05000,0.914
30,mlp.shared_mlp.gate_and_up_proj,0.0103392115,0.05000,0.946
30,mlp.shared_mlp.down_proj,0.0000904645,0.05000,0.699
30,mlp.experts.0.gate_and_up_proj,0.0071999457,0.05000,2.476
30,mlp.experts.2.gate_and_up_proj,0.0009665322,0.05000,2.498
30,mlp.experts.1.gate_and_up_proj,0.0103452443,0.05000,2.602
30,mlp.experts.3.gate_and_up_proj,0.0028543561,0.05000,2.535
30,mlp.experts.4.gate_and_up_proj,0.0001775734,0.05000,2.582
30,mlp.experts.5.gate_and_up_proj,0.0000919024,0.05000,2.489
30,mlp.experts.6.gate_and_up_proj,0.0004494474,0.05000,2.302
30,mlp.experts.7.gate_and_up_proj,0.0000000000,0.05000,2.322
30,mlp.experts.8.gate_and_up_proj,0.0021434672,0.05000,2.317
30,mlp.experts.9.gate_and_up_proj,0.0008696828,0.05000,2.256
30,mlp.experts.11.gate_and_up_proj,0.0000610239,0.05000,2.283
30,mlp.experts.10.gate_and_up_proj,0.0001894219,0.05000,2.348
30,mlp.experts.13.gate_and_up_proj,0.0015152631,0.05000,2.463
30,mlp.experts.14.gate_and_up_proj,0.0017533317,0.05000,2.498
30,mlp.experts.12.gate_and_up_proj,0.0009307822,0.05000,2.751
30,mlp.experts.15.gate_and_up_proj,0.0007826828,0.05000,2.333
30,mlp.experts.16.gate_and_up_proj,0.0005103017,0.05000,2.349
30,mlp.experts.17.gate_and_up_proj,0.0007037575,0.05000,2.389
30,mlp.experts.18.gate_and_up_proj,0.0026191906,0.05000,2.311
30,mlp.experts.19.gate_and_up_proj,0.0000655824,0.05000,2.328
30,mlp.experts.20.gate_and_up_proj,0.0016808851,0.05000,2.330
30,mlp.experts.21.gate_and_up_proj,0.0004100615,0.05000,2.267
30,mlp.experts.22.gate_and_up_proj,0.0001036433,0.05000,2.315
30,mlp.experts.23.gate_and_up_proj,0.0017324763,0.05000,2.335
30,mlp.experts.24.gate_and_up_proj,0.0008494185,0.05000,2.270
30,mlp.experts.25.gate_and_up_proj,0.0028066433,0.05000,2.315
30,mlp.experts.26.gate_and_up_proj,0.0001469732,0.05000,2.311
30,mlp.experts.27.gate_and_up_proj,0.0004357517,0.05000,2.261
30,mlp.experts.28.gate_and_up_proj,0.0073394775,0.05000,2.309
30,mlp.experts.29.gate_and_up_proj,0.0001954359,0.05000,2.305
30,mlp.experts.30.gate_and_up_proj,0.0008133540,0.05000,2.334
30,mlp.experts.31.gate_and_up_proj,0.0006131812,0.05000,2.359
30,mlp.experts.32.gate_and_up_proj,0.0011415954,0.05000,2.376
30,mlp.experts.33.gate_and_up_proj,0.0001010440,0.05000,2.253
30,mlp.experts.34.gate_and_up_proj,0.0017468376,0.05000,2.276
30,mlp.experts.35.gate_and_up_proj,0.0008553029,0.05000,2.281
30,mlp.experts.36.gate_and_up_proj,0.0000670200,0.05000,2.308
30,mlp.experts.37.gate_and_up_proj,0.0003906457,0.05000,2.392
30,mlp.experts.38.gate_and_up_proj,0.0011321131,0.05000,2.372
30,mlp.experts.39.gate_and_up_proj,0.0006432824,0.05000,2.368
30,mlp.experts.40.gate_and_up_proj,0.0004904560,0.05000,2.466
30,mlp.experts.41.gate_and_up_proj,0.0013285561,0.05000,2.458
30,mlp.experts.42.gate_and_up_proj,0.0002034586,0.05000,2.344
30,mlp.experts.43.gate_and_up_proj,0.0069696808,0.05000,2.389
30,mlp.experts.44.gate_and_up_proj,0.0005163923,0.05000,2.344
30,mlp.experts.45.gate_and_up_proj,0.0000000000,0.05000,2.264
30,mlp.experts.46.gate_and_up_proj,0.0000000000,0.05000,2.311
30,mlp.experts.47.gate_and_up_proj,0.0006289961,0.05000,2.288
30,mlp.experts.48.gate_and_up_proj,0.0008762792,0.05000,2.363
30,mlp.experts.49.gate_and_up_proj,0.0000000000,0.05000,2.472
30,mlp.experts.50.gate_and_up_proj,0.0008063322,0.05000,2.467
30,mlp.experts.51.gate_and_up_proj,0.0000673434,0.05000,2.351
30,mlp.experts.52.gate_and_up_proj,0.0000000000,0.05000,2.373
30,mlp.experts.53.gate_and_up_proj,0.0000000000,0.05000,2.345
30,mlp.experts.54.gate_and_up_proj,0.0023998322,0.05000,2.284
30,mlp.experts.56.gate_and_up_proj,0.0031392322,0.05000,2.342
30,mlp.experts.55.gate_and_up_proj,0.0001702676,0.05000,2.426
30,mlp.experts.57.gate_and_up_proj,0.0001357749,0.05000,2.308
30,mlp.experts.58.gate_and_up_proj,0.0049617913,0.05000,2.315
30,mlp.experts.59.gate_and_up_proj,0.0009293535,0.05000,2.326
30,mlp.experts.60.gate_and_up_proj,0.0005920902,0.05000,2.323
30,mlp.experts.61.gate_and_up_proj,0.0003764122,0.05000,2.362
30,mlp.experts.62.gate_and_up_proj,0.0009402886,0.05000,2.371
30,mlp.experts.63.gate_and_up_proj,0.0004593751,0.05000,1.483
30,mlp.experts.1.down_proj,0.0004967801,0.05000,1.775
30,mlp.experts.2.down_proj,0.0000270752,0.05000,1.776
30,mlp.experts.0.down_proj,0.0002957332,0.05000,1.897
30,mlp.experts.4.down_proj,0.0000130433,0.05000,1.817
30,mlp.experts.3.down_proj,0.0000599302,0.05000,1.884
30,mlp.experts.5.down_proj,0.0000056425,0.05000,1.875
30,mlp.experts.6.down_proj,0.0000149724,0.05000,1.822
30,mlp.experts.7.down_proj,0.0000000000,0.05000,1.837
30,mlp.experts.8.down_proj,0.0001209277,0.05000,1.828
30,mlp.experts.9.down_proj,0.0000478267,0.05000,1.890
30,mlp.experts.10.down_proj,0.0000078513,0.05000,1.906
30,mlp.experts.11.down_proj,0.0000023757,0.05000,1.935
30,mlp.experts.12.down_proj,0.0000693994,0.05000,2.018
30,mlp.experts.13.down_proj,0.0000799799,0.05000,2.027
30,mlp.experts.14.down_proj,0.0000425649,0.05000,2.020
30,mlp.experts.15.down_proj,0.0000231134,0.05000,1.956
30,mlp.experts.16.down_proj,0.0000131957,0.05000,2.024
30,mlp.experts.17.down_proj,0.0000231379,0.05000,1.963
30,mlp.experts.18.down_proj,0.0001143685,0.05000,1.935
30,mlp.experts.19.down_proj,0.0000015113,0.05000,1.966
30,mlp.experts.20.down_proj,0.0000461032,0.05000,1.924
30,mlp.experts.21.down_proj,0.0000435721,0.05000,1.892
30,mlp.experts.22.down_proj,0.0000035235,0.05000,1.924
30,mlp.experts.23.down_proj,0.0000678124,0.05000,1.925
30,mlp.experts.24.down_proj,0.0000270323,0.05000,1.985
30,mlp.experts.25.down_proj,0.0002122464,0.05000,1.991
30,mlp.experts.26.down_proj,0.0000040509,0.05000,2.000
30,mlp.experts.27.down_proj,0.0000124756,0.05000,1.962
30,mlp.experts.28.down_proj,0.0002713634,0.05000,1.950
30,mlp.experts.29.down_proj,0.0000044587,0.05000,1.944
30,mlp.experts.30.down_proj,0.0000378339,0.05000,2.001
30,mlp.experts.31.down_proj,0.0000404979,0.05000,1.898
30,mlp.experts.32.down_proj,0.0000943670,0.05000,1.979
30,mlp.experts.33.down_proj,0.0000039370,0.05000,1.822
30,mlp.experts.34.down_proj,0.0000799150,0.05000,1.801
30,mlp.experts.35.down_proj,0.0000437608,0.05000,1.798
30,mlp.experts.36.down_proj,0.0000043588,0.05000,1.759
30,mlp.experts.37.down_proj,0.0000083228,0.05000,1.758
30,mlp.experts.38.down_proj,0.0001417401,0.05000,1.745
30,mlp.experts.39.down_proj,0.0000219716,0.05000,1.831
30,mlp.experts.40.down_proj,0.0000152077,0.05000,1.753
30,mlp.experts.41.down_proj,0.0000673508,0.05000,1.755
30,mlp.experts.43.down_proj,0.0002136572,0.05000,1.786
30,mlp.experts.42.down_proj,0.0000122820,0.05000,1.842
30,mlp.experts.44.down_proj,0.0000191828,0.05000,1.793
30,mlp.experts.45.down_proj,0.0000000000,0.05000,1.777
30,mlp.experts.46.down_proj,0.0000000000,0.05000,1.790
30,mlp.experts.47.down_proj,0.0000399810,0.05000,1.825
30,mlp.experts.48.down_proj,0.0000215692,0.05000,1.771
30,mlp.experts.49.down_proj,0.0000000000,0.05000,1.797
30,mlp.experts.50.down_proj,0.0000294693,0.05000,1.780
30,mlp.experts.51.down_proj,0.0000025347,0.05000,1.710
30,mlp.experts.52.down_proj,0.0000000000,0.05000,1.723
30,mlp.experts.53.down_proj,0.0000000000,0.05000,1.705
30,mlp.experts.54.down_proj,0.0000918362,0.05000,1.732
30,mlp.experts.55.down_proj,0.0000075535,0.05000,1.740
30,mlp.experts.56.down_proj,0.0001238396,0.05000,1.724
30,mlp.experts.57.down_proj,0.0000032438,0.05000,1.770
30,mlp.experts.58.down_proj,0.0002020004,0.05000,1.745
30,mlp.experts.59.down_proj,0.0000383583,0.05000,1.746
30,mlp.experts.60.down_proj,0.0000172024,0.05000,1.843
30,mlp.experts.61.down_proj,0.0000090902,0.05000,1.800
30,mlp.experts.62.down_proj,0.0000295491,0.05000,1.792
30,mlp.experts.63.down_proj,0.0000155158,0.05000,0.727
31,self_attn.qkv_proj,0.0001703759,0.05000,0.941
31,self_attn.o_proj,0.0002437127,0.05000,0.937
31,mlp.gate.wg,0.0000527157,0.05000,0.915
31,mlp.shared_mlp.gate_and_up_proj,0.0053055715,0.05000,0.936
31,mlp.shared_mlp.down_proj,0.0000430565,0.05000,0.696
31,mlp.experts.0.gate_and_up_proj,0.0003008674,0.05000,2.452
31,mlp.experts.1.gate_and_up_proj,0.0002841171,0.05000,2.467
31,mlp.experts.2.gate_and_up_proj,0.0000643425,0.05000,2.471
31,mlp.experts.3.gate_and_up_proj,0.0015920949,0.05000,2.312
31,mlp.experts.4.gate_and_up_proj,0.0028519975,0.05000,2.397
31,mlp.experts.5.gate_and_up_proj,0.0008355768,0.05000,2.415
31,mlp.experts.6.gate_and_up_proj,0.0000350230,0.05000,2.326
31,mlp.experts.7.gate_and_up_proj,0.0010815872,0.05000,2.373
31,mlp.experts.8.gate_and_up_proj,0.0001158145,0.05000,2.375
31,mlp.experts.9.gate_and_up_proj,0.0003090802,0.05000,2.449
31,mlp.experts.10.gate_and_up_proj,0.0024325092,0.05000,2.456
31,mlp.experts.11.gate_and_up_proj,0.0011693582,0.05000,2.442
31,mlp.experts.12.gate_and_up_proj,0.0002066774,0.05000,2.482
31,mlp.experts.13.gate_and_up_proj,0.0002167655,0.05000,2.466
31,mlp.experts.14.gate_and_up_proj,0.0004499408,0.05000,2.464
31,mlp.experts.15.gate_and_up_proj,0.0003696301,0.05000,2.286
31,mlp.experts.17.gate_and_up_proj,0.0001334601,0.05000,2.306
31,mlp.experts.16.gate_and_up_proj,0.0005945489,0.05000,2.376
31,mlp.experts.18.gate_and_up_proj,0.0009811211,0.05000,2.369
31,mlp.experts.19.gate_and_up_proj,0.0001520989,0.05000,2.415
31,mlp.experts.20.gate_and_up_proj,0.0000000000,0.05000,2.424
31,mlp.experts.21.gate_and_up_proj,0.0025319926,0.05000,2.331
31,mlp.experts.22.gate_and_up_proj,0.0012675796,0.05000,2.333
31,mlp.experts.23.gate_and_up_proj,0.0006916572,0.05000,2.349
31,mlp.experts.24.gate_and_up_proj,0.0000000000,0.05000,2.348
31,mlp.experts.25.gate_and_up_proj,0.0007259616,0.05000,2.394
31,mlp.experts.26.gate_and_up_proj,0.0006096121,0.05000,2.446
31,mlp.experts.27.gate_and_up_proj,0.0006619770,0.05000,2.719
31,mlp.experts.28.gate_and_up_proj,0.0001577560,0.05000,2.769
31,mlp.experts.29.gate_and_up_proj,0.0008710883,0.05000,2.864
31,mlp.experts.30.gate_and_up_proj,0.0000000000,0.05000,2.775
31,mlp.experts.31.gate_and_up_proj,0.0000000000,0.05000,2.783
31,mlp.experts.32.gate_and_up_proj,0.0001665735,0.05000,2.896
31,mlp.experts.33.gate_and_up_proj,0.0001921929,0.05000,2.892
31,mlp.experts.34.gate_and_up_proj,0.0001630016,0.05000,2.781
31,mlp.experts.35.gate_and_up_proj,0.0003707495,0.05000,2.860
31,mlp.experts.36.gate_and_up_proj,0.0000577065,0.05000,2.737
31,mlp.experts.37.gate_and_up_proj,0.0002077327,0.05000,2.723
31,mlp.experts.38.gate_and_up_proj,0.0000216539,0.05000,2.750
31,mlp.experts.39.gate_and_up_proj,0.0002683019,0.05000,2.718
31,mlp.experts.40.gate_and_up_proj,0.0028169608,0.05000,2.796
31,mlp.experts.41.gate_and_up_proj,0.0000605013,0.05000,2.857
31,mlp.experts.42.gate_and_up_proj,0.0001555402,0.05000,3.082
31,mlp.experts.43.gate_and_up_proj,0.0003893076,0.05000,2.997
31,mlp.experts.44.gate_and_up_proj,0.0008369043,0.05000,3.060
31,mlp.experts.45.gate_and_up_proj,0.0056125480,0.05000,2.848
31,mlp.experts.46.gate_and_up_proj,0.0020796856,0.05000,2.784
31,mlp.experts.47.gate_and_up_proj,0.0004712624,0.05000,2.792
31,mlp.experts.48.gate_and_up_proj,0.0003161796,0.05000,2.627
31,mlp.experts.49.gate_and_up_proj,0.0008893618,0.05000,2.683
31,mlp.experts.50.gate_and_up_proj,0.0001701678,0.05000,2.619
31,mlp.experts.51.gate_and_up_proj,0.0000000000,0.05000,2.378
31,mlp.experts.52.gate_and_up_proj,0.0002378390,0.05000,2.398
31,mlp.experts.53.gate_and_up_proj,0.0000262242,0.05000,2.378
31,mlp.experts.54.gate_and_up_proj,0.0002133290,0.05000,2.378
31,mlp.experts.55.gate_and_up_proj,0.0015304136,0.05000,2.382
31,mlp.experts.56.gate_and_up_proj,0.0006173670,0.05000,2.383
31,mlp.experts.57.gate_and_up_proj,0.0000000000,0.05000,2.293
31,mlp.experts.58.gate_and_up_proj,0.0002705629,0.05000,2.324
31,mlp.experts.59.gate_and_up_proj,0.0007435871,0.05000,2.391
31,mlp.experts.60.gate_and_up_proj,0.0000000000,0.05000,2.368
31,mlp.experts.61.gate_and_up_proj,0.0006656586,0.05000,2.399
31,mlp.experts.62.gate_and_up_proj,0.0007689067,0.05000,2.200
31,mlp.experts.63.gate_and_up_proj,0.0018067377,0.05000,1.230
31,mlp.experts.1.down_proj,0.0000216627,0.05000,1.964
31,mlp.experts.0.down_proj,0.0000138249,0.05000,1.988
31,mlp.experts.2.down_proj,0.0000046864,0.05000,2.013
31,mlp.experts.3.down_proj,0.0000723486,0.05000,1.923
31,mlp.experts.4.down_proj,0.0000922065,0.05000,1.923
31,mlp.experts.5.down_proj,0.0000390851,0.05000,1.986
31,mlp.experts.6.down_proj,0.0000034246,0.05000,1.972
31,mlp.experts.7.down_proj,0.0000531051,0.05000,1.988
31,mlp.experts.8.down_proj,0.0000123065,0.05000,2.025
31,mlp.experts.10.down_proj,0.0000838462,0.05000,2.046
31,mlp.experts.9.down_proj,0.0000138458,0.05000,2.090
31,mlp.experts.11.down_proj,0.0000287059,0.05000,2.009
31,mlp.experts.12.down_proj,0.0000310987,0.05000,1.929
31,mlp.experts.14.down_proj,0.0000205653,0.05000,1.943
31,mlp.experts.13.down_proj,0.0000091454,0.05000,2.052
31,mlp.experts.15.down_proj,0.0000248884,0.05000,1.964
31,mlp.experts.16.down_proj,0.0000512711,0.05000,1.908
31,mlp.experts.17.down_proj,0.0000046872,0.05000,1.908
31,mlp.experts.18.down_proj,0.0000376250,0.05000,1.869
31,mlp.experts.19.down_proj,0.0000048805,0.05000,1.850
31,mlp.experts.20.down_proj,0.0000000000,0.05000,1.851
31,mlp.experts.21.down_proj,0.0001207678,0.05000,1.811
31,mlp.experts.22.down_proj,0.0000365174,0.05000,1.798
31,mlp.experts.23.down_proj,0.0000757158,0.05000,1.785
31,mlp.experts.24.down_proj,0.0000000000,0.05000,1.768
31,mlp.experts.26.down_proj,0.0000428682,0.05000,1.803
31,mlp.experts.25.down_proj,0.0000276342,0.05000,1.847
31,mlp.experts.27.down_proj,0.0000185959,0.05000,1.743
31,mlp.experts.28.down_proj,0.0000123269,0.05000,1.754
31,mlp.experts.29.down_proj,0.0000415783,0.05000,1.767
31,mlp.experts.30.down_proj,0.0000000000,0.05000,1.693
31,mlp.experts.31.down_proj,0.0000000000,0.05000,1.726
31,mlp.experts.32.down_proj,0.0000038417,0.05000,1.742
31,mlp.experts.33.down_proj,0.0000093200,0.05000,1.697
31,mlp.experts.34.down_proj,0.0000051152,0.05000,1.738
31,mlp.experts.35.down_proj,0.0000156678,0.05000,1.744
31,mlp.experts.36.down_proj,0.0000033981,0.05000,1.705
31,mlp.experts.37.down_proj,0.0000055464,0.05000,1.759
31,mlp.experts.38.down_proj,0.0000004219,0.05000,1.770
31,mlp.experts.39.down_proj,0.0000099749,0.05000,1.777
31,mlp.experts.40.down_proj,0.0000902386,0.05000,1.848
31,mlp.experts.41.down_proj,0.0000040595,0.05000,1.828
31,mlp.experts.42.down_proj,0.0000104981,0.05000,1.793
31,mlp.experts.43.down_proj,0.0000340765,0.05000,1.837
31,mlp.experts.44.down_proj,0.0000212817,0.05000,1.850
31,mlp.experts.45.down_proj,0.0001813159,0.05000,1.909
31,mlp.experts.46.down_proj,0.0000488775,0.05000,1.951
31,mlp.experts.47.down_proj,0.0000164601,0.05000,1.961
31,mlp.experts.48.down_proj,0.0000484968,0.05000,1.961
31,mlp.experts.49.down_proj,0.0000679595,0.05000,1.946
31,mlp.experts.50.down_proj,0.0000114642,0.05000,1.968
31,mlp.experts.51.down_proj,0.0000000000,0.05000,1.967
31,mlp.experts.52.down_proj,0.0000090859,0.05000,2.004
31,mlp.experts.53.down_proj,0.0000005202,0.05000,2.025
31,mlp.experts.54.down_proj,0.0000103265,0.05000,2.050
31,mlp.experts.55.down_proj,0.0001025669,0.05000,2.057
31,mlp.experts.56.down_proj,0.0000195529,0.05000,2.053
31,mlp.experts.57.down_proj,0.0000000000,0.05000,2.026
31,mlp.experts.58.down_proj,0.0000129145,0.05000,2.062
31,mlp.experts.59.down_proj,0.0000420109,0.05000,2.065
31,mlp.experts.60.down_proj,0.0000000000,0.05000,2.075
31,mlp.experts.61.down_proj,0.0000312128,0.05000,2.070
31,mlp.experts.62.down_proj,0.0000369926,0.05000,2.035
31,mlp.experts.63.down_proj,0.0000613165,0.05000,1.004