tinystories-attn-ablations-v2 / results_compute.csv
genaiquest's picture
upload results_compute.csv
5f7f964 verified
Raw History Blame Contribute Delete
37.4 kB
config_name,attn_kind,batch,context,q_heads,kv_heads,head_dim,latent_dim,cache_mb,bytes_per_token_kb,arithmetic_intensity,ms_mean,ms_p50,ms_p99,tokens_per_sec,approx_gb_s,speedup_vs_mha,cache_reduction_vs_mha
MHA+RoPE,mha,1,1024,8,8,48,,1.5,1536.0,1.0,0.04196031980216503,0.041120000183582306,0.05967487871646878,23832.039524837055,37.4845570151933,1.0,1.0
MHA+RoPE,mha,1,4096,8,8,48,,6.0,6144.0,1.0,0.04208320006728172,0.041200000792741776,0.054456319957971545,23762.451486607988,149.50041798012876,1.0,1.0
MHA+RoPE,mha,1,8192,8,8,48,,12.0,12288.0,1.0,0.043237759917974475,0.04295999929308891,0.04845119982957839,23127.93266573201,291.0167414748313,1.0,1.0
MHA+RoPE,mha,1,16384,8,8,48,,24.0,24576.0,1.0,0.04435264006257057,0.04390399903059006,0.05143648028373718,22546.57216772774,567.4030669763348,1.0,1.0
MHA+RoPE,mha,1,32768,8,8,48,,48.0,49152.0,1.0,0.0694252796471119,0.06919999793171883,0.07606304094195364,14403.975109398067,724.975805006985,1.0,1.0
MHA+RoPE,mha,4,1024,8,8,48,,6.0,1536.0,1.0,0.04083263993263245,0.040383998304605484,0.0479654399305582,97960.84717028785,154.07908992364762,1.0,1.0
MHA+RoPE,mha,4,4096,8,8,48,,24.0,6144.0,1.0,0.04331200025975704,0.04295999929308891,0.04800640039145947,92353.15792414613,581.0358295408166,1.0,1.0
MHA+RoPE,mha,4,8192,8,8,48,,48.0,12288.0,1.0,0.04689600005745888,0.046720001846551895,0.0534063997864723,85295.12101456495,1073.2610017556215,1.0,1.0
MHA+RoPE,mha,4,16384,8,8,48,,96.0,24576.0,1.0,0.08197824001312255,0.08169600367546082,0.08909215949475763,48793.43590884246,1227.9270204372094,1.0,1.0
MHA+RoPE,mha,4,32768,8,8,48,,192.0,49152.0,1.0,0.14707712024450303,0.1470079943537712,0.15302880600094793,27196.616260573668,1368.8505164182702,1.0,1.0
MHA+RoPE,mha,16,1024,8,8,48,,24.0,1536.0,1.0,0.044017280116677285,0.043248001486063004,0.05641471881419419,363493.6088188219,571.7260115412075,1.0,1.0
MHA+RoPE,mha,16,4096,8,8,48,,96.0,6144.0,1.0,0.06801855996251106,0.06780800223350525,0.07406176075339316,235229.91384731638,1479.9386528541816,1.0,1.0
MHA+RoPE,mha,16,8192,8,8,48,,192.0,12288.0,1.0,0.10949375957250596,0.10929599776864052,0.11598207943141459,146127.04927174337,1838.703801806011,1.0,1.0
MHA+RoPE,mha,16,16384,8,8,48,,384.0,24576.0,1.0,0.1987967997789383,0.19838400185108185,0.2064918431639671,80484.19299401184,2025.451035669335,1.0,1.0
MHA+RoPE,mha,16,32768,8,8,48,,768.0,49152.0,1.0,0.37680127859115603,0.37671999633312225,0.3862249684333801,42462.700922415446,2137.2177159562893,1.0,1.0
MQA+RoPE,mqa,1,1024,8,1,48,,0.1875,192.0,8.0,0.06660416036844254,0.06304000318050385,0.14472960166633106,15014.077115726335,2.951887673568723,0.6299954773102446,8.0
MQA+RoPE,mqa,1,4096,8,1,48,,0.75,768.0,8.0,0.06222527988255024,0.0615679994225502,0.0694271995127201,16070.638844654337,12.638464647879202,0.6763039097086176,8.0
MQA+RoPE,mqa,1,8192,8,1,48,,1.5,1536.0,8.0,0.06280703984200954,0.06201599910855293,0.0677638417482376,15921.7820568441,25.042797813056037,0.688422190040139,8.0
MQA+RoPE,mqa,1,16384,8,1,48,,3.0,3072.0,8.0,0.06329408004879951,0.06207999959588051,0.06879455849528313,15799.26589072791,49.70019309190773,0.7007391533043034,8.0
MQA+RoPE,mqa,1,32768,8,1,48,,6.0,6144.0,8.0,0.06314688012003898,0.06270400062203407,0.06916384115815162,15836.095118223597,99.63209564811856,1.0994253321009368,8.0
MQA+RoPE,mqa,4,1024,8,1,48,,0.75,192.0,8.0,0.06692607939243317,0.06625599786639214,0.07795744061470031,59767.433507426555,11.75075556702812,0.610115523026575,8.0
MQA+RoPE,mqa,4,4096,8,1,48,,3.0,768.0,8.0,0.06898879975080491,0.06830399855971336,0.07622655898332595,57980.42601767878,45.59766239393515,0.6278120566846318,8.0
MQA+RoPE,mqa,4,8192,8,1,48,,6.0,1536.0,8.0,0.07014207929372787,0.06940799951553345,0.07992223687469957,57027.108980467325,89.69588673945377,0.6685858265061774,8.0
MQA+RoPE,mqa,4,16384,8,1,48,,12.0,3072.0,8.0,0.07037567988038063,0.06931199878454208,0.08951520167291159,56837.81679692337,178.79631175695218,1.1648660468000183,8.0
MQA+RoPE,mqa,4,32768,8,1,48,,24.0,6144.0,8.0,0.09821567967534066,0.09751999750733376,0.11645151741802685,40726.69469093226,256.2302076734339,1.4974912430548517,8.0
MQA+RoPE,mqa,16,1024,8,1,48,,3.0,192.0,8.0,0.06035328008234501,0.05991999991238117,0.06658784009516239,265105.7238010903,52.121906145084765,0.7293270565679421,8.0
MQA+RoPE,mqa,16,4096,8,1,48,,12.0,768.0,8.0,0.06872127994894982,0.0682239979505539,0.07722624234855173,232824.5342910629,183.1006641515892,0.9897743466512735,8.0
MQA+RoPE,mqa,16,8192,8,1,48,,24.0,1536.0,8.0,0.06989888012409211,0.06911999732255936,0.08328896000981328,228902.09358998394,360.0318625323165,1.56645942507406,8.0
MQA+RoPE,mqa,16,16384,8,1,48,,48.0,3072.0,8.0,0.08694527938961982,0.08670399710536003,0.09960544049739833,184023.7918875467,578.8887948068284,2.2864588069018517,8.0
MQA+RoPE,mqa,16,32768,8,1,48,,96.0,6144.0,8.0,0.15306880086660385,0.15276800096035004,0.16431615814566608,104528.15929448388,657.6343149622364,2.461646504433847,8.0
"GQA(g=2, kv_h=4)+RoPE",gqa,1,1024,8,4,48,,0.75,768.0,2.0,0.06814591944217682,0.0671519972383976,0.0808550403267145,14674.39295244259,11.540412198375332,0.6157422211871277,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,1,4096,8,4,48,,3.0,3072.0,2.0,0.06971647977828979,0.06913599744439125,0.08045567788183688,14343.810863373614,45.121727459618555,0.6036334622906008,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,1,8192,8,4,48,,6.0,6144.0,2.0,0.07107200011610985,0.07009599730372429,0.08475679993629454,14070.23860826073,88.52228711337362,0.6083655989325928,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,1,16384,8,4,48,,12.0,12288.0,2.0,0.07069184020161629,0.06960000097751617,0.08892160095274443,14145.904211121897,177.99666784897624,0.6274081978354908,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,1,32768,8,4,48,,24.0,24576.0,2.0,0.09634176045656204,0.09559999778866768,0.11551904141902916,10379.714832498557,261.21407664484815,0.7206146049034876,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,4,1024,8,4,48,,3.0,768.0,2.0,0.06719808027148247,0.06643199920654297,0.07789888001978396,59525.51001218885,46.8127658899057,0.6076459292835038,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,4,4096,8,4,48,,12.0,3072.0,2.0,0.07003072008490563,0.0692799985408783,0.07701920121908187,57117.7905232215,179.6770329470325,0.6184714394946295,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,4,8192,8,4,48,,24.0,6144.0,2.0,0.0713542404770851,0.06985600292682648,0.09288671918213366,56058.33617253022,352.68855546268225,0.6572279340920065,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,4,16384,8,4,48,,48.0,12288.0,2.0,0.07130880057811737,0.07051200047135353,0.08870400294661517,56094.058062554,705.8265963240076,1.149623038790521,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,4,32768,8,4,48,,96.0,24576.0,2.0,0.13714047968387605,0.13684800267219543,0.14905344083905214,29167.17229821889,734.0159246346522,1.0724559268243194,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,16,1024,8,4,48,,12.0,768.0,2.0,0.06099840000271797,0.06043200008571148,0.06622816130518913,262301.96200698824,206.28265658507974,0.7216136835509778,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,16,4096,8,4,48,,48.0,3072.0,2.0,0.06372031971812248,0.06313599646091461,0.07215967752039432,251097.29629070734,789.8837956659742,1.0674547815108677,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,16,8192,8,4,48,,96.0,6144.0,2.0,0.09086464002728462,0.09062399715185165,0.10041823953390118,176086.09900612116,1107.837944108655,1.2050205617897944,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,16,16384,8,4,48,,192.0,12288.0,2.0,0.1630707198381424,0.1626719981431961,0.17318432226777072,98116.93979079122,1234.5968190968242,1.2190833521570041,2.0
"GQA(g=2, kv_h=4)+RoPE",gqa,16,32768,8,4,48,,384.0,24576.0,2.0,0.3070284801721573,0.3065919876098633,0.318209912776947,52112.42941055001,1311.4522267583252,1.227251877024163,2.0
"GQA(g=4, kv_h=2)+RoPE",gqa,1,1024,8,2,48,,0.375,384.0,4.0,0.06800192028284073,0.06704000011086464,0.08058592177927491,14705.467078586824,5.782424942773597,0.6170461014577127,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,1,4096,8,2,48,,1.5,1536.0,4.0,0.06943551987409592,0.06903999671339989,0.0762691231071949,14401.85083676556,22.652152714518426,0.6060759701027537,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,1,8192,8,2,48,,3.0,3072.0,4.0,0.07039871990680695,0.06956800073385239,0.08149759776890277,14204.803742508231,44.68444886731293,0.6141838939005162,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,1,16384,8,2,48,,6.0,6144.0,4.0,0.07118336036801338,0.07003199681639671,0.09164543978869912,14048.226928738184,88.38380160017142,0.6230759524876359,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,1,32768,8,2,48,,12.0,12288.0,4.0,0.09191871970891953,0.09100800007581711,0.11340255975723258,10879.176767982799,136.89172390397198,0.7552898894475688,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,4,1024,8,2,48,,1.5,384.0,4.0,0.06696191981434822,0.06641599908471107,0.0744655966758728,59735.44383270359,23.488932282120377,0.6097889673091939,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,4,4096,8,2,48,,6.0,1536.0,4.0,0.07039359971880912,0.06913599744439125,0.08967455893754954,56823.34780403626,89.3753981204477,0.6152832137121708,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,4,8192,8,2,48,,12.0,3072.0,4.0,0.0710252806544304,0.0698080025613308,0.08722272105515003,56317.97527787015,177.16103173490393,0.6602719429667415,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,4,16384,8,2,48,,24.0,6144.0,4.0,0.0707065600156784,0.0695200003683567,0.09046943783760066,56571.83716918266,355.91922438907727,1.159414911359636,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,4,32768,8,2,48,,48.0,12288.0,4.0,0.09844735980033875,0.09780799970030785,0.11237216055393213,40630.850924924816,511.25442167344755,1.4939671367804113,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,16,1024,8,2,48,,6.0,384.0,4.0,0.061764480024576185,0.061135999858379364,0.07123455777764319,259048.56632215757,101.8620410549335,0.712663331726629,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,16,4096,8,2,48,,24.0,1536.0,4.0,0.06377600073814392,0.06302400305867195,0.07351616039872168,250878.07035273264,394.5970852472805,1.0665228169729009,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,16,8192,8,2,48,,48.0,3072.0,4.0,0.07105727940797806,0.07046400010585785,0.08250016070902344,225170.4559097372,708.3250079280258,1.54092248513827,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,16,16384,8,2,48,,96.0,6144.0,4.0,0.10639360040426254,0.10571200028061867,0.11788287840783593,150384.98499162527,946.1405161354709,1.868503359446173,4.0
"GQA(g=4, kv_h=2)+RoPE",gqa,16,32768,8,2,48,,192.0,12288.0,4.0,0.19697728067636489,0.19675200432538986,0.20817632049322124,81227.6418126013,1022.0802688954827,1.9129174557457884,4.0
MLA(r=48)+RoPE,mla,1,1024,8,8,48,48.0,1.5,1536.0,2.0,0.08214143946766854,0.08116799965500832,0.0915439999103546,12174.123152463224,19.148240038075915,0.5108301007882984,1.0
MLA-abs(r=48),mla_absorbed,1,1024,8,8,48,48.0,1.5,1536.0,1.046875,0.08234047994017601,0.0817439965903759,0.08989727862179278,12144.694817500991,19.10195326943388,0.5095952784420379,1.0
MLA(r=48)+RoPE,mla,1,4096,8,8,48,48.0,6.0,6144.0,2.0,0.08173183992505073,0.08107199892401695,0.09114784039556978,12235.133834219496,76.97680617210325,0.5148935849954268,1.0
MLA-abs(r=48),mla_absorbed,1,4096,8,8,48,48.0,6.0,6144.0,1.01171875,0.08209536015987397,0.0814560018479824,0.08939167842268943,12180.956366505758,76.63595101779086,0.5126136237824909,1.0
MLA(r=48)+RoPE,mla,1,8192,8,8,48,48.0,12.0,12288.0,2.0,0.08370943993330002,0.08311999961733818,0.08979008175432682,11946.083987621989,150.31652356085658,0.5165219114167587,1.0
MLA-abs(r=48),mla_absorbed,1,8192,8,8,48,48.0,12.0,12288.0,1.005859375,0.08304384022951126,0.08246400207281113,0.09077727802097797,12041.832328999524,151.52131651455605,0.5206618552137848,1.0
MLA(r=48)+RoPE,mla,1,16384,8,8,48,48.0,24.0,24576.0,2.0,0.0856844799220562,0.08503999933600426,0.09480831876397132,11670.724977378175,293.70341073310317,0.5176274641909063,1.0
MLA-abs(r=48),mla_absorbed,1,16384,8,8,48,48.0,24.0,24576.0,1.0029296875,0.08280448019504547,0.08214399963617325,0.09449152305722233,12076.641235407866,303.9186278414169,0.5356309218788426,1.0
MLA(r=48)+RoPE,mla,1,32768,8,8,48,48.0,48.0,49152.0,2.0,0.1317382401227951,0.13140799850225449,0.14074400141835208,7590.810375695664,382.05799586426195,0.5269941330808701,1.0
MLA-abs(r=48),mla_absorbed,1,32768,8,8,48,48.0,48.0,49152.0,1.00146484375,0.08601407960057258,0.08560000360012054,0.09916767776012418,11626.003610615198,585.1559213762132,0.8071385518452928,1.0
MLA(r=48)+RoPE,mla,4,1024,8,8,48,48.0,6.0,1536.0,2.0,0.10542912006378174,0.10475200042128563,0.11342847898602484,37940.1819685122,59.67474637172197,0.3872994473247032,1.0
MLA-abs(r=48),mla_absorbed,4,1024,8,8,48,48.0,6.0,1536.0,1.046875,0.09193919971585274,0.09132800251245499,0.09884063951671122,43507.013465011645,68.43061522663207,0.44412655384025307,1.0
MLA(r=48)+RoPE,mla,4,4096,8,8,48,48.0,24.0,6144.0,2.0,0.15052415996789933,0.1504960060119629,0.15690496176481245,26573.807160611537,167.18793850347245,0.2877411856607851,1.0
MLA-abs(r=48),mla_absorbed,4,4096,8,8,48,48.0,24.0,6144.0,1.01171875,0.09212864086031913,0.09116800129413605,0.10587808080017565,43417.5514003794,273.1596142632254,0.4701252493828118,1.0
MLA(r=48)+RoPE,mla,4,8192,8,8,48,48.0,48.0,12288.0,2.0,0.28791295886039736,0.2876479923725128,0.29479872137308116,13893.087743714626,174.8155004874397,0.16288256090688058,1.0
MLA-abs(r=48),mla_absorbed,4,8192,8,8,48,48.0,48.0,12288.0,1.005859375,0.09175167977809906,0.09110400080680847,0.09955200158059596,43595.93208183195,548.5637769436681,0.511118708353641,1.0
MLA(r=48)+RoPE,mla,4,16384,8,8,48,48.0,96.0,24576.0,2.0,0.5355449569225311,0.5351999998092651,0.5439724707603454,7469.027479944353,187.96423101144313,0.15307443185387154,1.0
MLA-abs(r=48),mla_absorbed,4,16384,8,8,48,48.0,96.0,24576.0,1.0029296875,0.09569600030779839,0.09468799829483032,0.11609119527041907,41799.030128054736,1051.907035573323,0.856652731038353,1.0
MLA(r=48)+RoPE,mla,4,32768,8,8,48,48.0,192.0,49152.0,2.0,1.0225177597999573,1.0222079753875732,1.0294419419765473,3911.9124940994175,196.89300265981396,0.14383820607015846,1.0
MLA-abs(r=48),mla_absorbed,4,32768,8,8,48,48.0,192.0,49152.0,1.00146484375,0.15815872102975845,0.1573440060019493,0.17686175927519793,25291.049231786452,1272.9401874849461,0.9299336722432755,1.0
MLA(r=48)+RoPE,mla,16,1024,8,8,48,48.0,24.0,1536.0,2.0,0.14810816019773484,0.1478400006890297,0.15617951989173887,108029.15908643299,169.91517527732333,0.2971968597672884,1.0
MLA-abs(r=48),mla_absorbed,16,1024,8,8,48,48.0,24.0,1536.0,1.046875,0.09313664004206658,0.0926399976015091,0.10111520193517207,171790.60778629503,270.20326252518316,0.4726097065214744,1.0
MLA(r=48)+RoPE,mla,16,4096,8,8,48,48.0,96.0,6144.0,2.0,0.5187660813331604,0.5185439884662628,0.5257276701927185,30842.417374092984,194.04371184274154,0.13111605097178353,1.0
MLA-abs(r=48),mla_absorbed,16,4096,8,8,48,48.0,96.0,6144.0,1.01171875,0.0958886405825615,0.09512000158429146,0.10843776173889635,166860.2235133761,1049.7937543845712,0.7093495074001607,1.0
MLA(r=48)+RoPE,mla,16,8192,8,8,48,48.0,192.0,12288.0,2.0,0.9817433595657349,0.9810720086097717,0.9914598405361176,16297.538296645524,205.07049020332053,0.11152992124228833,1.0
MLA-abs(r=48),mla_absorbed,16,8192,8,8,48,48.0,192.0,12288.0,1.005859375,0.12335743948817253,0.122895997017622,0.14100159823894495,129704.37832032071,1632.0587784193035,0.8876137509566594,1.0
MLA(r=48)+RoPE,mla,16,16384,8,8,48,48.0,384.0,24576.0,2.0,1.8987660837173461,1.8986079692840576,1.908333088159561,8426.525066571492,212.06044675692644,0.1046978885307136,1.0
MLA-abs(r=48),mla_absorbed,16,16384,8,8,48,48.0,384.0,24576.0,1.0029296875,0.21205887883901597,0.21142399311065674,0.23062175899744028,75450.74315019068,1898.780122786904,0.9374603924500348,1.0
MLA(r=48)+RoPE,mla,16,32768,8,8,48,48.0,768.0,49152.0,2.0,3.7548435306549073,3.7545759677886963,3.7637599968910216,4261.162913813704,214.47135184872567,0.10035072713813872,1.0
MLA-abs(r=48),mla_absorbed,16,32768,8,8,48,48.0,768.0,49152.0,1.00146484375,0.387085440158844,0.3861280083656311,0.40730495959520335,41334.54359180819,2080.4356983035455,0.9734318047109501,1.0
MLA(r=24)+RoPE,mla,1,1024,8,8,48,24.0,0.75,768.0,3.0,0.08235648006200791,0.0814560018479824,0.09198816329240797,12142.33536021791,9.549121082006891,0.5094962748598802,2.0
MLA-abs(r=24),mla_absorbed,1,1024,8,8,48,24.0,0.75,768.0,1.046875,0.08196288034319878,0.081216000020504,0.08992160260677337,12200.644923808846,9.59497758872084,0.5119429627956806,2.0
MLA(r=24)+RoPE,mla,1,4096,8,8,48,24.0,3.0,3072.0,3.0,0.08297407999634743,0.081216000020504,0.11621472202241412,12051.956466935466,37.91217691281996,0.507184895200215,2.0
MLA-abs(r=24),mla_absorbed,1,4096,8,8,48,24.0,3.0,3072.0,1.01171875,0.08151871994137765,0.08087999746203423,0.08895231917500496,12267.120984224575,38.5890259594628,0.5162397066286727,2.0
MLA(r=24)+RoPE,mla,1,8192,8,8,48,24.0,6.0,6144.0,3.0,0.0835392002761364,0.0830559991300106,0.08989727891981601,11970.428214473313,75.31142241251742,0.517574501252745,2.0
MLA-abs(r=24),mla_absorbed,1,8192,8,8,48,24.0,6.0,6144.0,1.005859375,0.08421888038516044,0.08193599805235863,0.12953888513147818,11873.822062543142,74.70362905831942,0.5133974676489889,2.0
MLA(r=24)+RoPE,mla,1,16384,8,8,48,24.0,12.0,12288.0,3.0,0.08618944019079208,0.08574399724602699,0.09767327852547168,11602.349403666663,145.99134153959008,0.5145948269810079,2.0
MLA-abs(r=24),mla_absorbed,1,16384,8,8,48,24.0,12.0,12288.0,1.0029296875,0.08268288001418114,0.08191999793052673,0.09168287761509418,12094.402128088517,152.18279767035057,0.5364186643590976,2.0
MLA(r=24)+RoPE,mla,1,32768,8,8,48,24.0,24.0,24576.0,3.0,0.12219007924199105,0.1218239963054657,0.13274783603847023,8183.970468007901,205.95636041908446,0.5681744383651538,2.0
MLA-abs(r=24),mla_absorbed,1,32768,8,8,48,24.0,24.0,24576.0,1.00146484375,0.08581952080130577,0.08503999933600426,0.09646624013781546,11652.36056625458,293.2412551949031,0.8089683908612033,2.0
MLA(r=24)+RoPE,mla,4,1024,8,8,48,24.0,3.0,768.0,3.0,0.105382399559021,0.10414400324225426,0.12008319705724714,37957.0024666191,29.850601363828194,0.3874711536603749,2.0
MLA-abs(r=24),mla_absorbed,4,1024,8,8,48,24.0,3.0,768.0,1.046875,0.09175103977322578,0.09095999971032143,0.0987312027812004,43596.23618311577,34.285475213960105,0.44503735362079216,2.0
MLA(r=24)+RoPE,mla,4,4096,8,8,48,24.0,12.0,3072.0,3.0,0.13290495932102203,0.1327200010418892,0.14237312242388722,30096.69481436203,94.67601558499344,0.32588701340436915,2.0
MLA-abs(r=24),mla_absorbed,4,4096,8,8,48,24.0,12.0,3072.0,1.01171875,0.09140928030014038,0.09063999727368355,0.10001567766070366,43759.23305452233,137.65464467813644,0.47382497835605997,2.0
MLA(r=24)+RoPE,mla,4,8192,8,8,48,24.0,24.0,6144.0,3.0,0.2421433612704277,0.24198399484157562,0.24892575830221175,16519.139649394587,103.92944026202147,0.19367039348679493,2.0
MLA-abs(r=24),mla_absorbed,4,8192,8,8,48,24.0,24.0,6144.0,1.005859375,0.09180671989917755,0.09092799946665764,0.10030047953128814,43569.79537437797,274.11745052690253,0.5108122815950752,2.0
MLA(r=24)+RoPE,mla,4,16384,8,8,48,24.0,48.0,12288.0,3.0,0.4591929614543915,0.4588000029325485,0.46702400147914885,8710.934913572914,109.60892745521558,0.17852677827089233,2.0
MLA-abs(r=24),mla_absorbed,4,16384,8,8,48,24.0,48.0,12288.0,1.0029296875,0.09158463940024376,0.09070399776101112,0.09880511969327926,43675.44629967013,549.5642973494749,0.8951090548586509,2.0
MLA(r=24)+RoPE,mla,4,32768,8,8,48,24.0,96.0,24576.0,3.0,0.876243200302124,0.8760479986667633,0.8838012701272965,4564.942699265251,114.88054453979423,0.16784965657227538,2.0
MLA-abs(r=24),mla_absorbed,4,32768,8,8,48,24.0,96.0,24576.0,1.00146484375,0.09530624017119407,0.09403200075030327,0.11740735843777653,41969.96957192929,1056.2088675325278,1.5432055653471943,2.0
MLA(r=24)+RoPE,mla,16,1024,8,8,48,24.0,12.0,768.0,3.0,0.1310303997993469,0.1308160051703453,0.14071904122829434,122109.0680063677,96.03047857038376,0.3359318157014177,2.0
MLA-abs(r=24),mla_absorbed,16,1024,8,8,48,24.0,12.0,768.0,1.046875,0.09288064032793045,0.09152000024914742,0.10922367997467515,172264.10093114513,135.47400142348235,0.4739123240458615,2.0
MLA(r=24)+RoPE,mla,16,4096,8,8,48,24.0,48.0,3072.0,3.0,0.44329984188079835,0.4431360065937042,0.45022880405187604,36092.95219262077,113.53861031498855,0.1534369145586138,2.0
MLA-abs(r=24),mla_absorbed,16,4096,8,8,48,24.0,48.0,3072.0,1.01171875,0.09144959971308708,0.09083199873566628,0.09874079905450343,174959.75980428798,550.3758152896233,0.7437819320796559,2.0
MLA(r=24)+RoPE,mla,16,8192,8,8,48,24.0,96.0,6144.0,3.0,0.8341657567024231,0.8340959846973419,0.8409113550186157,19180.84010454984,120.6754115608107,0.13126139342539125,2.0
MLA-abs(r=24),mla_absorbed,16,8192,8,8,48,24.0,96.0,6144.0,1.005859375,0.09508416041731835,0.09404800087213516,0.10971967980265615,168271.98063039116,1058.6757621689583,1.1515457368708393,2.0
MLA(r=24)+RoPE,mla,16,16384,8,8,48,24.0,192.0,12288.0,3.0,1.6088217568397523,1.607919991016388,1.6187644934654235,9945.166350453384,125.13915301311609,0.12356670273370725,2.0
MLA-abs(r=24),mla_absorbed,16,16384,8,8,48,24.0,192.0,12288.0,1.0029296875,0.1415782403945923,0.14126399904489517,0.15957440569996828,113011.71673984962,1422.0164867064548,1.4041479765878735,2.0
MLA(r=24)+RoPE,mla,16,32768,8,8,48,24.0,384.0,24576.0,3.0,3.1803743982315065,3.180079936981201,3.1882086324691774,5030.854231783853,126.60559216672766,0.11847701918386773,2.0
MLA-abs(r=24),mla_absorbed,16,32768,8,8,48,24.0,384.0,24576.0,1.00146484375,0.2669036787748337,0.2654560059309006,0.2902035224437713,59946.71963100959,1508.6085956113322,1.4117500377693728,2.0
MLA(r=12)+RoPE,mla,1,1024,8,8,48,12.0,0.375,384.0,5.0,0.08312384009361268,0.08233600109815598,0.09192383900284766,12030.243055106896,4.730484053156913,0.5047928458900602,4.0
MLA-abs(r=12),mla_absorbed,1,1024,8,8,48,12.0,0.375,384.0,1.046875,0.08215679988265037,0.08131200075149536,0.09453472048044204,12171.84702213769,4.7861649986568935,0.5107345936319276,4.0
MLA(r=12)+RoPE,mla,1,4096,8,8,48,12.0,1.5,1536.0,5.0,0.08248768016695976,0.08188799768686295,0.09137983851134776,12123.022468033325,19.067865611160766,0.5101755799423979,4.0
MLA-abs(r=12),mla_absorbed,1,4096,8,8,48,12.0,1.5,1536.0,1.01171875,0.08235072001814842,0.08118399977684021,0.09709440067410466,12143.18465921877,19.099577995837475,0.5110240694678497,4.0
MLA(r=12)+RoPE,mla,1,8192,8,8,48,12.0,3.0,3072.0,5.0,0.08419584035873413,0.08350400254130363,0.09133343845605849,11877.071310640635,37.362035779878944,0.5135379578581422,4.0
MLA-abs(r=12),mla_absorbed,1,8192,8,8,48,12.0,3.0,3072.0,1.005859375,0.08216256037354469,0.08153600245714188,0.08870144061744212,12170.993643011974,38.28663549064478,0.5262465010997452,4.0
MLA(r=12)+RoPE,mla,1,16384,8,8,48,12.0,6.0,6144.0,5.0,0.08639807999134064,0.08596799895167351,0.09430239871144294,11574.331282595937,72.8193959938759,0.5133521493419284,4.0
MLA-abs(r=12),mla_absorbed,1,16384,8,8,48,12.0,6.0,6144.0,1.0029296875,0.08206975996494294,0.0814560018479824,0.08918271712958813,12184.75599815525,76.65985623312984,0.5404260970364276,4.0
MLA(r=12)+RoPE,mla,1,32768,8,8,48,12.0,12.0,12288.0,5.0,0.1266470408439636,0.1263040006160736,0.1362694406509399,7895.960247757049,99.35417295302516,0.5481792482830121,4.0
MLA-abs(r=12),mla_absorbed,1,32768,8,8,48,12.0,12.0,12288.0,1.00146484375,0.08159744009375572,0.08113599941134453,0.08898975931107997,12255.28642627755,154.2071906366449,0.8508266872997735,4.0
MLA(r=12)+RoPE,mla,4,1024,8,8,48,12.0,1.5,384.0,5.0,0.10585472017526626,0.10494400188326836,0.11554848052561283,37787.639449399154,14.858704433734937,0.3857422688853633,4.0
MLA-abs(r=12),mla_absorbed,4,1024,8,8,48,12.0,1.5,384.0,1.046875,0.09205760017037391,0.09136000275611877,0.1003478403389454,43451.05664928342,17.085650691404627,0.4435553377131512,4.0
MLA(r=12)+RoPE,mla,4,4096,8,8,48,12.0,6.0,1536.0,5.0,0.12920896023511885,0.12904000282287598,0.14052544295787808,30957.605360505055,48.69210299774542,0.3352089528539127,4.0
MLA-abs(r=12),mla_absorbed,4,4096,8,8,48,12.0,6.0,1536.0,1.01171875,0.09263423979282379,0.09172799810767174,0.10056831791996955,43180.577818158694,67.91717634938036,0.4675592994191371,4.0
MLA(r=12)+RoPE,mla,4,8192,8,8,48,12.0,12.0,3072.0,5.0,0.23108735978603362,0.23076800256967545,0.23843808040022849,17309.47120475843,54.45088823400233,0.2029362406532335,4.0
MLA-abs(r=12),mla_absorbed,4,8192,8,8,48,12.0,12.0,3072.0,1.005859375,0.09218111991882325,0.09151999652385712,0.10161503814160822,43392.83362496018,136.50205173337875,0.508737582042359,4.0
MLA(r=12)+RoPE,mla,4,16384,8,8,48,12.0,24.0,6144.0,5.0,0.4330259209871292,0.43276798725128174,0.439752322435379,9237.322308284847,58.116206860392545,0.18931485631678663,4.0
MLA-abs(r=12),mla_absorbed,4,16384,8,8,48,12.0,24.0,6144.0,1.0029296875,0.0933900797367096,0.0926399976015091,0.10252991765737532,42831.10166815381,269.4699915767163,0.8778045831445918,4.0
MLA(r=12)+RoPE,mla,4,32768,8,8,48,12.0,48.0,12288.0,5.0,0.8324044787883759,0.8322080075740814,0.8399599987268448,4805.3561723049415,60.46537384476992,0.1766894868929397,4.0
MLA-abs(r=12),mla_absorbed,4,32768,8,8,48,12.0,48.0,12288.0,1.00146484375,0.09601024016737938,0.09481599926948547,0.11776736058294768,41662.222623613925,524.2320809973431,1.5318899316166301,4.0
MLA(r=12)+RoPE,mla,16,1024,8,8,48,12.0,6.0,384.0,5.0,0.12749632000923156,0.12723200023174286,0.14023392438888546,125493.81816543016,49.346177203737795,0.34524353419369397,4.0
MLA-abs(r=12),mla_absorbed,16,1024,8,8,48,12.0,6.0,384.0,1.046875,0.09373951956629753,0.09296000003814697,0.10216863885521887,170685.74784708553,67.11636702543959,0.4695701484318569,4.0
MLA(r=12)+RoPE,mla,16,4096,8,8,48,12.0,24.0,1536.0,5.0,0.4197670412063599,0.4195359945297241,0.4263734337687492,38116.37987112549,59.95188170961793,0.16203882936362493,4.0
MLA-abs(r=12),mla_absorbed,16,4096,8,8,48,12.0,24.0,1536.0,1.01171875,0.09300032004714012,0.09243200346827507,0.10093088068068026,172042.418691569,270.599326832896,0.7313798482417451,4.0
MLA(r=12)+RoPE,mla,16,8192,8,8,48,12.0,48.0,3072.0,5.0,0.7916153585910797,0.7911359965801239,0.8015776002407073,20211.83624895412,63.58093921974996,0.1383168711726152,4.0
MLA-abs(r=12),mla_absorbed,16,8192,8,8,48,12.0,48.0,3072.0,1.005859375,0.09578431993722916,0.09478399902582169,0.10840416163206099,167041.95436669973,525.4685530260496,1.1431282243718084,4.0
MLA(r=12)+RoPE,mla,16,16384,8,8,48,12.0,96.0,6144.0,5.0,1.5235603189468383,1.5237759947776794,1.5336736249923706,10501.717458131232,66.0710933122645,0.13048173892869344,4.0
MLA-abs(r=12),mla_absorbed,16,16384,8,8,48,12.0,96.0,6144.0,1.0029296875,0.1254329600930214,0.12481599673628807,0.1451897604763507,127558.17919097467,802.5266718201327,1.5848848630496326,4.0
MLA(r=12)+RoPE,mla,16,32768,8,8,48,12.0,192.0,12288.0,5.0,3.00061758518219,3.0001440048217773,3.0117018032073974,5332.235630095636,67.09505169675795,0.12557457519808463,4.0
MLA-abs(r=12),mla_absorbed,16,32768,8,8,48,12.0,192.0,12288.0,1.00146484375,0.22368575870990753,0.22307199984788895,0.2424540796875953,71528.91669223341,900.042064193704,1.6845117041171145,4.0
MLA(r=6)+RoPE,mla,1,1024,8,8,48,6.0,0.1875,192.0,9.0,0.08267839953303337,0.08156800270080566,0.09855616100132464,12095.057544025867,2.377985073615838,0.5075124825729141,8.0
MLA-abs(r=6),mla_absorbed,1,1024,8,8,48,6.0,0.1875,192.0,1.046875,0.08198783963918686,0.0811999998986721,0.08864928051829338,12196.930720467972,2.398014155089767,0.5117871136356873,8.0
MLA(r=6)+RoPE,mla,1,4096,8,8,48,6.0,0.75,768.0,9.0,0.08203519985079766,0.08134400099515915,0.09102688021957875,12189.889240457267,9.58651897515129,0.512989547704168,8.0
MLA-abs(r=6),mla_absorbed,1,4096,8,8,48,6.0,0.75,768.0,1.01171875,0.0814496000111103,0.08078399673104286,0.08851647950708866,12277.531134144218,9.655443364887304,0.5166777990504714,8.0
MLA(r=6)+RoPE,mla,1,8192,8,8,48,6.0,1.5,1536.0,9.0,0.08430976003408432,0.08375999704003334,0.09248832009732723,11861.022965736413,18.65577602598004,0.5128440633740926,8.0
MLA-abs(r=6),mla_absorbed,1,8192,8,8,48,6.0,1.5,1536.0,1.005859375,0.08155967995524406,0.08110399916768074,0.08846015892922877,12260.960324375363,19.28482309963833,0.5301364588691524,8.0
MLA(r=6)+RoPE,mla,1,16384,8,8,48,6.0,3.0,3072.0,9.0,0.08690687999129296,0.08555200323462486,0.09580095916986466,11506.568871189349,36.19653588202873,0.5103467074990402,8.0
MLA-abs(r=6),mla_absorbed,1,16384,8,8,48,6.0,3.0,3072.0,1.0029296875,0.08188479974865913,0.08134400099515915,0.08938176229596137,12212.278750994627,38.416507210808824,0.5416468037866435,8.0
MLA(r=6)+RoPE,mla,1,32768,8,8,48,6.0,6.0,6144.0,9.0,0.1242144002020359,0.12374400347471237,0.13448640197515485,8050.596375086065,50.64997286761347,0.5589149046663754,8.0
MLA-abs(r=6),mla_absorbed,1,32768,8,8,48,6.0,6.0,6144.0,1.00146484375,0.08291200041770935,0.08065600320696831,0.1121455995738506,12060.980255717095,75.88112659571286,0.8373369270714546,8.0
MLA(r=6)+RoPE,mla,4,1024,8,8,48,6.0,0.75,192.0,9.0,0.10591488018631935,0.10499200224876404,0.1161500810086727,37766.17594207188,7.425132319618869,0.3855231659687668,8.0
MLA-abs(r=6),mla_absorbed,4,1024,8,8,48,6.0,0.75,192.0,1.046875,0.09220031991600991,0.09121600165963173,0.10669440276920793,43383.797405950536,8.529601640389123,0.442868744596863,8.0
MLA(r=6)+RoPE,mla,4,4096,8,8,48,6.0,3.0,768.0,9.0,0.1241247995197773,0.12390399724245071,0.1381935960054397,32225.63110253132,25.343267519225915,0.3489391356709177,8.0
MLA-abs(r=6),mla_absorbed,4,4096,8,8,48,6.0,3.0,768.0,1.01171875,0.09201088026165963,0.09139200299978256,0.09915200233459473,43473.11957699828,34.18865237517791,0.47072694160284956,8.0
MLA(r=6)+RoPE,mla,4,8192,8,8,48,6.0,6.0,1536.0,9.0,0.2227385601401329,0.22244799882173538,0.2313718384504318,17958.273580844983,28.245922017462163,0.21054279971979215,8.0
MLA-abs(r=6),mla_absorbed,4,8192,8,8,48,6.0,6.0,1536.0,1.005859375,0.09181120008230209,0.09105600044131279,0.09931647978723049,43567.66926490766,68.52601855067971,0.5107873550876147,8.0
MLA(r=6)+RoPE,mla,4,16384,8,8,48,6.0,12.0,3072.0,9.0,0.4151891201734543,0.41492798924446106,0.4229603174328804,9634.16381992118,30.306458884913013,0.19744794848881003,8.0
MLA-abs(r=6),mla_absorbed,4,16384,8,8,48,6.0,12.0,3072.0,1.0029296875,0.09323199987411498,0.09110400080680847,0.12169663853943347,42903.724101177024,134.96344620934738,0.8792929479557702,8.0
MLA(r=6)+RoPE,mla,4,32768,8,8,48,6.0,24.0,6144.0,9.0,0.793672958612442,0.7931040227413177,0.8073657667636871,5039.859247558462,31.708052702207166,0.18531199614213162,8.0
MLA-abs(r=6),mla_absorbed,4,32768,8,8,48,6.0,24.0,6144.0,1.00146484375,0.09504511967301368,0.09372799843549728,0.11265311934053894,42085.275012134334,264.7776559867426,1.5474452633706652,8.0
MLA(r=6)+RoPE,mla,16,1024,8,8,48,6.0,3.0,192.0,9.0,0.12243583992123604,0.1223360002040863,0.13638943731784817,130680.6896599307,25.692869032659658,0.35951303266260887,8.0
MLA-abs(r=6),mla_absorbed,16,1024,8,8,48,6.0,3.0,192.0,1.046875,0.09244416013360024,0.09144000336527824,0.10313983879983424,173077.45537280897,34.02841234593722,0.47614992718916527,8.0
MLA(r=6)+RoPE,mla,16,4096,8,8,48,6.0,12.0,768.0,9.0,0.4013868808746338,0.40112000703811646,0.40948832660913465,39861.79111069981,31.34858810676987,0.16945885180476408,8.0
MLA-abs(r=6),mla_absorbed,16,4096,8,8,48,6.0,12.0,768.0,1.01171875,0.0925113607943058,0.0918239988386631,0.10104160197079182,172951.73114548784,136.0147758282083,0.735245480971216,8.0
MLA(r=6)+RoPE,mla,16,8192,8,8,48,6.0,24.0,1536.0,9.0,0.7547545611858368,0.7544959783554077,0.7637052643299103,21198.9444288505,33.34305653013951,0.14507200778021695,8.0
MLA-abs(r=6),mla_absorbed,16,8192,8,8,48,6.0,24.0,1536.0,1.005859375,0.09530496045947075,0.09440000355243683,0.10424000017344952,167882.1325024749,264.0557624563727,1.1488778657965986,8.0
MLA(r=6)+RoPE,mla,16,16384,8,8,48,6.0,48.0,3072.0,9.0,1.4532364797592163,1.4531359672546387,1.462404829263687,11009.908038264362,34.63417599339327,0.13679590524171023,8.0
MLA-abs(r=6),mla_absorbed,16,16384,8,8,48,6.0,48.0,3072.0,1.0029296875,0.11835583984851837,0.11790399998426437,0.13578592337667936,135185.55586676692,425.256988285653,1.6796534926656337,8.0
MLA(r=6)+RoPE,mla,16,32768,8,8,48,6.0,96.0,6144.0,9.0,2.8673907136917114,2.8668640851974487,2.8783778309822083,5579.985986423281,35.10623631419867,0.1314091158878141,8.0
MLA-abs(r=6),mla_absorbed,16,32768,8,8,48,6.0,96.0,6144.0,1.00146484375,0.20886592030525208,0.2083040028810501,0.22667072236537927,76604.167767611,481.95175092654284,1.8040342725154528,8.0
MLA(r=3)+RoPE,mla,1,1024,8,8,48,3.0,0.09375,96.0,17.0,0.08201792016625405,0.08131200075149536,0.09189151860773562,12192.457428485806,1.198567335049869,0.5115994128735472,16.0
MLA-abs(r=3),mla_absorbed,1,1024,8,8,48,3.0,0.09375,96.0,1.046875,0.08136512026190758,0.08073599636554718,0.08922944083809851,12290.27864496584,1.208183551914722,0.5157040224004862,16.0
MLA(r=3)+RoPE,mla,1,4096,8,8,48,3.0,0.375,384.0,17.0,0.08207104012370109,0.08126400038599968,0.09132384188473223,12184.565938152557,4.791166279936595,0.5127655261082603,16.0
MLA-abs(r=3),mla_absorbed,1,4096,8,8,48,3.0,0.375,384.0,1.01171875,0.0811686398088932,0.08086399734020233,0.08761215977370737,12320.029045139123,4.844432541013425,0.5184662471413115,16.0
MLA(r=3)+RoPE,mla,1,8192,8,8,48,3.0,0.75,768.0,17.0,0.08326143994927407,0.08275199681520462,0.09007359996438025,12010.361586458712,9.445332683161896,0.5193011308033648,16.0
MLA-abs(r=3),mla_absorbed,1,8192,8,8,48,3.0,0.75,768.0,1.005859375,0.08188032031059266,0.0814560018479824,0.089865278378129,12212.946849826043,9.604652217002394,0.5280604637837611,16.0
MLA(r=3)+RoPE,mla,1,16384,8,8,48,3.0,1.5,1536.0,17.0,0.0860377599298954,0.08471999689936638,0.10439519725739954,11622.803764472854,18.28108962020383,0.5155020318835548,16.0
MLA-abs(r=3),mla_absorbed,1,16384,8,8,48,3.0,1.5,1536.0,1.0029296875,0.08184767961502075,0.08108799904584885,0.08925472117960452,12217.817349295743,19.2169650672827,0.5418924552435441,16.0
MLA(r=3)+RoPE,mla,1,32768,8,8,48,3.0,3.0,3072.0,17.0,0.16303616046905517,0.1626880019903183,0.17366495862603185,6133.608624755387,19.294664391934514,0.42582749401958014,16.0
MLA-abs(r=3),mla_absorbed,1,32768,8,8,48,3.0,3.0,3072.0,1.00146484375,0.08129791989922523,0.0806720033288002,0.08905631996691227,12300.437714022373,38.69383132925616,0.853961328077885,16.0
MLA(r=3)+RoPE,mla,4,1024,8,8,48,3.0,0.375,96.0,17.0,0.10538176000118256,0.10473600029945374,0.11249183930456637,37957.23282620364,3.7313478157471223,0.3874735052078675,16.0
MLA-abs(r=3),mla_absorbed,4,1024,8,8,48,3.0,0.375,96.0,1.046875,0.09169344037771225,0.09105600044131279,0.09963616192340849,43623.62218630715,4.288376555402738,0.445316914322669,16.0
MLA(r=3)+RoPE,mla,4,4096,8,8,48,3.0,1.5,384.0,17.0,0.128860160112381,0.12822400033473969,0.14309152349829668,31041.401752966445,12.205975831694454,0.33611630019692634,16.0
MLA-abs(r=3),mla_absorbed,4,4096,8,8,48,3.0,1.5,384.0,1.01171875,0.09161919996142387,0.09097599983215332,0.0989433604478836,43658.97106375295,17.16740596580468,0.4727393415134982,16.0
MLA(r=3)+RoPE,mla,4,8192,8,8,48,3.0,3.0,768.0,17.0,0.25655487954616546,0.25619199872016907,0.2665222418308258,15591.206088443252,12.261423386546602,0.182791300404872,16.0
MLA-abs(r=3),mla_absorbed,4,8192,8,8,48,3.0,3.0,768.0,1.005859375,0.0905868798494339,0.08991999924182892,0.09811072155833243,44156.504856425934,34.726088427248754,0.5176908635710334,16.0
MLA(r=3)+RoPE,mla,4,16384,8,8,48,3.0,6.0,1536.0,17.0,0.48014783978462217,0.47987200319767,0.4897529625892639,8330.767460693487,13.103164231296201,0.17073541359656055,16.0
MLA-abs(r=3),mla_absorbed,4,16384,8,8,48,3.0,6.0,1536.0,1.0029296875,0.09216319933533669,0.09150399640202522,0.10092864081263542,43401.27110221035,68.26429687090696,0.8894899548228999,16.0
MLA(r=3)+RoPE,mla,4,32768,8,8,48,3.0,12.0,3072.0,17.0,0.9135571241378784,0.9134080111980438,0.9206137579679489,4378.4891982258805,13.773536068556702,0.1609938955741814,16.0
MLA-abs(r=3),mla_absorbed,4,32768,8,8,48,3.0,12.0,3072.0,1.00146484375,0.09484991982579231,0.09380799904465675,0.11437439769506451,42171.8859367163,132.6612824039347,1.5506298847129725,16.0
MLA(r=3)+RoPE,mla,16,1024,8,8,48,3.0,1.5,96.0,17.0,0.12842111945152282,0.12803199887275696,0.14093375846743578,124590.09910780116,12.247705102693285,0.34275733076204173,16.0
MLA-abs(r=3),mla_absorbed,16,1024,8,8,48,3.0,1.5,96.0,1.046875,0.09248896047472954,0.09174399822950363,0.10134720005095003,172993.6191073488,17.005964732728817,0.4759192869153717,16.0
MLA(r=3)+RoPE,mla,16,4096,8,8,48,3.0,6.0,384.0,17.0,0.46840063989162445,0.46807999908924103,0.47853375196456904,34158.79193440465,13.43178352927886,0.14521448983982763,16.0
MLA-abs(r=3),mla_absorbed,16,4096,8,8,48,3.0,6.0,384.0,1.01171875,0.0916806399822235,0.09097599983215332,0.10208767868578433,174518.85156018037,68.62360473508788,0.7419075605896684,16.0
MLA(r=3)+RoPE,mla,16,8192,8,8,48,3.0,12.0,768.0,17.0,0.8766547214984893,0.8761920034885406,0.8865148901939391,18251.19925510784,14.353327132592968,0.12489952644679235,16.0
MLA-abs(r=3),mla_absorbed,16,8192,8,8,48,3.0,12.0,768.0,1.005859375,0.09421440035104751,0.0939520001411438,0.10193984121084212,169825.41883600818,133.5561437860396,1.1621764737081253,16.0
MLA(r=3)+RoPE,mla,16,16384,8,8,48,3.0,24.0,1536.0,17.0,1.6973632049560547,1.6974560022354126,1.7053785812854767,9426.385556893372,14.826422492557539,0.11712095513705047,16.0
MLA-abs(r=3),mla_absorbed,16,16384,8,8,48,3.0,24.0,1536.0,1.0029296875,0.1199251201748848,0.11907199770212173,0.13842368096113197,133416.58508799048,209.84614368783704,1.657674384557933,16.0
MLA(r=3)+RoPE,mla,16,32768,8,8,48,3.0,48.0,3072.0,17.0,3.358778238296509,3.357792019844055,3.3746882939338683,4763.636913437552,14.985106020434085,0.1121840299829561,16.0
MLA-abs(r=3),mla_absorbed,16,32768,8,8,48,3.0,48.0,3072.0,1.00146484375,0.21906880050897598,0.2184479981660843,0.23798496291041368,73036.41578730618,229.7526981617711,1.7200134282732662,16.0