Hugging Face's logo

MinimaLabs
/
KLM-1.1

Text Generation
Transformers
Safetensors
English
qwen3
klm
keylm
small-language-model
base
pretrained
gqa
rope
swiglu
qk-norm
text-generation-inference
Model card Files
xet
 Community
KLM-1.1 / charts /progress.svg
Eclipse-Senpai's picture
Eclipse-Senpai
Progress chart: leader lines for the crowded labels
d8cce58 verified
preview code
|
Raw History Blame Contribute Delete
8.16 kB
<svg xmlns="http://www.w3.org/2000/svg" width="1000" height="420" viewBox="0 0 1000 420" font-family="'Helvetica Neue', Helvetica, Arial, sans-serif">
<rect width="1000" height="420" rx="14" fill="#FFFFFF"/>
<text x="32.0" y="46.0" font-size="22" fill="#15171C" text-anchor="start" font-weight="700">Training progress</text>
<text x="32.0" y="72.0" font-size="13" fill="#6B7180" text-anchor="start" font-weight="400">Left: Int Index across every stage from KeyLM-75M to KLM 1.1. Right: held-out loss during the 95B-token run.</text>
<rect x="523.8" y="112" width="71.1" height="238" fill="#EEF1FE"/>
<text x="529.8" y="342.0" font-size="11" fill="#2747E8" text-anchor="start" font-weight="400">LR decay</text>
<line x1="76" x2="600" y1="350.0" y2="350.0" stroke="#E4E7EC"/>
<text x="66.0" y="354.0" font-size="11" fill="#6B7180" text-anchor="end" font-weight="400">10</text>
<line x1="76" x2="600" y1="290.5" y2="290.5" stroke="#E4E7EC"/>
<text x="66.0" y="294.5" font-size="11" fill="#6B7180" text-anchor="end" font-weight="400">12</text>
<line x1="76" x2="600" y1="231.0" y2="231.0" stroke="#E4E7EC"/>
<text x="66.0" y="235.0" font-size="11" fill="#6B7180" text-anchor="end" font-weight="400">14</text>
<line x1="76" x2="600" y1="171.5" y2="171.5" stroke="#E4E7EC"/>
<text x="66.0" y="175.5" font-size="11" fill="#6B7180" text-anchor="end" font-weight="400">16</text>
<line x1="76" x2="600" y1="112.0" y2="112.0" stroke="#E4E7EC"/>
<text x="66.0" y="116.0" font-size="11" fill="#6B7180" text-anchor="end" font-weight="400">18</text>
<text x="76.0" y="370.0" font-size="11" fill="#6B7180" text-anchor="middle" font-weight="400">0B</text>
<text x="150.9" y="370.0" font-size="11" fill="#6B7180" text-anchor="middle" font-weight="400">20B</text>
<text x="225.7" y="370.0" font-size="11" fill="#6B7180" text-anchor="middle" font-weight="400">40B</text>
<text x="300.6" y="370.0" font-size="11" fill="#6B7180" text-anchor="middle" font-weight="400">60B</text>
<text x="375.4" y="370.0" font-size="11" fill="#6B7180" text-anchor="middle" font-weight="400">80B</text>
<text x="450.3" y="370.0" font-size="11" fill="#6B7180" text-anchor="middle" font-weight="400">100B</text>
<text x="525.1" y="370.0" font-size="11" fill="#6B7180" text-anchor="middle" font-weight="400">120B</text>
<text x="600.0" y="370.0" font-size="11" fill="#6B7180" text-anchor="middle" font-weight="400">140B</text>
<text x="338.0" y="392.0" font-size="12" fill="#6B7180" text-anchor="middle" font-weight="400">Total training tokens</text>
<text x="30.0" y="231.0" font-size="12" fill="#6B7180" text-anchor="middle" font-weight="400" transform="rotate(-90 30 231.0)">Int Index</text>
<polyline points="146.1,336.0 164.8,231.3 185.0,220.3 200.2,214.9 220.9,196.8 224.7,192.6 232.2,200.4 239.6,185.8 331.9,187.3 392.7,184.0 428.0,154.5 459.4,151.6 498.7,162.6 532.0,163.5 543.8,156.9 559.5,150.1 569.3,156.3 594.8,129.9" fill="none" stroke="#2747E8" stroke-width="2.5" stroke-linejoin="round"/>
<circle cx="146.1" cy="336.0" r="5" fill="#2747E8" stroke="#2747E8" stroke-width="2"/>
<text x="156.1" y="354.0" font-size="12" fill="#15171C" text-anchor="start" font-weight="400">KeyLM-75M 路 10.47</text>
<circle cx="164.8" cy="231.3" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="185.0" cy="220.3" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="200.2" cy="214.9" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="220.9" cy="196.8" r="5" fill="#2747E8" stroke="#2747E8" stroke-width="2"/>
<line x1="220.9" y1="202.8" x2="257.9" y2="244.8" stroke="#6B7180" stroke-width="1"/>
<text x="260.9" y="258.8" font-size="12" fill="#15171C" text-anchor="start" font-weight="400">24K tokenizer, +20B 路 15.15</text>
<circle cx="224.7" cy="192.6" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="232.2" cy="200.4" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="239.6" cy="185.8" r="5" fill="#2747E8" stroke="#2747E8" stroke-width="2"/>
<line x1="239.6" y1="179.8" x2="266.6" y2="132.8" stroke="#6B7180" stroke-width="1"/>
<text x="269.6" y="127.8" font-size="12" fill="#15171C" text-anchor="start" font-weight="400">start of the 95B run 路 15.52</text>
<circle cx="331.9" cy="187.3" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="392.7" cy="184.0" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="428.0" cy="154.5" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="459.4" cy="151.6" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="498.7" cy="162.6" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="532.0" cy="163.5" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="543.8" cy="156.9" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="559.5" cy="150.1" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="569.3" cy="156.3" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="594.8" cy="129.9" r="5" fill="#2747E8" stroke="#2747E8" stroke-width="2"/>
<text x="582.8" y="134.9" font-size="12" fill="#15171C" text-anchor="end" font-weight="700">KLM 1.1 路 17.40</text>
<rect x="898.6" y="112" width="54.7" height="238" fill="#EEF1FE"/>
<line x1="680" x2="968" y1="350.0" y2="350.0" stroke="#E4E7EC"/>
<text x="670.0" y="354.0" font-size="11" fill="#6B7180" text-anchor="end" font-weight="400">1.8</text>
<line x1="680" x2="968" y1="282.0" y2="282.0" stroke="#E4E7EC"/>
<text x="670.0" y="286.0" font-size="11" fill="#6B7180" text-anchor="end" font-weight="400">1.9</text>
<line x1="680" x2="968" y1="214.0" y2="214.0" stroke="#E4E7EC"/>
<text x="670.0" y="218.0" font-size="11" fill="#6B7180" text-anchor="end" font-weight="400">2.0</text>
<line x1="680" x2="968" y1="146.0" y2="146.0" stroke="#E4E7EC"/>
<text x="670.0" y="150.0" font-size="11" fill="#6B7180" text-anchor="end" font-weight="400">2.1</text>
<text x="680.0" y="370.0" font-size="11" fill="#6B7180" text-anchor="middle" font-weight="400">0B</text>
<text x="752.0" y="370.0" font-size="11" fill="#6B7180" text-anchor="middle" font-weight="400">25B</text>
<text x="824.0" y="370.0" font-size="11" fill="#6B7180" text-anchor="middle" font-weight="400">50B</text>
<text x="896.0" y="370.0" font-size="11" fill="#6B7180" text-anchor="middle" font-weight="400">75B</text>
<text x="968.0" y="370.0" font-size="11" fill="#6B7180" text-anchor="middle" font-weight="400">100B</text>
<text x="824.0" y="392.0" font-size="12" fill="#6B7180" text-anchor="middle" font-weight="400">Tokens into the 95B run</text>
<text x="634.0" y="231.0" font-size="12" fill="#6B7180" text-anchor="middle" font-weight="400" transform="rotate(-90 634 231.0)">Held-out loss (nats/token)</text>
<polyline points="680.0,124.9 751.0,221.5 797.8,233.0 825.0,241.9 849.1,244.6 879.3,248.2 905.0,294.4 914.0,306.2 926.1,315.8 933.7,323.1 953.3,340.2" fill="none" stroke="#2747E8" stroke-width="2.5" stroke-linejoin="round"/>
<circle cx="680.0" cy="124.9" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="751.0" cy="221.5" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="797.8" cy="233.0" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="825.0" cy="241.9" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="849.1" cy="244.6" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="879.3" cy="248.2" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="905.0" cy="294.4" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="914.0" cy="306.2" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="926.1" cy="315.8" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="933.7" cy="323.1" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<circle cx="953.3" cy="340.2" r="3" fill="#FFFFFF" stroke="#2747E8" stroke-width="2"/>
<text x="688.0" y="128.9" font-size="12" fill="#15171C" text-anchor="start" font-weight="400">2.131</text>
<text x="949.3" y="362.2" font-size="12" fill="#15171C" text-anchor="end" font-weight="700">1.814</text>
</svg>