Text Generation
Transformers
Safetensors
PyTorch
ONNX
English
qtensorformer
q-tensorformer
high-throughput-inference
tensor-train
tensor-networks
tensorrt-llm
vllm
continuous-batching
paged-attention
speculative-decoding
zero-svd
adaptive-computation
dynamic-neural-networks
kv-cache
kv-cache-quantization
attention-sinks
lyapunov-stability
control-theory
contextual-bandits
edge-ai
edge-inference
on-device-ai
model-compression
hopper-h100
blackwell-b200
jetson-orin
jetson-thor
grouped-query-attention
rotary-position-embeddings
custom_code
academic-paper
ieee
neurips
vit-ap-university
deepseek
deepseek-r1
reasoning
chain-of-thought
gsm8k
openai-compatible
fastapi
rest-api
sse-streaming
onnx-runtime
embedded-ai
hardware-telemetry
power-efficiency
green-ai
raspberry-pi
smollm2
qwen2.5
llama-3.2
apple-silicon
Premchandyadav369
feat: complete scientific repair, canonical checkpoint training, tokenizer, GQA zero-copy, and release gating
d279df0 | { | |
| "cache_config": { | |
| "n_heads": 4, | |
| "head_dim": 32, | |
| "sink_tokens_protected": 4, | |
| "eviction_capacity": 128 | |
| }, | |
| "sequence_length_sweep": { | |
| "L_64": { | |
| "seq_len": 64, | |
| "fp16_full": { | |
| "bytes": 32768, | |
| "update_ms": 0.201, | |
| "k_mse": 8e-08, | |
| "attn_out_cosine_sim": 1.0 | |
| }, | |
| "int8_full": { | |
| "bytes": 17408, | |
| "compression_ratio_vs_fp16": 1.88, | |
| "update_ms": 0.936, | |
| "k_mse": 5.3e-05, | |
| "attn_out_cosine_sim": 0.999963 | |
| }, | |
| "int4_packed_full": { | |
| "bytes": 10240, | |
| "compression_ratio_vs_fp16": 3.2, | |
| "update_ms": 1.794, | |
| "k_mse": 0.011081, | |
| "attn_out_cosine_sim": 0.996616 | |
| }, | |
| "sink_protected_int8_cap128": { | |
| "retained_tokens": 64, | |
| "evicted_tokens": 0, | |
| "bytes": 17408, | |
| "memory_reduction_vs_fp16_pct": 46.88, | |
| "update_ms": 0.754, | |
| "attn_out_cosine_sim_vs_full": 0.999963 | |
| }, | |
| "sliding_window_no_sink_cap128": { | |
| "retained_tokens": 64, | |
| "attn_out_cosine_sim_vs_full": 1.0 | |
| }, | |
| "scientific_classification": "MEASURED" | |
| }, | |
| "L_128": { | |
| "seq_len": 128, | |
| "fp16_full": { | |
| "bytes": 65536, | |
| "update_ms": 0.165, | |
| "k_mse": 6e-08, | |
| "attn_out_cosine_sim": 1.0 | |
| }, | |
| "int8_full": { | |
| "bytes": 34816, | |
| "compression_ratio_vs_fp16": 1.88, | |
| "update_ms": 1.005, | |
| "k_mse": 4.2e-05, | |
| "attn_out_cosine_sim": 0.999981 | |
| }, | |
| "int4_packed_full": { | |
| "bytes": 20480, | |
| "compression_ratio_vs_fp16": 3.2, | |
| "update_ms": 2.056, | |
| "k_mse": 0.009272, | |
| "attn_out_cosine_sim": 0.996034 | |
| }, | |
| "sink_protected_int8_cap128": { | |
| "retained_tokens": 128, | |
| "evicted_tokens": 0, | |
| "bytes": 34816, | |
| "memory_reduction_vs_fp16_pct": 46.88, | |
| "update_ms": 0.893, | |
| "attn_out_cosine_sim_vs_full": 0.999981 | |
| }, | |
| "sliding_window_no_sink_cap128": { | |
| "retained_tokens": 128, | |
| "attn_out_cosine_sim_vs_full": 1.0 | |
| }, | |
| "scientific_classification": "MEASURED" | |
| }, | |
| "L_256": { | |
| "seq_len": 256, | |
| "fp16_full": { | |
| "bytes": 131072, | |
| "update_ms": 0.233, | |
| "k_mse": 5e-08, | |
| "attn_out_cosine_sim": 1.0 | |
| }, | |
| "int8_full": { | |
| "bytes": 69632, | |
| "compression_ratio_vs_fp16": 1.88, | |
| "update_ms": 1.15, | |
| "k_mse": 3.6e-05, | |
| "attn_out_cosine_sim": 0.999962 | |
| }, | |
| "int4_packed_full": { | |
| "bytes": 40960, | |
| "compression_ratio_vs_fp16": 3.2, | |
| "update_ms": 2.259, | |
| "k_mse": 0.007604, | |
| "attn_out_cosine_sim": 0.995489 | |
| }, | |
| "sink_protected_int8_cap128": { | |
| "retained_tokens": 128, | |
| "evicted_tokens": 128, | |
| "bytes": 34816, | |
| "memory_reduction_vs_fp16_pct": 73.44, | |
| "update_ms": 2.132, | |
| "attn_out_cosine_sim_vs_full": 0.985176 | |
| }, | |
| "sliding_window_no_sink_cap128": { | |
| "retained_tokens": 128, | |
| "attn_out_cosine_sim_vs_full": 0.119727 | |
| }, | |
| "scientific_classification": "MEASURED" | |
| }, | |
| "L_512": { | |
| "seq_len": 512, | |
| "fp16_full": { | |
| "bytes": 262144, | |
| "update_ms": 0.454, | |
| "k_mse": 5e-08, | |
| "attn_out_cosine_sim": 1.0 | |
| }, | |
| "int8_full": { | |
| "bytes": 139264, | |
| "compression_ratio_vs_fp16": 1.88, | |
| "update_ms": 1.695, | |
| "k_mse": 3.3e-05, | |
| "attn_out_cosine_sim": 0.999982 | |
| }, | |
| "int4_packed_full": { | |
| "bytes": 81920, | |
| "compression_ratio_vs_fp16": 3.2, | |
| "update_ms": 2.986, | |
| "k_mse": 0.006855, | |
| "attn_out_cosine_sim": 0.996796 | |
| }, | |
| "sink_protected_int8_cap128": { | |
| "retained_tokens": 128, | |
| "evicted_tokens": 384, | |
| "bytes": 34816, | |
| "memory_reduction_vs_fp16_pct": 86.72, | |
| "update_ms": 1.206, | |
| "attn_out_cosine_sim_vs_full": 0.996568 | |
| }, | |
| "sliding_window_no_sink_cap128": { | |
| "retained_tokens": 128, | |
| "attn_out_cosine_sim_vs_full": 0.149448 | |
| }, | |
| "scientific_classification": "MEASURED" | |
| }, | |
| "L_1024": { | |
| "seq_len": 1024, | |
| "fp16_full": { | |
| "bytes": 524288, | |
| "update_ms": 0.583, | |
| "k_mse": 5e-08, | |
| "attn_out_cosine_sim": 1.0 | |
| }, | |
| "int8_full": { | |
| "bytes": 278528, | |
| "compression_ratio_vs_fp16": 1.88, | |
| "update_ms": 2.118, | |
| "k_mse": 3e-05, | |
| "attn_out_cosine_sim": 0.999973 | |
| }, | |
| "int4_packed_full": { | |
| "bytes": 163840, | |
| "compression_ratio_vs_fp16": 3.2, | |
| "update_ms": 4.185, | |
| "k_mse": 0.006429, | |
| "attn_out_cosine_sim": 0.995593 | |
| }, | |
| "sink_protected_int8_cap128": { | |
| "retained_tokens": 128, | |
| "evicted_tokens": 896, | |
| "bytes": 34816, | |
| "memory_reduction_vs_fp16_pct": 93.36, | |
| "update_ms": 1.263, | |
| "attn_out_cosine_sim_vs_full": 0.996337 | |
| }, | |
| "sliding_window_no_sink_cap128": { | |
| "retained_tokens": 128, | |
| "attn_out_cosine_sim_vs_full": 0.004714 | |
| }, | |
| "scientific_classification": "MEASURED" | |
| } | |
| } | |
| } |