Add DSpark drafter for Inkling-Small-NVFP4 + acceptance eval

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+acceptance_by_position.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,84 @@
+---
+library_name: speculators
+tags:
+- speculative-decoding
+- dspark
+- draft-model
+- vllm
+base_model: thinkingmachines/Inkling-Small-NVFP4
+---
+
+# DSpark drafter for Inkling-Small-NVFP4
+
+A [DSpark](https://github.com/neuralmagic/speculators) speculative-decoding drafter
+trained for the verifier
+[`thinkingmachines/Inkling-Small-NVFP4`](https://huggingface.co/thinkingmachines/Inkling-Small-NVFP4)
+(multimodal MoE, NVFP4). DSpark drafts a whole block in one parallel pass
+(DFlash-style backbone) and injects intra-block dependency with a lightweight
+sequential Markov head, plus a per-position confidence head.
+
+## Model details
+
+| | |
+|---|---|
+| Algorithm | DSpark (greedy proposal) |
+| Verifier | `thinkingmachines/Inkling-Small-NVFP4` |
+| Backbone layers | 6 |
+| Hidden size | 4096 |
+| Attention heads / KV heads | 32 / 8 |
+| Draft vocab size | 201024 (full padded vocab) |
+| Block size | 16 |
+| Markov rank | 256 (vanilla) |
+| Confidence head | enabled (with Markov) |
+| Aux hidden-state layers | 2, 10, 18, 26, 34, 39 |
+| dtype | bfloat16 |
+
+## Acceptance evaluation
+
+Inference acceptance measured with vLLM spec-decode over 9 task subsets
+(~300k drafts total). **Mean acceptance length (AL)** = expected number of
+tokens accepted per draft step (higher is better; block size 16).
+
+| Subset | Acceptance length | pos-0 accept |
+|---|---|---|
+| math_reasoning | **4.55** | 0.776 |
+| HumanEval | **3.90** | 0.740 |
+| rag | 3.73 | 0.736 |
+| translation | 3.56 | 0.736 |
+| summarization | 3.40 | 0.731 |
+| tool_call | 3.20 | 0.707 |
+| writing | 3.11 | 0.679 |
+| qa | 3.10 | 0.684 |
+| question | 3.09 | 0.679 |
+
+**Weighted mean acceptance length ≈ 3.40** across all subsets.
+
+Per-position acceptance rate by subset:
+
+![Per-position acceptance rate](./acceptance_by_position.png)
+
+Full per-position numbers are in [`acceptance_eval.csv`](./acceptance_eval.csv).
+
+## Usage (vLLM)
+
+Serve the verifier with this drafter as the speculative model:
+
+```bash
+vllm serve thinkingmachines/Inkling-Small-NVFP4 \
+    --speculative-config '{"model": "<this-repo>", "num_speculative_tokens": 16}' \
+    --enforce-eager \
+    --trust-remote-code
+```
+
+> **Note:** Inkling currently requires `--enforce-eager` (CUDA-graph capture of
+> the custom ops crashes). The drafter uses the full padded vocab (draft
+> `vocab_size` 201024 vs. the verifier's logical 200058); loading it in vLLM
+> needs [vllm-project/vllm#55133](https://github.com/vllm-project/vllm/pull/55133),
+> which treats a padded/full-vocab draft as an identity map (no `d2t` required).
+
+## Training
+
+Trained with [speculators](https://github.com/neuralmagic/speculators) for 3
+epochs (`checkpoint_best` = best validation epoch). Validation at the selected
+checkpoint: accept_len 3.67, accept_rate 0.40, pos-0 acc 0.776. Loss
+`{"ce": 0.1, "tv": 0.9}`, lr 1e-4, seq len 8192, block size 16.

acceptance_eval.csv

@@ -0,0 +1,10 @@
+subset,num_drafts,num_draft_tokens,num_accepted_tokens,acceptance_length,acceptance_at_pos_0,acceptance_at_pos_1,acceptance_at_pos_2,acceptance_at_pos_3,acceptance_at_pos_4,acceptance_at_pos_5,acceptance_at_pos_6,acceptance_at_pos_7,acceptance_at_pos_8,acceptance_at_pos_9,acceptance_at_pos_10,acceptance_at_pos_11,acceptance_at_pos_12,acceptance_at_pos_13,acceptance_at_pos_14,acceptance_at_pos_15
+HumanEval,53377.0,854032.0,154985.0,3.9035914345129923,0.740487475879124,0.5334320025479139,0.38891282762238416,0.2877643929033104,0.21533619349157876,0.16514603668246622,0.12870712104464468,0.10127957734604792,0.08095247016505236,0.06463458043726698,0.05305655994154786,0.04256514978361466,0.03460291886018323,0.027895910223504506,0.021750941416715065,0.017067276167637747
+math_reasoning,7168.0,114688.0,25469.0,4.553152901785714,0.7759486607142857,0.6106305803571429,0.4755859375,0.3755580357142857,0.2972935267857143,0.23158482142857142,0.18233816964285715,0.14104352678571427,0.11481584821428571,0.091796875,0.0712890625,0.056640625,0.04450334821428571,0.03655133928571429,0.0283203125,0.019252232142857144
+qa,55900.0,894400.0,117644.0,3.1045438282647586,0.6840608228980322,0.4641323792486583,0.31783542039355994,0.21556350626118068,0.1460107334525939,0.0981216457960644,0.06474060822898033,0.04255813953488372,0.02745974955277281,0.017173524150268335,0.010894454382826477,0.006708407871198569,0.004114490161001789,0.002701252236135957,0.00148479427549195,0.0009838998211091235
+question,40020.0,640320.0,83773.0,3.0932783608195904,0.6787606196901549,0.445552223888056,0.2939030484757621,0.1973263368315842,0.13555722138930534,0.09317841079460269,0.06664167916041978,0.04832583708145927,0.034907546226886556,0.026586706646676662,0.020889555222388806,0.016391804097951025,0.012693653173413293,0.009645177411294352,0.007296351824087956,0.005622188905547227
+rag,25011.0,400176.0,68386.0,3.734236935748271,0.7355563552037103,0.536803806325217,0.3943864699532206,0.28811323017872137,0.21246651473351724,0.15561153092639238,0.11267042501299428,0.08344328495461997,0.062172644036623884,0.0477389948422694,0.034824677142057496,0.025268881692055496,0.017992083483267363,0.012594458438287154,0.008556235256487146,0.006037343568829715
+summarization,13314.0,213024.0,31998.0,3.4033348355114916,0.7314856541985879,0.5298182364428421,0.37546943067447797,0.265209553853087,0.17823343848580442,0.11904761904761904,0.07683641279855791,0.0501727504882079,0.030118672074508038,0.01892744479495268,0.011566771819137749,0.007210455159981974,0.00405588102748986,0.0026288117770767614,0.0016523959741625357,0.0009013068949977468
+tool_call,42535.0,680560.0,93512.0,3.1984718467144706,0.707299870694722,0.4844481015634184,0.32829434583284356,0.2216762666039732,0.14844245915128718,0.09991771482308687,0.06686258375455507,0.04495121664511579,0.031150816974256496,0.021958387210532504,0.015399083108028682,0.010344422240507817,0.0072410955683554715,0.005007640766427648,0.003244386975431997,0.0022334548019278242
+translation,23167.0,370672.0,59377.0,3.5629990935382225,0.7360469633530453,0.5348556135882937,0.3816635731859973,0.26930547761902707,0.18819873095351147,0.13247291405879053,0.09353822247161911,0.06651702853196356,0.04825829844174904,0.034402382699529506,0.025424094617343636,0.01834505978331247,0.013078948504338068,0.009194112314930721,0.007122199680580136,0.004575473734190875
+writing,38853.0,621648.0,82092.0,3.112887035750135,0.6792783054075618,0.447481532957558,0.2956528453401282,0.19774534784958692,0.1360770082104342,0.09505057524515481,0.0681800633155741,0.050189174581113424,0.03747458368723136,0.02874938872159164,0.022237665045170257,0.017012843281085117,0.01310066146758294,0.010243739222196483,0.008339124391938846,0.006074177026227061

config.json

@@ -0,0 +1,85 @@
+{
+  "architectures": [
+    "DSparkDraftModel"
+  ],
+  "auto_map": {
+    "": "config.DSparkSpeculatorConfig"
+  },
+  "aux_hidden_state_layer_ids": [
+    2,
+    10,
+    18,
+    26,
+    34,
+    39
+  ],
+  "block_size": 16,
+  "confidence_head_with_markov": true,
+  "draft_vocab_size": 201024,
+  "dtype": "bfloat16",
+  "enable_confidence_head": true,
+  "markov_head_type": "vanilla",
+  "markov_rank": 256,
+  "mask_token_id": 200058,
+  "sample_from_anchor": true,
+  "sliding_window_non_causal": false,
+  "speculators_config": {
+    "algorithm": "dspark",
+    "default_proposal_method": "greedy",
+    "proposal_methods": [
+      {
+        "accept_tolerance": 0.0,
+        "proposal_type": "greedy",
+        "speculative_tokens": 16,
+        "verifier_accept_k": 1
+      }
+    ],
+    "verifier": {
+      "architectures": [
+        "InklingForConditionalGeneration"
+      ],
+      "name_or_path": "thinkingmachines/Inkling-Small-NVFP4"
+    }
+  },
+  "speculators_model_type": "dspark",
+  "speculators_version": "0.7.0.dev155",
+  "target_hidden_size": null,
+  "tie_word_embeddings": false,
+  "transformer_layer_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "eos_token_id": null,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "initializer_range": 0.02,
+    "intermediate_size": 16384,
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention"
+    ],
+    "max_position_embeddings": 1048576,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 6,
+    "num_key_value_heads": 8,
+    "pad_token_id": null,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    },
+    "sliding_window": 2048,
+    "tie_word_embeddings": false,
+    "use_cache": true,
+    "use_sliding_window": true,
+    "vocab_size": 201024
+  },
+  "transformers_version": "5.15.0"
+}

config.py

@@ -0,0 +1,67 @@
+from typing import Literal
+
+from pydantic import Field
+
+from speculators import SpeculatorModelConfig
+from speculators.models.dflash.config import DFlashSpeculatorConfig
+
+__all__ = [
+    "DSparkSpeculatorConfig",
+]
+
+
+@SpeculatorModelConfig.register("dspark")
+class DSparkSpeculatorConfig(DFlashSpeculatorConfig):
+    """DFlash config plus a Markov logit-bias head and a confidence head.
+
+    The Markov head lets each draft position condition on previously sampled
+    tokens within the block; the confidence head predicts the per-position
+    acceptance probability. All DFlash fields are inherited unchanged.
+    """
+
+    speculators_model_type: Literal["dspark"] = "dspark"  # type: ignore[assignment]
+    architectures: list[str] = Field(
+        default_factory=lambda: ["DSparkSpeculator"],
+        description="Model architectures that can load these weights",
+    )
+
+    sample_from_anchor: bool = Field(
+        default=True,
+        description=(
+            "Whether to sample from the anchor position. "
+            "False: anchor is the bonus token, only mask tokens predict "
+            "(block_size-1 speculative tokens). "
+            "True: sample from anchor and all mask positions "
+            "(block_size speculative tokens). "
+            "Default True matches DeepSeek/DeepSpec convention."
+        ),
+    )
+
+    # Sequential (Markov) head.
+    markov_rank: int = Field(
+        default=256,
+        description=(
+            "Low-rank dimension of the Markov logit-bias factorization B = W1 @ W2. "
+            "Set to 0 to disable the sequential head (pure DFlash drafting)."
+        ),
+    )
+    markov_head_type: Literal["vanilla", "gated", "rnn"] = Field(
+        default="vanilla",
+        description=(
+            "Sequential head variant: 'vanilla' (first-order Markov bias), 'gated' "
+            "(hidden-gated bias), or 'rnn' (recurrent state over the block)."
+        ),
+    )
+
+    # Confidence head.
+    enable_confidence_head: bool = Field(
+        default=True,
+        description="Whether to attach the per-position acceptance-probability head.",
+    )
+    confidence_head_with_markov: bool = Field(
+        default=True,
+        description=(
+            "Concatenate the Markov previous-token embedding with the backbone "
+            "hidden state as the confidence-head input."
+        ),
+    )

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f452ea94a09757cece52e723abf058ce0210e90f479a39231f95d81e19a3e5a
+size 6620122506