Update Feather H200 training runtime image

Dockerfile

@@ -22,7 +22,29 @@ RUN pip install --upgrade pip setuptools wheel && \
       rustbpe \
       pandas \
       tiktoken \
-      pydantic
+      pydantic \
+      ninja \
+      packaging \
+      einops \
+      triton==3.2.0
+
+# Mamba-3 fused CUDA kernel stack (mandatory — NO fallback allowed).
+# causal-conv1d must be installed BEFORE mamba-ssm. CUDA_HOME is required so the
+# build can find nvcc; TORCH_CUDA_ARCH_LIST targets Hopper (H100/H200) sm_90a
+# plus Ada (L4/A10G) sm_89 so the same image runs on any HF Jobs flavor we pick.
+ENV CUDA_HOME=/usr/local/cuda \
+    TORCH_CUDA_ARCH_LIST="8.9;9.0" \
+    MAMBA_FORCE_BUILD=TRUE \
+    CAUSAL_CONV1D_FORCE_BUILD=TRUE \
+    MAX_JOBS=4
+
+RUN pip install --no-build-isolation 'causal-conv1d>=1.4.0' && \
+    pip install --no-build-isolation 'mamba-ssm[causal-conv1d]>=2.2.5' && \
+    python -c "from mamba_ssm import Mamba3; print('mamba_ssm Mamba3 import OK')"
+
+# Optional tilelang for MIMO path — wrap in || true because some flavors may not
+# need it; single-pass SISO Mamba3 still runs without it.
+RUN pip install tilelang || echo "[dockerfile] tilelang optional install failed — continuing"
 
 WORKDIR /workspace
 COPY overlay /workspace/feather

README.md

@@ -1,10 +1,11 @@
 ---
-title: Feather H200 Runtime
-emoji: 😻
-colorFrom: green
-colorTo: purple
+title: Feather H200 Runtime Slim
+emoji: 📚
+colorFrom: blue
+colorTo: indigo
 sdk: docker
+app_port: 7860
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Feather runtime image used as a Docker Space source for Hugging Face Jobs.

entrypoint.py

@@ -1,7 +1,13 @@
 #!/usr/bin/env python3
+from __future__ import annotations
+
+import json
 import os
 import subprocess
+import time
+from http.server import BaseHTTPRequestHandler, HTTPServer
 from pathlib import Path
+from threading import Thread
 
 from huggingface_hub import HfApi
 
@@ -11,6 +17,38 @@ LOG_FILE = REPO_ROOT / 'run_domain_expanded.log'
 JOB_ID = os.environ.get('JOB_ID', 'local-job')
 OUTPUT_REPO = os.environ.get('HF_REPO_ID', 'icarus112/feather-pretrain-checkpoints')
 TOKEN = os.environ.get('HF_TOKEN')
+RUNTIME_MODE = os.environ.get('FEATHER_RUNTIME_MODE', 'space')
+APP_PORT = int(os.environ.get('PORT', '7860'))
+
+
+class _HealthHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path in ('/', '/health', '/healthz', '/ready'):
+            payload = {
+                'status': 'ok',
+                'mode': RUNTIME_MODE,
+                'job_id': JOB_ID,
+            }
+            body = json.dumps(payload).encode('utf-8')
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/json')
+            self.send_header('Content-Length', str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+            return
+        self.send_response(404)
+        self.end_headers()
+
+    def log_message(self, format, *args):
+        return
+
+
+def _start_health_server() -> HTTPServer:
+    server = HTTPServer(('0.0.0.0', APP_PORT), _HealthHandler)
+    thread = Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    print(f'[space] health server listening on 0.0.0.0:{APP_PORT}', flush=True)
+    return server
 
 
 def upload_artifact(api: HfApi, path: Path, dest: str) -> None:
@@ -26,7 +64,7 @@ def upload_artifact(api: HfApi, path: Path, dest: str) -> None:
     print(f'[upload] uploaded {path} -> {OUTPUT_REPO}/{dest}', flush=True)
 
 
-def main() -> int:
+def run_job_mode() -> int:
     os.chdir(REPO_ROOT)
     os.environ.setdefault('HYDRA_TIME_BUDGET', '43200')
     os.environ.setdefault('HYDRA_TARGET_SHARDS', '2048')
@@ -35,6 +73,7 @@ def main() -> int:
     os.environ.setdefault('HYDRA_RESUME_CKPT', str(CACHE_ROOT / 'latest.pt'))
 
     cmd = [
+        'bash',
         './scripts/run_domain_expanded_pretrain.sh',
         '--target-shards', os.environ['HYDRA_TARGET_SHARDS'],
         '--download-workers', os.environ['HYDRA_DOWNLOAD_WORKERS'],
@@ -62,5 +101,22 @@ def main() -> int:
     return proc.returncode
 
 
+def run_space_mode() -> int:
+    server = _start_health_server()
+    print('[space] Feather runtime image ready', flush=True)
+    try:
+        while True:
+            time.sleep(3600)
+    finally:
+        server.shutdown()
+        server.server_close()
+
+
+def main() -> int:
+    if RUNTIME_MODE == 'job':
+        return run_job_mode()
+    return run_space_mode()
+
+
 if __name__ == '__main__':
     raise SystemExit(main())

overlay/hydra/model.py

@@ -35,6 +35,7 @@ import os
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
+
 from mamba_ssm import Mamba3
 
 from subsystems.hestia_mini import HestiaQAT
@@ -70,7 +71,7 @@ class PostSemClawModel(nn.Module):
         # Token embedding
         self.wte = nn.Embedding(config.vocab_size, config.d_model)
 
-        # Mamba-3 blocks (official mamba-ssm with CUDA parallel scan kernel)
+        # Mamba-3 blocks (official mamba-ssm with CUDA parallel scan kernel — no fallbacks, no bypasses).
         self.blocks = nn.ModuleList([
             Mamba3(
                 d_model=config.d_model,

overlay/subsystems/sdr_retina.py

@@ -61,7 +61,7 @@ TARGET_ACTIVE = int(N_BITS * TARGET_SPARSITY)  # 327
 
 CONTEXT_WINDOW = 8                # +/- 8 tokens
 TOP_K_FEATURES = 64               # top-K context features per token
-TARGET_TRAIN_TOKENS = 10_000_000  # scan this many tokens from parquet
+TARGET_TRAIN_TOKENS = 10000000  # scan this many tokens from parquet
 MAX_DOCS_PER_SHARD = 200_000      # safety cap per shard
 
 # Kohonen SOM
@@ -498,105 +498,4 @@ def build_retina(target_tokens: int = TARGET_TRAIN_TOKENS) -> BuildReport:
     )
 
 
-# ---------------------------------------------------------------------------
-# Validation
-# ---------------------------------------------------------------------------
-
-def _jaccard(a: np.ndarray, b: np.ndarray) -> float:
-    inter = int(np.logical_and(a, b).sum())
-    uni = int(np.logical_or(a, b).sum())
-    if uni == 0:
-        return 0.0
-    return inter / uni
-
-
-def _token_id_for_word(tokenizer: Tokenizer, word: str) -> int:
-    """Return the single-token id for a leading-space word if possible.
-    We try " word" first (BPE merges usually favor space-prefixed single tokens),
-    then bare "word", then take the first token of the encoding as a fallback.
-    """
-    for candidate in (" " + word, word):
-        ids = tokenizer.enc.encode_ordinary(candidate)
-        if len(ids) == 1:
-            return ids[0]
-    # Fallback: use the first token id of the bare word, with a warning.
-    ids = tokenizer.enc.encode_ordinary(word)
-    return ids[0]
-
-
-def validate_retina() -> bool:
-    print("[validate] loading retina...")
-    with np.load(RETINA_PATH) as f:
-        sdr = f["sdr"]
-        vocab_size = int(f["vocab_size"])
-        n_bits = int(f["n_bits"])
-    assert sdr.shape == (vocab_size, n_bits), f"bad shape {sdr.shape}"
-    per_row = sdr.sum(axis=1)
-    print(f"[validate] shape={sdr.shape}  active/row min/max={int(per_row.min())}/{int(per_row.max())}  "
-          f"mean={per_row.mean():.1f}")
-
-    tokenizer = Tokenizer.from_directory(TOKENIZER_DIR)
-
-    pairs = [
-        ("the", "a", "zebra"),
-        ("man", "woman", "rock"),
-        ("king", "queen", "dinosaur"),
-    ]
-    all_pass = True
-    print(f"\n{'pair':<30}{'close':>10}{'far':>10}{'passed?':>10}")
-    print("-" * 60)
-    for anchor, close_w, far_w in pairs:
-        a_id = _token_id_for_word(tokenizer, anchor)
-        c_id = _token_id_for_word(tokenizer, close_w)
-        f_id = _token_id_for_word(tokenizer, far_w)
-        j_close = _jaccard(sdr[a_id], sdr[c_id])
-        j_far = _jaccard(sdr[a_id], sdr[f_id])
-        passed = j_close > j_far
-        all_pass = all_pass and passed
-        print(f"{anchor!r:<10} vs {close_w!r:<8} / {far_w!r:<8}"
-              f"{j_close:>10.4f}{j_far:>10.4f}{'PASS' if passed else 'FAIL':>10}")
-        # also print the ids we used, for debuggability
-        print(f"    token ids: {anchor!r}={a_id}  {close_w!r}={c_id}  {far_w!r}={f_id}")
-
-    if all_pass:
-        print("\n[validate] ALL 3 overlap tests PASSED")
-    else:
-        print("\n[validate] ONE OR MORE TESTS FAILED")
-    return all_pass
-
-
-# ---------------------------------------------------------------------------
-# Entry point
-# ---------------------------------------------------------------------------
-
-def main():
-    parser = argparse.ArgumentParser(description="Build / validate the semantic-folding SDR retina.")
-    parser.add_argument("--build", action="store_true", help="Build retina.npz from the corpus.")
-    parser.add_argument("--validate", action="store_true", help="Run analogy overlap tests against retina.npz.")
-    parser.add_argument("--target-tokens", type=int, default=TARGET_TRAIN_TOKENS,
-                        help="Approximate number of tokens to stream for cooccurrence.")
-    parser.add_argument("--epochs", type=int, default=SOM_EPOCHS, help="Kohonen SOM epochs.")
-    parser.add_argument("--top-k", type=int, default=TOP_K_FEATURES,
-                        help="Top-K context features per token (PMI).")
-    args = parser.parse_args()
-
-    if not args.build and not args.validate:
-        parser.error("specify --build and/or --validate")
-
-    # Override module-level knobs used inside the pipeline with CLI values.
-    globals()["SOM_EPOCHS"] = int(args.epochs)
-    globals()["TOP_K_FEATURES"] = int(args.top_k)
-
-    if args.build:
-        build_retina(target_tokens=args.target_tokens)
-
-    ok = True
-    if args.validate:
-        ok = validate_retina()
-
-    if not ok:
-        sys.exit(1)
-
 
-if __name__ == "__main__":
-    main()