launcher/recipes/qwen38-27b-nvfp4-sglang-dspark/recipe.toml

# Qwen3.8-27B (RadixArk NVFP4 + DSpark on SGLang) — 1× DGX Spark / GB10 (sm_121),
# TP=1.
#
# SGLang + DSpark block-diffusion speculative decoding. The community (MiaAI)
# measured DSpark on this box at ~51.5 tok/s on code vs vLLM+MTP's 22.8 — the
# drafter is 3× cheaper per guess and edits/rewrites much faster. Uses the
# pre-built cookbook image (no source build): lmsysorg/sglang:qwen38-27b.
#
# TARGET MUST BE RadixArk MODELOPT NVFP4: the cookbook image accelerates
# modelopt-quantized NVFP4. The unsloth/RedHatAI compressed-tensors builds
# fall back to a slow unquantized path ("Falling back to UnquantizedLinear-
# Method") — prefill crawls at ~0.5 tok/s. Use RadixArk here.
#
# Draft: RadixArk/Qwen3.8-27B-DSpark (bf16, block 7). SGLang loads it natively
# from the HF cache — no arch patch needed (unlike the vLLM qwen3_dspark path).
#
# Image: lmsysorg/sglang:qwen38-27b  (multi-arch, incl. arm64; cookbook's
#   DGX Spark Qwen3.8 cell, pinned image digest febfb971). Pre-built — no
#   source build.
#
# Model: RadixArk/Qwen3.8-27B-NVFP4  (modelopt, cached locally on the node)
#   https://huggingface.co/RadixArk/Qwen3.8-27B-NVFP4
#
# PREREQUISITES:
#   Target + draft cached on the node (dgx1 ~/.cache/huggingface → NAS):
#       hfrs download RadixArk/Qwen3.8-27B-NVFP4
#       hfrs download RadixArk/Qwen3.8-27B-DSpark
#
# REFERENCE:
#   https://github.com/MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark  (start-dspark.sh)
#   https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark

engine = "sglang"
name = "Qwen3.8-27B (RadixArk NVFP4, SGLang DSpark)"
docker_image = "lmsysorg/sglang:qwen38-27b"
model = "RadixArk/Qwen3.8-27B-NVFP4"
parent_model = "Qwen/Qwen3.8-27B"
required_nodes = 1
tensor_parallel_size = 1
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

entrypoint = ["python3", "-m", "sglang.launch_server"]

args = [
    # Memory — larger KV pool (mamba GDN state pool sized from concurrency)
    "--mem-fraction-static", "0.90",
    "--max-running-requests", "10",
    "--max-mamba-cache-size", "30",
    # Attention / KV
    "--attention-backend", "flashinfer",
    "--kv-cache-dtype", "fp8_e4m3",
    # Mamba GDN hybrid
    "--mamba-ssm-dtype", "bfloat16",
    "--mamba-full-memory-ratio", "4.21",
    "--mamba-radix-cache-strategy", "extra_buffer_lazy",
    # Chunked prefill
    "--chunked-prefill-size", "8192",
    # DSpark speculative decoding (block 7 / 8 draft tokens, unquant bf16 draft)
    "--speculative-algorithm", "DSPARK",
    "--speculative-draft-model-path", "RadixArk/Qwen3.8-27B-DSpark",
    "--speculative-dspark-block-size", "7",
    "--speculative-draft-model-quantization", "unquant",
    "--speculative-num-draft-tokens", "8",
    # Compile + decode graphs
    "--enable-torch-compile", "--torch-compile-max-bs", "4",
    "--cuda-graph-max-bs-decode", "4",
    "--num-continuous-decode-steps", "2",
    # Tokenizer / reasoning / tool calling (Qwen3 family)
    "--reasoning-parser", "qwen3",
    "--tool-call-parser", "qwen3_coder",
    # Observability
    "--enable-metrics",
    "--enable-cache-report",
]

[env]
# Offline — target + draft are both cached on the node
HF_HUB_OFFLINE = "1"

# Pin to GB10's Cortex-X5 big cores (scheduler/tokenizer off the A725 efficiency
# cores — +2-7% decode per the MiaAI build)
# CPUSET handled at the container layer; this is the SGLang-side hint.
SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK = "1"

# Architecture
TORCH_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
OMP_NUM_THREADS = "4"