launcher/recipes/qwen38-flash-next-nvfp4/recipe.toml

# Qwen3.8-Flash-Next (NVFP4, SGLang) — 2× DGX Spark / GB10 (sm_121), TP=2.
#
# Qwen4-architecture preview: 176B total (6B active) GDN + QSA hybrid MoE,
# 512 routed experts (top-10) + shared expert, 51B n-gram "engram" table
# (PLE, offloaded to host RAM), in-checkpoint MTP draft. Text + vision.
# Native 262144 ctx (YaRN to 1M). Always-thinking.
#
# QUANT: NVFP4 (RadixArk), NOT the official FP8. FP8 cannot serve on 2×
# Spark: its MoE weight_block_size=[128,128] + moe_intermediate_size=640
# violates SGLang's (intermediate/tp) % block_n == 0 at tp=2 (sgl-project/
# sglang #21810) — only tp in {1,5} align, and tp=1 can't hold 125B. NVFP4
# is the community-verified 2× Spark path. SGLang's own cookbook lists the
# FP8 quant's supported HW as H200/B200/B300/GB300 — not GB10.
#
# Image: qwen38-flashnext-nvfp4:local — built by the launcher from
#   docker-images/qwen38-flashnext-nvfp4/ (base lmsysorg/sglang:qwen38flashnext
#   + a SM121 QSA decode kernel fix). The stock day-0 image cannot serve
#   qwen4_exp on GB10: QSA's packed varlen decode resolves to flash-attn-4
#   CuTe-DSL kernels that fail MLIR compilation on SM121; the overlay swaps in
#   a CUDA-graph-safe Triton fallback. Provenance/validation:
#     https://github.com/MiaAI-Lab/Qwen3.8-Flash-Next-Dual-DGX-Sparks
#
# Model: RadixArk/Qwen3.8-Flash-Next-NVFP4 (~135 GB)
#   https://huggingface.co/RadixArk/Qwen3.8-Flash-Next-NVFP4
#
# REFERENCE:
#   https://forums.developer.nvidia.com/t/qwen3-8-flash-next/381228

engine = "sglang"
name = "Qwen3.8-Flash-Next NVFP4 (SGLang, 2× Spark)"
docker_image = "qwen38-flashnext-nvfp4:local"
docker_image_dir = "qwen38-flashnext-nvfp4"
model = "RadixArk/Qwen3.8-Flash-Next-NVFP4"
parent_model = "Qwen/Qwen3.8-Flash-Next"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

entrypoint = ["python3", "-m", "sglang.launch_server"]

args = [
    # Stable served name — quant-agnostic, shared with qwen38-flash-next-fp8 so
    # swapping quants never changes what agents/clients pin against.
    "--served-model-name", "qwen3.8-flash-next",
    "--trust-remote-code",
    # NVFP4 (ModelOpt) weights
    "--quantization", "modelopt_fp4",
    # flashinfer is required on SM120/SM121 (trtllm_mha is SM100-only)
    "--attention-backend", "flashinfer",
    # page 64 is pinned by SGLang for compressed QSA (full//ratio addressing)
    "--page-size", "64",
    # GDN (DeltaNet) hybrid state pool
    "--mamba-ssm-dtype", "bfloat16",
    "--mamba-radix-cache-strategy", "extra_buffer",
    "--mamba-track-interval", "64",
    # Memory — 0.70 leaves DRAM for the ~26 GB/rank pinned PLE (engram) table
    "--mem-fraction-static", "0.70",
    "--chunked-prefill-size", "4096",
    "--max-running-requests", "16",
    "--context-length", "262144",
    # NEXTN speculative decoding — draft = the in-checkpoint MTP layer
    "--speculative-algorithm", "NEXTN",
    "--speculative-num-steps", "3",
    "--speculative-eagle-topk", "1",
    "--speculative-num-draft-tokens", "4",
    # Parsers (auto = resolve from the checkpoint chat template)
    "--reasoning-parser", "auto",
    "--tool-call-parser", "auto",
    "--allow-auto-truncate",
    # Observability
    "--enable-metrics",
    "--enable-cache-report",
    # SM121 house rule: prefill CUDA graphs unreliable on GB10
    "--disable-prefill-cuda-graph",
    # Pin decode-graph batch list (default list OOMs the graph pool on GB10)
    "--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "10", "12", "14", "16",
]

[env]
# Offline — weights must be cached on each node
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
OMP_NUM_THREADS = "4"