recipes: qwen38-flash-next-nvfp4

recipes/qwen38-flash-next-nvfp4/recipe.toml

@@ -0,0 +1,90 @@
+# Qwen3.8-Flash-Next (NVFP4, SGLang) — 2× DGX Spark / GB10 (sm_121), TP=2.
+#
+# Qwen4-architecture preview: 176B total (6B active) GDN + QSA hybrid MoE,
+# 512 routed experts (top-10) + shared expert, 51B n-gram "engram" table
+# (PLE, offloaded to host RAM), in-checkpoint MTP draft. Text + vision.
+# Native 262144 ctx (YaRN to 1M). Always-thinking.
+#
+# QUANT: NVFP4 (RadixArk), NOT the official FP8. FP8 cannot serve on 2×
+# Spark: its MoE weight_block_size=[128,128] + moe_intermediate_size=640
+# violates SGLang's (intermediate/tp) % block_n == 0 at tp=2 (sgl-project/
+# sglang #21810) — only tp in {1,5} align, and tp=1 can't hold 125B. NVFP4
+# is the community-verified 2× Spark path. SGLang's own cookbook lists the
+# FP8 quant's supported HW as H200/B200/B300/GB300 — not GB10.
+#
+# Image: qwen38-flashnext-nvfp4:local — built by the launcher from
+#   docker-images/qwen38-flashnext-nvfp4/ (base lmsysorg/sglang:qwen38flashnext
+#   + a SM121 QSA decode kernel fix). The stock day-0 image cannot serve
+#   qwen4_exp on GB10: QSA's packed varlen decode resolves to flash-attn-4
+#   CuTe-DSL kernels that fail MLIR compilation on SM121; the overlay swaps in
+#   a CUDA-graph-safe Triton fallback. Provenance/validation:
+#     https://github.com/MiaAI-Lab/Qwen3.8-Flash-Next-Dual-DGX-Sparks
+#
+# Model: RadixArk/Qwen3.8-Flash-Next-NVFP4 (~135 GB)
+#   https://huggingface.co/RadixArk/Qwen3.8-Flash-Next-NVFP4
+#
+# REFERENCE:
+#   https://forums.developer.nvidia.com/t/qwen3-8-flash-next/381228
+
+engine = "sglang"
+name = "Qwen3.8-Flash-Next NVFP4 (SGLang, 2× Spark)"
+docker_image = "qwen38-flashnext-nvfp4:local"
+docker_image_dir = "qwen38-flashnext-nvfp4"
+model = "RadixArk/Qwen3.8-Flash-Next-NVFP4"
+parent_model = "Qwen/Qwen3.8-Flash-Next"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["python3", "-m", "sglang.launch_server"]
+
+args = [
+    # Stable served name — quant-agnostic, shared with qwen38-flash-next-fp8 so
+    # swapping quants never changes what agents/clients pin against.
+    "--served-model-name", "qwen3.8-flash-next",
+    "--trust-remote-code",
+    # NVFP4 (ModelOpt) weights
+    "--quantization", "modelopt_fp4",
+    # flashinfer is required on SM120/SM121 (trtllm_mha is SM100-only)
+    "--attention-backend", "flashinfer",
+    # page 64 is pinned by SGLang for compressed QSA (full//ratio addressing)
+    "--page-size", "64",
+    # GDN (DeltaNet) hybrid state pool
+    "--mamba-ssm-dtype", "bfloat16",
+    "--mamba-radix-cache-strategy", "extra_buffer",
+    "--mamba-track-interval", "64",
+    # Memory — 0.70 leaves DRAM for the ~26 GB/rank pinned PLE (engram) table
+    "--mem-fraction-static", "0.70",
+    "--chunked-prefill-size", "4096",
+    "--max-running-requests", "16",
+    "--context-length", "262144",
+    # NEXTN speculative decoding — draft = the in-checkpoint MTP layer
+    "--speculative-algorithm", "NEXTN",
+    "--speculative-num-steps", "3",
+    "--speculative-eagle-topk", "1",
+    "--speculative-num-draft-tokens", "4",
+    # Parsers (auto = resolve from the checkpoint chat template)
+    "--reasoning-parser", "auto",
+    "--tool-call-parser", "auto",
+    "--allow-auto-truncate",
+    # Observability
+    "--enable-metrics",
+    "--enable-cache-report",
+    # SM121 house rule: prefill CUDA graphs unreliable on GB10
+    "--disable-prefill-cuda-graph",
+    # Pin decode-graph batch list (default list OOMs the graph pool on GB10)
+    "--cuda-graph-bs-decode", "1", "2", "3", "4", "5", "6", "7", "8", "10", "12", "14", "16",
+]
+
+[env]
+# Offline — weights must be cached on each node
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"