recipes: qwen38-27b-nvfp4-sglang-dspark

recipes/qwen38-27b-nvfp4-sglang-dspark/recipe.toml

@@ -0,0 +1,90 @@
+# Qwen3.8-27B (RadixArk NVFP4 + DSpark on SGLang) — 1× DGX Spark / GB10 (sm_121),
+# TP=1.
+#
+# SGLang + DSpark block-diffusion speculative decoding. The community (MiaAI)
+# measured DSpark on this box at ~51.5 tok/s on code vs vLLM+MTP's 22.8 — the
+# drafter is 3× cheaper per guess and edits/rewrites much faster. Uses the
+# pre-built cookbook image (no source build): lmsysorg/sglang:qwen38-27b.
+#
+# TARGET MUST BE RadixArk MODELOPT NVFP4: the cookbook image accelerates
+# modelopt-quantized NVFP4. The unsloth/RedHatAI compressed-tensors builds
+# fall back to a slow unquantized path ("Falling back to UnquantizedLinear-
+# Method") — prefill crawls at ~0.5 tok/s. Use RadixArk here.
+#
+# Draft: RadixArk/Qwen3.8-27B-DSpark (bf16, block 7). SGLang loads it natively
+# from the HF cache — no arch patch needed (unlike the vLLM qwen3_dspark path).
+#
+# Image: lmsysorg/sglang:qwen38-27b  (multi-arch, incl. arm64; cookbook's
+#   DGX Spark Qwen3.8 cell, pinned image digest febfb971). Pre-built — no
+#   source build.
+#
+# Model: RadixArk/Qwen3.8-27B-NVFP4  (modelopt, cached locally on the node)
+#   https://huggingface.co/RadixArk/Qwen3.8-27B-NVFP4
+#
+# PREREQUISITES:
+#   Target + draft cached on the node (dgx1 ~/.cache/huggingface → NAS):
+#       hfrs download RadixArk/Qwen3.8-27B-NVFP4
+#       hfrs download RadixArk/Qwen3.8-27B-DSpark
+#
+# REFERENCE:
+#   https://github.com/MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark  (start-dspark.sh)
+#   https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark
+
+engine = "sglang"
+name = "Qwen3.8-27B (RadixArk NVFP4, SGLang DSpark)"
+docker_image = "lmsysorg/sglang:qwen38-27b"
+model = "RadixArk/Qwen3.8-27B-NVFP4"
+parent_model = "Qwen/Qwen3.8-27B"
+required_nodes = 1
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["python3", "-m", "sglang.launch_server"]
+
+args = [
+    # Memory — larger KV pool (mamba GDN state pool sized from concurrency)
+    "--mem-fraction-static", "0.90",
+    "--max-running-requests", "10",
+    "--max-mamba-cache-size", "30",
+    # Attention / KV
+    "--attention-backend", "flashinfer",
+    "--kv-cache-dtype", "fp8_e4m3",
+    # Mamba GDN hybrid
+    "--mamba-ssm-dtype", "bfloat16",
+    "--mamba-full-memory-ratio", "4.21",
+    "--mamba-radix-cache-strategy", "extra_buffer_lazy",
+    # Chunked prefill
+    "--chunked-prefill-size", "8192",
+    # DSpark speculative decoding (block 7 / 8 draft tokens, unquant bf16 draft)
+    "--speculative-algorithm", "DSPARK",
+    "--speculative-draft-model-path", "RadixArk/Qwen3.8-27B-DSpark",
+    "--speculative-dspark-block-size", "7",
+    "--speculative-draft-model-quantization", "unquant",
+    "--speculative-num-draft-tokens", "8",
+    # Compile + decode graphs
+    "--enable-torch-compile", "--torch-compile-max-bs", "4",
+    "--cuda-graph-max-bs-decode", "4",
+    "--num-continuous-decode-steps", "2",
+    # Tokenizer / reasoning / tool calling (Qwen3 family)
+    "--reasoning-parser", "qwen3",
+    "--tool-call-parser", "qwen3_coder",
+    # Observability
+    "--enable-metrics",
+    "--enable-cache-report",
+]
+
+[env]
+# Offline — target + draft are both cached on the node
+HF_HUB_OFFLINE = "1"
+
+# Pin to GB10's Cortex-X5 big cores (scheduler/tokenizer off the A725 efficiency
+# cores — +2-7% decode per the MiaAI build)
+# CPUSET handled at the container layer; this is the SGLang-side hint.
+SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK = "1"
+
+# Architecture
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"