| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475 |
- # Qwen3.8-27B (RadixArk NVFP4 + DFlash2 on SGLang) — 1× DGX Spark / GB10,
- # TP=1.
- #
- # SGLang + DFlash2 block-diffusion speculative decoding. Per the community
- # (MiaAI / forum 380350), DFlash2 on NVFP4 ties DSpark on code and *beats it*
- # on prose/essay and short-chat once counted right. This is the fork/engine
- # the forum's high tok/s numbers (52-61 code single, 87 TP=2) came from — NOT
- # the stock DSpark path.
- #
- # Image: lmsysorg/sglang:qwen38-27b-dflash2-minoverlay — a locally-built overlay
- # of the 5 DFlash2 modules onto the cookbook qwen38-27b image. The cookbook
- # tag itself predates DFlash2 (upstream merged commit c14312a66 on
- # 2026-08-19, after every published tag), so the modules are overlaid.
- # The minoverlay modules already bake in the quantized-head selector fix
- # (should_apply_lm_head_quant_method) so DFLASH runs the NVFP4 lm_head in
- # place — no dense dequant.
- #
- # Draft: z-lab/Qwen3.8-27B-DFlash2 @ 50307d4 (block-diffusion drafter, 3.6 GB).
- #
- # Model: RadixArk/Qwen3.8-27B-NVFP4 (modelopt)
- #
- # PREREQUISITES (on the node):
- # hfrs download RadixArk/Qwen3.8-27B-NVFP4
- # hfrs download z-lab/Qwen3.8-27B-DFlash2
- # docker build the minoverlay image (or use the recipe's docker_image as-is)
- #
- # REFERENCE:
- # https://github.com/MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark (start-dflash.sh)
- # https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
- engine = "sglang"
- name = "Qwen3.8-27B (RadixArk NVFP4, SGLang DFlash2)"
- docker_image = "lmsysorg/sglang:qwen38-27b-dflash2-minoverlay"
- model = "RadixArk/Qwen3.8-27B-NVFP4"
- required_nodes = 1
- tensor_parallel_size = 1
- pipeline_parallel_size = 1
- distributed_backend = "mp"
- shm_size = "32gb"
- entrypoint = ["python3", "-m", "sglang.launch_server"]
- args = [
- # Memory — NVFP4 needs 0.90 (0.95 hard-rebooted the box at graph capture)
- "--mem-fraction-static", "0.90",
- "--max-running-requests", "10",
- "--max-mamba-cache-size", "30",
- # Attention / KV
- "--attention-backend", "flashinfer",
- "--kv-cache-dtype", "fp8_e4m3",
- # Mamba GDN hybrid — DFLASH requires extra_buffer (not extra_buffer_lazy)
- "--mamba-ssm-dtype", "bfloat16",
- "--mamba-full-memory-ratio", "4.21",
- "--mamba-radix-cache-strategy", "extra_buffer",
- # Chunked prefill
- "--chunked-prefill-size", "8192",
- # DFlash2 block-diffusion speculative decoding
- "--speculative-algorithm", "DFLASH",
- "--speculative-draft-model-path", "z-lab/Qwen3.8-27B-DFlash2",
- "--speculative-draft-model-revision", "50307d4c4cde6860d4eee73e2547cd786fe8e8a4",
- "--speculative-num-draft-tokens", "8",
- # Tokenizer / reasoning / tool calling (Qwen3 family)
- "--reasoning-parser", "qwen3",
- "--tool-call-parser", "qwen3_coder",
- # Observability
- "--enable-metrics",
- "--enable-cache-report",
- ]
- [env]
- HF_HUB_OFFLINE = "1"
- SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK = "1"
- TORCH_CUDA_ARCH_LIST = "12.1a"
- CUTE_DSL_ARCH = "sm_121a"
- OMP_NUM_THREADS = "4"
|
