|
|
@@ -0,0 +1,76 @@
|
|
|
+# Qwen3.8-27B (RadixArk NVFP4 + DFlash2 on SGLang) — 1× DGX Spark / GB10,
|
|
|
+# TP=1.
|
|
|
+#
|
|
|
+# SGLang + DFlash2 block-diffusion speculative decoding. Per the community
|
|
|
+# (MiaAI / forum 380350), DFlash2 on NVFP4 ties DSpark on code and *beats it*
|
|
|
+# on prose/essay and short-chat once counted right. This is the fork/engine
|
|
|
+# the forum's high tok/s numbers (52-61 code single, 87 TP=2) came from — NOT
|
|
|
+# the stock DSpark path.
|
|
|
+#
|
|
|
+# Image: lmsysorg/sglang:qwen38-27b-dflash2-minoverlay — a locally-built overlay
|
|
|
+# of the 5 DFlash2 modules onto the cookbook qwen38-27b image. The cookbook
|
|
|
+# tag itself predates DFlash2 (upstream merged commit c14312a66 on
|
|
|
+# 2026-08-19, after every published tag), so the modules are overlaid.
|
|
|
+# The minoverlay modules already bake in the quantized-head selector fix
|
|
|
+# (should_apply_lm_head_quant_method) so DFLASH runs the NVFP4 lm_head in
|
|
|
+# place — no dense dequant.
|
|
|
+#
|
|
|
+# Draft: z-lab/Qwen3.8-27B-DFlash2 @ 50307d4 (block-diffusion drafter, 3.6 GB).
|
|
|
+#
|
|
|
+# Model: RadixArk/Qwen3.8-27B-NVFP4 (modelopt)
|
|
|
+#
|
|
|
+# PREREQUISITES (on the node):
|
|
|
+# hfrs download RadixArk/Qwen3.8-27B-NVFP4
|
|
|
+# hfrs download z-lab/Qwen3.8-27B-DFlash2
|
|
|
+# docker build the minoverlay image (or use the recipe's docker_image as-is)
|
|
|
+#
|
|
|
+# REFERENCE:
|
|
|
+# https://github.com/MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark (start-dflash.sh)
|
|
|
+# https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
|
|
|
+
|
|
|
+engine = "sglang"
|
|
|
+name = "Qwen3.8-27B (RadixArk NVFP4, SGLang DFlash2)"
|
|
|
+docker_image = "lmsysorg/sglang:qwen38-27b-dflash2-minoverlay"
|
|
|
+model = "RadixArk/Qwen3.8-27B-NVFP4"
|
|
|
+parent_model = "Qwen/Qwen3.8-27B"
|
|
|
+required_nodes = 1
|
|
|
+tensor_parallel_size = 1
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+shm_size = "32gb"
|
|
|
+
|
|
|
+entrypoint = ["python3", "-m", "sglang.launch_server"]
|
|
|
+
|
|
|
+args = [
|
|
|
+ # Memory — NVFP4 needs 0.90 (0.95 hard-rebooted the box at graph capture)
|
|
|
+ "--mem-fraction-static", "0.90",
|
|
|
+ "--max-running-requests", "10",
|
|
|
+ "--max-mamba-cache-size", "30",
|
|
|
+ # Attention / KV
|
|
|
+ "--attention-backend", "flashinfer",
|
|
|
+ "--kv-cache-dtype", "fp8_e4m3",
|
|
|
+ # Mamba GDN hybrid — DFLASH requires extra_buffer (not extra_buffer_lazy)
|
|
|
+ "--mamba-ssm-dtype", "bfloat16",
|
|
|
+ "--mamba-full-memory-ratio", "4.21",
|
|
|
+ "--mamba-radix-cache-strategy", "extra_buffer",
|
|
|
+ # Chunked prefill
|
|
|
+ "--chunked-prefill-size", "8192",
|
|
|
+ # DFlash2 block-diffusion speculative decoding
|
|
|
+ "--speculative-algorithm", "DFLASH",
|
|
|
+ "--speculative-draft-model-path", "z-lab/Qwen3.8-27B-DFlash2",
|
|
|
+ "--speculative-draft-model-revision", "50307d4c4cde6860d4eee73e2547cd786fe8e8a4",
|
|
|
+ "--speculative-num-draft-tokens", "8",
|
|
|
+ # Tokenizer / reasoning / tool calling (Qwen3 family)
|
|
|
+ "--reasoning-parser", "qwen3",
|
|
|
+ "--tool-call-parser", "qwen3_coder",
|
|
|
+ # Observability
|
|
|
+ "--enable-metrics",
|
|
|
+ "--enable-cache-report",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+HF_HUB_OFFLINE = "1"
|
|
|
+SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK = "1"
|
|
|
+TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
+OMP_NUM_THREADS = "4"
|
