|
|
@@ -0,0 +1,90 @@
|
|
|
+# Qwen3.8-27B (RadixArk NVFP4 + DSpark on SGLang) — 1× DGX Spark / GB10 (sm_121),
|
|
|
+# TP=1.
|
|
|
+#
|
|
|
+# SGLang + DSpark block-diffusion speculative decoding. The community (MiaAI)
|
|
|
+# measured DSpark on this box at ~51.5 tok/s on code vs vLLM+MTP's 22.8 — the
|
|
|
+# drafter is 3× cheaper per guess and edits/rewrites much faster. Uses the
|
|
|
+# pre-built cookbook image (no source build): lmsysorg/sglang:qwen38-27b.
|
|
|
+#
|
|
|
+# TARGET MUST BE RadixArk MODELOPT NVFP4: the cookbook image accelerates
|
|
|
+# modelopt-quantized NVFP4. The unsloth/RedHatAI compressed-tensors builds
|
|
|
+# fall back to a slow unquantized path ("Falling back to UnquantizedLinear-
|
|
|
+# Method") — prefill crawls at ~0.5 tok/s. Use RadixArk here.
|
|
|
+#
|
|
|
+# Draft: RadixArk/Qwen3.8-27B-DSpark (bf16, block 7). SGLang loads it natively
|
|
|
+# from the HF cache — no arch patch needed (unlike the vLLM qwen3_dspark path).
|
|
|
+#
|
|
|
+# Image: lmsysorg/sglang:qwen38-27b (multi-arch, incl. arm64; cookbook's
|
|
|
+# DGX Spark Qwen3.8 cell, pinned image digest febfb971). Pre-built — no
|
|
|
+# source build.
|
|
|
+#
|
|
|
+# Model: RadixArk/Qwen3.8-27B-NVFP4 (modelopt, cached locally on the node)
|
|
|
+# https://huggingface.co/RadixArk/Qwen3.8-27B-NVFP4
|
|
|
+#
|
|
|
+# PREREQUISITES:
|
|
|
+# Target + draft cached on the node (dgx1 ~/.cache/huggingface → NAS):
|
|
|
+# hfrs download RadixArk/Qwen3.8-27B-NVFP4
|
|
|
+# hfrs download RadixArk/Qwen3.8-27B-DSpark
|
|
|
+#
|
|
|
+# REFERENCE:
|
|
|
+# https://github.com/MiaAI-Lab/Qwen3.8-27B-SGLang-DGX-Spark (start-dspark.sh)
|
|
|
+# https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark
|
|
|
+
|
|
|
+engine = "sglang"
|
|
|
+name = "Qwen3.8-27B (RadixArk NVFP4, SGLang DSpark)"
|
|
|
+docker_image = "lmsysorg/sglang:qwen38-27b"
|
|
|
+model = "RadixArk/Qwen3.8-27B-NVFP4"
|
|
|
+parent_model = "Qwen/Qwen3.8-27B"
|
|
|
+required_nodes = 1
|
|
|
+tensor_parallel_size = 1
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+shm_size = "32gb"
|
|
|
+
|
|
|
+entrypoint = ["python3", "-m", "sglang.launch_server"]
|
|
|
+
|
|
|
+args = [
|
|
|
+ # Memory — larger KV pool (mamba GDN state pool sized from concurrency)
|
|
|
+ "--mem-fraction-static", "0.90",
|
|
|
+ "--max-running-requests", "10",
|
|
|
+ "--max-mamba-cache-size", "30",
|
|
|
+ # Attention / KV
|
|
|
+ "--attention-backend", "flashinfer",
|
|
|
+ "--kv-cache-dtype", "fp8_e4m3",
|
|
|
+ # Mamba GDN hybrid
|
|
|
+ "--mamba-ssm-dtype", "bfloat16",
|
|
|
+ "--mamba-full-memory-ratio", "4.21",
|
|
|
+ "--mamba-radix-cache-strategy", "extra_buffer_lazy",
|
|
|
+ # Chunked prefill
|
|
|
+ "--chunked-prefill-size", "8192",
|
|
|
+ # DSpark speculative decoding (block 7 / 8 draft tokens, unquant bf16 draft)
|
|
|
+ "--speculative-algorithm", "DSPARK",
|
|
|
+ "--speculative-draft-model-path", "RadixArk/Qwen3.8-27B-DSpark",
|
|
|
+ "--speculative-dspark-block-size", "7",
|
|
|
+ "--speculative-draft-model-quantization", "unquant",
|
|
|
+ "--speculative-num-draft-tokens", "8",
|
|
|
+ # Compile + decode graphs
|
|
|
+ "--enable-torch-compile", "--torch-compile-max-bs", "4",
|
|
|
+ "--cuda-graph-max-bs-decode", "4",
|
|
|
+ "--num-continuous-decode-steps", "2",
|
|
|
+ # Tokenizer / reasoning / tool calling (Qwen3 family)
|
|
|
+ "--reasoning-parser", "qwen3",
|
|
|
+ "--tool-call-parser", "qwen3_coder",
|
|
|
+ # Observability
|
|
|
+ "--enable-metrics",
|
|
|
+ "--enable-cache-report",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+# Offline — target + draft are both cached on the node
|
|
|
+HF_HUB_OFFLINE = "1"
|
|
|
+
|
|
|
+# Pin to GB10's Cortex-X5 big cores (scheduler/tokenizer off the A725 efficiency
|
|
|
+# cores — +2-7% decode per the MiaAI build)
|
|
|
+# CPUSET handled at the container layer; this is the SGLang-side hint.
|
|
|
+SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK = "1"
|
|
|
+
|
|
|
+# Architecture
|
|
|
+TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
+OMP_NUM_THREADS = "4"
|
