recipes: qwen38-27b-fp8-dspark

recipes/qwen38-27b-fp8-dspark/recipe.toml

@@ -0,0 +1,93 @@
+# Qwen3.8-27B (FP8 + DSpark draft) — 1× DGX Spark / GB10 (sm_121), TP=1.
+#
+# DSpark: a dedicated semi-autoregressive draft model (~2.6 GB, 5 layers,
+# block_size 7) proposes blocks of tokens that the full model verifies in one
+# weight-read. 3× cheaper per guess than the in-checkpoint MTP head, so it wins
+# on edit-heavy / structured workloads (blog: 45-48 t/s edit, 17 t/s free-gen,
+# 66 t/s aggregate @ c10 on the FP8 target). DSpark blocks are verifier-rejected,
+# so no quality risk.
+#
+# Target: Qwen/Qwen3.8-27B-FP8 (official FP8 quant, 28.75 GB).
+# Draft:  RadixArk/Qwen3.8-27B-DSpark (arch DSparkDraftModel, bf16, 5 layers,
+#         dspark_block_size 7). Downloaded via hfrs into the node HF cache.
+#
+# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
+#   vLLM 0.27.1 (installed 0.27.2.dev0+g6e448d0ea), supports method "dspark"
+#   (SpeculativeMethod Literal includes dspark) + Qwen3DSparkModel draft.
+#
+# PREREQUISITES:
+#   Target + draft cached on the node (dgx1 ~/.cache/huggingface → NAS):
+#       hfrs download Qwen/Qwen3.8-27B-FP8
+#       hfrs download RadixArk/Qwen3.8-27B-DSpark
+#
+# REFERENCE:
+#   https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark  (DSpark section)
+#   https://huggingface.co/RadixArk/Qwen3.8-27B-DSpark
+#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
+
+engine = "vllm"
+name = "Qwen3.8-27B (FP8 + DSpark)"
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
+model = "Qwen/Qwen3.8-27B-FP8"
+parent_model = "Qwen/Qwen3.8-27B"
+required_nodes = 1
+tensor_parallel_size = 1
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # KV cache & memory
+    "--kv-cache-dtype", "fp8",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "6",
+    "--max-num-batched-tokens", "8192",
+    "--gpu-memory-utilization", "0.75",
+    "--served-model-name", "qwen3.8-27b",
+    # Engine backends (dense GQA — FlashInfer)
+    "--attention-backend", "flashinfer",
+    # Compilation / perf
+    "--optimization-level", "3",
+    "--performance-mode", "throughput",
+    # Chunked prefill + prefix caching
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--enable-flashinfer-autotune",
+    # DSpark speculative decoding — dedicated draft model (not the in-checkpoint
+    # MTP head). num_speculative_tokens must equal the draft's block size (7).
+    # The draft lives in a local dir (arch-patched copy of RadixArk/Qwen3.8-27B-DSpark
+    # re-labelled `Qwen3DSparkModel` so vLLM routes it to qwen3_dspark, not the
+    # DeepSeek-V4 DSpark code path) bind-mounted at /dspark-draft below.
+    "--speculative-config", """
+'{"method":"dspark","model":"/dspark-draft","num_speculative_tokens":7}'""",
+    # Tokenizer / reasoning (Qwen3 family)
+    "--tool-call-parser", "qwen3_xml",
+    "--reasoning-parser", "qwen3",
+    "--enable-auto-tool-choice",
+    "--generation-config", "auto",
+    "--default-chat-template-kwargs.preserve_thinking=true",
+    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
+    # practices)
+    "--override-generation-config", """
+'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
+    "--trust-remote-code",
+]
+
+# DSpark draft model — arch-patched local copy of RadixArk/Qwen3.8-27B-DSpark,
+# mounted at /dspark-draft so vLLM loads it via the qwen3_dspark draft class.
+[[extra_mounts]]
+host = "/home/kilork/dspark-draft"
+container = "/dspark-draft"
+
+
+[env]
+HF_HUB_OFFLINE = "1"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+CUDA_VISIBLE_DEVICES = "0"
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
+CUDA_CACHE_MAXSIZE = "4294967296"