recipes: qwen38-flash-next-fp8

recipes/qwen38-flash-next-fp8/recipe.toml

@@ -0,0 +1,80 @@
+# Qwen3.8-Flash-Next-FP8 (vLLM expert-parallel) — 2× DGX Spark / GB10 (sm_121), TP=2.
+#
+# Official FP8 quant of the Qwen4-architecture preview (176B total / 6B
+# active, GDN + QSA hybrid MoE, 512 experts, 51B n-gram "engram" PLE table,
+# in-checkpoint MTP). Text + vision. Native 262144 ctx.
+#
+# WHY vLLM + expert-parallel: the stock vLLM MoE path shards each expert's
+# intermediate by TP, and this FP8 checkpoint (weight_block_size=[128,128],
+# moe_intermediate_size=640) violates (intermediate/tp) % block_n == 0 at
+# tp=2 -> boot crash. --enable-expert-parallel splits the 512 *experts*
+# across the ranks instead (256 each, full 640-wide intermediate), so the
+# divisibility rule never fires. This is the verified 2× Spark FP8 path.
+#
+# Image: vllm/vllm-openai:qwen38-flash-next — vLLM's day-0 model tag (arm64).
+#
+# Model: Qwen/Qwen3.8-Flash-Next-FP8 (~186 GB)
+#   https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8
+#
+# REFERENCE (provenance — verified ~30-45 tok/s on 2× Sparks):
+#   https://forums.developer.nvidia.com/t/qwen3-8-flash-next/381228/97  (foogitiff)
+#   answers #106 / #113 (m-emm): ENFORCE_EAGER=1, disable MM, MTP=3 -> ~45 tok/s
+#
+# NOTE: day-0 config. foogitiff's base boots; m-emm's ENFORCE_EAGER + no-MM
+# gives the better throughput. Expect to tune max-num-seqs / mm limits.
+
+engine = "vllm"
+name = "Qwen3.8-Flash-Next FP8 (vLLM expert-parallel, 2x Spark)"
+docker_image = "vllm/vllm-openai:qwen38-flash-next"
+model = "Qwen/Qwen3.8-Flash-Next-FP8"
+parent_model = "Qwen/Qwen3.8-Flash-Next"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Stable served name — quant-agnostic, shared with qwen38-flash-next-nvfp4
+    "--served-model-name", "qwen3.8-flash-next",
+    # Expert parallelism — the fix for the FP8 MoE block-divisibility crash
+    "--distributed-executor-backend", "mp",
+    "--enable-expert-parallel",
+    "--all2all-backend", "allgather_reducescatter",
+    # Weight loading (no fastsafetensors day-0; lazy strategy keeps boot sane)
+    "--load-format", "safetensors",
+    "--safetensors-load-strategy", "lazy",
+    # Context / batching (verified; raise max-num-seqs once stable)
+    "--max-model-len", "262144",
+    "--max-num-seqs", "1",
+    "--max-num-batched-tokens", "8192",
+    "--gpu-memory-utilization", "0.83",
+    "--enable-chunked-prefill",
+    # MTP speculative decoding (in-checkpoint draft)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":3}'""",
+    "--compilation-config", """
+'{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}'""",
+    # Tokenizer / reasoning / tools (Qwen3 family)
+    "--reasoning-parser", "qwen3",
+    "--enable-auto-tool-choice",
+    "--tool-call-parser", "qwen3_coder",
+    # m-emm perf (post #113): disable multimodal to save VRAM/bandwidth
+    "--limit-mm-per-prompt", """
+'{"image":0,"video":0}'""",
+]
+
+[env]
+# Offline — weights must be cached on each node
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+# m-emm (post #113): eager mode for the 45 tok/s config (overrides the
+# cudagraph request above; day-0 stability knob)
+ENFORCE_EAGER = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"