feat(recipes): Qwen3.8-Flash-Next FP8 on vLLM via expert parallelism

The official FP8 quant CAN serve on 2x Spark via vLLM — the trick is
--enable-expert-parallel + --all2all-backend allgather_reducescatter, which
splits the 512 experts across the ranks instead of sharding the intermediate,
dodging the (intermediate/tp) % block_n == 0 crash that blocks plain TP=2.

- recipe: recipes/qwen38-flash-next-fp8 (vllm/vllm-openai:qwen38-flash-next,
  day-0 arm64 image; --load-format safetensors/lazy, MTP k=3, mm disabled,
  ENFORCE_EAGER per the 45 tok/s answer)
- provenance: forum 381228 post #97 (foogitiff) + answers #106/#113 (m-emm)
- note: coexists with qwen38-flash-next-nvfp4 (SGLang NVFP4) to A/B

recipes/qwen38-flash-next-fp8/recipe.toml

@@ -0,0 +1,77 @@
+# Qwen3.8-Flash-Next-FP8 (vLLM expert-parallel) — 2× DGX Spark / GB10 (sm_121), TP=2.
+#
+# Official FP8 quant of the Qwen4-architecture preview (176B total / 6B
+# active, GDN + QSA hybrid MoE, 512 experts, 51B n-gram "engram" PLE table,
+# in-checkpoint MTP). Text + vision. Native 262144 ctx.
+#
+# WHY vLLM + expert-parallel: the stock vLLM MoE path shards each expert's
+# intermediate by TP, and this FP8 checkpoint (weight_block_size=[128,128],
+# moe_intermediate_size=640) violates (intermediate/tp) % block_n == 0 at
+# tp=2 -> boot crash. --enable-expert-parallel splits the 512 *experts*
+# across the ranks instead (256 each, full 640-wide intermediate), so the
+# divisibility rule never fires. This is the verified 2× Spark FP8 path.
+#
+# Image: vllm/vllm-openai:qwen38-flash-next — vLLM's day-0 model tag (arm64).
+#
+# Model: Qwen/Qwen3.8-Flash-Next-FP8 (~186 GB)
+#   https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8
+#
+# REFERENCE (provenance — verified ~30-45 tok/s on 2× Sparks):
+#   https://forums.developer.nvidia.com/t/qwen3-8-flash-next/381228/97  (foogitiff)
+#   answers #106 / #113 (m-emm): ENFORCE_EAGER=1, disable MM, MTP=3 -> ~45 tok/s
+#
+# NOTE: day-0 config. foogitiff's base boots; m-emm's ENFORCE_EAGER + no-MM
+# gives the better throughput. Expect to tune max-num-seqs / mm limits.
+
+engine = "vllm"
+name = "Qwen3.8-Flash-Next FP8 (vLLM expert-parallel, 2x Spark)"
+docker_image = "vllm/vllm-openai:qwen38-flash-next"
+model = "Qwen/Qwen3.8-Flash-Next-FP8"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Expert parallelism — the fix for the FP8 MoE block-divisibility crash
+    "--distributed-executor-backend", "mp",
+    "--enable-expert-parallel",
+    "--all2all-backend", "allgather_reducescatter",
+    # Weight loading (no fastsafetensors day-0; lazy strategy keeps boot sane)
+    "--load-format", "safetensors",
+    "--safetensors-load-strategy", "lazy",
+    # Context / batching (verified; raise max-num-seqs once stable)
+    "--max-model-len", "262144",
+    "--max-num-seqs", "1",
+    "--max-num-batched-tokens", "8192",
+    "--gpu-memory-utilization", "0.83",
+    "--enable-chunked-prefill",
+    # MTP speculative decoding (in-checkpoint draft)
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":3}'""",
+    "--compilation-config", """
+'{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}'""",
+    # Tokenizer / reasoning / tools (Qwen3 family)
+    "--reasoning-parser", "qwen3",
+    "--enable-auto-tool-choice",
+    "--tool-call-parser", "qwen3_coder",
+    # m-emm perf (post #113): disable multimodal to save VRAM/bandwidth
+    "--limit-mm-per-prompt", """
+'{"image":0,"video":0}'""",
+]
+
+[env]
+# Offline — weights must be cached on each node
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+# m-emm (post #113): eager mode for the 45 tok/s config (overrides the
+# cudagraph request above; day-0 stability knob)
+ENFORCE_EAGER = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"