| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980 |
- # Qwen3.8-Flash-Next-FP8 (vLLM expert-parallel) — 2× DGX Spark / GB10 (sm_121), TP=2.
- #
- # Official FP8 quant of the Qwen4-architecture preview (176B total / 6B
- # active, GDN + QSA hybrid MoE, 512 experts, 51B n-gram "engram" PLE table,
- # in-checkpoint MTP). Text + vision. Native 262144 ctx.
- #
- # WHY vLLM + expert-parallel: the stock vLLM MoE path shards each expert's
- # intermediate by TP, and this FP8 checkpoint (weight_block_size=[128,128],
- # moe_intermediate_size=640) violates (intermediate/tp) % block_n == 0 at
- # tp=2 -> boot crash. --enable-expert-parallel splits the 512 *experts*
- # across the ranks instead (256 each, full 640-wide intermediate), so the
- # divisibility rule never fires. This is the verified 2× Spark FP8 path.
- #
- # Image: vllm/vllm-openai:qwen38-flash-next — vLLM's day-0 model tag (arm64).
- #
- # Model: Qwen/Qwen3.8-Flash-Next-FP8 (~186 GB)
- # https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8
- #
- # REFERENCE (provenance — verified ~30-45 tok/s on 2× Sparks):
- # https://forums.developer.nvidia.com/t/qwen3-8-flash-next/381228/97 (foogitiff)
- # answers #106 / #113 (m-emm): ENFORCE_EAGER=1, disable MM, MTP=3 -> ~45 tok/s
- #
- # NOTE: day-0 config. foogitiff's base boots; m-emm's ENFORCE_EAGER + no-MM
- # gives the better throughput. Expect to tune max-num-seqs / mm limits.
- engine = "vllm"
- name = "Qwen3.8-Flash-Next FP8 (vLLM expert-parallel, 2x Spark)"
- docker_image = "vllm/vllm-openai:qwen38-flash-next"
- model = "Qwen/Qwen3.8-Flash-Next-FP8"
- parent_model = "Qwen/Qwen3.8-Flash-Next"
- required_nodes = 2
- tensor_parallel_size = 2
- pipeline_parallel_size = 1
- distributed_backend = "mp"
- shm_size = "32gb"
- entrypoint = ["vllm", "serve"]
- args = [
- # Stable served name — quant-agnostic, shared with qwen38-flash-next-nvfp4
- "--served-model-name", "qwen3.8-flash-next",
- # Expert parallelism — the fix for the FP8 MoE block-divisibility crash
- "--distributed-executor-backend", "mp",
- "--enable-expert-parallel",
- "--all2all-backend", "allgather_reducescatter",
- # Weight loading (no fastsafetensors day-0; lazy strategy keeps boot sane)
- "--load-format", "safetensors",
- "--safetensors-load-strategy", "lazy",
- # Context / batching (verified; raise max-num-seqs once stable)
- "--max-model-len", "262144",
- "--max-num-seqs", "1",
- "--max-num-batched-tokens", "8192",
- "--gpu-memory-utilization", "0.83",
- "--enable-chunked-prefill",
- # MTP speculative decoding (in-checkpoint draft)
- "--speculative-config", """
- '{"method":"mtp","num_speculative_tokens":3}'""",
- "--compilation-config", """
- '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}'""",
- # Tokenizer / reasoning / tools (Qwen3 family)
- "--reasoning-parser", "qwen3",
- "--enable-auto-tool-choice",
- "--tool-call-parser", "qwen3_coder",
- # m-emm perf (post #113): disable multimodal to save VRAM/bandwidth
- "--limit-mm-per-prompt", """
- '{"image":0,"video":0}'""",
- ]
- [env]
- # Offline — weights must be cached on each node
- HF_HUB_OFFLINE = "1"
- TRANSFORMERS_OFFLINE = "1"
- # m-emm (post #113): eager mode for the 45 tok/s config (overrides the
- # cudagraph request above; day-0 stability knob)
- ENFORCE_EAGER = "1"
- # Architecture (aarch64 / sm_121a DGX Spark)
- TORCH_CUDA_ARCH_LIST = "12.1a"
- CUTE_DSL_ARCH = "sm_121a"
- OMP_NUM_THREADS = "4"
|
