# Qwen3.8-Flash-Next-FP8 (vLLM expert-parallel) — 2× DGX Spark / GB10 (sm_121), TP=2.
#
# Official FP8 quant of the Qwen4-architecture preview (176B total / 6B
# active, GDN + QSA hybrid MoE, 512 experts, 51B n-gram "engram" PLE table,
# in-checkpoint MTP). Text + vision. Native 262144 ctx.
#
# WHY vLLM + expert-parallel: the stock vLLM MoE path shards each expert's
# intermediate by TP, and this FP8 checkpoint (weight_block_size=[128,128],
# moe_intermediate_size=640) violates (intermediate/tp) % block_n == 0 at
# tp=2 -> boot crash. --enable-expert-parallel splits the 512 *experts*
# across the ranks instead (256 each, full 640-wide intermediate), so the
# divisibility rule never fires. This is the verified 2× Spark FP8 path.
#
# Image: vllm/vllm-openai:qwen38-flash-next — vLLM's day-0 model tag (arm64).
#
# Model: Qwen/Qwen3.8-Flash-Next-FP8 (~186 GB)
#   https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8
#
# REFERENCE (provenance — verified ~30-45 tok/s on 2× Sparks):
#   https://forums.developer.nvidia.com/t/qwen3-8-flash-next/381228/97  (foogitiff)
#   answers #106 / #113 (m-emm): ENFORCE_EAGER=1, disable MM, MTP=3 -> ~45 tok/s
#
# NOTE: day-0 config. foogitiff's base boots; m-emm's ENFORCE_EAGER + no-MM
# gives the better throughput. Expect to tune max-num-seqs / mm limits.

engine = "vllm"
name = "Qwen3.8-Flash-Next FP8 (vLLM expert-parallel, 2x Spark)"
docker_image = "vllm/vllm-openai:qwen38-flash-next"
model = "Qwen/Qwen3.8-Flash-Next-FP8"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

entrypoint = ["vllm", "serve"]

args = [
    # Stable served name — quant-agnostic, shared with qwen38-flash-next-nvfp4
    "--served-model-name", "qwen3.8-flash-next",
    # Expert parallelism — the fix for the FP8 MoE block-divisibility crash
    "--distributed-executor-backend", "mp",
    "--enable-expert-parallel",
    "--all2all-backend", "allgather_reducescatter",
    # Weight loading (no fastsafetensors day-0; lazy strategy keeps boot sane)
    "--load-format", "safetensors",
    "--safetensors-load-strategy", "lazy",
    # Context / batching (verified; raise max-num-seqs once stable)
    "--max-model-len", "262144",
    "--max-num-seqs", "1",
    "--max-num-batched-tokens", "8192",
    "--gpu-memory-utilization", "0.83",
    "--enable-chunked-prefill",
    # MTP speculative decoding (in-checkpoint draft)
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":3}'""",
    "--compilation-config", """
'{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}'""",
    # Tokenizer / reasoning / tools (Qwen3 family)
    "--reasoning-parser", "qwen3",
    "--enable-auto-tool-choice",
    "--tool-call-parser", "qwen3_coder",
    # m-emm perf (post #113): disable multimodal to save VRAM/bandwidth
    "--limit-mm-per-prompt", """
'{"image":0,"video":0}'""",
]

[env]
# Offline — weights must be cached on each node
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
# m-emm (post #113): eager mode for the 45 tok/s config (overrides the
# cudagraph request above; day-0 stability knob)
ENFORCE_EAGER = "1"

# Architecture (aarch64 / sm_121a DGX Spark)
TORCH_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
OMP_NUM_THREADS = "4"