recipes/recipes/beam-1-0-nvfp4-4x/recipe.toml

# Beam-1.0-501B-A23B (NVFP4) on 4x DGX Spark — Reflection's 501B/23B-active MoE.
#
# From the official vLLM recipe (vllm-project/vllm,
# recipes/models/reflection/Beam-1.0-501B-A23B.yaml, 2026-10-16):
#   * image `vllm/vllm-openai:reflection-beam` — Beam support has not landed in
#     a vLLM release yet (min_vllm_version 0.33.0), so it ships only in this
#     Docker image. arm64 published (16.8 GB), which is what we need.
#   * NVFP4 default variant: ModelOpt NVFP4 on the routed experts, everything
#     else BF16 -> 353 GB of VRAM at TP4 (4 x ~88 GB per rank + KV).
#   * `--kv-cache-dtype fp8` for the NVFP4 variant.
#   * reasoning parser `beam` (Harmony `analysis` channel -> reasoning_content)
#     and tool-call parser `beam` with auto tool choice.
#   * the first NVFP4 start spends ~10 min autotuning FlashInfer MoE kernels —
#     well past vLLM's 600 s engine-ready default, hence the env bump.
#
# Weights live in the shared NAS HF cache (53 shards, 274 GB); the launcher
# mounts it at /root/.cache/huggingface so the repo id resolves offline.
#
# DFlash2 speculation (drafter reflection/Beam-1.0-501B-A23B-DFlash2, 7 tokens,
# FLASH_ATTN) — MEASURED 2026-10-21 and LEFT OFF: on GB10 (sm_121) fp8 KV is
# rejected by FlashAttention (needs FA3/SM90 or FA4/SM100 — the same wall as
# the GLM-5.3 DFlash2 A/B), so the lane can only run the drafter with BF16 KV
# (pool halves to 2.77M tokens), and adaptive verification conflicts with the
# FlashAttention cudagraph replay ("pass enable_adaptive_verification=false").
# With that forced off, decode was 23.6-27.1 tok/s vs 26.2 without any
# speculation — i.e. neutral, for half the KV. Not worth it here.

engine = "vllm"
name = "Beam-1.0-501B-A23B NVFP4 (4x Spark, TP4, beam parsers)"
docker_image = "vllm/vllm-openai:reflection-beam"
model = "reflection/Beam-1.0-501B-A23B-NVFP4"
parent_model = "reflection/Beam-1.0-501B-A23B"
port = 30000
required_nodes = 4
# dgx5-8: the experiment cluster (dgx1-4 runs the production 1017 lane).
cluster = "dgx58-switch"
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

# The launcher must know the process to exec: with an EMPTY entrypoint it
# renders `--entrypoint <model>` (the known "model executed as a command"
# trap). vllm/vllm-openai ships `vllm serve` as the image entrypoint.
entrypoint = ["vllm", "serve"]

args = [
    "--served-model-name", "beam",
    # Vendor default (fp8 KV) and MEASURED BEST on GB10: 6.8M-token KV pool,
    # 26 tok/s c1 / 41 c2 / 62 c4 decode, 90/100 quality.
    "--kv-cache-dtype", "fp8",
    # vendor: automatic tool choice, grammar derived from the checkpoint's
    # response_template; the reasoning parser splits the Harmony analysis channel
    "--enable-auto-tool-choice",
    "--tool-call-parser", "beam",
    "--reasoning-parser", "beam",
    # 384K of the model's 1M window: the measured KV pool is 5.27M tokens, so
    # this still allows ~13 concurrent full-length requests, and it leaves room
    # for the bench depth suite's 256K point (a 264K prompt does not fit a
    # 262144 cap — 2026-10-21 depth run).
    "--max-model-len", "393216",
    "--gpu-memory-utilization", "0.85",
    "--max-num-seqs", "8",
    "--trust-remote-code",
    "--speculative-config", """
'{"method":"dflash","model":"reflection/Beam-1.0-501B-A23B-DFlash2","num_speculative_tokens":7,"attention_backend":"FLASHINFER","enable_adaptive_verification":false}'""",
]

[env]
# First NVFP4 start autotunes FlashInfer MoE kernels (~10 min) — the default
# 600 s engine-ready timeout would kill the boot.
VLLM_ENGINE_READY_TIMEOUT_S = "3600"