recipes/recipes/nemotron-35-super-vl-120b-nvfp4/recipe.toml

# NVIDIA Nemotron-3.5-Super-VL-120B-A12B NVFP4 — single DGX Spark, VL
# (text+image+video), 1M context, LiLiCorr speculative decoding.
# Source: NVIDIA forum thread 385969 (the spark-vllm-docker recipe, post #3) +
# model card https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4
# (DGX Spark section). NVFP4 75.1 GB → one Spark. tool-eval-bench 90/100.
#
# Needs vLLM >= v0.32 (nemotron_v3 reasoning parser + qwen3_coder tool parser +
# the mamba flags are built in). Image: eugr/spark-vllm-b12x (vLLM nightly main,
# sm_121) — the card's `vllm-node` equivalent. LiLiCorr drafter (method dflash,
# 7 tokens) is a SECOND HF repo → resolved from the HF cache via the repo-ref
# below ({{lilicorr}}), like the kindling dflash mount. Image digest-pinned
# (arm64, built 2026-10-15); the card's `vllm-node`/`:latest` drift, we pin.

engine = "vllm"
name = "Nemotron-3.5-Super-VL-120B-A12B NVFP4 (1x Spark, LiLiCorr dflash)"
docker_image = "eugr/spark-vllm-b12x@sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595"
entrypoint = ["vllm", "serve"]
model = "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4"
parent_model = "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B"
port = 30000
solo = true
required_nodes = 1
tensor_parallel_size = 1
args = [
    "--served-model-name", "nemotron-3.5-super-vl",
    "--max-model-len", "1048576",
    "--max-num-seqs", "1",
    "--max-num-batched-tokens", "8192",
    "--gpu-memory-utilization", "0.75",
    "--trust-remote-code",
    "--mamba-backend", "flashinfer",
    "--mamba-ssm-cache-dtype", "float16",
    "--enable-mamba-cache-stochastic-rounding",
    "--mamba-cache-philox-rounds", "5",
    "--enable-mamba-shared-prefix-checkpoint",
    "--block-size", "64",
    "--prefix-match-unit", "64",
    "--video-pruning-rate", "0.5",
    "--media-io-kwargs", '{"video":{"fps":2,"num_frames":256}}',
    "--enable-auto-tool-choice",
    "--tool-call-parser", "qwen3_coder",
    "--reasoning-parser", "nemotron_v3",
    "--speculative-config", """
'{"method":"dflash","model":"{{lilicorr}}","num_speculative_tokens":7}'""",
]

# LiLiCorr drafter: resolved per node from the HF cache (repo-ref, no bind-mount).
[[extra_mounts]]
repo = "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4-LiLiCorr"
container = "lilicorr"

[env]
VLLM_ENGINE_READY_TIMEOUT_S = "3600"