|
@@ -0,0 +1,57 @@
|
|
|
|
|
+# NVIDIA Nemotron-3.5-Bolt-74B-A9B-24GB — single DGX Spark, VL, experts
|
|
|
|
|
+# quantized to 2-bit via QAD (quantization-aware distillation) to fit GB10.
|
|
|
|
|
+# Source: NVIDIA forum thread 385978 + model card
|
|
|
|
|
+# https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB (DGX Spark
|
|
|
|
|
+# section). 24 GB → one Spark (the "star" for Sparks without 128 GB RAM).
|
|
|
|
|
+# tool-eval-bench 88/100.
|
|
|
|
|
+#
|
|
|
|
|
+# Image: eugr/spark-vllm-b12x (vLLM nightly main + b12x, sm_121), digest-pinned
|
|
|
|
|
+# (arm64, built 2026-10-15). b12x linear/MoE
|
|
|
|
|
+# backends + flashinfer mamba. LiLiCorr drafter (dflash, 7) is a SECOND HF repo →
|
|
|
|
|
+# resolved via the repo-ref ({{lilicorr}}).
|
|
|
|
|
+#
|
|
|
|
|
+# CAVEAT: the card uses a reasoning-parser PLUGIN file `ultra_v3_reasoning_parser.py`
|
|
|
|
|
+# (ultra_v3 is NOT built into vLLM). The file ships in the model repo; vLLM
|
|
|
|
|
+# resolves it relative to the model dir under --trust-remote-code. If the plugin
|
|
|
|
|
+# path does not resolve at launch, point --reasoning-parser-plugin at the
|
|
|
|
|
+# snapshot's absolute path (or drop the parser for a text-only smoke).
|
|
|
|
|
+
|
|
|
|
|
+engine = "vllm"
|
|
|
|
|
+name = "Nemotron-3.5-Bolt-74B-A9B-24GB (1x Spark, QAD 2-bit, LiLiCorr)"
|
|
|
|
|
+docker_image = "eugr/spark-vllm-b12x@sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595"
|
|
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
|
|
+model = "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB"
|
|
|
|
|
+parent_model = "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B"
|
|
|
|
|
+port = 30000
|
|
|
|
|
+solo = true
|
|
|
|
|
+required_nodes = 1
|
|
|
|
|
+tensor_parallel_size = 1
|
|
|
|
|
+args = [
|
|
|
|
|
+ "--served-model-name", "nemotron-3.5-bolt",
|
|
|
|
|
+ "--trust-remote-code",
|
|
|
|
|
+ "--max-model-len", "262144",
|
|
|
|
|
+ "--max-num-seqs", "1",
|
|
|
|
|
+ "--enable-chunked-prefill",
|
|
|
|
|
+ "--kv-cache-dtype", "fp8",
|
|
|
|
|
+ "--linear-backend", "b12x",
|
|
|
|
|
+ "--moe-backend", "b12x",
|
|
|
|
|
+ "--mamba-backend", "flashinfer",
|
|
|
|
|
+ "--mamba-ssm-cache-dtype", "bfloat16",
|
|
|
|
|
+ "--mamba-cache-dtype", "bfloat16",
|
|
|
|
|
+ "--mamba-cache-mode", "align",
|
|
|
|
|
+ "--limit-mm-per-prompt", '{"video":1,"image":1}',
|
|
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
|
|
+ "--tool-call-parser", "qwen3_coder",
|
|
|
|
|
+ "--reasoning-parser-plugin", "ultra_v3_reasoning_parser.py",
|
|
|
|
|
+ "--reasoning-parser", "ultra_v3",
|
|
|
|
|
+ "--speculative-config", """
|
|
|
|
|
+'{"method":"dflash","model":"{{lilicorr}}","num_speculative_tokens":7,"rejection_sample_method":"standard","attention_backend":"FLASHINFER","draft_load_config":{"load_format":"auto"}}'""",
|
|
|
|
|
+]
|
|
|
|
|
+
|
|
|
|
|
+# LiLiCorr drafter: resolved per node from the HF cache (repo-ref, no bind-mount).
|
|
|
|
|
+[[extra_mounts]]
|
|
|
|
|
+repo = "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr"
|
|
|
|
|
+container = "lilicorr"
|
|
|
|
|
+
|
|
|
|
|
+[env]
|
|
|
|
|
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|