Prechádzať zdrojové kódy

Add 3 single-Spark recipes from NVIDIA forum (10-16): Qwen3.8-Flash-Next-FP4-Local (b12x+MTP n4, 89/100), Nemotron-3.5-Super-VL-120B-NVFP4 (LiLiCorr dflash7, 1M ctx, 90/100), Nemotron-3.5-Bolt-74B-A9B-24GB (QAD 2-bit, LiLiCorr, 88/100). All solo/TP1, eugr/spark-vllm-b12x digest-pinned, port 30000.

Alexander Korolev 6 dní pred
rodič
commit
903ed35881

+ 57 - 0
recipes/nemotron-35-bolt-74b-24gb/recipe.toml

@@ -0,0 +1,57 @@
+# NVIDIA Nemotron-3.5-Bolt-74B-A9B-24GB — single DGX Spark, VL, experts
+# quantized to 2-bit via QAD (quantization-aware distillation) to fit GB10.
+# Source: NVIDIA forum thread 385978 + model card
+# https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB (DGX Spark
+# section). 24 GB → one Spark (the "star" for Sparks without 128 GB RAM).
+# tool-eval-bench 88/100.
+#
+# Image: eugr/spark-vllm-b12x (vLLM nightly main + b12x, sm_121), digest-pinned
+# (arm64, built 2026-10-15). b12x linear/MoE
+# backends + flashinfer mamba. LiLiCorr drafter (dflash, 7) is a SECOND HF repo →
+# resolved via the repo-ref ({{lilicorr}}).
+#
+# CAVEAT: the card uses a reasoning-parser PLUGIN file `ultra_v3_reasoning_parser.py`
+# (ultra_v3 is NOT built into vLLM). The file ships in the model repo; vLLM
+# resolves it relative to the model dir under --trust-remote-code. If the plugin
+# path does not resolve at launch, point --reasoning-parser-plugin at the
+# snapshot's absolute path (or drop the parser for a text-only smoke).
+
+engine = "vllm"
+name = "Nemotron-3.5-Bolt-74B-A9B-24GB (1x Spark, QAD 2-bit, LiLiCorr)"
+docker_image = "eugr/spark-vllm-b12x@sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595"
+entrypoint = ["vllm", "serve"]
+model = "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB"
+parent_model = "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B"
+port = 30000
+solo = true
+required_nodes = 1
+tensor_parallel_size = 1
+args = [
+    "--served-model-name", "nemotron-3.5-bolt",
+    "--trust-remote-code",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "1",
+    "--enable-chunked-prefill",
+    "--kv-cache-dtype", "fp8",
+    "--linear-backend", "b12x",
+    "--moe-backend", "b12x",
+    "--mamba-backend", "flashinfer",
+    "--mamba-ssm-cache-dtype", "bfloat16",
+    "--mamba-cache-dtype", "bfloat16",
+    "--mamba-cache-mode", "align",
+    "--limit-mm-per-prompt", '{"video":1,"image":1}',
+    "--enable-auto-tool-choice",
+    "--tool-call-parser", "qwen3_coder",
+    "--reasoning-parser-plugin", "ultra_v3_reasoning_parser.py",
+    "--reasoning-parser", "ultra_v3",
+    "--speculative-config", """
+'{"method":"dflash","model":"{{lilicorr}}","num_speculative_tokens":7,"rejection_sample_method":"standard","attention_backend":"FLASHINFER","draft_load_config":{"load_format":"auto"}}'""",
+]
+
+# LiLiCorr drafter: resolved per node from the HF cache (repo-ref, no bind-mount).
+[[extra_mounts]]
+repo = "nvidia/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-LiLiCorr"
+container = "lilicorr"
+
+[env]
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"

+ 53 - 0
recipes/nemotron-35-super-vl-120b-nvfp4/recipe.toml

@@ -0,0 +1,53 @@
+# NVIDIA Nemotron-3.5-Super-VL-120B-A12B NVFP4 — single DGX Spark, VL
+# (text+image+video), 1M context, LiLiCorr speculative decoding.
+# Source: NVIDIA forum thread 385969 (the spark-vllm-docker recipe, post #3) +
+# model card https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4
+# (DGX Spark section). NVFP4 75.1 GB → one Spark. tool-eval-bench 90/100.
+#
+# Needs vLLM >= v0.32 (nemotron_v3 reasoning parser + qwen3_coder tool parser +
+# the mamba flags are built in). Image: eugr/spark-vllm-b12x (vLLM nightly main,
+# sm_121) — the card's `vllm-node` equivalent. LiLiCorr drafter (method dflash,
+# 7 tokens) is a SECOND HF repo → resolved from the HF cache via the repo-ref
+# below ({{lilicorr}}), like the kindling dflash mount. Image digest-pinned
+# (arm64, built 2026-10-15); the card's `vllm-node`/`:latest` drift, we pin.
+
+engine = "vllm"
+name = "Nemotron-3.5-Super-VL-120B-A12B NVFP4 (1x Spark, LiLiCorr dflash)"
+docker_image = "eugr/spark-vllm-b12x@sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595"
+entrypoint = ["vllm", "serve"]
+model = "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4"
+parent_model = "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B"
+port = 30000
+solo = true
+required_nodes = 1
+tensor_parallel_size = 1
+args = [
+    "--served-model-name", "nemotron-3.5-super-vl",
+    "--max-model-len", "1048576",
+    "--max-num-seqs", "1",
+    "--max-num-batched-tokens", "8192",
+    "--gpu-memory-utilization", "0.75",
+    "--trust-remote-code",
+    "--mamba-backend", "flashinfer",
+    "--mamba-ssm-cache-dtype", "float16",
+    "--enable-mamba-cache-stochastic-rounding",
+    "--mamba-cache-philox-rounds", "5",
+    "--enable-mamba-shared-prefix-checkpoint",
+    "--block-size", "64",
+    "--prefix-match-unit", "64",
+    "--video-pruning-rate", "0.5",
+    "--media-io-kwargs", '{"video":{"fps":2,"num_frames":256}}',
+    "--enable-auto-tool-choice",
+    "--tool-call-parser", "qwen3_coder",
+    "--reasoning-parser", "nemotron_v3",
+    "--speculative-config", """
+'{"method":"dflash","model":"{{lilicorr}}","num_speculative_tokens":7}'""",
+]
+
+# LiLiCorr drafter: resolved per node from the HF cache (repo-ref, no bind-mount).
+[[extra_mounts]]
+repo = "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4-LiLiCorr"
+container = "lilicorr"
+
+[env]
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"

+ 54 - 0
recipes/qwen38-flash-next-fp4-local/recipe.toml

@@ -0,0 +1,54 @@
+# Qwen3.8-Flash-Next FP4-Local — NVIDIA official single-DGX-Spark checkpoint.
+# Source: NVIDIA forum thread 385994 + model card
+# https://huggingface.co/nvidia/Qwen3.8-Flash-Next-FP4-Local (DGX Spark section,
+# transcribed verbatim). VL model (text+image+video). Single Spark (TP1): model
+# ~70 GiB + FP8 KV ~23 GiB → ~1.23M tokens KV. tool-eval-bench hardmode 89/100.
+#
+# Image: eugr/spark-vllm-b12x (vLLM nightly main + b12x MoE backend, sm_121),
+# digest-pinned (arm64, built 2026-10-15 = the card's d20261015 build). The card
+# uses :latest; we pin the digest per the recipe contract (floating tags drift).
+# b12x load path (--load-format b12x, --*-backend b12x) + the B12X_* env from the
+# card. MTP drafter is in-model (method mtp, n=4) — no separate repo.
+# Raise VLLM_ENGINE_READY_TIMEOUT_S: first launch (weight load + b12x autotune)
+# exceeds vLLM's default 600 s (forum: use 3600).
+
+engine = "vllm"
+name = "Qwen3.8-Flash-Next FP4-Local (1x Spark, b12x, MTP)"
+docker_image = "eugr/spark-vllm-b12x@sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595"
+entrypoint = ["vllm", "serve"]
+model = "nvidia/Qwen3.8-Flash-Next-FP4-Local"
+parent_model = "nvidia/Qwen3.8-Flash-Next"
+port = 30000
+solo = true
+required_nodes = 1
+tensor_parallel_size = 1
+args = [
+    "--served-model-name", "qwen38-flash-next",
+    "--mamba-cache-mode", "align",
+    "--enable-prefix-caching",
+    "--enable-chunked-prefill",
+    "--kv-cache-dtype", "fp8",
+    "--block-size", "16",
+    "--load-format", "b12x",
+    "--max-model-len", "262144",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "4096",
+    "--gdn-decode-kernel", "b12x",
+    "--linear-backend", "b12x",
+    "--moe-backend", "b12x",
+    "--reasoning-parser", "qwen3",
+    "--tool-call-parser", "qwen3_xml",
+    "--enable-auto-tool-choice",
+    "--compilation-config", '{"pass_config":{"fuse_act_quant":true}}',
+    "--gpu-memory-utilization", "0.8",
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":4}'""",
+]
+
+[env]
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+B12X_DISK_BACKEND = "io_uring"
+B12X_POLICY_MODE = "auto"
+VLLM_B12X_MOE_FP4_LAYER_MAX_INPUT_SCALE = "w13"
+VLLM_B12X_MOE_FP4_FORCE_A16 = "1"
+B12X_W4A16_A4_PREFILL = "1"