|
|
@@ -0,0 +1,98 @@
|
|
|
+# DeepSeek-V4-Flash-DSpark — 2× GB10 / DGX Spark, TP=2 over RoCE.
|
|
|
+# Uses the DSpark speculative decoding method with aidendle94's production-3.6 image.
|
|
|
+#
|
|
|
+# Image: aidendle94/sparkrun-vllm-ds4-gb10:production-3.6
|
|
|
+# Latest GB10-native image with DSpark support, b12x MoE, and multi-node fixes.
|
|
|
+# Model: deepseek-ai/DeepSeek-V4-Flash-DSpark
|
|
|
+# The DSpark variant tuned for speculative decoding on DGX Spark.
|
|
|
+#
|
|
|
+# Reference: Nvidia Developer Forum — DeepSeek-V4-Flash-DSpark recipe
|
|
|
+
|
|
|
+engine = "vllm"
|
|
|
+name = "DeepSeek V4 Flash DSpark (vLLM)"
|
|
|
+docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-3.6"
|
|
|
+model = "deepseek-ai/DeepSeek-V4-Flash-DSpark"
|
|
|
+port = 8000
|
|
|
+required_nodes = 2
|
|
|
+tensor_parallel_size = 2
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+shm_size = "32gb"
|
|
|
+
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+
|
|
|
+args = [
|
|
|
+ "--kv-cache-dtype", "fp8",
|
|
|
+ "--block-size", "256",
|
|
|
+ "--max-model-len", "393216",
|
|
|
+ "--max-num-seqs", "128",
|
|
|
+ "--max-num-batched-tokens", "4096",
|
|
|
+ "--max-cudagraph-capture-size", "128",
|
|
|
+ "--gpu-memory-utilization", "0.8",
|
|
|
+ "--served-model-name", "ChatGPTN",
|
|
|
+ # Engine backends
|
|
|
+ "--attention-backend", "FLASHINFER_MLA_SPARSE_DSV4",
|
|
|
+ "--moe-backend", "b12x",
|
|
|
+ # Chunked prefill + prefix caching
|
|
|
+ "--enable-chunked-prefill",
|
|
|
+ "--enable-prefix-caching",
|
|
|
+ "--enable-flashinfer-autotune",
|
|
|
+ # Compilation
|
|
|
+ "--compilation-config", """
|
|
|
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
|
|
|
+ # Async scheduling
|
|
|
+ "--async-scheduling",
|
|
|
+ "--no-scheduler-reserve-full-isl",
|
|
|
+ # DSpark speculative decoding
|
|
|
+ "--speculative-config", """
|
|
|
+'{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic"}'""",
|
|
|
+ # Tokenizer / reasoning
|
|
|
+ "--tokenizer-mode", "deepseek_v4",
|
|
|
+ "--tool-call-parser", "deepseek_v4",
|
|
|
+ "--reasoning-parser", "deepseek_v4",
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
+ "--default-chat-template-kwargs.thinking=true",
|
|
|
+ "--default-chat-template-kwargs.reasoning_effort=max",
|
|
|
+ # Disable custom all-reduce (not needed on GB10 single-NIC)
|
|
|
+ "--disable-custom-all-reduce",
|
|
|
+ "--trust-remote-code",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+# DSpark speculative decoding
|
|
|
+VLLM_DSPARK_REPLICATE_MARKOV_W1 = "1"
|
|
|
+# b12x MoE (this image's primary path)
|
|
|
+VLLM_USE_B12X_MOE = "1"
|
|
|
+
|
|
|
+# b12x suite = REQUIRED by this image
|
|
|
+VLLM_USE_B12X_MHC = "1"
|
|
|
+VLLM_USE_B12X_WO_PROJECTION = "1"
|
|
|
+VLLM_USE_B12X_FP8_GEMM = "1"
|
|
|
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
|
|
|
+VLLM_USE_V2_MODEL_RUNNER = "1"
|
|
|
+VLLM_USE_FLASHINFER_SAMPLER = "1"
|
|
|
+VLLM_USE_AOT_COMPILE = "1"
|
|
|
+VLLM_USE_MEGA_AOT_ARTIFACT = "0"
|
|
|
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
|
|
|
+VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
|
|
|
+VLLM_PREFIX_CACHE_RETENTION_INTERVAL = "4096"
|
|
|
+B12X_MLA_SM120_UNIFIED = "1"
|
|
|
+B12X_DENSE_SPLITK_TURBO = "1"
|
|
|
+B12X_W4A16_TC_DECODE = "1"
|
|
|
+B12X_MHC_MAX_TOKENS = "16384"
|
|
|
+
|
|
|
+# GPU memory management
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
+OMP_NUM_THREADS = "4"
|
|
|
+
|
|
|
+# Offline mode
|
|
|
+HF_HUB_OFFLINE = "1"
|
|
|
+
|
|
|
+# Compile-time arch hints
|
|
|
+TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
+
|
|
|
+# NCCL overrides (cluster config ib_hca/socket_ifname used for most; recipe wins last)
|
|
|
+NCCL_P2P_LEVEL = "SYS"
|
|
|
+NCCL_CUMEM_ENABLE = "0"
|
|
|
+
|