|
|
@@ -0,0 +1,97 @@
|
|
|
+# EXPERIMENTAL — DeepSeek-V4-Flash-0731 DSpark, built FROM SOURCE from the
|
|
|
+# newest local-inference-lab b12x fork ("eldritch"), instead of the prebuilt
|
|
|
+# aidendle94 image.
|
|
|
+#
|
|
|
+# fork branch: feat/gg-ds4-0731-launcher (e63a190ee) — torch 2.12.0,
|
|
|
+# flashinfer 0.6.15, cutlass-dsl 4.6.0, sm_121a
|
|
|
+# image: vllm-ds4-dspark-src:probe (built on demand via docker_image_dir)
|
|
|
+#
|
|
|
+# Serving flags/env mirror the fork's authoritative serve-ds4-flash-spark.sh at
|
|
|
+# that commit (DSpark k=5, draft attention backend B12X_MLA_SPARSE, b12x MoE/MLA,
|
|
|
+# cudagraph capture size = max_num_seqs * (k+1)). Adapt cluster-side: TP=2,
|
|
|
+# 1 node per rank, no DCP.
|
|
|
+#
|
|
|
+# ⚠️ NOT production. The image is a multi-hour from-source build. Boot may need
|
|
|
+# tuning; verify acceptance/throughput before any real traffic.
|
|
|
+
|
|
|
+engine = "vllm"
|
|
|
+name = "DeepSeek V4 Flash DSpark (EXPERIMENTAL, from source)"
|
|
|
+docker_image = "vllm-ds4-dspark-src:probe"
|
|
|
+model = "deepseek-ai/DeepSeek-V4-Flash-0731"
|
|
|
+required_nodes = 2
|
|
|
+tensor_parallel_size = 2
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+shm_size = "32gb"
|
|
|
+docker_image_dir = "deepseek-v4-flash-dspark-src"
|
|
|
+
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+
|
|
|
+args = [
|
|
|
+ "--kv-cache-dtype", "fp8",
|
|
|
+ "--block-size", "256",
|
|
|
+ "--load-format", "auto",
|
|
|
+ "--max-model-len", "524288",
|
|
|
+ "--max-num-seqs", "4",
|
|
|
+ "--max-num-batched-tokens", "8192",
|
|
|
+ # fork rule: max_num_seqs * (k+1) = 4 * 6
|
|
|
+ "--max-cudagraph-capture-size", "24",
|
|
|
+ # 0.78, not 0.80: the SvangenStudios agent-serving benchmark (v0.1.1) on
|
|
|
+ # this exact stack found "0.80 boots but dies under traffic". GMU is a KV
|
|
|
+ # lever, not a safety lever, when weights dominate memory. See
|
|
|
+ # specs/tech-architecture/dgx-spark-agent-serving-findings.md.
|
|
|
+ "--gpu-memory-utilization", "0.78",
|
|
|
+ "--served-model-name", "deepseek-v4-flash", "default",
|
|
|
+ "--attention-backend", "B12X_MLA_SPARSE",
|
|
|
+ "--moe-backend", "b12x",
|
|
|
+ "--enable-chunked-prefill",
|
|
|
+ "--enable-prefix-caching",
|
|
|
+ "--enable-flashinfer-autotune",
|
|
|
+ "--compilation-config", """
|
|
|
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
|
|
|
+ "--async-scheduling",
|
|
|
+ "--no-scheduler-reserve-full-isl",
|
|
|
+ # DSpark k=5 (dspark_block_size == 5); draft attention backend per fork
|
|
|
+ "--speculative-config", """
|
|
|
+'{"method":"dspark","num_speculative_tokens":5,"attention_backend":"B12X_MLA_SPARSE"}'""",
|
|
|
+ "--generation-config", "vllm",
|
|
|
+ "--default-chat-template-kwargs.thinking=true",
|
|
|
+ "--default-chat-template-kwargs.reasoning_effort=high",
|
|
|
+ "--default-chat-template-kwargs.temperature=1",
|
|
|
+ "--default-chat-template-kwargs.top_p=1",
|
|
|
+ "--disable-custom-all-reduce",
|
|
|
+ "--trust-remote-code",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+# --- from the fork's serve-ds4-flash-spark.sh (e63a190ee) ---
|
|
|
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
|
|
|
+VLLM_USE_AOT_COMPILE = "1"
|
|
|
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
|
|
|
+VLLM_USE_MEGA_AOT_ARTIFACT = "1"
|
|
|
+VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS = "1"
|
|
|
+VLLM_USE_FLASHINFER_SAMPLER = "1"
|
|
|
+VLLM_USE_B12X_WO_PROJECTION = "1"
|
|
|
+VLLM_USE_B12X_MHC = "1"
|
|
|
+VLLM_USE_B12X_FP8_GEMM = "1"
|
|
|
+VLLM_USE_B12X_MOE = "1"
|
|
|
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
|
|
|
+VLLM_USE_V2_MODEL_RUNNER = "1"
|
|
|
+VLLM_PCIE_ALLREDUCE_BACKEND = "b12x"
|
|
|
+VLLM_ENABLE_PCIE_ALLREDUCE = "1"
|
|
|
+B12X_MLA_SM120_UNIFIED = "1"
|
|
|
+B12X_DENSE_SPLITK_TURBO = "1"
|
|
|
+B12X_W4A16_TC_DECODE = "1"
|
|
|
+B12X_MOE_FORCE_A8 = "1"
|
|
|
+
|
|
|
+# --- build/arch + memory ---
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
+DG_JIT_USE_NVRTC = "0"
|
|
|
+USE_CUDNN = "1"
|
|
|
+CUDA_VISIBLE_DEVICES = "0"
|
|
|
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
|
|
|
+OMP_NUM_THREADS = "4"
|
|
|
+
|
|
|
+# --- offline ---
|
|
|
+HF_HUB_OFFLINE = "1"
|
