| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091 |
- # DeepSeek-V4.1-Flash (552B MoE, MXFP4 experts, Engram-on-disk) — 4x DGX Spark.
- #
- # Image: dsv41-flash:local (docker-images/dsv41-flash/ — tonyd2wild overlay
- # chain as clone-at-pin build: vllm dsv41-feat@e47aa780 + sm121
- # _C_stable_libtorch + FlashInfer 0.7.0rc1 + 7 boot-10 patches baked in,
- # MIT provenance).
- #
- # Model: deepseek-ai/DeepSeek-V4.1-Flash (476 GB on the NFS HF cache,
- # refs/main). The two 203 GB FP8 Engram n-gram tables are NEVER loaded into
- # HBM — the baked engram.py patch keeps them on disk (DSV41_ENGRAM_DISK=1)
- # and reads rows on demand. Node-local sparse copies of each rank's rows
- # (48 GB, engram-local.json) accelerate decode once created (DSV41_ENGRAM_DIR
- # below, added after the copies exist — first boot reads straight from NFS).
- #
- # Config mirrors their boot-10 serving setup (launch/dsv41-tp4.sh):
- # CUDA graphs FULL_AND_PIECEWISE + exact DSpark capture sizes (no padded
- # rows — padded speculative batches hang SM120 sparse MLA, FlashInfer
- # #5015), DSpark k=5 with the checkpoint's own draft layers, GMU 0.80,
- # 300K ctx (KV pool ~1.07M tokens = 3.57x headroom), tools + vision on,
- # thinking off by default.
- engine = "vllm"
- name = "DeepSeek-V4.1-Flash MXFP4 (dsv41 img, 4x, DSpark k=5, Engram-on-disk)"
- docker_image = "dsv41-flash:local"
- docker_image_dir = "dsv41-flash"
- model = "deepseek-ai/DeepSeek-V4.1-Flash"
- required_nodes = 4
- tensor_parallel_size = 4
- pipeline_parallel_size = 1
- distributed_backend = "mp"
- shm_size = "32gb"
- entrypoint = ["vllm", "serve"]
- args = [
- "--served-model-name", "dsv41",
- "--trust-remote-code",
- "--distributed-executor-backend", "mp",
- "--gpu-memory-utilization", "0.80",
- "--max-model-len", "300000",
- "--max-num-seqs", "8",
- "--max-num-batched-tokens", "8192",
- # boot-4 fix: the V4 indexer backend refuses the smallest-listed block
- "--block-size", "128",
- # boot-10: exact capture sizes for DSpark k=5, SEQS=8 (multiples of 5
- # and 6 up to 48 — every decode batch gets an exact FULL graph, no padding)
- "--compilation-config", """
- '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[5,6,10,12,15,18,20,24,25,30,35,36,40,42,48]}'""",
- "--speculative-config", """
- '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}'""",
- "--tool-call-parser", "deepseek_v41",
- "--enable-auto-tool-choice",
- "--reasoning-parser", "deepseek_v41",
- "--default-chat-template-kwargs", """
- '{"thinking": false}'""",
- ]
- [env]
- HF_HUB_OFFLINE = "1"
- # Engram tables stay on disk (the baked engram.py patch); 32 parallel row
- # reads, 16 MB chunks (their launch defaults)
- DSV41_ENGRAM_DISK = "1"
- DSV41_ENGRAM_DISK_THREADS = "32"
- DSV41_ENGRAM_DISK_CHUNK = "16"
- # CUDA graphs: the Engram rows are staged in prepare_inputs (baked
- # model_state.py prestage patch) so the whole decode step is capturable
- VLLM_USE_BREAKABLE_CUDAGRAPH = "1"
- VLLM_HAS_FLASHINFER_CUBIN = "1"
- VLLM_USE_RUST_FRONTEND = "0"
- # runtime JIT wedge countermeasure (their boot-3 lesson): kernels are
- # prebuilt in the image, but any residual JIT must not OOM the box
- MAX_JOBS = "2"
- VLLM_ENGINE_READY_TIMEOUT_S = "3600"
- PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
- TORCH_CUDA_ARCH_LIST = "12.1a"
- FLASHINFER_CUDA_ARCH_LIST = "12.1a"
- OMP_NUM_THREADS = "2"
- # JIT caches persist per-node (self-namespace by engine version key; the
- # image prebakes the flashinfer 0.7.0rc1 SM120 kernels, first boots re-JIT
- # the rest)
- [[extra_mounts]]
- host = "~/sm121-cache/jit-121f/flashinfer"
- container = "/root/.cache/flashinfer"
- [[extra_mounts]]
- host = "~/sm121-cache/jit-121f/torchinductor"
- container = "/root/.cache/torchinductor"
- [[extra_mounts]]
- host = "~/sm121-cache/jit-121f/vllm"
- container = "/root/.cache/vllm"
|
