| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647 |
- engine = "vllm"
- name = "DeepSeek V4 Flash (vLLM)"
- docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-ready"
- model = "deepseek-ai/DeepSeek-V4-Flash"
- required_nodes = 2
- tensor_parallel_size = 2
- pipeline_parallel_size = 1
- distributed_backend = "mp"
- # DeepSeek V4 with 1M context needs much more shared memory than the
- # default 10g. KV cache + activations for 1M tokens fills up fast.
- shm_size = "64gb"
- args = [
- "--kv-cache-dtype", "fp8",
- "--block-size", "256",
- "--max-model-len", "1048576",
- "--max-num-seqs", "6",
- "--max-num-batched-tokens", "8192",
- "--gpu-memory-utilization", "0.82",
- "--enable-prefix-caching",
- "--speculative-config", '{"method":"mtp","num_speculative_tokens":2}',
- "--tokenizer-mode", "deepseek_v4",
- "--tool-call-parser", "deepseek_v4",
- "--enable-auto-tool-choice",
- "--reasoning-parser", "deepseek_v4",
- "--reasoning-config", '{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}',
- "--default-chat-template-kwargs.thinking", "true",
- "--default-chat-template-kwargs.reasoning_effort", "high",
- "--enable-flashinfer-autotune",
- "--trust-remote-code",
- "--served-model-name", "deepseek-v4-flash", "default",
- ]
- [env]
- # HuggingFace / cache config
- HF_HUB_OFFLINE = "0"
- VLLM_CACHE_ROOT = "/cache/huggingface/vllm-cache"
- # vLLM features for DeepSeek V4 on Blackwell (GB10 / sm_120)
- VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
- VLLM_USE_B12X_MOE = "1"
- VLLM_SPARSE_INDEXER_MAX_LOGITS_MB = "256"
- # Compile-time arch hints for PyTorch and FlashInfer on GB10
- TORCH_CUDA_ARCH_LIST = "12.1a"
- FLASHINFER_CUDA_ARCH_LIST = "12.1a"
|
