engine = "vllm"
name = "DeepSeek V4 Flash (vLLM)"
docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-ready"
model = "deepseek-ai/DeepSeek-V4-Flash"
port = 8000
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"

# DeepSeek V4 with 1M context needs much more shared memory than the
# default 10g.  KV cache + activations for 1M tokens fills up fast.
shm_size = "64gb"

args = [
    "--kv-cache-dtype", "fp8",
    "--block-size", "256",
    "--max-model-len", "1000000",
    "--max-num-seqs", "6",
    "--max-num-batched-tokens", "8192",
    "--gpu-memory-utilization", "0.82",
    "--enable-prefix-caching",
    "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":2}",
    "--tokenizer-mode", "deepseek_v4",
    "--tool-call-parser", "deepseek_v4",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "deepseek_v4",
    "--reasoning-config", "{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}",
    "--default-chat-template-kwargs.thinking", "true",
    "--default-chat-template-kwargs.reasoning_effort", "high",
    "--enable-flashinfer-autotune",
    "--host", "0.0.0.0",
    "--port", "8000",
    "--trust-remote-code",
    "--served-model-name", "deepseek-v4-flash",
]

[env]
# HuggingFace / cache config
HF_HUB_OFFLINE = "0"
VLLM_CACHE_ROOT = "/cache/huggingface/vllm-cache"

# vLLM features for DeepSeek V4 on Blackwell (GB10 / sm_120)
VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
VLLM_USE_B12X_MOE = "1"
VLLM_SPARSE_INDEXER_MAX_LOGITS_MB = "256"

# Compile-time arch hints for PyTorch and FlashInfer on GB10
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1a"