# Qwen3.8-27B (BF16) — 1× DGX Spark / GB10 (sm_121), TP=1.
#
# Native BF16 dense 27B (52 GB). Quality reference baseline; the quantized
# variants (FP8 / NVFP4 / Int4) are the speed candidates. Runs text + vision.
#
# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
#   vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
#   instanttensor. Stock vLLM — supports the qwen3_5 arch + MTP.
#
# Model: Qwen/Qwen3.8-27B
#   https://huggingface.co/Qwen/Qwen3.8-27B
#   Dense 27B, 64 layers, hidden 5120, GQA (4 KV heads × 256), Gated
#   DeltaNet + Gated Attention hybrid. MTP-1. Native 262,144 ctx (YaRN to 1M).
#   Multimodal (vision tower, `language_model_only: False`).
#
# PREREQUISITES:
#   Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
#       huggingface-cli download Qwen/Qwen3.8-27B
#
# REFERENCE:
#   https://forums.developer.nvidia.com/t/qwen3-8-27b-mixedint4-autoround-optimized-for-a-single-dgx-spark/380248
#   https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
#   https://forums.developer.nvidia.com/t/qwen3-8-27b-at-34-38-tok-s-on-dgx-spark-open-source-one-command-setup-sglang-nvfp4-dspark/380257

engine = "vllm"
name = "Qwen3.8-27B (BF16)"
docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
model = "Qwen/Qwen3.8-27B"
required_nodes = 1
tensor_parallel_size = 1
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

entrypoint = ["vllm", "serve"]

args = [
    # KV cache & memory
    "--kv-cache-dtype", "fp8",
    "--max-model-len", "262144",
    "--max-num-seqs", "6",
    "--max-num-batched-tokens", "8192",
    "--gpu-memory-utilization", "0.75",
    "--served-model-name", "qwen3.8-27b",
    # Engine backends (dense GQA — FlashInfer, not the DeepSeek MLA path)
    "--attention-backend", "flashinfer",
    # Compilation / perf
    "--optimization-level", "3",
    "--performance-mode", "throughput",
    # Chunked prefill + prefix caching
    "--enable-chunked-prefill",
    "--enable-prefix-caching",
    "--enable-flashinfer-autotune",
    # MTP speculative decoding (in-checkpoint draft, mtp_num_hidden_layers=1
    # cycled across steps)
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":5}'""",
    # Tokenizer / reasoning (Qwen3 family)
    "--tool-call-parser", "qwen3_xml",
    "--reasoning-parser", "qwen3",
    "--enable-auto-tool-choice",
    "--generation-config", "auto",
    "--default-chat-template-kwargs.preserve_thinking=true",
    # Proven generation defaults (thinking mode; Qwen3.8 model-card best
    # practices). repetition_penalty 1.0 = identity: accepted but never
    # deviates from no-op during req handling (DSpark/MLA bug caveat — see
    # findings doc for the DeepSeek path; irrelevant to dense Qwen).
    "--override-generation-config", """
'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
    "--trust-remote-code",
]

[env]
# Offline mode — weights must be cached on each node
HF_HUB_OFFLINE = "1"

# Architecture
TORCH_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
OMP_NUM_THREADS = "4"

# GPU memory management
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"

# GPU device isolation
CUDA_VISIBLE_DEVICES = "0"
CUDA_DEVICE_ORDER = "PCI_BUS_ID"
CUDA_CACHE_MAXSIZE = "4294967296"