|
|
@@ -0,0 +1,75 @@
|
|
|
+# Qwen3.8-27B (FP8) — 1× DGX Spark / GB10 (sm_121), TP=1.
|
|
|
+#
|
|
|
+# Official Qwen FP8 quant (29 GB). Dense 27B, GQA, hybrid DeltaNet+Attention,
|
|
|
+# MTP-1. Text + vision. As fast as NVFP4 typically, with full FP8 quality.
|
|
|
+#
|
|
|
+# Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
|
|
|
+# vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
|
|
|
+# instanttensor. Stock vLLM — supports the qwen3_5 arch + MTP.
|
|
|
+#
|
|
|
+# Model: Qwen/Qwen3.8-27B-FP8
|
|
|
+# https://huggingface.co/Qwen/Qwen3.8-27B-FP8
|
|
|
+#
|
|
|
+# PREREQUISITES:
|
|
|
+# Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
|
|
|
+# huggingface-cli download Qwen/Qwen3.8-27B-FP8
|
|
|
+#
|
|
|
+# REFERENCE:
|
|
|
+# https://forums.developer.nvidia.com/t/qwen3-8-27b-mixedint4-autoround-optimized-for-a-single-dgx-spark/380248
|
|
|
+# https://forums.developer.nvidia.com/t/qwen3-8-27b-on-dual-sparks/380350
|
|
|
+
|
|
|
+engine = "vllm"
|
|
|
+name = "Qwen3.8-27B (FP8)"
|
|
|
+docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
|
|
|
+model = "Qwen/Qwen3.8-27B-FP8"
|
|
|
+parent_model = "Qwen/Qwen3.8-27B"
|
|
|
+required_nodes = 1
|
|
|
+tensor_parallel_size = 1
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+shm_size = "32gb"
|
|
|
+
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+
|
|
|
+args = [
|
|
|
+ # KV cache & memory
|
|
|
+ "--kv-cache-dtype", "fp8",
|
|
|
+ "--max-model-len", "262144",
|
|
|
+ "--max-num-seqs", "6",
|
|
|
+ "--max-num-batched-tokens", "8192",
|
|
|
+ "--gpu-memory-utilization", "0.75",
|
|
|
+ "--served-model-name", "qwen3.8-27b",
|
|
|
+ # Engine backends (dense GQA — FlashInfer)
|
|
|
+ "--attention-backend", "flashinfer",
|
|
|
+ # Compilation / perf
|
|
|
+ "--optimization-level", "3",
|
|
|
+ "--performance-mode", "throughput",
|
|
|
+ # Chunked prefill + prefix caching
|
|
|
+ "--enable-chunked-prefill",
|
|
|
+ "--enable-prefix-caching",
|
|
|
+ "--enable-flashinfer-autotune",
|
|
|
+ # MTP speculative decoding (in-checkpoint draft)
|
|
|
+ "--speculative-config", """
|
|
|
+'{"method":"mtp","num_speculative_tokens":5}'""",
|
|
|
+ # Tokenizer / reasoning (Qwen3 family)
|
|
|
+ "--tool-call-parser", "qwen3_xml",
|
|
|
+ "--reasoning-parser", "qwen3",
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
+ "--generation-config", "auto",
|
|
|
+ "--default-chat-template-kwargs.preserve_thinking=true",
|
|
|
+ # Proven generation defaults (thinking mode; Qwen3.8 model-card best
|
|
|
+ # practices)
|
|
|
+ "--override-generation-config", """
|
|
|
+'{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
|
|
|
+ "--trust-remote-code",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+HF_HUB_OFFLINE = "1"
|
|
|
+TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
+OMP_NUM_THREADS = "4"
|
|
|
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
+CUDA_VISIBLE_DEVICES = "0"
|
|
|
+CUDA_DEVICE_ORDER = "PCI_BUS_ID"
|
|
|
+CUDA_CACHE_MAXSIZE = "4294967296"
|
