| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697 |
- # Qwen3.8-27B (Pilcothink Int4 AutoRound + MTP) — 1× DGX Spark / GB10 (sm_121),
- # TP=1.
- #
- # Mixed 4-bit AutoRound quant (20 GB) tuned for a single DGX Spark. Sensitive
- # layers kept FP8/FP16; vision layers unquantized. MTP quantized + included.
- # Dense 27B, GQA, hybrid DeltaNet+Attention. Text + vision. The community has
- # heavily tuned vLLM flags for this exact build (thread 380248 post #39).
- #
- # Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
- # vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
- # instanttensor. Stock vLLM — supports the qwen3_5 arch + MTP.
- #
- # Model: Pilcothink/Qwen3.8-27B-MixedInt4-AutoRound
- # https://huggingface.co/Pilcothink/Qwen3.8-27B-MixedInt4-AutoRound
- #
- # PREREQUISITES:
- # Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
- # huggingface-cli download Pilcothink/Qwen3.8-27B-MixedInt4-AutoRound
- #
- # REFERENCE:
- # https://forums.developer.nvidia.com/t/qwen3-8-27b-mixedint4-autoround-optimized-for-a-single-dgx-spark/380248
- engine = "vllm"
- name = "Qwen3.8-27B (Int4 AutoRound)"
- docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
- model = "Pilcothink/Qwen3.8-27B-MixedInt4-AutoRound"
- parent_model = "Qwen/Qwen3.8-27B"
- required_nodes = 1
- tensor_parallel_size = 1
- pipeline_parallel_size = 1
- distributed_backend = "mp"
- shm_size = "32gb"
- entrypoint = ["vllm", "serve"]
- args = [
- # KV cache & memory
- "--kv-cache-dtype", "fp8",
- "--max-model-len", "262144",
- "--max-num-seqs", "6",
- "--max-num-batched-tokens", "8192",
- "--gpu-memory-utilization", "0.75",
- "--served-model-name", "qwen3.8-27b",
- # Engine backends (dense GQA — FlashInfer)
- "--attention-backend", "flashinfer",
- # Compilation / perf (AutoRound community tuning)
- "--optimization-level", "3",
- "--performance-mode", "throughput",
- # Chunked prefill + prefix caching
- "--enable-chunked-prefill",
- "--enable-prefix-caching",
- "--enable-flashinfer-autotune",
- # MTP speculative decoding (quantized draft)
- "--speculative-config", """
- '{"method":"mtp","num_speculative_tokens":4}'""",
- # Tokenizer / reasoning (Qwen3 family)
- "--tool-call-parser", "qwen3_xml",
- "--reasoning-parser", "qwen3",
- "--enable-auto-tool-choice",
- "--generation-config", "auto",
- "--default-chat-template-kwargs.preserve_thinking=true",
- # Proven generation defaults (thinking mode; Qwen3.8 model-card best
- # practices)
- "--override-generation-config", """
- '{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
- "--trust-remote-code",
- ]
- [env]
- # AutoRound community tuning (thread 380248 post #39): faster load + higher
- # compile level for the 4-bit kernels.
- VLLM_TORCH_COMPILE_LEVEL = "3"
- VLLM_TRITON_COMPILE_ON_LOAD = "1"
- VLLM_WARMUP_BEFORE_SERVING = "1"
- VLLM_SKIP_WARMUP = "false"
- TORCH_MATMUL_PRECISION = "high"
- NVIDIA_FORWARD_COMPAT = "1"
- NVIDIA_DISABLE_REQUIRE = "1"
- CUDA_DEVICE_MAX_CONNECTIONS = "1"
- VLLM_MARLIN_USE_ATOMIC_ADD = "1"
- FLASHINFER_DISABLE_VERSION_CHECK = "1"
- # Offline mode
- HF_HUB_OFFLINE = "1"
- # Architecture
- TORCH_CUDA_ARCH_LIST = "12.1a"
- CUTE_DSL_ARCH = "sm_121a"
- OMP_NUM_THREADS = "4"
- # GPU memory management
- PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
- # GPU device isolation
- CUDA_VISIBLE_DEVICES = "0"
- CUDA_DEVICE_ORDER = "PCI_BUS_ID"
- CUDA_CACHE_MAXSIZE = "4294967296"
|
