| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990 |
- # Qwen3.8-27B (unsloth NVFP4 + DSpark) — 1× DGX Spark / GB10 (sm_121), TP=1.
- #
- # NVFP4 compressed-tensors quant (22 GB) with a dedicated DSpark draft model
- # (RadixArk, 2.6 GB, block_size 7) for speculative decoding. Dense 27B, GQA,
- # hybrid DeltaNet+Attention. Text + vision. Unsloth Dynamic V3.0 (preview)
- # quant — SOTA quality for NVFP4. vLLM-only (lm_head FP8).
- #
- # DSpark vs the in-checkpoint MTP head (qwen38-27b-nvfp4-unsloth recipe): the
- # draft is 3× cheaper per guess and edits/rewrites much faster, at the cost of
- # a 2.6 GB draft + a local arch-patched copy. Benchmark both.
- #
- # Image: ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2
- # vLLM 0.27.1, torch 2.13+cuda130, FlashInfer built from source,
- # instanttensor. Stock vLLM — supports qwen3_5 arch + method "dspark".
- #
- # Model: unsloth/Qwen3.8-27B-NVFP4
- # https://huggingface.co/unsloth/Qwen3.8-27B-NVFP4
- #
- # PREREQUISITES:
- # Weights cached on every node (dgx1 ~/.cache/huggingface → NAS):
- # huggingface-cli download unsloth/Qwen3.8-27B-NVFP4
- # hfrs download RadixArk/Qwen3.8-27B-DSpark (then cp -aL + patch arch → /home/kilork/dspark-draft)
- #
- # REFERENCE:
- # https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark (DSpark section)
- engine = "vllm"
- name = "Qwen3.8-27B (NVFP4 unsloth + DSpark)"
- docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.27.1-gb10.2"
- model = "unsloth/Qwen3.8-27B-NVFP4"
- required_nodes = 1
- tensor_parallel_size = 1
- pipeline_parallel_size = 1
- distributed_backend = "mp"
- shm_size = "32gb"
- entrypoint = ["vllm", "serve"]
- args = [
- # KV cache & memory
- "--kv-cache-dtype", "fp8",
- "--max-model-len", "262144",
- "--max-num-seqs", "6",
- "--max-num-batched-tokens", "16384",
- "--gpu-memory-utilization", "0.75",
- "--served-model-name", "qwen3.8-27b",
- # Engine backends (dense GQA — FlashInfer; NVFP4 route via compressed-tensors)
- "--attention-backend", "flashinfer",
- # Compilation / perf
- "--optimization-level", "3",
- "--performance-mode", "throughput",
- # Chunked prefill + prefix caching
- "--enable-chunked-prefill",
- "--enable-prefix-caching",
- "--enable-flashinfer-autotune",
- # DSpark speculative decoding — dedicated draft model (arch-patched local
- # copy of RadixArk/Qwen3.8-27B-DSpark, re-labelled `Qwen3DSparkModel` so
- # vLLM routes it to qwen3_dspark not the DeepSeek-V4 path). block_size 7.
- # batched-tokens raised to 16384 to fit the draft token slots (warn:
- # max_num_scheduled_tokens=8156 at 8192).
- "--speculative-config", """
- '{"method":"dspark","model":"/dspark-draft","num_speculative_tokens":7}'""",
- # Tokenizer / reasoning (Qwen3 family)
- "--tool-call-parser", "qwen3_xml",
- "--reasoning-parser", "qwen3",
- "--enable-auto-tool-choice",
- "--generation-config", "auto",
- "--default-chat-template-kwargs.preserve_thinking=true",
- # Proven generation defaults (thinking mode; Qwen3.8 model-card best
- # practices)
- "--override-generation-config", """
- '{"temperature":1.0,"top_p":0.95,"top_k":20,"min_p":0.0,"presence_penalty":0.0,"repetition_penalty":1.0}'""",
- "--trust-remote-code",
- ]
- # DSpark draft model — arch-patched local copy of RadixArk/Qwen3.8-27B-DSpark.
- [[extra_mounts]]
- host = "/home/kilork/dspark-draft"
- container = "/dspark-draft"
- [env]
- HF_HUB_OFFLINE = "1"
- TORCH_CUDA_ARCH_LIST = "12.1a"
- CUTE_DSL_ARCH = "sm_121a"
- OMP_NUM_THREADS = "4"
- PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
- CUDA_VISIBLE_DEVICES = "0"
- CUDA_DEVICE_ORDER = "PCI_BUS_ID"
- CUDA_CACHE_MAXSIZE = "4294967296"
|
