| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172 |
- # Qwen3.8-Flash-Next-FP8 (SGLang day-0) — 2× DGX Spark / GB10 (sm_121), TP=2.
- #
- # Qwen4-architecture preview: 125B-A6B MoE (512 experts, 10 routed + 1 shared),
- # Gated DeltaNet + Qwen Sparse Attention, 51B n-gram "engram" table, 4B
- # in-checkpoint MTP. FP8 fine-grained (block 128). Text + vision. Native 262K
- # ctx (YaRN to 1M). Model: https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8
- #
- # Engine: SGLang day-0 — the ONLY engine with qwen4_exp today. vLLM support
- # is an open PR (#53896), merged into no GB10 fork image (aidendle94 / eugr /
- # local-inference-lab are all pre-release pins).
- #
- # Image: lmsysorg/sglang:qwen38flashnext — multi-arch incl. arm64 (digest
- # sha256:14ed58251…; re-verify at serve time). Day-0 = boots, not tuned.
- #
- # Memory: 172.8 GiB weights → 86.4 GiB/node. SGLang claims the 51B engram
- # table offloads to host RAM (async prefetch) — if active, the GPU budget
- # shrinks and a larger KV pool / longer ctx is possible.
- #
- # PREREQUISITES:
- # Weights cached on every node (dgx1/dgx2 ~/.cache/huggingface → NAS):
- # huggingface-cli download Qwen/Qwen3.8-Flash-Next-FP8
- #
- # REFERENCE:
- # https://forums.developer.nvidia.com/t/qwen3-8-flash-next/381228
- engine = "sglang"
- name = "Qwen3.8-Flash-Next FP8 (SGLang day-0)"
- docker_image = "lmsysorg/sglang:qwen38flashnext"
- model = "Qwen/Qwen3.8-Flash-Next-FP8"
- required_nodes = 2
- tensor_parallel_size = 2
- pipeline_parallel_size = 1
- distributed_backend = "mp"
- shm_size = "32gb"
- entrypoint = ["python3", "-m", "sglang.launch_server"]
- args = [
- # Memory — weights ~86.4 GiB of 119.6 GiB/node; the rest goes to KV +
- # activations. Undervalue to be safe on day-0; raise after measuring.
- "--mem-fraction-static", "0.80",
- "--max-running-requests", "10",
- # GDN (DeltaNet) hybrid state pool
- "--max-mamba-cache-size", "30",
- # Attention / KV
- "--attention-backend", "flashinfer",
- "--kv-cache-dtype", "fp8_e4m3",
- "--mamba-ssm-dtype", "bfloat16",
- "--mamba-full-memory-ratio", "4.21",
- "--mamba-radix-cache-strategy", "extra_buffer_lazy",
- # Chunked prefill
- "--chunked-prefill-size", "8192",
- # Compile + decode graphs
- "--enable-torch-compile", "--torch-compile-max-bs", "4",
- "--cuda-graph-max-bs-decode", "4",
- "--num-continuous-decode-steps", "2",
- # Tokenizer / reasoning / tool calling (Qwen3 family)
- "--reasoning-parser", "qwen3",
- "--tool-call-parser", "qwen3_coder",
- # Observability
- "--enable-metrics",
- "--enable-cache-report",
- ]
- [env]
- # Offline — weights must be cached on each node
- HF_HUB_OFFLINE = "1"
- # Architecture
- TORCH_CUDA_ARCH_LIST = "12.1a"
- CUTE_DSL_ARCH = "sm_121a"
- OMP_NUM_THREADS = "4"
|
