| 123456789101112131415161718192021222324252627282930313233343536373839 |
- # Ling-3.1-flash (inclusionAI) fp8 on 8x DGX Spark.
- #
- # 2026-10-16: the fp8 checkpoint is on the NAS HF cache (535 GB, snapshot
- # a4ccd6b4). Arch `BailingMoeV3ForCausalLM` (+ `BailingMoeV3MTPModel`) is
- # registered in stock vLLM v0.31.0 — no nightly, no custom image. 535 GB / 8 =
- # ~67 GB weights/rank, fits a GB10 with KV headroom.
- #
- # Base: stock `vllm/vllm-openai:v0.31.0` (prebuilt, arch built-in) — NOT the
- # kindling image (its overlays are GLM-5.3-specific). `--trust-remote-code` for
- # the model's custom_code auto_map (vLLM still prefers its native bailing_moe_v3
- # impl). max-model-len kept modest (65536) to stay clear of the GB10 unified-
- # memory boot gate (vLLM #48140 reads MemFree; 128K+ can refuse to boot) — raise
- # once it's proven up. GMU 0.78 (the measured GB10 safe ceiling).
- #
- # GB10/sm_121a fp8-MoE feasibility is UNVERIFIED for this arch — first live
- # probe. See specs/tech-architecture/upstream-watch-2026-10-16.md.
- engine = "vllm"
- name = "Ling-3.1-flash fp8 (8x, stock v0.31.0)"
- docker_image = "vllm/vllm-openai:v0.31.0"
- entrypoint = ["vllm", "serve"]
- model = "inclusionAI/Ling-3.1-flash"
- parent_model = "inclusionAI/Ling-3.1-flash"
- port = 30000
- required_nodes = 8
- tensor_parallel_size = 8
- pipeline_parallel_size = 1
- distributed_backend = "mp"
- args = [
- "--served-model-name", "inclusionAI/Ling-3.1-flash",
- "--trust-remote-code",
- "--max-model-len", "65536",
- "--gpu-memory-utilization", "0.78",
- "--max-num-seqs", "32",
- "--max-num-batched-tokens", "8192",
- ]
- [env]
- VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|
