| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112 |
- # GLM-5.2 QuantTrio Int4-Int8Mix — 4× DGX Spark / GB10 (sm_121), TP=4.
- #
- # Image: sparkrun-vllm-ds4-gb10:glm52
- # Built from docker-images/glm52-int4-int8/Dockerfile, which overlays
- # sparkrun-vllm-ds4-gb10:fixed with an entrypoint wrapper that unsets
- # the baked VLLM_PREFIX_CACHE_RETENTION_INTERVAL before calling the
- # original /opt/nvidia/nvidia_entrypoint.sh.
- #
- # Model: QuantTrio/GLM-5.2-Int4-Int8Mix (mixed Int4/Int8, 10% expert-pruned)
- # → https://huggingface.co/QuantTrio/GLM-5.2-Int4-Int8Mix
- # Includes in-checkpoint MTP draft model.
- #
- # PREREQUISITES:
- # - Weights cached on every node:
- # huggingface-cli download QuantTrio/GLM-5.2-Int4-Int8Mix
- #
- # REFERENCE:
- # https://forums.developer.nvidia.com/t/glm-5-2-int4-int8-on-8x-gb10-1200-t-s-prefill-33-54-t-s-avg-decode-generic-coding-structured/376831
- # https://github.com/ciprianveg/gb10-glm-5.2
- engine = "vllm"
- name = "GLM-5.2 Int4-Int8Mix (vLLM, DS4 image)"
- docker_image = "sparkrun-vllm-ds4-gb10:glm52"
- model = "QuantTrio/GLM-5.2-Int4-Int8Mix"
- port = 8210
- required_nodes = 4
- tensor_parallel_size = 4
- pipeline_parallel_size = 1
- distributed_backend = "mp"
- shm_size = "64gb"
- docker_image_dir = "glm52-int4-int8"
- entrypoint = ["vllm", "serve"]
- args = [
- # KV cache & memory
- "--kv-cache-dtype", "fp8",
- "--max-model-len", "150000",
- "--max-num-seqs", "1",
- "--max-num-batched-tokens", "4096",
- "--gpu-memory-utilization", "0.91",
- # Engine backends
- "--attention-backend", "B12X_MLA_SPARSE",
- "--moe-backend", "b12x",
- # Compilation
- "--compilation-config", """
- '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
- # Chunked prefill + prefix caching
- "--enable-chunked-prefill",
- "--enable-prefix-caching",
- "--enable-flashinfer-autotune",
- # MTP speculative decoding (in-checkpoint draft model)
- # "--speculative-config", """
- #'{"method":"mtp","num_speculative_tokens":4,"moe_backend":"b12x","draft_sample_method":"probabilistic"}'""",
- # Tokenizer / reasoning (GLM-5.2 family)
- "--tool-call-parser", "glm47",
- "--reasoning-parser", "glm45",
- "--enable-auto-tool-choice",
- "--trust-remote-code",
- "--served-model-name", "glm52",
- ]
- [env]
- # b12x suite (image requires this)
- VLLM_USE_B12X_MOE = "1"
- VLLM_USE_B12X_MHC = "1"
- VLLM_USE_B12X_WO_PROJECTION = "1"
- VLLM_USE_B12X_FP8_GEMM = "1"
- VLLM_USE_B12X_SPARSE_INDEXER = "1"
- VLLM_USE_V2_MODEL_RUNNER = "1"
- VLLM_USE_FLASHINFER_SAMPLER = "1"
- VLLM_USE_AOT_COMPILE = "1"
- VLLM_USE_MEGA_AOT_ARTIFACT = "0"
- VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
- VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
- B12X_MLA_SM120_UNIFIED = "1"
- B12X_DENSE_SPLITK_TURBO = "1"
- B12X_W4A16_TC_DECODE = "1"
- B12X_MHC_MAX_TOKENS = "16384"
- # GLM-5.2 support flags
- VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
- VLLM_SPARSE_INDEXER_MAX_LOGITS_MB = "256"
- # Timeout safety net for cold boot (cudagraph + JIT)
- VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "1800"
- # Offline mode — weights must be cached on each node
- HF_HUB_OFFLINE = "1"
- # Architecture
- TORCH_CUDA_ARCH_LIST = "12.1a"
- CUTE_DSL_ARCH = "sm_121a"
- OMP_NUM_THREADS = "4"
- # GPU memory management
- PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
- # NCCL overrides
- NCCL_CROSS_NIC = "1"
- NCCL_CUMEM_ENABLE = "0"
- NCCL_IGNORE_CPU_AFFINITY = "1"
- NCCL_DEBUG = "WARN"
- # PCIe allreduce not needed on GB10 single-NIC
- VLLM_ENABLE_PCIE_ALLREDUCE = "0"
- VLLM_PCIE_ALLREDUCE_BACKEND = "cpp"
- # GPU device isolation
- CUDA_VISIBLE_DEVICES = "0"
- CUDA_DEVICE_ORDER = "PCI_BUS_ID"
- CUDA_CACHE_MAXSIZE = "4294967296"
|
