| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374 |
- # GLM-5.3-Flash NVFP4 — local-inference-lab quant (vLLM) — 2× DGX Spark, TP=2.
- #
- # local-inference-lab's mixed-precision quant: main MoE experts NVFP4
- # (group-16), MTP layer-45 routed experts in MXFP8, W4A4 input scales
- # included. tool-eval-bench 89/100 (vs 85 LibertAIDAI, 90 official FP8) —
- # best quant for agents so far.
- # Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4
- # (~186 GB → ~93 GB/node on 2 Sparks; GMU 0.85 = ~101 GiB budget)
- #
- # Image: shared glm53-flash-nvfp4:local overlay (day-0 vLLM image + the
- # NoPE-MLA / sm120 sparse-MLA mod). Rebuilt from docker-images/glm53-flash-nvfp4/.
- #
- # ⚠ KNOWN DAY-0 ISSUE (forum 381350 #41): vLLM needs a small `modelopt.py`
- # fix for THIS quant — the MTP weights are stored under
- # `language_model.model.layers.45.*` while the quant config lists
- # `model.language_model.layers.45.*`, so the drafter's scale lookup misses.
- # Until the fix is vendored into the overlay (as patch part 5), a boot with
- # --speculative-config may fail at MTP weight load. Fallback: comment the
- # speculative-config line → clean baseline serve (~15 tok/s class).
- #
- # GB10 gotchas handled here (same as glm53-flash-nvfp4 / LibertAIDAI):
- # --moe-backend marlin (silent FP4 MoE corruption fix)
- # VLLM_ENGINE_READY_TIMEOUT_S (60-min FlashInfer autotune is not a hang)
- # --language-model-only (vision front-end OOM-kills on GB10)
- #
- # PREREQUISITES: weights cached on both nodes:
- # huggingface-cli download local-inference-lab/GLM-5.3-Flash-NVFP4
- #
- # REFERENCE:
- # https://forums.developer.nvidia.com/t/glm-5-3-flash-320b-total-parameters-18b-active/381350/41
- # https://github.com/kingjones30/GLM-5.3-Flash-2x-DGX-Spark (mod + base config)
- engine = "vllm"
- name = "GLM-5.3-Flash NVFP4-lab (vLLM, 2x Spark)"
- docker_image = "glm53-flash-nvfp4:local"
- docker_image_dir = "glm53-flash-nvfp4"
- model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
- required_nodes = 2
- tensor_parallel_size = 2
- pipeline_parallel_size = 1
- distributed_backend = "mp"
- shm_size = "32gb"
- args = [
- # Stable served name — quant-agnostic (shared across glm53 quants)
- "--served-model-name", "glm53",
- "--trust-remote-code",
- "--distributed-executor-backend", "mp",
- # ~93 GiB weights/node of 119.6 GiB; drop to 0.83 or 262144→131072 if the
- # UMA MemFree gate (vLLM #48140) refuses boot
- "--gpu-memory-utilization", "0.85",
- "--max-model-len", "262144",
- "--max-num-seqs", "1",
- "--kv-cache-dtype", "fp8_ds_mla",
- "--language-model-only",
- "--moe-backend", "marlin",
- # MTP-5 — REQUIRES the modelopt.py prefix fix (header). Remove if load aborts.
- "--speculative-config", """
- '{"method":"mtp","num_speculative_tokens":5}'""",
- "--tool-call-parser", "glm47",
- "--enable-auto-tool-choice",
- "--reasoning-parser", "glm45",
- ]
- [env]
- HF_HUB_OFFLINE = "1"
- # Activates the NoPE-MLA rope-pad mod baked into the image
- VLLM_MLA_NOPE_PAD_ROPE = "1"
- # FlashInfer autotune can legitimately loop 60+ min on first boot
- VLLM_ENGINE_READY_TIMEOUT_S = "3600"
- TORCH_CUDA_ARCH_LIST = "12.1a"
- CUTE_DSL_ARCH = "sm_121a"
- OMP_NUM_THREADS = "4"
|
