|
|
@@ -0,0 +1,66 @@
|
|
|
+# NVIDIA GLM-5.3-Flash-FP4-Local — official NVFP4 (QAD) GLM-5.3-Flash, 2x DGX Spark.
|
|
|
+# Source: model card https://huggingface.co/nvidia/GLM-5.3-Flash-FP4-Local
|
|
|
+# ("2 x NVIDIA DGX Spark" section, transcribed). Quantized with nvidia-modelopt
|
|
|
+# v0.47.0 via quantization-aware distillation (QAD). NVFP4 routed experts
|
|
|
+# (E2M1 + E4M3 block scales + FP32 global scale). 2M tokens FP8 KV on 2x Spark.
|
|
|
+#
|
|
|
+# CRITICAL: the NVFP4 scale factors use a NEW lossless-compression format that
|
|
|
+# upstream vLLM does NOT yet support — MUST run on eugr/spark-vllm-b12x (b12x
|
|
|
+# backend + --load-format b12x). Digest-pinned (arm64, built 2026-10-15).
|
|
|
+# This is the official NVFP4 counterpart to our kindling 2x lane (our own r35
|
|
|
+# build) — a direct A/B vs the 95/100 kindling anchor.
|
|
|
+#
|
|
|
+# The launcher supplies the NCCL/socket/IB env from the cluster config
|
|
|
+# (NCCL_IB_HCA, NCCL_SOCKET_IFNAME, MN_IF_NAME, UCX_NET_DEVICES, GLOO/TP_SOCKET,
|
|
|
+# NCCL_IB_DISABLE, master addr/port). Only the b12x + RoCE-allreduce + AOT env
|
|
|
+# from the card's "both nodes" block is added here.
|
|
|
+
|
|
|
+engine = "vllm"
|
|
|
+name = "GLM-5.3-Flash-FP4-Local (2x Spark, NVFP4 QAD, MTP n5)"
|
|
|
+docker_image = "eugr/spark-vllm-b12x@sha256:03a956544e9f9b4b7ba91690469fbe7848bdbef29a7d2e0f5788368467bb3595"
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+model = "nvidia/GLM-5.3-Flash-FP4-Local"
|
|
|
+parent_model = "zai-org/GLM-5.3-Flash"
|
|
|
+port = 30000
|
|
|
+required_nodes = 2
|
|
|
+tensor_parallel_size = 2
|
|
|
+args = [
|
|
|
+ "--served-model-name", "glm53-fp4-local",
|
|
|
+ "--mamba-cache-mode", "align",
|
|
|
+ "--enable-prefix-caching",
|
|
|
+ "--enable-chunked-prefill",
|
|
|
+ "--kv-cache-dtype", "fp8",
|
|
|
+ "--attention-backend", "B12X",
|
|
|
+ "--block-size", "256",
|
|
|
+ "--moe-backend", "b12x",
|
|
|
+ "--linear-backend", "b12x",
|
|
|
+ "--load-format", "b12x",
|
|
|
+ "--max-model-len", "500000",
|
|
|
+ "--reasoning-parser", "glm45",
|
|
|
+ "--tool-call-parser", "glm47",
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
+ "--gpu-memory-utilization", "0.87",
|
|
|
+ "--speculative-config", """
|
|
|
+'{"method":"mtp","num_speculative_tokens":5,"moe_backend":"humming","attention_backend":"B12X"}'""",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+# b12x MoE/FP4 path (card "both nodes" block)
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
+SAFETENSORS_FAST_GPU = "1"
|
|
|
+B12X_POLICY_MODE = "auto"
|
|
|
+VLLM_B12X_MOE_FP4_FORCE_A16 = "1"
|
|
|
+B12X_W4A16_A4_PREFILL = "1"
|
|
|
+# RoCE allreduce + NCCL tuning (card)
|
|
|
+VLLM_ENABLE_ROCE_ALLREDUCE = "1"
|
|
|
+VLLM_ROCE_ALLREDUCE_MAX_SIZE = "2MB"
|
|
|
+NCCL_BUFFSIZE = "1048576"
|
|
|
+NCCL_MAX_CTAS = "8"
|
|
|
+VLLM_ENABLE_PCIE_ALLREDUCE = "0"
|
|
|
+# vLLM runner / compile flags (card)
|
|
|
+VLLM_WORKER_MULTIPROC_METHOD = "spawn"
|
|
|
+VLLM_SSM_CONV_STATE_LAYOUT = "DS"
|
|
|
+VLLM_USE_AOT_COMPILE = "1"
|
|
|
+VLLM_USE_MEGA_AOT_ARTIFACT = "1"
|
|
|
+VLLM_USE_V2_MODEL_RUNNER = "1"
|
|
|
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|
