recipes: glm52-nf4-lloyd-experts-gptq

recipes/glm52-nf4-lloyd-experts-gptq/recipe.toml

@@ -0,0 +1,206 @@
+# GLM-5.2 NF4 Lloyd-Experts-GPTQ — 4× DGX Spark (GB10, sm_121a), TP=4, DCP=4.
+#
+# Model: aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
+#   → https://huggingface.co/aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
+#
+# Lloyd-codebook NF4 routed-expert overlay for GLM-5.2 (744B MoE, 256
+# experts/layer, 8 activated/token, 78 layers, sparse-MLA). Expert weights
+# quantized to 4-bit with a fitted Lloyd–Max codebook (−16.8% expert
+# reconstruction error vs E2M1/MXFP4 baseline on the same GPTQ pipeline).
+# Codes are indices into per-layer grids/L{n}.json — NOT the E2M1 table.
+#
+# LOADING via [composed_model]: the Lloyd repo is experts-only (no config,
+# no tokenizer, no non-expert weights). The launcher builds a /tmp dir per
+# node with symlinks into the HF cache and bind-mounts it at /root/model:
+#   1. aidendle94/GLM-5.2-MXFP4-Experts-GPTQ (base checkpoint — config.json
+#      with quant_method="hybrid_mxfp4_ct", hybrid-ct-*.safetensors non-expert
+#      weights, tokenizer, mtp-draft) → all files symlinked.
+#   2. aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ (Lloyd NF4 experts) →
+#      L*.safetensors symlinked, OVERWRITING the MXFP4 E2M1 experts
+#      (later sources win). Same filenames, same tensor names — the Lloyd
+#      codes replace the E2M1 codes.
+#   3. nf4_layers.json (generated) — tells the kernel which layers use the
+#      nf4_e8m0_k32 decode path vs the default fp4_e8m0_k32 E2M1 bit-twiddle.
+#
+# The symlinks point to container-side paths (/root/.cache/huggingface/...)
+# so they resolve inside the container where the HF cache is mounted. The
+# _NF4_CODEBOOK is GLOBAL and hardcoded in the b12x binary (fitted as the
+# mean of 71 per-layer Lloyd–Max tables) — the per-layer grids/L{n}.json
+# files are reference artifacts, NOT loaded at runtime.
+#
+# Image: sparkrun-vllm-ds4-gb10:glm52
+#   Built from docker-images/glm52/Dockerfile, which FROMs
+#   aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.4 (bakes the
+#   nf4_e8m0_k32 / nf4_packed kernel — inert without VLLM_NF4_LAYERS).
+#
+# PREREQUISITES:
+#   - Both repos cached on every node via HF_HOME:
+#       huggingface-cli download aidendle94/GLM-5.2-MXFP4-Experts-GPTQ
+#       huggingface-cli download aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
+#     (MXFP4 base ~390 GB, Lloyd experts ~385 GB)
+#   No manual overlay or JSON creation — [composed_model] handles everything.
+
+engine = "vllm"
+name = "GLM-5.2 NF4 Lloyd-Experts-GPTQ (vLLM)"
+docker_image = "sparkrun-vllm-ds4-gb10:glm52"
+model = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
+parent_model = "zai-org/GLM-5.2"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "10gb"
+docker_image_dir = "glm52"
+
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Model name
+    "--served-model-name", "glm52",
+    # General
+    "--trust-remote-code",
+    "--reasoning-parser", "glm45",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--enable-prefix-caching",
+    # DCP (Decode Context Parallelism)
+    "--decode-context-parallel-size", "4",
+    "--dcp-comm-backend", "ag_rs",
+    # Engine backends
+    "--attention-backend", "B12X_MLA_SPARSE",
+    # HF overrides — 78-char index_topk_pattern (GLM-5.2 MLP/expert routing)
+    "--hf-overrides", """
+'{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'""",
+    # KV cache — NVFP4 with per-token scaling (nvfp4_ds_mla)
+    "--kv-cache-dtype", "nvfp4_ds_mla",
+    "--kv-cache-memory-bytes", "5500000000",
+    # Context & batch config
+    "--max-model-len", "262144",
+    "--max-num-batched-tokens", "2048",
+    "--max-num-seqs", "3",
+    "--gpu-memory-utilization", "0.89",
+    # Speculative decode — MTP k=4 with adaptive depths
+    # /root/model/mtp-draft is symlinked from the MXFP4 base checkpoint
+    # (the composed dir includes it via source #1).
+    #
+    # NOTE: MTP + reasoning parser (glm45) + response_format (guided decoding)
+    # requires the vLLM structured-output reasoning-boundary fix (upstream
+    # PRs #44297 + #44993, forward-ported in docker-images/glm52/). Without
+    # it, the grammar FSM fails to advance across the reasoning-end marker
+    # (|◊|) under MTP, so the content channel is unconstrained and the model
+    # emits prose/markdown-fence prefixes before JSON. The fix is baked into
+    # the sparkrun-vllm-ds4-gb10:glm52 image.
+    "--speculative-config", """
+'{"model":"/root/model/mtp-draft","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
+    # Compilation — FULL_AND_PIECEWISE with all custom ops, 8 capture sizes
+    "--compilation-config", """
+'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,8]}'""",
+    # Timeout safety net (cold boot: ranks finish minutes apart)
+    "--distributed-timeout-seconds", "3600",
+    "--cpu-distributed-timeout-seconds", "3600",
+]
+
+[composed_model]
+mount = "/root/model"
+
+[[composed_model.sources]]
+repo = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
+
+[[composed_model.sources]]
+repo = "aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ"
+glob = "L*.safetensors"
+# Exclude the 5 layers with mis-packed experts (per model card known-defect
+# workaround). These layers' Lloyd repo L{n}.safetensors contain 255 Lloyd-
+# coded experts + 1 E2M1 expert — decoding as E2M1 (which nf4_layers.json
+# does) would garbage-decode the 255 Lloyd experts. Instead, skip the Lloyd
+# overlay for these files so the MXFP4 base's all-E2M1 L{n}.safetensors
+# survives, and the excluded-from-NF4 E2M1 decode path matches their bytes.
+exclude = [
+    "L5.safetensors",
+    "L8.safetensors",
+    "L9.safetensors",
+    "L32.safetensors",
+    "L46.safetensors",
+]
+
+# NF4 layer list — all 75 MoE layers (3–77) EXCEPT the 5 layers with a
+# known packing defect (per model card 2026-07-27). Layers 5, 8, 9, 32, 46
+# each contain 1 expert whose bytes were packed with E2M1/MXFP4 nibbles
+# (quantizer arm won per-expert, but decoding is per-layer). These layers
+# are ALSO excluded from the Lloyd source above (exclude = [...]) so the
+# MXFP4 base's all-E2M1 L{n}.safetensors survives — decoding as E2M1
+# (which this list does) matches their bytes. Layer 10 had the same defect
+# but was already caught + fixed upstream by the packing guard, so it
+# stays in the NF4 set.
+# Ref: https://huggingface.co/aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
+# ("Known defect: 5 experts are packed on the wrong grid")
+[[composed_model.files]]
+name = "nf4_layers.json"
+content = '{"layers":[3,4,6,7,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,33,34,35,36,37,38,39,40,41,42,43,44,45,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77]}'
+
+# Persistent JIT/torch-inductor cache (avoids ~20 min recompilation on cold boot).
+# `alias = "jit_cache"` resolves via [mount_aliases] in config.toml; if absent,
+# falls back to host "~/.cache/jit" (expanded to /home/<ssh_user>/.cache/jit).
+# `env` injects -e XDG_CACHE_HOME=/root/.cache/jit so vLLM/torch write there.
+[[extra_mounts]]
+alias = "jit_cache"
+host = "~/.cache/jit"
+container = "/root/.cache/jit"
+env = "XDG_CACHE_HOME"
+
+
+[env]
+# ── NF4 Lloyd expert overlay ──────────────────────────────────────────────
+# Path to a JSON file listing which layers use the nf4_e8m0_k32 decode path
+# (vs the default fp4_e8m0_k32 E2M1 bit-twiddle). Generated by [composed_model]
+# at /root/model/nf4_layers.json — no placeholder needed (known container path).
+VLLM_NF4_LAYERS = "/root/model/nf4_layers.json"
+
+# ── b12x suite ────────────────────────────────────────────────────────────
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_DCP_SHARD_DRAFT = "1"
+VLLM_DCP_GLOBAL_TOPK = "1"
+VLLM_DSV4_INDEXER_SP = "1"
+
+# ── MLA CKV gather (sparse gather for decode) ─────────────────────────────
+VLLM_B12X_MLA_CKV_GATHER = "1"
+VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS = "262144"
+VLLM_B12X_MLA_DECODE_SPARSE_GATHER = "1"
+VLLM_B12X_MLA_DECODE_GATHER_V2 = "1"
+
+# ── Speculative decode ────────────────────────────────────────────────────
+VLLM_ADAPTIVE_SPEC_DEPTHS = "2,4"
+
+# ── NVFP4 KV cache ────────────────────────────────────────────────────────
+VLLM_NVFP4_MLA_PER_TOKEN_SCALE = "1"
+
+# ── Timeout safety net ────────────────────────────────────────────────────
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "1800"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+
+# ── MUST unset baked VLLM_PREFIX_CACHE_RETENTION_INTERVAL ─────────────────
+# The image bakes a value that GLM-5.2's KV coordinator rejects.
+# Empty string = unset at runtime.
+VLLM_PREFIX_CACHE_RETENTION_INTERVAL = ""
+
+# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+
+# ── JIT cache ──────────────────────────────────────────────────────────────
+# Persisted via the [[extra_mounts]] jit_cache entry above: XDG_CACHE_HOME is
+# injected from that mount, pointing at /root/.cache/jit (no hardcoded dupe).
+
+# ── NCCL overrides ─────────────────────────────────────────────────────────
+# Cluster config provides NCCL_IB_HCA, NCCL_IB_GID_INDEX, NCCL_SOCKET_IFNAME
+# Launcher defaults: NCCL_IGNORE_CPU_AFFINITY=1, NCCL_NET=IB, NCCL_DEBUG=WARN
+NCCL_IB_DISABLE = "0"             # enable IB (override launcher default)
+NCCL_MAX_NCHANNELS = "4"
+NCCL_MIN_NCHANNELS = "4"
+NCCL_CUMEM_ENABLE = "0"
+
+# ── Compute ────────────────────────────────────────────────────────────────
+OMP_NUM_THREADS = "4"