# GLM-5.2 NF4 Lloyd-Experts-GPTQ — 4× DGX Spark (GB10, sm_121a), TP=4, DCP=4. # # Model: aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ # → https://huggingface.co/aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ # # Lloyd-codebook NF4 routed-expert overlay for GLM-5.2 (744B MoE, 256 # experts/layer, 8 activated/token, 78 layers, sparse-MLA). Expert weights # quantized to 4-bit with a fitted Lloyd–Max codebook (−16.8% expert # reconstruction error vs E2M1/MXFP4 baseline on the same GPTQ pipeline). # Codes are indices into per-layer grids/L{n}.json — NOT the E2M1 table. # # LOADING: the Lloyd repo is experts-only (L{n}.safetensors + grids/). It # cannot be served standalone. The base checkpoint is # aidendle94/GLM-5.2-MXFP4-Experts-GPTQ (full model: config.json with # quant_method="hybrid_mxfp4_ct", hybrid-ct-*.safetensors non-expert weights # in block-FP8/NVFP4, tokenizer, mtp-draft). The Lloyd NF4 experts REPLACE # the MXFP4 E2M1 experts for layers 3–77 (all 75 MoE layers): the Lloyd # L{n}.safetensors files have the same filenames and tensor names as the # MXFP4 repo's, so they must be copied into the MXFP4 snapshot directory # (filesystem overlay). vLLM loads them through the standard weight loader; # the nf4_e8m0_k32 kernel (baked in production-hybrid-1.4) decodes them with # the global _NF4_CODEBOOK instead of the E2M1 bit-twiddle. Which layers use # the NF4 path is controlled by VLLM_NF4_LAYERS — a path to a JSON file # {"layers": [3, 4, ..., 77]}, NOT a range string. # # Image: sparkrun-vllm-ds4-gb10:glm52 # Built from docker-images/glm52/Dockerfile, which FROMs # aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.4 (bakes the # nf4_e8m0_k32 / nf4_packed kernel — inert without VLLM_NF4_LAYERS). # # PREREQUISITES (manual, on EVERY node): # 1. Download both repos via HF_HOME: # huggingface-cli download aidendle94/GLM-5.2-MXFP4-Experts-GPTQ # huggingface-cli download aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ # (MXFP4 base ~390 GB, Lloyd experts ~385 GB) # 2. Overlay the Lloyd NF4 experts onto the MXFP4 snapshot directory (same # filenames, same tensor names — they replace the E2M1 experts with # NF4-packed codes). Use RELATIVE symlinks (ln -snrf), not copies: the # HF cache is mounted at /root/.cache/huggingface inside the container # (vs /home//.cache/huggingface on the host), so absolute symlinks # would break — relative paths resolve the same under both roots, like # HF's own snapshot→blob links. -n is critical: it stops ln from # dereferencing the existing MXFP4 symlink (without -n, ln would replace # the BLOB the link points to, corrupting the 390 GB MXFP4 cache). # Lightweight: ~75 symlinks, no 385 GB copy. # MXFP4=$(readlink -f ~/.cache/huggingface/hub/models--aidendle94--GLM-5.2-MXFP4-Experts-GPTQ/snapshots/main) # LLOYD=$(readlink -f ~/.cache/huggingface/hub/models--aidendle94--GLM-5.2-NF4-Lloyd-Experts-GPTQ/snapshots/main) # for f in "$LLOYD"/L*.safetensors; do ln -snrf "$f" "$MXFP4/$(basename "$f")"; done # Without this overlay the kernel decodes E2M1 weights with the NF4 # codebook → garbage output (the model card warns about this). # 3. Create the NF4 layer-list JSON in the Lloyd snapshot dir (the kernel # reads this via VLLM_NF4_LAYERS — {{lloyd_experts}}/nf4_layers.json): # echo "{\"layers\":[$(seq -s, 3 77)]}" > "$LLOYD"/nf4_layers.json # # REFERENCE: # https://huggingface.co/aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ # https://huggingface.co/aidendle94/GLM-5.2-MXFP4-Experts-GPTQ engine = "vllm" name = "GLM-5.2 NF4 Lloyd-Experts-GPTQ (vLLM)" docker_image = "sparkrun-vllm-ds4-gb10:glm52" model = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ" required_nodes = 4 tensor_parallel_size = 4 pipeline_parallel_size = 1 distributed_backend = "mp" shm_size = "10gb" docker_image_dir = "glm52" entrypoint = ["vllm", "serve"] args = [ # Model name "--served-model-name", "glm52", # General "--trust-remote-code", "--reasoning-parser", "glm45", "--tool-call-parser", "glm47", "--enable-auto-tool-choice", "--enable-prefix-caching", # DCP (Decode Context Parallelism) "--decode-context-parallel-size", "4", "--dcp-comm-backend", "ag_rs", # Engine backends "--attention-backend", "B12X_MLA_SPARSE", # HF overrides — 78-char index_topk_pattern (GLM-5.2 MLP/expert routing) "--hf-overrides", """ '{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'""", # KV cache — NVFP4 with per-token scaling (nvfp4_ds_mla) "--kv-cache-dtype", "nvfp4_ds_mla", "--kv-cache-memory-bytes", "5500000000", # Context & batch config "--max-model-len", "262144", "--max-num-batched-tokens", "2048", "--max-num-seqs", "3", "--gpu-memory-utilization", "0.89", # Speculative decode — MTP k=4 with adaptive depths # {{mtp_draft}} is substituted per-node with the resolved HF cache path # (refs/main → latest snapshot) so symlinks resolve correctly. "--speculative-config", """ '{"model":"{{mtp_draft}}","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""", # Compilation — FULL_AND_PIECEWISE with all custom ops, 8 capture sizes "--compilation-config", """ '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,8]}'""", # Timeout safety net (cold boot: ranks finish minutes apart) "--distributed-timeout-seconds", "3600", "--cpu-distributed-timeout-seconds", "3600", ] # Persistent JIT/torch-inductor cache (avoids ~20 min recompilation on cold boot). # `alias = "jit_cache"` resolves via [mount_aliases] in config.toml; if absent, # falls back to host "~/.cache/jit" (expanded to /home//.cache/jit). # `env` injects -e XDG_CACHE_HOME=/root/.cache/jit so vLLM/torch write there. [[extra_mounts]] alias = "jit_cache" host = "~/.cache/jit" container = "/root/.cache/jit" env = "XDG_CACHE_HOME" # MTP draft model — from the MXFP4 base checkpoint's repo. Resolved at serve # time, per node, from the HF cache (refs/main → latest snapshot), so the # path survives `huggingface-cli download` updates without a fragile symlink. # The recipe references {{mtp_draft}} in --speculative-config; the launcher # substitutes the resolved full HF cache path per-node (no bind-mount — the # HF cache is already mounted at /root/.cache/huggingface, so symlinks # resolve correctly). [[extra_mounts]] repo = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ" subfolder = "mtp-draft" container = "mtp_draft" # Lloyd repo path reference — resolved at serve time, per node, from the HF # cache (refs/main → latest snapshot). {{lloyd_experts}} is substituted into # VLLM_NF4_LAYERS below so the kernel can find the nf4_layers.json file # (created by the operator in step 3 of the prerequisites above). This is a # path reference, NOT a bind-mount — the HF cache is already mounted at # /root/.cache/huggingface, so the resolved path's symlinks resolve correctly. [[extra_mounts]] repo = "aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ" container = "lloyd_experts" [env] # ── NF4 Lloyd expert overlay ────────────────────────────────────────────── # Path to a JSON file listing which layers use the nf4_e8m0_k32 decode path # (vs the default fp4_e8m0_k32 E2M1 bit-twiddle). The kernel reads this file # at init: {"layers": [3, 4, ..., 77]}. {{lloyd_experts}} is substituted # per-node with the resolved HF cache path to the Lloyd repo snapshot, where # the operator creates nf4_layers.json (see PREREQUISITES step 3 above). VLLM_NF4_LAYERS = "{{lloyd_experts}}/nf4_layers.json" # ── b12x suite ──────────────────────────────────────────────────────────── VLLM_USE_V2_MODEL_RUNNER = "1" VLLM_USE_B12X_MOE = "1" VLLM_USE_B12X_SPARSE_INDEXER = "1" VLLM_DCP_SHARD_DRAFT = "1" VLLM_DCP_GLOBAL_TOPK = "1" VLLM_DSV4_INDEXER_SP = "1" # ── MLA CKV gather (sparse gather for decode) ───────────────────────────── VLLM_B12X_MLA_CKV_GATHER = "1" VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS = "262144" VLLM_B12X_MLA_DECODE_SPARSE_GATHER = "1" VLLM_B12X_MLA_DECODE_GATHER_V2 = "1" # ── Speculative decode ──────────────────────────────────────────────────── VLLM_ADAPTIVE_SPEC_DEPTHS = "2,4" # ── NVFP4 KV cache ──────────────────────────────────────────────────────── VLLM_NVFP4_MLA_PER_TOKEN_SCALE = "1" # ── Timeout safety net ──────────────────────────────────────────────────── VLLM_ENGINE_READY_TIMEOUT_S = "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "1800" VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1" # ── MUST unset baked VLLM_PREFIX_CACHE_RETENTION_INTERVAL ───────────────── # The image bakes a value that GLM-5.2's KV coordinator rejects. # Empty string = unset at runtime. VLLM_PREFIX_CACHE_RETENTION_INTERVAL = "" # ── Architecture (aarch64/sm_121a DGX Spark) ────────────────────────────── CUTE_DSL_ARCH = "sm_121a" TORCH_CUDA_ARCH_LIST = "12.1a" # ── JIT cache ────────────────────────────────────────────────────────────── # Persisted via the [[extra_mounts]] jit_cache entry above: XDG_CACHE_HOME is # injected from that mount, pointing at /root/.cache/jit (no hardcoded dupe). # ── NCCL overrides ───────────────────────────────────────────────────────── # Cluster config provides NCCL_IB_HCA, NCCL_IB_GID_INDEX, NCCL_SOCKET_IFNAME # Launcher defaults: NCCL_IGNORE_CPU_AFFINITY=1, NCCL_NET=IB, NCCL_DEBUG=WARN NCCL_IB_DISABLE = "0" # enable IB (override launcher default) NCCL_MAX_NCHANNELS = "4" NCCL_MIN_NCHANNELS = "4" NCCL_CUMEM_ENABLE = "0" # ── Compute ──────────────────────────────────────────────────────────────── OMP_NUM_THREADS = "4"