launcher/recipes/glm52-nf4-lloyd-experts-gptq/recipe.toml

# GLM-5.2 NF4 Lloyd-Experts-GPTQ — 4× DGX Spark (GB10, sm_121a), TP=4, DCP=4.
#
# Model: aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
#   → https://huggingface.co/aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
#
# Lloyd-codebook NF4 routed-expert overlay for GLM-5.2 (744B MoE, 256
# experts/layer, 8 activated/token, 78 layers, sparse-MLA). Expert weights
# quantized to 4-bit with a fitted Lloyd–Max codebook (−16.8% expert
# reconstruction error vs E2M1/MXFP4 baseline on the same GPTQ pipeline).
# Codes are indices into per-layer grids/L{n}.json — NOT the E2M1 table.
#
# LOADING: the Lloyd repo is experts-only (L{n}.safetensors + grids/). It
# cannot be served standalone. The base checkpoint is
# aidendle94/GLM-5.2-MXFP4-Experts-GPTQ (full model: config.json with
# quant_method="hybrid_mxfp4_ct", hybrid-ct-*.safetensors non-expert weights
# in block-FP8/NVFP4, tokenizer, mtp-draft). The Lloyd NF4 experts REPLACE
# the MXFP4 E2M1 experts for layers 3–77 (all 75 MoE layers): the Lloyd
# L{n}.safetensors files have the same filenames and tensor names as the
# MXFP4 repo's, so they must be copied into the MXFP4 snapshot directory
# (filesystem overlay). vLLM loads them through the standard weight loader;
# the nf4_e8m0_k32 kernel (baked in production-hybrid-1.4) decodes them with
# the global _NF4_CODEBOOK instead of the E2M1 bit-twiddle. Which layers use
# the NF4 path is controlled by VLLM_NF4_LAYERS — a path to a JSON file
# {"layers": [3, 4, ..., 77]}, NOT a range string.
#
# Image: sparkrun-vllm-ds4-gb10:glm52
#   Built from docker-images/glm52/Dockerfile, which FROMs
#   aidendle94/sparkrun-vllm-ds4-gb10:production-hybrid-1.4 (bakes the
#   nf4_e8m0_k32 / nf4_packed kernel — inert without VLLM_NF4_LAYERS).
#
# PREREQUISITES (manual, on EVERY node):
#   1. Download both repos via HF_HOME:
#        huggingface-cli download aidendle94/GLM-5.2-MXFP4-Experts-GPTQ
#        huggingface-cli download aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
#      (MXFP4 base ~390 GB, Lloyd experts ~385 GB)
#   2. Overlay the Lloyd NF4 experts onto the MXFP4 snapshot directory:
#        MXFP4=$(readlink -f ~/.cache/huggingface/hub/models--aidendle94--GLM-5.2-MXFP4-Experts-GPTQ/snapshots/main)
#        LLOYD=$(readlink -f ~/.cache/huggingface/hub/models--aidendle94--GLM-5.2-NF4-Lloyd-Experts-GPTQ/snapshots/main)
#        cp "$LLOYD"/L*.safetensors "$MXFP4"/
#      Without this overlay the kernel decodes E2M1 weights with the NF4
#      codebook → garbage output (the model card warns about this).
#   3. Create the NF4 layer-list JSON in the Lloyd snapshot dir:
#        echo '{"layers":[3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77]}' > "$LLOYD"/nf4_layers.json
#
# REFERENCE:
#   https://huggingface.co/aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
#   https://huggingface.co/aidendle94/GLM-5.2-MXFP4-Experts-GPTQ

engine = "vllm"
name = "GLM-5.2 NF4 Lloyd-Experts-GPTQ (vLLM)"
docker_image = "sparkrun-vllm-ds4-gb10:glm52"
model = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "10gb"
docker_image_dir = "glm52"

entrypoint = ["vllm", "serve"]

args = [
    # Model name
    "--served-model-name", "glm52",
    # General
    "--trust-remote-code",
    "--reasoning-parser", "glm45",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--enable-prefix-caching",
    # DCP (Decode Context Parallelism)
    "--decode-context-parallel-size", "4",
    "--dcp-comm-backend", "ag_rs",
    # Engine backends
    "--attention-backend", "B12X_MLA_SPARSE",
    # HF overrides — 78-char index_topk_pattern (GLM-5.2 MLP/expert routing)
    "--hf-overrides", """
'{"index_topk_pattern":"FFFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSSFSSS"}'""",
    # KV cache — NVFP4 with per-token scaling (nvfp4_ds_mla)
    "--kv-cache-dtype", "nvfp4_ds_mla",
    "--kv-cache-memory-bytes", "5500000000",
    # Context & batch config
    "--max-model-len", "262144",
    "--max-num-batched-tokens", "2048",
    "--max-num-seqs", "3",
    "--gpu-memory-utilization", "0.89",
    # Speculative decode — MTP k=4 with adaptive depths
    # {{mtp_draft}} is substituted per-node with the resolved HF cache path
    # (refs/main → latest snapshot) so symlinks resolve correctly.
    "--speculative-config", """
'{"model":"{{mtp_draft}}","method":"mtp","num_speculative_tokens":4,"draft_attention_backend":"B12X_MLA_SPARSE","adaptive_speculative_tokens_window":32}'""",
    # Compilation — FULL_AND_PIECEWISE with all custom ops, 8 capture sizes
    "--compilation-config", """
'{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"],"cudagraph_capture_sizes":[1,2,3,4,5,6,8]}'""",
    # Timeout safety net (cold boot: ranks finish minutes apart)
    "--distributed-timeout-seconds", "3600",
    "--cpu-distributed-timeout-seconds", "3600",
]

# Persistent JIT/torch-inductor cache (avoids ~20 min recompilation on cold boot).
# `alias = "jit_cache"` resolves via [mount_aliases] in config.toml; if absent,
# falls back to host "~/.cache/jit" (expanded to /home/<ssh_user>/.cache/jit).
# `env` injects -e XDG_CACHE_HOME=/root/.cache/jit so vLLM/torch write there.
[[extra_mounts]]
alias = "jit_cache"
host = "~/.cache/jit"
container = "/root/.cache/jit"
env = "XDG_CACHE_HOME"

# MTP draft model — from the MXFP4 base checkpoint's repo. Resolved at serve
# time, per node, from the HF cache (refs/main → latest snapshot), so the
# path survives `huggingface-cli download` updates without a fragile symlink.
# The recipe references {{mtp_draft}} in --speculative-config; the launcher
# substitutes the resolved full HF cache path per-node (no bind-mount — the
# HF cache is already mounted at /root/.cache/huggingface, so symlinks
# resolve correctly).
[[extra_mounts]]
repo = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
subfolder = "mtp-draft"
container = "mtp_draft"

# Lloyd repo path reference — resolved at serve time, per node, from the HF
# cache (refs/main → latest snapshot). {{lloyd_experts}} is substituted into
# VLLM_NF4_LAYERS below so the kernel can find the nf4_layers.json file
# (created by the operator in step 3 of the prerequisites above). This is a
# path reference, NOT a bind-mount — the HF cache is already mounted at
# /root/.cache/huggingface, so the resolved path's symlinks resolve correctly.
[[extra_mounts]]
repo = "aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ"
container = "lloyd_experts"


[env]
# ── NF4 Lloyd expert overlay ──────────────────────────────────────────────
# Path to a JSON file listing which layers use the nf4_e8m0_k32 decode path
# (vs the default fp4_e8m0_k32 E2M1 bit-twiddle). The kernel reads this file
# at init: {"layers": [3, 4, ..., 77]}. {{lloyd_experts}} is substituted
# per-node with the resolved HF cache path to the Lloyd repo snapshot, where
# the operator creates nf4_layers.json (see PREREQUISITES step 3 above).
VLLM_NF4_LAYERS = "{{lloyd_experts}}/nf4_layers.json"

# ── b12x suite ────────────────────────────────────────────────────────────
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_USE_B12X_MOE = "1"
VLLM_USE_B12X_SPARSE_INDEXER = "1"
VLLM_DCP_SHARD_DRAFT = "1"
VLLM_DCP_GLOBAL_TOPK = "1"
VLLM_DSV4_INDEXER_SP = "1"

# ── MLA CKV gather (sparse gather for decode) ─────────────────────────────
VLLM_B12X_MLA_CKV_GATHER = "1"
VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS = "262144"
VLLM_B12X_MLA_DECODE_SPARSE_GATHER = "1"
VLLM_B12X_MLA_DECODE_GATHER_V2 = "1"

# ── Speculative decode ────────────────────────────────────────────────────
VLLM_ADAPTIVE_SPEC_DEPTHS = "2,4"

# ── NVFP4 KV cache ────────────────────────────────────────────────────────
VLLM_NVFP4_MLA_PER_TOKEN_SCALE = "1"

# ── Timeout safety net ────────────────────────────────────────────────────
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "1800"
VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"

# ── MUST unset baked VLLM_PREFIX_CACHE_RETENTION_INTERVAL ─────────────────
# The image bakes a value that GLM-5.2's KV coordinator rejects.
# Empty string = unset at runtime.
VLLM_PREFIX_CACHE_RETENTION_INTERVAL = ""

# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
CUTE_DSL_ARCH = "sm_121a"
TORCH_CUDA_ARCH_LIST = "12.1a"

# ── JIT cache ──────────────────────────────────────────────────────────────
# Persisted via the [[extra_mounts]] jit_cache entry above: XDG_CACHE_HOME is
# injected from that mount, pointing at /root/.cache/jit (no hardcoded dupe).

# ── NCCL overrides ─────────────────────────────────────────────────────────
# Cluster config provides NCCL_IB_HCA, NCCL_IB_GID_INDEX, NCCL_SOCKET_IFNAME
# Launcher defaults: NCCL_IGNORE_CPU_AFFINITY=1, NCCL_NET=IB, NCCL_DEBUG=WARN
NCCL_IB_DISABLE = "0"             # enable IB (override launcher default)
NCCL_MAX_NCHANNELS = "4"
NCCL_MIN_NCHANNELS = "4"
NCCL_CUMEM_ENABLE = "0"

# ── Compute ────────────────────────────────────────────────────────────────
OMP_NUM_THREADS = "4"