launcher/recipes/glm52-nf3-hybrid-context/recipe.toml

# GLM-5.2 NF3-Hybrid — 🧠 CONTEXT LANE (DCP4, 800K max-len)
# 4× DGX Spark (GB10, aarch64/sm_121a), TP=4, DCP=4.
#
# Model: madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
#   → https://huggingface.co/madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
#
# Unpruned GLM-5.2 (753B/40B MoE, all 256 experts) in 3-format hybrid quant:
#   327GB — top-64 experts/layer in NVFP4, remaining 192 in NF3 (3-bit),
#   non-expert layers in MXFP8.
#
# BUILD THE IMAGE FIRST:
#   cd docker-images/glm52-nf3-hybrid && bash build.sh
#   cd docker-images/glm52-nf3-hybrid && bash build.sh
#   Produces image: vllm-nf3-hybrid:probe
#
# DCP4 lane: KV sharded across all 4 nodes → 877K-token pool.
#   Every decode step pays ag_rs allreduce on all 78 layers.
#   MTP k=3 (needs fuse_allreduce_rms) — near-perfect acceptance ~3.0
#   beat k=4/5 by ~8% on long network-taxed steps.
#
# Performance (measured 2026-07-07, 512-tok gens, temp 0, shallow ctx):
#   c1: 21.4–21.6 tok/s  |  c2: 32.9  |  c3: 38.9
#   c4: 45.4  |  c5: 37.4  |  c6: 40.1 aggregate
#   KV pool: 877,056 tokens  |  max-model-len: 800,000
#   Boot ~20 min, MTP k=3 acceptance ~3.0
#
# TUNING NOTES:
#   - gpu-memory-utilization 0.88 (GB10 unified memory; author's 0.96 is
#     dedicated-VRAM territory)
#   - max-num-batched-tokens ≥2048 or NF3 graph capture breaks
#   - max-cudagraph-capture-size must cover seqs×(1+k) = 4×4 = 16
#   - VLLM_DCP_SHARD_DRAFT=1 mandatory for MTP draft KV under DCP
#   - k is lane-specific: CONTEXT lane runs k=3 + fuse_allreduce_rms
#
# PREREQUISITES:
#   - Weights cached on every node:
#       huggingface-cli download madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid
#     (327GB, 334GB on disk, 96 safetensors shards)

engine = "vllm"
name = "GLM-5.2 NF3-Hybrid CONTEXT LANE (vLLM)"
docker_image = "vllm-nf3-hybrid:probe"
model = "madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid"
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "64gb"
docker_image_dir = "glm52-nf3-hybrid"

entrypoint = ["vllm", "serve"]

args = [
    # Quantization & KV cache
    "--quantization", "modelopt_fp4",
    "--kv-cache-dtype", "fp8",
    "--load-format", "auto",
#    "--load-format", "fastsafetensors",
    # Context length (800K max-len, 877K KV pool)
    "--max-model-len", "800000",
    # Batch & memory
    "--max-num-seqs", "4",
    "--max-num-batched-tokens", "4096",
    "--max-cudagraph-capture-size", "32",
    "--gpu-memory-utilization", "0.92",
    # Engine backends
    "--attention-backend", "B12X_MLA_SPARSE",
    "--moe-backend", "b12x",
    # Scheduling
    "--async-scheduling",
    "--enable-chunked-prefill",
    "--enable-prefix-caching",
    # Compilation (CONTEXT: fuse_allreduce_rms for DCP network tax)
    "--compilation-config", """
'{"pass_config":{"fuse_allreduce_rms":true}}'""",
    # Speculative decode — CONTEXT: MTP k=3 (near-perfect acceptance ~3.0)
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":3,"moe_backend":"b12x","draft_sample_method":"probabilistic"}'""",
    # Tool calling & reasoning (verified working from first boot)
    "--tool-call-parser", "glm47",
    "--reasoning-parser", "glm45",
    "--enable-auto-tool-choice",
    # General
    "--trust-remote-code",
    "--enable-flashinfer-autotune",
    "--served-model-name", "glm-5.2",
]

[env]
# ── DCP (Decode Context Parallelism) — CONTEXT: DCP4 ──────────────────────
# KV sharded across all 4 nodes — 4× pool, decode pays ag_rs gather
DECODE_CONTEXT_PARALLEL_SIZE = "4"
DCP_COMM_BACKEND = "ag_rs"
DCP_KV_CACHE_INTERLEAVE_SIZE = "1"

# ── b12x suite (required) ─────────────────────────────────────────────────
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_USE_B12X_FP8_GEMM = "1"
VLLM_USE_B12X_MOE = "1"
VLLM_USE_B12X_SPARSE_INDEXER = "1"
VLLM_DCP_GLOBAL_TOPK = "1"
VLLM_DCP_SHARD_DRAFT = "1"
B12X_W4A16_TC_DECODE = "1"
B12X_MOE_FORCE_A16 = "1"
B12X_DENSE_SPLITK_TURBO = "1"

# ── Hybrid loader knobs (v2) ──────────────────────────────────────────────
HYBRID_TIER = "both"
HYBRID_KEPT = "b12x_nf3"
HYBRID_NF3 = "b12x_nf3"
HYBRID_B12X_MAX_TOKENS = "4096"
HYBRID_MXFP8_NATIVE = "1"
# Real path to tier json inside the image (loader hardcodes /opt/venv layout)
HYBRID_MXFP8_TIER_JSON = "/usr/local/lib/python3.12/dist-packages/mxfp8_tier.json"

# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
TORCH_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
FLASHINFER_CUDA_ARCH_LIST = "12.1a"

# ── Offline mode (weights must be cached on each node) ────────────────────
HF_HOME = "/root/.cache/huggingface"
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"

# ── Timeout safety net (cold boot: cudagraph + JIT) ───────────────────────
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "3600"

# ── NCCL / RoCE (cluster config ib_hca/socket_ifname used for most) ──────
NCCL_CROSS_NIC = "1"
NCCL_CUMEM_ENABLE = "0"
NCCL_IGNORE_CPU_AFFINITY = "1"
NCCL_DEBUG = "WARN"

# ── Compute ────────────────────────────────────────────────────────────────
OMP_NUM_THREADS = "4"