launcher/recipes/glm53-flash-exl3/recipe.toml

# GLM-5.3-Flash EXL3 4bpw + DFlash2 on the Entrpi community image — 2× DGX
# Spark, TP=2.
#
# NOT our lab stack: a fully different serving lane — 4-bit EXL3/TR3 fused
# trellis MoE kernels (fork: Entrpi/vllm-glm-5.3-flash-spark, branch
# glm53-on-infernal) with DFlash2 block-diffusion speculative decoding,
# CUDA graphs (their validated default), 524k context over a packed-fp8
# KV pool (~1.43M tokens). Upstream reference + numbers:
#   https://github.com/Entrpi/glm-5.3-flash-exl3-2x-spark
# Image provenance: docs/BUILD.md in that repo. Image manifest digest
# (arm64-only, checked via `docker manifest inspect`):
#   ghcr.io/entrpi/glm-5.3-flash-exl3-2x-spark:v1-dflash2
#   sha256:284142c5833cbfd540ad42bb8f32cb340451db05a3b84029eaebae54579e9135
#
# Model: brandonmusic/GLM-5.3-Flash-tr3-4bpw (EXL3/TR3 uniform-K4, ~176 GiB,
# KLD parity with official FP8). Drafter: incoai/GLM-5.3-Flash-DFlash2
# (CC BY-NC-ND 4.0 — non-commercial; keep to research/personal use).
# DFlash2 drafter already cached site-wide; the EXL3 weights must be
# downloaded once into the NAS hub (see PREREQUISITES).
#
# Flag provenance: parsed verbatim from the upstream
# scripts/launch-glm53-vllm-tp2.sh defaults (validated production config).
# DEVIATIONS from their kit — all forced by our launcher/fabric:
#   * no NCCL/*, master-addr/port, drop-caches ritual: the launcher handles
#     rendezvous + the [nccl.ds4] table + drop_caches (same ritual upstream).
#   * no --skip-mm-profiling... included; port = launcher 30000 convention
#     (their kit serves 8000; adjust --port if you must expose 8000).
#   * their hotfix-overlay hooks (glm53-hotfix etc.) are not used — this is
#     the shipped image as-is; re-add if tracking their issue tracker.
#   * weights stream from the NAS hub via HF_HOME (their kit prefers local
#     NVMe per box; NFS mode exists upstream too). First load is slow;
#     JIT caches persist per node (see extra_mounts).
#
# =====================================================================
# STATUS 2026-08-30 (2nd entry): FIXED and verified serving on ds4.
# ROOT CAUSE of the three OOM kills: the stock loader balloons ~80 GiB of
# CPU-side anon buffers while streaming the 164 GiB checkpoint from the
# NAS — unified memory (GPU pool + loader anon) hits the 121.6 GiB wall.
# FIX: --load-format instanttensor (overlay image, this build context adds
# instanttensor==0.1.9 --no-deps to the pinned Entrpi image): streams at
# ~175 MB/s with tiny buffers; boot ~20 min end-to-end, no OOM (third+
# boot clean at 16:40-17:00). Serving: ~22.8 tok/s with DFlash2 k=7
# (accept 48-81% by content), math smoke exact. Recipe now builds its own
# glm53-flash-exl3:local FROM the digest-pinned upstream image.
# ---- first-entry diagnosis (kept for context) ----
# boots fully to weight-stream, then global-OOM kills
# the worker at 75-95% of shard load — REPRODUCED TWICE (12:22:25 and
# 13:12:24, dmesg oom-killer) while streaming the 164 GiB checkpoint from
# the shared NAS on BOTH nodes. GB10 unified memory + NFS page cache
# cannot keep up (their kit streams NFS to the HEAD only; the worker reads
# local NVMe, and even their local mode "wedges into swap at ~90%" by
# their own caveat). Fixed knobs so far: entrypoint (NGC wrapper execs
# first CMD arg as a file — `vllm serve` must be explicit), drafter repo
# (incoai raw drafter is NOT in the NAS hub — we serve the MXFP8 mirror),
# GMU 0.86 (their 0.85 left 1.14 GiB free when warmup wanted 1.12).
# REMAINING FIX: per-node local mirror (sm121 --mirror after pointing
# hf_cache_dir at the NAS). dgx3 has 400+ GiB free; dgx4 is 98% full —
# blocked on local-cache cleanup there.
#
# Boot markers (engine log): "Model loading took ~88 GiB"-scale BEFORE any
# EXL3 trellis lines → "GPU KV cache size" ≈ 1.4M tokens (fp8_e4m3 @524k)
# → "Application startup complete". A "Setting attention block size to
# 4608" line is expected (fp8 KV auto-bump, documented upstream).
#
# PREREQUISITES: weights cached on the NAS hub once:
#   docker run --rm --entrypoint python3 -v /mnt/nas/ai/huggingface:/root/.cache/huggingface \
#     glm53-flash-nvfp4-lab:local -c "from huggingface_hub import snapshot_download; \
#     snapshot_download('brandonmusic/GLM-5.3-Flash-tr3-4bpw')"
# (DFlash2: already in the NAS hub.)

engine = "vllm"
name = "GLM-5.3-Flash EXL3-4bpw + DFlash2 (Entrpi image, 2x Spark, TP=2)"
docker_image = "glm53-flash-exl3:local"
docker_image_dir = "glm53-flash-exl3"
model = "brandonmusic/GLM-5.3-Flash-tr3-4bpw"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
# The image's NGC entrypoint wrapper (/opt/nvidia/nvidia_entrypoint.sh)
# execs the first CMD argument as a FILE unless it is a known command —
# so `vllm serve` must be passed explicitly, as the upstream launch
# script does (verified: first arg landed as a relative model path).
entrypoint = ["vllm", "serve"]

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    # EXL3 fused trellis MoE (fork-baked kernels)
    "--quantization", "exl3",
    # instanttensor loader (installed by the overlay): streams shards with
    # tiny CPU buffers — the stock loader OOM-kills this box by ballooning
    # ~80 GiB anon while streaming 164 GiB from the NAS (3 kernel kills,
    # 2026-08-30). Also drops load time from ~13 min to ~1.
    "--load-format", "instanttensor",
    "--distributed-executor-backend", "mp",
    # Their validated defaults; 0.88+ risks unified-memory swap on GB10
    # 0.85 upstream default left 1.14 GiB when the warmup wanted 1.12 GiB
# (OOM at engine init, 2026-08-30); 0.86 = one notch of slack. Their
# 0.88+ warning is about swap wedging — 0.86 stays below that.
    "--gpu-memory-utilization", "0.86",
    "--max-model-len", "524288",
    "--max-num-seqs", "4",
    # vLLM auto-bumps to 4608 under fp8 KV (KDA state page parity)
    "--block-size", "2304",
    "--mm-processor-cache-gb", "0.5",
    "--max-num-batched-tokens", "8192",
    "--kv-cache-dtype", "fp8_e4m3",
    # 1,435,070-token pool at 524k fp8; do NOT raise (2.26 GiB floor @13.4e9)
    "--kv-cache-memory", "12400000000",
    # Drafter by hub id (resolved from the mounted NAS HF_HOME like the
    # target model; 7 = trained block size 8 minus the bonus token).
    # incoai/GLM-5.3-Flash-DFlash2 = the exact upstream drafter (raw
    # fp8-e4m3, CC BY-NC-ND 4.0), downloaded to the NAS hub 2026-08-30 at
    # the operator's request; the MXFP8 mirror is the fallback.
    "--speculative-config", """
'{"method":"dflash","model":"incoai/GLM-5.3-Flash-DFlash2","num_speculative_tokens":7}'""",
    # required at 524288 on GB10 unified memory (upstream default SKIP_MM_PROFILING=1)
    "--skip-mm-profiling",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
    "--default-chat-template-kwargs", """
'{"enable_thinking": false}'""",
]

[env]
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_DISABLE_VERSION_CHECK = "1"
VLLM_NO_USAGE_STATS = "1"

# Weights + drafter both resolve out of the site NAS hub (offline mode).
# NOTE: [[extra_mounts]] must stay AFTER args — a table header would
# swallow the args array into it (TOML).
[[extra_mounts]]
host = "/mnt/nas/ai/huggingface"
container = "/mnt/nas/ai/huggingface"
env = "HF_HOME"

# Long-lived JIT caches (upstream mounts $CACHE_HOST_PATH per layer);
# hash-keyed so stale entries are inert, and first-boot compiles persist.
[[extra_mounts]]
host = "~/.cache/glm53-exl3-jit/triton"
container = "/root/.triton"
[[extra_mounts]]
host = "~/.cache/glm53-exl3-jit/flashinfer"
container = "/root/.cache/flashinfer"
[[extra_mounts]]
host = "~/.cache/glm53-exl3-jit/vllm"
container = "/root/.cache/vllm"