recipes: glm53-flash-exl3

recipes/glm53-flash-exl3/recipe.toml

@@ -0,0 +1,158 @@
+# GLM-5.3-Flash EXL3 4bpw + DFlash2 on the Entrpi community image — 2× DGX
+# Spark, TP=2.
+#
+# NOT our lab stack: a fully different serving lane — 4-bit EXL3/TR3 fused
+# trellis MoE kernels (fork: Entrpi/vllm-glm-5.3-flash-spark, branch
+# glm53-on-infernal) with DFlash2 block-diffusion speculative decoding,
+# CUDA graphs (their validated default), 524k context over a packed-fp8
+# KV pool (~1.43M tokens). Upstream reference + numbers:
+#   https://github.com/Entrpi/glm-5.3-flash-exl3-2x-spark
+# Image provenance: docs/BUILD.md in that repo. Image manifest digest
+# (arm64-only, checked via `docker manifest inspect`):
+#   ghcr.io/entrpi/glm-5.3-flash-exl3-2x-spark:v1-dflash2
+#   sha256:284142c5833cbfd540ad42bb8f32cb340451db05a3b84029eaebae54579e9135
+#
+# Model: brandonmusic/GLM-5.3-Flash-tr3-4bpw (EXL3/TR3 uniform-K4, ~176 GiB,
+# KLD parity with official FP8). Drafter: incoai/GLM-5.3-Flash-DFlash2
+# (CC BY-NC-ND 4.0 — non-commercial; keep to research/personal use).
+# DFlash2 drafter already cached site-wide; the EXL3 weights must be
+# downloaded once into the NAS hub (see PREREQUISITES).
+#
+# Flag provenance: parsed verbatim from the upstream
+# scripts/launch-glm53-vllm-tp2.sh defaults (validated production config).
+# DEVIATIONS from their kit — all forced by our launcher/fabric:
+#   * no NCCL/*, master-addr/port, drop-caches ritual: the launcher handles
+#     rendezvous + the [nccl.ds4] table + drop_caches (same ritual upstream).
+#   * no --skip-mm-profiling... included; port = launcher 30000 convention
+#     (their kit serves 8000; adjust --port if you must expose 8000).
+#   * their hotfix-overlay hooks (glm53-hotfix etc.) are not used — this is
+#     the shipped image as-is; re-add if tracking their issue tracker.
+#   * weights stream from the NAS hub via HF_HOME (their kit prefers local
+#     NVMe per box; NFS mode exists upstream too). First load is slow;
+#     JIT caches persist per node (see extra_mounts).
+#
+# =====================================================================
+# STATUS 2026-08-30 (2nd entry): FIXED and verified serving on ds4.
+# ROOT CAUSE of the three OOM kills: the stock loader balloons ~80 GiB of
+# CPU-side anon buffers while streaming the 164 GiB checkpoint from the
+# NAS — unified memory (GPU pool + loader anon) hits the 121.6 GiB wall.
+# FIX: --load-format instanttensor (overlay image, this build context adds
+# instanttensor==0.1.9 --no-deps to the pinned Entrpi image): streams at
+# ~175 MB/s with tiny buffers; boot ~20 min end-to-end, no OOM (third+
+# boot clean at 16:40-17:00). Serving: ~22.8 tok/s with DFlash2 k=7
+# (accept 48-81% by content), math smoke exact. Recipe now builds its own
+# glm53-flash-exl3:local FROM the digest-pinned upstream image.
+# ---- first-entry diagnosis (kept for context) ----
+# boots fully to weight-stream, then global-OOM kills
+# the worker at 75-95% of shard load — REPRODUCED TWICE (12:22:25 and
+# 13:12:24, dmesg oom-killer) while streaming the 164 GiB checkpoint from
+# the shared NAS on BOTH nodes. GB10 unified memory + NFS page cache
+# cannot keep up (their kit streams NFS to the HEAD only; the worker reads
+# local NVMe, and even their local mode "wedges into swap at ~90%" by
+# their own caveat). Fixed knobs so far: entrypoint (NGC wrapper execs
+# first CMD arg as a file — `vllm serve` must be explicit), drafter repo
+# (incoai raw drafter is NOT in the NAS hub — we serve the MXFP8 mirror),
+# GMU 0.86 (their 0.85 left 1.14 GiB free when warmup wanted 1.12).
+# REMAINING FIX: per-node local mirror (sm121 --mirror after pointing
+# hf_cache_dir at the NAS). dgx3 has 400+ GiB free; dgx4 is 98% full —
+# blocked on local-cache cleanup there.
+#
+# Boot markers (engine log): "Model loading took ~88 GiB"-scale BEFORE any
+# EXL3 trellis lines → "GPU KV cache size" ≈ 1.4M tokens (fp8_e4m3 @524k)
+# → "Application startup complete". A "Setting attention block size to
+# 4608" line is expected (fp8 KV auto-bump, documented upstream).
+#
+# PREREQUISITES: weights cached on the NAS hub once:
+#   docker run --rm --entrypoint python3 -v /mnt/nas/ai/huggingface:/root/.cache/huggingface \
+#     glm53-flash-nvfp4-lab:local -c "from huggingface_hub import snapshot_download; \
+#     snapshot_download('brandonmusic/GLM-5.3-Flash-tr3-4bpw')"
+# (DFlash2: already in the NAS hub.)
+
+engine = "vllm"
+name = "GLM-5.3-Flash EXL3-4bpw + DFlash2 (Entrpi image, 2x Spark, TP=2)"
+docker_image = "glm53-flash-exl3:local"
+docker_image_dir = "glm53-flash-exl3"
+model = "brandonmusic/GLM-5.3-Flash-tr3-4bpw"
+parent_model = "zai-org/GLM-5.3-Flash"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+# The image's NGC entrypoint wrapper (/opt/nvidia/nvidia_entrypoint.sh)
+# execs the first CMD argument as a FILE unless it is a known command —
+# so `vllm serve` must be passed explicitly, as the upstream launch
+# script does (verified: first arg landed as a relative model path).
+entrypoint = ["vllm", "serve"]
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    # EXL3 fused trellis MoE (fork-baked kernels)
+    "--quantization", "exl3",
+    # instanttensor loader (installed by the overlay): streams shards with
+    # tiny CPU buffers — the stock loader OOM-kills this box by ballooning
+    # ~80 GiB anon while streaming 164 GiB from the NAS (3 kernel kills,
+    # 2026-08-30). Also drops load time from ~13 min to ~1.
+    "--load-format", "instanttensor",
+    "--distributed-executor-backend", "mp",
+    # Their validated defaults; 0.88+ risks unified-memory swap on GB10
+    # 0.85 upstream default left 1.14 GiB when the warmup wanted 1.12 GiB
+# (OOM at engine init, 2026-08-30); 0.86 = one notch of slack. Their
+# 0.88+ warning is about swap wedging — 0.86 stays below that.
+    "--gpu-memory-utilization", "0.86",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "4",
+    # vLLM auto-bumps to 4608 under fp8 KV (KDA state page parity)
+    "--block-size", "2304",
+    "--mm-processor-cache-gb", "0.5",
+    "--max-num-batched-tokens", "8192",
+    "--kv-cache-dtype", "fp8_e4m3",
+    # 1,435,070-token pool at 524k fp8; do NOT raise (2.26 GiB floor @13.4e9)
+    "--kv-cache-memory", "12400000000",
+    # Drafter by hub id (resolved from the mounted NAS HF_HOME like the
+    # target model; 7 = trained block size 8 minus the bonus token).
+    # incoai/GLM-5.3-Flash-DFlash2 = the exact upstream drafter (raw
+    # fp8-e4m3, CC BY-NC-ND 4.0), downloaded to the NAS hub 2026-08-30 at
+    # the operator's request; the MXFP8 mirror is the fallback.
+    "--speculative-config", """
+'{"method":"dflash","model":"incoai/GLM-5.3-Flash-DFlash2","num_speculative_tokens":7}'""",
+    # required at 524288 on GB10 unified memory (upstream default SKIP_MM_PROFILING=1)
+    "--skip-mm-profiling",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+    "--default-chat-template-kwargs", """
+'{"enable_thinking": false}'""",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_DISABLE_VERSION_CHECK = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Weights + drafter both resolve out of the site NAS hub (offline mode).
+# NOTE: [[extra_mounts]] must stay AFTER args — a table header would
+# swallow the args array into it (TOML).
+[[extra_mounts]]
+host = "/mnt/nas/ai/huggingface"
+container = "/mnt/nas/ai/huggingface"
+env = "HF_HOME"
+
+# Long-lived JIT caches (upstream mounts $CACHE_HOST_PATH per layer);
+# hash-keyed so stale entries are inert, and first-boot compiles persist.
+[[extra_mounts]]
+host = "~/.cache/glm53-exl3-jit/triton"
+container = "/root/.triton"
+[[extra_mounts]]
+host = "~/.cache/glm53-exl3-jit/flashinfer"
+container = "/root/.cache/flashinfer"
+[[extra_mounts]]
+host = "~/.cache/glm53-exl3-jit/vllm"
+container = "/root/.cache/vllm"