recipe: GLM-5.3-Flash EXL3-4bpw + DFlash2 on Entrpi community image

Community stack (ghcr.io/entrpi/glm-5.3-flash-exl3-2x-spark:v1-dflash2,
digest 284142c5..., arm64): EXL3/TR3 4bpw fused-trellis MoE with DFlash2
k=7 + CUDA graphs + plain fp8_e4m3 KV (1.43M-token pool @524k). Flags
parsed verbatim from upstream launch-glm53-vllm-tp2.sh defaults; NCCL/
rendezvous/drop-caches delegated to the launcher; weights + drafter read
from the site NAS hub (HF_HOME), JIT caches persist per node.

recipes/glm53-flash-exl3/recipe.toml

@@ -0,0 +1,114 @@
+# GLM-5.3-Flash EXL3 4bpw + DFlash2 on the Entrpi community image — 2× DGX
+# Spark, TP=2.
+#
+# NOT our lab stack: a fully different serving lane — 4-bit EXL3/TR3 fused
+# trellis MoE kernels (fork: Entrpi/vllm-glm-5.3-flash-spark, branch
+# glm53-on-infernal) with DFlash2 block-diffusion speculative decoding,
+# CUDA graphs (their validated default), 524k context over a packed-fp8
+# KV pool (~1.43M tokens). Upstream reference + numbers:
+#   https://github.com/Entrpi/glm-5.3-flash-exl3-2x-spark
+# Image provenance: docs/BUILD.md in that repo. Image manifest digest
+# (arm64-only, checked via `docker manifest inspect`):
+#   ghcr.io/entrpi/glm-5.3-flash-exl3-2x-spark:v1-dflash2
+#   sha256:284142c5833cbfd540ad42bb8f32cb340451db05a3b84029eaebae54579e9135
+#
+# Model: brandonmusic/GLM-5.3-Flash-tr3-4bpw (EXL3/TR3 uniform-K4, ~176 GiB,
+# KLD parity with official FP8). Drafter: incoai/GLM-5.3-Flash-DFlash2
+# (CC BY-NC-ND 4.0 — non-commercial; keep to research/personal use).
+# DFlash2 drafter already cached site-wide; the EXL3 weights must be
+# downloaded once into the NAS hub (see PREREQUISITES).
+#
+# Flag provenance: parsed verbatim from the upstream
+# scripts/launch-glm53-vllm-tp2.sh defaults (validated production config).
+# DEVIATIONS from their kit — all forced by our launcher/fabric:
+#   * no NCCL/*, master-addr/port, drop-caches ritual: the launcher handles
+#     rendezvous + the [nccl.ds4] table + drop_caches (same ritual upstream).
+#   * no --skip-mm-profiling... included; port = launcher 30000 convention
+#     (their kit serves 8000; adjust --port if you must expose 8000).
+#   * their hotfix-overlay hooks (glm53-hotfix etc.) are not used — this is
+#     the shipped image as-is; re-add if tracking their issue tracker.
+#   * weights stream from the NAS hub via HF_HOME (their kit prefers local
+#     NVMe per box; NFS mode exists upstream too). First load is slow;
+#     JIT caches persist per node (see extra_mounts).
+#
+# Boot markers (engine log): "Model loading took ~88 GiB"-scale BEFORE any
+# EXL3 trellis lines → "GPU KV cache size" ≈ 1.4M tokens (fp8_e4m3 @524k)
+# → "Application startup complete". A "Setting attention block size to
+# 4608" line is expected (fp8 KV auto-bump, documented upstream).
+#
+# PREREQUISITES: weights cached on the NAS hub once:
+#   docker run --rm --entrypoint python3 -v /mnt/nas/ai/huggingface:/root/.cache/huggingface \
+#     glm53-flash-nvfp4-lab:local -c "from huggingface_hub import snapshot_download; \
+#     snapshot_download('brandonmusic/GLM-5.3-Flash-tr3-4bpw')"
+# (DFlash2: already in the NAS hub.)
+
+engine = "vllm"
+name = "GLM-5.3-Flash EXL3-4bpw + DFlash2 (Entrpi image, 2x Spark, TP=2)"
+docker_image = "ghcr.io/entrpi/glm-5.3-flash-exl3-2x-spark:v1-dflash2"
+model = "brandonmusic/GLM-5.3-Flash-tr3-4bpw"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    # EXL3 fused trellis MoE (fork-baked kernels)
+    "--quantization", "exl3",
+    "--distributed-executor-backend", "mp",
+    # Their validated defaults; 0.88+ risks unified-memory swap on GB10
+    "--gpu-memory-utilization", "0.85",
+    "--max-model-len", "524288",
+    "--max-num-seqs", "4",
+    # vLLM auto-bumps to 4608 under fp8 KV (KDA state page parity)
+    "--block-size", "2304",
+    "--mm-processor-cache-gb", "0.5",
+    "--max-num-batched-tokens", "8192",
+    "--kv-cache-dtype", "fp8_e4m3",
+    # 1,435,070-token pool at 524k fp8; do NOT raise (2.26 GiB floor @13.4e9)
+    "--kv-cache-memory", "12400000000",
+    # Drafter by hub id (resolved from the mounted NAS HF_HOME like the
+    # target model; 7 = trained block size 8 minus the bonus token)
+    "--speculative-config", """
+'{"method":"dflash","model":"incoai/GLM-5.3-Flash-DFlash2","num_speculative_tokens":7}'""",
+    # required at 524288 on GB10 unified memory (upstream default SKIP_MM_PROFILING=1)
+    "--skip-mm-profiling",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+    "--default-chat-template-kwargs", """
+'{"enable_thinking": false}'""",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_DISABLE_VERSION_CHECK = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Weights + drafter both resolve out of the site NAS hub (offline mode).
+# NOTE: [[extra_mounts]] must stay AFTER args — a table header would
+# swallow the args array into it (TOML).
+[[extra_mounts]]
+host = "/mnt/nas/ai/huggingface"
+container = "/mnt/nas/ai/huggingface"
+env = "HF_HOME"
+
+# Long-lived JIT caches (upstream mounts $CACHE_HOST_PATH per layer);
+# hash-keyed so stale entries are inert, and first-boot compiles persist.
+[[extra_mounts]]
+host = "~/.cache/glm53-exl3-jit/triton"
+container = "/root/.triton"
+[[extra_mounts]]
+host = "~/.cache/glm53-exl3-jit/flashinfer"
+container = "/root/.cache/flashinfer"
+[[extra_mounts]]
+host = "~/.cache/glm53-exl3-jit/vllm"
+container = "/root/.cache/vllm"