launcher/recipes/glm53-flash-exl3/recipe.toml

# GLM-5.3-Flash EXL3 4bpw + DFlash2 on the Entrpi community image — 2× DGX
# Spark, TP=2.
#
# NOT our lab stack: a fully different serving lane — 4-bit EXL3/TR3 fused
# trellis MoE kernels (fork: Entrpi/vllm-glm-5.3-flash-spark, branch
# glm53-on-infernal) with DFlash2 block-diffusion speculative decoding,
# CUDA graphs (their validated default), 524k context over a packed-fp8
# KV pool (~1.43M tokens). Upstream reference + numbers:
#   https://github.com/Entrpi/glm-5.3-flash-exl3-2x-spark
# Image provenance: docs/BUILD.md in that repo. Image manifest digest
# (arm64-only, checked via `docker manifest inspect`):
#   ghcr.io/entrpi/glm-5.3-flash-exl3-2x-spark:v1-dflash2
#   sha256:284142c5833cbfd540ad42bb8f32cb340451db05a3b84029eaebae54579e9135
#
# Model: brandonmusic/GLM-5.3-Flash-tr3-4bpw (EXL3/TR3 uniform-K4, ~176 GiB,
# KLD parity with official FP8). Drafter: incoai/GLM-5.3-Flash-DFlash2
# (CC BY-NC-ND 4.0 — non-commercial; keep to research/personal use).
# DFlash2 drafter already cached site-wide; the EXL3 weights must be
# downloaded once into the NAS hub (see PREREQUISITES).
#
# Flag provenance: parsed verbatim from the upstream
# scripts/launch-glm53-vllm-tp2.sh defaults (validated production config).
# DEVIATIONS from their kit — all forced by our launcher/fabric:
#   * no NCCL/*, master-addr/port, drop-caches ritual: the launcher handles
#     rendezvous + the [nccl.ds4] table + drop_caches (same ritual upstream).
#   * no --skip-mm-profiling... included; port = launcher 30000 convention
#     (their kit serves 8000; adjust --port if you must expose 8000).
#   * their hotfix-overlay hooks (glm53-hotfix etc.) are not used — this is
#     the shipped image as-is; re-add if tracking their issue tracker.
#   * weights stream from the NAS hub via HF_HOME (their kit prefers local
#     NVMe per box; NFS mode exists upstream too). First load is slow;
#     JIT caches persist per node (see extra_mounts).
#
# Boot markers (engine log): "Model loading took ~88 GiB"-scale BEFORE any
# EXL3 trellis lines → "GPU KV cache size" ≈ 1.4M tokens (fp8_e4m3 @524k)
# → "Application startup complete". A "Setting attention block size to
# 4608" line is expected (fp8 KV auto-bump, documented upstream).
#
# PREREQUISITES: weights cached on the NAS hub once:
#   docker run --rm --entrypoint python3 -v /mnt/nas/ai/huggingface:/root/.cache/huggingface \
#     glm53-flash-nvfp4-lab:local -c "from huggingface_hub import snapshot_download; \
#     snapshot_download('brandonmusic/GLM-5.3-Flash-tr3-4bpw')"
# (DFlash2: already in the NAS hub.)

engine = "vllm"
name = "GLM-5.3-Flash EXL3-4bpw + DFlash2 (Entrpi image, 2x Spark, TP=2)"
docker_image = "ghcr.io/entrpi/glm-5.3-flash-exl3-2x-spark:v1-dflash2"
model = "brandonmusic/GLM-5.3-Flash-tr3-4bpw"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    # EXL3 fused trellis MoE (fork-baked kernels)
    "--quantization", "exl3",
    "--distributed-executor-backend", "mp",
    # Their validated defaults; 0.88+ risks unified-memory swap on GB10
    "--gpu-memory-utilization", "0.85",
    "--max-model-len", "524288",
    "--max-num-seqs", "4",
    # vLLM auto-bumps to 4608 under fp8 KV (KDA state page parity)
    "--block-size", "2304",
    "--mm-processor-cache-gb", "0.5",
    "--max-num-batched-tokens", "8192",
    "--kv-cache-dtype", "fp8_e4m3",
    # 1,435,070-token pool at 524k fp8; do NOT raise (2.26 GiB floor @13.4e9)
    "--kv-cache-memory", "12400000000",
    # Drafter by hub id (resolved from the mounted NAS HF_HOME like the
    # target model; 7 = trained block size 8 minus the bonus token)
    "--speculative-config", """
'{"method":"dflash","model":"incoai/GLM-5.3-Flash-DFlash2","num_speculative_tokens":7}'""",
    # required at 524288 on GB10 unified memory (upstream default SKIP_MM_PROFILING=1)
    "--skip-mm-profiling",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
    "--default-chat-template-kwargs", """
'{"enable_thinking": false}'""",
]

[env]
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
TORCH_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_CUDA_ARCH_LIST = "12.1a"
FLASHINFER_DISABLE_VERSION_CHECK = "1"
VLLM_NO_USAGE_STATS = "1"

# Weights + drafter both resolve out of the site NAS hub (offline mode).
# NOTE: [[extra_mounts]] must stay AFTER args — a table header would
# swallow the args array into it (TOML).
[[extra_mounts]]
host = "/mnt/nas/ai/huggingface"
container = "/mnt/nas/ai/huggingface"
env = "HF_HOME"

# Long-lived JIT caches (upstream mounts $CACHE_HOST_PATH per layer);
# hash-keyed so stale entries are inert, and first-boot compiles persist.
[[extra_mounts]]
host = "~/.cache/glm53-exl3-jit/triton"
container = "/root/.triton"
[[extra_mounts]]
host = "~/.cache/glm53-exl3-jit/flashinfer"
container = "/root/.cache/flashinfer"
[[extra_mounts]]
host = "~/.cache/glm53-exl3-jit/vllm"
container = "/root/.cache/vllm"