launcher/docker-images/glm53-kindling/compose/glm53.yaml

# GLM-5.3-Flash NVFP4, TP=4 across four GB10 nodes (one head, three workers),
# with the DFlash2 drafter.
#
# Run the SAME file on every node:
#   docker compose -f compose/glm53.yaml up -d
# compose/.env is optional. Addresses and roles come from mentat, and paths
# and the image have defaults (.env.example lists the overrides). Compose
# reads .env from this file's directory, whatever the working directory.
#
# Ordering: from cold, any rank may start first -- registration retries
# forever, and the host-level mentatd (mentatd.yaml from
# https://github.com/mmastrac/mentat) just has to be running on every box. Replacing a running stack is different: take all four
# ranks down first (README, "Replacing a running stack").
#
# Nothing else fits beside it. GLM's weights alone are ~90.7 GiB per node out
# of 121.6 GiB, so stop any other model on these boxes first.
#
# network_mode: host is required -- RoCE/NCCL needs the real interfaces, and
# Ray's actor ports are not practically mappable. It is also how the
# container reaches the box's mentatd, on 127.0.0.1:6379 and :6380.
#
# Pinned so the project is not named after this directory ("compose"). Before
# the first `up`, take down a stack started under another project name: this
# one reuses container_name glm53 and would collide with it.
name: glm53
services:
  glm53:
    # image/build.sh's default tag. Keep the two in step: the entrypoint's
    # defaults live in the image, and an older tag boots silently mistuned.
    image: ${IMAGE:-spark-glm53:v9}
    container_name: glm53
    network_mode: host
    ipc: host
    ulimits:
      memlock: -1
      stack: 67108864
      # A crash must not dump core. The engine maps ~90 GiB, and the dumping
      # thread holds the NVIDIA RM lock the whole time it writes: on
      # 2026-08-27 that turned one CUDA OOM on a node into a box with no
      # ssh, no userspace and no way back except a power cycle, while the
      # kernel sat in nvidia_close for 860+ seconds. core_pattern on these
      # hosts is a plain file, so nothing rate-limits or discards it either.
      core: 0
    devices:
      - /dev/infiniband:/dev/infiniband
    volumes:
      # The full 181 GiB checkpoint, read-only. There is no transcoded cache:
      # this image loads plain safetensors, so EVERY rank reads the whole
      # checkpoint and it must be present on EACH node's local disk.
      # Local rather than NFS on purpose -- an NFS mount races the network at
      # boot and fails (seen on one node on 2026-08-26), and a model that only
      # starts when the NAS is awake will eventually fail to start.
      - ${MODEL_HOST_DIR:-/srv/models/glm-5.3-flash-nvfp4}:/models/glm-5.3-flash-nvfp4:ro
      # The host's list of running systemd units, read by the preflight to warn
      # about a desktop session (README, "What you need").
      - /run/systemd/units:/host/systemd-units:ro
      # The DFlash2 drafter, also on every node: speculation runs on every TP
      # rank. A separate Qwen3-based model that reads auxiliary hidden states
      # from target layers 5, 14, 24, 33 and 42 (image/patches/glm53_eagle3_aux.py
      # supplies them; upstream GLM5next does not).
      - ${DFLASH_HOST_DIR:-/srv/models/glm-5.3-flash-dflash2}:/models/glm-5.3-flash-dflash2:ro
      # JIT caches (FlashInfer, torch.compile, Triton, TileLang), and the
      # experimental weight snapshots (~48 GB a node). WITHOUT this they live
      # inside the container and are discarded on every `compose down`, so
      # each restart recompiles. The FlashInfer autotune cache is the
      # exception: at TP>1 the entrypoint keeps it in /tmp and wipes it.
      # Unset, a named volume. `down -v` deletes it.
      - ${CACHE_HOME:-cache}:/root/.cache
      # The engine log and the allocator counters behind /memory, read through
      # the status server's MCP tools. Unset, a named volume.
      - ${LOG_DIR:-logs}:/logs
    # RULE: any knob that also has a default in entrypoint.sh is left EMPTY
    # here. A default in both places means two places to change and one
    # silently winning -- which happened repeatedly on 2026-08-27, with the
    # compose defaults from the pre-SM90 attempts quietly overriding tuned
    # values (MTP off, GPU_MEM_UTIL 0.86, MAX_MODEL_LEN 131072). The exceptions
    # are the knobs the status server reads itself (SERVICE_NAME, SERVED_NAME,
    # PEERS, MCP_LOG_DIR, THROUGHPUT_INTERVAL_S). Its own defaults are
    # generic, not this model's, so they are set here explicitly.
    environment:
      # Passed through only when .env (or the shell running compose) sets
      # them. Unset, the entrypoint takes them from the box's mentatd.
      #
      # ROLE and HEAD_HOST, both unset: the boxes elect the live agent with
      # the lowest address as head. Set, the roles are fixed: ROLE is head on
      # one box and worker on the rest, and HEAD_HOST is the head's LAN
      # address on every box, the head included.
      - ROLE
      - HEAD_HOST
      # This node's LAN address. mentat identifies a node by it, and the agent
      # and daemon must agree on one string. Unset = the address mentatd tags
      # lan.
      - VLLM_HOST_IP
      # Address prefix of the fabric, with its trailing dot. Unset = the first
      # of FABRIC_SUBNETS.
      - CLUSTER_SUBNET
      # The status server reads these two itself, and its default name is
      # "vllm", so they carry a default here too.
      - SERVICE_NAME=${SERVICE_NAME:-glm53}
      - SERVED_NAME=${SERVED_NAME:-glm53}
      # mentat subcluster identity; empty = SERVICE_NAME (the entrypoint's
      # default). Set it only to run the same model as a second parallel
      # stack, which needs a distinct group per stack.
      - MENTAT_GROUP=${MENTAT_GROUP:-}
      # Empty = 127.0.0.1:6379 when the boxes elect a head, else HEAD_HOST:6379.
      - RAY_ADDRESS=${RAY_ADDRESS:-}
      # One subnet per ConnectX PCIe root NCCL should use, space-separated. Each
      # names an interface; the entrypoint maps it to its RDMA device and GID.
      # Both roots made a 126k prefill 11% faster on TP=4 (2026-09-26). Empty =
      # CLUSTER_SUBNET when that is set, else one subnet per address mentatd
      # tags rdma.
      - FABRIC_SUBNETS=${FABRIC_SUBNETS:-}
      # How long to wait for the fabric and for the peer's GPUs before giving
      # up. Both default to waiting forever, because the usual reason either is
      # missing is a switch reboot, and a container that exits then just races
      # it. Set a bound only where a stuck stack must surface as a failure.
      - FABRIC_WAIT_S=${FABRIC_WAIT_S:-}
      - WORKER_WAIT_S=${WORKER_WAIT_S:-}
      # How long the RoCE device and its GID table get to appear once the
      # address is already up. Local state, so it settles in seconds or is
      # broken: this one is deliberately bounded, and the container exits so
      # the failure is visible rather than a container that waits forever.
      - ROCE_SETTLE_S=${ROCE_SETTLE_S:-}
      # LEFT EMPTY ON PURPOSE so the entrypoint derives it at start. There is
      # no correct constant: the GID table is indexed by (address, RoCE version)
      # with first-free allocation, so the index differs per node AND per boot.
      # On 2026-08-26 the right entry was 6 on one node and 5 on its peer, and the
      # first node's table had an empty slot 3 left by four failed DHCP cycles plus
      # a profile replacement during an earlier boot. Reboot that node with the static
      # profile in place and its correct index moves 6 -> 5, so a pinned value
      # is a latent NCCL failure rather than a fix. Set it only to override.
      - NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-}
      # Empty = all four RDMA devices, which is what reaches 196 Gb/s rather
      # than 112. Override only to restrict NCCL to one PCIe root.
      - NCCL_IB_HCA=${NCCL_IB_HCA:-}
      # Empty = the interface holding VLLM_HOST_IP, derived in the entrypoint.
      - NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-}
      # Channel count and NCCL logging; defaults and why in the entrypoint.
      - NCCL_MIN_NCHANNELS=${NCCL_MIN_NCHANNELS:-}
      - NCCL_MAX_NCHANNELS=${NCCL_MAX_NCHANNELS:-}
      - NCCL_DEBUG=${NCCL_DEBUG:-}
      - TP=${TP:-}
      # 1 lets the stack start without any experimental/ overlay (stock
      # kernels, about half the README's speed); otherwise the entrypoint refuses.
      - ALLOW_STOCK=${ALLOW_STOCK:-}
      - PREFLIGHT=${PREFLIGHT:-}
      - FABRIC_CHECK=${FABRIC_CHECK:-}
      # Speculation. SPEC_METHOD is dflash, mtp or none (entrypoint default
      # dflash, k=7); MTP=0 turns it off whichever is named. An MTP drafter
      # costs ~3.4 GiB and a rejection-sampler warmup, which were what killed
      # rank 0 on 2026-08-27 while this comment wrongly said MTP was off.
      - MTP=${MTP:-}
      - SPEC_METHOD=${SPEC_METHOD:-}
      - SPEC_TOKENS=${SPEC_TOKENS:-}
      - DFLASH_MODEL=${DFLASH_MODEL:-}
      - MODEL_DIR=${MODEL_DIR:-}
      # Default and KV arithmetic in the entrypoint. Raise it against the
      # "GPU KV cache size" line at boot.
      - MAX_MODEL_LEN=${MAX_MODEL_LEN:-}
      # Per-prompt multimodal caps. Raising the image cap raises the peak a
      # single request can demand. With SKIP_MM_PROFILING=1 the vision encoder
      # is never budgeted at startup, so those bytes come out of the same
      # headroom a long prefill needs.
      - LIMIT_MM=${LIMIT_MM:-}
      - SKIP_MM_PROFILING=${SKIP_MM_PROFILING:-}
      # lazy|eager|prefetch|torchao; eager in the entrypoint, which says why.
      - SAFETENSORS_LOAD_STRATEGY=${SAFETENSORS_LOAD_STRATEGY:-}
      # Which NVFP4 MoE kernel: flashinfer_cutlass (entrypoint default) or
      # marlin. The entrypoint has the history.
      - MOE_BACKEND=${MOE_BACKEND:-}
      # 0.88 is the measured ceiling under mentat (2026-08-27): CUDA-visible
      # free memory on the head is 108.0 GiB regardless of page cache, 0.90
      # wants 109.46 and failed the startup check four times, 0.88 wants
      # 107.03 and served with 0.63 GiB to spare -- KV 1,842,951 tokens vs
      # ~1.07M at the old 0.84. Under real Ray the ceiling was lower still
      # (its object store and daemons sat in this pool).
      #
      # 0.90 starts on a freshly booted box, which is the trap. It is worth
      # real KV -- 1,541,820 tokens against 1,200,343 here, +28%, because KV
      # gets only what the weights leave -- and it still must not be used: it
      # ran the driver out of memory 90 seconds into serving
      # (NV_ERR_NO_MEMORY in dmesg, hours before anything visible went wrong)
      # and both ranks died with the boxes needing a power cycle. A driver OOM
      # during startup PROFILING is normal at any setting; one during steady
      # serving is the signal.
      - GPU_MEM_UTIL=${GPU_MEM_UTIL:-}
      # Allocator and per-worker memory ceiling; defaults and why in the
      # entrypoint.
      - PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-}
      - TORCH_MEM_FRACTION=${TORCH_MEM_FRACTION:-}
      # Set to a writable directory to record an allocation stack per block
      # and dump a snapshot the first time the ceiling above is crossed. The
      # stack is taken in the allocating thread, so it names the caller. It
      # costs time and memory to record, so leave it empty outside an
      # investigation. /logs is already mounted and MCP-readable.
      - TORCH_MEM_TRACE_DIR=${TORCH_MEM_TRACE_DIR:-}
      # Where each worker publishes its allocator counters, which is how the
      # status page's /memory answers whether torch or something else is
      # holding the node's memory. One counter read per worker per interval,
      # so unlike the trace above this is cheap enough to leave on.
      - TORCH_MEM_STATS_DIR=${TORCH_MEM_STATS_DIR:-/logs}
      - TORCH_MEM_STATS_INTERVAL=${TORCH_MEM_STATS_INTERVAL:-}
      # Tool-call parser. Unset = the image's default: the stock glm47, or
      # glm47_failclosed on an image built before that became the default
      # (the default lives in the image's entrypoint). Set glm47 or
      # glm47_failclosed (image/patches/glm47_failclosed.py) to choose.
      - TOOL_PARSER=${TOOL_PARSER:-}
      # Appended to the serve command verbatim. Nothing the entrypoint selects
      # belongs here: EXTRA_ARGS is one string, and an override restating it
      # drops whatever else it carried -- how the fail-closed tool parser once
      # vanished from production.
      - EXTRA_ARGS=${EXTRA_ARGS:-}
      # KV pool in bytes; the entrypoint pins 26 GiB. Pinned, vLLM "skips
      # memory profiling" AND "does not respect the gpu_memory_utilization
      # config", so the pin is not a fix for a warmup OOM -- lower GPU_MEM_UTIL
      # or the pin for that.
      - KV_CACHE_MEMORY=${KV_CACHE_MEMORY:-}
      # Empty = fp8_e4m3, the entrypoint's default and what every measurement
      # here used. Never plain `fp8`: that selects the fp8_ds_mla packed format,
      # which hardcodes pe_dim == 64. This model is NoPE sparse MLA
      # (qk_rope_head_dim is 0), so that kernel refuses at KV cache init --
      # after a full 10-minute weight load.
      - KV_CACHE_DTYPE=${KV_CACHE_DTYPE:-}
      # Left EMPTY on purpose so the entrypoint owns the defaults. A default
      # here as well as there means two places to change and one silently
      # winning.
      - MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-}
      # Capped by the KDA recurrent state, not by throughput. 34 of 45 layers
      # are linear attention and each keeps one fixed-size state per sequence
      # ("Mamba cache block"); exactly 32 fit after weights at GPU_MEM_UTIL
      # 0.86. vLLM's default of 256 aborts at CUDA graph capture AFTER the full
      # weight load. Empty here so the entrypoint owns the default.
      - MAX_NUM_SEQS=${MAX_NUM_SEQS:-}
      - LONG_PREFILL_TOKEN_THRESHOLD=${LONG_PREFILL_TOKEN_THRESHOLD:-}
      - DECODE_RESERVE_TOKENS=${DECODE_RESERVE_TOKENS:-}
      - CUDAGRAPH_CAPTURE_SIZES=${CUDAGRAPH_CAPTURE_SIZES:-}
      - CUDAGRAPH_METRICS=${CUDAGRAPH_METRICS:-}
      - ITERATION_DETAILS=${ITERATION_DETAILS:-}
      # Empty = 8002 (API) and 8082 (status page and MCP), from the
      # entrypoint.
      - API_PORT=${API_PORT:-}
      - STATUS_PORT=${STATUS_PORT:-}
      # Status-page peers, host:port, comma-separated. Empty shows this node
      # alone; mentatd-serve gives the cluster view.
      - PEERS=${PEERS:-}
      - MCP_LOG_DIR=${MCP_LOG_DIR:-/logs}
      # mentat's agent id is group@container@node_ip. Without this the agent
      # falls back to the hostname, which under network_mode: host is the
      # HOST's name, so every container on a node would claim the same id.
      - CONTAINER_NAME=${CONTAINER_NAME:-glm53}
      # What the agent announces to mentatd-serve. Empty = port form (8002/v1,
      # 8082/mcp), which the router resolves against every address the node
      # announces. A URL pins one address. A worker holds a TP rank and
      # answers no requests, so the entrypoint drops its OpenAI endpoint
      # however this is set.
      - MENTAT_OPENAI_API=${MENTAT_OPENAI_API:-}
      - MENTAT_MCP_API=${MENTAT_MCP_API:-}
      - MENTAT_MODEL_PROVIDER=${MENTAT_MODEL_PROVIDER:-}
      - MAX_JOBS=${MAX_JOBS:-}
      - SELF_TEST=${SELF_TEST:-}
      - THROUGHPUT_INTERVAL_S=${THROUGHPUT_INTERVAL_S:-60}
      - NVIDIA_VISIBLE_DEVICES=all
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    restart: unless-stopped

# Used only when CACHE_HOME or LOG_DIR is unset.
volumes:
  cache:
  logs: