launcher/recipes/glm53-kindling-2x/recipe.toml

# kindlingai/glm-5.3-flash-gx10 — integration mode B (operator 2026-10-01:
# "image + patches first, their full stack later"). Their IMAGE (15 anchored
# patches + FlashKDA rebuild, baked experimental/ overlay stack via
# docker-images/glm53-kindling/build.sh) launched with sm121's mp rendezvous.
# Every arg/env below is a 1:1 transcription of upstream `image/entrypoint.sh`
# (`exec vllm serve …` and its `case "$TP"` block) plus the
# `experimental/compose/*.yaml` env blocks (arx, snapshot, adaptive-k, fp8,
# megamoe, fixes, sp, recoverssm, tp3-pads) — the stack the upstream README
# headline numbers were measured with. Deviation ledger:
# docker-images/glm53-kindling/README.md.
#
# NOT used (mode A territory): their entrypoint (mentatd roles/election,
# preflight, fabric-check, status server, self-test, GID derivation) — sm121
# owns orchestration, NCCL env comes from the cluster [nccl] block (auto GID
# detect included). Weights: NAS HF cache (upstream demands local disks per
# box — operator call, documented). KV pin: upstream per-TP value on GB10 UMA
# (profiled-unpinned pools starve FlashInfer autotune at boot — upstream).
# TP=2 knobs are the entrypoint's `case "$TP"` TP=2 branch:
# pin 8 GiB, MML 163840 (their measured longest request 160k),
# batched 8192, seqs 16, adaptive prior 32.1,1.32,0.427 — each
# chosen because a rank reads twice the weights it does at TP=4.
# Build: `sm121 build` in docker-images/glm53-kindling (tag spark-glm53:v9).
# Bench: seed 42 hardmode + perf c1,c2 + depth suite, as everywhere.

engine = "vllm"
name = "GLM-5.3-Flash NVFP4 (kindlingai gx10 stack, 2x, DFlash2)"
docker_image = "spark-glm53:v9"
# shared image build dir (all three TP variants build the same image)
docker_image_dir = "glm53-kindling"
entrypoint = ["vllm", "serve"]
# Floating model: the engine resolves the org/model id via the NAS HF cache
# (refs/main → latest snapshot), so a model re-upload is picked up without
# touching the recipe. The NAS copy stood at snapshot da920bb0 on 2026-10-02
# (informational only; the path follows refs/main).
model = "nvidia/GLM-5.3-Flash-NVFP4"
parent_model = "nvidia/GLM-5.3-Flash-NVFP4"
port = 30000
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # --- upstream entrypoint.sh `exec vllm serve …`, TP=2 case block -----
    "--served-model-name", "glm53",
    "--distributed-executor-backend", "mp",
    "--load-format", "auto",
    "--safetensors-load-strategy", "eager",
    "--gpu-memory-utilization", "0.86",
    "--max-model-len", "524288",
    "--kv-cache-dtype", "fp8_e4m3",
    "--block-size", "2304",
    "--kv-cache-memory", "10737418240",
    "--trust-remote-code",
    "--enable-prefix-caching",
    "--max-num-batched-tokens", "6144",
    "--max-num-seqs", "6",
    "--long-prefill-token-threshold", "2304",
    # flashinfer_cutlass — native NVFP4 MoE on the checkpoint's per-projection
    # input scales (marlin branch of their entrypoint not used). NO explicit
    # cudagraph cap: adaptive-k's num_speculative_tokens_per_batch_size makes
    # vLLM pick its own ceiling (upstream entrypoint deliberately leaves it).
    "--moe-backend", "flashinfer_cutlass",
    "--limit-mm-per-prompt", '{"image":16,"video":0}',
    # upstream's chat template: thinking:false -> low reasoning effort (the
    # model has no real non-thinking mode; an open  breaks long output)
    "--chat-template", "/usr/local/share/glm53-chat-template.jinja",
    # entrypoint TOOL_ARGS / TOPK_ARGS (vendor parser-name notes apply)
    "--enable-auto-tool-choice",
    "--tool-call-parser", "glm47",
    "--sparse-indexer-topk-backend", "per_row",
    "--reasoning-parser", "glm45",
    # adaptive-k overlay EXTRA_ARGS — upstream appends it AFTER the entrypoint's
    # own spec line, so this JSON is the effective one (same race resolved).
    "--scheduler-cls", "vllm.v1.core.sched.adaptive_k.AdaptiveKScheduler",
    "--speculative-config", """
'{"method":"dflash","disable_eagle_block_drop":true,"model":"{{dflash}}","num_speculative_tokens":7,"attention_backend":"TRITON_ATTN","num_speculative_tokens_per_batch_size":[[1,1,7],[2,2,5],[3,3,4],[4,4,3],[5,5,2],[6,32,7]]}'""",
]

# DFlash2 drafter: {{dflash}} resolves per node from the NAS HF
# cache (upstream mounts DFLASH_HOST_DIR on every box instead).
[[extra_mounts]]
repo = "incoai/GLM-5.3-Flash-DFlash2"
container = "dflash"

# Node-local persistence, upstream CACHE_HOME equivalent (processed weight
# snapshots ~48 GB/rank for their ~3-min restarts + JIT tiers + adaptive-k/
# draft-trunc control state). One dir per cache root — NOT /root/.cache whole
# (that would nest over the HF mount).
[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling-tp2/snapshots"
container = "/root/.cache/weight-snapshots"

[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling-tp2/triton"
container = "/root/.cache/triton"

[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling-tp2/tilelang"
container = "/root/.cache/tilelang"

[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling-tp2/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling-tp2/vllm"
container = "/root/.cache/vllm"

[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling-tp2/adaptive-k"
container = "/root/.cache/adaptive-k"

[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling-tp2/draft-trunc"
container = "/root/.cache/draft-trunc"

# Last-moment drop_caches: the built-in (pre-container) one is too early —
# by worker init the container stack (torch+CUDA+code page cache from
# overlay/NFS) refills ~11 GiB, and the GB10 UMA gate (free >= GMU*total)
# is a ~0.75 GiB coin flip per node (observed: 106.28/106.92 vs 107.03 on
# random nodes). Dropping caches right before the engine exec reclaims the
# code-cache part. GMU 0.88->0.86 below gives the extra margin; the hard
# --kv-cache-memory pin is untouched, so the KV pool is unchanged.
[mods]
commands = [
  { command = "sh -c 'echo 3 > /proc/sys/vm/drop_caches 2>/dev/null || true'", on_failure = "warn" },
]

[env]
HF_HUB_OFFLINE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
TORCH_MEM_FRACTION = "0.92"
MAX_JOBS = "4"
# entrypoint cache layout (dirs above): autotune per-boot and EPHEMERAL — a
# persisted autotune cache deadlocks the next TP>1 boot (upstream measurement)
TRITON_CACHE_DIR = "/root/.cache/triton"
TILELANG_CACHE_DIR = "/root/.cache/tilelang"
VLLM_CACHE_ROOT = "/root/.cache/vllm"
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR = "/tmp/flashinfer_autotune"
VLLM_WEIGHT_SNAPSHOT_DIR = "/root/.cache/weight-snapshots"
# TP-scoped tag (OURS — upstream snapshots are per TP rank; its single shared
# tag is safe only on a fixed-topology box set; TP2/TP4 must not collide)
VLLM_WEIGHT_SNAPSHOT_TAG = "fp8w4c-tp2-2026-09-28"
# arx.yaml — RDMA collectives: decode two-shot, prefill RS gather
VLLM_ARX_ALLREDUCE = "1"
VLLM_ARXBIG = "1"
VLLM_ARXBIG_RS = "1"
VLLM_ARXBIG_AG = "0"
VLLM_ARX_TWO_SHOT_MAX_KB = "2048"
# adaptive-k.yaml — online step-cost learner + draft truncation
VLLM_ADAPTIVE_K_CONTROL = "/root/.cache/adaptive-k/control"
VLLM_ADAPTIVE_K_MODEL = "32.1,1.32,0.427"
VLLM_ADAPTIVE_K_PER_REQUEST = "0"
VLLM_ADAPTIVE_K_CODE_MODE = "gated"
VLLM_ADAPTIVE_K_ONLINE = "1"
VLLM_ADAPTIVE_K_COST_MS = "2:29.7,3:33.1,4:36.5,5:39.8,6:43.1,7:46.0,8:48.9"
VLLM_DRAFT_TRUNC_TAU = "0.3"
VLLM_DRAFT_TRUNC_EXPLORE = "4"
VLLM_DRAFT_TRUNC_CONTROL = "/root/.cache/draft-trunc/control"
VLLM_DRAFT_TRUNC_LOG = ""
# fp8.yaml — dense linears the checkpoint kept in bf16 -> FP8/NVFP4 (the
# measured ~1% NLL-on-prose cost is upstream's documented tuned default)
VLLM_DENSE_FP8 = "1"
VLLM_DENSE_FP8_LM_HEAD = "1"
VLLM_DENSE_W4 = 'in_proj_qkvbfg_a$|^model\.|self_attn\.o_proj$|shared_experts\.(gate_up_proj|down_proj)$'  # literal string (TOML backslash escapes)
# megamoe.yaml — W4A16 decode MoE (batch<=8) + fused prefill >=1024 tokens
VLLM_MEGAMOE = "1"
VLLM_MEGAMOE_MAX_TOKENS = "8"
VLLM_MEGAMOE_VARIANT = "20"
VLLM_MOE_PREFILL = "1"
VLLM_MOE_PREFILL_MIN_TOKENS = "1024"
VLLM_MOE_PREFILL_CHECK = "0"
VLLM_MOE_PREFILL_Y8 = "1"
# fixes.yaml — triton sparse MLA (3.3x vs FA2), drafter KV pool, fused mHC
# cap, KDA/conv1d copy elision, KV coordinator files
VLLM_MHC_FUSED_MAX_TOKENS = "32"
VLLM_TRITON_SPARSE_MLA = "1"
VLLM_GLM5NEXT_DRAFT_POOL = "1"
VLLM_GLM5NEXT_DRAFT_BLOCK_SIZE = ""
# sp.yaml — sequence-parallel prefill over TP, quantized gathers, arx prefetch
VLLM_GLM_SP_TP = "1"
VLLM_GLM_SP_MIN_TOKENS = "1024"
VLLM_GLM_SP_FP8_GATHER = "1"
VLLM_GLM_ARX_PREFETCH = "1"
VLLM_GLM_SP_MOE_FUSED = "1"
VLLM_GLM_SP_MOE_QUANT_GATHER = "1"
VLLM_GLM_SP_MOE_CHECK = "0"
# recoverssm.yaml — KDA verify from one state per request (seqs raised above)
VLLM_GLM5NEXT_RECOVERSSM = "1"