launcher/recipes/glm53-kindling-4x-karmic/recipe.toml

# kindlingai/glm-5.3-flash-gx10 — integration mode B (operator 2026-10-01:
# "image + patches first, their full stack later"). Their IMAGE (15 anchored
# patches + FlashKDA rebuild, baked experimental/ overlay stack via
# docker-images/glm53-kindling/build.sh) launched with sm121's mp rendezvous.
# Every arg/env below is a 1:1 transcription of upstream `image/entrypoint.sh`
# (`exec vllm serve …` and its `case "$TP"` block) plus the
# `experimental/compose/*.yaml` env blocks (arx, snapshot, adaptive-k, fp8,
# megamoe, fixes, sp, recoverssm, tp3-pads) — the stack the upstream README
# headline numbers were measured with. Deviation ledger:
# docker-images/glm53-kindling/README.md.
#
# NOT used (mode A territory): their entrypoint (mentatd roles/election,
# preflight, fabric-check, status server, self-test, GID derivation) — sm121
# owns orchestration, NCCL env comes from the cluster [nccl] block (auto GID
# detect included). Weights: NAS HF cache (upstream demands local disks per
# box — operator call, documented). KV pin: upstream per-TP value on GB10 UMA
# (profiled-unpinned pools starve FlashInfer autotune at boot — upstream).

# Build: `sm121 build` in docker-images/glm53-kindling (tag spark-glm53:v9).
# Bench: seed 42 hardmode + perf c1,c2 + depth suite, as everywhere.

engine = "vllm"
name = "GLM-5.3-Flash NVFP4 (kindlingai gx10 stack, 4x, DFlash2, karmic-kraken)"
docker_image = "spark-glm53-karmic:local"
# shared image build dir (all three TP variants build the same image)
docker_image_dir = "karmic-kraken"
entrypoint = ["vllm", "serve"]
# Floating model: the engine resolves the org/model id via the NAS HF cache
# (refs/main → latest snapshot), so a model re-upload is picked up without
# touching the recipe. The NAS copy stood at snapshot da920bb0 on 2026-10-02
# (the 94/100 campaign) — informational only; the path follows refs/main.
model = "nvidia/GLM-5.3-Flash-NVFP4"
parent_model = "nvidia/GLM-5.3-Flash-NVFP4"
port = 30000
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # --- upstream entrypoint.sh `exec vllm serve …`, TP=4 case block -----
    "--served-model-name", "glm53",
    "--distributed-executor-backend", "mp",
    "--load-format", "b12x",
    "--gpu-memory-utilization", "0.86",
    "--max-model-len", "524288",
    "--kv-cache-dtype", "fp8",
    "--block-size", "256",
    "--kv-cache-memory", "27917287424",
    "--trust-remote-code",
    "--enable-prefix-caching",
    "--max-num-batched-tokens", "16384",
    "--max-num-seqs", "64",
    "--long-prefill-token-threshold", "2304",
    # Karmic-qualified B12X stack (local-inference-lab docker configurator,
    # glm53-flash profile): the B12X attention backend is the one that handles
    # GLM-5.3's hybrid fp8_ds_mla KV (both the MHA and the sparse-MLA parts);
    # fp8 KV + B12X linear/MoE backends. (The kindling transcription used
    # flashinfer_cutlass + auto backend, which the karmic vLLM tip can't
    # reconcile for this model.)
    "--attention-backend", "B12X",
    "--linear-backend", "b12x",
    "--moe-backend", "b12x",
    "--mamba-cache-mode", "align",
    "--recurrent-checkpoint-policy", "request_boundaries",
    "--cp-kv-cache-interleave-size", "4",
    "--additional-config", "{\"glm53_kda_decode_backend\": \"auto\", \"kda_prefill_backend\": \"b12x\"}",
    "--limit-mm-per-prompt", '{"image":16,"video":0}',
    # upstream's chat template: thinking:false -> low reasoning effort (the
    # model has no real non-thinking mode; an open  breaks long output)
    "--chat-template", "/usr/local/share/glm53-chat-template.jinja",
    # entrypoint TOOL_ARGS / TOPK_ARGS (vendor parser-name notes apply)
    "--enable-auto-tool-choice",
    "--tool-call-parser", "glm47",
    "--sparse-indexer-topk-backend", "per_row",
    "--reasoning-parser", "glm45",
    # adaptive-k overlay EXTRA_ARGS — upstream appends it AFTER the entrypoint's
    # own spec line, so this JSON is the effective one (same race resolved).
    "--scheduler-cls", "vllm.v1.core.sched.adaptive_k.AdaptiveKScheduler",
    "--speculative-config", """
'{"method":"dflash","disable_eagle_block_drop":true,"model":"{{dflash}}","num_speculative_tokens":7,"num_speculative_tokens_per_batch_size":[[1,1,7],[2,2,5],[3,3,4],[4,4,3],[5,5,2],[6,32,7]]}'""",
]

# DFlash2 drafter: {{dflash}} resolves per node from the NAS HF
# cache (upstream mounts DFLASH_HOST_DIR on every box instead).
[[extra_mounts]]
repo = "incoai/GLM-5.3-Flash-DFlash2"
container = "dflash"

# Node-local persistence, upstream CACHE_HOME equivalent (processed weight
# snapshots ~48 GB/rank for their ~3-min restarts + JIT tiers + adaptive-k/
# draft-trunc control state). One dir per cache root — NOT /root/.cache whole
# (that would nest over the HF mount).
[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling/snapshots"
container = "/root/.cache/weight-snapshots"

[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling/triton"
container = "/root/.cache/triton"

[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling/tilelang"
container = "/root/.cache/tilelang"

[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling/vllm"
container = "/root/.cache/vllm"

[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling/adaptive-k"
container = "/root/.cache/adaptive-k"

[[extra_mounts]]
host = "~/sm121-cache/glm53-kindling/draft-trunc"
container = "/root/.cache/draft-trunc"

# Last-moment drop_caches: the built-in (pre-container) one is too early —
# by worker init the container stack (torch+CUDA+code page cache from
# overlay/NFS) refills ~11 GiB, and the GB10 UMA gate (free >= GMU*total)
# is a ~0.75 GiB coin flip per node (observed: 106.28/106.92 vs 107.03 on
# random nodes). Dropping caches right before the engine exec reclaims the
# code-cache part. GMU 0.88->0.86 below gives the extra margin; the hard
# --kv-cache-memory pin is untouched, so the KV pool is unchanged.
[mods]
commands = [
  { command = "sh -c 'echo 3 > /proc/sys/vm/drop_caches 2>/dev/null || true'", on_failure = "warn" },
]

[env]
HF_HUB_OFFLINE = "1"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
TORCH_MEM_FRACTION = "0.92"
MAX_JOBS = "4"
# entrypoint cache layout (dirs above): autotune per-boot and EPHEMERAL — a
# persisted autotune cache deadlocks the next TP>1 boot (upstream measurement)
TRITON_CACHE_DIR = "/root/.cache/triton"
TILELANG_CACHE_DIR = "/root/.cache/tilelang"
VLLM_CACHE_ROOT = "/root/.cache/vllm"
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR = "/tmp/flashinfer_autotune"
# Karmic B12X stack env (lil-catalog glm53-flash profile): the dynamic-split
# MHC/KDA kernels that handle GLM-5.3's hybrid attention on the B12X backend.
B12X_DYNAMIC_SPLIT_B12X_MHC_TILE_LANG = "0"
B12X_MHC_TILE_LANG = "1"
B12X_MHC_TILE_LANG_FAST = "1"
B12X_MHC_TILE_LANG_PREFILL = "1"
B12X_MHC_TILE_LANG_PREFILL_FAST = "1"
B12X_TQ_TILE_LANG = "1"
VLLM_WEIGHT_SNAPSHOT_DIR = "/root/.cache/weight-snapshots"
# TP-scoped tag (OURS — upstream snapshots are per TP rank; its single shared
# tag is safe only on a fixed-topology box set; TP2/TP4 must not collide)
VLLM_WEIGHT_SNAPSHOT_TAG = "fp8w4c-tp4-2026-09-28"
# arx.yaml — RDMA collectives: decode two-shot, prefill RS gather
VLLM_ARX_ALLREDUCE = "1"
VLLM_ARXBIG = "1"
VLLM_ARXBIG_RS = "1"
VLLM_ARXBIG_AG = "0"
VLLM_ARX_TWO_SHOT_MAX_KB = "2048"
# adaptive-k.yaml — online step-cost learner + draft truncation
VLLM_ADAPTIVE_K_CONTROL = "/root/.cache/adaptive-k/control"
VLLM_ADAPTIVE_K_MODEL = "26.4,0.660,0.484"
VLLM_ADAPTIVE_K_PER_REQUEST = "0"
VLLM_ADAPTIVE_K_CODE_MODE = "gated"
VLLM_ADAPTIVE_K_ONLINE = "1"
VLLM_ADAPTIVE_K_COST_MS = "2:29.7,3:33.1,4:36.5,5:39.8,6:43.1,7:46.0,8:48.9"
VLLM_DRAFT_TRUNC_TAU = "0.3"
VLLM_DRAFT_TRUNC_EXPLORE = "4"
VLLM_DRAFT_TRUNC_CONTROL = "/root/.cache/draft-trunc/control"
VLLM_DRAFT_TRUNC_LOG = ""
# fp8.yaml — dense linears the checkpoint kept in bf16 -> FP8/NVFP4 (the
# measured ~1% NLL-on-prose cost is upstream's documented tuned default)
VLLM_DENSE_FP8 = "1"
VLLM_DENSE_FP8_LM_HEAD = "1"
VLLM_DENSE_W4 = 'in_proj_qkvbfg_a$|^model\.|self_attn\.o_proj$|shared_experts\.(gate_up_proj|down_proj)$'  # literal string (TOML backslash escapes)
# megamoe.yaml — W4A16 decode MoE (batch<=8) + fused prefill >=1024 tokens
VLLM_MEGAMOE = "1"
VLLM_MEGAMOE_MAX_TOKENS = "8"
VLLM_MEGAMOE_VARIANT = "20"
VLLM_MOE_PREFILL = "1"
VLLM_MOE_PREFILL_MIN_TOKENS = "1024"
VLLM_MOE_PREFILL_CHECK = "0"
VLLM_MOE_PREFILL_Y8 = "1"
# fixes.yaml — triton sparse MLA (3.3x vs FA2), drafter KV pool, fused mHC
# cap, KDA/conv1d copy elision, KV coordinator files
VLLM_MHC_FUSED_MAX_TOKENS = "32"
VLLM_TRITON_SPARSE_MLA = "1"
VLLM_GLM5NEXT_DRAFT_POOL = "1"
VLLM_GLM5NEXT_DRAFT_BLOCK_SIZE = ""
# sp.yaml — sequence-parallel prefill over TP, quantized gathers, arx prefetch
VLLM_GLM_SP_TP = "1"
VLLM_GLM_SP_MIN_TOKENS = "1024"
VLLM_GLM_SP_FP8_GATHER = "1"
VLLM_GLM_ARX_PREFETCH = "1"
VLLM_GLM_SP_MOE_FUSED = "1"
VLLM_GLM_SP_MOE_QUANT_GATHER = "1"
VLLM_GLM_SP_MOE_CHECK = "0"
# recoverssm.yaml — KDA verify from one state per request (seqs raised above)
VLLM_GLM5NEXT_RECOVERSSM = "1"