recipes: glm53-kindling-4x-karmic

recipes/glm53-kindling-4x-karmic/recipe.toml

@@ -0,0 +1,205 @@
+
+# kindlingai/glm-5.3-flash-gx10 — integration mode B (operator 2026-10-01:
+# "image + patches first, their full stack later"). Their IMAGE (15 anchored
+# patches + FlashKDA rebuild, baked experimental/ overlay stack via
+# docker-images/glm53-kindling/build.sh) launched with sm121's mp rendezvous.
+# Every arg/env below is a 1:1 transcription of upstream `image/entrypoint.sh`
+# (`exec vllm serve …` and its `case "$TP"` block) plus the
+# `experimental/compose/*.yaml` env blocks (arx, snapshot, adaptive-k, fp8,
+# megamoe, fixes, sp, recoverssm, tp3-pads) — the stack the upstream README
+# headline numbers were measured with. Deviation ledger:
+# docker-images/glm53-kindling/README.md.
+#
+# NOT used (mode A territory): their entrypoint (mentatd roles/election,
+# preflight, fabric-check, status server, self-test, GID derivation) — sm121
+# owns orchestration, NCCL env comes from the cluster [nccl] block (auto GID
+# detect included). Weights: NAS HF cache (upstream demands local disks per
+# box — operator call, documented). KV pin: upstream per-TP value on GB10 UMA
+# (profiled-unpinned pools starve FlashInfer autotune at boot — upstream).
+
+# Build: `sm121 build` in docker-images/glm53-kindling (tag spark-glm53:v9).
+# Bench: seed 42 hardmode + perf c1,c2 + depth suite, as everywhere.
+
+engine = "vllm"
+name = "GLM-5.3-Flash NVFP4 (kindlingai gx10 stack, 4x, DFlash2, karmic-kraken)"
+docker_image = "spark-glm53-karmic:local"
+# shared image build dir (all three TP variants build the same image)
+docker_image_dir = "karmic-kraken"
+entrypoint = ["vllm", "serve"]
+# Floating model: the engine resolves the org/model id via the NAS HF cache
+# (refs/main → latest snapshot), so a model re-upload is picked up without
+# touching the recipe. The NAS copy stood at snapshot da920bb0 on 2026-10-02
+# (the 94/100 campaign) — informational only; the path follows refs/main.
+model = "nvidia/GLM-5.3-Flash-NVFP4"
+parent_model = "nvidia/GLM-5.3-Flash-NVFP4"
+port = 30000
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # --- upstream entrypoint.sh `exec vllm serve …`, TP=4 case block -----
+    "--served-model-name", "glm53",
+    "--distributed-executor-backend", "mp",
+    "--load-format", "b12x",
+    "--gpu-memory-utilization", "0.86",
+    "--max-model-len", "524288",
+    "--kv-cache-dtype", "fp8",
+    "--block-size", "256",
+    "--kv-cache-memory", "27917287424",
+    "--trust-remote-code",
+    "--enable-prefix-caching",
+    "--max-num-batched-tokens", "16384",
+    "--max-num-seqs", "64",
+    "--long-prefill-token-threshold", "2304",
+    # Karmic-qualified B12X stack (local-inference-lab docker configurator,
+    # glm53-flash profile): the B12X attention backend is the one that handles
+    # GLM-5.3's hybrid fp8_ds_mla KV (both the MHA and the sparse-MLA parts);
+    # fp8 KV + B12X linear/MoE backends. (The kindling transcription used
+    # flashinfer_cutlass + auto backend, which the karmic vLLM tip can't
+    # reconcile for this model.)
+    "--attention-backend", "B12X",
+    "--linear-backend", "b12x",
+    "--moe-backend", "b12x",
+    "--mamba-cache-mode", "align",
+    "--recurrent-checkpoint-policy", "request_boundaries",
+    "--cp-kv-cache-interleave-size", "4",
+    "--additional-config", "{\"glm53_kda_decode_backend\": \"auto\", \"kda_prefill_backend\": \"b12x\"}",
+    "--limit-mm-per-prompt", '{"image":16,"video":0}',
+    # upstream's chat template: thinking:false -> low reasoning effort (the
+    # model has no real non-thinking mode; an open  breaks long output)
+    "--chat-template", "/usr/local/share/glm53-chat-template.jinja",
+    # entrypoint TOOL_ARGS / TOPK_ARGS (vendor parser-name notes apply)
+    "--enable-auto-tool-choice",
+    "--tool-call-parser", "glm47",
+    "--sparse-indexer-topk-backend", "per_row",
+    "--reasoning-parser", "glm45",
+    # adaptive-k overlay EXTRA_ARGS — upstream appends it AFTER the entrypoint's
+    # own spec line, so this JSON is the effective one (same race resolved).
+    "--scheduler-cls", "vllm.v1.core.sched.adaptive_k.AdaptiveKScheduler",
+    "--speculative-config", """
+'{"method":"dflash","disable_eagle_block_drop":true,"model":"{{dflash}}","num_speculative_tokens":7,"num_speculative_tokens_per_batch_size":[[1,1,7],[2,2,5],[3,3,4],[4,4,3],[5,5,2],[6,32,7]]}'""",
+]
+
+# DFlash2 drafter: {{dflash}} resolves per node from the NAS HF
+# cache (upstream mounts DFLASH_HOST_DIR on every box instead).
+[[extra_mounts]]
+repo = "incoai/GLM-5.3-Flash-DFlash2"
+container = "dflash"
+
+# Node-local persistence, upstream CACHE_HOME equivalent (processed weight
+# snapshots ~48 GB/rank for their ~3-min restarts + JIT tiers + adaptive-k/
+# draft-trunc control state). One dir per cache root — NOT /root/.cache whole
+# (that would nest over the HF mount).
+[[extra_mounts]]
+host = "~/sm121-cache/glm53-kindling/snapshots"
+container = "/root/.cache/weight-snapshots"
+
+[[extra_mounts]]
+host = "~/sm121-cache/glm53-kindling/triton"
+container = "/root/.cache/triton"
+
+[[extra_mounts]]
+host = "~/sm121-cache/glm53-kindling/tilelang"
+container = "/root/.cache/tilelang"
+
+[[extra_mounts]]
+host = "~/sm121-cache/glm53-kindling/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/glm53-kindling/vllm"
+container = "/root/.cache/vllm"
+
+[[extra_mounts]]
+host = "~/sm121-cache/glm53-kindling/adaptive-k"
+container = "/root/.cache/adaptive-k"
+
+[[extra_mounts]]
+host = "~/sm121-cache/glm53-kindling/draft-trunc"
+container = "/root/.cache/draft-trunc"
+
+# Last-moment drop_caches: the built-in (pre-container) one is too early —
+# by worker init the container stack (torch+CUDA+code page cache from
+# overlay/NFS) refills ~11 GiB, and the GB10 UMA gate (free >= GMU*total)
+# is a ~0.75 GiB coin flip per node (observed: 106.28/106.92 vs 107.03 on
+# random nodes). Dropping caches right before the engine exec reclaims the
+# code-cache part. GMU 0.88->0.86 below gives the extra margin; the hard
+# --kv-cache-memory pin is untouched, so the KV pool is unchanged.
+[mods]
+commands = [
+  { command = "sh -c 'echo 3 > /proc/sys/vm/drop_caches 2>/dev/null || true'", on_failure = "warn" },
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+TORCH_MEM_FRACTION = "0.92"
+MAX_JOBS = "4"
+# entrypoint cache layout (dirs above): autotune per-boot and EPHEMERAL — a
+# persisted autotune cache deadlocks the next TP>1 boot (upstream measurement)
+TRITON_CACHE_DIR = "/root/.cache/triton"
+TILELANG_CACHE_DIR = "/root/.cache/tilelang"
+VLLM_CACHE_ROOT = "/root/.cache/vllm"
+VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR = "/tmp/flashinfer_autotune"
+# Karmic B12X stack env (lil-catalog glm53-flash profile): the dynamic-split
+# MHC/KDA kernels that handle GLM-5.3's hybrid attention on the B12X backend.
+B12X_DYNAMIC_SPLIT_B12X_MHC_TILE_LANG = "0"
+B12X_MHC_TILE_LANG = "1"
+B12X_MHC_TILE_LANG_FAST = "1"
+B12X_MHC_TILE_LANG_PREFILL = "1"
+B12X_MHC_TILE_LANG_PREFILL_FAST = "1"
+B12X_TQ_TILE_LANG = "1"
+VLLM_WEIGHT_SNAPSHOT_DIR = "/root/.cache/weight-snapshots"
+# TP-scoped tag (OURS — upstream snapshots are per TP rank; its single shared
+# tag is safe only on a fixed-topology box set; TP2/TP4 must not collide)
+VLLM_WEIGHT_SNAPSHOT_TAG = "fp8w4c-tp4-2026-09-28"
+# arx.yaml — RDMA collectives: decode two-shot, prefill RS gather
+VLLM_ARX_ALLREDUCE = "1"
+VLLM_ARXBIG = "1"
+VLLM_ARXBIG_RS = "1"
+VLLM_ARXBIG_AG = "0"
+VLLM_ARX_TWO_SHOT_MAX_KB = "2048"
+# adaptive-k.yaml — online step-cost learner + draft truncation
+VLLM_ADAPTIVE_K_CONTROL = "/root/.cache/adaptive-k/control"
+VLLM_ADAPTIVE_K_MODEL = "26.4,0.660,0.484"
+VLLM_ADAPTIVE_K_PER_REQUEST = "0"
+VLLM_ADAPTIVE_K_CODE_MODE = "gated"
+VLLM_ADAPTIVE_K_ONLINE = "1"
+VLLM_ADAPTIVE_K_COST_MS = "2:29.7,3:33.1,4:36.5,5:39.8,6:43.1,7:46.0,8:48.9"
+VLLM_DRAFT_TRUNC_TAU = "0.3"
+VLLM_DRAFT_TRUNC_EXPLORE = "4"
+VLLM_DRAFT_TRUNC_CONTROL = "/root/.cache/draft-trunc/control"
+VLLM_DRAFT_TRUNC_LOG = ""
+# fp8.yaml — dense linears the checkpoint kept in bf16 -> FP8/NVFP4 (the
+# measured ~1% NLL-on-prose cost is upstream's documented tuned default)
+VLLM_DENSE_FP8 = "1"
+VLLM_DENSE_FP8_LM_HEAD = "1"
+VLLM_DENSE_W4 = 'in_proj_qkvbfg_a$|^model\.|self_attn\.o_proj$|shared_experts\.(gate_up_proj|down_proj)$'  # literal string (TOML backslash escapes)
+# megamoe.yaml — W4A16 decode MoE (batch<=8) + fused prefill >=1024 tokens
+VLLM_MEGAMOE = "1"
+VLLM_MEGAMOE_MAX_TOKENS = "8"
+VLLM_MEGAMOE_VARIANT = "20"
+VLLM_MOE_PREFILL = "1"
+VLLM_MOE_PREFILL_MIN_TOKENS = "1024"
+VLLM_MOE_PREFILL_CHECK = "0"
+VLLM_MOE_PREFILL_Y8 = "1"
+# fixes.yaml — triton sparse MLA (3.3x vs FA2), drafter KV pool, fused mHC
+# cap, KDA/conv1d copy elision, KV coordinator files
+VLLM_MHC_FUSED_MAX_TOKENS = "32"
+VLLM_TRITON_SPARSE_MLA = "1"
+VLLM_GLM5NEXT_DRAFT_POOL = "1"
+VLLM_GLM5NEXT_DRAFT_BLOCK_SIZE = ""
+# sp.yaml — sequence-parallel prefill over TP, quantized gathers, arx prefetch
+VLLM_GLM_SP_TP = "1"
+VLLM_GLM_SP_MIN_TOKENS = "1024"
+VLLM_GLM_SP_FP8_GATHER = "1"
+VLLM_GLM_ARX_PREFETCH = "1"
+VLLM_GLM_SP_MOE_FUSED = "1"
+VLLM_GLM_SP_MOE_QUANT_GATHER = "1"
+VLLM_GLM_SP_MOE_CHECK = "0"
+# recoverssm.yaml — KDA verify from one state per request (seqs raised above)
+VLLM_GLM5NEXT_RECOVERSSM = "1"