recipes: glm53-flash-r28-dflash2-2x

recipes/glm53-flash-r28-dflash2-2x/recipe.toml

@@ -0,0 +1,156 @@
+# BLOCKED 2026-09-08 (r28 re-test): the r28 tree advanced the DCP+DFlash
+# boundary logic (fa4720ea) but the drafter still dies — now at
+# backend selection, not the old C4 assert: the DFlash attention kernel
+# is BF16-KV-only ("K and V caches must be bfloat16") while the GLM-5.3
+# engine runs fp8_ds_mla KV (fork default; no per-drafter dtype override
+# in this tree) -> "No valid attention backend found ... kv_cache_dtype=
+# fp8_ds_mla, use_mla=False, block_size=256". Fork-side work: teach
+# DFlashAttention the fp8_ds_mla KV, or add a drafter bf16-KV override.
+#
+# DFlash2 re-test on the r28 full stack (2026-09-08, operator: "maybe it
+# works now"). The 2026-09-06 blocker (C4 parent-page stride assert + FP8-KV
+# drafter backend rejection) targeted the r27 tree; r28's vLLM carries
+# fa4720ea "fix(cache): restore GLM boundaries across DCP and DFlash groups"
+# + the #646 2048/256 page split, so the geometry may now line up.
+# Recipe = the r27-dflash2 2x shape (deleted at 17a91a9) retargeted:
+# method "dflash", incoai/GLM-5.3-Flash-DFlash2 draft, k=7, B12X backend,
+# block-size 256, 10 GiB KV pin (2x profile).
+#
+engine = "vllm"
+name = "GLM-5.3-Flash NVFP4 (r28 img, DFlash2 k=7, 2x re-test)"
+docker_image = "jovian-judgement:r28-glm53"  # full r28 lock via build_profile
+build_profile = "r28"
+docker_image_dir = "jovian-judgement"
+# Required: the clean r12 image follows the solar contract (ENTRYPOINT []
+# + CMD bash), so the recipe carries the entrypoint (the old day-0-based
+# r12 image had ["vllm","serve"] baked in, which is why this was absent
+# until the clean rebuild).
+entrypoint = ["vllm", "serve"]
+model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
+parent_model = "zai-org/GLM-5.3-Flash"
+required_nodes = 2
+tensor_parallel_size = 2
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 quants)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--distributed-executor-backend", "mp",
+    # GMU is ONLY a startup guardrail here (KV pool is pinned below): 0.90
+    # demands 109.46 GiB free on GB10's 121.63 GiB, but driver reserve +
+    # page cache routinely leave ~108.5 -> flaky
+    # "Free memory ... less than desired" boot deaths (this was the class
+    # of the old "54-second death"), and post-crash reruns see ~106.6 GiB.
+    # 0.82 = 99.7 GiB, below any realistic dip; does NOT change what gets
+    # allocated (weights ~93 + pinned KV 10 + activations).
+    "--gpu-memory-utilization", "0.90",
+    # Pinned 10 GiB KV (≈1.16M tokens) — GMU sizing has no headroom at
+    # 92.72 GiB weights/node; pinning also keeps warmup off the swap path
+    "--kv-cache-memory-bytes", "10737418240",
+    # No --kv-cache-dtype: let the r12 fork pick its default (the explicit
+    # fp8 / fp8_ds_mla both failed auto-selection on this build).
+    # The GLM5Next model wires the B12X sparse-MLA backend (sm_120/sm_121)
+    # when the global backend is B12X (glm5next/nvidia/attention.py:
+    # _select_sparse_backend); the sm12x auto-priority list never offers it
+    # (FLASHINFER_MLA_SPARSE_SM120 only supports head 576, not GLM's 512).
+    "--attention-backend", "B12X",
+    # pure I/O loader (kept across every config; 171 GB in ~5.5 min vs ~16 NFS)
+    "--load-format", "instanttensor",
+    # [R17 FULL-CLUSTER TEST] lil manifest backends (quality gate: hardmode
+    # must stay ≥88; r12-era this cluster dropped it to 74 — r17 may carry
+    # the fix through — one boot decides).
+    "--block-size", "256",
+    # FINAL BISECT (2026-09-03, hardmode 176pt, r17 img):
+    #   knobs-only 87: d0 c1 24.1, d8192 23.8
+    #   +moe b12x  88: d0 c1 23.5, d8192 26.6  <- SHIP (2 pts d8192)
+    #   +linear b12x 44: W4A4 linear kernel destroys numerics (param-precision 0%)
+    #   +align     88: but d0 c1 18.6 (-5) — perf-negative, dropped
+    # (all three together = 44-74; the lab needs its private packed package)
+    "--max-model-len", "524288",
+    "--max-num-seqs", "16",
+    "--max-num-batched-tokens", "4096",
+    "--enable-chunked-prefill",
+    "--async-scheduling",
+    "--enable-prefix-caching",
+    "--disable-custom-all-reduce",
+    # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup
+    "--kernel-config", """
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
+    # Built-in MTP draft, 3 tokens (the r12-qualified "mtp:3" mode — their
+    # x86 numbers: 254 tok/s CC1, 2.37 accept; the checkpoint embeds up to
+    # 5 via the model-mtp-0000x shards but 3 is the verified depth). This is
+    # the checkpoint's own MTP head (Glm5NextMTPModel), NOT dspark — dspark
+    # is the DeepSeek-V4 family's mechanism; GLM's other r12 mode is
+    # dflash2:7 with a separate trained draft checkpoint.
+    # The draft model's vllm_config takes its attention backend from the
+    # SPECULATIVE config, not the global flag (eagle/utils.py:
+    # _make_eagle_draft_vllm_config) — B12X here mirrors the main model's.
+    # r18 used draft_model/FLASH_ATTN/kv auto — both rejected on that fork
+# (FP8-KV-drafter backend check + C4-page stride assert). r27 qualifies
+# dflash2 upstream (the #646 2048+256 page split is in this tree); the
+# drafter/depth follow the working aiden GB10 lane (method "dflash").
+"--speculative-config", """
+'{"method":"dflash","model":"incoai/GLM-5.3-Flash-DFlash2","num_speculative_tokens":7}'""",
+    "--language-model-only",
+    # MTP + CUDA graphs degrades OUTPUT QUALITY on the glm53 day-0 dev build
+    # (forum #381350/#44, CONFIRMED by A/B bench 2026-08-28: same seed/flags,
+    # graphs on = 89/100, eager = 92/100; decode unchanged 28.4 t/s @ c1,
+    # c2 -7%, TTFT +1.6s). The b12x glm52 stack does NOT need this (its
+    # graphs+MTP combo benches clean at 100/100) — bug is build-specific.
+    # graphs moved BEFORE tool-call-parser; enforce-eager removed for the
+    # r17-graphs test (bisect A2 was 88/100 eager; graphs+MTP fix is public
+    # in r17 refs? verify by bench).
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--reasoning-parser", "glm45",
+]
+archived = true
+
+[env]
+HF_HUB_OFFLINE = "1"
+# FlashInfer JIT (mxfp8_gemm_cutlass_sm120, ~12 kernels) compiles at first
+# boot with 12 parallel nvcc jobs -> OOM-killed (code 137) on GB10's
+# 121 GB unified while the 93 GB weights + 10 GB KV pin are resident.
+# MAX_JOBS caps ninja parallelism so the JIT fits in the remaining ~20 GB.
+MAX_JOBS = "2"
+# NoPE-MLA rope-pad runtime gate (mod baked into the image)
+VLLM_MLA_NOPE_PAD_ROPE = "1"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+# Upstream launch-script env (memory + compile knobs)
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_USE_AOT_COMPILE = "0"
+VLLM_USE_STANDALONE_COMPILE = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
+TORCH_USE_RTLD_GLOBAL = "1"
+VLLM_NO_USAGE_STATS = "1"
+
+# Architecture (aarch64 / sm_121a DGX Spark)
+TORCH_CUDA_ARCH_LIST = "12.1a"
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
+CUTE_DSL_ARCH = "sm_121a"
+CMAKE_CUDA_ARCHITECTURES = "121"
+OMP_NUM_THREADS = "2"
+
+[[extra_mounts]]
+host = "~/sm121-cache/glm53-jit-121f/b12x"
+container = "/root/.cache/b12x"
+
+# JIT caches persist per-node across serves (was: cold /root/.cache per
+# container -> ~20 min flashinfer/torch recompile every serve). The host
+# dirs are shared between lanes of the same arch — each cache
+# self-namespaces by its engine version key internally.
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"