glm53-mxfp4-fp8-4x-r39: GLM-5.3 (full) RedHat MXFP4xFP8 boots at TP4 on four Sparks

Working boot recipe after four attempts, each failing differently:
- kindling spark-glm53:v10: multi-node worker init dies (gloo is_in_the_same_node)
- jovian r39 + --attention-backend B12X: TypeError DeepseekV32B12xAttention got
  unexpected 'index_group_builder' (re-cut indexer API newer than r39's class)
- r39 + --linear-backend marlin: prepare_fp8_layer_for_marlin on ReplicatedLinear
  -> AttributeError output_size_per_partition
- r39 default dispatch: FlashInfer mm_fp4 rejects capability 121 (cute-dsl)
FIX: VLLM_DISABLED_KERNELS=FlashInferMxFp4LinearKernel — MXFP4 linears fall to
Marlin, fp8_block layers keep their own backend. Loads 96.34 GiB/rank,
KV 79,616 tokens at 65536 ctx, ~19 t/s decode single request.

Plus the eugr/vllm-main fallback recipe (image verified: vLLM 8244d1c44,
compressed-tensors 0.17.0 with mxfp4/mxfp8 schemes, GlmMoeDsa registered).

recipes/glm53-mxfp4-fp8-4x-eugr/recipe.toml

@@ -0,0 +1,61 @@
+# GLM-5.3 (full, not Flash) — RedHatAI mixed MXFP4×FP8, TP4 on four DGX Spark.
+#
+# Weights: RedHatAI/GLM-5.3-MXFP4xFP8 (2026-10-21, 415.4 GB, 141 shards,
+# GlmMoeDsaForCausalLM, ~4.5 bits/param: routed experts + dense MLPs at MXFP4
+# weights with FP8 dynamic activations, attention/shared-experts/MTP at FP8
+# block-wise). Base: zai-org/GLM-5.3 — the *full* model, 78 layers, 1M window,
+# not the Flash variant our other glm53-* lanes serve.
+#
+# Why 4 nodes: the vendor card serves it with --tensor-parallel-size 8 on
+# 1507 GB-class hardware trimmed to 415 GB. At TP4 each rank holds ~97 GiB of
+# weights inside a ~119 GiB unified pool — tight, so this lane starts with a
+# small window (32768) and few sequences; raise only after the boot numbers
+# show how much KV is really left.
+#
+# Engine: spark-glm53:v10 (kindlingai gx10 stack, vLLM 0.30.1 dev fork) — its
+# model registry already carries GlmMoeDsaForCausalLM + the glm47 parsers this
+# checkpoint expects. The card's "vLLM v0.32" refers to their Hopper-tuned path;
+# the arch class is what matters for a first boot.
+
+engine = "vllm"
+name = "GLM-5.3 (full) MXFP4xFP8, 4x Spark TP4"
+docker_image = "eugr/spark-vllm-b12x:nightly-20261015"
+docker_image_dir = "glm53-kindling"
+entrypoint = ["vllm", "serve"]
+model = "RedHatAI/GLM-5.3-MXFP4xFP8"
+parent_model = "zai-org/GLM-5.3"
+port = 30000
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+# the free stand: dgx5-8 (dgx1-4 runs the production 1017 lane)
+cluster = "dgx58-switch"
+
+args = [
+    "--served-model-name", "glm53-full",
+    # 415 GB of shards are read from the shared NAS cache: eager streaming keeps
+    # the peak host RSS down (each rank touches the whole bundle, keeps 1/4).
+    "--load-format", "auto",
+    "--safetensors-load-strategy", "eager",
+    "--kv-cache-dtype", "fp8",
+    "--gpu-memory-utilization", "0.90",
+    "--max-model-len", "32768",
+    "--max-num-seqs", "4",
+    "--trust-remote-code",
+    "--enable-auto-tool-choice",
+    "--tool-call-parser", "glm47",
+    "--reasoning-parser", "glm47",
+    # the checkpoint ships MTP blocks: the vendor card runs k=5
+    "--speculative-config", '{"method":"mtp","num_speculative_tokens":5}',
+]
+
+[env]
+# 415 GB from the NAS per rank: the default 600 s engine-ready timeout is far
+# too short for a first load (and the FlashInfer/CUTLASS autotune pass).
+VLLM_ENGINE_READY_TIMEOUT_S = "5400"
+# keep the loader from pulling anything: the bundle is already in the cache
+HF_HUB_OFFLINE = "1"
+TORCH_MEM_FRACTION = "0.92"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"

recipes/glm53-mxfp4-fp8-4x-r39/recipe.toml

@@ -0,0 +1,121 @@
+# GLM-5.3 (FULL, non-Flash) — RedHatAI MXFP4×FP8 re-cut — 4× DGX Spark, TP=4.
+#
+# The 2026-10-21 re-cut of the RedHat quant: `RedHatAI/GLM-5.3-MXFP4xFP8`
+# (415.4 GB, 141 shards) instead of the 2026-09 `RedHatAI/GLM-5.3-MXFP4`
+# (403.4 GB). Mixed precision by layer group: routed experts + dense MLPs at
+# MXFP4 weights (FP4-e2m1, E8M0 group-32) with dynamic FP8 activations,
+# attention / shared experts / MTP head at FP8 block-wise (128). Architecture
+# and geometry are unchanged: GlmMoeDsaForCausalLM, 78 layers, MLA + DSA
+# lightning indexer, 1M native window, MTP head n=1 in-checkpoint.
+#
+# Stack: our own jovian r39 lock (`jovian-judgement:r39-glm53`) — the same
+# image that booted and scored **91/100** on this checkpoint family at 4x on
+# 2026-09-14 (`glm53-full-mxfp4-4x-probe`, harness dev65). r39 registers
+# GlmMoeDsaForCausalLM through the NEW `deepseek_v32` NVIDIA path (the GB10
+# target), ships the compressed-tensors W4A4-MXFP4 scheme and fp8_block, and
+# is our reproducible clone-at-pin build.
+#
+# Rejected for this lane (measured or verified 2026-10-22):
+#   * `spark-glm53:v10` (kindling gx10 fork) — registers the arch, loads all
+#     141 shards twice (~45 min/pass) and then dies in multi-node worker init:
+#     `is_in_the_same_node: gloo/tcp/pair.cc` → "WorkerProc failed to start".
+#     The fork is tuned for TP2 on two nodes.
+#   * `eugr/spark-vllm-b12x:nightly-20261015` (vLLM main 8244d1c44) — has the
+#     arch, `mxfp4`/`mxfp8`/`mxfp4_csf` methods and ct 0.17.0 schemes; kept as
+#     the fallback (`glm53-mxfp4-fp8-4x-eugr`) if r39 trips on the re-cut.
+#   * `ghcr.io/fujitsupolycom/glm53-flash-sparkring:v14-arm64` — a pinned
+#     contract for GLM-5.3-**Flash** NVFP4 (InstantTensor + MTP4 sampling +
+#     SparkRing NCCL), not a generic vLLM; nothing to reuse for this checkpoint
+#     beyond its ideas.
+#
+# MEMORY (the binding constraint — 121.63 GiB unified/node):
+#   weights 103.9 GB = 96.7 GiB/rank (was 93.9 GiB on the old cut, +2.8 GiB)
+#   vLLM procs + docker ~12 GiB · KV pin 4 GiB · activations/graphs ~4 GiB
+#   ⇒ ~117 GiB, inside the node but with no room for a second surprise, so
+#   this first boot is 64K context and 4 sequences. Ladder after a clean
+#   boot: 131072 (needs 6.6 GiB/rank) → DCP4 + 512K (see the probe's header).
+#
+# BOOT HAZARD (2026-09-08): a post-load crash with auto-restart began a
+# SECOND full load on top of the first → all four nodes OOM-thrashed. Watch
+# for a second "Loading safetensors" pass and kill immediately.
+
+engine = "vllm"
+name = "GLM-5.3 (full) MXFP4xFP8, 4x TP4 (jovian r39)"
+docker_image = "jovian-judgement:r39-glm53"
+docker_image_dir = "jovian-judgement"
+entrypoint = ["vllm", "serve"]
+model = "RedHatAI/GLM-5.3-MXFP4xFP8"
+parent_model = "zai-org/GLM-5.3"
+port = 30000
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+cluster = "dgx58-switch"
+
+args = [
+    "--served-model-name", "glm53-full",
+    "--trust-remote-code",
+    "--prefill-compute-share", "0.4",
+    # Admission guardrail only (the KV pool is pinned): 0.85 × 121.63 = 103.4 GiB.
+    "--gpu-memory-utilization", "0.85",
+    # 4 GiB/rank: enough for one 65536-token request (51.6 KB/token/rank).
+    "--kv-cache-memory-bytes", "4294967296",
+    # 2026-10-22 05:41: explicit B12X dies in the worker with
+    #   TypeError: DeepseekV32B12xAttention.__init__() got an unexpected
+    #   keyword argument 'index_group_builder'
+    # (the re-cut's indexer API is newer than r39's B12X attention class), so
+    # the first boot lets the fork pick its default backend.
+    # "--attention-backend", "B12X",
+    # MXFP4 GEMM: the FlashInfer kernel accepts capability 121 in is_supported()
+    # and then fails at runtime (mm_fp4, cute-dsl); the native b12x warmup
+    # crashes on this checkpoint's layer ranks. Marlin is the working path.
+    # 2026-10-22 06:15: forcing marlin kills the boot inside
+    #   prepare_fp8_layer_for_marlin: AttributeError: 'ReplicatedLinear' object
+    #   has no attribute 'output_size_per_partition'
+    # — the re-cut puts fp8_block on ReplicatedLinear modules (shared experts /
+    # indexer) and marlin's fp8 prep only handles the parallel-linear shapes.
+    # Marlin IS the right kernel for the MXFP4 linears on GB10 (weight-only A16
+    # dequant), but the flag is global, so the default per-scheme dispatch runs
+    # instead.
+    # "--linear-backend", "marlin",
+    # (2026-10-22 06:44, no flag at all: default dispatch sends the MXFP4
+    # linears to FlashInfer's mm_fp4, which rejects capability 121 at runtime
+    # — flashinfer.utils.BackendSupportedError: mm_fp4 does not support backend
+    # 'cute-dsl' with capability 121. The surgical fix is the env lever below:
+    # disable ONLY that kernel so MXFP4 falls through to Marlin while the
+    # fp8_block layers keep their own (working) backend.)
+    "--load-format", "instanttensor",
+    "--block-size", "256",
+    "--max-model-len", "65536",
+    "--max-num-seqs", "4",
+    "--max-num-batched-tokens", "4096",
+    "--enable-chunked-prefill",
+    "--async-scheduling",
+    "--enable-prefix-caching",
+    # No CuTe-DSL warmup / FlashInfer autotune: their scratch at large shapes
+    # does not fit this headroom.
+    "--kernel-config", """
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
+    "--language-model-only",
+    "--enable-auto-tool-choice",
+    "--tool-call-parser", "glm47",
+    "--reasoning-parser", "glm45",
+    # No --speculative-config on the first boot: the MTP head is real
+    # (num_nextn_predict_layers=1) but draft KV + graph shapes need headroom
+    # this lane does not have yet. Add mtp k=3 after a clean boot.
+]
+
+[env]
+VLLM_ENGINE_READY_TIMEOUT_S = "5400"
+HF_HUB_OFFLINE = "1"
+NCCL_NCHANNELS = "16"
+VLLM_ENABLE_ROCE_ALLREDUCE = "1"
+# compile-OOM guard: 12 parallel nvcc jobs × ~3 GiB behind ~97 GiB of weights
+MAX_JOBS = "2"
+# MXFP4 linears: FlashInfer's cute-dsl mm_fp4 does not support GB10 (cap 121),
+# and forcing --linear-backend marlin globally trips on the fp8_block
+# ReplicatedLinear modules. Disabling just the FlashInfer MXFP4 kernel makes
+# the dispatcher fall through to Marlin for FP4 and leave FP8 alone.
+VLLM_DISABLED_KERNELS = "FlashInferMxFp4LinearKernel"