recipes: glm53-full-native-8x-dspark8

recipes/glm53-full-native-8x-dspark8/recipe.toml

@@ -0,0 +1,127 @@
+# GLM-5.3 (full, NATIVE zai-org FP8) + DSpark K8 speculator — 8x @64K.
+#
+# The operator's "whole model" lane (2026-09-18 night): NOT the RedHatAI
+# MXFP4 quant — the original `zai-org/GLM-5.3` checkpoint (753.3 GB, FP8
+# e4m3 block-128, 141 shards, already on the NAS: blobs 704 GB, refs/main
+# aca966e4). Same glm_moe_dsa architecture as the GLM-5.2 lane (the
+# operator's read: the architecture did not change between 5.2 and 5.3),
+# so the Aiden GLM image family carries it.
+#
+# Image: aidendle94/sparkrun-vllm-glm53-gb10:production-1.2 (31.2 GB,
+# Aiden's GLM-5.2/5.3 production stack — the "aiden 5.2" image). Verified
+# in-image (2026-09-18): vLLM dev20051+g487ecf187 has both the
+# DSparkDraftModel speculators loader (config/speculative.py +
+# models/registry.py) and the glm_moe_dsa path
+# (models/deepseek_v32/nvidia/model.py). Their DFlash2 drafter runs on
+# this image WITHOUT the r37 workarounds (no skip-layers flag, no draft
+# page patch) -> the shared-cache-contamination bug is absent from this
+# tree, so the 4x-dspark8 r37 workarounds are NOT ported.
+#
+# Loader (operator call): the standard safetensors loader
+# (`--load-format auto`), NOT InstantTensor. InstantTensor streams the
+# WHOLE checkpoint per rank (~750 GB/rank over the 10G NFS mgmt NIC,
+# ~90-100 min); the standard loader reads each rank's TP slices
+# (page-cache shared) and the dspark38 Aiden precedent already runs on
+# `auto`.
+#
+# Memory math (121.63 GiB/node): weights 753.3/8 = 94.2 GB/node;
+# GMU 0.85 -> 111.0 GB budget; 111.0 - 94.2 - ~10 (process) = ~6.8 GB KV.
+# 64K request = 3.45 GB/rank (52.7 KB/token) -> ~1.9x; 96K = 5.1 GB ->
+# ~1.3x; 512K (26.3 GB) DOES NOT FIT — physical limit, hence
+# max-model-len 65536 (the lab r34 doc's qualified Model limit; their
+# GMU 0.97/0.98 numbers are RTX 6000 (96 GB, smaller baseline reservation)
+# and do not pass the GB10 admission check — measured free 108.78 GiB).
+#
+# Speculator: RedHatAI/GLM-5.3-speculator.dspark (2.3 GB, NAS snapshot
+# 198b3ced) at K8, `method: dspark` — the same drafter as the 4x dspark8
+# lane (that one on the r37 image; here on Aiden's tree, DSpark support
+# verified in-image).
+#
+# Contract mirrors the proven Aiden precedents on this image family:
+# glm53-flash-aiden-2x (same image, DFlash2: kv fp8_e4m3, moe-backend
+# marlin, block 2304, no --attention-backend — the image routes B12X
+# natively) + deepseek-v4-flash-dspark38 (DSpark + CUDA graphs without
+# enforce-eager, --load-format auto, block 256) + the r34 doc's scheduler
+# bounds (seqs 8, batched tokens 2048).
+engine = "vllm"
+name = "GLM-5.3 (full native FP8) + DSpark K8 (aiden prod-1.2 img, zai-org ckpt, 8x @64K)"
+docker_image = "aidendle94/sparkrun-vllm-glm53-gb10:production-1.2"
+entrypoint = []
+model = "zai-org/GLM-5.3"
+parent_model = "zai-org/GLM-5.3"
+required_nodes = 8
+tensor_parallel_size = 8
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+
+args = [
+    # Stable served name — quant-agnostic (shared across glm53 lanes)
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--reasoning-parser", "glm45",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--default-chat-template-kwargs", """
+'{"enable_thinking": true, "reasoning_effort": "high"}'""",
+    # Aiden-image-proven spellings (glm53-flash-aiden-2x baseline):
+    "--kv-cache-dtype", "fp8_e4m3",
+    "--moe-backend", "marlin",
+    # the plain safetensors loader (operator call 2026-09-18): InstantTensor
+    # streams the WHOLE checkpoint per rank (~704 GB/rank at 130 MB/s = ~90
+    # min, and it hung in the first Aiden boot); `auto` on this image =
+    # InstantTensor (verified in the boot log), so the format is spelled
+    # explicitly. The standard loader reads each rank's TP slices with mmap
+    # (page-cache shared) — per-node NFS traffic ≈ the model, not 8x it.
+    "--load-format", "safetensors",
+    # block 256 (dspark38 precedent; the baseline's 2304 is an NVFP4-Flash
+    # knob)
+    "--block-size", "256",
+    # No KV pin: the pool is GMU-driven (~6.8 GB, see header).
+    "--max-model-len", "65536",
+    "--max-num-seqs", "8",
+    "--max-num-batched-tokens", "2048",
+    "--enable-chunked-prefill",
+    "--enable-prefix-caching",
+    "--gpu-memory-utilization", "0.85",
+    # DSpark K8 speculator (RedHatAI/GLM-5.3-speculator.dspark, 2.3 GB, in
+    # the shared HF cache; HF_HUB_OFFLINE=1 resolves it there).
+    "--speculative-config", """
+'{"model":"RedHatAI/GLM-5.3-speculator.dspark","num_speculative_tokens":8,"method":"dspark"}'""",
+    "--language-model-only",
+]
+
+[env]
+
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+TORCH_USE_RTLD_GLOBAL = "1"
+TORCHINDUCTOR_COMPILE_THREADS = "1"
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+VLLM_NO_USAGE_STATS = "1"
+OMP_NUM_THREADS = "2"
+# Compile-OOM guard (cold /root/.cache compiles behind 94 GB of resident
+# weights; the JIT mounts below keep it warm after the first boot).
+MAX_JOBS = "2"
+
+# JIT caches persist per-node across serves (cold /root/.cache -> ~20 min
+# recompile every serve). The host dirs are shared between lanes of the
+# same arch — each cache self-namespaces by its engine version internally.
+[[extra_mounts]]
+host = "~/sm121-cache/glm53-jit-121f/b12x"
+container = "/root/.cache/b12x"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/flashinfer"
+container = "/root/.cache/flashinfer"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/torchinductor"
+container = "/root/.cache/torchinductor"
+
+[[extra_mounts]]
+host = "~/sm121-cache/jit-121f/vllm"
+container = "/root/.cache/vllm"