launcher/recipes/glm53-full-native-8x-dspark8/recipe.toml

# GLM-5.3 (full, NATIVE zai-org FP8) + DSpark K8 speculator — 8x @64K.
#
# The operator's "whole model" lane (2026-09-18 night): NOT the RedHatAI
# MXFP4 quant — the original `zai-org/GLM-5.3` checkpoint (753.3 GB, FP8
# e4m3 block-128, 141 shards, already on the NAS: blobs 704 GB, refs/main
# aca966e4). Same glm_moe_dsa architecture as the GLM-5.2 lane (the
# operator's read: the architecture did not change between 5.2 and 5.3),
# so the Aiden GLM image family carries it.
#
# Image: aidendle94/sparkrun-vllm-glm53-gb10:production-1.2 (31.2 GB,
# Aiden's GLM-5.2/5.3 production stack — the "aiden 5.2" image). Verified
# in-image (2026-09-18): vLLM dev20051+g487ecf187 has both the
# DSparkDraftModel speculators loader (config/speculative.py +
# models/registry.py) and the glm_moe_dsa path
# (models/deepseek_v32/nvidia/model.py). Their DFlash2 drafter runs on
# this image WITHOUT the r37 workarounds (no skip-layers flag, no draft
# page patch) -> the shared-cache-contamination bug is absent from this
# tree, so the 4x-dspark8 r37 workarounds are NOT ported.
#
# Loader (operator call): the standard safetensors loader
# (`--load-format auto`), NOT InstantTensor. InstantTensor streams the
# WHOLE checkpoint per rank (~750 GB/rank over the 10G NFS mgmt NIC,
# ~90-100 min); the standard loader reads each rank's TP slices
# (page-cache shared) and the dspark38 Aiden precedent already runs on
# `auto`.
#
# Memory math (121.63 GiB/node): weights 753.3/8 = 94.2 GB/node;
# GMU 0.85 -> 111.0 GB budget; 111.0 - 94.2 - ~10 (process) = ~6.8 GB KV.
# 64K request = 3.45 GB/rank (52.7 KB/token) -> ~1.9x; 96K = 5.1 GB ->
# ~1.3x; 512K (26.3 GB) DOES NOT FIT — physical limit, hence
# max-model-len 65536 (the lab r34 doc's qualified Model limit; their
# GMU 0.97/0.98 numbers are RTX 6000 (96 GB, smaller baseline reservation)
# and do not pass the GB10 admission check — measured free 108.78 GiB).
#
# Speculator: RedHatAI/GLM-5.3-speculator.dspark (2.3 GB, NAS snapshot
# 198b3ced) at K8, `method: dspark` — the same drafter as the 4x dspark8
# lane (that one on the r37 image; here on Aiden's tree, DSpark support
# verified in-image).
#
# Contract mirrors the proven Aiden precedents on this image family:
# glm53-flash-aiden-2x (same image, DFlash2: kv fp8_e4m3, moe-backend
# marlin, block 2304, no --attention-backend — the image routes B12X
# natively) + deepseek-v4-flash-dspark38 (DSpark + CUDA graphs without
# enforce-eager, --load-format auto, block 256) + the r34 doc's scheduler
# bounds (seqs 8, batched tokens 2048).
engine = "vllm"
name = "GLM-5.3 (full native FP8) + DSpark K8 (aiden prod-1.2 img, zai-org ckpt, 8x @64K)"
docker_image = "aidendle94/sparkrun-vllm-glm53-gb10:production-1.2"
entrypoint = []
model = "zai-org/GLM-5.3"
parent_model = "zai-org/GLM-5.3"
required_nodes = 8
tensor_parallel_size = 8
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 lanes)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--reasoning-parser", "glm45",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--default-chat-template-kwargs", """
'{"enable_thinking": true, "reasoning_effort": "high"}'""",
    # Aiden-image-proven spellings (glm53-flash-aiden-2x baseline):
    "--kv-cache-dtype", "fp8_e4m3",
    "--moe-backend", "marlin",
    # the plain safetensors loader (operator call 2026-09-18): InstantTensor
    # streams the WHOLE checkpoint per rank (~704 GB/rank at 130 MB/s = ~90
    # min, and it hung in the first Aiden boot); `auto` on this image =
    # InstantTensor (verified in the boot log), so the format is spelled
    # explicitly. The standard loader reads each rank's TP slices with mmap
    # (page-cache shared) — per-node NFS traffic ≈ the model, not 8x it.
    "--load-format", "safetensors",
    # block 256 (dspark38 precedent; the baseline's 2304 is an NVFP4-Flash
    # knob)
    "--block-size", "256",
    # No KV pin: the pool is GMU-driven (~6.8 GB, see header).
    "--max-model-len", "65536",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "2048",
    "--enable-chunked-prefill",
    "--enable-prefix-caching",
    "--gpu-memory-utilization", "0.85",
    # DSpark K8 speculator (RedHatAI/GLM-5.3-speculator.dspark, 2.3 GB, in
    # the shared HF cache; HF_HUB_OFFLINE=1 resolves it there).
    "--speculative-config", """
'{"model":"RedHatAI/GLM-5.3-speculator.dspark","num_speculative_tokens":8,"method":"dspark"}'""",
    "--language-model-only",
]

[env]

HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
TORCH_USE_RTLD_GLOBAL = "1"
TORCHINDUCTOR_COMPILE_THREADS = "1"
CUTE_DSL_ARCH = "sm_121a"
TORCH_CUDA_ARCH_LIST = "12.1a"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
VLLM_NO_USAGE_STATS = "1"
OMP_NUM_THREADS = "2"
# Compile-OOM guard (cold /root/.cache compiles behind 94 GB of resident
# weights; the JIT mounts below keep it warm after the first boot).
MAX_JOBS = "2"

# JIT caches persist per-node across serves (cold /root/.cache -> ~20 min
# recompile every serve). The host dirs are shared between lanes of the
# same arch — each cache self-namespaces by its engine version internally.
[[extra_mounts]]
host = "~/sm121-cache/glm53-jit-121f/b12x"
container = "/root/.cache/b12x"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/torchinductor"
container = "/root/.cache/torchinductor"

[[extra_mounts]]
host = "~/sm121-cache/jit-121f/vllm"
container = "/root/.cache/vllm"