launcher/recipes/glm53-full-nvfp4-spark-aiden/recipe.toml

# GLM-5.3-Flash on 4× DGX Spark — aidendle94 sparkrun image (TP=4). DEFERRED.
#
# aiden's production 4-node profile, verbatim translation of
# https://aidenle.com/recipes/glm-5-3-flash-4x-dgx-spark into our 4x launcher
# shape. Reference numbers (his, single stream, idle cluster, thinking off):
#   ~84 tok/s decode (FP8, DFlash2 k=7) — NVFP4 +23% (110.7 structured);
#   prefill ~2,100 cold; TTFT 4 s @ 8K → 40 s @ 94K; KV 2.25M tokens at 16 GiB
#   pin; 393K context verified. Decode flat with context (34/45 KDA layers +
#   11 sparse-MLA @ index_topk 2048).
#
# DFlash2: acceptance 0.5–0.8; k=7 (drafter block 8 − 1 target token); k=6
# wins prose, loses structured. Native MTP fallback (production-1.0 image or
# spec omitted) optimum k=5. ~15 min full relaunch on FP8 (963 s measured).
# FP8-MLA + DFlash2 combination is STRUCTURALLY broken upstream (three trees
# fail differently) — hence plain fp8_e4m3 KV.
#
# NOT READY TO DEPLOY: needs a 4-node cluster (ds4 + 2nodes-dac freed) and the
# unsloth/GLM-5.3-Flash-FP8 checkpoint (328 GB — background download running
# on dgx3, /tmp/dl_fp8.log; ~340 GB free per node required for weights + Triton
# cache). Fabric: RoCE dual-HCA NCCL profile comes from the launcher (config
# [nccl.*] / GID auto-detect) — his NCCL_* env block intentionally omitted.
# NOTE: his launcher unsets VLLM_PREFIX_CACHE_RETENTION_INTERVAL via bash -lc
# before exec; if our boot rejects it in the KV coordinator, add a mods script.

engine = "vllm"
name = "GLM-5.3 (full) NVFP4-Spark, aiden image (vLLM, 4x Spark)"
docker_image = "aidendle94/sparkrun-vllm-glm53-gb10:production-1.2"
model = "local-inference-lab/GLM-5.3-NVFP4-Spark"
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
# image ships ENTRYPOINT ["vllm","serve"]; our launcher passes the recipe
# entrypoint as CMD words after the image, which would double it. Empty clears.
entrypoint = []

args = [
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--reasoning-parser", "glm45",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--default-chat-template-kwargs", """
'{"enable_thinking": true, "reasoning_effort": "high"}'""",
    "--kv-cache-dtype", "fp8_e4m3",
    "--block-size", "2304",
    "--moe-backend", "marlin",
    # full-model weights ~101.5 GiB/node (406G/4): 4 GiB KV pin, ~79K-token
    # pool (same sizing the lab image computed); 393K context won't fit
    "--max-model-len", "79616",
    "--max-num-batched-tokens", "8192",
    "--max-num-seqs", "16",
    "--enable-chunked-prefill",
    "--enforce-eager",
    "--gpu-memory-utilization", "0.85",
    "--kv-cache-memory-bytes", "4294967296",
    # no spec: the DFlash2 drafter is the Flash model's; full-model plain run
    "--language-model-only",
    # PARKED 2026-09-04 (user call: stop full-model experiments): the
    # MIXED_PRECISION full model loads clean on this image (85/85 shards),
    # but dgx2.lan went down TWICE mid-full-model-boot (once during the first
    # compile-OOM episode, once during the MAX_JOBS=2 retry's init phase —
    # whether node instability under this workload or coincidence is open).
    # Resume when: dgx2 proves stable under this load, and/or the JIT cache
    # is pre-built (extra_mounts below persist it) / the image ships
    # prebuilt sm121a mxfp8 gemm cubins.
    # image requests need the mm template shipped/in-wallpaper on aiden's rigs:
    # "--chat-template", "<path-to>/chat_template_mm.jinja",
]

[env]
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
TORCH_USE_RTLD_GLOBAL = "1"
TORCHINDUCTOR_COMPILE_THREADS = "1"
CUTE_DSL_ARCH = "sm_121a"
TORCH_CUDA_ARCH_LIST = "12.1a"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
VLLM_NO_USAGE_STATS = "1"
OMP_NUM_THREADS = "2"
# Compile-OOM fix (2026-09-04): first boot of the MIXED_PRECISION checkpoint
# JIT-compiles mxfp8_gemm_cutlass_sm120 (12 nvcc/cicc jobs, ~3 GB RSS each)
# while 101.5 GiB of weights sit in unified memory -> cicc OOM-killed.
# flashinfer's run_ninja reads MAX_JOBS and passes -j N: cap at 2.
MAX_JOBS = "2"

# Persistent JIT caches (containers run with --rm; without these every boot
# re-runs the ~10 min nvcc compile of the mxfp8 gemm module)
[[extra_mounts]]
host = "~/.cache/glm53-full-flashinfer"
container = "/root/.cache/flashinfer"

[[extra_mounts]]
host = "~/.cache/glm53-full-triton"
container = "/root/.triton"