launcher/recipes/glm53-flash-aiden-2x/recipe.toml

# GLM-5.3-Flash on 2× DGX Spark — aidendle94 sparkrun image (TP=2).
#
# aiden's 4x guide (https://aidenle.com/recipes/glm-5-3-flash-4x-dgx-spark)
# translated to our 2-node ds4 launcher: same image (production-1.1 — SM90
# NoPE-MLA-to-capability-12 route + three functional fixes + DFlash2 overlay),
# same serve flags, TP2 instead of TP4.
#
# Image: aidendle94/sparkrun-vllm-glm53-gb10:production-1.1 (Docker Hub).
#   production-1.0 = MTP-only fallback; 1.1 ships the DFlash2 drafter overlay
#   (drafter package + aux-capture patch + KV-page LCM patch + backend-registry
#   patch). Grep the log for "B12X PCIe|DFlash" style markers per his guide.
#
# Weights: LibertAIDAI/GLM-5.3-Flash-NVFP4 (194 GB, 97 GiB/node at TP2; cached
# on both nodes). aiden: NVFP4 is ~23% faster than unsloth FP8 (which needs 4
# nodes at 82 GB/rank). Drafter: incoai/GLM-5.3-Flash-DFlash2 (2.34 GB, cached).
#   spec k MUST be 7 = block_size(2304? no: drafter block 8) − 1; acceptance
#   0.5–0.8, ~0.15 = broken aux capture (silent).
#
# Memory plan (2x): 97 GiB weights/node + KV pin 4 GiB + eager activations
# ≈ 103 GiB vs GMU 0.85 × 121.63 = 103.4 GiB. If a boot dies on free-memory,
# drop KV pin to 3 GiB or GMU to 0.83. KV pool ≈ 400-600K tokens — fine for
# agent-class benches, NOT for the 393K context the 4x profile serves.
#
# WARNING (from aiden's guide, untested on our 2x boot): the image bakes
# VLLM_PREFIX_CACHE_RETENTION_INTERVAL; his launcher unsets it via bash -lc.
# If the boot rejects the value in the KV coordinator, add a mods script that
# unsets it (ours cannot env-unset across the exec boundary).
#
# Boot markers: "Loading safetensors" (~2-4 min sequential from NVMe) →
# "GPU KV cache size" → "Application startup complete". DFlash2 acceptance
# 0.5-0.8 in SpecDecoding metrics.

engine = "vllm"
name = "GLM-5.3-Flash aiden image (vLLM, 2x Spark, DFlash2-7)"
docker_image = "aidendle94/sparkrun-vllm-glm53-gb10:production-1.1"
model = "LibertAIDAI/GLM-5.3-Flash-NVFP4"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
entrypoint = ["vllm", "serve"]

args = [
    "--served-model-name", "glm53",
    "--trust-remote-code",
    # functional parity (silent client gaps in aiden's guide)
    "--reasoning-parser", "glm45",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    # aiden ships thinking ON (his default template kwargs)
    "--default-chat-template-kwargs", """
'{"enable_thinking": true, "reasoning_effort": "high"}'""",
    # the SM90-NoPE + FA2 route: plain fp8 cache, real qk_rope_head_dim=0
    "--kv-cache-dtype", "fp8_e4m3",
    "--block-size", "2304",
    "--moe-backend", "marlin",
    "--max-model-len", "393216",
    "--max-num-batched-tokens", "8192",
    "--max-num-seqs", "16",
    "--enable-chunked-prefill",
    # deliberate per aiden (graphs neutral at TP4+collective-bound step;
    # saves 0.6 GiB + 28 s capture — and we need every GiB on 2 nodes)
    "--enforce-eager",
    "--gpu-memory-utilization", "0.85",
    "--kv-cache-memory-bytes", "4294967296",
    "--speculative-config", """
'{"method":"dflash","model":"incoai/GLM-5.3-Flash-DFlash2","num_speculative_tokens":7}'""",
    "--language-model-only",
]

[env]
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
TORCH_USE_RTLD_GLOBAL = "1"
TORCHINDUCTOR_COMPILE_THREADS = "1"
CUTE_DSL_ARCH = "sm_121a"
TORCH_CUDA_ARCH_LIST = "12.1a"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
VLLM_NO_USAGE_STATS = "1"
OMP_NUM_THREADS = "2"