launcher/recipes/glm53-flash/recipe.toml

# GLM-5.3-Flash (SGLang day-0) — 4× DGX Spark / GB10 (sm_121), TP=4.
#
# 320B-A18B native FP8, hybrid sparse + linear attention + mHC. First
# natively multimodal GLM-5 (text + vision). No MTP/DFlash draft shipped —
# plain decode until a drafter lands. Model:
# https://huggingface.co/zai-org/GLM-5.3-Flash
#
# Engine: SGLang day-0 — no vLLM has glm5_next: local-inference-lab fork
# idle since Aug 11; aidendle94 / eugr images all pre-release.
#
# Image: lmsysorg/sglang:glm-5.3-flash — multi-arch incl. arm64 (digest
# sha256:989e2fff0…; re-verify at serve time). Day-0 = boots, not tuned.
#
# Memory: 305.8 GiB → 76.5 GiB/node on 4× Spark; ~43 GiB/node left for KV.
# Sparse+linear attention keeps KV small, so long context should be feasible.
#
# PREREQUISITES:
#   Weights cached on every node (~/.cache/huggingface → NAS, 328 GB):
#       huggingface-cli download zai-org/GLM-5.3-Flash
#
# REFERENCE:
#   https://forums.developer.nvidia.com/t/glm-5-3-flash-320b-total-parameters-18b-active/381350

engine = "sglang"
name = "GLM-5.3-Flash (SGLang day-0)"
docker_image = "lmsysorg/sglang:glm-5.3-flash"
model = "zai-org/GLM-5.3-Flash"
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "64gb"

entrypoint = ["python3", "-m", "sglang.launch_server"]

args = [
    # Memory — weights ~76.5 GiB of 119.6 GiB/node; KV gets the rest.
    "--mem-fraction-static", "0.87",
    "--max-running-requests", "10",
    # Context — 262K start; model card claims up to 1M, scale after measuring.
    "--max-model-len", "262144",
    # Attention / KV
    "--attention-backend", "flashinfer",
    "--kv-cache-dtype", "fp8_e4m3",
    # Chunked prefill
    "--chunked-prefill-size", "8192",
    # Compile + decode graphs
    "--enable-torch-compile", "--torch-compile-max-bs", "4",
    "--cuda-graph-max-bs-decode", "4",
    "--num-continuous-decode-steps", "2",
    # Tokenizer / reasoning / tool calling (GLM family; parser names carry
    # over from the vLLM glm52 recipes — verify they exist in this SGLang)
    "--reasoning-parser", "glm45",
    "--tool-call-parser", "glm47",
    # Observability
    "--enable-metrics",
    "--enable-cache-report",
]

[env]
# Offline — weights must be cached on each node
HF_HUB_OFFLINE = "1"

# Architecture
TORCH_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
OMP_NUM_THREADS = "4"