launcher/recipes/glm53-flash-sparkring-2x/recipe.toml

# SparkRing shared image (ghcr.io/fujitsupolycom/sparkring, digest-pinned)
# — the sparkring TP2 profile `glm53-flash-spark-tp2-mtp3` from
# runtime/profiles/glm53-flash-spark-tp2 (profile.json), adapted to sm121:
# structural flags (--tp/--pp/--nnodes/--node-rank/--master-addr/--master-port/
# --host/--port) come from the launcher; everything else is the profile's.
# Model: local-inference-lab/GLM-5.3-Flash-NVFP4-Spark @ df116c4f (already in
# the shared HF cache; mounted at /models/target as the profile expects).
# Transport: the image ships the RoCEnante TP2 transport bundle; our DAC pair
# is on the f1 HCA pair (rocep1s0f1, roceP2p1s0f1), which the launcher
# injects via NCCL_IB_HCA from the 2nodes-dac [nccl] block.

engine = "vllm"
name = "GLM-5.3-Flash NVFP4-Spark (SparkRing shared img, TP2 RoCEnante profile)"
docker_image = "ghcr.io/fujitsupolycom/sparkring:glm53-shared-8b32ef2"  # prebuilt (no docker_image_dir); the 4774a7c5 memory-guard build (2026-09-11 re-run)
model = "local-inference-lab/GLM-5.3-Flash-NVFP4-Spark"
parent_model = "zai-org/GLM-5.3-Flash"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"

args = [
    "--decode-context-parallel-size", "1",
    "--mamba-cache-mode", "align",
    "--enable-prefix-caching",
    "--enable-chunked-prefill",
    "--dtype", "bfloat16",
    "--attention-backend", "B12X",
    "--moe-backend", "b12x",
    "--no-enable-flashinfer-autotune",
    "--reasoning-parser", "glm45",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--served-model-name", "GLM-5.3-Flash-NVFP4-Spark",
    "--max-model-len", "262144",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "8192",
    "--prefill-schedule-interval", "8",
    "--prefix-cache-retention-interval", "0",
    "--recurrent-checkpoint-policy", "aligned",
    "--block-size", "256",
    "--kv-cache-memory-bytes", "9395240960",
    "--kv-cache-dtype", "fp8",
    "--mm-processor-cache-gb", "0",
    "--mm-encoder-tp-mode", "data",
    "--linear-backend", "b12x",
    "--quantization", "modelopt_mixed",
    "--kda-prefill-backend", "b12x",
    "--gdn-decode-kernel", "b12x",
    "--generation-config", "auto",
    "--load-format", "b12x",
    "--model-loader-extra-config", "{\"allocation\":\"managed\"}",
    "--gpu-memory-utilization", "0.87",
    "--speculative-config", "{\"method\": \"mtp\", \"num_speculative_tokens\": 3, \"moe_backend\": \"humming\", \"attention_backend\": \"B12X\"}",
    "--compilation-config", "{\"mode\":0,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[1,2,4,8,12,16,20,24,28,32],\"max_cudagraph_capture_size\":32}",
    "--limit-mm-per-prompt", "{\"image\": 4, \"video\": 0}",
]


[env]
B12X_POLICY_MODE = "auto"
CUDA_DEVICE_MAX_CONNECTIONS = "32"
CUDA_DEVICE_ORDER = "PCI_BUS_ID"
CUTE_DSL_ARCH = "sm_121a"
FLASHINFER_CUDA_ARCH_LIST = "12.1f"
FLASHINFER_DISABLE_VERSION_CHECK = "1"
HF_HUB_OFFLINE = "1"
NCCL_CUMEM_ENABLE = "0"
NCCL_DEBUG = "WARN"
NCCL_IB_EXTENDED_IPV4_GIDS = "1"
NCCL_IB_GID_INDEX = "3"
NCCL_IB_PRESERVE_PCI_DOMAIN = "1"
NCCL_IB_ROUTE_DIAGNOSTICS = "1"
NCCL_IB_SUBNET_AWARE_ROUTING = "0"
NCCL_LOCAL_INFERENCE_PATH = "/opt/sparkring/nccl-pci/libnccl.so.2.30.7"
NCCL_NET = "IB"
NCCL_NET_PLUGIN = "none"
NCCL_P2P_LEVEL = "SYS"
NCCL_PROTO = "LL,LL128,Simple"
OMP_NUM_THREADS = "2"
PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
TORCH_USE_RTLD_GLOBAL = "1"
TRANSFORMERS_OFFLINE = "1"
VLLM_B12X_KDA_PREFILL_COALESCING = "1"
VLLM_B12X_MOE_FP4_FORCE_A16 = "0"
VLLM_ENABLE_PCIE_ALLREDUCE = "0"
VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH = "1"
VLLM_GLM53_KDA_GATE_SIDE_STREAM = "0"
VLLM_GLM53_MHC_PREFILL_SHARD = "1"
VLLM_GLM53_SPLIT_MAMBA_BLOCK_SIZE = "256"
VLLM_GLM53_SPLIT_TARGET_BLOCK_SIZE = "2048"
VLLM_LM_HEAD_A16 = "1"
VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS = "1"
VLLM_MTP_NVFP4_LM_HEAD = "1"
VLLM_MXFP8_LM_HEAD = "1"
VLLM_NO_USAGE_STATS = "1"
VLLM_NCCL_SO_PATH = "/opt/sparkring/nccl-pci/libnccl.so.2.30.7"
VLLM_PLUGINS = "b12x_loader"
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
VLLM_DISABLED_KERNELS = "MarlinFP8ScaledMMLinearKernel"
PYTHONDONTWRITEBYTECODE = "1"
# New image (8b32ef2/4774a7c5): b12x loader JIT-compiles its native module at
# first boot; the linker needs the CUDA lib path (the old image prebaked it).
# The build is cached in B12X_CACHE_DIR — persisted per node via the mount below.
CUDA_HOME = "/usr/local/cuda"
LIBRARY_PATH = "/usr/local/cuda/lib64"
B12X_CACHE_DIR = "/root/.cache/b12x"
LD_PRELOAD = "/opt/sparkring/nccl-pci/libnccl.so.2.30.7"
# The serve-with-warmup wrapper polls the API at 127.0.0.1:$PORT — the value
# resolves from the recipe's `port` field, so the two can't drift apart.
PORT = { from = "port" }
SERVED_MODEL_NAME = "GLM-5.3-Flash-NVFP4-Spark"
DFLASH_WARMUP = "1"
DFLASH_WARMUP_CONCURRENCIES = "1"
DFLASH_WARMUP_SHAPE_WORDS = "8"
DFLASH_WARMUP_MAX_TOKENS = "16"
DFLASH_WARMUP_TIMEOUT_SECONDS = "1800"
SPARKRING_LIVENESS_ENABLED = "1"
SPARKRING_LIVENESS_PORT = "8001"

# b12x JIT build cache (persists the first-boot native compile)
[[extra_mounts]]
host = "~/sm121-cache/jit-121f/sparkring-b12x"
container = "/root/.cache/b12x"