launcher/recipes/glm53-flash-sparkring-4x/recipe.toml

# SparkRing shared image — TP4 SWITCHED profile
# (`/opt/sparkring/profiles/glm53-flash-spark-tp4-switched/profile.json`,
# "glm53-flash-spark-tp4-switched-mtp3", status: research-only — "not validated
# on switched hardware"). Plain NCCL (RoCEnante OFF), NCCL_ALGO=Ring,
# KV 24 GiB fp8, max-len 1M, MTP3 with draft TP4. Structural flags
# (--tp/--pp/--nnodes/--node-rank/--master-addr/--master-port/--host/--port)
# come from the launcher (master-port: 25000 vs their 29795 — cosmetic).
# Model: local-inference-lab/GLM-5.3-Flash-NVFP4-Spark @ df116c4f from the
# shared NFS cache, mounted read-only at /models/target as the profile expects.

engine = "vllm"
name = "GLM-5.3-Flash NVFP4-Spark (SparkRing shared img, TP4 switched profile)"
docker_image = "sparkring-shared:local"
docker_image_dir = "sparkring-shared"
model = "local-inference-lab/GLM-5.3-Flash-NVFP4-Spark"
parent_model = "zai-org/GLM-5.3-Flash"
required_nodes = 4
tensor_parallel_size = 4
pipeline_parallel_size = 1
distributed_backend = "mp"

args = [
    "--decode-context-parallel-size", "1",
    "--cp-kv-cache-interleave-size", "1",
    "--disable-custom-all-reduce",
    "--mamba-cache-mode", "align",
    "--limit-mm-per-prompt", "{\"image\":4,\"video\":1}",
    "--enable-chunked-prefill",
    "--dtype", "bfloat16",
    "--kv-cache-dtype", "fp8",
    "--quantization", "modelopt_mixed",
    "--attention-backend", "B12X",
    "--block-size", "512",
    "--mamba-block-size", "512",
    "--recurrent-checkpoint-policy", "aligned",
    "--prefix-cache-retention-interval", "0",
    "--moe-backend", "b12x",
    "--linear-backend", "b12x",
    "--no-enable-flashinfer-autotune",
    "--load-format", "fastsafetensors",
    "--enable-auto-tool-choice",
    "--tool-call-parser", "glm47",
    "--reasoning-parser", "glm45",
    "--kda-prefill-backend", "b12x",
    "--gpu-memory-utilization", "0.80",
    "--kv-cache-memory-bytes", "25769803776",
    "--max-model-len", "1048576",
    "--max-num-seqs", "16",
    "--max-num-batched-tokens", "8192",
    "--prefill-schedule-interval", "2",
    "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"draft_tensor_parallel_size\":4,\"kv_cache_dtype\":\"auto\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\",\"draft_load_config\":{\"load_format\":\"safetensors\"},\"attention_backend\":\"B12X\"}",
    "--compilation-config", "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],\"custom_ops\":[\"all\"],\"pass_config\":{\"fuse_allreduce_rms\":false}}",
    "--max-cudagraph-capture-size", "64",
    "--async-scheduling",
    "--enable-prefix-caching",
    "--cudagraph-metrics",
    "--enable-prompt-tokens-details",
    "--served-model-name", "glm-5.3-flash-spark",
]


[env]
PYTHONDONTWRITEBYTECODE = "1"
VLLM_SPARK_SHARED_CAPTURE_STREAM = "0"
SPARK_TP4_HEALTH_GATE = "0"
SPARK_CUDAGRAPH_REPLAY_TIMING = "0"
SPARK_TP4_DCP_COLLECTIVE_AUDIT = "0"
SPARK_GDN_PREFILL_CHECKPOINTS = "0"
SPARK_GDN_CONTINUATION_CHECKPOINTS = "0"
SPARK_MHC_PREFILL_SHARD = "0"
SPARK_MTP_CAPTURE_TAIL_MAX_REQS = "0"
VLLM_ENABLE_PCIE_ALLREDUCE = "0"
VLLM_ENABLE_ROCE_ALLREDUCE = "0"
VLLM_ALLREDUCE_USE_FLASHINFER = "0"
VLLM_ALLREDUCE_USE_SYMM_MEM = "0"
VLLM_USE_NCCL_SYMM_MEM = "0"
NCCL_SWITCHLESS_RING_ONLY = "0"
NCCL_IB_SUBNET_AWARE_ROUTING = "0"
NCCL_IB_EXTENDED_IPV4_GIDS = "1"
NCCL_IB_PRESERVE_PCI_DOMAIN = "1"
NCCL_IB_ROUTE_DIAGNOSTICS = "0"
NCCL_NET = "IB"
NCCL_NET_PLUGIN = "none"
NCCL_IB_DISABLE = "0"
NCCL_ALGO = "Ring"
NCCL_PROTO = "LL,LL128,Simple"
NCCL_DEBUG = "WARN"
NCCL_CUMEM_ENABLE = "0"
NCCL_IB_MERGE_NICS = "0"
NCCL_CROSS_NIC = "1"
VLLM_NCCL_SO_PATH = "/opt/sparkring/nccl-pci/libnccl.so.2.30.7"
LD_PRELOAD = "/opt/sparkring/nccl-pci/libnccl.so.2.30.7"
VLLM_USE_V2_MODEL_RUNNER = "1"
VLLM_B12X_KDA_PREFILL_COALESCING = "1"
VLLM_GLM53_MHC_PREFILL_SHARD = "1"
VLLM_GLM53_MHC_PREFILL_DIAGNOSTICS = "1"
VLLM_GLM53_KDA_GATE_SIDE_STREAM = "1"
VLLM_DCP_TOPK_OWNER_MERGE = "0"
VLLM_DCP_OWNER_FUSED_ENDPOINTS = "0"
VLLM_DCP_COMPACT_INDEX_CACHE_OWNER = "0"
VLLM_DCP_COMPACT_INDEX_TENSOR_VOTE = "0"
VLLM_DCP_COMPACT_INDEX_LOCAL_WIDTHS = "0"
VLLM_DCP_COMPACT_INDEX_PROFILE = "0"
SPARKCACHE_ENABLED = "0"
SPARKCACHE_ASYNC_PAGE_CAPTURE = "0"
VLLM_GLM53_SPLIT_TARGET_BLOCK_SIZE = "512"
VLLM_GLM53_SPLIT_MAMBA_BLOCK_SIZE = "512"
VLLM_B12X_MLA_CKV_GATHER = "1"
VLLM_MTP_NVFP4_LM_HEAD = "1"
VLLM_LM_HEAD_A16 = "1"
VLLM_MXFP8_LM_HEAD = "0"
VLLM_B12X_MOE_FP4_FORCE_A16 = "0"
VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH = "1"
VLLM_FASTSAFETENSORS_QUEUE_SIZE = "1"
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
VLLM_NO_USAGE_STATS = "1"
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"
# The serve-with-warmup wrapper polls the API at 127.0.0.1:$PORT — the value
# resolves from the recipe's `port` field, so the two can't drift apart.
PORT = { from = "port" }
SERVED_MODEL_NAME = "glm-5.3-flash-spark"
DFLASH_WARMUP = "1"
DFLASH_WARMUP_CONCURRENCIES = "1"
DFLASH_WARMUP_SHAPE_WORDS = "8"
DFLASH_WARMUP_MAX_TOKENS = "16"
DFLASH_WARMUP_TIMEOUT_SECONDS = "1800"
SPARKRING_WARMUP_TEMPERATURE = "1"
SPARKRING_LIVENESS_ENABLED = "1"
SPARKRING_LIVENESS_PORT = "8001"