# GLM-5.3-Flash NVFP4-Spark — 4x DGX Spark, the lab's current Spark quant on # the LATEST jovian-judgement (r38 cut, 2026-09-19). # # Operator direction (2026-09-19): after the NAS disk death the interim # serving mode is 2x + 4x on local NVMe with the current lab quants. This # lane uses the latest lab engine (r38 = dev/jovian-judgement tip 8e1f1e5, # see profiles/r38.env) instead of the legacy r12 rebuild # (the r12 image is gone from the nodes; the r12 composition is a "phantom" # — the lab mainline keeps the split GDN naming the current quants carry, # and the r38 tip adds the B12X MXFP8 linear kernel that the r35-era # flashinfer-only path lacked for the small GDN GEMMs). # # Model: local-inference-lab/GLM-5.3-Flash-NVFP4-Spark (154.1 GiB, snapshot # a6082410, 09-17) — the "DSpark speculative embedded" checkpoint: the MTP # draft head lives in the model-mtp-*.safetensors shards (no separate # drafter download). NVFP4 routed experts + NVFP4 MTP expert layer. Split # GDN naming (q/k/g_a/g_b/f_a/f_b/b_proj) — loads on the JJ fork lineage # (r12..r38); the Aiden fork's fused in_proj_qkvbfg naming KeyErrors on it. # # Image: glm53-jovian-r38:local — the r27-lineage clone-at-pin build at the # r38 pins (normal in-build git, no pre-staged tarballs). Flags mirror the # proven r27-lineage GLM recipe (glm53-flash-r27-4x-dcp4, 160/176 bench) # minus the DCP4 pair: # * --attention-backend B12X — required: the sm12x auto-priority list never # offers the B12X sparse-MLA backend for GLM's head-512 # * no --kv-cache-dtype — let the fork pick its default (explicit fp8 / # fp8_ds_mla both failed auto-selection on this build lineage) # * --load-format instanttensor — baked into the r27-lineage images # * no --moe-backend — the b12x MoE routing is the fork default for this # quant (the FINAL BISECT "+moe b12x" gain is in the default path) # * speculative-config carries attention_backend B12X — the draft model's # vllm_config takes its attention backend from the SPECULATIVE config # * --language-model-only, no --enforce-eager (r27-lineage: graphs+MTP is # clean on the r17+ refs; the day-0 graph bug does not apply) # # PREREQUISITES: weights on each node's LOCAL hub (the NAS disk died # 2026-09-19): /home/kilork/hf-local, ~/.cache/huggingface symlink # re-pointed per node (snapshot a6082410). engine = "vllm" name = "GLM-5.3-Flash NVFP4-Spark (vLLM, 4x Spark, MTP-3, r38 tip)" docker_image = "jovian-judgement:r35-glm53" # r35 loads the Spark checkpoint (r38 pool-deadlock, see bug report) docker_image_dir = "jovian-judgement" build_profile = "r35" entrypoint = ["vllm", "serve"] model = "local-inference-lab/GLM-5.3-Flash-NVFP4-Spark" parent_model = "zai-org/GLM-5.3-Flash" required_nodes = 4 tensor_parallel_size = 4 pipeline_parallel_size = 1 distributed_backend = "mp" shm_size = "32gb" args = [ # Stable served name — quant-agnostic (shared across glm53 quants) "--served-model-name", "glm53", "--trust-remote-code", "--distributed-executor-backend", "mp", # GMU is ONLY a startup guardrail here (KV pool is pinned below): # 0.88 = 107.0 GiB, below any realistic free-memory dip on GB10's # 121.63 GiB (the 0.90 admission flake, #48140). "--gpu-memory-utilization", "0.88", # 16 GiB KV (4x NVFP4 has the headroom: 38.5 GiB weights/node at the # Spark quant vs ~108 GiB usable) "--kv-cache-memory-bytes", "17179869184", "--block-size", "256", "--max-model-len", "524288", "--max-num-seqs", "16", "--max-num-batched-tokens", "4096", "--enable-chunked-prefill", "--async-scheduling", "--enable-prefix-caching", # B12X sparse-MLA (required for head-512 on sm_12x) "--attention-backend", "B12X", # pure I/O loader (baked into the r27-lineage images; ~5.5 min for the # full checkpoint off shared NFS, ~1 min off local NVMe) "--load-format", "instanttensor", # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup "--kernel-config", """ '{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""", # Built-in MTP draft, 3 tokens (the r12-qualified "mtp:3" mode). The # draft model's vllm_config takes its attention backend from the # SPECULATIVE config, not the global flag — B12X mirrors the main model. "--speculative-config", """ '{"method":"mtp","num_speculative_tokens":3,"attention_backend":"B12X"}'""", "--language-model-only", "--tool-call-parser", "glm47", "--enable-auto-tool-choice", "--reasoning-parser", "glm45", ] [env] HF_HUB_OFFLINE = "1" # RoCEnante (b12x one-shot RoCE all-reduce/all-gather) — default on for the # r27 lanes since the 2026-09-08 A/B (4x: decode +31..+90%, quality 160 vs # 155). Needs the RoCE fabric; single-node serves skip it silently. VLLM_ENABLE_ROCE_ALLREDUCE = "1" # Route mxfp8 linears away from flashinfer cutlass (rejects small GDN GEMMs: min n/k=128) VLLM_DISABLED_KERNELS = "FlashInferCutlassMxfp8LinearKernel" # FlashInfer JIT compiles at first boot with N parallel nvcc jobs -> # OOM-killed on GB10 while weights+KV are resident; cap the ninja jobs. MAX_JOBS = "2" # NoPE-MLA rope-pad runtime gate (mod baked into the image lineage) VLLM_MLA_NOPE_PAD_ROPE = "1" VLLM_ENGINE_READY_TIMEOUT_S = "3600" # Upstream launch-script env (memory + compile knobs) PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True" VLLM_USE_AOT_COMPILE = "0" VLLM_USE_STANDALONE_COMPILE = "1" VLLM_USE_FLASHINFER_SAMPLER = "1" VLLM_USE_BREAKABLE_CUDAGRAPH = "0" TORCH_USE_RTLD_GLOBAL = "1" VLLM_NO_USAGE_STATS = "1" # Architecture (aarch64 / sm_121a DGX Spark) TORCH_CUDA_ARCH_LIST = "12.1a" FLASHINFER_CUDA_ARCH_LIST = "12.1f" CUTE_DSL_ARCH = "sm_121a" CMAKE_CUDA_ARCHITECTURES = "121" OMP_NUM_THREADS = "2" # JIT caches persist per-node across serves (cold /root/.cache per container # = ~20 min flashinfer/torch recompile every serve) [[extra_mounts]] host = "~/sm121-cache/glm53-jit-121f/b12x" container = "/root/.cache/b12x" [[extra_mounts]] host = "~/sm121-cache/jit-121f/flashinfer" container = "/root/.cache/flashinfer" [[extra_mounts]] host = "~/sm121-cache/jit-121f/torchinductor" container = "/root/.cache/torchinductor" [[extra_mounts]] host = "~/sm121-cache/jit-121f/vllm" container = "/root/.cache/vllm"