|
|
@@ -0,0 +1,327 @@
|
|
|
+# GLM-5.3 (FULL, non-Flash) MXFP4 — RedHatAI official quant — 4× DGX Spark, TP=4.
|
|
|
+#
|
|
|
+# REPLACES the failed full-model variant: `local-inference-lab/GLM-5.3-NVFP4-Spark`
|
|
|
+# (406 GiB → 101.5 GiB weights/node → OOM-proven on 4 nodes 2026-09-08/09: the
|
|
|
+# engine crash-restarted and began a SECOND full load while the first copy was
|
|
|
+# still resident → 2×101.5 > 121.63 GiB → all four nodes OOM-thrashed, sshd down
|
|
|
+# on 3 of 4, HA power-cycle required). That checkpoint is abandoned (recipes
|
|
|
+# glm53-full-nvfp4-spark-lab / -aiden / glm53-spark-nvfp4-4x deleted).
|
|
|
+#
|
|
|
+# THIS checkpoint: RedHatAI/GLM-5.3-MXFP4 — 403.4 GB (375.6 GiB) → 100.85 GB
|
|
|
+# (93.9 GiB) weights/node on 4x. ~7.6 GiB/node more headroom than the NVFP4
|
|
|
+# attempt, which is what makes 4x plausible at all. LLM-Compressor
|
|
|
+# (compressed-tensors) quant: "mxfp4-pack-quantized", weights FP4-e2m1 group-32
|
|
|
+# with E8M0 scales, dynamic per-group FP4 activations; mlp.gate / lm_head /
|
|
|
+# embed_tokens / eh_proj / indexer.{wk,weights_proj} stay bf16.
|
|
|
+# Model: https://huggingface.co/RedHatAI/GLM-5.3-MXFP4
|
|
|
+# Parent: zai-org/GLM-5.3 (FP8). Card: gsm8k 96.36 vs 97.60 (FP8) = 98.7%
|
|
|
+# recovery; ifeval 89.25 vs 93.53 = 95.4%.
|
|
|
+# Card's serving recipe (4× B200): --kv-cache-dtype fp8_e4m3 --tp 4
|
|
|
+# --reasoning-parser glm45 --speculative-config '{"method":"mtp","num_speculative_tokens":5}'
|
|
|
+#
|
|
|
+# GEOMETRY (config.json, verified 2026-09-13): arch GlmMoeDsaForCausalLM
|
|
|
+# (model_type glm_moe_dsa), 78 layers, hidden 6144, 256 routed experts + 1
|
|
|
+# shared, top-8, moe_inter 2048, MLA (kv_lora 512 / q_lora 2048 / qk_nope 192 /
|
|
|
+# qk_rope 64 / v 256), DSA lightning indexer (index_topk 2048), 1M native ctx,
|
|
|
+# first_k_dense_replace 3, num_nextn_predict_layers 1 (MTP head IN the
|
|
|
+# checkpoint), 141 shards, no separate drafter.
|
|
|
+# ~740B total params / ~40B active. The Flash variant we serve in prod is
|
|
|
+# 320B/18B (45 layers, hidden 4096, 288 experts) — this is a different model.
|
|
|
+#
|
|
|
+# ── BASE STACK: our own jovian r35 build (docker-images/glm53-jovian-r27 +
|
|
|
+# build_profile glm53-r35 → image glm53-jovian-r35:local) ────────────────
|
|
|
+# Selected 2026-09-13. Evidence (all verified by read-only inspection of the
|
|
|
+# images on dgx1):
|
|
|
+# * registers the full-model arch: registry.py →
|
|
|
+# "GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", ...) — the NEW
|
|
|
+# `deepseek_v32` implementation, which is "MLA + a lightning indexer that
|
|
|
+# selects the top-k tokens for a sparse MLA" and dispatches to an
|
|
|
+# NVIDIA-specific path (deepseek_v32/nvidia/model.py:
|
|
|
+# DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM). That is exactly the
|
|
|
+# GB10/sm_121a target; the lab fork and aiden's image register the same
|
|
|
+# class from the older `deepseek_v2` module instead.
|
|
|
+# * ships the MXFP4 scheme: model_executor/layers/quantization/
|
|
|
+# compressed_tensors/schemes/compressed_tensors_w4a4_mxfp4.py
|
|
|
+# ("mxfp4-pack-quantized") → the RedHat checkpoint loads natively.
|
|
|
+# * it is OUR reproducible clone-at-pin build (vLLM de982a50 + b12x 98086604,
|
|
|
+# r35 lock 2026-09-11 = newest published lab composition), all gates green,
|
|
|
+# patches ours to write — and it is the SAME image our GLM-5.3-Flash 4x
|
|
|
+# prod runs, so the A/B vs prod is clean and the JIT caches are warm.
|
|
|
+# * the b12x MoE/attention backends are the GB10-tuned ones our GLM prod
|
|
|
+# uses (--moe-backend b12x / --attention-backend B12X).
|
|
|
+# Rejected: `glm53-flash-nvfp4-lab:local` and aiden's
|
|
|
+# `aidendle94/sparkrun-vllm-glm53-gb10:production-1.2` — both older vLLM
|
|
|
+# (0.1.dev20051+g487ecf187, `deepseek_v2` GLM class); the lab fork needed two
|
|
|
+# SM120 mla_attention hotfix mods to boot the full model at all
|
|
|
+# (masked_mha_available AttributeError + forward_mha NotImplementedError, both
|
|
|
+# never re-tested — the lane parked). Keep them as a fallback reference only:
|
|
|
+# if the r35 deepseek_v32 path trips on the DSA/indexer kernels, port those
|
|
|
+# two sed mods (documented in the deleted glm53-full-nvfp4-spark-lab history)
|
|
|
+# onto the lab image and re-point docker_image/docker_image_dir.
|
|
|
+#
|
|
|
+# ── MEMORY PLAN (the binding constraint — read before the first boot) ──────
|
|
|
+# 121.63 GiB unified/node. Idle free: 116-117 GiB.
|
|
|
+# weights 93.9 GiB (403.4 GB / 4) ← ~101.5 was fatal
|
|
|
+# vLLM procs+docker ~12 GiB
|
|
|
+# KV pin 8 GiB (--kv-cache-memory-bytes)
|
|
|
+# activations + CUDA graphs ~4 GiB
|
|
|
+# ⇒ ~118 GiB — tight but inside the node. GMU 0.85 is the ADMISSION
|
|
|
+# guardrail only (free ≥ 0.85×121.63 = 103.4 GiB; 0.88 = 107.0 GiB fails
|
|
|
+# once the ~12 GiB of starting processes are counted, the 2026-09-08
|
|
|
+# #48140 class) — the KV pool is pinned, so GMU does not change capacity.
|
|
|
+#
|
|
|
+# WHY DCP4: the full model's attention KV is NOT the Flash lane's compact
|
|
|
+# MLA cache. The parked run measured ~54 KB/token (4 GiB ↔ ~79.6K tokens;
|
|
|
+# "524288 would need 26.34 GiB") vs the Flash lane's ~2 KB/token at 512K.
|
|
|
+# Decode-context-parallel-4 shards the KV across the 4 ranks, so the 8 GiB
|
|
|
+# pin buys 32 GiB aggregate → a single 524288-token request needs ~27 GiB
|
|
|
+# aggregate (≈6.8 GiB/rank) and just fits. Without DCP a 512K request is
|
|
|
+# impossible behind these weights. CAVEAT: DCP4 is qualified in r35 for the
|
|
|
+# Flash model (CP1/CP4 shapes); it is UNVERIFIED for the full model — if the
|
|
|
+# boot rejects it (the GLM C4 page-stride / interleave-multiple-of-4 gates
|
|
|
+# that block DFlash2 on 2x), drop --decode-context-parallel-size +
|
|
|
+# --cp-kv-cache-interleave-size and lower --max-model-len to fit ~10 GiB of
|
|
|
+# aggregate KV (~180K tokens).
|
|
|
+#
|
|
|
+# --max-model-len 524288 = the operator's floor (128K/131K profiles are not
|
|
|
+# usable for us). --max-num-seqs 8 keeps the pool from being shredded by
|
|
|
+# short requests; the admission scheduler handles the rest.
|
|
|
+#
|
|
|
+# ── BOOT PLAN / HAZARD ─────────────────────────────────────────────────────
|
|
|
+# The 2026-09-08 disaster was NOT a bad weight footprint per se — it was a
|
|
|
+# post-load crash + auto-restart that started a SECOND full load on top of
|
|
|
+# the first. Watch the first boot: if the engine logs a second
|
|
|
+# "Loading safetensors" pass, kill immediately (`sm121 stop`), do not wait.
|
|
|
+# Always `sm121 serve --keep` for the first/unknown boot (default is --rm,
|
|
|
+# which wipes the crash logs).
|
|
|
+# Ladder if the boot dies: (1) disable DCP4, max-model-len 131072, KV pin
|
|
|
+# 4 GiB (prove stack+checkpoint end-to-end, the parked lane's approach);
|
|
|
+# (2) then re-add DCP4 + 512K; (3) only then add spec.
|
|
|
+# Boot markers to record: "Model loading took X GiB", the KV line
|
|
|
+# ("GPU KV cache size: N tokens, Maximum concurrency for 524288 tokens per
|
|
|
+# request: Mx"), "Application startup complete". MAX_JOBS=2 below is the
|
|
|
+# compile-OOM guard (12 parallel nvcc jobs × ~3 GiB behind 94 GiB of weights
|
|
|
+# = cicc OOM-killed, 2026-09-04 lesson).
|
|
|
+#
|
|
|
+# ── MXFP4 vs NVFP4: what changed, and what is already in our tree ──────────
|
|
|
+# The full-model NVFP4 lane is retired (see above), so this checkpoint is the
|
|
|
+# lab's MXFP4 family: routed-expert weights FP4-e2m1 with power-of-two (E8M0)
|
|
|
+# group-32 scales — the same shape as their own GLM-5.1 MXFP4 repack, which is
|
|
|
+# why their GLM-5.1 runbook (models/glm-5.1-mxfp4.md) is the reference for
|
|
|
+# this lane even though GLM-5.3 moved on: the MXFP4 ingestion path, the
|
|
|
+# FlashInfer Cutlass MXFP4xMXFP8 MoE route, the standard-SwiGLU rule and the
|
|
|
+# "do not set B12X_MOE_FORCE_A16" warning all still apply. That runbook also
|
|
|
+# unsets empty NCCL graph variables before launch
|
|
|
+# (NCCL_GRAPH_FILE / NCCL_GRAPH_DUMP_FILE / VLLM_B12X_MLA_EXTEND_MAX_CHUNKS)
|
|
|
+# — unset them in the shell if a previous lane exported them empty; an empty
|
|
|
+# value is not the same as an unset one to NCCL.
|
|
|
+#
|
|
|
+# ── SPECULATIVE DECODING ───────────────────────────────────────────────────
|
|
|
+# NOT enabled on the first boot: the checkpoint's MTP head is real
|
|
|
+# (num_nextn_predict_layers=1) and the card suggests mtp k=5, but spec adds
|
|
|
+# draft KV + graph shapes onto a memory budget that has no room for
|
|
|
+# surprises, and the full-model MTP path is unproven on this stack. After a
|
|
|
+# clean boot + smoke + bench, add:
|
|
|
+# "--speculative-config", """'{"method":"mtp","num_speculative_tokens":3,"attention_backend":"B12X"}'"""
|
|
|
+# (k=3 = our Flash prod's verified depth; try 5 after k=3 is clean).
|
|
|
+#
|
|
|
+# PREREQUISITE: weights in the shared NAS HF cache on all 4 nodes
|
|
|
+# (models--RedHatAI--GLM-5.3-MXFP4, 403.4 GB, 141 shards) —
|
|
|
+# `hf download RedHatAI/GLM-5.3-MXFP4`; HF_HUB_OFFLINE=1 below makes a
|
|
|
+# missing shard a hard load failure. NAS had 16 TB free at 2026-09-13.
|
|
|
+#
|
|
|
+# A/B TARGET: our GLM-5.3-Flash r35 MTP3 4x prod (dev65/seed 42, hardmode):
|
|
|
+# 163/176 (93), 1,554 pp / 49.6 tg / 1,544 ms TTFT @d0 c1, KV 8.06M @512K.
|
|
|
+# This lane is a capability/quality lane first (does the full model fit and
|
|
|
+# hold 512K on 4 nodes), throughput second.
|
|
|
+
|
|
|
+engine = "vllm"
|
|
|
+name = "GLM-5.3 (full) MXFP4 (r37 img, 4x, DSpark K8 @131K)"
|
|
|
+# Image: the r37 DS4.1 build (2026-09-14, night shift) - the r35 tree's DCP4
|
|
|
+# sparse-INDEXER path asserts interleave==1 and then crashes in the DCP sparse
|
|
|
+# decode reshape during graph capture ("shape '[16, 16, 512]' is invalid for
|
|
|
+# input of size 524288" - the x4 DCP factor in the attention.py _sparse_
|
|
|
+# decode). r37 is the newer DS4.1 lock whose DCP sparse-MLA path is the
|
|
|
+# qualified one (same tree family as our DCP2/DFlash2 work); the GlmMoeDsa
|
|
|
+# arch + MTP + b12x ship in the r37 image (verified 2026-09-13).
|
|
|
+docker_image = "jovian-r37-ds41:local"
|
|
|
+docker_image_dir = "glm53-jovian-r27"
|
|
|
+# NOTE: build_profile must stay a TOP-LEVEL key — a bare key after
|
|
|
+# [[extra_mounts]] would parse into the last mount table and be silently
|
|
|
+# ignored (2026-09-08).
|
|
|
+build_profile = "jovian-r37-ds41"
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+model = "RedHatAI/GLM-5.3-MXFP4"
|
|
|
+parent_model = "zai-org/GLM-5.3"
|
|
|
+required_nodes = 4
|
|
|
+tensor_parallel_size = 4
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+shm_size = "32gb"
|
|
|
+
|
|
|
+args = [
|
|
|
+ # R31/r35 delta (their qualified locks): adaptive prefill/decode
|
|
|
+ # compute-time sharing with a fixed 40% prefill floor.
|
|
|
+ "--prefill-compute-share", "0.4",
|
|
|
+ # Stable served name — quant-agnostic (shared across glm53 lanes)
|
|
|
+ "--served-model-name", "glm53",
|
|
|
+ "--trust-remote-code",
|
|
|
+ "--distributed-executor-backend", "mp",
|
|
|
+ # Admission guardrail only (KV is pinned below): 0.85×121.63 = 103.4 GiB.
|
|
|
+ # 0.88 (107.0) is the Flash lane's value and is too tight here — the full
|
|
|
+ # model's weights leave ~24 GiB before the engine's own ~12 GiB.
|
|
|
+ "--gpu-memory-utilization", "0.85",
|
|
|
+ # 9 GiB/rank pin (DCP1, no context parallel). The no-spec 131K anchor ran
|
|
|
+ # at 8 GiB (pool 159,232 tokens); +1 GiB covers the draft model's own
|
|
|
+ # KV group: the engine needs 8.08 GiB for one 131072-token request + the
|
|
|
+ # drafter, and the 8.0 GiB pin was 80 MB short (boot 5, 2026-09-18).
|
|
|
+ "--kv-cache-memory-bytes", "9663676416",
|
|
|
+ # NO DCP here: the DCP>1 sparse-decode path (deepseek_v32/attention.py)
|
|
|
+ # crashes on both r35 and r37 trees (the 512K-on-4x blocker). 512K on
|
|
|
+ # 4x is out; long context is the 8x DCP1 story (P2).
|
|
|
+ # No --kv-cache-dtype: let the r35 fork pick its native default for
|
|
|
+ # GlmMoeDsa attention (the Flash lane's explicit fp8/fp8_ds_mla spellings
|
|
|
+ # both failed auto-selection on this build, and those were Flash-MLA
|
|
|
+ # knobs). The card's --kv-cache-dtype fp8_e4m3 is the fallback spelling
|
|
|
+ # if the pool needs to shrink.
|
|
|
+ # DSpark drafter fix (boot 1, 2026-09-18): the target's B12X sparse-MLA
|
|
|
+ # layer normalizes the shared cache config to fp8_ds_mla (mla_attention
|
|
|
+ # _canonicalize_sparse_mla_kv_cache_dtype rewrites cache_config.cache
|
|
|
+ # dtype in place); the draft model's non-MLA DFlash attention (head 64)
|
|
|
+ # is built after that and the r37 selector finds no backend for
|
|
|
+ # (head 64, bf16, fp8_ds_mla, non-MLA) → ValueError at init. The
|
|
|
+ # drafter's DFlash layers are sliding-window (2048), the target's DSA
|
|
|
+ # layers are not — skipping KV quant for the sliding_window pattern
|
|
|
+ # gives the draft layers auto (bf16) KV while the target keeps
|
|
|
+ # fp8_ds_mla. Explicit --kv-cache-dtype is NOT the lever: the MLA
|
|
|
+ # normalization overrode fp8_e4m3 back to fp8_ds_mla (boot 2).
|
|
|
+ "--kv-cache-dtype-skip-layers", "sliding_window",
|
|
|
+ "--attention-backend", "B12X",
|
|
|
+ # MXFP4 kernel routing (measured on the first boot, 2026-09-13 ~23:10 UTC):
|
|
|
+ # - MoE: the compressed-tensors W4A4-MXFP4 scheme picks MarlinExperts on
|
|
|
+ # its own (A16 dequant GEMM - correct on GB10). The --moe-backend flag
|
|
|
+ # does not reach that scheme (verified inert: it still logged "Using
|
|
|
+ # MarlinExperts for MXFP4 MoE" with the flag set), so we do not set it.
|
|
|
+ # - LINEAR: the default priority [FlashInfer, Marlin, Humming, B12x]
|
|
|
+ # starts with the FlashInfer kernel, whose is_supported() wrongly
|
|
|
+ # accepts capability 121 (the numeric has_device_capability(100) check
|
|
|
+ # passes on GB10) while the cute-dsl mm_fp4 GEMM it calls rejects it at
|
|
|
+ # runtime with:
|
|
|
+ # BackendSupportedError: mm_fp4 does not support backend 'cute-dsl'
|
|
|
+ # with capability 121
|
|
|
+ # (the first probe boot died there, kv_b_proj dequant in mla_attention.py).
|
|
|
+ # --linear-backend b12x forces the B12xMxFp4LinearKernel - "MXFP4 linear
|
|
|
+ # through the native B12X SM120 dense GEMM" - the right path on GB10,
|
|
|
+ # verified per component: is_device_capability_family(120)=True, b12x
|
|
|
+ # blockscaled.is_supported()=True, and the cute-dsl input quantize
|
|
|
+ # (flashinfer_mxfp4_quantize) works on capability 121; only the cute-dsl
|
|
|
+ # mm_fp4 GEMM is unsupported there. Boot log to look for:
|
|
|
+ # "Using B12xMxFp4LinearKernel for MXFP4 GEMM"
|
|
|
+ # (The lab's GLM-5.1 runbook describes the same kernel family for the
|
|
|
+ # quant_method=mxfp4 ingestion path - this checkpoint goes through the
|
|
|
+ # compressed-tensors path, which is why --moe-backend was the wrong lever.)
|
|
|
+ # First attempt: b12x (the native SM120 GEMM) - correct selection, but the
|
|
|
+ # unconditional b12x warmup (b12x_warmup.py, no config gate) crashes on
|
|
|
+ # this checkpoint's layer shapes during kernel compile:
|
|
|
+ # RuntimeError: Expected source and destination tensors to have the same
|
|
|
+ # rank, but got 3 and 4 (b12x/gemm/blockscaled/_linear.py dense_gemm,
|
|
|
+ # via _apply_b12x_mxfp4_linear on a synthetic zero source)
|
|
|
+ # so the mxfp4 LINEAR layers run on Marlin (weight-only A16 dequant GEMM -
|
|
|
+ # the same family as the MarlinExperts the MoE scheme picked). Slower than
|
|
|
+ # native FP4 for the attention projections, but the MoE experts (the
|
|
|
+ # dominant cost) are Marlin either way. The b12x warmup rank bug is
|
|
|
+ # filed-worthy for the lab (RedHatAI GLM-5.3-MXFP4 on GB10/sm_121a).
|
|
|
+ "--linear-backend", "marlin",
|
|
|
+ # pure I/O loader (kept across every GLM config)
|
|
|
+ "--load-format", "instanttensor",
|
|
|
+ "--block-size", "256",
|
|
|
+ "--max-model-len", "131072",
|
|
|
+ "--max-num-seqs", "8",
|
|
|
+ "--max-num-batched-tokens", "4096",
|
|
|
+ "--enable-chunked-prefill",
|
|
|
+ "--async-scheduling",
|
|
|
+ "--enable-prefix-caching",
|
|
|
+ # Their proven boot: no flashinfer autotune, no CuTe-DSL warmup (the
|
|
|
+ # autotune scratch at 512K shapes does not fit the headroom, 2026-08-28).
|
|
|
+ "--kernel-config", """
|
|
|
+'{"enable_cutedsl_warmup":false,"enable_flashinfer_autotune":false}'""",
|
|
|
+ "--language-model-only",
|
|
|
+ # A/B vs the no-spec 131K anchor (158/176, 934 pp / 19.0 tg / 2,124 ms):
|
|
|
+ # external DSpark K8 drafter, RedHatAI/GLM-5.3-speculator.dspark (09-16,
|
|
|
+ # v1.0) — a 3-layer Qwen3-style 2.5 GB drafter with Markov + confidence
|
|
|
+ # heads, trained on zai-org/GLM-5.3 (BF16) hidden states, layers
|
|
|
+ # 2/20/39/58/75 aux. Card contract (num_speculative_tokens 8, method
|
|
|
+ # dspark). r37 tree verified to carry the integration (DSparkDraftModel
|
|
|
+ # in config/speculative.py + speculators config loader). Drafter rides
|
|
|
+ # the TP group (draft_tensor_parallel_size defaults to TP). The repo is
|
|
|
+ # in the shared HF cache (snapshot 198b3ced), HF_HUB_OFFLINE=1 below
|
|
|
+ # resolves it from there.
|
|
|
+ "--speculative-config", """
|
|
|
+'{"model":"RedHatAI/GLM-5.3-speculator.dspark","num_speculative_tokens":8,"method":"dspark"}'""",
|
|
|
+ "--tool-call-parser", "glm47",
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
+ "--reasoning-parser", "glm45",
|
|
|
+]
|
|
|
+
|
|
|
+[mods]
|
|
|
+commands = [
|
|
|
+ # r37 tree: the draft SlidingWindowSpec inherits the target sparse-MLA
|
|
|
+ # shared-page padding (cache_config.skip_page_size_padded) and the KV
|
|
|
+ # layout assert fails (page_size_padded < unpadded bf16 draft page).
|
|
|
+ # The patcher lives on the shared HF cache (sm121-patches/) and is
|
|
|
+ # idempotent per boot; repo copy: patches/patch_dspark_draft_page.py.
|
|
|
+ "python3 /root/.cache/huggingface/sm121-patches/patch_dspark_draft_page.py",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+
|
|
|
+# R31 lock: 16 NCCL channels (their qualified runtime).
|
|
|
+NCCL_NCHANNELS = "16"
|
|
|
+HF_HUB_OFFLINE = "1"
|
|
|
+# RoCEnante (b12x one-shot RoCE all-reduce/all-gather) — default on for r27+
|
|
|
+# lanes since the 2026-09-08 A/B (4x DCP4: decode +31..+90%, quality 160 vs
|
|
|
+# 155; specs/tech-architecture/glm53-recipe-state.md). Needs the CX-7 RoCE
|
|
|
+# fabric (all of ours); single-node serves skip it silently.
|
|
|
+VLLM_ENABLE_ROCE_ALLREDUCE = "1"
|
|
|
+# Compile-OOM guard: first boot JIT-compiles the mxfp8 gemm module with 12
|
|
|
+# parallel nvcc jobs (~3 GiB each) behind ~94 GiB of resident weights →
|
|
|
+# cicc OOM-killed (2026-09-04). flashinfer's run_ninja reads MAX_JOBS.
|
|
|
+MAX_JOBS = "2"
|
|
|
+# NoPE-MLA rope-pad runtime gate (mod baked into the image)
|
|
|
+VLLM_MLA_NOPE_PAD_ROPE = "1"
|
|
|
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|
|
|
+# Upstream launch-script env (memory + compile knobs)
|
|
|
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
|
|
|
+VLLM_USE_AOT_COMPILE = "0"
|
|
|
+VLLM_USE_STANDALONE_COMPILE = "1"
|
|
|
+VLLM_USE_FLASHINFER_SAMPLER = "1"
|
|
|
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"
|
|
|
+TORCH_USE_RTLD_GLOBAL = "1"
|
|
|
+VLLM_NO_USAGE_STATS = "1"
|
|
|
+
|
|
|
+# Architecture (aarch64 / sm_121a DGX Spark)
|
|
|
+TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
+FLASHINFER_CUDA_ARCH_LIST = "12.1f"
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
+CMAKE_CUDA_ARCHITECTURES = "121"
|
|
|
+OMP_NUM_THREADS = "2"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/glm53-jit-121f/b12x"
|
|
|
+container = "/root/.cache/b12x"
|
|
|
+
|
|
|
+# JIT caches persist per-node across serves (was: cold /root/.cache per
|
|
|
+# container -> ~20 min flashinfer/torch recompile every serve). The host
|
|
|
+# dirs are shared between lanes of the same arch — each cache
|
|
|
+# self-namespaces by its engine version key internally.
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/flashinfer"
|
|
|
+container = "/root/.cache/flashinfer"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/torchinductor"
|
|
|
+container = "/root/.cache/torchinductor"
|
|
|
+
|
|
|
+[[extra_mounts]]
|
|
|
+host = "~/sm121-cache/jit-121f/vllm"
|
|
|
+container = "/root/.cache/vllm"
|
