|
|
@@ -70,6 +70,11 @@ engine = "vllm"
|
|
|
name = "GLM-5.3-Flash NVFP4-Spark (vLLM, 2x Spark, MTP-5)"
|
|
|
docker_image = "glm53-jovian-r12:local" # r12 fork build (see docker-images/glm53-jovian-r12)
|
|
|
docker_image_dir = "glm53-jovian-r12"
|
|
|
+# Required: the clean r12 image follows the solar contract (ENTRYPOINT []
|
|
|
+# + CMD bash), so the recipe carries the entrypoint (the old day-0-based
|
|
|
+# r12 image had ["vllm","serve"] baked in, which is why this was absent
|
|
|
+# until the clean rebuild).
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
model = "local-inference-lab/GLM-5.3-Flash-NVFP4-Spark"
|
|
|
required_nodes = 2
|
|
|
tensor_parallel_size = 2
|
|
|
@@ -136,6 +141,11 @@ args = [
|
|
|
|
|
|
[env]
|
|
|
HF_HUB_OFFLINE = "1"
|
|
|
+# FlashInfer JIT (mxfp8_gemm_cutlass_sm120, ~12 kernels) compiles at first
|
|
|
+# boot with 12 parallel nvcc jobs -> OOM-killed (code 137) on GB10's
|
|
|
+# 121 GB unified while the 93 GB weights + 10 GB KV pin are resident.
|
|
|
+# MAX_JOBS caps ninja parallelism so the JIT fits in the remaining ~20 GB.
|
|
|
+MAX_JOBS = "2"
|
|
|
# NoPE-MLA rope-pad runtime gate (mod baked into the image)
|
|
|
VLLM_MLA_NOPE_PAD_ROPE = "1"
|
|
|
VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|