# GLM-5.3-Flash NVFP4 — local-inference-lab quant (vLLM) — 2× DGX Spark, TP=2.
#
# local-inference-lab's mixed-precision quant: main MoE experts NVFP4
# (group-16), MTP layer-45 routed experts in MXFP8, W4A4 input scales
# included. tool-eval-bench 89/100 (vs 85 LibertAIDAI, 90 official FP8) —
# best quant for agents so far.
# Model: https://huggingface.co/local-inference-lab/GLM-5.3-Flash-NVFP4
#        (~186 GB → ~93 GB/node on 2 Sparks; GMU 0.85 = ~101 GiB budget)
#
# Image: shared glm53-flash-nvfp4:local overlay (day-0 vLLM image + the
# NoPE-MLA / sm120 sparse-MLA mod). Rebuilt from docker-images/glm53-flash-nvfp4/.
#
# ⚠ KNOWN DAY-0 ISSUE (forum 381350 #41): vLLM needs a small `modelopt.py`
#   fix for THIS quant — the MTP weights are stored under
#   `language_model.model.layers.45.*` while the quant config lists
#   `model.language_model.layers.45.*`, so the drafter's scale lookup misses.
#   Until the fix is vendored into the overlay (as patch part 5), a boot with
#   --speculative-config may fail at MTP weight load. Fallback: comment the
#   speculative-config line → clean baseline serve (~15 tok/s class).
#
# GB10 gotchas handled here (same as glm53-flash-nvfp4 / LibertAIDAI):
#   --moe-backend marlin        (silent FP4 MoE corruption fix)
#   VLLM_ENGINE_READY_TIMEOUT_S (60-min FlashInfer autotune is not a hang)
#   --language-model-only       (vision front-end OOM-kills on GB10)
#
# PREREQUISITES: weights cached on both nodes:
#   huggingface-cli download local-inference-lab/GLM-5.3-Flash-NVFP4
#
# REFERENCE:
#   https://forums.developer.nvidia.com/t/glm-5-3-flash-320b-total-parameters-18b-active/381350/41
#   https://github.com/kingjones30/GLM-5.3-Flash-2x-DGX-Spark (mod + base config)

engine = "vllm"
name = "GLM-5.3-Flash NVFP4-lab (vLLM, 2x Spark)"
docker_image = "glm53-flash-nvfp4:local"
docker_image_dir = "glm53-flash-nvfp4"
model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

args = [
    # Stable served name — quant-agnostic (shared across glm53 quants)
    "--served-model-name", "glm53",
    "--trust-remote-code",
    "--distributed-executor-backend", "mp",
    # ~93 GiB weights/node of 119.6 GiB; drop to 0.83 or 262144→131072 if the
    # UMA MemFree gate (vLLM #48140) refuses boot
    "--gpu-memory-utilization", "0.85",
    "--max-model-len", "262144",
    "--max-num-seqs", "1",
    "--kv-cache-dtype", "fp8_ds_mla",
    "--language-model-only",
    "--moe-backend", "marlin",
    # MTP-5 — REQUIRES the modelopt.py prefix fix (header). Remove if load aborts.
    "--speculative-config", """
'{"method":"mtp","num_speculative_tokens":5}'""",
    "--tool-call-parser", "glm47",
    "--enable-auto-tool-choice",
    "--reasoning-parser", "glm45",
]

[env]
HF_HUB_OFFLINE = "1"
# Activates the NoPE-MLA rope-pad mod baked into the image
VLLM_MLA_NOPE_PAD_ROPE = "1"
# FlashInfer autotune can legitimately loop 60+ min on first boot
VLLM_ENGINE_READY_TIMEOUT_S = "3600"

TORCH_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
OMP_NUM_THREADS = "4"