recipes: glm53-flash-aiden-4x-labdrafter

recipes/glm53-flash-aiden-4x-labdrafter/recipe.toml

@@ -0,0 +1,93 @@
+# GLM-5.3-Flash on 4× DGX Spark — aiden image with the LAB's own DFlash2
+# drafter (local-inference-lab/GLM-5.3-Flash-DFlash2, 1.2 GB, on the shared
+# NAS HF) instead of incoai's — the competitive A/B the user asked for:
+# same image, same NVFP4 checkpoint, only the speculative model differs.
+# (This recipe = glm53-flash-aiden-4x with the drafter swapped.)
+#
+# PARKED 2026-09-04: with the MXFP8→bf16 dequantized lab drafter the target
+# loads clean and the drafter loads (no missing params), but the FIRST draft
+# pass at warmup dies in _project_context_kv: "mat1 and mat2 same dtype —
+# BFloat16 vs Float8_e4m3fn" (the fused KV-proj weight ends up fp8 at
+# runtime — the NVFP4 target's quant config bleeds into the draft's linears,
+# and the lab drafter's layout doesn't survive that path; incoai's does).
+# The lab drafter belongs to the lab's own vllm fork (newer dflash + aux-
+# capture). Revisit with a lab-fork image build.
+#
+# aiden's production 4-node profile, verbatim translation of
+# https://aidenle.com/recipes/glm-5-3-flash-4x-dgx-spark into our 4x launcher
+# shape. Reference numbers (his, single stream, idle cluster, thinking off):
+#   ~84 tok/s decode (FP8, DFlash2 k=7) — NVFP4 +23% (110.7 structured);
+#   prefill ~2,100 cold; TTFT 4 s @ 8K → 40 s @ 94K; KV 2.25M tokens at 16 GiB
+#   pin; 393K context verified. Decode flat with context (34/45 KDA layers +
+#   11 sparse-MLA @ index_topk 2048).
+#
+# DFlash2: acceptance 0.5–0.8; k=7 (drafter block 8 − 1 target token); k=6
+# wins prose, loses structured. Native MTP fallback (production-1.0 image or
+# spec omitted) optimum k=5. ~15 min full relaunch on FP8 (963 s measured).
+# FP8-MLA + DFlash2 combination is STRUCTURALLY broken upstream (three trees
+# fail differently) — hence plain fp8_e4m3 KV.
+#
+# NOT READY TO DEPLOY: needs a 4-node cluster (ds4 + 2nodes-dac freed) and the
+# unsloth/GLM-5.3-Flash-FP8 checkpoint (328 GB — background download running
+# on dgx3, /tmp/dl_fp8.log; ~340 GB free per node required for weights + Triton
+# cache). Fabric: RoCE dual-HCA NCCL profile comes from the launcher (config
+# [nccl.*] / GID auto-detect) — his NCCL_* env block intentionally omitted.
+# NOTE: his launcher unsets VLLM_PREFIX_CACHE_RETENTION_INTERVAL via bash -lc
+# before exec; if our boot rejects it in the KV coordinator, add a mods script.
+
+engine = "vllm"
+name = "GLM-5.3-Flash aiden image (vLLM, 4x Spark, LAB DFlash2-7, NVFP4)"
+docker_image = "sparkrun-vllm-glm53-flash-aiden:glm53"
+docker_image_dir = "glm53-flash-aiden"
+model = "local-inference-lab/GLM-5.3-Flash-NVFP4"
+parent_model = "zai-org/GLM-5.3-Flash"
+required_nodes = 4
+tensor_parallel_size = 4
+pipeline_parallel_size = 1
+distributed_backend = "mp"
+shm_size = "32gb"
+# image ships ENTRYPOINT ["vllm","serve"]; our launcher passes the recipe
+# entrypoint as CMD words after the image, which would double it. Empty clears.
+entrypoint = []
+
+args = [
+    "--served-model-name", "glm53",
+    "--trust-remote-code",
+    "--reasoning-parser", "glm45",
+    "--tool-call-parser", "glm47",
+    "--enable-auto-tool-choice",
+    "--default-chat-template-kwargs", """
+'{"enable_thinking": true, "reasoning_effort": "high"}'""",
+    "--kv-cache-dtype", "fp8_e4m3",
+#    "--block-size", "2304",
+    "--moe-backend", "flashinfer_cutlass",
+    "--max-model-len", "524288",
+    "--max-num-batched-tokens", "8192",
+    "--max-num-seqs", "16",
+    "--enable-chunked-prefill",
+#    "--enforce-eager",
+    "--gpu-memory-utilization", "0.85",
+    "--kv-cache-memory-bytes", "17179869184",
+    "--speculative-config", """
+'{"method":"mtp","num_speculative_tokens":3}'""",
+#    "--speculative-config", """
+#'{"method":"dflash","model":"local-inference-lab/GLM-5.3-Flash-DFlash2-bf16","num_speculative_tokens":7}'""",
+    "--language-model-only",
+    "--load-format", "instanttensor",
+    # image requests need the mm template shipped/in-wallpaper on aiden's rigs:
+    # "--chat-template", "<path-to>/chat_template_mm.jinja",
+]
+
+[env]
+HF_HUB_OFFLINE = "1"
+TRANSFORMERS_OFFLINE = "1"
+TORCH_USE_RTLD_GLOBAL = "1"
+TORCHINDUCTOR_COMPILE_THREADS = "1"
+CUTE_DSL_ARCH = "sm_121a"
+TORCH_CUDA_ARCH_LIST = "12.1a"
+PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:True"
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
+VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
+VLLM_NO_USAGE_STATS = "1"
+OMP_NUM_THREADS = "2"
+MAX_JOBS = "2"