|
|
@@ -0,0 +1,48 @@
|
|
|
+# GLM-5.3 (full) on 4x DGX Spark — the gr8 engine lane.
|
|
|
+#
|
|
|
+# gr8 (github.com/kindlingai/gr8) is a from-scratch CUDA inference engine for
|
|
|
+# the FULL GLM-5.3 (256-expert MoE, MLA + sparse indexer, one MTP layer) on
|
|
|
+# 3/4/6 Sparks, tensor-parallel across all of them. No vLLM/PyTorch/NCCL at
|
|
|
+# serve time: each box runs one `g6m4` process, ranks talk over RoCE v2, and an
|
|
|
+# OpenAI-compatible API runs on the first box (port 8100).
|
|
|
+#
|
|
|
+# First-class `engine = "gr8"`: the launcher generates the per-node
|
|
|
+# `gr8 glm53 up --tp <N> --peers <mgmt-LAN-ips>` (each rank finds itself in
|
|
|
+# --peers), passes the model via --model, and skips --host/--port (gr8's API
|
|
|
+# port is fixed at 8100). Readiness is the uniform /v1/models probe on :8100.
|
|
|
+#
|
|
|
+# Image: ghcr.io/kindlingai/gr8 (CI-published; TP is a runtime --tp flag, one
|
|
|
+# image for all shapes). No docker_image_dir — prebuilt external image.
|
|
|
+#
|
|
|
+# Prereqs (gr8 README): a RoCE v2 2-rail fabric (the switch clusters have it;
|
|
|
+# `--shape ring` runs switchless at TP3/4), TCP 7340/7341 open on the LAN,
|
|
|
+# ~90 GB disk/box, and proactive memory compaction OFF on every box (a rank
|
|
|
+# fills the box to ~2 GB under its limit; kernel compaction bursts delay every
|
|
|
+# collective). The sysctl is a before_serve host hook below.
|
|
|
+#
|
|
|
+# The DFlash2 drafter (incoai/GLM-5.3-DFlash2, CC BY-NC-ND) is TP=4-only (its 8
|
|
|
+# KV heads don't split 3/6 ways); each box downloads + converts its own rank.
|
|
|
+
|
|
|
+engine = "gr8"
|
|
|
+name = "GLM-5.3 full (gr8 engine, TP=4 switched)"
|
|
|
+docker_image = "ghcr.io/kindlingai/gr8"
|
|
|
+entrypoint = ["/opt/gr8/gr8", "glm53", "up"]
|
|
|
+# gr8 fetches its own per-rank slice into --model-cache (below); the model is
|
|
|
+# passed via --model, not the HF cache mount.
|
|
|
+model = "kindlingai/glm-5.3-exlr8-k2-k3-k4"
|
|
|
+parent_model = "ZhipuAI/GLM-5.3"
|
|
|
+port = 8100
|
|
|
+required_nodes = 4
|
|
|
+tensor_parallel_size = 4
|
|
|
+distributed_backend = "mp"
|
|
|
+# --tp and --peers are generated by the gr8 driver; --model comes from `model`.
|
|
|
+args = [
|
|
|
+ "--model-cache", "/opt/gr8/cache",
|
|
|
+ "--drafter", "https://huggingface.co/incoai/GLM-5.3-DFlash2",
|
|
|
+]
|
|
|
+
|
|
|
+# gr8 requires proactive memory compaction off on every box (holds across
|
|
|
+# reboots; idempotent). See gr8 README "What you need".
|
|
|
+before_serve = [
|
|
|
+ { command = "sudo sysctl -w vm.compaction_proactiveness=0", on_failure = "warn" },
|
|
|
+]
|
