kindling recipes: kill the GB10 UMA startup gate coin flip — GMU 0.88->0.86 (required 104.60 vs observed free 106.28-106.92 at worker init; the container stack refills ~11 GiB between the pre-container drop_caches and the check) + last-moment drop_caches mod right before the engine exec; the hard --kv-cache-memory pin is untouched so the KV pool is unchanged

recipes/glm53-kindling-2x/recipe.toml

@@ -44,7 +44,7 @@ args = [
     "--distributed-executor-backend", "mp",
     "--load-format", "auto",
     "--safetensors-load-strategy", "eager",
-    "--gpu-memory-utilization", "0.88",
+    "--gpu-memory-utilization", "0.86",
     "--max-model-len", "163840",
     "--kv-cache-dtype", "fp8_e4m3",
     "--block-size", "2304",
@@ -113,6 +113,18 @@ container = "/root/.cache/adaptive-k"
 host = "~/sm121-cache/glm53-kindling-tp2/draft-trunc"
 container = "/root/.cache/draft-trunc"
 
+# Last-moment drop_caches: the built-in (pre-container) one is too early —
+# by worker init the container stack (torch+CUDA+code page cache from
+# overlay/NFS) refills ~11 GiB, and the GB10 UMA gate (free >= GMU*total)
+# is a ~0.75 GiB coin flip per node (observed: 106.28/106.92 vs 107.03 on
+# random nodes). Dropping caches right before the engine exec reclaims the
+# code-cache part. GMU 0.88->0.86 below gives the extra margin; the hard
+# --kv-cache-memory pin is untouched, so the KV pool is unchanged.
+[mods]
+commands = [
+  { command = "sh -c 'echo 3 > /proc/sys/vm/drop_caches 2>/dev/null || true'", on_failure = "warn" },
+]
+
 [env]
 HF_HUB_OFFLINE = "1"
 VLLM_ENGINE_READY_TIMEOUT_S = "3600"

recipes/glm53-kindling-4x/recipe.toml

@@ -41,7 +41,7 @@ args = [
     "--distributed-executor-backend", "mp",
     "--load-format", "auto",
     "--safetensors-load-strategy", "eager",
-    "--gpu-memory-utilization", "0.88",
+    "--gpu-memory-utilization", "0.86",
     "--max-model-len", "524288",
     "--kv-cache-dtype", "fp8_e4m3",
     "--block-size", "2304",
@@ -110,6 +110,18 @@ container = "/root/.cache/adaptive-k"
 host = "~/sm121-cache/glm53-kindling/draft-trunc"
 container = "/root/.cache/draft-trunc"
 
+# Last-moment drop_caches: the built-in (pre-container) one is too early —
+# by worker init the container stack (torch+CUDA+code page cache from
+# overlay/NFS) refills ~11 GiB, and the GB10 UMA gate (free >= GMU*total)
+# is a ~0.75 GiB coin flip per node (observed: 106.28/106.92 vs 107.03 on
+# random nodes). Dropping caches right before the engine exec reclaims the
+# code-cache part. GMU 0.88->0.86 below gives the extra margin; the hard
+# --kv-cache-memory pin is untouched, so the KV pool is unchanged.
+[mods]
+commands = [
+  { command = "sh -c 'echo 3 > /proc/sys/vm/drop_caches 2>/dev/null || true'", on_failure = "warn" },
+]
+
 [env]
 HF_HUB_OFFLINE = "1"
 VLLM_ENGINE_READY_TIMEOUT_S = "3600"

recipes/glm53-kindling-6x/recipe.toml

@@ -50,7 +50,7 @@ args = [
     "--distributed-executor-backend", "mp",
     "--load-format", "auto",
     "--safetensors-load-strategy", "eager",
-    "--gpu-memory-utilization", "0.88",
+    "--gpu-memory-utilization", "0.86",
     "--max-model-len", "524288",
     "--kv-cache-dtype", "fp8_e4m3",
     "--block-size", "1792",
@@ -115,6 +115,18 @@ container = "/root/.cache/adaptive-k"
 host = "~/sm121-cache/glm53-kindling-tp6/draft-trunc"
 container = "/root/.cache/draft-trunc"
 
+# Last-moment drop_caches: the built-in (pre-container) one is too early —
+# by worker init the container stack (torch+CUDA+code page cache from
+# overlay/NFS) refills ~11 GiB, and the GB10 UMA gate (free >= GMU*total)
+# is a ~0.75 GiB coin flip per node (observed: 106.28/106.92 vs 107.03 on
+# random nodes). Dropping caches right before the engine exec reclaims the
+# code-cache part. GMU 0.88->0.86 below gives the extra margin; the hard
+# --kv-cache-memory pin is untouched, so the KV pool is unchanged.
+[mods]
+commands = [
+  { command = "sh -c 'echo 3 > /proc/sys/vm/drop_caches 2>/dev/null || true'", on_failure = "warn" },
+]
+
 [env]
 HF_HUB_OFFLINE = "1"
 VLLM_ENGINE_READY_TIMEOUT_S = "3600"