Browse Source

vision 2x: his verified TP2 values (GMU 0.85, KV pin off) — 0.80+pin hit a DeepGEMM layout assert

Alexander Korolev 1 month ago
parent
commit
33cde9c3ef
1 changed files with 4 additions and 2 deletions
  1. 4 2
      recipes/dsv4-vision-aiden-2x/recipe.toml

+ 4 - 2
recipes/dsv4-vision-aiden-2x/recipe.toml

@@ -40,8 +40,10 @@ args = [
     "--linear-backend", "b12x",
     "--kv-cache-dtype", "fp8",
     "--block-size", "256",
-    "--gpu-memory-utilization", "0.80",
-    "--kv-cache-memory-bytes", "12884901888",
+    # his verified 2x (TP2) values: 0.75 fails KV alloc, 0.82 leaves one rank
+    # short, 0.85 works; KV pin left OFF (a pin caps the pool ~26 GB/rank and
+    # hides the layout headroom)
+    "--gpu-memory-utilization", "0.85",
     "--max-model-len", "393216",
     "--max-num-seqs", "8",
     "--max-num-batched-tokens", "8192",