fix: add initial boot delay and retry logic to auto-power-on

- After powering on the HA switch, wait 60s for nodes to boot (POST → OS → SSH)
- Then poll every 10s with run_ssh_with_retry(2 retries) instead of failing immediately
- Use 'echo ok' as the health-check command for simplicity

recipes/deepseek-v4-flash/recipe.toml

@@ -1,6 +1,6 @@
 engine = "vllm"
 name = "DeepSeek V4 Flash (vLLM)"
-docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-v2"
+docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-2.9"
 model = "deepseek-ai/DeepSeek-V4-Flash"
 required_nodes = 2
 tensor_parallel_size = 2
@@ -16,8 +16,12 @@ args = [
     "--kv-cache-dtype", "fp8",
     "--block-size", "256",
     "--max-model-len", "524288",
-    "--max-num-seqs", "8",
+    "--max-num-seqs", "6",
     "--max-num-batched-tokens", "8192",
+    "--max-cudagraph-capture-size", "192",
+    "--attention-backend", "B12X_MLA_SPARSE",
+    "--moe-backend", "flashinfer_cutlass",
+    "--linear-backend", "b12x",
     "--enable-chunked-prefill",
     "--gpu-memory-utilization", "0.85",
     "--compilation-config", """
@@ -33,8 +37,9 @@ args = [
     "--reasoning-config", """
 '{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
     "--default-chat-template-kwargs", """
-'{"thinking":true,"preserve_thinking":true}'""",
-    "--default-chat-template-kwargs.reasoning_effort", "high",
+'{"thinking":true,"preserve_thinking":true,"reasoning_effort":"high"}'""",
+    "--override-generation-config", """
+'{"temperature": 1.0, "top_p": 1.0}'""",
     "--enable-flashinfer-autotune",
     "--trust-remote-code",
     "--served-model-name", "deepseek-v4-flash", "default",
@@ -48,18 +53,42 @@ RAY_object_store_memory = "1073741824"
 
 # vLLM features for DeepSeek V4 on Blackwell (GB10 / sm_120)
 VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
-VLLM_USE_B12X_MOE = "1"
+# --- backend selection (the working config) ---
+USES_B12X = "True"
+VLLM_USE_B12X_MOE = "0"              # b12x MoE OFF -> cutlass W4A8 (b12x MoE prefill stalls on GB10)
+MOE_BACKEND = "flashinfer_cutlass"   # W4A8 cutlass MoE
+ATTN_BACKEND = "B12X_MLA_SPARSE"     # the only sparse-MLA that runs on GB10/sm_121a here
+ASYNC_SCHED = "0"                    # async-scheduling crashes 2-node TP — KEEP OFF
+# --- b12x suite = REQUIRED by this image (disabling any crashes engine init) ---
+VLLM_USE_B12X_MHC = "1"
+VLLM_USE_B12X_WO_PROJECTION = "1"
+VLLM_USE_B12X_FP8_GEMM = "1"
+VLLM_USE_B12X_SPARSE_INDEXER = "1"
+VLLM_USE_V2_MODEL_RUNNER = "1"
+VLLM_USE_FLASHINFER_SAMPLER = "1"
+VLLM_USE_AOT_COMPILE = "1"
+VLLM_USE_MEGA_AOT_ARTIFACT = "1"
+VLLM_USE_BREAKABLE_CUDAGRAPH = "0"   # breakable mode regresses prefill on GB10 — keep off
+VLLM_MEMORY_PROFILE_INCLUDE_ATTN = "1"
+VLLM_PREFIX_CACHE_RETENTION_INTERVAL = "4096"
+VLLM_CACHE_ROOT = "/cache/jit/vllm-cache"   # persist AOT/compile artifacts
+B12X_MLA_SM120_UNIFIED = "1"
+B12X_DENSE_SPLITK_TURBO = "1"
+B12X_W4A16_TC_DECODE = "1"
+B12X_MHC_MAX_TOKENS = "16384"
 VLLM_SPARSE_INDEXER_MAX_LOGITS_MB = "256"
 
 # Compile-time arch hints for PyTorch and FlashInfer on GB10
 TORCH_CUDA_ARCH_LIST = "12.1a"
 FLASHINFER_CUDA_ARCH_LIST = "12.1a"
+CUTE_DSL_ARCH = "sm_121a"
+OMP_NUM_THREADS = "4"
 
-CUDA_HOME = "/opt/env/targets/sbsa-linux"
+#CUDA_HOME = "/opt/env/targets/sbsa-linux"
 FLASHINFER_DISABLE_VERSION_CHECK = "1"
 TILELANG_CLEANUP_TEMP_FILES = "1"
 DG_JIT_USE_NVRTC = "0"
 DG_JIT_PRINT_COMPILER_COMMAND = "1"
-VLLM_NCCL_SO_PATH = "/opt/env/lib/python3.12/site-packages/nvidia/nccl/lib/libnccl.so.2"
+#VLLM_NCCL_SO_PATH = "/opt/env/lib/python3.12/site-packages/nvidia/nccl/lib/libnccl.so.2"
 HF_HUB_OFFLINE = "1"
 TRANSFORMERS_OFFLINE = "1"

src/cluster_power.rs

@@ -5,7 +5,7 @@ use tracing::info;
 
 use crate::config::{Cluster, Config, NcclConfig, Recipe, resolve_cluster};
 use crate::home_assistant::Client as HaClient;
-use crate::ssh::run_ssh;
+use crate::ssh::run_ssh_with_retry;
 
 /// Power on cluster if switch is off.
 pub async fn power_on_cluster_if_off(
@@ -81,15 +81,23 @@ pub async fn power_on_cluster_if_off(
 
                 // Wait for nodes to become available
                 info!("{}", fluent_i18n::t!("cluster-waiting-for-nodes"));
-                let max_wait_secs = 120;
-                let wait_interval_secs = 5;
+
+                // Initial boot delay — nodes need time for POST → bootloader → OS → SSH
+                let boot_wait_secs = 60;
+                info!("⏳ Waiting {}s for nodes to boot...", boot_wait_secs);
+                tokio::time::sleep(std::time::Duration::from_secs(boot_wait_secs)).await;
+
+                let max_poll_secs = 120;
+                let poll_interval_secs = 10;
                 let mut waited_secs = 0;
-                while waited_secs < max_wait_secs {
-                    // Check if all nodes are online
+                while waited_secs < max_poll_secs {
                     let mut all_online = true;
                     for node in &cluster_to_check.nodes {
-                        let cmd = "ip link show up 2>/dev/null | head -1 || echo ok";
-                        if run_ssh(config, node, cmd.to_string(), false).await.is_err() {
+                        let cmd = "echo ok".to_string();
+                        if run_ssh_with_retry(config, node, cmd, false, 2)
+                            .await
+                            .is_err()
+                        {
                             all_online = false;
                             break;
                         }
@@ -98,10 +106,10 @@ pub async fn power_on_cluster_if_off(
                         info!("{}", fluent_i18n::t!("cluster-nodes-available"));
                         break;
                     }
-                    tokio::time::sleep(std::time::Duration::from_secs(wait_interval_secs)).await;
-                    waited_secs += wait_interval_secs;
+                    tokio::time::sleep(std::time::Duration::from_secs(poll_interval_secs)).await;
+                    waited_secs += poll_interval_secs;
                 }
-                if waited_secs >= max_wait_secs {
+                if waited_secs >= max_poll_secs {
                     anyhow::bail!("{}", fluent_i18n::t!("cluster-nodes-not-available-timeout"));
                 }
             } else {