fix: auto-power-on now powers on per-node entities, not just cluster extras

Two bugs fixed:

1. `occupied_nodes()` treated unreachable nodes as "occupied", causing
   `select_cluster` to think 0 nodes were free and bail with "No suitable
   cluster found". Now only nodes with running containers are "occupied";
   unreachable nodes are left out of the set.

2. `power_on_cluster_if_off()` only checked cluster-level `extra_entities`
   to decide whether to power on. For clusters like `2nodes-dac` that have
   no extras (only per-node entities like `switch.dgx1`), `ha_switch_off`
   was always false and nothing got powered on. Now it collects both
   per-node entities and cluster extras, matching `power::power_on`.

recipes/deepseek-v4-flash-dspark/recipe.toml

@@ -1,7 +1,7 @@
 # DeepSeek-V4-Flash-DSpark — 2× GB10 / DGX Spark, TP=2 over RoCE.
-# Uses the DSpark speculative decoding method with aidendle94's production-3.6 image.
+# Uses the DSpark speculative decoding method with aidendle94's production-3.7 image.
 #
-# Image: aidendle94/sparkrun-vllm-ds4-gb10:production-3.6
+# Image: aidendle94/sparkrun-vllm-ds4-gb10:production-3.7
 #   Latest GB10-native image with DSpark support, b12x MoE, and multi-node fixes.
 # Model: deepseek-ai/DeepSeek-V4-Flash-DSpark
 #   The DSpark variant tuned for speculative decoding on DGX Spark.
@@ -10,7 +10,7 @@
 
 engine = "vllm"
 name = "DeepSeek V4 Flash DSpark (vLLM)"
-docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-3.6"
+docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-3.7"
 model = "deepseek-ai/DeepSeek-V4-Flash-DSpark"
 required_nodes = 2
 tensor_parallel_size = 2
@@ -23,12 +23,12 @@ entrypoint = ["vllm", "serve"]
 args = [
     "--kv-cache-dtype", "fp8",
     "--block-size", "256",
-    "--max-model-len", "393216",
-    "--max-num-seqs", "128",
-    "--max-num-batched-tokens", "4096",
-    "--max-cudagraph-capture-size", "128",
-    "--gpu-memory-utilization", "0.8",
-    "--served-model-name", "deepseek-v4-flash-dspark", "default",
+    "--max-model-len", "1048576",
+    "--max-num-seqs", "16",
+    "--max-num-batched-tokens", "16536",
+    "--max-cudagraph-capture-size", "16",
+    "--gpu-memory-utilization", "0.84",
+    "--served-model-name", "deepseek-v4-flash", "default",
     # Engine backends
     "--attention-backend", "FLASHINFER_MLA_SPARSE_DSV4",
     "--moe-backend", "b12x",
@@ -43,25 +43,36 @@ args = [
     "--async-scheduling",
     "--no-scheduler-reserve-full-isl",
     # DSpark speculative decoding
+#    "--speculative-config", """
+#'{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic"}'""",
     "--speculative-config", """
-'{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic"}'""",
+'{"method":"dspark","num_speculative_tokens":7,"draft_sample_method":"greedy"}'""",
     # Tokenizer / reasoning
     "--tokenizer-mode", "deepseek_v4",
     "--tool-call-parser", "deepseek_v4",
     "--reasoning-parser", "deepseek_v4",
     "--enable-auto-tool-choice",
     "--default-chat-template-kwargs.thinking=true",
-    "--default-chat-template-kwargs.reasoning_effort=max",
+    #"--default-chat-template-kwargs.reasoning_effort=max",
+    # temp 0.4 norm
+    "--override-generation-config", """
+'{"temperature": 0.4, "top_p": 1.0}'""",
+    "--default-chat-template-kwargs.reasoning_effort=high",
     # Disable custom all-reduce (not needed on GB10 single-NIC)
     "--disable-custom-all-reduce",
     "--trust-remote-code",
+    # From model card
+# does not work with b12x MoE
+#    "--attention-config", """
+#'{"use_fp4_indexer_cache": true}'""",
 ]
 
 [env]
 # DSpark speculative decoding
 VLLM_DSPARK_REPLICATE_MARKOV_W1 = "1"
 # b12x MoE (this image's primary path)
-VLLM_USE_B12X_MOE = "1"
+# VLLM_USE_B12X_MOE = "1"
+VLLM_USE_B12X_MOE = "0" # cutlass decode
 
 # b12x suite = REQUIRED by this image
 VLLM_USE_B12X_MHC = "1"

src/cluster_power.rs

@@ -7,7 +7,10 @@ use crate::config::{Cluster, Config, NcclConfig, Recipe, resolve_cluster};
 use crate::home_assistant::Client as HaClient;
 use crate::ssh::run_ssh_with_retry;
 
-/// Power on cluster if switch is off.
+/// Power on cluster if the smart plug(s) are off.
+///
+/// Mirrors `power::power_on` but checks entity states first so it's a
+/// no-op when everything is already on.
 pub async fn power_on_cluster_if_off(
     config: &Config,
     recipe: &Recipe,
@@ -42,82 +45,92 @@ pub async fn power_on_cluster_if_off(
         (c, Some(n.clone()))
     };
 
-    let ha_switch_off = if let Some(ha_cfg) = &config.home_assistant {
-        let ha = HaClient::new(ha_cfg).ok();
-        let entities = cluster_to_check.extra_entities.clone();
-        if !entities.is_empty() {
-            if let Some(client) = ha {
-                let mut switch_off = false;
-                for e in &entities {
-                    match client.get_state(e).await {
-                        Some(ref s) if s == "on" => continue,
-                        _ => {
-                            switch_off = true;
-                            break;
-                        }
-                    }
+    let Some(ha_cfg) = &config.home_assistant else {
+        // No HA → can't power on via smart plugs; assume node is already
+        // reachable or the user handles power separately.
+        return Ok(());
+    };
+    let ha = HaClient::new(ha_cfg)?;
+    crate::home_assistant::warn_if_token_looks_short(&ha_cfg.token);
+
+    // Collect all entity IDs for this cluster (per-node + cluster-level extras).
+    let mut all_entities: Vec<String> = Vec::new();
+    for node in &cluster_to_check.nodes {
+        if let Some(entity) = &node.entity {
+            all_entities.push(entity.clone());
+        }
+    }
+    all_entities.extend(cluster_to_check.extra_entities.iter().cloned());
+
+    if all_entities.is_empty() {
+        // No entities configured → nothing to power on.
+        return Ok(());
+    }
+
+    // Check if any entity is NOT "on".
+    let any_off = {
+        let mut off = false;
+        for e in &all_entities {
+            match ha.get_state(e).await {
+                Some(ref s) if s == "on" => {}
+                _ => {
+                    off = true;
                 }
-                switch_off
-            } else {
-                false
             }
-        } else {
-            false
         }
-    } else {
-        false
+        off
     };
 
-    if ha_switch_off {
-        info!("{}", fluent_i18n::t!("cluster-switch-off-detected"));
-        if !dry_run {
-            if let Some(ha_cfg) = &config.home_assistant {
-                let ha = HaClient::new(ha_cfg)?;
-                crate::home_assistant::warn_if_token_looks_short(&ha_cfg.token);
-                for e in &cluster_to_check.extra_entities {
-                    ha.turn_on(e).await?;
-                }
-                info!("{}", fluent_i18n::t!("cluster-switch-powered-on"));
+    if !any_off {
+        // All entities already on; nodes should be reachable.
+        return Ok(());
+    }
 
-                // Wait for nodes to become available
-                info!("{}", fluent_i18n::t!("cluster-waiting-for-nodes"));
+    info!("{}", fluent_i18n::t!("cluster-switch-off-detected"));
+    if dry_run {
+        info!("{}", fluent_i18n::t!("power-on-dryrun"));
+        return Ok(());
+    }
 
-                // Initial boot delay — nodes need time for POST → bootloader → OS → SSH
-                let boot_wait_secs = 60;
-                info!("⏳ Waiting {}s for nodes to boot...", boot_wait_secs);
-                tokio::time::sleep(std::time::Duration::from_secs(boot_wait_secs)).await;
+    // Power on each entity (per-node first, then cluster extras).
+    for e in &all_entities {
+        ha.turn_on(e).await?;
+    }
+    info!("{}", fluent_i18n::t!("cluster-switch-powered-on"));
 
-                let max_poll_secs = 120;
-                let poll_interval_secs = 10;
-                let mut waited_secs = 0;
-                while waited_secs < max_poll_secs {
-                    let mut all_online = true;
-                    for node in &cluster_to_check.nodes {
-                        let cmd = "echo ok".to_string();
-                        if run_ssh_with_retry(config, node, cmd, false, 2)
-                            .await
-                            .is_err()
-                        {
-                            all_online = false;
-                            break;
-                        }
-                    }
-                    if all_online {
-                        info!("{}", fluent_i18n::t!("cluster-nodes-available"));
-                        break;
-                    }
-                    tokio::time::sleep(std::time::Duration::from_secs(poll_interval_secs)).await;
-                    waited_secs += poll_interval_secs;
-                }
-                if waited_secs >= max_poll_secs {
-                    anyhow::bail!("{}", fluent_i18n::t!("cluster-nodes-not-available-timeout"));
-                }
-            } else {
-                anyhow::bail!("{}", fluent_i18n::t!("ha-config-required-for-power-on"));
+    // Wait for nodes to become available via SSH.
+    info!("{}", fluent_i18n::t!("cluster-waiting-for-nodes"));
+
+    // Initial boot delay — nodes need time for POST → bootloader → OS → SSH.
+    let boot_wait_secs = 60;
+    info!("⏳ Waiting {}s for nodes to boot...", boot_wait_secs);
+    tokio::time::sleep(std::time::Duration::from_secs(boot_wait_secs)).await;
+
+    let max_poll_secs = 120;
+    let poll_interval_secs = 10;
+    let mut waited_secs = 0;
+    while waited_secs < max_poll_secs {
+        let mut all_online = true;
+        for node in &cluster_to_check.nodes {
+            let cmd = "echo ok".to_string();
+            if run_ssh_with_retry(config, node, cmd, false, 2)
+                .await
+                .is_err()
+            {
+                all_online = false;
+                break;
             }
-        } else {
-            info!("{}", fluent_i18n::t!("power-on-dryrun"));
         }
+        if all_online {
+            info!("{}", fluent_i18n::t!("cluster-nodes-available"));
+            break;
+        }
+        tokio::time::sleep(std::time::Duration::from_secs(poll_interval_secs)).await;
+        waited_secs += poll_interval_secs;
     }
+    if waited_secs >= max_poll_secs {
+        anyhow::bail!("{}", fluent_i18n::t!("cluster-nodes-not-available-timeout"));
+    }
+
     Ok(())
 }

src/docker/mod.rs

@@ -240,7 +240,7 @@ pub async fn occupied_nodes(config: &Config) -> Result<HashSet<String>> {
     let state = crate::cluster_state::ClusterState::probe(config).await;
     let mut result: HashSet<String> = HashSet::new();
     for (name, ns) in &state.nodes {
-        if !ns.reachable || ns.occupied {
+        if ns.occupied {
             result.insert(name.clone());
         }
     }