native/dgpp: fix provisioning from the first live DGPP w2 run

Live end-to-end test on 2nodes-dac surfaced four provisioning bugs, all fixed:
- tee logging: the inline `cmd && ... && echo DONE 2>&1 | tee log` piped only
  the final echo, so build output was lost on failure. Write the build script
  to a file (heredoc) and run `bash <file> 2>&1 | tee <log>` — whole build logged.
- install needs a deployment config: dgpp-cluster install <tarball> --config
  <json>. Stage the generated cluster JSON on every node before provisioning.
- install is HEAD-COORDINATED (installs on rank 0 AND SSHes to peers): run
  provisioning on rank 0 only, not every node (peers got double-installed).
- install is not idempotent ("release already exists"): tolerate it via a
  `releases | grep <rev-abbrev>` fallback, so a re-serve at the same pin is safe.
- DGPP_NODES must be SPACE-separated (was comma); stage .env on every node
  (install validates DGPP_NODES per node).
Provisioning now completes and hands off to dgpp-cluster up, which runs DGPP's
own preflight. Unit tests updated. make check green.

src/native/mod.rs

@@ -102,7 +102,7 @@ pub fn generate_site_env(ssh_user: &str, cluster: &Cluster, recipe: &Recipe) ->
         .iter()
         .map(|n| n.ssh_ip.clone())
         .collect::<Vec<_>>()
-        .join(",");
+        .join(" ");
     let port = http_port(recipe);
     format!(
         "DGPP_NODES={nodes}\nDGPP_SSH_USER={user}\nDGPP_HTTP_PORT={port}\nDGPP_FABRIC_PORT=29970\nDGPP_JOURNAL_PORT=29971\n",
@@ -193,12 +193,6 @@ pub async fn native_serve(
     // native analog of building the docker image on demand): clone-at-pin +
     // release.sh + install, with a rev-marker skip. No-op when the recipe
     // declares no [dgpp].repo/rev (assume pre-installed). Nodes in parallel.
-    let prov = cluster
-        .nodes
-        .iter()
-        .map(|n| provision::provision_node(config, n, recipe_name, &dgpp, dry_run));
-    futures::future::try_join_all(prov).await?;
-
     let dir = expand_home(&dgpp_dir(&dgpp));
     let cfg_name = format!("cluster_sm121_{recipe_name}.json");
     let cfg_path = format!("{dir}/{cfg_name}");
@@ -206,15 +200,31 @@ pub async fn native_serve(
     let json = generate_cluster_json(&recipe_obj, cluster.nodes.len())?;
     let env = generate_site_env(&config.ssh_user, &cluster, &recipe_obj);
 
-    crate::ssh::run_ssh(
-        config,
-        &head,
-        format!("mkdir -p \"{dir}\" && mkdir -p \"{dir}/log\""),
-        dry_run,
-    )
-    .await?;
-    write_remote(config, &head, &cfg_path, &json, dry_run).await?;
-    write_remote(config, &head, &env_path, &env, dry_run).await?;
+    // Stage the generated cluster JSON on EVERY launch node — DGPP's
+    // `dgpp-cluster install` needs a deployment config, and each node runs a
+    // rank — plus the site `.env` on every node too (install validates
+    // DGPP_NODES per node; the rank-0 manager reads it for `up`).
+    for n in &cluster.nodes {
+        crate::ssh::run_ssh(
+            config,
+            n,
+            format!("mkdir -p \"{dir}\" && mkdir -p \"{dir}/log\""),
+            dry_run,
+        )
+        .await?;
+        write_remote(config, n, &cfg_path, &json, dry_run).await?;
+        write_remote(config, n, &env_path, &env, dry_run).await?;
+    }
+
+    // Provision (clone-at-pin + release.sh + install --config <staged json>) on
+    // RANK 0 ONLY — `dgpp-cluster install` is head-coordinated: it installs the
+    // release on rank 0 AND SSHes to the peers (from DGPP_NODES) to install
+    // there too, like the docker build-once + sync model. Running it on every
+    // node would double-install the peers ("release already exists").
+    let prov = provision::provision_node(config, &head, recipe_name, &dgpp, &cfg_path, dry_run);
+    if let Err(e) = prov.await {
+        anyhow::bail!("DGPP provisioning failed on head '{}': {e}", head.name);
+    }
 
     // Launch on rank 0 only; the DGPP manager SSHes to the peers. Detached so
     // it survives the SSH session (nohup + log file).
@@ -345,7 +355,7 @@ mod tests {
             extra_entities: vec![],
         };
         let env = generate_site_env("kilork", &cluster, &r);
-        assert!(env.contains("DGPP_NODES=192.168.88.10,192.168.88.14"));
+        assert!(env.contains("DGPP_NODES=192.168.88.10 192.168.88.14"));
         assert!(env.contains("DGPP_SSH_USER=kilork"));
         assert!(env.contains("DGPP_HTTP_PORT=18080"));
     }

src/native/provision.rs

@@ -26,26 +26,43 @@ const DONE_MARKER: &str = "SM121-DGPP-DONE";
 /// are single-quoted literals. The rev marker + DONE echo run only if every
 /// prior step succeeded (`set -e`), so a failed build leaves no marker and the
 /// next serve retries.
-pub fn build_script(repo: &str, rev: &str, dir: &str) -> String {
+pub fn build_script(repo: &str, rev: &str, dir: &str, config_path: &str) -> String {
+    // `dgpp-cluster install` is not idempotent (a re-install of the same
+    // release errors "already exists"). Tolerate that only when the release is
+    // actually installed: fall back to `releases | grep <rev-abbrev>` (the
+    // version string carries the first 12 rev chars). A genuinely missing
+    // release still fails (grep misses) → no marker → retry.
+    let rev12: String = rev.chars().take(12).collect();
     format!(
         "set -e; \
          if [ -d \"{dir}/.git\" ]; then cd \"{dir}\" && git fetch --all --tags && git checkout '{rev}'; \
          else git clone '{repo}' \"{dir}\" && cd \"{dir}\" && git checkout '{rev}'; fi; \
          scripts/release.sh && \
-         scripts/dgpp-cluster install dist/dgpp-*.tar.zst && \
+         (scripts/dgpp-cluster install dist/dgpp-*.tar.zst --config \"{config_path}\" \
+            || scripts/dgpp-cluster releases --config \"{config_path}\" | grep -qF '{rev12}') && \
          echo '{rev}' > .sm121-dgpp-rev && echo {DONE_MARKER}"
     )
 }
 
-/// The detached tmux new-session command wrapping the build script (tee to a
-/// per-recipe log). Session/log names are per-recipe, like the docker build.
-pub fn tmux_build(recipe_name: &str, dir: &str, repo: &str, rev: &str) -> String {
+/// Path of the build script written to the node (heredoc) and its log.
+pub fn script_path(recipe_name: &str) -> String {
+    format!("/tmp/{recipe_name}-dgpp-build.sh")
+}
+pub fn log_path(recipe_name: &str) -> String {
+    format!("/tmp/{recipe_name}-dgpp-build.log")
+}
+
+/// The detached tmux command: run the (already-uploaded) build script through
+/// bash and tee its WHOLE output to the log. Writing the script to a file
+/// first avoids the quoting trap where `cmd && ... && echo DONE 2>&1 | tee`
+/// pipes only the final command — the file keeps the build output in the log.
+pub fn tmux_build(recipe_name: &str) -> String {
     let session = format!("sm121-dgpp-{recipe_name}");
-    let logfile = format!("/tmp/{recipe_name}-dgpp-build.log");
     format!(
         "tmux kill-session -t '{session}' 2>/dev/null || true; \
-         tmux new-session -d -s '{session}' '{script} 2>&1 | tee {logfile}'",
-        script = build_script(repo, rev, dir).replace('\'', r"'\''"),
+         tmux new-session -d -s '{session}' 'bash {script} 2>&1 | tee {log}'",
+        script = script_path(recipe_name),
+        log = log_path(recipe_name),
     )
 }
 
@@ -63,6 +80,7 @@ pub async fn provision_node(
     node: &Node,
     recipe_name: &str,
     d: &DgppConfig,
+    config_path: &str,
     dry_run: bool,
 ) -> Result<()> {
     let (repo, rev) = match (d.repo.as_deref(), d.rev.as_deref()) {
@@ -80,7 +98,15 @@ pub async fn provision_node(
             true,
         )
         .await?;
-        run_ssh(config, node, tmux_build(recipe_name, &dir, repo, rev), true).await?;
+        super::write_remote(
+            config,
+            node,
+            &script_path(recipe_name),
+            &build_script(repo, rev, &dir, config_path),
+            true,
+        )
+        .await?;
+        run_ssh(config, node, tmux_build(recipe_name), true).await?;
         return Ok(());
     }
 
@@ -101,16 +127,18 @@ pub async fn provision_node(
         "Provisioning DGPP '{rev}' on '{}' (tmux: sm121-dgpp-{recipe_name})...",
         node.name
     );
-    run_ssh(
+    super::write_remote(
         config,
         node,
-        tmux_build(recipe_name, &dir, repo, rev),
+        &script_path(recipe_name),
+        &build_script(repo, rev, &dir, config_path),
         false,
     )
     .await?;
+    run_ssh(config, node, tmux_build(recipe_name), false).await?;
 
     let session = format!("sm121-dgpp-{recipe_name}");
-    let logfile = format!("/tmp/{recipe_name}-dgpp-build.log");
+    let logfile = log_path(recipe_name);
     let start = std::time::Instant::now();
     loop {
         tokio::time::sleep(std::time::Duration::from_secs(30)).await;
@@ -177,27 +205,28 @@ mod tests {
             "https://github.com/HawkBearPig/dgpp.git",
             "v1.2.3",
             "$HOME/dgpp",
+            "$HOME/dgpp/cluster_sm121_x.json",
         );
         // dir double-quoted ($HOME expands), repo/rev single-quoted literals.
         assert!(s.contains("git clone 'https://github.com/HawkBearPig/dgpp.git' \"$HOME/dgpp\""));
         assert!(s.contains("git checkout 'v1.2.3'"));
         assert!(s.contains("scripts/release.sh"));
-        assert!(s.contains("scripts/dgpp-cluster install dist/dgpp-*.tar.zst"));
+        // install needs the deployment config (double-quoted → $HOME expands),
+        // and tolerates "already exists" via a releases|grep fallback.
+        assert!(s.contains("scripts/dgpp-cluster install dist/dgpp-*.tar.zst --config \"$HOME/dgpp/cluster_sm121_x.json\""));
+        assert!(s.contains("|| scripts/dgpp-cluster releases --config \"$HOME/dgpp/cluster_sm121_x.json\" | grep -qF 'v1.2.3'"));
         // marker + DONE come last (after `&&`), so a failed build leaves none.
         assert!(s.contains("echo 'v1.2.3' > .sm121-dgpp-rev && echo SM121-DGPP-DONE"));
         assert!(s.find("scripts/release.sh").unwrap() < s.find(".sm121-dgpp-rev").unwrap());
     }
 
     #[test]
-    fn tmux_build_detaches_and_tees_to_per_recipe_log() {
-        let t = tmux_build(
-            "glm53-flash-nvfp4-fp8-dgpp-2x",
-            "$HOME/dgpp",
-            "https://x/dgpp.git",
-            "abc123",
-        );
+    fn tmux_build_runs_the_uploaded_script_file_and_tees_whole_output() {
+        let t = tmux_build("glm53-flash-nvfp4-fp8-dgpp-2x");
         assert!(t.contains("tmux new-session -d -s 'sm121-dgpp-glm53-flash-nvfp4-fp8-dgpp-2x'"));
-        assert!(t.contains("tee /tmp/glm53-flash-nvfp4-fp8-dgpp-2x-dgpp-build.log"));
+        // the whole script's output is teed (bash <file> 2>&1 | tee <log>), not
+        // just the final echo — the file avoids the pipe-binds-to-last-cmd trap.
+        assert!(t.contains("bash /tmp/glm53-flash-nvfp4-fp8-dgpp-2x-dgpp-build.sh 2>&1 | tee /tmp/glm53-flash-nvfp4-fp8-dgpp-2x-dgpp-build.log"));
         // stale-session guard so a rerun doesn't hit "duplicate session".
         assert!(t.contains("tmux kill-session -t 'sm121-dgpp-glm53-flash-nvfp4-fp8-dgpp-2x'"));
     }