fix(cluster): resolve DNS failures on systemd-resolved hosts

crates/openshell-bootstrap/src/docker.rs

@@ -236,6 +236,57 @@ fn home_dir() -> Option<String> {
     std::env::var("HOME").ok()
 }
 
+/// Discover upstream DNS resolvers from systemd-resolved's configuration.
+///
+/// Only reads `/run/systemd/resolve/resolv.conf` — the upstream resolver file
+/// maintained by systemd-resolved. This file is only present on Linux hosts
+/// running systemd-resolved (e.g., Ubuntu), so the function is a no-op on
+/// macOS, Windows/WSL, and non-systemd Linux distributions.
+///
+/// We intentionally do NOT fall back to `/etc/resolv.conf` here. On Docker
+/// Desktop (macOS/Windows), `/etc/resolv.conf` may contain non-loopback
+/// resolvers that appear valid but are unreachable via direct UDP from inside
+/// the container's network stack. Those environments rely on the entrypoint's
+/// iptables DNAT proxy to Docker's embedded DNS — sniffing host resolvers
+/// would bypass that proxy and break DNS.
+///
+/// Returns an empty vec if no usable resolvers are found.
+fn resolve_upstream_dns() -> Vec<String> {
+    let paths = ["/run/systemd/resolve/resolv.conf"];
+
+    for path in &paths {
+        if let Ok(contents) = std::fs::read_to_string(path) {
+            let resolvers: Vec<String> = contents
+                .lines()
+                .filter_map(|line| {
+                    let line = line.trim();
+                    if !line.starts_with("nameserver") {
+                        return None;
+                    }
+                    let ip = line.split_whitespace().nth(1)?;
+                    if ip.starts_with("127.") || ip == "::1" {
+                        return None;
+                    }
+                    Some(ip.to_string())
+                })
+                .collect();
+
+            if !resolvers.is_empty() {
+                tracing::debug!(
+                    "Discovered {} upstream DNS resolver(s) from {}: {}",
+                    resolvers.len(),
+                    path,
+                    resolvers.join(", "),
+                );
+                return resolvers;
+            }
+        }
+    }
+
+    tracing::debug!("No upstream DNS resolvers found in host resolver config");
+    Vec::new()
+}
+
 /// Create an SSH Docker client from remote options.
 pub async fn create_ssh_docker_client(remote: &RemoteOptions) -> Result<Docker> {
     // Ensure destination has ssh:// prefix
@@ -455,6 +506,7 @@ pub async fn ensure_container(
     registry_username: Option<&str>,
     registry_token: Option<&str>,
     gpu: bool,
+    is_remote: bool,
 ) -> Result<()> {
     let container_name = container_name(name);
 
@@ -675,6 +727,17 @@ pub async fn ensure_container(
         env_vars.push("GPU_ENABLED=true".to_string());
     }
 
+    // Pass upstream DNS resolvers discovered on the host so the entrypoint
+    // can configure k3s without probing files inside the container.
+    // Skip for remote deploys — the local host's resolvers are likely wrong
+    // for the remote Docker host (different network, split-horizon DNS, etc.).
+    if !is_remote {
+        let upstream_dns = resolve_upstream_dns();
+        if !upstream_dns.is_empty() {
+            env_vars.push(format!("UPSTREAM_DNS={}", upstream_dns.join(",")));
+        }
+    }
+
     let env = Some(env_vars);
 
     let config = ContainerCreateBody {
@@ -1195,4 +1258,112 @@ mod tests {
             "should return a reasonable number of sockets"
         );
     }
+
+    #[test]
+    fn resolve_upstream_dns_filters_loopback() {
+        // This test validates the function runs without panic on the current host.
+        // The exact output depends on the host's DNS config, but loopback
+        // addresses must never appear in the result.
+        let resolvers = resolve_upstream_dns();
+        for r in &resolvers {
+            assert!(
+                !r.starts_with("127."),
+                "IPv4 loopback should be filtered: {r}"
+            );
+            assert_ne!(r, "::1", "IPv6 loopback should be filtered");
+        }
+    }
+
+    #[test]
+    fn resolve_upstream_dns_returns_vec() {
+        // Verify the function returns a vec (may be empty in some CI environments
+        // where no resolv.conf exists, but should never panic).
+        let resolvers = resolve_upstream_dns();
+        assert!(
+            resolvers.len() <= 20,
+            "should return a reasonable number of resolvers"
+        );
+    }
+
+    /// Helper: parse resolv.conf content using the same logic as resolve_upstream_dns().
+    /// Allows deterministic testing without depending on host DNS config.
+    fn parse_resolv_conf(contents: &str) -> Vec<String> {
+        contents
+            .lines()
+            .filter_map(|line| {
+                let line = line.trim();
+                if !line.starts_with("nameserver") {
+                    return None;
+                }
+                let ip = line.split_whitespace().nth(1)?;
+                if ip.starts_with("127.") || ip == "::1" {
+                    return None;
+                }
+                Some(ip.to_string())
+            })
+            .collect()
+    }
+
+    #[test]
+    fn parse_resolv_conf_filters_ipv4_loopback() {
+        let input = "nameserver 127.0.0.1\nnameserver 127.0.0.53\nnameserver 127.0.0.11\n";
+        assert!(parse_resolv_conf(input).is_empty());
+    }
+
+    #[test]
+    fn parse_resolv_conf_filters_ipv6_loopback() {
+        let input = "nameserver ::1\n";
+        assert!(parse_resolv_conf(input).is_empty());
+    }
+
+    #[test]
+    fn parse_resolv_conf_passes_real_resolvers() {
+        let input = "nameserver 8.8.8.8\nnameserver 1.1.1.1\n";
+        assert_eq!(parse_resolv_conf(input), vec!["8.8.8.8", "1.1.1.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_mixed_loopback_and_real() {
+        let input =
+            "nameserver 127.0.0.53\nnameserver ::1\nnameserver 10.0.0.1\nnameserver 172.16.0.1\n";
+        assert_eq!(parse_resolv_conf(input), vec!["10.0.0.1", "172.16.0.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_ignores_comments_and_other_lines() {
+        let input =
+            "# nameserver 8.8.8.8\nsearch example.com\noptions ndots:5\nnameserver 1.1.1.1\n";
+        assert_eq!(parse_resolv_conf(input), vec!["1.1.1.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_handles_tabs_and_extra_spaces() {
+        let input = "nameserver\t8.8.8.8\nnameserver     1.1.1.1\n";
+        assert_eq!(parse_resolv_conf(input), vec!["8.8.8.8", "1.1.1.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_empty_input() {
+        assert!(parse_resolv_conf("").is_empty());
+        assert!(parse_resolv_conf("   \n\n").is_empty());
+    }
+
+    #[test]
+    fn parse_resolv_conf_bare_nameserver_keyword() {
+        assert!(parse_resolv_conf("nameserver\n").is_empty());
+        assert!(parse_resolv_conf("nameserver   \n").is_empty());
+    }
+
+    #[test]
+    fn parse_resolv_conf_systemd_resolved_typical() {
+        let input =
+            "# This is /run/systemd/resolve/resolv.conf\nnameserver 192.168.1.1\nsearch lan\n";
+        assert_eq!(parse_resolv_conf(input), vec!["192.168.1.1"]);
+    }
+
+    #[test]
+    fn parse_resolv_conf_crlf_line_endings() {
+        let input = "nameserver 8.8.8.8\r\nnameserver 1.1.1.1\r\n";
+        assert_eq!(parse_resolv_conf(input), vec!["8.8.8.8", "1.1.1.1"]);
+    }
 }

crates/openshell-bootstrap/src/lib.rs

@@ -417,6 +417,7 @@ where
             registry_username.as_deref(),
             registry_token.as_deref(),
             gpu,
+            remote_opts.is_some(),
         )
         .await?;
         start_container(&target_docker, &name).await?;

deploy/docker/cluster-entrypoint.sh

@@ -69,7 +69,46 @@ wait_for_default_route() {
 #   3. Adding DNAT rules so traffic to <eth0_ip>:53 reaches Docker's DNS
 #   4. Writing that IP into the k3s resolv.conf
 
+# Extract upstream DNS resolvers reachable from k3s pod namespaces.
+# Docker's embedded DNS (127.0.0.11) is namespace-local — DNAT to it from
+# pod traffic is dropped as a martian packet. Use real upstream servers instead.
+#
+# Priority:
+#   1. UPSTREAM_DNS env var (set by bootstrap, comma-separated)
+#   2. /etc/resolv.conf (fallback for non-bootstrap launches)
+get_upstream_resolvers() {
+    local resolvers=""
+
+    # Bootstrap-provided resolvers (sniffed from host by the Rust bootstrap crate)
+    if [ -n "${UPSTREAM_DNS:-}" ]; then
+        resolvers=$(printf '%s\n' "$UPSTREAM_DNS" | tr ',' '\n' | \
+            awk '{ip=$1; if(ip !~ /^127\./ && ip != "::1" && ip != "") print ip}')
+    fi
+
+    # Fallback: Docker-generated resolv.conf may have non-loopback servers
+    if [ -z "$resolvers" ]; then
+        resolvers=$(awk '/^nameserver/{ip=$2; gsub(/\r/,"",ip); if(ip !~ /^127\./ && ip != "::1") print ip}' \
+            /etc/resolv.conf)
+    fi
+
+    echo "$resolvers"
+}
+
 setup_dns_proxy() {
+    # Prefer upstream resolvers that work across network namespaces.
+    # This avoids the DNAT-to-loopback problem on systemd-resolved hosts.
+    UPSTREAM_DNS=$(get_upstream_resolvers)
+    if [ -n "$UPSTREAM_DNS" ]; then
+        : > "$RESOLV_CONF"
+        echo "$UPSTREAM_DNS" | while read -r ns; do
+            [ -n "$ns" ] && echo "nameserver $ns" >> "$RESOLV_CONF"
+        done
+        echo "DNS: using upstream resolvers directly (avoids cross-namespace DNAT)"
+        cat "$RESOLV_CONF"
+        return 0
+    fi
+
+    # Fall back to DNAT proxy when no upstream resolvers are available.
     # Extract Docker's actual DNS listener ports from the DOCKER_OUTPUT chain.
     # Docker sets up rules like:
     #   -A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp --dport 53 -j DNAT --to-destination 127.0.0.11:<port>
@@ -160,6 +199,8 @@ verify_dns() {
         sleep 1
         i=$((i + 1))
     done
+    echo "Warning: DNS verification failed for $lookup_host after $attempts attempts"
+    echo "  resolv.conf: $(head -3 "$RESOLV_CONF" 2>/dev/null)"
     return 1
 }