fix(e2e): separate Podman Machine loopback listeners (#2622)

* fix(e2e): separate Podman Machine loopback listeners

Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>

* test(e2e): remove shallow harness checks

Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>

* refactor(e2e): trim Podman listener workaround

Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>

* fix(e2e): bypass proxies for Podman health probe

Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>

---------

Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>

Author: matthewgrossman

.agents/skills/debug-openshell-cluster/SKILL.md

@@ -208,9 +208,13 @@ Common findings:
   host's IPv4 default route. Rootless pasta uses the private source address
   selected by that route; rootful Podman uses the bridge gateway address.
 - Callback discovery reports that the requested address equals the primary
-  listener: configure a distinct primary address. For Podman Machine, keep the
-  IPv4 loopback callback separate by using an IPv6-loopback primary such as
-  `[::1]:17670`.
+  listener: configure a distinct primary address. For Podman Machine, bind the
+  primary listener to IPv6 loopback, for example
+  `bind_address = "[::1]:17670"`, and register the CLI endpoint as
+  `https://localhost:17670`. The generated certificate includes `localhost`,
+  while a raw `https://[::1]:17670` endpoint can fail TLS setup with
+  `invalid dns name`. This leaves `127.0.0.1:17670` available for the
+  callback-only listener.
 - Rootless slirp4netns, another named helper, or missing helper metadata
   requires an explicitly remote `grpc_endpoint`. An explicit `host_gateway_ip`
   cannot bypass slirp4netns host-loopback isolation. Do not work around

crates/openshell-core/src/forward.rs

@@ -745,11 +745,11 @@ pub fn resolve_ssh_gateway(
             // Remote cluster: use the remote host but keep the cluster URL port.
             return (host.to_string(), cluster_port);
         }
-        // Both endpoints loopback. The unspecified addresses (0.0.0.0 / ::)
-        // are bind-only — they aren't valid connect targets and aren't in TLS
-        // cert SANs, so fall back to the cluster URL's host (which the CLI
-        // is already using to reach the gateway).
-        if gateway_host == "0.0.0.0" || gateway_host == "::" {
+        // Unspecified addresses are bind-only, and tonic cannot use an IPv6
+        // literal as a TLS DNS name. In those cases, keep the cluster URL's
+        // already-reachable authority. Other loopback addresses retain the
+        // gateway-reported host.
+        if matches!(gateway_host, "0.0.0.0" | "::" | "::1") {
             return (host.to_string(), cluster_port);
         }
         return (gateway_host.to_string(), cluster_port);
@@ -1026,6 +1026,13 @@ mod tests {
         assert_eq!(port, 443);
     }
 
+    #[test]
+    fn resolve_ssh_gateway_preserves_loopback_tls_authority() {
+        let (host, port) = resolve_ssh_gateway("::1", 8080, "https://localhost:8443");
+        assert_eq!(host, "localhost");
+        assert_eq!(port, 8443);
+    }
+
     #[test]
     fn resolve_ssh_gateway_swaps_zeros_for_loopback_cluster_host() {
         // The gateway binds 0.0.0.0 but advertises that bind address via the

docs/reference/sandbox-compute-drivers.mdx

@@ -114,8 +114,10 @@ reflection, inference-route management, and HTTP requests. A
 is expected for those requests. The gateway fails startup if a callback
 requirement resolves to the exact primary listener address because one socket
 cannot preserve both authorization scopes. For the IPv4-loopback callback used
-by Podman Machine, bind the primary listener to a distinct address such as
-`[::1]:17670`.
+by Podman Machine, set `bind_address = "[::1]:17670"` for the primary listener
+and register `https://localhost:17670` as the CLI endpoint. The hostname matches
+the generated certificate and avoids the TLS transport error produced by a raw
+IPv6-literal endpoint. Do not broaden the primary listener to `0.0.0.0`.
 
 ## Docker Driver
 

e2e/with-podman-gateway.sh

@@ -386,6 +386,16 @@ export OPENSHELL_E2E_GATEWAY_CA_CERT="${PKI_DIR}/ca.crt"
 
 HOST_PORT=$(e2e_pick_port)
 HEALTH_PORT=$(e2e_pick_port)
+if [ "$(uname -s)" = "Darwin" ]; then
+  # Podman Machine reserves IPv4 loopback for its callback-only listener.
+  PRIMARY_BIND_IP="::1"
+  CLI_ENDPOINT_HOST="localhost"
+  HEALTH_ENDPOINT_HOST="[::1]"
+else
+  PRIMARY_BIND_IP="127.0.0.1"
+  CLI_ENDPOINT_HOST="127.0.0.1"
+  HEALTH_ENDPOINT_HOST="127.0.0.1"
+fi
 STATE_DIR="${WORKDIR}/state"
 mkdir -p "${STATE_DIR}"
 export XDG_STATE_HOME="${STATE_DIR}"
@@ -415,11 +425,11 @@ toml_string() {
 
 GATEWAY_CONFIG="${STATE_DIR}/gateway.toml"
 
-# Start from the RPM default template so this e2e test exercises the same
-# TOML config path that RPM users get on first start. The template leaves
-# bind_address unset and sets compute_drivers = ["podman"], so this test
-# exercises the built-in loopback listener plus the callback listener
-# requested by the Podman driver.
+# Start from the RPM default template so this e2e test exercises the same TOML
+# config path that RPM users get on first start. The template leaves
+# bind_address unset and sets compute_drivers = ["podman"]. On Podman Machine,
+# the driver reserves IPv4 loopback for its callback-only listener, so the
+# primary listener uses IPv6 loopback. Native Linux keeps the IPv4 default.
 #
 # We append the driver-specific table and override the port via CLI flag
 # (CLI > TOML in the merge precedence) so the test can use an ephemeral port.
@@ -458,8 +468,9 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}"
 
 GATEWAY_ARGS=(
   --config "${GATEWAY_CONFIG}"
-  # compute_drivers comes from the RPM template, while bind_address uses the
-  # built-in loopback default. Override only the port for ephemeral selection.
+  # compute_drivers comes from the RPM template. Override the loopback address
+  # and port so Podman Machine can keep its IPv4 callback listener distinct.
+  --bind-address "${PRIMARY_BIND_IP}"
   --port "${HOST_PORT}"
   --health-port "${HEALTH_PORT}"
   --tls-cert "${PKI_DIR}/server/tls.crt"
@@ -495,10 +506,10 @@ printf '%s\n' "${GATEWAY_PID}" >"${GATEWAY_PID_FILE}"
 
 GATEWAY_NAME="openshell-e2e-podman-${HOST_PORT}"
 if [ "${OIDC_MODE}" = "1" ]; then
-  CLI_GATEWAY_ENDPOINT="https://127.0.0.1:${HOST_PORT}"
+  CLI_GATEWAY_ENDPOINT="https://${CLI_ENDPOINT_HOST}:${HOST_PORT}"
   export OPENSHELL_E2E_OIDC_GATEWAY_ENDPOINT="${CLI_GATEWAY_ENDPOINT}"
 else
-  CLI_GATEWAY_ENDPOINT="https://127.0.0.1:${HOST_PORT}"
+  CLI_GATEWAY_ENDPOINT="https://${CLI_ENDPOINT_HOST}:${HOST_PORT}"
   e2e_register_mtls_gateway \
     "${XDG_CONFIG_HOME}" \
     "${GATEWAY_NAME}" \
@@ -524,7 +535,8 @@ while [ "${elapsed}" -lt "${timeout}" ]; do
     echo "ERROR: openshell-gateway exited before becoming healthy"
     exit 1
   fi
-  if curl -sf "http://127.0.0.1:${HEALTH_PORT}/healthz" >/dev/null 2>&1; then
+  # Keep this loopback probe direct even when ::1 is absent from NO_PROXY.
+  if curl --noproxy '*' -sf "http://${HEALTH_ENDPOINT_HOST}:${HEALTH_PORT}/healthz" >/dev/null 2>&1; then
     echo "Gateway healthy after ${elapsed}s."
     break
   fi