feat(compute): negotiate gateway callback listeners

.agents/skills/debug-openshell-cluster/SKILL.md

@@ -176,6 +176,18 @@ Common findings:
 - Sandbox image missing or pull denied: verify image reference and registry credentials.
 - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Root and missing identities are rejected.
 - Supervisor cannot call back: check callback endpoint and gateway logs.
+- Gateway exits before becoming healthy with a callback-listener discovery
+  error: inspect `podman info --debug`, the configured Podman network, and the
+  host's IPv4 default route. Rootless pasta uses the private source address
+  selected by that route; rootful Podman uses the bridge gateway address.
+- Callback discovery reports that the requested address equals the primary
+  listener: configure a distinct primary address. For Podman Machine, keep the
+  IPv4 loopback callback separate by using an IPv6-loopback primary such as
+  `[::1]:17670`.
+- Rootless slirp4netns, another named helper, or missing helper metadata
+  requires an explicitly remote `grpc_endpoint`. An explicit `host_gateway_ip`
+  cannot bypass slirp4netns host-loopback isolation. Do not work around
+  discovery failures by broadening the primary gateway listener to `0.0.0.0`.
 
 ### Step 6: Check Kubernetes Helm Gateways
 
@@ -392,6 +404,8 @@ openshell logs <sandbox-name>
 |---|---|---|
 | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs |
 | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs |
+| Gateway exits while resolving compute-driver listener requirements | Callback alias topology is unsupported, the Podman network cannot be inspected, or the selected address is not private/authorized | Gateway startup error, `podman info --debug`, Podman network inspection, host IPv4 default route |
+| Admin, health, reflection, or HTTP request is denied on a Docker/Podman callback address | Negotiated callback listeners intentionally expose only sandbox-callable gRPC methods | Retry through the gateway's primary endpoint; inspect the listener-purpose startup log if the address was unexpected |
 | Docker or Podman sandbox never registers | Wrong callback endpoint or supervisor startup failure | Gateway logs and sandbox container logs |
 | Docker GPU e2e fails before GPU sandbox comparison | NVIDIA CDI specs are missing or Docker has not discovered them | `docker info --format '{{json .DiscoveredDevices}}'`, `/etc/cdi`, `/var/run/cdi`, `nvidia-cdi-refresh.service` |
 | Kubernetes gateway pod pending | PVC unbound, taint, selector, or insufficient resources | `kubectl -n openshell describe pod <pod>` |

.github/workflows/e2e-test.yml

@@ -131,20 +131,17 @@ jobs:
     # Run directly on the Ubuntu host so the test observes the host's AppArmor
     # and unprivileged-user-namespace policy. A privileged job container masks
     # the restrictions that production rootless Podman installations enforce.
+    # Ubuntu 26.04 provides the supported Podman 5.x and pasta combination.
+    # Re-add older/slirp4netns environments when direct callbacks through a
+    # rootless-network namespace relay are supported.
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
         include:
-          # Ubuntu 24.04 matches the environment reported in #2069 and ships
-          # Podman 4.x. The probe records whether AppArmor blocks the drop.
-          - runner: ubuntu-24.04
-            podman_major: "4"
-            podman_package_version: "4.9.3+ds1-1ubuntu0.2"
-            conmon_package_version: "2.1.10+ds1-1build2"
-          # Ubuntu 26.04 provides the supported Podman 5.x coverage for
-          # comparison with the Ubuntu 24.04 environment.
+          # Keep package versions explicit so hosted-runner tool overrides
+          # cannot silently change the supported test environment.
           - runner: ubuntu-26.04
             podman_major: "5"
             podman_package_version: "5.7.0+ds2-3build1"
@@ -199,7 +196,6 @@ jobs:
             pkg-config \
             "conmon=${{ matrix.conmon_package_version }}" \
             "podman=${{ matrix.podman_package_version }}" \
-            slirp4netns \
             uidmap
           # Hosted runners can place newer Podman and conmon binaries under
           # /usr/local ahead of Ubuntu's packages. Select the distro CLI and
@@ -238,6 +234,7 @@ jobs:
           test "$(command -v podman)" = "/usr/bin/podman"
           test "$(podman info --format '{{.Host.Conmon.Path}}')" = "/usr/bin/conmon"
           test "$(podman info --format '{{.Host.Security.Rootless}}')" = "true"
+          test "$(podman info --format '{{.Host.RootlessNetworkCmd}}')" = "pasta"
           test "$(sudo sysctl -n kernel.apparmor_restrict_unprivileged_userns)" = "1"
           echo "=== host ==="
           uname -a

Cargo.toml

@@ -67,6 +67,7 @@ metrics-exporter-prometheus = { version = "0.18", default-features = false, feat
 # Unix/Process
 nix = { version = "0.29", features = ["signal", "process", "user", "fs", "term"] }
 rustix = { version = "1.1", features = ["process"] }
+socket2 = "0.6"
 
 # Serialization
 serde = { version = "1", features = ["derive"] }

architecture/gateway.md

@@ -37,6 +37,27 @@ health, metrics, or tunnel routes. The plaintext service router also rejects
 browser requests whose Fetch Metadata, Origin, or Referer headers indicate a
 cross-origin or sibling-subdomain request.
 
+Docker and Podman may negotiate additional listeners that make the gateway
+reachable from their local sandbox network topology. Those listeners accept
+only gRPC methods classified as sandbox-callable by the gateway's generated
+authorization metadata. They reject user and administrator APIs, health,
+reflection, non-callback inference APIs, and HTTP routes before normal request
+authentication. The operator-configured primary listener retains the full
+multiplexed API surface.
+
+The gateway rejects a callback requirement that resolves to the exact primary
+listener address because one socket cannot preserve two authorization scopes.
+A wildcard primary listener may cover a callback address because the accepted
+connection's concrete local address still selects the callback-only scope.
+
+The `rpc_auth` classification is also the source of truth for negotiated
+listener exposure: marking an RPC as `sandbox` or `dual` makes it callable on
+these listeners. Review such changes as both authorization and network-surface
+changes. Listener requirements are currently authorized only for the built-in
+Docker and Podman drivers. Operator-granted listener capabilities for external
+drivers are tracked in
+[#2539](https://github.com/NVIDIA/OpenShell/issues/2539).
+
 Operators can configure a gateway-wide gRPC request rate limit. The limit is
 applied only to gRPC API traffic after protocol multiplexing; health, metrics,
 and local sandbox-service HTTP routes are not rate limited by this control.
@@ -630,7 +651,12 @@ system entry instead of pretending to delete package-manager owned state.
 - Podman-backed macOS gateways use gvproxy's host-loopback IP for sandbox host
   aliases by default so stale Podman machine images do not need Podman's
   `host-gateway` resolver. Linux Podman keeps the resolver unless
-  `host_gateway_ip` is configured.
+  `host_gateway_ip` is configured. Rootful Podman can request its exact bridge
+  gateway listener. Rootless Podman explicitly reporting pasta requests the
+  private IPv4 source selected by the host default route rather than an
+  arbitrary private interface. Slirp4netns, other helpers, and missing helper
+  metadata fail closed for local callbacks until a rootless-network namespace
+  relay is available.
 - Gateway restarts recover persisted objects from storage, but live relay
   streams must be re-established by supervisors.
 - User-facing behavior changes must update published docs in `docs/`; this file

crates/openshell-driver-docker/src/lib.rs

@@ -39,12 +39,14 @@ use openshell_core::progress::{
 use openshell_core::proto::compute::v1::{
     CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse,
     DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus,
-    DriverSandboxTemplate, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest,
-    GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse,
-    StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest,
-    ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent,
-    WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent,
-    compute_driver_server::ComputeDriver, watch_sandboxes_event,
+    DriverSandboxTemplate, GatewayListenerRequirement, GetCapabilitiesRequest,
+    GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest,
+    GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse,
+    GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest,
+    StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse,
+    WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesPlatformEvent,
+    WatchSandboxesRequest, WatchSandboxesSandboxEvent, compute_driver_server::ComputeDriver,
+    gateway_listener_requirement::Selector, watch_sandboxes_event,
 };
 use openshell_core::proto_struct::{
     deserialize_optional_non_empty_string_list, struct_to_json_value,
@@ -399,14 +401,6 @@ impl DockerComputeDriver {
         Ok(driver)
     }
 
-    #[must_use]
-    pub fn gateway_bind_addresses(&self) -> Vec<SocketAddr> {
-        match self.config.gateway_route {
-            DockerGatewayRoute::Bridge { bind_address, .. } => vec![bind_address],
-            DockerGatewayRoute::HostGateway => Vec::new(),
-        }
-    }
-
     fn capabilities(&self) -> GetCapabilitiesResponse {
         openshell_core::driver_utils::build_capabilities_response(
             "docker",
@@ -1386,6 +1380,24 @@ impl ComputeDriver for DockerComputeDriver {
         Ok(Response::new(self.capabilities()))
     }
 
+    async fn get_gateway_listener_requirements(
+        &self,
+        _request: Request<GetGatewayListenerRequirementsRequest>,
+    ) -> Result<Response<GetGatewayListenerRequirementsResponse>, Status> {
+        let requirements = match self.config.gateway_route {
+            DockerGatewayRoute::Bridge { bind_address, .. } => {
+                vec![GatewayListenerRequirement {
+                    reason: "docker managed bridge gateway".to_string(),
+                    selector: Some(Selector::ExactBindAddress(bind_address.to_string())),
+                }]
+            }
+            DockerGatewayRoute::HostGateway => Vec::new(),
+        };
+        Ok(Response::new(GetGatewayListenerRequirementsResponse {
+            requirements,
+        }))
+    }
+
     async fn validate_sandbox_create(
         &self,
         request: Request<ValidateSandboxCreateRequest>,

crates/openshell-driver-docker/src/tests.rs

@@ -13,8 +13,9 @@ use openshell_core::progress::{
     PROGRESS_STEP_STARTING_SANDBOX,
 };
 use openshell_core::proto::compute::v1::{
-    DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, GpuResourceRequirements,
-    ResourceRequirements,
+    DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate,
+    GetGatewayListenerRequirementsRequest, GpuResourceRequirements, ResourceRequirements,
+    gateway_listener_requirement::Selector,
 };
 use std::fs;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr};
@@ -158,6 +159,43 @@ fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDr
     }
 }
 
+#[tokio::test]
+async fn gateway_listener_requirements_report_managed_bridge_address() {
+    let config = runtime_config();
+    let expected_address = match config.gateway_route {
+        DockerGatewayRoute::Bridge { bind_address, .. } => bind_address,
+        DockerGatewayRoute::HostGateway => panic!("test config must use a managed bridge"),
+    };
+    let driver = test_driver_with_config(config);
+
+    let response = driver
+        .get_gateway_listener_requirements(Request::new(GetGatewayListenerRequirementsRequest {}))
+        .await
+        .unwrap()
+        .into_inner();
+
+    assert_eq!(response.requirements.len(), 1);
+    assert_eq!(
+        response.requirements[0].selector,
+        Some(Selector::ExactBindAddress(expected_address.to_string()))
+    );
+}
+
+#[tokio::test]
+async fn gateway_listener_requirements_are_empty_for_host_gateway_route() {
+    let mut config = runtime_config();
+    config.gateway_route = DockerGatewayRoute::HostGateway;
+    let driver = test_driver_with_config(config);
+
+    let response = driver
+        .get_gateway_listener_requirements(Request::new(GetGatewayListenerRequirementsRequest {}))
+        .await
+        .unwrap()
+        .into_inner();
+
+    assert!(response.requirements.is_empty());
+}
+
 #[test]
 fn container_visible_endpoint_rewrites_loopback_hosts() {
     assert_eq!(

crates/openshell-driver-kubernetes/src/grpc.rs

@@ -6,7 +6,8 @@
 use futures::{Stream, StreamExt};
 use openshell_core::proto::compute::v1::{
     CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse,
-    GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, GetSandboxResponse,
+    GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest,
+    GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse,
     ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse,
     ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent,
     WatchSandboxesRequest, compute_driver_server::ComputeDriver,
@@ -40,6 +41,15 @@ impl ComputeDriver for ComputeDriverService {
             .map_err(Status::internal)
     }
 
+    async fn get_gateway_listener_requirements(
+        &self,
+        _request: Request<GetGatewayListenerRequirementsRequest>,
+    ) -> Result<Response<GetGatewayListenerRequirementsResponse>, Status> {
+        Ok(Response::new(GetGatewayListenerRequirementsResponse {
+            requirements: Vec::new(),
+        }))
+    }
+
     async fn validate_sandbox_create(
         &self,
         request: Request<ValidateSandboxCreateRequest>,

crates/openshell-driver-podman/Cargo.toml

@@ -34,6 +34,7 @@ tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 thiserror = { workspace = true }
 miette = { workspace = true }
+url = { workspace = true }
 
 [dev-dependencies]
 prost-types = { workspace = true }

crates/openshell-driver-podman/NETWORKING.md

@@ -255,6 +255,34 @@ if config.grpc_endpoint.is_empty() {
 The bridge gateway IP is not a stable substitute in rootless mode because it
 can live inside the user namespace rather than on the host.
 
+Before the gateway binds its serving sockets, the driver reports the callback
+listener required by the selected topology:
+
+- Rootful Linux Podman reports the configured bridge's gateway address exactly.
+- Rootless Linux Podman explicitly reporting pasta requests the private IPv4
+  source address selected by the host's default route. This avoids guessing
+  among private interfaces on a multihomed host.
+- Rootless Linux Podman reporting slirp4netns, another named helper, or no
+  helper cannot use a direct local callback listener. The driver fails startup
+  unless `grpc_endpoint` names an explicitly remote endpoint. Supporting
+  slirp4netns requires a relay inside Podman's rootless network namespace.
+- Podman Machine requests IPv4 loopback because gvproxy terminates the host
+  forwarding path there.
+- An explicitly remote callback endpoint requests no additional local listener.
+
+On Linux, an explicit `host_gateway_ip` is reported exactly for rootful Podman
+and rootless pasta because the driver maps both local callback aliases to that
+literal. Other rootless helpers still fail closed. Podman Machine requests
+gateway loopback because its configured address is guest-visible and gvproxy
+terminates that route on host loopback. The gateway validates and binds every
+accepted callback listener. A callback address cannot equal the exact primary
+listener address because the gateway could not distinguish their authorization
+scopes. In particular, a Podman Machine gateway using the IPv4 loopback
+callback must place its primary listener on another address, such as IPv6
+loopback (`[::1]:17670`). Negotiated callback listeners expose only the
+gateway's sandbox-callable gRPC methods. Operator, health, reflection, and HTTP
+requests must use the primary listener.
+
 ### Layer 3 Inner Sandbox Network Namespace
 
 Inside the container, the supervisor creates another network namespace for the

crates/openshell-driver-podman/src/client.rs

@@ -262,6 +262,8 @@ pub struct HostInfo {
     #[serde(default)]
     pub network_backend: String,
     #[serde(default)]
+    pub rootless_network_cmd: String,
+    #[serde(default)]
     pub security: SecurityInfo,
 }
 
@@ -902,6 +904,24 @@ mod tests {
         assert!(validate_name(&exact_name).is_ok());
     }
 
+    #[test]
+    fn system_info_parses_rootless_network_helper() {
+        let info: SystemInfo = serde_json::from_str(
+            r#"{
+                "host": {
+                    "cgroupVersion": "v2",
+                    "networkBackend": "netavark",
+                    "rootlessNetworkCmd": "pasta",
+                    "security": {"rootless": true}
+                }
+            }"#,
+        )
+        .unwrap();
+
+        assert!(info.host.security.rootless);
+        assert_eq!(info.host.rootless_network_cmd, "pasta");
+    }
+
     #[tokio::test]
     async fn inspect_volume_parses_driver_options() {
         let (socket_path, request_log, handle) = spawn_podman_stub(