From 5da41a6e383ed883f11b29db0022d65fd538c46a Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 27 Jul 2026 12:57:47 +0200 Subject: [PATCH 01/24] feat(compute): query gateway listener requirements Signed-off-by: Evan Lezar --- crates/openshell-driver-docker/src/lib.rs | 40 +++-- crates/openshell-driver-docker/src/tests.rs | 42 ++++- .../openshell-driver-kubernetes/src/grpc.rs | 12 +- crates/openshell-driver-podman/src/grpc.rs | 12 +- crates/openshell-driver-vm/src/driver.rs | 10 ++ crates/openshell-server/src/compute/mod.rs | 169 +++++++++++++++--- .../openshell-server/src/gateway_listener.rs | 150 ++++++++++++++-- crates/openshell-server/src/lib.rs | 24 ++- crates/openshell-server/src/test_support.rs | 56 +++++- proto/compute_driver.proto | 24 +++ 10 files changed, 475 insertions(+), 64 deletions(-) diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index a196ba6ad..502f4ae8f 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -39,12 +39,14 @@ use openshell_core::progress::{ use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, - DriverSandboxTemplate, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, - GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, - WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, - compute_driver_server::ComputeDriver, watch_sandboxes_event, + DriverSandboxTemplate, GatewayListenerRequirement, GetCapabilitiesRequest, + GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, + GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, + StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, + WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesPlatformEvent, + WatchSandboxesRequest, WatchSandboxesSandboxEvent, compute_driver_server::ComputeDriver, + gateway_listener_requirement::Selector, watch_sandboxes_event, }; use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, @@ -399,14 +401,6 @@ impl DockerComputeDriver { Ok(driver) } - #[must_use] - pub fn gateway_bind_addresses(&self) -> Vec { - match self.config.gateway_route { - DockerGatewayRoute::Bridge { bind_address, .. } => vec![bind_address], - DockerGatewayRoute::HostGateway => Vec::new(), - } - } - fn capabilities(&self) -> GetCapabilitiesResponse { openshell_core::driver_utils::build_capabilities_response( "docker", @@ -1386,6 +1380,24 @@ impl ComputeDriver for DockerComputeDriver { Ok(Response::new(self.capabilities())) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + let requirements = match self.config.gateway_route { + DockerGatewayRoute::Bridge { bind_address, .. } => { + vec![GatewayListenerRequirement { + reason: "docker managed bridge gateway".to_string(), + selector: Some(Selector::ExactBindAddress(bind_address.to_string())), + }] + } + DockerGatewayRoute::HostGateway => Vec::new(), + }; + Ok(Response::new(GetGatewayListenerRequirementsResponse { + requirements, + })) + } + async fn validate_sandbox_create( &self, request: Request, diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index a86a9936f..fdf850dc6 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -13,8 +13,9 @@ use openshell_core::progress::{ PROGRESS_STEP_STARTING_SANDBOX, }; use openshell_core::proto::compute::v1::{ - DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, GpuResourceRequirements, - ResourceRequirements, + DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, + GetGatewayListenerRequirementsRequest, GpuResourceRequirements, ResourceRequirements, + gateway_listener_requirement::Selector, }; use std::fs; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; @@ -158,6 +159,43 @@ fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDr } } +#[tokio::test] +async fn gateway_listener_requirements_report_managed_bridge_address() { + let config = runtime_config(); + let expected_address = match config.gateway_route { + DockerGatewayRoute::Bridge { bind_address, .. } => bind_address, + DockerGatewayRoute::HostGateway => panic!("test config must use a managed bridge"), + }; + let driver = test_driver_with_config(config); + + let response = driver + .get_gateway_listener_requirements(Request::new(GetGatewayListenerRequirementsRequest {})) + .await + .unwrap() + .into_inner(); + + assert_eq!(response.requirements.len(), 1); + assert_eq!( + response.requirements[0].selector, + Some(Selector::ExactBindAddress(expected_address.to_string())) + ); +} + +#[tokio::test] +async fn gateway_listener_requirements_are_empty_for_host_gateway_route() { + let mut config = runtime_config(); + config.gateway_route = DockerGatewayRoute::HostGateway; + let driver = test_driver_with_config(config); + + let response = driver + .get_gateway_listener_requirements(Request::new(GetGatewayListenerRequirementsRequest {})) + .await + .unwrap() + .into_inner(); + + assert!(response.requirements.is_empty()); +} + #[test] fn container_visible_endpoint_rewrites_loopback_hosts() { assert_eq!( diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index fccfa9464..6eeb51cd7 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -6,7 +6,8 @@ use futures::{Stream, StreamExt}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, GetSandboxResponse, + GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_server::ComputeDriver, @@ -40,6 +41,15 @@ impl ComputeDriver for ComputeDriverService { .map_err(Status::internal) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(GetGatewayListenerRequirementsResponse { + requirements: Vec::new(), + })) + } + async fn validate_sandbox_create( &self, request: Request, diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index 8e68a91e7..4d04db54b 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -6,7 +6,8 @@ use futures::{Stream, StreamExt}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, GetSandboxResponse, + GetCapabilitiesRequest, GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_server::ComputeDriver, @@ -40,6 +41,15 @@ impl ComputeDriver for ComputeDriverService { .map_err(Status::from) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(GetGatewayListenerRequirementsResponse { + requirements: Vec::new(), + })) + } + async fn validate_sandbox_create( &self, request: Request, diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 7af0ddc38..841457f38 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -41,6 +41,7 @@ use openshell_core::proto::compute::v1::{ DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent, DriverSandbox as Sandbox, DriverSandboxStatus as SandboxStatus, DriverSandboxTemplate as SandboxTemplate, GetCapabilitiesRequest, GetCapabilitiesResponse, + GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, @@ -2989,6 +2990,15 @@ impl ComputeDriver for VmDriver { Ok(Response::new(self.capabilities())) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(GetGatewayListenerRequirementsResponse { + requirements: Vec::new(), + })) + } + async fn validate_sandbox_create( &self, request: Request, diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index b87f1b5fa..0b239d15d 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -27,11 +27,14 @@ use openshell_core::ComputeDriverKind; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, DeleteSandboxRequest, DriverCondition, DriverPlatformEvent, DriverResourceRequirements, DriverSandbox, DriverSandboxSpec, DriverSandboxStatus, - DriverSandboxTemplate, GetCapabilitiesRequest, GetSandboxRequest, + DriverSandboxTemplate, GatewayListenerRequirement as ProtoGatewayListenerRequirement, + GetCapabilitiesRequest, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GpuResourceRequirements as DriverGpuResourceRequirements, ListSandboxesRequest, ResourceRequirements as DriverSandboxResourceRequirements, ValidateSandboxCreateRequest, WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_client::ComputeDriverClient, - compute_driver_server::ComputeDriver, watch_sandboxes_event, + compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, + watch_sandboxes_event, }; use openshell_core::proto::{ PlatformEvent, Sandbox, SandboxCondition, SandboxPhase, SandboxSpec, SandboxStatus, @@ -65,6 +68,13 @@ type SharedComputeDriver = const DELETE_PHASE_CAS_RETRY_LIMIT: usize = 3; +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct GatewayListenerRequirement { + pub address: SocketAddr, + pub driver_name: String, + pub reason: String, +} + /// Serializes request-side deletes for the same stable sandbox ID. /// /// Watch events deliberately do not use these gates, so a slow driver delete @@ -288,6 +298,14 @@ impl ComputeDriver for RemoteComputeDriver { client.get_capabilities(request).await } + async fn get_gateway_listener_requirements( + &self, + request: Request, + ) -> Result, Status> { + let mut client = self.client(); + client.get_gateway_listener_requirements(request).await + } + async fn validate_sandbox_create( &self, request: Request, @@ -370,7 +388,7 @@ pub struct ComputeRuntime { supervisor_sessions: Arc, sync_lock: Arc>, delete_gates: Arc, - gateway_bind_addresses: Vec, + gateway_listener_requirements: Vec, replica_id: String, } @@ -393,7 +411,6 @@ impl ComputeRuntime { sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, - gateway_bind_addresses: Vec, ) -> Result { let capabilities = driver .get_capabilities(Request::new(GetCapabilitiesRequest {})) @@ -408,11 +425,48 @@ impl ComputeRuntime { "Compute driver connected" ); let driver_info = ComputeDriverInfoSnapshot { - name: driver_name, + name: driver_name.clone(), driver_name: capabilities.driver_name, driver_version: capabilities.driver_version, }; let default_image = capabilities.default_image; + let gateway_listener_requirements = match driver + .get_gateway_listener_requirements(Request::new( + GetGatewayListenerRequirementsRequest {}, + )) + .await + { + Ok(response) => response + .into_inner() + .requirements + .into_iter() + .map(|requirement: ProtoGatewayListenerRequirement| { + let Some(Selector::ExactBindAddress(bind_address)) = requirement.selector else { + return Err(ComputeError::Message(format!( + "compute driver '{driver_name}' returned a gateway listener requirement without an exact bind address" + ))); + }; + let address = bind_address.parse::().map_err(|err| { + ComputeError::Message(format!( + "compute driver '{driver_name}' returned invalid gateway listener address '{bind_address}': {err}" + )) + })?; + Ok(GatewayListenerRequirement { + address, + driver_name: driver_name.clone(), + reason: requirement.reason, + }) + }) + .collect::, ComputeError>>()?, + Err(status) if status.code() == Code::Unimplemented => { + debug!( + driver = %driver_name, + "Compute driver does not implement gateway listener requirements" + ); + Vec::new() + } + Err(status) => return Err(compute_error_from_status(status)), + }; Ok(Self { driver, driver_info, @@ -427,7 +481,7 @@ impl ComputeRuntime { supervisor_sessions, sync_lock: Arc::new(Mutex::new(())), delete_gates: Arc::new(DeleteGateRegistry::default()), - gateway_bind_addresses, + gateway_listener_requirements, replica_id: lease::replica_id(), }) } @@ -471,7 +525,6 @@ impl ComputeRuntime { .await .map_err(|err| ComputeError::Message(err.to_string()))?, ); - let gateway_bind_addresses = driver.gateway_bind_addresses(); let shutdown_cleanup: Arc = driver.clone(); let startup_resume: Arc = driver.clone(); let driver: SharedComputeDriver = driver; @@ -486,7 +539,6 @@ impl ComputeRuntime { sandbox_watch_bus, tracing_log_bus, supervisor_sessions, - gateway_bind_addresses, ) .await } @@ -514,7 +566,6 @@ impl ComputeRuntime { sandbox_watch_bus, tracing_log_bus, supervisor_sessions, - Vec::new(), ) .await } @@ -539,7 +590,6 @@ impl ComputeRuntime { sandbox_watch_bus, tracing_log_bus, supervisor_sessions, - Vec::new(), ) .await } @@ -567,7 +617,6 @@ impl ComputeRuntime { sandbox_watch_bus, tracing_log_bus, supervisor_sessions, - Vec::new(), ) .await } @@ -588,8 +637,8 @@ impl ComputeRuntime { } #[must_use] - pub fn gateway_bind_addresses(&self) -> &[SocketAddr] { - &self.gateway_bind_addresses + pub(crate) fn gateway_listener_requirements(&self) -> &[GatewayListenerRequirement] { + &self.gateway_listener_requirements } pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), Status> { @@ -2752,6 +2801,15 @@ impl ComputeDriver for NoopTestDriver { )) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new( + GetGatewayListenerRequirementsResponse::default(), + )) + } + async fn validate_sandbox_create( &self, _request: Request, @@ -2847,7 +2905,7 @@ pub async fn new_test_runtime_for_driver(store: Arc, driver_name: &str) - supervisor_sessions: Arc::new(SupervisorSessionRegistry::new()), sync_lock: Arc::new(Mutex::new(())), delete_gates: Arc::new(DeleteGateRegistry::default()), - gateway_bind_addresses: Vec::new(), + gateway_listener_requirements: Vec::new(), replica_id: "test-replica".to_string(), } } @@ -3001,6 +3059,15 @@ mod tests { })) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new( + GetGatewayListenerRequirementsResponse::default(), + )) + } + async fn validate_sandbox_create( &self, _request: Request, @@ -3183,6 +3250,15 @@ mod tests { })) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + Ok(tonic::Response::new( + GetGatewayListenerRequirementsResponse::default(), + )) + } + async fn validate_sandbox_create( &self, _request: Request, @@ -3316,7 +3392,7 @@ mod tests { supervisor_sessions: Arc::new(SupervisorSessionRegistry::new()), sync_lock: Arc::new(Mutex::new(())), delete_gates: Arc::new(DeleteGateRegistry::default()), - gateway_bind_addresses: Vec::new(), + gateway_listener_requirements: Vec::new(), replica_id: "test-replica".to_string(), } } @@ -6085,7 +6161,11 @@ mod tests { let socket_path = dir.path().join("compute-driver.sock"); let driver = FakeComputeDriver::new() .with_driver_name("fake-remote-driver") - .with_default_image("openshell/sandbox:remote"); + .with_default_image("openshell/sandbox:remote") + .with_gateway_listener_requirement( + "172.19.0.1:17670", + "external driver managed bridge", + ); let _server = driver.serve_uds(&socket_path).unwrap(); let endpoint = connect_remote_compute_driver("external-test", &socket_path) @@ -6102,6 +6182,14 @@ mod tests { ) .await .unwrap(); + assert_eq!( + runtime.gateway_listener_requirements(), + &[GatewayListenerRequirement { + address: "172.19.0.1:17670".parse().unwrap(), + driver_name: "external-test".to_string(), + reason: "external driver managed bridge".to_string(), + }] + ); let mut sandbox = sandbox_record("sb-uds", "uds-sandbox", SandboxPhase::Provisioning); sandbox.spec = Some(SandboxSpec { @@ -6138,10 +6226,14 @@ mod tests { ); let calls = driver.calls(); - assert_eq!(calls.len(), 4, "unexpected calls: {calls:?}"); + assert_eq!(calls.len(), 5, "unexpected calls: {calls:?}"); assert!(matches!(calls[0], FakeComputeDriverCall::GetCapabilities)); + assert!(matches!( + calls[1], + FakeComputeDriverCall::GetGatewayListenerRequirements + )); - let validated = match &calls[1] { + let validated = match &calls[2] { FakeComputeDriverCall::ValidateSandboxCreate { sandbox: Some(sandbox), } => sandbox, @@ -6158,7 +6250,7 @@ mod tests { assert!(driver_config.fields.contains_key("pool")); assert!(!driver_config.fields.contains_key("network_mode")); - let created = match &calls[2] { + let created = match &calls[3] { FakeComputeDriverCall::CreateSandbox { sandbox: Some(sandbox), } => sandbox, @@ -6167,7 +6259,7 @@ mod tests { assert_eq!(created.id, "sb-uds"); assert_eq!(created.name, "uds-sandbox"); - match &calls[3] { + match &calls[4] { FakeComputeDriverCall::DeleteSandbox { sandbox_id, sandbox_name, @@ -6179,6 +6271,43 @@ mod tests { } } + #[tokio::test] + #[cfg(unix)] + async fn remote_compute_driver_accepts_unimplemented_listener_requirements_api() { + use crate::test_support::{FakeComputeDriver, FakeComputeDriverCall}; + + let dir = tempfile::tempdir().unwrap(); + let socket_path = dir.path().join("compute-driver.sock"); + let driver = FakeComputeDriver::new() + .with_driver_name("legacy-remote-driver") + .without_gateway_listener_requirements_api(); + let _server = driver.serve_uds(&socket_path).unwrap(); + + let endpoint = connect_remote_compute_driver("external-test", &socket_path) + .await + .unwrap(); + let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); + let runtime = ComputeRuntime::new_remote_driver( + endpoint, + store, + SandboxIndex::new(), + SandboxWatchBus::new(), + TracingLogBus::new(), + Arc::new(SupervisorSessionRegistry::new()), + ) + .await + .unwrap(); + + assert!(runtime.gateway_listener_requirements().is_empty()); + assert_eq!( + driver.calls(), + vec![ + FakeComputeDriverCall::GetCapabilities, + FakeComputeDriverCall::GetGatewayListenerRequirements, + ] + ); + } + #[tokio::test] async fn create_sandbox_returns_resource_version_one() { let runtime = test_runtime(Arc::new(TestDriver::default())).await; diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index 0957c4ebe..21100de52 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -1,7 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use openshell_core::{Error, Result}; +use crate::compute::GatewayListenerRequirement; +use openshell_core::{ComputeDriverKind, Error, Result}; use std::net::SocketAddr; use tokio::net::TcpListener; use tracing::info; @@ -24,6 +25,14 @@ pub struct GatewayListenerSpec { pub address: SocketAddr, pub scope: GatewayListenerScope, covered_addresses: Vec, + provenance: Option, +} + +/// Diagnostic source of a driver-requested listener. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct GatewayListenerProvenance { + pub driver_name: String, + pub reason: String, } /// A gateway listener together with the context needed to serve it. @@ -38,6 +47,7 @@ impl GatewayListenerSpec { address, scope, covered_addresses: Vec::new(), + provenance: None, } } @@ -59,42 +69,88 @@ impl GatewayListenerSpec { fn gateway_listener_specs( bind_address: SocketAddr, - extra_addresses: &[SocketAddr], -) -> Vec { + requirements: &[GatewayListenerRequirement], +) -> Result> { let mut specs = vec![GatewayListenerSpec::new( bind_address, GatewayListenerScope::Primary, )]; - for address in extra_addresses { + for requirement in requirements { + validate_gateway_listener_requirement(bind_address, requirement)?; + let address = requirement.address; let scope = GatewayListenerScope::ComputeDriverCallback; if let Some(existing) = specs .iter() - .position(|existing| listener_covers(existing.address, *address)) + .position(|existing| listener_covers(existing.address, address)) { let existing = &mut specs[existing]; - if existing.address != *address + if existing.address != address && !existing .covered_addresses .iter() - .any(|covered| covered.address == *address) + .any(|covered| covered.address == address) { existing.covered_addresses.push(CoveredGatewayAddress { - address: *address, + address, scope, }); } } else { - specs.push(GatewayListenerSpec::new(*address, scope)); + specs.push(GatewayListenerSpec { + address, + scope, + covered_addresses: Vec::new(), + provenance: Some(GatewayListenerProvenance { + driver_name: requirement.driver_name.clone(), + reason: requirement.reason.clone(), + }), + }); } } - specs + Ok(specs) +} + +fn validate_gateway_listener_requirement( + primary_listener: SocketAddr, + requirement: &GatewayListenerRequirement, +) -> Result<()> { + let requested_listener = requirement.address; + if requirement.driver_name != ComputeDriverKind::Docker.as_str() { + return Err(Error::config(format!( + "compute driver '{}' is not authorized to request gateway listeners in this proof of concept", + requirement.driver_name + ))); + } + if requested_listener.ip().is_unspecified() { + return Err(Error::config(format!( + "compute driver requested wildcard gateway listener {requested_listener}" + ))); + } + if requested_listener.ip().is_multicast() { + return Err(Error::config(format!( + "compute driver requested multicast gateway listener {requested_listener}" + ))); + } + if requested_listener.port() == 0 { + return Err(Error::config(format!( + "compute driver requested zero-port gateway listener {requested_listener}" + ))); + } + if requested_listener.port() != primary_listener.port() { + return Err(Error::config(format!( + "compute driver requested gateway listener {requested_listener} with port {}, but the primary listener uses port {}", + requested_listener.port(), + primary_listener.port() + ))); + } + Ok(()) } pub async fn bind_gateway_listeners( bind_address: SocketAddr, - extra_addresses: &[SocketAddr], + requirements: &[GatewayListenerRequirement], ) -> Result> { - let specs = gateway_listener_specs(bind_address, extra_addresses); + let specs = gateway_listener_specs(bind_address, requirements)?; let mut listeners = Vec::with_capacity(specs.len()); for spec in specs { let listener = TcpListener::bind(spec.address) @@ -158,9 +214,10 @@ fn listener_covers(existing: SocketAddr, requested: SocketAddr) -> bool { #[cfg(test)] mod tests { use super::{ - CoveredGatewayAddress, GatewayListenerScope, GatewayListenerSpec, bind_gateway_listeners, - gateway_listener_specs, + CoveredGatewayAddress, GatewayListenerProvenance, GatewayListenerScope, + GatewayListenerSpec, bind_gateway_listeners, gateway_listener_specs, }; + use crate::compute::GatewayListenerRequirement; use std::net::SocketAddr; use std::sync::atomic::{AtomicBool, Ordering}; use tokio::net::TcpListener; @@ -169,9 +226,13 @@ mod tests { fn gateway_listener_specs_track_driver_address_covered_by_wildcard() { let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); let docker: SocketAddr = "172.18.0.1:8080".parse().unwrap(); + let requirements = [ + docker_listener_requirement(docker), + docker_listener_requirement(docker), + ]; assert_eq!( - gateway_listener_specs(primary, &[docker, docker]), + gateway_listener_specs(primary, &requirements).unwrap(), vec![GatewayListenerSpec { address: primary, scope: GatewayListenerScope::Primary, @@ -179,6 +240,7 @@ mod tests { address: docker, scope: GatewayListenerScope::ComputeDriverCallback, }], + provenance: None, }] ); } @@ -188,7 +250,8 @@ mod tests { let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); let docker: SocketAddr = "172.18.0.1:8080".parse().unwrap(); let loopback: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let [spec] = gateway_listener_specs(primary, &[docker]) + let [spec] = gateway_listener_specs(primary, &[docker_listener_requirement(docker)]) + .unwrap() .try_into() .unwrap(); @@ -206,24 +269,63 @@ mod tests { fn gateway_listener_specs_preserve_driver_callback_scope() { let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); let docker: SocketAddr = "172.18.0.1:8080".parse().unwrap(); + let requirements = [ + docker_listener_requirement(docker), + docker_listener_requirement(docker), + ]; assert_eq!( - gateway_listener_specs(primary, &[docker, docker]), + gateway_listener_specs(primary, &requirements).unwrap(), vec![ GatewayListenerSpec { address: primary, scope: GatewayListenerScope::Primary, covered_addresses: Vec::new(), + provenance: None, }, GatewayListenerSpec { address: docker, scope: GatewayListenerScope::ComputeDriverCallback, covered_addresses: Vec::new(), + provenance: Some(GatewayListenerProvenance { + driver_name: "docker".to_string(), + reason: "managed bridge".to_string(), + }), }, ] ); } + #[test] + fn gateway_listener_specs_reject_unauthorized_external_driver() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + let requirement = GatewayListenerRequirement { + address: "172.18.0.1:8080".parse().unwrap(), + driver_name: "external-test".to_string(), + reason: "external bridge".to_string(), + }; + + let err = gateway_listener_specs(primary, &[requirement]).unwrap_err(); + assert!(err.to_string().contains("not authorized")); + } + + #[test] + fn gateway_listener_specs_reject_invalid_exact_addresses() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + for address in [ + "0.0.0.0:8080", + "224.0.0.1:8080", + "172.18.0.1:0", + "172.18.0.1:9090", + ] { + let requirement = docker_listener_requirement(address.parse().unwrap()); + assert!( + gateway_listener_specs(primary, &[requirement]).is_err(), + "{address} should be rejected" + ); + } + } + #[tokio::test] async fn failed_bind_does_not_return_partially_bound_listeners() { let occupied_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -232,7 +334,11 @@ mod tests { let primary_address: SocketAddr = "127.0.0.1:0".parse().unwrap(); let result: openshell_core::Result<()> = async { - let _listeners = bind_gateway_listeners(primary_address, &[occupied_address]).await?; + let _listeners = bind_gateway_listeners( + primary_address, + &[docker_listener_requirement(occupied_address)], + ) + .await?; continuation_reached.store(true, Ordering::SeqCst); Ok(()) } @@ -247,4 +353,12 @@ mod tests { "binding must fail before returning a partial listener set" ); } + + fn docker_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { + GatewayListenerRequirement { + address, + driver_name: "docker".to_string(), + reason: "managed bridge".to_string(), + } + } } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index e079f460e..c3b4040c4 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -71,7 +71,7 @@ use tracing::{debug, error, info, warn}; #[cfg(test)] pub(crate) static TEST_ENV_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); -use compute::ComputeRuntime; +use compute::{ComputeRuntime, GatewayListenerRequirement}; #[cfg(test)] use gateway_listener::GatewayListenerSpec; use gateway_listener::{BoundGatewayListener, GatewayListenerScope, bind_gateway_listeners}; @@ -443,8 +443,11 @@ pub(crate) async fn run_server( // snapshot on its first poll. ensure_default_workspace(&store).await?; - let gateway_listeners = - bind_gateway_listeners(config.bind_address, state.compute.gateway_bind_addresses()).await?; + let gateway_listeners = bind_gateway_listeners( + config.bind_address, + state.compute.gateway_listener_requirements(), + ) + .await?; if let Err(err) = state.compute.resume_persisted_sandboxes().await { warn!(error = %err, "Failed to resume persisted sandboxes during startup"); @@ -1013,6 +1016,7 @@ mod tests { allow_plaintext_service_http, bind_gateway_listeners, classify_initial_bytes, configured_compute_driver, is_benign_tls_handshake_failure, kubernetes_sandbox_jwt_expiry_disabled, serve_gateway_listener, + GatewayListenerRequirement, }; use openshell_core::{ ComputeDriverKind, Config, @@ -1495,7 +1499,11 @@ mod tests { let primary_address: SocketAddr = "127.0.0.1:0".parse().unwrap(); let result: openshell_core::Result<()> = async { - let _listeners = bind_gateway_listeners(primary_address, &[occupied_address]).await?; + let _listeners = bind_gateway_listeners( + primary_address, + &[docker_listener_requirement(occupied_address)], + ) + .await?; resume_attempted.store(true, Ordering::SeqCst); Ok(()) } @@ -1510,4 +1518,12 @@ mod tests { "persisted sandbox resume must not run before every gateway listener is bound" ); } + + fn docker_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { + GatewayListenerRequirement { + address, + driver_name: "docker".to_string(), + reason: "managed bridge".to_string(), + } + } } diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs index 9cd80d6ed..2bfa9998a 100644 --- a/crates/openshell-server/src/test_support.rs +++ b/crates/openshell-server/src/test_support.rs @@ -8,10 +8,12 @@ use futures::{Stream, stream}; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverSandbox, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, - GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, - StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, - WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_server::ComputeDriver, + DriverSandbox, GatewayListenerRequirement, GetCapabilitiesRequest, GetCapabilitiesResponse, + GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, + GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, + StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, + compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, }; use std::collections::HashMap; #[cfg(unix)] @@ -33,6 +35,7 @@ type WatchStream = Pin #[derive(Debug, Clone, PartialEq)] pub enum FakeComputeDriverCall { GetCapabilities, + GetGatewayListenerRequirements, ValidateSandboxCreate { sandbox: Option, }, @@ -65,6 +68,8 @@ struct FakeComputeDriverState { driver_name: String, driver_version: String, default_image: String, + gateway_listener_requirements: Vec, + gateway_listener_requirements_supported: bool, sandboxes: HashMap, calls: Vec, } @@ -83,6 +88,8 @@ impl FakeComputeDriver { driver_name: "fake-compute-driver".to_string(), driver_version: "test".to_string(), default_image: "openshell/sandbox:test".to_string(), + gateway_listener_requirements: Vec::new(), + gateway_listener_requirements_supported: true, sandboxes: HashMap::new(), calls: Vec::new(), })), @@ -107,6 +114,29 @@ impl FakeComputeDriver { self } + #[must_use] + pub fn with_gateway_listener_requirement( + self, + bind_address: impl Into, + reason: impl Into, + ) -> Self { + self.with_state(|state| { + state + .gateway_listener_requirements + .push(GatewayListenerRequirement { + reason: reason.into(), + selector: Some(Selector::ExactBindAddress(bind_address.into())), + }); + }); + self + } + + #[must_use] + pub fn without_gateway_listener_requirements_api(self) -> Self { + self.with_state(|state| state.gateway_listener_requirements_supported = false); + self + } + #[must_use] pub fn calls(&self) -> Vec { self.with_state(|state| state.calls.clone()) @@ -194,6 +224,24 @@ impl ComputeDriver for FakeComputeDriver { Ok(Response::new(response)) } + async fn get_gateway_listener_requirements( + &self, + _request: Request, + ) -> Result, Status> { + self.with_state(|state| { + state + .calls + .push(FakeComputeDriverCall::GetGatewayListenerRequirements); + state + .gateway_listener_requirements_supported + .then(|| GetGatewayListenerRequirementsResponse { + requirements: state.gateway_listener_requirements.clone(), + }) + .map(Response::new) + .ok_or_else(|| Status::unimplemented("listener requirements unsupported")) + }) + } + async fn validate_sandbox_create( &self, request: Request, diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index c99b7756b..7eef98e0c 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -20,6 +20,13 @@ service ComputeDriver { // Report driver capabilities and defaults. rpc GetCapabilities(GetCapabilitiesRequest) returns (GetCapabilitiesResponse); + // Report additional gateway listeners required by this driver instance. + // + // A requirement is not authorization to expose the gateway. The gateway + // owns validation, authorization, and the authoritative bind. + rpc GetGatewayListenerRequirements(GetGatewayListenerRequirementsRequest) + returns (GetGatewayListenerRequirementsResponse); + // Validate a sandbox before create-time provisioning. rpc ValidateSandboxCreate(ValidateSandboxCreateRequest) returns (ValidateSandboxCreateResponse); @@ -57,6 +64,23 @@ message GetCapabilitiesResponse { string default_image = 3; } +message GetGatewayListenerRequirementsRequest {} + +message GatewayListenerRequirement { + // Untrusted human-readable driver rationale for diagnostics. + string reason = 1; + + oneof selector { + // Concrete IP:port address requested by the driver. The POC requires the + // port to match the gateway's fixed primary listener port. + string exact_bind_address = 2; + } +} + +message GetGatewayListenerRequirementsResponse { + repeated GatewayListenerRequirement requirements = 1; +} + // Driver-owned sandbox model used for create requests and platform observations. // // This intentionally omits gateway-owned lifecycle fields such as the public From 6b51f125fdfcecd64ce87b71a60bcb62efd65db3 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 27 Jul 2026 13:58:50 +0200 Subject: [PATCH 02/24] feat(compute): add Podman listener requirements Signed-off-by: Evan Lezar --- Cargo.lock | 1 + architecture/gateway.md | 4 +- crates/openshell-driver-podman/Cargo.toml | 1 + crates/openshell-driver-podman/NETWORKING.md | 17 + crates/openshell-driver-podman/src/client.rs | 20 + crates/openshell-driver-podman/src/driver.rs | 230 +++++++++- crates/openshell-driver-podman/src/grpc.rs | 5 +- crates/openshell-server/src/compute/mod.rs | 78 +++- .../openshell-server/src/gateway_listener.rs | 411 ++++++++++++++++-- crates/openshell-server/src/lib.rs | 6 +- proto/compute_driver.proto | 9 + 11 files changed, 715 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f028df5f4..9f454e3e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3976,6 +3976,7 @@ dependencies = [ "tonic", "tracing", "tracing-subscriber", + "url", ] [[package]] diff --git a/architecture/gateway.md b/architecture/gateway.md index 5a84177e1..a95ee171a 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -630,7 +630,9 @@ system entry instead of pretending to delete package-manager owned state. - Podman-backed macOS gateways use gvproxy's host-loopback IP for sandbox host aliases by default so stale Podman machine images do not need Podman's `host-gateway` resolver. Linux Podman keeps the resolver unless - `host_gateway_ip` is configured. + `host_gateway_ip` is configured. Rootful Podman can request its exact bridge + gateway listener; rootless pasta requests the private IPv4 source selected by + the host default route rather than an arbitrary private interface. - Gateway restarts recover persisted objects from storage, but live relay streams must be re-established by supervisors. - User-facing behavior changes must update published docs in `docs/`; this file diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index ed798c0ab..5f108fd9c 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -34,6 +34,7 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } thiserror = { workspace = true } miette = { workspace = true } +url = { workspace = true } [dev-dependencies] prost-types = { workspace = true } diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 93c5ba096..62761b60b 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -255,6 +255,23 @@ if config.grpc_endpoint.is_empty() { The bridge gateway IP is not a stable substitute in rootless mode because it can live inside the user namespace rather than on the host. +Before the gateway binds its serving sockets, the driver reports the callback +listener required by the selected topology: + +- Rootful Linux Podman reports the configured bridge's gateway address exactly. +- Rootless Linux Podman using pasta requests the private IPv4 source address + selected by the host's default route. This matches pasta's default upstream + interface without guessing among private interfaces on a multihomed host. +- Podman Machine requests IPv4 loopback because gvproxy terminates the host + forwarding path there. +- An explicitly remote callback endpoint requests no additional local listener. + +On Linux, an explicit `host_gateway_ip` is reported exactly because the driver +maps both local callback aliases to that literal. Podman Machine still requests +gateway loopback because its configured address is guest-visible and gvproxy +terminates that route on host loopback. The gateway validates and binds every +accepted callback listener; the primary listener can remain on loopback. + ### Layer 3 Inner Sandbox Network Namespace Inside the container, the supervisor creates another network namespace for the diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 952938d47..5291f09ea 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -262,6 +262,8 @@ pub struct HostInfo { #[serde(default)] pub network_backend: String, #[serde(default)] + pub rootless_network_cmd: String, + #[serde(default)] pub security: SecurityInfo, } @@ -902,6 +904,24 @@ mod tests { assert!(validate_name(&exact_name).is_ok()); } + #[test] + fn system_info_parses_rootless_network_helper() { + let info: SystemInfo = serde_json::from_str( + r#"{ + "host": { + "cgroupVersion": "v2", + "networkBackend": "netavark", + "rootlessNetworkCmd": "pasta", + "security": {"rootless": true} + } + }"#, + ) + .unwrap(); + + assert!(info.host.security.rootless); + assert_eq!(info.host.rootless_network_cmd, "pasta"); + } + #[tokio::test] async fn inspect_volume_parses_driver_options() { let (socket_path, request_log, handle) = spawn_podman_stub( diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index aa3df9cbd..f8b0351bc 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -16,13 +16,21 @@ use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, effective_driver_gpu_count, validate_specific_gpu_device_request, }; +#[cfg(target_os = "linux")] +use openshell_core::proto::compute::v1::GatewayDefaultRouteInterfaceRequirement; +#[cfg(target_os = "macos")] +use openshell_core::proto::compute::v1::GatewayLoopbackInterfaceRequirement; use openshell_core::proto::compute::v1::{ - DriverSandbox, GetCapabilitiesResponse, GpuResourceRequirements, + DriverSandbox, GatewayListenerRequirement, GetCapabilitiesResponse, GpuResourceRequirements, + gateway_listener_requirement::Selector, }; +#[cfg(target_os = "linux")] +use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; use tracing::{debug, info, warn}; +use url::Url; impl From for ComputeDriverError { fn from(value: PodmanApiError) -> Self { @@ -39,9 +47,13 @@ impl From for ComputeDriverError { pub struct PodmanComputeDriver { client: PodmanClient, config: PodmanComputeConfig, - /// The host's IP on the bridge network. Sandbox containers use this to - /// reach the gateway server when no explicit gRPC endpoint is configured. + /// The host's IP on the bridge network, when that bridge exists in the + /// gateway's network namespace (notably rootful Podman). network_gateway_ip: Option, + /// Whether Podman's service is running without root privileges. + rootless: bool, + /// Rootless network helper reported by Podman, such as `pasta`. + rootless_network_cmd: String, gpu_selector: Arc, gpu_inventory_refresh: Arc (CdiGpuInventory, bool) + Send + Sync>, } @@ -52,6 +64,8 @@ impl std::fmt::Debug for PodmanComputeDriver { .field("socket_path", &self.config.socket_path) .field("default_image", &self.config.default_image) .field("network_name", &self.config.network_name) + .field("rootless", &self.rootless) + .field("rootless_network_cmd", &self.rootless_network_cmd) .field("gpu_inventory", &self.gpu_selector.device_ids()) .finish() } @@ -289,7 +303,7 @@ impl PodmanComputeDriver { } // Verify cgroups v2, detect rootless mode, and log system info. - match client.system_info().await { + let (rootless, rootless_network_cmd) = match client.system_info().await { Ok(info) => { if info.host.cgroup_version != "v2" { return Err(PodmanApiError::Connection(format!( @@ -303,15 +317,17 @@ impl PodmanComputeDriver { cgroup_version = %info.host.cgroup_version, network_backend = %info.host.network_backend, rootless = info.host.security.rootless, + rootless_network_cmd = %info.host.rootless_network_cmd, "Connected to Podman" ); + (info.host.security.rootless, info.host.rootless_network_cmd) } Err(e) => { return Err(PodmanApiError::Connection(format!( "failed to query Podman system info: {e}" ))); } - } + }; // Rootless pre-flight: warn if subuid/subgid ranges look missing. // Not a hard error because some systems configure these via LDAP or @@ -368,6 +384,8 @@ impl PodmanComputeDriver { client, config, network_gateway_ip, + rootless, + rootless_network_cmd, gpu_selector: Arc::new(CdiGpuDefaultSelector::new( gpu_inventory, allow_all_default_gpu, @@ -378,8 +396,8 @@ impl PodmanComputeDriver { /// The host's IP on the bridge network, if available. /// - /// Used by the server to auto-detect the gRPC callback endpoint when - /// no explicit `--grpc-endpoint` is configured. + /// Used to request the exact rootful gateway callback listener when no + /// explicit host-gateway override is configured. #[must_use] pub fn network_gateway_ip(&self) -> Option<&str> { self.network_gateway_ip.as_deref() @@ -394,6 +412,90 @@ impl PodmanComputeDriver { )) } + /// Report the gateway exposure needed by Podman's standard local callback aliases. + /// + /// Rootful Podman binds the exact bridge address behind the sandbox alias. + /// Rootless pasta follows the host's default-route interface, while Podman + /// Machine forwards the alias to gateway loopback. + pub fn gateway_listener_requirements( + &self, + ) -> Result, ComputeDriverError> { + let endpoint = Url::parse(&self.config.grpc_endpoint).map_err(|err| { + ComputeDriverError::Precondition(format!( + "invalid Podman gateway callback endpoint '{}': {err}", + self.config.grpc_endpoint + )) + })?; + let uses_local_callback_alias = endpoint.host_str().is_some_and(|host| { + matches!(host, "host.containers.internal" | "host.openshell.internal") + }); + if !uses_local_callback_alias { + return Ok(Vec::new()); + } + + #[cfg(target_os = "linux")] + { + if self.config.host_gateway_ip.trim().is_empty() + && self.rootless + && self.rootless_network_cmd == "pasta" + { + return Ok(vec![GatewayListenerRequirement { + reason: "Podman rootless pasta callback uses the host default-route interface" + .to_string(), + selector: Some(Selector::DefaultRouteInterface( + GatewayDefaultRouteInterfaceRequirement {}, + )), + }]); + } + if self.config.host_gateway_ip.trim().is_empty() && self.rootless { + return Ok(Vec::new()); + } + + let gateway_ip = if self.config.host_gateway_ip.trim().is_empty() { + self.network_gateway_ip.as_deref().ok_or_else(|| { + ComputeDriverError::Precondition(format!( + "Podman network '{}' did not report a host bridge gateway address for local callback alias '{}'", + self.config.network_name, + endpoint.host_str().unwrap_or_default() + )) + })? + } else { + self.config.host_gateway_ip.trim() + }; + let gateway_ip = gateway_ip.parse::().map_err(|err| { + ComputeDriverError::Precondition(format!( + "Podman callback gateway address '{gateway_ip}' is invalid: {err}" + )) + })?; + let port = endpoint.port_or_known_default().ok_or_else(|| { + ComputeDriverError::Precondition(format!( + "Podman gateway callback endpoint '{}' has no port", + self.config.grpc_endpoint + )) + })?; + Ok(vec![GatewayListenerRequirement { + reason: format!("Podman network '{}' host gateway", self.config.network_name), + selector: Some(Selector::ExactBindAddress( + SocketAddr::new(gateway_ip, port).to_string(), + )), + }]) + } + #[cfg(target_os = "macos")] + { + Ok(vec![GatewayListenerRequirement { + reason: "Podman machine callback forwarding terminates on gateway loopback" + .to_string(), + selector: Some(Selector::LoopbackInterface( + GatewayLoopbackInterfaceRequirement {}, + )), + }]) + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + Ok(Vec::new()) + } + } + #[must_use] pub fn default_image(&self) -> &str { &self.config.default_image @@ -892,6 +994,8 @@ impl PodmanComputeDriver { client, config, network_gateway_ip: None, + rootless: false, + rootless_network_cmd: String::new(), gpu_selector: Arc::new(CdiGpuDefaultSelector::new( gpu_inventory, allow_all_default_gpu, @@ -1168,6 +1272,118 @@ mod tests { assert_eq!(cfg.grpc_endpoint, "https://gateway.internal:9000"); } + #[test] + #[cfg(target_os = "linux")] + fn rootful_local_callback_alias_requests_discovered_network_gateway() { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + ..PodmanComputeConfig::default() + }); + driver.network_gateway_ip = Some("10.89.1.1".to_string()); + + let requirements = driver.gateway_listener_requirements().unwrap(); + + assert_eq!(requirements.len(), 1); + assert_eq!( + requirements[0].selector, + Some(Selector::ExactBindAddress("10.89.1.1:17670".to_string())) + ); + } + + #[test] + #[cfg(target_os = "linux")] + fn configured_host_gateway_overrides_discovered_network_gateway() { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.containers.internal:17670".to_string(), + host_gateway_ip: "10.90.1.1".to_string(), + ..PodmanComputeConfig::default() + }); + driver.network_gateway_ip = Some("10.89.1.1".to_string()); + driver.rootless = true; + driver.rootless_network_cmd = "pasta".to_string(); + + let requirements = driver.gateway_listener_requirements().unwrap(); + + assert_eq!( + requirements[0].selector, + Some(Selector::ExactBindAddress("10.90.1.1:17670".to_string())) + ); + } + + #[test] + #[cfg(target_os = "linux")] + fn rootless_pasta_requests_default_route_interface() { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + ..PodmanComputeConfig::default() + }); + driver.rootless = true; + driver.rootless_network_cmd = "pasta".to_string(); + + let requirements = driver.gateway_listener_requirements().unwrap(); + + assert!(matches!( + requirements[0].selector, + Some(Selector::DefaultRouteInterface(_)) + )); + } + + #[test] + #[cfg(target_os = "linux")] + fn rootless_non_pasta_does_not_request_additional_listener() { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + ..PodmanComputeConfig::default() + }); + driver.rootless = true; + driver.rootless_network_cmd = "slirp4netns".to_string(); + + assert!(driver.gateway_listener_requirements().unwrap().is_empty()); + } + + #[test] + #[cfg(target_os = "linux")] + fn rootful_local_callback_alias_requires_concrete_gateway_address() { + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + ..PodmanComputeConfig::default() + }); + + let err = driver.gateway_listener_requirements().unwrap_err(); + + assert!( + err.to_string() + .contains("did not report a host bridge gateway address") + ); + } + + #[test] + #[cfg(target_os = "macos")] + fn podman_machine_callback_alias_requests_loopback_listener() { + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + ..PodmanComputeConfig::default() + }); + + let requirements = driver.gateway_listener_requirements().unwrap(); + + assert_eq!(requirements.len(), 1); + assert!(matches!( + requirements[0].selector, + Some(Selector::LoopbackInterface(_)) + )); + } + + #[test] + fn explicit_remote_callback_does_not_request_gateway_listener() { + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "https://gateway.example.test:17670".to_string(), + ..PodmanComputeConfig::default() + }); + + assert!(driver.gateway_listener_requirements().unwrap().is_empty()); + } + #[test] fn local_podman_cdi_gpu_inventory_maps_nvidia_device_nodes() { let root = std::env::temp_dir().join(format!( diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index 4d04db54b..2d0792d44 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -46,7 +46,10 @@ impl ComputeDriver for ComputeDriverService { _request: Request, ) -> Result, Status> { Ok(Response::new(GetGatewayListenerRequirementsResponse { - requirements: Vec::new(), + requirements: self + .driver + .gateway_listener_requirements() + .map_err(Status::from)?, })) } diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 0b239d15d..5d3dda0a7 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -69,10 +69,38 @@ type SharedComputeDriver = const DELETE_PHASE_CAS_RETRY_LIMIT: usize = 3; #[derive(Clone, Debug, Eq, PartialEq)] -pub struct GatewayListenerRequirement { - pub address: SocketAddr, - pub driver_name: String, - pub reason: String, +pub enum GatewayListenerRequirement { + Exact { + address: SocketAddr, + driver_name: String, + reason: String, + }, + DefaultRouteInterface { + driver_name: String, + reason: String, + }, + LoopbackInterface { + driver_name: String, + reason: String, + }, +} + +impl GatewayListenerRequirement { + pub fn driver_name(&self) -> &str { + match self { + Self::Exact { driver_name, .. } + | Self::DefaultRouteInterface { driver_name, .. } + | Self::LoopbackInterface { driver_name, .. } => driver_name, + } + } + + pub fn reason(&self) -> &str { + match self { + Self::Exact { reason, .. } + | Self::DefaultRouteInterface { reason, .. } + | Self::LoopbackInterface { reason, .. } => reason, + } + } } /// Serializes request-side deletes for the same stable sandbox ID. @@ -441,21 +469,37 @@ impl ComputeRuntime { .requirements .into_iter() .map(|requirement: ProtoGatewayListenerRequirement| { - let Some(Selector::ExactBindAddress(bind_address)) = requirement.selector else { + let Some(selector) = requirement.selector else { return Err(ComputeError::Message(format!( - "compute driver '{driver_name}' returned a gateway listener requirement without an exact bind address" + "compute driver '{driver_name}' returned a gateway listener requirement without a selector" ))); }; - let address = bind_address.parse::().map_err(|err| { - ComputeError::Message(format!( - "compute driver '{driver_name}' returned invalid gateway listener address '{bind_address}': {err}" - )) - })?; - Ok(GatewayListenerRequirement { - address, - driver_name: driver_name.clone(), - reason: requirement.reason, - }) + match selector { + Selector::ExactBindAddress(bind_address) => { + let address = bind_address.parse::().map_err(|err| { + ComputeError::Message(format!( + "compute driver '{driver_name}' returned invalid gateway listener address '{bind_address}': {err}" + )) + })?; + Ok(GatewayListenerRequirement::Exact { + address, + driver_name: driver_name.clone(), + reason: requirement.reason, + }) + } + Selector::DefaultRouteInterface(_) => { + Ok(GatewayListenerRequirement::DefaultRouteInterface { + driver_name: driver_name.clone(), + reason: requirement.reason, + }) + } + Selector::LoopbackInterface(_) => { + Ok(GatewayListenerRequirement::LoopbackInterface { + driver_name: driver_name.clone(), + reason: requirement.reason, + }) + } + } }) .collect::, ComputeError>>()?, Err(status) if status.code() == Code::Unimplemented => { @@ -6184,7 +6228,7 @@ mod tests { .unwrap(); assert_eq!( runtime.gateway_listener_requirements(), - &[GatewayListenerRequirement { + &[GatewayListenerRequirement::Exact { address: "172.19.0.1:17670".parse().unwrap(), driver_name: "external-test".to_string(), reason: "external driver managed bridge".to_string(), diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index 21100de52..dd6c7c151 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -3,7 +3,7 @@ use crate::compute::GatewayListenerRequirement; use openshell_core::{ComputeDriverKind, Error, Result}; -use std::net::SocketAddr; +use std::net::{IpAddr, SocketAddr}; use tokio::net::TcpListener; use tracing::info; @@ -70,57 +70,149 @@ impl GatewayListenerSpec { fn gateway_listener_specs( bind_address: SocketAddr, requirements: &[GatewayListenerRequirement], +) -> Result> { + let needs_default_route_resolution = requirements.iter().any(|requirement| { + matches!( + requirement, + GatewayListenerRequirement::DefaultRouteInterface { .. } + ) + }); + let default_route_ip = if needs_default_route_resolution { + Some(gateway_default_route_ip()?) + } else { + None + }; + gateway_listener_specs_with_default_route_ip(bind_address, requirements, default_route_ip) +} + +fn gateway_listener_specs_with_default_route_ip( + bind_address: SocketAddr, + requirements: &[GatewayListenerRequirement], + default_route_ip: Option, ) -> Result> { let mut specs = vec![GatewayListenerSpec::new( bind_address, GatewayListenerScope::Primary, )]; + + // Resolve exact requirements first so they can satisfy a later semantic + // requirement regardless of driver response ordering. for requirement in requirements { + let GatewayListenerRequirement::Exact { address, .. } = requirement else { + continue; + }; validate_gateway_listener_requirement(bind_address, requirement)?; - let address = requirement.address; - let scope = GatewayListenerScope::ComputeDriverCallback; - if let Some(existing) = specs - .iter() - .position(|existing| listener_covers(existing.address, address)) - { - let existing = &mut specs[existing]; - if existing.address != address - && !existing - .covered_addresses - .iter() - .any(|covered| covered.address == address) - { - existing.covered_addresses.push(CoveredGatewayAddress { - address, - scope, - }); - } - } else { - specs.push(GatewayListenerSpec { - address, - scope, - covered_addresses: Vec::new(), - provenance: Some(GatewayListenerProvenance { - driver_name: requirement.driver_name.clone(), - reason: requirement.reason.clone(), - }), - }); + add_callback_listener_spec(&mut specs, *address, requirement); + } + + for requirement in requirements { + let GatewayListenerRequirement::DefaultRouteInterface { .. } = requirement else { + continue; + }; + validate_gateway_listener_requirement(bind_address, requirement)?; + let Some(ip) = default_route_ip else { + return Err(Error::config(format!( + "compute driver '{}' requested the gateway default-route interface, but no IPv4 source address was resolved (reason: {})", + requirement.driver_name(), + requirement.reason() + ))); + }; + if !gateway_default_route_ip_is_usable(ip) { + return Err(Error::config(format!( + "compute driver '{}' requested the gateway default-route interface, but its resolved address {ip} is not a private IPv4 address (reason: {})", + requirement.driver_name(), + requirement.reason() + ))); } + let address = SocketAddr::new(ip, bind_address.port()); + validate_resolved_gateway_listener(bind_address, address)?; + add_callback_listener_spec(&mut specs, address, requirement); } + + for requirement in requirements { + let GatewayListenerRequirement::LoopbackInterface { .. } = requirement else { + continue; + }; + validate_gateway_listener_requirement(bind_address, requirement)?; + let address = SocketAddr::from(([127, 0, 0, 1], bind_address.port())); + validate_resolved_gateway_listener(bind_address, address)?; + add_callback_listener_spec(&mut specs, address, requirement); + } + Ok(specs) } +fn add_callback_listener_spec( + specs: &mut Vec, + address: SocketAddr, + requirement: &GatewayListenerRequirement, +) { + let scope = GatewayListenerScope::ComputeDriverCallback; + if let Some(existing) = specs + .iter_mut() + .find(|existing| listener_covers(existing.address, address)) + { + if existing.address != address + && !existing + .covered_addresses + .iter() + .any(|covered| covered.address == address) + { + existing + .covered_addresses + .push(CoveredGatewayAddress { address, scope }); + } + return; + } + specs.push(callback_listener_spec(address, requirement)); +} + +fn callback_listener_spec( + address: SocketAddr, + requirement: &GatewayListenerRequirement, +) -> GatewayListenerSpec { + GatewayListenerSpec { + address, + scope: GatewayListenerScope::ComputeDriverCallback, + covered_addresses: Vec::new(), + provenance: Some(GatewayListenerProvenance { + driver_name: requirement.driver_name().to_string(), + reason: requirement.reason().to_string(), + }), + } +} + fn validate_gateway_listener_requirement( primary_listener: SocketAddr, requirement: &GatewayListenerRequirement, ) -> Result<()> { - let requested_listener = requirement.address; - if requirement.driver_name != ComputeDriverKind::Docker.as_str() { - return Err(Error::config(format!( - "compute driver '{}' is not authorized to request gateway listeners in this proof of concept", - requirement.driver_name - ))); + match requirement { + GatewayListenerRequirement::Exact { + address, + driver_name, + .. + } if driver_name == ComputeDriverKind::Docker.as_str() + || driver_name == ComputeDriverKind::Podman.as_str() => + { + validate_resolved_gateway_listener(primary_listener, *address) + } + GatewayListenerRequirement::DefaultRouteInterface { driver_name, .. } + | GatewayListenerRequirement::LoopbackInterface { driver_name, .. } + if driver_name == ComputeDriverKind::Podman.as_str() => + { + Ok(()) + } + _ => Err(Error::config(format!( + "compute driver '{}' is not authorized for this gateway listener requirement in this proof of concept", + requirement.driver_name() + ))), } +} + +fn validate_resolved_gateway_listener( + primary_listener: SocketAddr, + requested_listener: SocketAddr, +) -> Result<()> { if requested_listener.ip().is_unspecified() { return Err(Error::config(format!( "compute driver requested wildcard gateway listener {requested_listener}" @@ -146,6 +238,35 @@ fn validate_gateway_listener_requirement( Ok(()) } +fn gateway_default_route_ip_is_usable(address: IpAddr) -> bool { + matches!(address, IpAddr::V4(address) if address.is_private()) +} + +#[cfg(target_os = "linux")] +fn gateway_default_route_ip() -> Result { + // UDP connect performs a local route lookup without sending a packet. The + // selected source address follows the IPv4 default route, matching pasta's + // default upstream-interface selection. + let socket = + std::net::UdpSocket::bind((std::net::Ipv4Addr::UNSPECIFIED, 0)).map_err(|err| { + Error::config(format!("failed to open default-route probe socket: {err}")) + })?; + socket + .connect((std::net::Ipv4Addr::new(192, 0, 2, 1), 9)) + .map_err(|err| Error::config(format!("failed to resolve IPv4 default route: {err}")))?; + socket + .local_addr() + .map(|address| address.ip()) + .map_err(|err| Error::config(format!("failed to read IPv4 default-route address: {err}"))) +} + +#[cfg(not(target_os = "linux"))] +fn gateway_default_route_ip() -> Result { + Err(Error::config( + "default-route gateway listener requirements are supported only on Linux", + )) +} + pub async fn bind_gateway_listeners( bind_address: SocketAddr, requirements: &[GatewayListenerRequirement], @@ -205,8 +326,8 @@ fn listener_covers(existing: SocketAddr, requested: SocketAddr) -> bool { } match (existing.ip(), requested.ip()) { - (std::net::IpAddr::V4(existing), std::net::IpAddr::V4(_)) => existing.is_unspecified(), - (std::net::IpAddr::V6(existing), std::net::IpAddr::V6(_)) => existing.is_unspecified(), + (IpAddr::V4(existing), IpAddr::V4(_)) => existing.is_unspecified(), + (IpAddr::V6(existing), IpAddr::V6(_)) => existing.is_unspecified(), _ => false, } } @@ -216,6 +337,7 @@ mod tests { use super::{ CoveredGatewayAddress, GatewayListenerProvenance, GatewayListenerScope, GatewayListenerSpec, bind_gateway_listeners, gateway_listener_specs, + gateway_listener_specs_with_default_route_ip, }; use crate::compute::GatewayListenerRequirement; use std::net::SocketAddr; @@ -299,7 +421,7 @@ mod tests { #[test] fn gateway_listener_specs_reject_unauthorized_external_driver() { let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let requirement = GatewayListenerRequirement { + let requirement = GatewayListenerRequirement::Exact { address: "172.18.0.1:8080".parse().unwrap(), driver_name: "external-test".to_string(), reason: "external bridge".to_string(), @@ -326,6 +448,155 @@ mod tests { } } + #[test] + fn gateway_listener_specs_use_exact_podman_network_gateway() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + let podman_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); + + assert_eq!( + gateway_listener_specs(primary, &[podman_listener_requirement(podman_gateway)]) + .unwrap(), + vec![ + primary_listener_spec(primary), + callback_listener_spec(podman_gateway, "podman", "Podman managed bridge",), + ] + ); + } + + #[test] + fn gateway_listener_specs_track_podman_exact_when_primary_covers_it() { + let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); + let podman_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); + + assert_eq!( + gateway_listener_specs( + primary, + &[podman_listener_requirement(podman_gateway)], + ) + .unwrap(), + vec![primary_listener_spec_with_covered( + primary, + podman_gateway, + )] + ); + } + + #[test] + fn gateway_listener_specs_resolve_podman_default_route_source() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + let default_route_ip = "192.168.20.20".parse().unwrap(); + + assert_eq!( + gateway_listener_specs_with_default_route_ip( + primary, + &[podman_default_route_listener_requirement()], + Some(default_route_ip), + ) + .unwrap(), + vec![ + primary_listener_spec(primary), + callback_listener_spec( + "192.168.20.20:8080".parse().unwrap(), + "podman", + "rootless pasta upstream interface", + ), + ] + ); + } + + #[test] + fn gateway_listener_specs_reject_public_default_route_source() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + + let err = gateway_listener_specs_with_default_route_ip( + primary, + &[podman_default_route_listener_requirement()], + Some("203.0.113.20".parse().unwrap()), + ) + .unwrap_err(); + + assert!(err.to_string().contains("not a private IPv4 address")); + } + + #[test] + fn gateway_listener_specs_track_default_route_when_primary_is_ipv4_wildcard() { + let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); + let default_route_ip = "192.168.20.20".parse().unwrap(); + let callback = "192.168.20.20:8080".parse().unwrap(); + + assert_eq!( + gateway_listener_specs_with_default_route_ip( + primary, + &[podman_default_route_listener_requirement()], + Some(default_route_ip), + ) + .unwrap(), + vec![primary_listener_spec_with_covered(primary, callback)] + ); + } + + #[test] + fn gateway_listener_specs_resolve_podman_loopback_separately() { + let primary: SocketAddr = "192.168.20.20:8080".parse().unwrap(); + + assert_eq!( + gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(), + vec![ + primary_listener_spec(primary), + callback_listener_spec( + "127.0.0.1:8080".parse().unwrap(), + "podman", + "Podman machine host forwarder", + ), + ] + ); + } + + #[test] + fn gateway_listener_specs_track_podman_loopback_when_wildcard_primary_covers_it() { + let primary = "0.0.0.0:8080".parse().unwrap(); + let loopback = "127.0.0.1:8080".parse().unwrap(); + + assert_eq!( + gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(), + vec![primary_listener_spec_with_covered(primary, loopback)] + ); + } + + #[test] + fn gateway_listener_specs_skip_podman_loopback_when_primary_is_same_address() { + let primary = "127.0.0.1:8080".parse().unwrap(); + + assert_eq!( + gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(), + vec![primary_listener_spec(primary)] + ); + } + + #[test] + fn gateway_listener_specs_do_not_use_ipv6_listener_for_ipv4_loopback_requirement() { + for primary in ["[::1]:8080", "[::]:8080"] { + let primary = primary.parse().unwrap(); + let specs = + gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(); + + assert_eq!(specs.len(), 2); + assert_eq!(specs[1].address, SocketAddr::from(([127, 0, 0, 1], 8080))); + } + } + + #[test] + fn gateway_listener_specs_reject_cross_driver_selector_authority() { + let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); + let requirement = GatewayListenerRequirement::LoopbackInterface { + driver_name: "docker".to_string(), + reason: "wrong selector".to_string(), + }; + + let err = gateway_listener_specs(primary, &[requirement]).unwrap_err(); + assert!(err.to_string().contains("not authorized")); + } + #[tokio::test] async fn failed_bind_does_not_return_partially_bound_listeners() { let occupied_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -355,10 +626,72 @@ mod tests { } fn docker_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { - GatewayListenerRequirement { + GatewayListenerRequirement::Exact { address, driver_name: "docker".to_string(), reason: "managed bridge".to_string(), } } + + fn podman_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { + GatewayListenerRequirement::Exact { + address, + driver_name: "podman".to_string(), + reason: "Podman managed bridge".to_string(), + } + } + + fn podman_default_route_listener_requirement() -> GatewayListenerRequirement { + GatewayListenerRequirement::DefaultRouteInterface { + driver_name: "podman".to_string(), + reason: "rootless pasta upstream interface".to_string(), + } + } + + fn podman_loopback_listener_requirement() -> GatewayListenerRequirement { + GatewayListenerRequirement::LoopbackInterface { + driver_name: "podman".to_string(), + reason: "Podman machine host forwarder".to_string(), + } + } + + fn primary_listener_spec(address: SocketAddr) -> GatewayListenerSpec { + GatewayListenerSpec { + address, + scope: GatewayListenerScope::Primary, + covered_addresses: Vec::new(), + provenance: None, + } + } + + fn primary_listener_spec_with_covered( + address: SocketAddr, + covered_address: SocketAddr, + ) -> GatewayListenerSpec { + GatewayListenerSpec { + address, + scope: GatewayListenerScope::Primary, + covered_addresses: vec![CoveredGatewayAddress { + address: covered_address, + scope: GatewayListenerScope::ComputeDriverCallback, + }], + provenance: None, + } + } + + fn callback_listener_spec( + address: SocketAddr, + driver_name: &str, + reason: &str, + ) -> GatewayListenerSpec { + GatewayListenerSpec { + address, + scope: GatewayListenerScope::ComputeDriverCallback, + covered_addresses: Vec::new(), + provenance: Some(GatewayListenerProvenance { + driver_name: driver_name.to_string(), + reason: reason.to_string(), + }), + } + } } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index c3b4040c4..61706c47b 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -71,7 +71,9 @@ use tracing::{debug, error, info, warn}; #[cfg(test)] pub(crate) static TEST_ENV_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); -use compute::{ComputeRuntime, GatewayListenerRequirement}; +use compute::ComputeRuntime; +#[cfg(test)] +use compute::GatewayListenerRequirement; #[cfg(test)] use gateway_listener::GatewayListenerSpec; use gateway_listener::{BoundGatewayListener, GatewayListenerScope, bind_gateway_listeners}; @@ -1520,7 +1522,7 @@ mod tests { } fn docker_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { - GatewayListenerRequirement { + GatewayListenerRequirement::Exact { address, driver_name: "docker".to_string(), reason: "managed bridge".to_string(), diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index 7eef98e0c..47d883846 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -74,9 +74,18 @@ message GatewayListenerRequirement { // Concrete IP:port address requested by the driver. The POC requires the // port to match the gateway's fixed primary listener port. string exact_bind_address = 2; + // Ask the gateway to bind the IPv4 address selected by its default route. + // This matches rootless pasta's default upstream-interface selection. + GatewayDefaultRouteInterfaceRequirement default_route_interface = 3; + // Ask the gateway to ensure an IPv4 loopback listener is present. This + // covers runtimes whose host forwarder terminates on gateway loopback. + GatewayLoopbackInterfaceRequirement loopback_interface = 4; } } +message GatewayDefaultRouteInterfaceRequirement {} +message GatewayLoopbackInterfaceRequirement {} + message GetGatewayListenerRequirementsResponse { repeated GatewayListenerRequirement requirements = 1; } From 11a60e324e1fa8388c8b5d12bf93a9524d8f0f6d Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 27 Jul 2026 14:13:32 +0200 Subject: [PATCH 03/24] test(docker): use default gateway bind address Signed-off-by: Evan Lezar --- e2e/with-docker-gateway.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index f6d01cb88..15e9d3466 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -512,7 +512,6 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" - --bind-address 0.0.0.0 --port "${HOST_PORT}" --health-port "${HEALTH_PORT}" --drivers docker From b1aeb29e1c654b239e87dccf0ce21c8061339c7c Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 27 Jul 2026 14:20:37 +0200 Subject: [PATCH 04/24] fix(gateway): avoid wildcard primary listener Signed-off-by: Evan Lezar --- crates/openshell-server/src/config_file.rs | 23 ++++++++-------------- deploy/rpm/CONFIGURATION.md | 21 ++++++++++---------- deploy/rpm/QUICKSTART.md | 10 +++++----- deploy/rpm/TROUBLESHOOTING.md | 7 ++++--- deploy/rpm/gateway.toml.default | 8 +++----- e2e/with-podman-gateway.sh | 12 +++++------ tasks/scripts/gateway.sh | 9 --------- 7 files changed, 36 insertions(+), 54 deletions(-) diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index c4e0cbc95..2980fbe56 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -720,7 +720,8 @@ version = 2 /// `load()` path that the gateway uses at runtime, catching: /// - template corruption or unknown fields (`deny_unknown_fields`) /// - schema drift (version bump or field renames) - /// - accidental changes to the bind address or compute driver list + /// - accidental addition of a wildcard bind-address override + /// - accidental changes to the compute driver list #[test] fn rpm_default_config_parses_and_has_podman_defaults() { let path = @@ -729,20 +730,12 @@ version = 2 load(&path).expect("deploy/rpm/gateway.toml.default must parse against current schema"); let gw = &config.openshell.gateway; - let addr = gw - .bind_address - .expect("bind_address must be explicitly set in the RPM default config"); - assert!( - addr.ip().is_unspecified(), - "RPM default bind_address must be 0.0.0.0 so Podman sandbox containers \ - can reach the gateway over the host network bridge, got {addr}" - ); - assert_eq!( - addr.port(), - openshell_core::config::DEFAULT_SERVER_PORT, - "RPM default port must match DEFAULT_SERVER_PORT ({})", - openshell_core::config::DEFAULT_SERVER_PORT - ); + if let Some(addr) = gw.bind_address { + assert!( + !addr.ip().is_unspecified(), + "RPM default config must not expose the primary listener on every interface" + ); + } let drivers = gw .compute_drivers diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index a144cac8e..4fc18e621 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -20,14 +20,13 @@ The defaults are tuned for rootless Podman use: version = 1 [openshell.gateway] -bind_address = "0.0.0.0:17670" compute_drivers = ["podman"] ``` -`bind_address = "0.0.0.0:17670"` is required because Podman sandbox -containers reach the gateway over the host network bridge and cannot -connect to `127.0.0.1` inside the gateway's network namespace. mTLS is -enabled by default and protects all connections. +The RPM does not override `bind_address`. The primary listener uses the +built-in `127.0.0.1:17670` default. The Podman driver reports the callback +interface it needs, and the gateway adds a separate listener scoped to that +interface. This keeps the general API off unrelated host interfaces. `compute_drivers = ["podman"]` pins the compute driver to Podman. Without this, the gateway auto-detects in order: Kubernetes, Podman, Docker. Pinning @@ -43,8 +42,8 @@ To apply environment variable overrides that persist across upgrades without editing the TOML file, add them to `~/.config/openshell/gateway.env`: ```shell -# Example: restrict to loopback only -OPENSHELL_BIND_ADDRESS=127.0.0.1 +# Example: explicitly expose the primary listener on one host interface +OPENSHELL_BIND_ADDRESS=192.168.1.10 ``` To override the path to the TOML config file entirely: @@ -63,8 +62,9 @@ systemctl --user edit openshell-gateway ## TLS (mTLS) The RPM enables mutual TLS by default. The gateway requires a valid -client certificate for all API connections and listens on -`0.0.0.0:17670` by default (see "Default configuration" above). +client certificate for all API connections. Its primary listener uses +`127.0.0.1:17670`; Podman callback traffic uses the additional listener +described in "Default configuration" above. ### Auto-generated certificates @@ -214,7 +214,7 @@ overrides that persist across package upgrades. | TOML option | Default | Description | |-------------|---------|-------------| -| `bind_address` | `0.0.0.0:17670` (RPM default) | Address for the gRPC/HTTP API. | +| `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. | | `compute_drivers` | `["podman"]` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman. | | `default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | | `supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | @@ -235,7 +235,6 @@ settings: version = 1 [openshell.gateway] -bind_address = "0.0.0.0:17670" compute_drivers = ["podman"] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" diff --git a/deploy/rpm/QUICKSTART.md b/deploy/rpm/QUICKSTART.md index c6634ced9..442458d09 100644 --- a/deploy/rpm/QUICKSTART.md +++ b/deploy/rpm/QUICKSTART.md @@ -65,11 +65,11 @@ On first start, the gateway automatically generates: - A self-signed PKI bundle (CA, server cert, client cert) for mTLS -> **Note:** The RPM default configuration binds to `0.0.0.0:17670` so -> Podman sandbox containers can reach the gateway over the host network -> bridge. Mutual TLS (mTLS) is enabled automatically on first start, -> requiring a valid client certificate for every connection. See -> CONFIGURATION.md for details. +> **Note:** The primary gateway listener uses the loopback default, +> `127.0.0.1:17670`. The Podman driver requests a separate callback listener +> scoped to the interface its sandboxes can reach. Mutual TLS (mTLS) is +> enabled automatically on first start, requiring a valid client certificate +> for every connection. See CONFIGURATION.md for details. Verify the service is running: diff --git a/deploy/rpm/TROUBLESHOOTING.md b/deploy/rpm/TROUBLESHOOTING.md index 68a1f4946..103ce3bf9 100644 --- a/deploy/rpm/TROUBLESHOOTING.md +++ b/deploy/rpm/TROUBLESHOOTING.md @@ -85,7 +85,7 @@ Generate certificates that include the server's hostname or IP in the SANs. See "Using externally-managed certificates" in CONFIGURATION.md. Then change `bind_address` in `~/.config/openshell/gateway.toml` to the interface the remote CLI -can reach, for example `0.0.0.0:17670`, and restart the gateway. +can reach, for example `192.168.1.10:17670`, and restart the gateway. After placing the server and client certs, register from the remote CLI: @@ -274,11 +274,12 @@ Other breaking changes in this release: - **Default bind address changed from `0.0.0.0` to `127.0.0.1`.** If you relied on network-accessible access without an explicit bind - address, add the following to `~/.config/openshell/gateway.toml`: + address, bind the specific reachable interface in + `~/.config/openshell/gateway.toml`: ```toml [openshell.gateway] - bind_address = "0.0.0.0:17670" + bind_address = "192.168.1.10:17670" ``` Also update your firewall rule if applicable: diff --git a/deploy/rpm/gateway.toml.default b/deploy/rpm/gateway.toml.default index d85379964..cd7e0d99c 100644 --- a/deploy/rpm/gateway.toml.default +++ b/deploy/rpm/gateway.toml.default @@ -18,11 +18,9 @@ version = 1 [openshell.gateway] -# Podman sandbox containers reach the gateway over the host network bridge, -# which requires binding to all interfaces. Override to 127.0.0.1:17670 if -# you don't use Podman or want loopback-only access (e.g. behind a reverse -# proxy). mTLS is enabled by default and protects all connections. -bind_address = "0.0.0.0:17670" +# Keep the primary listener on the built-in 127.0.0.1:17670 default. The +# Podman driver reports the callback interface it needs, and the gateway +# adds a separate listener scoped to that interface. # Pin to the Podman compute driver. Without this, the gateway auto-detects # in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index ba04d63b3..8c598c88f 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -416,10 +416,10 @@ toml_string() { GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" # Start from the RPM default template so this e2e test exercises the same -# TOML config path that RPM users get on first start. The template sets -# bind_address = "0.0.0.0:17670" and compute_drivers = ["podman"]; those -# values must be correct for Podman e2e to pass, which means a regression -# to the template (wrong bind address, wrong driver) will surface here. +# TOML config path that RPM users get on first start. The template leaves +# bind_address unset and sets compute_drivers = ["podman"], so this test +# exercises the built-in loopback listener plus the callback listener +# requested by the Podman driver. # # We append the driver-specific table and override the port via CLI flag # (CLI > TOML in the merge precedence) so the test can use an ephemeral port. @@ -458,8 +458,8 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" - # bind_address and compute_drivers come from the RPM template; no CLI flags - # needed. Port is overridden via CLI (CLI > TOML) for ephemeral port selection. + # compute_drivers comes from the RPM template, while bind_address uses the + # built-in loopback default. Override only the port for ephemeral selection. --port "${HOST_PORT}" --health-port "${HEALTH_PORT}" --tls-cert "${PKI_DIR}/server/tls.crt" diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index 3e94afe10..5d3adae2a 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -306,15 +306,6 @@ if [[ "${DRIVER}" == "podman" ]]; then SUPERVISOR_IMAGE="${OPENSHELL_SUPERVISOR_IMAGE:-openshell/supervisor:dev}" ensure_podman_supervisor_image "${SUPERVISOR_IMAGE}" export OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_IMAGE}" - - # Rootless Podman containers reach the host via pasta's local connection - # bypass, which translates to host L4 sockets. The gateway must listen on - # 0.0.0.0 so pasta can reach it — 127.0.0.1 is not routable through pasta. - if [[ -z "${OPENSHELL_BIND_ADDRESS:-}" ]]; then - if podman info --format '{{.Host.Security.Rootless}}' 2>/dev/null | grep -q true; then - export OPENSHELL_BIND_ADDRESS="0.0.0.0" - fi - fi fi if [[ ! "${GATEWAY_NAME}" =~ ^[A-Za-z0-9._-]+$ ]]; then From 46fea27d8383b4602eba6e4983e4a81366a24138 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 28 Jul 2026 10:54:22 +0200 Subject: [PATCH 05/24] fix(podman): validate callback listener discovery Signed-off-by: Evan Lezar --- crates/openshell-driver-podman/NETWORKING.md | 3 + crates/openshell-driver-podman/src/driver.rs | 118 +++++++++++++++---- docs/reference/sandbox-compute-drivers.mdx | 2 +- 3 files changed, 98 insertions(+), 25 deletions(-) diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 62761b60b..7d98496e0 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -262,6 +262,9 @@ listener required by the selected topology: - Rootless Linux Podman using pasta requests the private IPv4 source address selected by the host's default route. This matches pasta's default upstream interface without guessing among private interfaces on a multihomed host. +- Rootless Linux Podman using another helper cannot infer a safe local callback + listener. The driver fails startup unless `host_gateway_ip` is set or + `grpc_endpoint` names an explicitly remote endpoint. - Podman Machine requests IPv4 loopback because gvproxy terminates the host forwarding path there. - An explicitly remote callback endpoint requests no additional local listener. diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index f8b0351bc..a4a21f8a6 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -338,10 +338,7 @@ impl PodmanComputeDriver { // Ensure the bridge network exists. client.ensure_network(&config.network_name).await?; - let network_gateway_ip = client - .network_gateway_ip(&config.network_name) - .await - .unwrap_or(None); + let network_gateway_ip = client.network_gateway_ip(&config.network_name).await?; info!( network = %config.network_name, gateway_ip = ?network_gateway_ip, @@ -435,20 +432,27 @@ impl PodmanComputeDriver { #[cfg(target_os = "linux")] { - if self.config.host_gateway_ip.trim().is_empty() - && self.rootless - && self.rootless_network_cmd == "pasta" - { - return Ok(vec![GatewayListenerRequirement { - reason: "Podman rootless pasta callback uses the host default-route interface" - .to_string(), - selector: Some(Selector::DefaultRouteInterface( - GatewayDefaultRouteInterfaceRequirement {}, - )), - }]); - } if self.config.host_gateway_ip.trim().is_empty() && self.rootless { - return Ok(Vec::new()); + let rootless_network_cmd = self.rootless_network_cmd.trim(); + if rootless_network_cmd == "pasta" { + return Ok(vec![GatewayListenerRequirement { + reason: + "Podman rootless pasta callback uses the host default-route interface" + .to_string(), + selector: Some(Selector::DefaultRouteInterface( + GatewayDefaultRouteInterfaceRequirement {}, + )), + }]); + } + + let reported = if rootless_network_cmd.is_empty() { + "" + } else { + rootless_network_cmd + }; + return Err(ComputeDriverError::Precondition(format!( + "Podman rootless network helper '{reported}' is unsupported for local gateway callback aliases; configure pasta, set host_gateway_ip, or use an explicitly remote grpc_endpoint" + ))); } let gateway_ip = if self.config.host_gateway_ip.trim().is_empty() { @@ -1330,15 +1334,81 @@ mod tests { #[test] #[cfg(target_os = "linux")] - fn rootless_non_pasta_does_not_request_additional_listener() { - let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + fn rootless_unsupported_network_helper_is_rejected() { + for (rootless_network_cmd, expected) in [("slirp4netns", "slirp4netns"), ("", "")] + { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + ..PodmanComputeConfig::default() + }); + driver.rootless = true; + driver.rootless_network_cmd = rootless_network_cmd.to_string(); + + let err = driver.gateway_listener_requirements().unwrap_err(); + + assert!( + matches!(err, ComputeDriverError::Precondition(_)), + "unsupported helper should fail precondition: {err}" + ); + assert!(err.to_string().contains(expected)); + assert!(err.to_string().contains("configure pasta")); + } + } + + #[tokio::test] + async fn constructor_preserves_network_gateway_discovery_error() { + let (socket_path, request_log, handle) = spawn_podman_stub( + "network-gateway-error", + vec![ + StubResponse::new(StatusCode::OK, ""), + StubResponse::new( + StatusCode::OK, + r#"{ + "host": { + "cgroupVersion": "v2", + "networkBackend": "netavark", + "security": {"rootless": false} + } + }"#, + ), + StubResponse::new(StatusCode::CREATED, "{}"), + StubResponse::new( + StatusCode::INTERNAL_SERVER_ERROR, + r#"{"message":"network gateway inspection failed"}"#, + ), + ], + ); + let config = PodmanComputeConfig { + socket_path: Some(socket_path.clone()), ..PodmanComputeConfig::default() - }); - driver.rootless = true; - driver.rootless_network_cmd = "slirp4netns".to_string(); + }; + let network_name = config.network_name.clone(); - assert!(driver.gateway_listener_requirements().unwrap().is_empty()); + let Err(err) = PodmanComputeDriver::new(config).await else { + panic!("network gateway discovery failure should prevent startup"); + }; + + assert!( + err.to_string() + .contains("network gateway inspection failed"), + "unexpected startup error: {err}" + ); + handle.await.expect("stub task should finish"); + assert_eq!( + request_log + .lock() + .expect("request log lock should not be poisoned") + .as_slice(), + [ + "GET /_ping".to_string(), + format!("GET {}", api_path("/libpod/info")), + format!("POST {}", api_path("/libpod/networks/create")), + format!( + "GET {}", + api_path(&format!("/libpod/networks/{network_name}/json")) + ), + ] + ); } #[test] diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 6700a22c9..648219357 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -186,7 +186,7 @@ Podman sandboxes default to a 45-second graceful stop window before Podman escal For proxy-required networks, the Podman driver also accepts the corporate egress proxy keys `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, and `proxy_connect_by_hostname`. The supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. -On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. +On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. Automatic local callback listener discovery for rootless Podman requires the default pasta network helper. When Podman reports another helper, set `host_gateway_ip` or configure an explicitly remote `grpc_endpoint`; otherwise the gateway fails startup rather than leaving sandbox callbacks unreachable. ### Podman Driver Config Mounts From 41afe8cd1536e725dbee1326e7c7b27c096659f3 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 28 Jul 2026 10:54:50 +0200 Subject: [PATCH 06/24] fix(server): support split dual-stack listeners Signed-off-by: Evan Lezar --- Cargo.lock | 1 + Cargo.toml | 1 + crates/openshell-server/Cargo.toml | 1 + .../openshell-server/src/gateway_listener.rs | 51 ++++++++++++++++++- 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9f454e3e0..e8c990db4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4218,6 +4218,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "socket2 0.6.3", "sqlx", "tempfile", "thiserror 2.0.18", diff --git a/Cargo.toml b/Cargo.toml index 4ec6a0d44..78af61d43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ metrics-exporter-prometheus = { version = "0.18", default-features = false, feat # Unix/Process nix = { version = "0.29", features = ["signal", "process", "user", "fs", "term"] } rustix = { version = "1.1", features = ["process"] } +socket2 = "0.6" # Serialization serde = { version = "1", features = ["derive"] } diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 989377b2a..af1bfbbf1 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -35,6 +35,7 @@ k8s-openapi = { workspace = true } # Async runtime tokio = { workspace = true } +socket2 = { workspace = true } # gRPC tonic = { workspace = true, features = ["channel", "tls-native-roots"] } diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index dd6c7c151..a6bd6eb09 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -3,6 +3,7 @@ use crate::compute::GatewayListenerRequirement; use openshell_core::{ComputeDriverKind, Error, Result}; +use socket2::{Domain, Protocol, Socket, Type}; use std::net::{IpAddr, SocketAddr}; use tokio::net::TcpListener; use tracing::info; @@ -273,8 +274,14 @@ pub async fn bind_gateway_listeners( ) -> Result> { let specs = gateway_listener_specs(bind_address, requirements)?; let mut listeners = Vec::with_capacity(specs.len()); - for spec in specs { - let listener = TcpListener::bind(spec.address) + for spec in &specs { + let ipv6_only = matches!( + spec.address.ip(), + IpAddr::V6(address) if address.is_unspecified() + ) && specs.iter().any(|candidate| { + candidate.address.port() == spec.address.port() && candidate.address.is_ipv4() + }); + let listener = bind_gateway_listener(spec.address, ipv6_only) .await .map_err(|e| Error::transport(format!("failed to bind to {}: {e}", spec.address)))?; let local_addr = listener.local_addr().unwrap_or(spec.address); @@ -317,6 +324,24 @@ fn resolve_ephemeral_port( } } +async fn bind_gateway_listener( + address: SocketAddr, + ipv6_only: bool, +) -> std::io::Result { + if ipv6_only { + let socket = Socket::new(Domain::IPV6, Type::STREAM, Some(Protocol::TCP))?; + socket.set_reuse_address(true)?; + socket.set_only_v6(true)?; + socket.set_nonblocking(true)?; + socket.bind(&address.into())?; + socket.listen(1024)?; + let listener: std::net::TcpListener = socket.into(); + return TcpListener::from_std(listener); + } + + TcpListener::bind(address).await +} + fn listener_covers(existing: SocketAddr, requested: SocketAddr) -> bool { if existing == requested { return true; @@ -625,6 +650,28 @@ mod tests { ); } + #[tokio::test] + #[cfg(target_os = "linux")] + async fn gateway_listeners_bind_ipv6_wildcard_and_ipv4_callback_on_same_port() { + let probe = TcpListener::bind("[::1]:0") + .await + .expect("IPv6 loopback probe should bind"); + let port = probe.local_addr().unwrap().port(); + drop(probe); + + let primary = format!("[::]:{port}").parse().unwrap(); + let listeners = bind_gateway_listeners(primary, &[podman_loopback_listener_requirement()]) + .await + .expect("IPv6 wildcard and IPv4 callback listeners should both bind"); + + assert_eq!(listeners.len(), 2); + assert_eq!(listeners[0].address, primary); + assert_eq!( + listeners[1].address, + SocketAddr::from(([127, 0, 0, 1], port)) + ); + } + fn docker_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { GatewayListenerRequirement::Exact { address, From 5b03f5bcf46ae8cf12352905d3a66427d6117792 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 28 Jul 2026 12:28:34 +0200 Subject: [PATCH 07/24] fix(podman): support legacy rootless listener discovery Signed-off-by: Evan Lezar --- architecture/gateway.md | 5 +- crates/openshell-driver-podman/NETWORKING.md | 14 ++-- crates/openshell-driver-podman/src/driver.rs | 73 +++++++++++--------- 3 files changed, 51 insertions(+), 41 deletions(-) diff --git a/architecture/gateway.md b/architecture/gateway.md index a95ee171a..9c2399790 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -631,8 +631,9 @@ system entry instead of pretending to delete package-manager owned state. aliases by default so stale Podman machine images do not need Podman's `host-gateway` resolver. Linux Podman keeps the resolver unless `host_gateway_ip` is configured. Rootful Podman can request its exact bridge - gateway listener; rootless pasta requests the private IPv4 source selected by - the host default route rather than an arbitrary private interface. + gateway listener; rootless pasta, slirp4netns, and legacy Podman versions that + do not report their helper request the private IPv4 source selected by the + host default route rather than an arbitrary private interface. - Gateway restarts recover persisted objects from storage, but live relay streams must be re-established by supervisors. - User-facing behavior changes must update published docs in `docs/`; this file diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 7d98496e0..a24767673 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -259,12 +259,14 @@ Before the gateway binds its serving sockets, the driver reports the callback listener required by the selected topology: - Rootful Linux Podman reports the configured bridge's gateway address exactly. -- Rootless Linux Podman using pasta requests the private IPv4 source address - selected by the host's default route. This matches pasta's default upstream - interface without guessing among private interfaces on a multihomed host. -- Rootless Linux Podman using another helper cannot infer a safe local callback - listener. The driver fails startup unless `host_gateway_ip` is set or - `grpc_endpoint` names an explicitly remote endpoint. +- Rootless Linux Podman using pasta or slirp4netns requests the private IPv4 + source address selected by the host's default route. Podman 4 does not report + its selected helper through the API, so an empty helper value uses the same + legacy-compatible requirement. This avoids guessing among private interfaces + on a multihomed host. +- Rootless Linux Podman reporting another named helper cannot infer a safe local + callback listener. The driver fails startup unless `host_gateway_ip` is set + or `grpc_endpoint` names an explicitly remote endpoint. - Podman Machine requests IPv4 loopback because gvproxy terminates the host forwarding path there. - An explicitly remote callback endpoint requests no additional local listener. diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index a4a21f8a6..2fdcb1f31 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -412,8 +412,8 @@ impl PodmanComputeDriver { /// Report the gateway exposure needed by Podman's standard local callback aliases. /// /// Rootful Podman binds the exact bridge address behind the sandbox alias. - /// Rootless pasta follows the host's default-route interface, while Podman - /// Machine forwards the alias to gateway loopback. + /// Rootless networking follows the host's default-route interface, while + /// Podman Machine forwards the alias to gateway loopback. pub fn gateway_listener_requirements( &self, ) -> Result, ComputeDriverError> { @@ -434,11 +434,16 @@ impl PodmanComputeDriver { { if self.config.host_gateway_ip.trim().is_empty() && self.rootless { let rootless_network_cmd = self.rootless_network_cmd.trim(); - if rootless_network_cmd == "pasta" { + if matches!(rootless_network_cmd, "" | "pasta" | "slirp4netns") { + let helper = if rootless_network_cmd.is_empty() { + "legacy Podman" + } else { + rootless_network_cmd + }; return Ok(vec![GatewayListenerRequirement { - reason: - "Podman rootless pasta callback uses the host default-route interface" - .to_string(), + reason: format!( + "Podman rootless {helper} callback uses the host default-route interface" + ), selector: Some(Selector::DefaultRouteInterface( GatewayDefaultRouteInterfaceRequirement {}, )), @@ -451,7 +456,7 @@ impl PodmanComputeDriver { rootless_network_cmd }; return Err(ComputeDriverError::Precondition(format!( - "Podman rootless network helper '{reported}' is unsupported for local gateway callback aliases; configure pasta, set host_gateway_ip, or use an explicitly remote grpc_endpoint" + "Podman rootless network helper '{reported}' is unsupported for local gateway callback aliases; configure pasta or slirp4netns, set host_gateway_ip, or use an explicitly remote grpc_endpoint" ))); } @@ -1316,27 +1321,8 @@ mod tests { #[test] #[cfg(target_os = "linux")] - fn rootless_pasta_requests_default_route_interface() { - let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: "http://host.openshell.internal:17670".to_string(), - ..PodmanComputeConfig::default() - }); - driver.rootless = true; - driver.rootless_network_cmd = "pasta".to_string(); - - let requirements = driver.gateway_listener_requirements().unwrap(); - - assert!(matches!( - requirements[0].selector, - Some(Selector::DefaultRouteInterface(_)) - )); - } - - #[test] - #[cfg(target_os = "linux")] - fn rootless_unsupported_network_helper_is_rejected() { - for (rootless_network_cmd, expected) in [("slirp4netns", "slirp4netns"), ("", "")] - { + fn rootless_supported_helpers_request_default_route_interface() { + for rootless_network_cmd in ["pasta", "slirp4netns", ""] { let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { grpc_endpoint: "http://host.openshell.internal:17670".to_string(), ..PodmanComputeConfig::default() @@ -1344,17 +1330,38 @@ mod tests { driver.rootless = true; driver.rootless_network_cmd = rootless_network_cmd.to_string(); - let err = driver.gateway_listener_requirements().unwrap_err(); + let requirements = driver.gateway_listener_requirements().unwrap(); assert!( - matches!(err, ComputeDriverError::Precondition(_)), - "unsupported helper should fail precondition: {err}" + matches!( + requirements[0].selector, + Some(Selector::DefaultRouteInterface(_)) + ), + "rootless helper '{rootless_network_cmd}' should use the default route" ); - assert!(err.to_string().contains(expected)); - assert!(err.to_string().contains("configure pasta")); } } + #[test] + #[cfg(target_os = "linux")] + fn rootless_unsupported_network_helper_is_rejected() { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + ..PodmanComputeConfig::default() + }); + driver.rootless = true; + driver.rootless_network_cmd = "unknown-helper".to_string(); + + let err = driver.gateway_listener_requirements().unwrap_err(); + + assert!( + matches!(err, ComputeDriverError::Precondition(_)), + "unsupported helper should fail precondition: {err}" + ); + assert!(err.to_string().contains("unknown-helper")); + assert!(err.to_string().contains("configure pasta or slirp4netns")); + } + #[tokio::test] async fn constructor_preserves_network_gateway_discovery_error() { let (socket_path, request_log, handle) = spawn_podman_stub( From 0bbfe77195d9f0eba08ca39cec7142026ae6d5c9 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 28 Jul 2026 12:28:56 +0200 Subject: [PATCH 08/24] test(e2e): accept loopback plaintext rejection Signed-off-by: Evan Lezar --- e2e/python/test_security_tls.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/e2e/python/test_security_tls.py b/e2e/python/test_security_tls.py index fa9059fa4..529404a6e 100644 --- a/e2e/python/test_security_tls.py +++ b/e2e/python/test_security_tls.py @@ -234,11 +234,14 @@ def test_plaintext_connection_rejected( stub = openshell_pb2_grpc.OpenShellStub(channel) with pytest.raises(grpc.RpcError) as exc_info: stub.Health(openshell_pb2.HealthRequest(), timeout=10) - # Plaintext to a TLS port will fail at the transport level. + # The loopback listener may intentionally accept plaintext service + # HTTP. A gRPC request is still rejected, either at the transport + # boundary or as an unimplemented HTTP route. assert exc_info.value.code() in ( grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.UNKNOWN, grpc.StatusCode.INTERNAL, - ), f"expected transport failure, got {exc_info.value.code()}" + grpc.StatusCode.UNIMPLEMENTED, + ), f"expected plaintext gRPC rejection, got {exc_info.value.code()}" finally: channel.close() From 54cdb79d0fb09965bbde932a35344fa38630280c Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 28 Jul 2026 12:32:27 +0200 Subject: [PATCH 09/24] docs(agent): add callback listener diagnostics Signed-off-by: Evan Lezar --- .agents/skills/debug-openshell-cluster/SKILL.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 0de85fb0a..5413cd175 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -176,6 +176,14 @@ Common findings: - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Root and missing identities are rejected. - Supervisor cannot call back: check callback endpoint and gateway logs. +- Gateway exits before becoming healthy with a callback-listener discovery + error: inspect `podman info --debug`, the configured Podman network, and the + host's IPv4 default route. Rootless pasta, slirp4netns, and Podman 4's + unreported legacy helper use the private source address selected by that + route; rootful Podman uses the bridge gateway address. +- An unsupported rootless helper requires an explicit `host_gateway_ip` or an + explicitly remote `grpc_endpoint`. Do not work around discovery failures by + broadening the primary gateway listener to `0.0.0.0`. ### Step 6: Check Kubernetes Helm Gateways @@ -392,6 +400,7 @@ openshell logs |---|---|---| | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs | | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs | +| Gateway exits while resolving compute-driver listener requirements | Callback alias topology is unsupported, the Podman network cannot be inspected, or the selected address is not private/authorized | Gateway startup error, `podman info --debug`, Podman network inspection, host IPv4 default route | | Docker or Podman sandbox never registers | Wrong callback endpoint or supervisor startup failure | Gateway logs and sandbox container logs | | Docker GPU e2e fails before GPU sandbox comparison | NVIDIA CDI specs are missing or Docker has not discovered them | `docker info --format '{{json .DiscoveredDevices}}'`, `/etc/cdi`, `/var/run/cdi`, `nvidia-cdi-refresh.service` | | Kubernetes gateway pod pending | PVC unbound, taint, selector, or insufficient resources | `kubectl -n openshell describe pod ` | From 92e8c0f6c193431ff7c1ccc83d07d98622dfe77b Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 28 Jul 2026 16:31:03 +0200 Subject: [PATCH 10/24] fix(server): restrict compute callback listeners Signed-off-by: Evan Lezar --- .../skills/debug-openshell-cluster/SKILL.md | 1 + architecture/gateway.md | 8 + crates/openshell-driver-podman/NETWORKING.md | 3 + crates/openshell-server/src/lib.rs | 39 +++- crates/openshell-server/src/multiplex.rs | 178 +++++++++++++++++- docs/reference/sandbox-compute-drivers.mdx | 6 + 6 files changed, 226 insertions(+), 9 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 5413cd175..ea6000ffd 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -401,6 +401,7 @@ openshell logs | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs | | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs | | Gateway exits while resolving compute-driver listener requirements | Callback alias topology is unsupported, the Podman network cannot be inspected, or the selected address is not private/authorized | Gateway startup error, `podman info --debug`, Podman network inspection, host IPv4 default route | +| Admin, health, reflection, or HTTP request is denied on a Docker/Podman callback address | Negotiated callback listeners intentionally expose only sandbox-callable gRPC methods | Retry through the gateway's primary endpoint; inspect the listener-purpose startup log if the address was unexpected | | Docker or Podman sandbox never registers | Wrong callback endpoint or supervisor startup failure | Gateway logs and sandbox container logs | | Docker GPU e2e fails before GPU sandbox comparison | NVIDIA CDI specs are missing or Docker has not discovered them | `docker info --format '{{json .DiscoveredDevices}}'`, `/etc/cdi`, `/var/run/cdi`, `nvidia-cdi-refresh.service` | | Kubernetes gateway pod pending | PVC unbound, taint, selector, or insufficient resources | `kubectl -n openshell describe pod ` | diff --git a/architecture/gateway.md b/architecture/gateway.md index 9c2399790..a9701ec85 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -37,6 +37,14 @@ health, metrics, or tunnel routes. The plaintext service router also rejects browser requests whose Fetch Metadata, Origin, or Referer headers indicate a cross-origin or sibling-subdomain request. +Docker and Podman may negotiate additional listeners that make the gateway +reachable from their local sandbox network topology. Those listeners accept +only gRPC methods classified as sandbox-callable by the gateway's generated +authorization metadata. They reject user and administrator APIs, health, +reflection, non-callback inference APIs, and HTTP routes before normal request +authentication. The operator-configured primary listener retains the full +multiplexed API surface. + Operators can configure a gateway-wide gRPC request rate limit. The limit is applied only to gRPC API traffic after protocol multiplexing; health, metrics, and local sandbox-service HTTP routes are not rate limited by this control. diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index a24767673..4d82c1316 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -276,6 +276,9 @@ maps both local callback aliases to that literal. Podman Machine still requests gateway loopback because its configured address is guest-visible and gvproxy terminates that route on host loopback. The gateway validates and binds every accepted callback listener; the primary listener can remain on loopback. +Negotiated callback listeners expose only the gateway's sandbox-callable gRPC +methods. Operator, health, reflection, and HTTP requests must use the primary +listener. ### Layer 3 Inner Sandbox Network Namespace diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 61706c47b..57bb7f3d7 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -661,8 +661,12 @@ fn allow_plaintext_service_http( enabled: bool, listen_addr: SocketAddr, peer_addr: SocketAddr, + listener_purpose: &GatewayListenerPurpose, ) -> bool { - enabled && listen_addr.ip().is_loopback() && peer_addr.ip().is_loopback() + enabled + && matches!(listener_purpose, GatewayListenerPurpose::Primary) + && listen_addr.ip().is_loopback() + && peer_addr.ip().is_loopback() } fn spawn_gateway_connection( @@ -682,6 +686,7 @@ fn spawn_gateway_connection( enable_loopback_service_http, listen_addr, addr, + &listener_purpose, ) => { if let Err(e) = service @@ -696,7 +701,12 @@ fn spawn_gateway_connection( } } Ok(ConnectionProtocol::PlainHttp) => { - warn!(client = %addr, listen = %listen_addr, "Rejected plaintext HTTP on non-loopback gateway listener"); + warn!( + client = %addr, + listen = %listen_addr, + purpose = ?listener_purpose, + "Rejected plaintext HTTP on gateway listener" + ); } Ok(ConnectionProtocol::Tls | ConnectionProtocol::Unknown) => { // acceptor.acceptor() snapshots the current TLS config; @@ -1212,11 +1222,28 @@ mod tests { let peer: SocketAddr = "127.0.0.1:54000".parse().unwrap(); let wildcard: SocketAddr = "0.0.0.0:8080".parse().unwrap(); let remote_peer: SocketAddr = "192.0.2.10:54000".parse().unwrap(); + let primary = GatewayListenerPurpose::Primary; + let callback = GatewayListenerPurpose::ComputeDriverCallback { + driver_name: "test-driver".to_string(), + reason: "test callback requirement".to_string(), + }; - assert!(allow_plaintext_service_http(true, loopback, peer)); - assert!(!allow_plaintext_service_http(false, loopback, peer)); - assert!(!allow_plaintext_service_http(true, wildcard, peer)); - assert!(!allow_plaintext_service_http(true, loopback, remote_peer)); + assert!(allow_plaintext_service_http(true, loopback, peer, &primary)); + assert!(!allow_plaintext_service_http( + false, loopback, peer, &primary + )); + assert!(!allow_plaintext_service_http( + true, wildcard, peer, &primary + )); + assert!(!allow_plaintext_service_http( + true, + loopback, + remote_peer, + &primary + )); + assert!(!allow_plaintext_service_http( + true, loopback, peer, &callback + )); } #[tokio::test] diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index 44ec231d2..0972c5342 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -7,7 +7,7 @@ //! to either the gRPC service or HTTP endpoints based on the request headers. use bytes::{Bytes, BytesMut}; -use http::{Extensions, HeaderValue, Request, Response}; +use http::{Extensions, HeaderValue, Request, Response, StatusCode}; use http_body::Body; use http_body_util::{BodyExt, Full, LengthLimitError, Limited, StreamBody}; use hyper::body::Incoming; @@ -995,6 +995,38 @@ impl MultiplexedService { } } +fn listener_allows_request( + listener_purpose: Option<&GatewayListenerPurpose>, + is_grpc: bool, + path: &str, +) -> bool { + match listener_purpose { + Some(GatewayListenerPurpose::ComputeDriverCallback { .. }) => { + is_grpc && crate::auth::sandbox_methods::is_sandbox_callable(path) + } + Some(GatewayListenerPurpose::Primary) | None => true, + } +} + +fn callback_listener_rejection(is_grpc: bool) -> Response { + if is_grpc { + let response: Response = tonic::Status::permission_denied( + "compute-driver callback listeners accept sandbox callback RPCs only", + ) + .into_http(); + let (parts, body) = response.into_parts(); + let body = body.map_err(Into::into).boxed_unsync(); + Response::from_parts(parts, BoxBody(body)) + } else { + Response::builder() + .status(StatusCode::FORBIDDEN) + .body(boxed_body_from_bytes(Bytes::from_static( + b"compute-driver callback listeners accept gRPC callbacks only", + ))) + .expect("static callback listener rejection response must be valid") + } +} + impl hyper::service::Service> for MultiplexedService where G: tower::Service, Response = Response> + Clone + Send + 'static, @@ -1018,6 +1050,15 @@ where .get("content-type") .is_some_and(|v| v.as_bytes().starts_with(b"application/grpc")); + if !listener_allows_request( + req.extensions().get::(), + is_grpc, + req.uri().path(), + ) { + let response = callback_listener_rejection(is_grpc); + return Box::pin(async move { Ok(response) }); + } + if is_grpc { let method = grpc_method_from_path(req.uri().path()); let start = Instant::now(); @@ -1205,6 +1246,88 @@ mod tests { ); } + fn callback_listener_purpose() -> GatewayListenerPurpose { + GatewayListenerPurpose::ComputeDriverCallback { + driver_name: "test-driver".to_string(), + reason: "test callback requirement".to_string(), + } + } + + #[test] + fn callback_listener_allows_sandbox_callback_rpcs() { + let purpose = callback_listener_purpose(); + let callback_paths = [ + "/openshell.v1.OpenShell/ConnectSupervisor", + "/openshell.v1.OpenShell/RelayStream", + "/openshell.v1.OpenShell/GetSandboxConfig", + "/openshell.v1.OpenShell/ReportPolicyStatus", + "/openshell.v1.OpenShell/PushSandboxLogs", + "/openshell.v1.OpenShell/GetSandboxProviderEnvironment", + "/openshell.v1.OpenShell/SubmitPolicyAnalysis", + "/openshell.v1.OpenShell/RefreshSandboxToken", + "/openshell.inference.v1.Inference/GetInferenceBundle", + ]; + + for path in callback_paths { + assert!( + listener_allows_request(Some(&purpose), true, path), + "callback listener should allow {path}" + ); + } + } + + #[test] + fn callback_listener_rejects_non_callback_routes() { + let purpose = callback_listener_purpose(); + let rejected_grpc_paths = [ + "/grpc.health.v1.Health/Check", + "/grpc.reflection.v1.ServerReflection/ServerReflectionInfo", + "/openshell.v1.OpenShell/ListSandboxes", + "/openshell.v1.OpenShell/DeleteSandbox", + "/openshell.v1.OpenShell/CreateProvider", + "/openshell.inference.v1.Inference/GetInferenceRoute", + "/openshell.inference.v1.Inference/SetInferenceRoute", + ]; + + for path in rejected_grpc_paths { + assert!( + !listener_allows_request(Some(&purpose), true, path), + "callback listener should reject {path}" + ); + } + assert!(!listener_allows_request(Some(&purpose), false, "/health")); + assert!(!listener_allows_request(Some(&purpose), false, "/service")); + } + + #[test] + fn primary_listener_routing_is_unchanged() { + let primary = GatewayListenerPurpose::Primary; + let paths = [ + "/grpc.health.v1.Health/Check", + "/openshell.v1.OpenShell/ListSandboxes", + "/openshell.inference.v1.Inference/GetInferenceRoute", + "/health", + "/service", + ]; + + for path in paths { + assert!(listener_allows_request(Some(&primary), true, path)); + assert!(listener_allows_request(Some(&primary), false, path)); + assert!(listener_allows_request(None, true, path)); + assert!(listener_allows_request(None, false, path)); + } + } + + #[test] + fn callback_listener_rejections_use_protocol_appropriate_statuses() { + let grpc = callback_listener_rejection(true); + assert_eq!(grpc.status(), StatusCode::OK); + assert_eq!(grpc.headers().get("grpc-status").unwrap(), "7"); + + let http = callback_listener_rejection(false); + assert_eq!(http.status(), StatusCode::FORBIDDEN); + } + #[derive(Clone)] struct PostCommitTestInterceptor; @@ -1306,6 +1429,12 @@ mod tests { } async fn start_http_server_with_middleware() -> std::net::SocketAddr { + start_http_server_with_middleware_on_listener(GatewayListenerPurpose::Primary).await + } + + async fn start_http_server_with_middleware_on_listener( + listener_purpose: GatewayListenerPurpose, + ) -> std::net::SocketAddr { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -1313,6 +1442,7 @@ mod tests { let http_service = request_id_middleware!(http_service); let service = MultiplexedService::new(http_service.clone(), http_service); + let service = GatewayListenerContextService::new(service, listener_purpose); tokio::spawn(async move { loop { @@ -1331,8 +1461,9 @@ mod tests { addr } - async fn http1_get( + async fn http1_request( addr: std::net::SocketAddr, + method: &str, path: &str, headers: &[(&str, &str)], ) -> Response { @@ -1346,7 +1477,7 @@ mod tests { }); let mut builder = Request::builder() - .method("GET") + .method(method) .uri(format!("http://{addr}{path}")); for (k, v) in headers { builder = builder.header(*k, *v); @@ -1355,6 +1486,47 @@ mod tests { sender.send_request(req).await.unwrap() } + async fn http1_get( + addr: std::net::SocketAddr, + path: &str, + headers: &[(&str, &str)], + ) -> Response { + http1_request(addr, "GET", path, headers).await + } + + #[tokio::test] + async fn callback_listener_filter_is_applied_before_route_dispatch() { + let addr = start_http_server_with_middleware_on_listener(callback_listener_purpose()).await; + + let health = http1_get(addr, "/healthz", &[]).await; + assert_eq!(health.status(), StatusCode::FORBIDDEN); + + let admin = http1_request( + addr, + "POST", + "/openshell.v1.OpenShell/ListSandboxes", + &[("content-type", "application/grpc")], + ) + .await; + assert_eq!(admin.status(), StatusCode::OK); + assert_eq!(admin.headers().get("grpc-status").unwrap(), "7"); + + let callback = http1_request( + addr, + "POST", + "/openshell.v1.OpenShell/ConnectSupervisor", + &[("content-type", "application/grpc")], + ) + .await; + assert_ne!( + callback + .headers() + .get("grpc-status") + .and_then(|value| value.to_str().ok()), + Some("7") + ); + } + #[tokio::test] async fn intercepted_grpc_body_collection_rejects_oversized_body() { let oversized = Bytes::from(vec![0_u8; MAX_INTERCEPTED_GRPC_BODY_SIZE + 1]); diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 648219357..7d8dc6eaa 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -107,6 +107,12 @@ It overrides the gateway's configured default runtime class for that sandbox, while a typed `SandboxTemplate.runtime_class_name` value from the API still takes precedence. +Docker and Podman callback listeners accept only supervisor callback gRPC +methods. Use the gateway's primary endpoint for CLI, administrator, health, +reflection, inference-route management, and HTTP requests. A +`PermissionDenied` response from one of the sandbox-visible callback addresses +is expected for those requests. + ## Docker Driver [Docker](https://www.docker.com/get-started/)-backed sandboxes run as containers on the gateway host. Use Docker for local development, single-machine gateways, and hosts that already use Docker Desktop or Docker Engine. From cc0399195d637472d94facf79dc02882832bde4f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 29 Jul 2026 10:44:36 +0200 Subject: [PATCH 11/24] fix(podman): validate local callback port Signed-off-by: Evan Lezar --- crates/openshell-driver-podman/src/driver.rs | 49 ++++++++++++++++---- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 2fdcb1f31..ae04d3d39 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -429,6 +429,18 @@ impl PodmanComputeDriver { if !uses_local_callback_alias { return Ok(Vec::new()); } + let callback_port = endpoint.port_or_known_default().ok_or_else(|| { + ComputeDriverError::Precondition(format!( + "Podman gateway callback endpoint '{}' has no port", + self.config.grpc_endpoint + )) + })?; + if callback_port != self.config.gateway_port { + return Err(ComputeDriverError::Precondition(format!( + "Podman local callback endpoint '{}' uses port {callback_port}, but the gateway primary listener uses port {}; configure grpc_endpoint with the gateway primary listener port", + self.config.grpc_endpoint, self.config.gateway_port + ))); + } #[cfg(target_os = "linux")] { @@ -476,16 +488,10 @@ impl PodmanComputeDriver { "Podman callback gateway address '{gateway_ip}' is invalid: {err}" )) })?; - let port = endpoint.port_or_known_default().ok_or_else(|| { - ComputeDriverError::Precondition(format!( - "Podman gateway callback endpoint '{}' has no port", - self.config.grpc_endpoint - )) - })?; Ok(vec![GatewayListenerRequirement { reason: format!("Podman network '{}' host gateway", self.config.network_name), selector: Some(Selector::ExactBindAddress( - SocketAddr::new(gateway_ip, port).to_string(), + SocketAddr::new(gateway_ip, callback_port).to_string(), )), }]) } @@ -1454,13 +1460,40 @@ mod tests { #[test] fn explicit_remote_callback_does_not_request_gateway_listener() { let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: "https://gateway.example.test:17670".to_string(), + grpc_endpoint: "https://gateway.example.test:9443".to_string(), + gateway_port: 17670, ..PodmanComputeConfig::default() }); assert!(driver.gateway_listener_requirements().unwrap().is_empty()); } + #[test] + fn local_callback_alias_requires_primary_listener_port() { + for grpc_endpoint in [ + "http://host.openshell.internal:17671", + "http://host.containers.internal", + ] { + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: grpc_endpoint.to_string(), + gateway_port: 17670, + ..PodmanComputeConfig::default() + }); + + let err = driver.gateway_listener_requirements().unwrap_err(); + + assert!( + matches!(err, ComputeDriverError::Precondition(_)), + "mismatched local callback port should fail precondition: {err}" + ); + assert!( + err.to_string() + .contains("gateway primary listener uses port 17670"), + "unexpected error for {grpc_endpoint}: {err}" + ); + } + } + #[test] fn local_podman_cdi_gpu_inventory_maps_nvidia_device_nodes() { let root = std::env::temp_dir().join(format!( From afb61c5ec7a29b63d5abcb3001c56e7b273ab727 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 29 Jul 2026 15:16:22 +0200 Subject: [PATCH 12/24] test(server): clarify callback listener contract Signed-off-by: Evan Lezar --- .../openshell-server/src/gateway_listener.rs | 27 ++++++++- crates/openshell-server/src/lib.rs | 27 ++++----- crates/openshell-server/src/multiplex.rs | 58 +++++++++++-------- proto/compute_driver.proto | 4 +- 4 files changed, 72 insertions(+), 44 deletions(-) diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index a6bd6eb09..701d6419c 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -204,7 +204,7 @@ fn validate_gateway_listener_requirement( Ok(()) } _ => Err(Error::config(format!( - "compute driver '{}' is not authorized for this gateway listener requirement in this proof of concept", + "compute driver '{}' is not authorized to request this gateway listener selector", requirement.driver_name() ))), } @@ -285,7 +285,30 @@ pub async fn bind_gateway_listeners( .await .map_err(|e| Error::transport(format!("failed to bind to {}: {e}", spec.address)))?; let local_addr = listener.local_addr().unwrap_or(spec.address); - info!(address = %local_addr, "Server listening"); + match spec.scope { + GatewayListenerScope::Primary => { + info!( + address = %local_addr, + listener_purpose = "primary", + authorization_scope = "full-multiplexed-api", + "Gateway listener bound" + ); + } + GatewayListenerScope::ComputeDriverCallback => { + let provenance = spec + .provenance + .as_ref() + .expect("callback listener spec must include provenance"); + info!( + address = %local_addr, + listener_purpose = "compute-driver-callback", + driver = %provenance.driver_name, + reason = %provenance.reason, + authorization_scope = "sandbox-callable-grpc-only", + "Gateway listener bound" + ); + } + } listeners.push(BoundGatewayListener { listener, spec: spec.bind_to(local_addr), diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 57bb7f3d7..03f278c4c 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -661,10 +661,10 @@ fn allow_plaintext_service_http( enabled: bool, listen_addr: SocketAddr, peer_addr: SocketAddr, - listener_purpose: &GatewayListenerPurpose, + listener_scope: GatewayListenerScope, ) -> bool { enabled - && matches!(listener_purpose, GatewayListenerPurpose::Primary) + && matches!(listener_scope, GatewayListenerScope::Primary) && listen_addr.ip().is_loopback() && peer_addr.ip().is_loopback() } @@ -686,7 +686,7 @@ fn spawn_gateway_connection( enable_loopback_service_http, listen_addr, addr, - &listener_purpose, + listener_scope, ) => { if let Err(e) = service @@ -704,7 +704,7 @@ fn spawn_gateway_connection( warn!( client = %addr, listen = %listen_addr, - purpose = ?listener_purpose, + scope = ?listener_scope, "Rejected plaintext HTTP on gateway listener" ); } @@ -1222,27 +1222,22 @@ mod tests { let peer: SocketAddr = "127.0.0.1:54000".parse().unwrap(); let wildcard: SocketAddr = "0.0.0.0:8080".parse().unwrap(); let remote_peer: SocketAddr = "192.0.2.10:54000".parse().unwrap(); - let primary = GatewayListenerPurpose::Primary; - let callback = GatewayListenerPurpose::ComputeDriverCallback { - driver_name: "test-driver".to_string(), - reason: "test callback requirement".to_string(), - }; + let primary = GatewayListenerScope::Primary; + let callback = GatewayListenerScope::ComputeDriverCallback; - assert!(allow_plaintext_service_http(true, loopback, peer, &primary)); - assert!(!allow_plaintext_service_http( - false, loopback, peer, &primary - )); + assert!(allow_plaintext_service_http(true, loopback, peer, primary)); assert!(!allow_plaintext_service_http( - true, wildcard, peer, &primary + false, loopback, peer, primary )); + assert!(!allow_plaintext_service_http(true, wildcard, peer, primary)); assert!(!allow_plaintext_service_http( true, loopback, remote_peer, - &primary + primary )); assert!(!allow_plaintext_service_http( - true, loopback, peer, &callback + true, loopback, peer, callback )); } diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index 0972c5342..97f634af2 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -149,7 +149,7 @@ impl MultiplexService { .await } - /// Serve a connection and preserve its listener purpose in request + /// Serve a connection and preserve its listener scope in request /// extensions for downstream routing and policy decisions. pub(crate) async fn serve_on_listener( &self, @@ -180,7 +180,7 @@ impl MultiplexService { .await } - /// Serve a TLS connection and preserve its listener purpose in request + /// Serve a TLS connection and preserve its listener scope in request /// extensions for downstream routing and policy decisions. pub(crate) async fn serve_with_peer_identity_on_listener( &self, @@ -260,7 +260,7 @@ impl MultiplexService { } /// Serve a plaintext service HTTP connection and preserve its listener - /// purpose in request extensions. + /// scope in request extensions. pub(crate) async fn serve_service_http_on_listener( &self, stream: S, @@ -996,15 +996,15 @@ impl MultiplexedService { } fn listener_allows_request( - listener_purpose: Option<&GatewayListenerPurpose>, + listener_scope: Option<&GatewayListenerScope>, is_grpc: bool, path: &str, ) -> bool { - match listener_purpose { - Some(GatewayListenerPurpose::ComputeDriverCallback { .. }) => { + match listener_scope { + Some(GatewayListenerScope::ComputeDriverCallback) => { is_grpc && crate::auth::sandbox_methods::is_sandbox_callable(path) } - Some(GatewayListenerPurpose::Primary) | None => true, + Some(GatewayListenerScope::Primary) | None => true, } } @@ -1051,7 +1051,7 @@ where .is_some_and(|v| v.as_bytes().starts_with(b"application/grpc")); if !listener_allows_request( - req.extensions().get::(), + req.extensions().get::(), is_grpc, req.uri().path(), ) { @@ -1246,16 +1246,13 @@ mod tests { ); } - fn callback_listener_purpose() -> GatewayListenerPurpose { - GatewayListenerPurpose::ComputeDriverCallback { - driver_name: "test-driver".to_string(), - reason: "test callback requirement".to_string(), - } + fn callback_listener_scope() -> GatewayListenerScope { + GatewayListenerScope::ComputeDriverCallback } #[test] fn callback_listener_allows_sandbox_callback_rpcs() { - let purpose = callback_listener_purpose(); + let scope = callback_listener_scope(); let callback_paths = [ "/openshell.v1.OpenShell/ConnectSupervisor", "/openshell.v1.OpenShell/RelayStream", @@ -1270,15 +1267,28 @@ mod tests { for path in callback_paths { assert!( - listener_allows_request(Some(&purpose), true, path), + listener_allows_request(Some(&scope), true, path), "callback listener should allow {path}" ); } } + #[test] + fn callback_listener_surface_matches_rpc_auth_metadata() { + let scope = callback_listener_scope(); + + for path in crate::auth::method_authz::all_paths() { + assert_eq!( + listener_allows_request(Some(&scope), true, path), + crate::auth::method_authz::is_sandbox_callable(path), + "callback listener exposure must follow rpc_auth metadata for {path}" + ); + } + } + #[test] fn callback_listener_rejects_non_callback_routes() { - let purpose = callback_listener_purpose(); + let scope = callback_listener_scope(); let rejected_grpc_paths = [ "/grpc.health.v1.Health/Check", "/grpc.reflection.v1.ServerReflection/ServerReflectionInfo", @@ -1291,17 +1301,17 @@ mod tests { for path in rejected_grpc_paths { assert!( - !listener_allows_request(Some(&purpose), true, path), + !listener_allows_request(Some(&scope), true, path), "callback listener should reject {path}" ); } - assert!(!listener_allows_request(Some(&purpose), false, "/health")); - assert!(!listener_allows_request(Some(&purpose), false, "/service")); + assert!(!listener_allows_request(Some(&scope), false, "/health")); + assert!(!listener_allows_request(Some(&scope), false, "/service")); } #[test] fn primary_listener_routing_is_unchanged() { - let primary = GatewayListenerPurpose::Primary; + let primary = GatewayListenerScope::Primary; let paths = [ "/grpc.health.v1.Health/Check", "/openshell.v1.OpenShell/ListSandboxes", @@ -1429,11 +1439,11 @@ mod tests { } async fn start_http_server_with_middleware() -> std::net::SocketAddr { - start_http_server_with_middleware_on_listener(GatewayListenerPurpose::Primary).await + start_http_server_with_middleware_on_listener(GatewayListenerScope::Primary).await } async fn start_http_server_with_middleware_on_listener( - listener_purpose: GatewayListenerPurpose, + listener_scope: GatewayListenerScope, ) -> std::net::SocketAddr { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -1442,7 +1452,7 @@ mod tests { let http_service = request_id_middleware!(http_service); let service = MultiplexedService::new(http_service.clone(), http_service); - let service = GatewayListenerContextService::new(service, listener_purpose); + let service = GatewayListenerContextService::new(service, listener_scope); tokio::spawn(async move { loop { @@ -1496,7 +1506,7 @@ mod tests { #[tokio::test] async fn callback_listener_filter_is_applied_before_route_dispatch() { - let addr = start_http_server_with_middleware_on_listener(callback_listener_purpose()).await; + let addr = start_http_server_with_middleware_on_listener(callback_listener_scope()).await; let health = http1_get(addr, "/healthz", &[]).await; assert_eq!(health.status(), StatusCode::FORBIDDEN); diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index 47d883846..e3f18af19 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -71,8 +71,8 @@ message GatewayListenerRequirement { string reason = 1; oneof selector { - // Concrete IP:port address requested by the driver. The POC requires the - // port to match the gateway's fixed primary listener port. + // Concrete IP:port address requested by the driver. The port must match + // the gateway's configured primary listener port. string exact_bind_address = 2; // Ask the gateway to bind the IPv4 address selected by its default route. // This matches rootless pasta's default upstream-interface selection. From 816610a3039440f3d7dfd8ae441b8cc01e9c23e9 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 29 Jul 2026 15:17:02 +0200 Subject: [PATCH 13/24] fix(podman): require pasta for local callbacks Signed-off-by: Evan Lezar --- .../skills/debug-openshell-cluster/SKILL.md | 12 +- .github/workflows/e2e-test.yml | 15 +-- architecture/gateway.md | 16 ++- crates/openshell-driver-podman/NETWORKING.md | 20 +-- crates/openshell-driver-podman/src/driver.rs | 126 +++++++++++------- docs/reference/sandbox-compute-drivers.mdx | 2 +- 6 files changed, 116 insertions(+), 75 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index ea6000ffd..acdaa7b3b 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -178,12 +178,12 @@ Common findings: - Supervisor cannot call back: check callback endpoint and gateway logs. - Gateway exits before becoming healthy with a callback-listener discovery error: inspect `podman info --debug`, the configured Podman network, and the - host's IPv4 default route. Rootless pasta, slirp4netns, and Podman 4's - unreported legacy helper use the private source address selected by that - route; rootful Podman uses the bridge gateway address. -- An unsupported rootless helper requires an explicit `host_gateway_ip` or an - explicitly remote `grpc_endpoint`. Do not work around discovery failures by - broadening the primary gateway listener to `0.0.0.0`. + host's IPv4 default route. Rootless pasta uses the private source address + selected by that route; rootful Podman uses the bridge gateway address. +- Rootless slirp4netns, another named helper, or missing helper metadata + requires an explicitly remote `grpc_endpoint`. An explicit `host_gateway_ip` + cannot bypass slirp4netns host-loopback isolation. Do not work around + discovery failures by broadening the primary gateway listener to `0.0.0.0`. ### Step 6: Check Kubernetes Helm Gateways diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index a9270c378..c4b331d7a 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -131,20 +131,17 @@ jobs: # Run directly on the Ubuntu host so the test observes the host's AppArmor # and unprivileged-user-namespace policy. A privileged job container masks # the restrictions that production rootless Podman installations enforce. + # Ubuntu 26.04 provides the supported Podman 5.x and pasta combination. + # Re-add older/slirp4netns environments when direct callbacks through a + # rootless-network namespace relay are supported. runs-on: ${{ matrix.runner }} timeout-minutes: 30 strategy: fail-fast: false matrix: include: - # Ubuntu 24.04 matches the environment reported in #2069 and ships - # Podman 4.x. The probe records whether AppArmor blocks the drop. - - runner: ubuntu-24.04 - podman_major: "4" - podman_package_version: "4.9.3+ds1-1ubuntu0.2" - conmon_package_version: "2.1.10+ds1-1build2" - # Ubuntu 26.04 provides the supported Podman 5.x coverage for - # comparison with the Ubuntu 24.04 environment. + # Keep package versions explicit so hosted-runner tool overrides + # cannot silently change the supported test environment. - runner: ubuntu-26.04 podman_major: "5" podman_package_version: "5.7.0+ds2-3build1" @@ -199,7 +196,6 @@ jobs: pkg-config \ "conmon=${{ matrix.conmon_package_version }}" \ "podman=${{ matrix.podman_package_version }}" \ - slirp4netns \ uidmap # Hosted runners can place newer Podman and conmon binaries under # /usr/local ahead of Ubuntu's packages. Select the distro CLI and @@ -238,6 +234,7 @@ jobs: test "$(command -v podman)" = "/usr/bin/podman" test "$(podman info --format '{{.Host.Conmon.Path}}')" = "/usr/bin/conmon" test "$(podman info --format '{{.Host.Security.Rootless}}')" = "true" + test "$(podman info --format '{{.Host.RootlessNetworkCmd}}')" = "pasta" test "$(sudo sysctl -n kernel.apparmor_restrict_unprivileged_userns)" = "1" echo "=== host ===" uname -a diff --git a/architecture/gateway.md b/architecture/gateway.md index a9701ec85..5e2489c2b 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -45,6 +45,14 @@ reflection, non-callback inference APIs, and HTTP routes before normal request authentication. The operator-configured primary listener retains the full multiplexed API surface. +The `rpc_auth` classification is also the source of truth for negotiated +listener exposure: marking an RPC as `sandbox` or `dual` makes it callable on +these listeners. Review such changes as both authorization and network-surface +changes. Listener requirements are currently authorized only for the built-in +Docker and Podman drivers. Operator-granted listener capabilities for external +drivers are tracked in +[#2539](https://github.com/NVIDIA/OpenShell/issues/2539). + Operators can configure a gateway-wide gRPC request rate limit. The limit is applied only to gRPC API traffic after protocol multiplexing; health, metrics, and local sandbox-service HTTP routes are not rate limited by this control. @@ -639,9 +647,11 @@ system entry instead of pretending to delete package-manager owned state. aliases by default so stale Podman machine images do not need Podman's `host-gateway` resolver. Linux Podman keeps the resolver unless `host_gateway_ip` is configured. Rootful Podman can request its exact bridge - gateway listener; rootless pasta, slirp4netns, and legacy Podman versions that - do not report their helper request the private IPv4 source selected by the - host default route rather than an arbitrary private interface. + gateway listener. Rootless Podman explicitly reporting pasta requests the + private IPv4 source selected by the host default route rather than an + arbitrary private interface. Slirp4netns, other helpers, and missing helper + metadata fail closed for local callbacks until a rootless-network namespace + relay is available. - Gateway restarts recover persisted objects from storage, but live relay streams must be re-established by supervisors. - User-facing behavior changes must update published docs in `docs/`; this file diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 4d82c1316..28357f1c4 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -259,20 +259,20 @@ Before the gateway binds its serving sockets, the driver reports the callback listener required by the selected topology: - Rootful Linux Podman reports the configured bridge's gateway address exactly. -- Rootless Linux Podman using pasta or slirp4netns requests the private IPv4 - source address selected by the host's default route. Podman 4 does not report - its selected helper through the API, so an empty helper value uses the same - legacy-compatible requirement. This avoids guessing among private interfaces - on a multihomed host. -- Rootless Linux Podman reporting another named helper cannot infer a safe local - callback listener. The driver fails startup unless `host_gateway_ip` is set - or `grpc_endpoint` names an explicitly remote endpoint. +- Rootless Linux Podman explicitly reporting pasta requests the private IPv4 + source address selected by the host's default route. This avoids guessing + among private interfaces on a multihomed host. +- Rootless Linux Podman reporting slirp4netns, another named helper, or no + helper cannot use a direct local callback listener. The driver fails startup + unless `grpc_endpoint` names an explicitly remote endpoint. Supporting + slirp4netns requires a relay inside Podman's rootless network namespace. - Podman Machine requests IPv4 loopback because gvproxy terminates the host forwarding path there. - An explicitly remote callback endpoint requests no additional local listener. -On Linux, an explicit `host_gateway_ip` is reported exactly because the driver -maps both local callback aliases to that literal. Podman Machine still requests +On Linux, an explicit `host_gateway_ip` is reported exactly for rootful Podman +and rootless pasta because the driver maps both local callback aliases to that +literal. Other rootless helpers still fail closed. Podman Machine requests gateway loopback because its configured address is guest-visible and gvproxy terminates that route on host loopback. The gateway validates and binds every accepted callback listener; the primary listener can remain on loopback. diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index ae04d3d39..39c312c24 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -412,8 +412,9 @@ impl PodmanComputeDriver { /// Report the gateway exposure needed by Podman's standard local callback aliases. /// /// Rootful Podman binds the exact bridge address behind the sandbox alias. - /// Rootless networking follows the host's default-route interface, while - /// Podman Machine forwards the alias to gateway loopback. + /// Rootless pasta follows the host's default-route interface, while Podman + /// Machine forwards the alias to gateway loopback. Other rootless helpers + /// cannot use a direct host listener. pub fn gateway_listener_requirements( &self, ) -> Result, ComputeDriverError> { @@ -444,32 +445,19 @@ impl PodmanComputeDriver { #[cfg(target_os = "linux")] { - if self.config.host_gateway_ip.trim().is_empty() && self.rootless { - let rootless_network_cmd = self.rootless_network_cmd.trim(); - if matches!(rootless_network_cmd, "" | "pasta" | "slirp4netns") { - let helper = if rootless_network_cmd.is_empty() { - "legacy Podman" - } else { - rootless_network_cmd - }; + if self.rootless { + validate_rootless_local_callback_helper(&self.rootless_network_cmd)?; + + if self.config.host_gateway_ip.trim().is_empty() { return Ok(vec![GatewayListenerRequirement { - reason: format!( - "Podman rootless {helper} callback uses the host default-route interface" - ), + reason: + "Podman rootless pasta callback uses the host default-route interface" + .to_string(), selector: Some(Selector::DefaultRouteInterface( GatewayDefaultRouteInterfaceRequirement {}, )), }]); } - - let reported = if rootless_network_cmd.is_empty() { - "" - } else { - rootless_network_cmd - }; - return Err(ComputeDriverError::Precondition(format!( - "Podman rootless network helper '{reported}' is unsupported for local gateway callback aliases; configure pasta or slirp4netns, set host_gateway_ip, or use an explicitly remote grpc_endpoint" - ))); } let gateway_ip = if self.config.host_gateway_ip.trim().is_empty() { @@ -1069,6 +1057,25 @@ fn check_subuid_range() { } } +#[cfg(any(target_os = "linux", test))] +fn validate_rootless_local_callback_helper( + rootless_network_cmd: &str, +) -> Result<(), ComputeDriverError> { + let rootless_network_cmd = rootless_network_cmd.trim(); + if rootless_network_cmd == "pasta" { + return Ok(()); + } + + let reported = if rootless_network_cmd.is_empty() { + "" + } else { + rootless_network_cmd + }; + Err(ComputeDriverError::Precondition(format!( + "Podman rootless network helper '{reported}' does not support direct local gateway callbacks; configure pasta or use an explicitly remote grpc_endpoint" + ))) +} + #[cfg(test)] mod tests { use super::*; @@ -1287,6 +1294,20 @@ mod tests { assert_eq!(cfg.grpc_endpoint, "https://gateway.internal:9000"); } + #[test] + fn rootless_slirp_allows_remote_callback_endpoint() { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "https://gateway.internal:9000".to_string(), + ..PodmanComputeConfig::default() + }); + driver.rootless = true; + driver.rootless_network_cmd = "slirp4netns".to_string(); + + let requirements = driver.gateway_listener_requirements().unwrap(); + + assert!(requirements.is_empty()); + } + #[test] #[cfg(target_os = "linux")] fn rootful_local_callback_alias_requests_discovered_network_gateway() { @@ -1327,45 +1348,58 @@ mod tests { #[test] #[cfg(target_os = "linux")] - fn rootless_supported_helpers_request_default_route_interface() { - for rootless_network_cmd in ["pasta", "slirp4netns", ""] { - let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { - grpc_endpoint: "http://host.openshell.internal:17670".to_string(), - ..PodmanComputeConfig::default() - }); - driver.rootless = true; - driver.rootless_network_cmd = rootless_network_cmd.to_string(); + fn rootless_pasta_requests_default_route_interface() { + let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + ..PodmanComputeConfig::default() + }); + driver.rootless = true; + driver.rootless_network_cmd = "pasta".to_string(); - let requirements = driver.gateway_listener_requirements().unwrap(); + let requirements = driver.gateway_listener_requirements().unwrap(); - assert!( - matches!( - requirements[0].selector, - Some(Selector::DefaultRouteInterface(_)) - ), - "rootless helper '{rootless_network_cmd}' should use the default route" - ); + assert!(matches!( + requirements[0].selector, + Some(Selector::DefaultRouteInterface(_)) + )); + } + + #[test] + fn rootless_non_pasta_helpers_are_rejected() { + for (rootless_network_cmd, reported) in [ + ("slirp4netns", "slirp4netns"), + ("", ""), + ("unknown-helper", "unknown-helper"), + ] { + let err = validate_rootless_local_callback_helper(rootless_network_cmd).unwrap_err(); + + assert!(matches!(err, ComputeDriverError::Precondition(_))); + assert!(err.to_string().contains(reported)); + assert!(err.to_string().contains("configure pasta")); + assert!(err.to_string().contains("remote grpc_endpoint")); } } + #[test] + fn rootless_pasta_is_accepted_for_local_callbacks() { + validate_rootless_local_callback_helper("pasta").unwrap(); + } + #[test] #[cfg(target_os = "linux")] - fn rootless_unsupported_network_helper_is_rejected() { + fn rootless_slirp_rejects_explicit_host_gateway_override() { let mut driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { grpc_endpoint: "http://host.openshell.internal:17670".to_string(), + host_gateway_ip: "10.90.1.1".to_string(), ..PodmanComputeConfig::default() }); driver.rootless = true; - driver.rootless_network_cmd = "unknown-helper".to_string(); + driver.rootless_network_cmd = "slirp4netns".to_string(); let err = driver.gateway_listener_requirements().unwrap_err(); - assert!( - matches!(err, ComputeDriverError::Precondition(_)), - "unsupported helper should fail precondition: {err}" - ); - assert!(err.to_string().contains("unknown-helper")); - assert!(err.to_string().contains("configure pasta or slirp4netns")); + assert!(matches!(err, ComputeDriverError::Precondition(_))); + assert!(err.to_string().contains("slirp4netns")); } #[tokio::test] diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 7d8dc6eaa..167374333 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -192,7 +192,7 @@ Podman sandboxes default to a 45-second graceful stop window before Podman escal For proxy-required networks, the Podman driver also accepts the corporate egress proxy keys `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, and `proxy_connect_by_hostname`. The supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. -On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. Automatic local callback listener discovery for rootless Podman requires the default pasta network helper. When Podman reports another helper, set `host_gateway_ip` or configure an explicitly remote `grpc_endpoint`; otherwise the gateway fails startup rather than leaving sandbox callbacks unreachable. +On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. Direct local callbacks from rootless Podman require Podman to report the pasta network helper. Slirp4netns, other helpers, and Podman versions that do not report their helper require an explicitly remote `grpc_endpoint`; otherwise the gateway fails startup rather than leaving sandbox callbacks unreachable. Rootful Podman continues to use the configured network's bridge gateway address. ### Podman Driver Config Mounts From fbc9b8ec1151c50d140ecfc5b5d60f9f2c55ace4 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 29 Jul 2026 15:33:53 +0200 Subject: [PATCH 14/24] docs(gateway): document RPM listener default Signed-off-by: Evan Lezar --- docs/reference/gateway-config.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 27a1d3d45..02de4b39d 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -29,6 +29,8 @@ Package-managed gateways do not require a TOML file. Create one at the package's | Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | | Snap | `$SNAP_COMMON/gateway.toml`, usually `/var/snap/openshell/common/gateway.toml`. | +The Fedora/RHEL RPM template leaves `[openshell.gateway].bind_address` unset. The gateway therefore uses its built-in `127.0.0.1:17670` primary listener. The Podman driver negotiates separate, restricted listeners for sandbox callbacks, so the primary listener does not need a wildcard address. Set `bind_address` explicitly only when clients must reach the primary multiplexed API through another interface. + ## Layout The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Shared keys set at gateway scope are inherited into driver tables when not overridden. From f1c4137df8411af39aee76e52c0e9a93ab8d91e1 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 29 Jul 2026 16:22:36 +0200 Subject: [PATCH 15/24] refactor(server): keep listener provenance diagnostic-only Signed-off-by: Evan Lezar --- .../openshell-server/src/gateway_listener.rs | 14 ++++---------- crates/openshell-server/src/lib.rs | 19 +++++++++---------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index 701d6419c..08a875389 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -311,7 +311,7 @@ pub async fn bind_gateway_listeners( } listeners.push(BoundGatewayListener { listener, - spec: spec.bind_to(local_addr), + spec: spec.clone().bind_to(local_addr), }); } Ok(listeners) @@ -517,15 +517,9 @@ mod tests { let podman_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); assert_eq!( - gateway_listener_specs( - primary, - &[podman_listener_requirement(podman_gateway)], - ) - .unwrap(), - vec![primary_listener_spec_with_covered( - primary, - podman_gateway, - )] + gateway_listener_specs(primary, &[podman_listener_requirement(podman_gateway)],) + .unwrap(), + vec![primary_listener_spec_with_covered(primary, podman_gateway,)] ); } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 03f278c4c..fade49e23 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -72,10 +72,6 @@ use tracing::{debug, error, info, warn}; pub(crate) static TEST_ENV_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); use compute::ComputeRuntime; -#[cfg(test)] -use compute::GatewayListenerRequirement; -#[cfg(test)] -use gateway_listener::GatewayListenerSpec; use gateway_listener::{BoundGatewayListener, GatewayListenerScope, bind_gateway_listeners}; pub use grpc::OpenShellService; pub use http::{health_router, http_router, metrics_router, service_http_router}; @@ -1024,11 +1020,10 @@ pub(crate) async fn ensure_default_workspace(store: &Store) -> Result<()> { mod tests { use super::{ BoundGatewayListener, ConfiguredComputeDriver, ConnectionProtocol, GatewayListenerScope, - GatewayListenerSpec, MultiplexService, ServerState, TlsAcceptor, - allow_plaintext_service_http, bind_gateway_listeners, classify_initial_bytes, - configured_compute_driver, is_benign_tls_handshake_failure, - kubernetes_sandbox_jwt_expiry_disabled, serve_gateway_listener, - GatewayListenerRequirement, + MultiplexService, ServerState, TlsAcceptor, allow_plaintext_service_http, + bind_gateway_listeners, classify_initial_bytes, configured_compute_driver, + is_benign_tls_handshake_failure, kubernetes_sandbox_jwt_expiry_disabled, + serve_gateway_listener, }; use openshell_core::{ ComputeDriverKind, Config, @@ -1046,7 +1041,11 @@ mod tests { use tokio::net::{TcpListener, TcpStream}; use tokio::sync::watch; - use crate::tls_test_utils::{generate_test_certs_with_ca, install_rustls_provider}; + use crate::{ + compute::GatewayListenerRequirement, + gateway_listener::GatewayListenerSpec, + tls_test_utils::{generate_test_certs_with_ca, install_rustls_provider}, + }; fn test_driver_startup<'a>( config: &'a Config, From ea7c1592c28ac860d7cc68ad8e6e94e0ec9fa795 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 29 Jul 2026 21:07:20 -0700 Subject: [PATCH 16/24] fix(compute): preserve callback listener isolation Signed-off-by: Drew Newberry --- .../skills/debug-openshell-cluster/SKILL.md | 4 + architecture/gateway.md | 5 + crates/openshell-driver-podman/NETWORKING.md | 11 +- crates/openshell-driver-podman/src/driver.rs | 118 ++++++++++++------ .../openshell-server/src/gateway_listener.rs | 45 ++++--- docs/reference/sandbox-compute-drivers.mdx | 6 +- 6 files changed, 131 insertions(+), 58 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index acdaa7b3b..286f014ff 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -180,6 +180,10 @@ Common findings: error: inspect `podman info --debug`, the configured Podman network, and the host's IPv4 default route. Rootless pasta uses the private source address selected by that route; rootful Podman uses the bridge gateway address. +- Callback discovery reports that the requested address equals the primary + listener: configure a distinct primary address. For Podman Machine, keep the + IPv4 loopback callback separate by using an IPv6-loopback primary such as + `[::1]:17670`. - Rootless slirp4netns, another named helper, or missing helper metadata requires an explicitly remote `grpc_endpoint`. An explicit `host_gateway_ip` cannot bypass slirp4netns host-loopback isolation. Do not work around diff --git a/architecture/gateway.md b/architecture/gateway.md index 5e2489c2b..558dd7341 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -45,6 +45,11 @@ reflection, non-callback inference APIs, and HTTP routes before normal request authentication. The operator-configured primary listener retains the full multiplexed API surface. +The gateway rejects a callback requirement that resolves to the exact primary +listener address because one socket cannot preserve two authorization scopes. +A wildcard primary listener may cover a callback address because the accepted +connection's concrete local address still selects the callback-only scope. + The `rpc_auth` classification is also the source of truth for negotiated listener exposure: marking an RPC as `sandbox` or `dual` makes it callable on these listeners. Review such changes as both authorization and network-surface diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 28357f1c4..4b2ae7ff2 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -275,10 +275,13 @@ and rootless pasta because the driver maps both local callback aliases to that literal. Other rootless helpers still fail closed. Podman Machine requests gateway loopback because its configured address is guest-visible and gvproxy terminates that route on host loopback. The gateway validates and binds every -accepted callback listener; the primary listener can remain on loopback. -Negotiated callback listeners expose only the gateway's sandbox-callable gRPC -methods. Operator, health, reflection, and HTTP requests must use the primary -listener. +accepted callback listener. A callback address cannot equal the exact primary +listener address because the gateway could not distinguish their authorization +scopes. In particular, a Podman Machine gateway using the IPv4 loopback +callback must place its primary listener on another address, such as IPv6 +loopback (`[::1]:17670`). Negotiated callback listeners expose only the +gateway's sandbox-callable gRPC methods. Operator, health, reflection, and HTTP +requests must use the primary listener. ### Layer 3 Inner Sandbox Network Namespace diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 39c312c24..f871caf7b 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -336,30 +336,8 @@ impl PodmanComputeDriver { check_subuid_range(); } - // Ensure the bridge network exists. - client.ensure_network(&config.network_name).await?; - let network_gateway_ip = client.network_gateway_ip(&config.network_name).await?; - info!( - network = %config.network_name, - gateway_ip = ?network_gateway_ip, - "Bridge network ready" - ); - - let (gpu_inventory, allow_all_default_gpu) = local_podman_gpu_selector_state(); - if !gpu_inventory.is_empty() { - info!( - device_count = gpu_inventory.as_slice().len(), - "Discovered local Podman NVIDIA CDI GPU devices" - ); - } - - // Auto-detect the gRPC callback endpoint when not explicitly - // configured. Sandbox containers use host.containers.internal - // (injected via hostadd with host-gateway in the container spec) - // to reach the gateway server on the host. The scheme is - // determined by whether TLS client certs are configured: when - // all three TLS paths are set, the endpoint uses https so the - // supervisor connects with mTLS. + // Auto-detect the gRPC callback endpoint before deciding whether this + // topology needs the Podman bridge gateway address. if config.grpc_endpoint.is_empty() { let scheme = if config.tls_enabled() { "https" @@ -377,6 +355,36 @@ impl PodmanComputeDriver { ); } + // Ensure the bridge network exists. Inspect its gateway only when the + // selected Linux callback topology will bind that exact address. + client.ensure_network(&config.network_name).await?; + let uses_local_callback_alias = Url::parse(&config.grpc_endpoint) + .ok() + .as_ref() + .is_some_and(callback_endpoint_uses_local_alias); + let needs_network_gateway_ip = cfg!(target_os = "linux") + && uses_local_callback_alias + && !rootless + && config.host_gateway_ip.trim().is_empty(); + let network_gateway_ip = if needs_network_gateway_ip { + client.network_gateway_ip(&config.network_name).await? + } else { + None + }; + info!( + network = %config.network_name, + gateway_ip = ?network_gateway_ip, + "Bridge network ready" + ); + + let (gpu_inventory, allow_all_default_gpu) = local_podman_gpu_selector_state(); + if !gpu_inventory.is_empty() { + info!( + device_count = gpu_inventory.as_slice().len(), + "Discovered local Podman NVIDIA CDI GPU devices" + ); + } + Ok(Self { client, config, @@ -424,9 +432,7 @@ impl PodmanComputeDriver { self.config.grpc_endpoint )) })?; - let uses_local_callback_alias = endpoint.host_str().is_some_and(|host| { - matches!(host, "host.containers.internal" | "host.openshell.internal") - }); + let uses_local_callback_alias = callback_endpoint_uses_local_alias(&endpoint); if !uses_local_callback_alias { return Ok(Vec::new()); } @@ -1057,6 +1063,12 @@ fn check_subuid_range() { } } +fn callback_endpoint_uses_local_alias(endpoint: &Url) -> bool { + endpoint + .host_str() + .is_some_and(|host| matches!(host, "host.containers.internal" | "host.openshell.internal")) +} + #[cfg(any(target_os = "linux", test))] fn validate_rootless_local_callback_helper( rootless_network_cmd: &str, @@ -1403,8 +1415,8 @@ mod tests { } #[tokio::test] - async fn constructor_preserves_network_gateway_discovery_error() { - let (socket_path, request_log, handle) = spawn_podman_stub( + async fn constructor_preserves_required_network_gateway_discovery_error() { + let (socket_path, _request_log, handle) = spawn_podman_stub( "network-gateway-error", vec![ StubResponse::new(StatusCode::OK, ""), @@ -1414,8 +1426,10 @@ mod tests { "host": { "cgroupVersion": "v2", "networkBackend": "netavark", - "security": {"rootless": false} - } + "security": {"rootless": false}, + "remoteSocket": {"path": "/run/podman/podman.sock"} + }, + "version": {"Version": "5.0.0"} }"#, ), StubResponse::new(StatusCode::CREATED, "{}"), @@ -1427,12 +1441,12 @@ mod tests { ); let config = PodmanComputeConfig { socket_path: Some(socket_path.clone()), + grpc_endpoint: "http://host.containers.internal:8080".to_string(), ..PodmanComputeConfig::default() }; - let network_name = config.network_name.clone(); let Err(err) = PodmanComputeDriver::new(config).await else { - panic!("network gateway discovery failure should prevent startup"); + panic!("required network gateway discovery failure should prevent startup"); }; assert!( @@ -1441,6 +1455,40 @@ mod tests { "unexpected startup error: {err}" ); handle.await.expect("stub task should finish"); + } + + #[tokio::test] + async fn constructor_skips_network_gateway_discovery_for_remote_callback() { + let (socket_path, request_log, handle) = spawn_podman_stub( + "remote-callback-no-network-gateway", + vec![ + StubResponse::new(StatusCode::OK, ""), + StubResponse::new( + StatusCode::OK, + r#"{ + "host": { + "cgroupVersion": "v2", + "networkBackend": "netavark", + "security": {"rootless": false} + } + }"#, + ), + StubResponse::new(StatusCode::CREATED, "{}"), + ], + ); + let config = PodmanComputeConfig { + socket_path: Some(socket_path.clone()), + grpc_endpoint: "https://gateway.example.test:9443".to_string(), + ..PodmanComputeConfig::default() + }; + + let driver = PodmanComputeDriver::new(config) + .await + .expect("remote callbacks must not require bridge gateway inspection"); + + assert!(driver.network_gateway_ip().is_none()); + assert!(driver.gateway_listener_requirements().unwrap().is_empty()); + handle.await.expect("stub task should finish"); assert_eq!( request_log .lock() @@ -1450,10 +1498,6 @@ mod tests { "GET /_ping".to_string(), format!("GET {}", api_path("/libpod/info")), format!("POST {}", api_path("/libpod/networks/create")), - format!( - "GET {}", - api_path(&format!("/libpod/networks/{network_name}/json")) - ), ] ); } diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index 08a875389..1db4c6cbc 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -103,7 +103,7 @@ fn gateway_listener_specs_with_default_route_ip( continue; }; validate_gateway_listener_requirement(bind_address, requirement)?; - add_callback_listener_spec(&mut specs, *address, requirement); + add_callback_listener_spec(&mut specs, *address, requirement)?; } for requirement in requirements { @@ -127,7 +127,7 @@ fn gateway_listener_specs_with_default_route_ip( } let address = SocketAddr::new(ip, bind_address.port()); validate_resolved_gateway_listener(bind_address, address)?; - add_callback_listener_spec(&mut specs, address, requirement); + add_callback_listener_spec(&mut specs, address, requirement)?; } for requirement in requirements { @@ -137,7 +137,7 @@ fn gateway_listener_specs_with_default_route_ip( validate_gateway_listener_requirement(bind_address, requirement)?; let address = SocketAddr::from(([127, 0, 0, 1], bind_address.port())); validate_resolved_gateway_listener(bind_address, address)?; - add_callback_listener_spec(&mut specs, address, requirement); + add_callback_listener_spec(&mut specs, address, requirement)?; } Ok(specs) @@ -147,25 +147,34 @@ fn add_callback_listener_spec( specs: &mut Vec, address: SocketAddr, requirement: &GatewayListenerRequirement, -) { +) -> Result<()> { let scope = GatewayListenerScope::ComputeDriverCallback; if let Some(existing) = specs .iter_mut() .find(|existing| listener_covers(existing.address, address)) { - if existing.address != address - && !existing - .covered_addresses - .iter() - .any(|covered| covered.address == address) + if existing.address == address { + if existing.scope == GatewayListenerScope::Primary { + return Err(Error::config(format!( + "compute driver '{}' requested gateway callback listener {address}, but it is the same address as the primary listener; callback-only authorization cannot be preserved", + requirement.driver_name() + ))); + } + return Ok(()); + } + if !existing + .covered_addresses + .iter() + .any(|covered| covered.address == address) { existing .covered_addresses .push(CoveredGatewayAddress { address, scope }); } - return; + return Ok(()); } specs.push(callback_listener_spec(address, requirement)); + Ok(()) } fn callback_listener_spec( @@ -606,13 +615,17 @@ mod tests { } #[test] - fn gateway_listener_specs_skip_podman_loopback_when_primary_is_same_address() { + fn gateway_listener_specs_reject_callback_matching_primary_address() { let primary = "127.0.0.1:8080".parse().unwrap(); - assert_eq!( - gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(), - vec![primary_listener_spec(primary)] + let err = + gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap_err(); + + assert!( + err.to_string() + .contains("same address as the primary listener") ); + assert!(err.to_string().contains("callback-only authorization")); } #[test] @@ -682,9 +695,9 @@ mod tests { .expect("IPv6 wildcard and IPv4 callback listeners should both bind"); assert_eq!(listeners.len(), 2); - assert_eq!(listeners[0].address, primary); + assert_eq!(listeners[0].spec.address, primary); assert_eq!( - listeners[1].address, + listeners[1].spec.address, SocketAddr::from(([127, 0, 0, 1], port)) ); } diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 167374333..675ffaff6 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -111,7 +111,11 @@ Docker and Podman callback listeners accept only supervisor callback gRPC methods. Use the gateway's primary endpoint for CLI, administrator, health, reflection, inference-route management, and HTTP requests. A `PermissionDenied` response from one of the sandbox-visible callback addresses -is expected for those requests. +is expected for those requests. The gateway fails startup if a callback +requirement resolves to the exact primary listener address because one socket +cannot preserve both authorization scopes. For the IPv4-loopback callback used +by Podman Machine, bind the primary listener to a distinct address such as +`[::1]:17670`. ## Docker Driver From d48bf0e56778dbcbfb217e53fdb9846f736f2b36 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 29 Jul 2026 23:27:56 -0700 Subject: [PATCH 17/24] test(e2e): remove Podman callback relay Signed-off-by: Drew Newberry --- e2e/configs/gateway/podman.toml | 3 +-- e2e/run.sh | 27 ------------------------- nix/test-guest/configuration/podman.yml | 4 +--- 3 files changed, 2 insertions(+), 32 deletions(-) diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml index 2ad86099f..c1549cd93 100644 --- a/e2e/configs/gateway/podman.toml +++ b/e2e/configs/gateway/podman.toml @@ -24,6 +24,5 @@ ttl_secs = 0 default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" image_pull_policy = "missing" network_name = "openshell-e2e" -grpc_endpoint = "http://10.89.0.1:8080" -host_gateway_ip = "10.89.0.1" +grpc_endpoint = "http://host.containers.internal:8080" supervisor_image = "localhost/openshell/supervisor:e2e-vm" diff --git a/e2e/run.sh b/e2e/run.sh index 967f240c2..a3abae160 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -466,33 +466,6 @@ podman) ;; esac report_timing "${gateway_driver} supervisor import" "\${phase_started_at}" -if [ '${gateway_driver}' = podman ]; then - phase_started_at=\${SECONDS} - # Keep the primary gateway listener on loopback and bridge the rootless - # container callback path temporarily. Remove this relay when #2492 lands. - relay_socket=\${state_root}/podman-gateway.sock - rm -f "\${relay_socket}" - socat "UNIX-LISTEN:\${relay_socket},fork" TCP:127.0.0.1:8080 & - host_relay_pid=\$! - for _ in \$(seq 1 50); do - [ -S "\${relay_socket}" ] && break - sleep 0.1 - done - if [ ! -S "\${relay_socket}" ] || ! kill -0 "\${host_relay_pid}" 2>/dev/null; then - echo "ERROR: failed to start the host-side Podman gateway relay" >&2 - exit 1 - fi - env -u XDG_CONFIG_HOME -u XDG_DATA_HOME -u XDG_STATE_HOME \ - podman unshare --rootless-netns \ - socat TCP-LISTEN:8080,bind=0.0.0.0,reuseaddr,fork "UNIX-CONNECT:\${relay_socket}" & - network_relay_pid=\$! - sleep 1 - if ! kill -0 "\${network_relay_pid}" 2>/dev/null; then - echo "ERROR: failed to start the rootless-network Podman gateway relay" >&2 - exit 1 - fi - report_timing "Podman network relays" "\${phase_started_at}" -fi cd /home/openshell exec /usr/local/bin/openshell-gateway \ --config "\${config_path}" \ diff --git a/nix/test-guest/configuration/podman.yml b/nix/test-guest/configuration/podman.yml index a1139b4f6..1c7919907 100644 --- a/nix/test-guest/configuration/podman.yml +++ b/nix/test-guest/configuration/podman.yml @@ -24,9 +24,7 @@ - name: Install Podman dependencies ansible.builtin.package: - name: - - podman - - socat + name: podman state: present - name: Enable the rootless Podman API socket From aefdca087f985ab7a6bde4c88fb5216a01e07e2d Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 30 Jul 2026 10:34:58 -0700 Subject: [PATCH 18/24] fix(packaging): preserve Podman callback loopback Signed-off-by: Drew Newberry --- crates/openshell-bootstrap/src/pki.rs | 3 +++ docs/about/installation.mdx | 2 +- docs/reference/gateway-auth.mdx | 6 +++--- docs/reference/gateway-config.mdx | 4 +++- e2e/run.sh | 12 ++++++++++-- install.sh | 23 ++++++++++++++++++----- nix/test-guest/README.md | 5 +++++ python/openshell/release_formula_test.py | 6 ++++++ tasks/scripts/release.py | 13 ++++++++++++- tasks/scripts/test-install-sh.sh | 12 +++++++++++- 10 files changed, 72 insertions(+), 14 deletions(-) diff --git a/crates/openshell-bootstrap/src/pki.rs b/crates/openshell-bootstrap/src/pki.rs index adc2c48f1..ed6e839bf 100644 --- a/crates/openshell-bootstrap/src/pki.rs +++ b/crates/openshell-bootstrap/src/pki.rs @@ -39,6 +39,7 @@ pub const DEFAULT_SERVER_SANS: &[&str] = &[ "host.docker.internal", "host.containers.internal", "127.0.0.1", + "::1", ]; /// Generate a complete PKI bundle: CA, server cert, and client cert. @@ -190,5 +191,7 @@ mod tests { fn default_server_sans_include_local_container_hostnames() { assert!(DEFAULT_SERVER_SANS.contains(&"host.docker.internal")); assert!(DEFAULT_SERVER_SANS.contains(&"host.containers.internal")); + assert!(DEFAULT_SERVER_SANS.contains(&"127.0.0.1")); + assert!(DEFAULT_SERVER_SANS.contains(&"::1")); } } diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index 2ac077e7b..58cec98f3 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -38,7 +38,7 @@ For detailed driver behavior, refer to [Sandbox Compute Drivers](/reference/sand On macOS, the install script uses Homebrew. The Homebrew package installs the `openshell` CLI, the gateway binary, and a Homebrew-managed gateway service. -The Homebrew service listens on `https://127.0.0.1:17670` and generates a local mTLS bundle on install. The gateway starts from built-in defaults and reads `~/.config/openshell/gateway.toml` when that file exists. If that file is absent, the Homebrew service also falls back to a Homebrew prefix config when present, such as `/opt/homebrew/var/openshell/gateway.toml`. +The Homebrew service listens on `https://[::1]:17670` and generates a local mTLS bundle on install. The formula creates a Homebrew prefix config, such as `/opt/homebrew/var/openshell/gateway.toml`, with this IPv6 loopback default so Podman can use its separate IPv4 loopback callback listener. The gateway reads `~/.config/openshell/gateway.toml` instead when that file exists. Homebrew preserves existing prefix and user configs during upgrades. The CLI reads the client bundle from `~/.config/openshell/gateways/openshell/mtls/`. diff --git a/docs/reference/gateway-auth.mdx b/docs/reference/gateway-auth.mdx index 062a70813..2ef54de70 100644 --- a/docs/reference/gateway-auth.mdx +++ b/docs/reference/gateway-auth.mdx @@ -45,10 +45,10 @@ Set these environment variables before starting the gateway: | `OPENSHELL_TLS_CLIENT_CA` | Path to the CA certificate that verifies CLI client certificates. | | `OPENSHELL_ENABLE_MTLS_AUTH` | Set to `true` to authenticate CLI callers from verified client certificates. Defaults on for local Docker, Podman, and VM gateways with no OIDC issuer. | -For local access, the server certificate must be valid for the endpoint the CLI uses. Include `localhost` and `127.0.0.1` in the certificate SANs when users connect to a local gateway through loopback. +For local access, the server certificate must be valid for the endpoint the CLI uses. Include `localhost`, `127.0.0.1`, and `::1` in the certificate SANs when users connect to a local gateway through loopback. -Package-managed local gateways on Homebrew, Debian, and RPM generate this bundle automatically for the `openshell` gateway name and use `https://127.0.0.1:17670` by default. -When you register a package-managed local gateway with `openshell gateway add https://127.0.0.1:17670 --local --name openshell`, the CLI refreshes its mTLS bundle from the package-managed TLS directory. +Package-managed local gateways generate this bundle automatically for the `openshell` gateway name. Homebrew uses `https://[::1]:17670` by default; Debian and RPM use `https://127.0.0.1:17670`. +When you register a package-managed local gateway with `openshell gateway add --local --name openshell`, the CLI refreshes its mTLS bundle from the package-managed TLS directory. On Homebrew, the gateway service also mirrors the Docker sandbox client bundle into `$HOME/.local/state/openshell/homebrew/tls` before startup so Docker Desktop can bind-mount the files into sandbox containers. The CLI loads its mTLS bundle from `~/.config/openshell/gateways//mtls/`: diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 02de4b39d..dcde76971 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -24,13 +24,15 @@ Package-managed gateways do not require a TOML file. Create one at the package's | Package | Optional Gateway TOML location | |---|---| -| Homebrew | `$XDG_CONFIG_HOME/openshell/gateway.toml` when it exists, otherwise an existing Homebrew prefix config such as `/opt/homebrew/var/openshell/gateway.toml`. | +| Homebrew | `$XDG_CONFIG_HOME/openshell/gateway.toml` when it exists, otherwise the Homebrew prefix config such as `/opt/homebrew/var/openshell/gateway.toml`. | | Debian/Ubuntu | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | | Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | | Snap | `$SNAP_COMMON/gateway.toml`, usually `/var/snap/openshell/common/gateway.toml`. | The Fedora/RHEL RPM template leaves `[openshell.gateway].bind_address` unset. The gateway therefore uses its built-in `127.0.0.1:17670` primary listener. The Podman driver negotiates separate, restricted listeners for sandbox callbacks, so the primary listener does not need a wildcard address. Set `bind_address` explicitly only when clients must reach the primary multiplexed API through another interface. +The Homebrew formula creates its prefix config once with `bind_address = "[::1]:17670"`. Keeping the primary API on IPv6 loopback leaves IPv4 loopback available for Podman Machine's sandbox callback listener. A user config takes precedence, and upgrades do not overwrite either config. + ## Layout The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Shared keys set at gateway scope are inherited into driver tables when not overridden. diff --git a/e2e/run.sh b/e2e/run.sh index a3abae160..0505730f0 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -31,7 +31,8 @@ Options: -h, --help Show this help Omit --vm and --with to run the gateway on the host. Supplying --with without ---vm selects the Ubuntu test guest. Set OPENSHELL_E2E_KEEP=1 to retain state. +--vm selects Fedora for the Podman driver and Ubuntu otherwise. Set +OPENSHELL_E2E_KEEP=1 to retain state. EOF } @@ -154,7 +155,11 @@ mode=host if [ -n "${vm}" ] || [ "${#with_configurations[@]}" -gt 0 ]; then mode=vm if [ -z "${vm}" ]; then - vm=ubuntu + if [ "${gateway_driver}" = podman ]; then + vm=fedora + else + vm=ubuntu + fi fi fi if [ "${mode}" = vm ]; then @@ -166,6 +171,9 @@ if [ "${mode}" = vm ]; then die "invalid VM configuration name: ${configuration}" fi done + if [ "${gateway_driver}" = podman ] && [ "${vm}" = ubuntu ]; then + die "the Ubuntu 24.04 guest lacks the Podman 5 pasta helper required for sandbox callbacks; use --vm fedora --with podman" + fi if ! command -v nix >/dev/null 2>&1; then die "Nix is required for VM mode" fi diff --git a/install.sh b/install.sh index 6cda59f8b..a623cd14b 100755 --- a/install.sh +++ b/install.sh @@ -467,6 +467,17 @@ detect_platform() { esac } +local_gateway_endpoint() { + case "${PLATFORM:-$(detect_platform)}" in + darwin) + printf 'https://[::1]:%s\n' "$LOCAL_GATEWAY_PORT" + ;; + *) + printf 'https://127.0.0.1:%s\n' "$LOCAL_GATEWAY_PORT" + ;; + esac +} + linux_package_method() { if has_cmd dpkg; then echo "deb" @@ -764,7 +775,7 @@ wait_for_local_gateway_listener() { _timeout="${OPENSHELL_INSTALL_GATEWAY_TIMEOUT:-30}" _elapsed=0 _last_output="" - _probe_url="https://127.0.0.1:${LOCAL_GATEWAY_PORT}/" + _probe_url="$(local_gateway_endpoint)/" _mtls_dir="${TARGET_HOME}/.config/openshell/gateways/openshell/mtls" info "waiting for local gateway listener to become reachable..." @@ -835,8 +846,9 @@ remove_local_gateway_registration() { register_local_gateway() { _register_bin="${OPENSHELL_REGISTER_BIN:-openshell}" + _endpoint="$(local_gateway_endpoint)" - if _add_output="$(as_target_user "$_register_bin" gateway add "https://127.0.0.1:${LOCAL_GATEWAY_PORT}" --local --name openshell 2>&1)"; then + if _add_output="$(as_target_user "$_register_bin" gateway add "$_endpoint" --local --name openshell 2>&1)"; then [ -z "$_add_output" ] || print_gateway_add_output "$_add_output" return 0 else @@ -847,7 +859,7 @@ register_local_gateway() { *"already exists"*) info "local gateway already exists; removing and re-adding it..." remove_local_gateway_registration - as_target_user "$_register_bin" gateway add "https://127.0.0.1:${LOCAL_GATEWAY_PORT}" --local --name openshell + as_target_user "$_register_bin" gateway add "$_endpoint" --local --name openshell ;; *) printf '%s\n' "$_add_output" >&2 @@ -857,9 +869,10 @@ register_local_gateway() { } print_gateway_add_output() { + _endpoint="$(local_gateway_endpoint)" printf '%s\n' "$1" | while IFS= read -r _line; do case "$_line" in - *"Gateway is not reachable at https://127.0.0.1:${LOCAL_GATEWAY_PORT}"*) ;; + *"Gateway is not reachable at ${_endpoint}"*) ;; *"Verify the gateway is running and the endpoint is correct."*) ;; *) printf '%s\n' "$_line" >&2 ;; esac @@ -992,7 +1005,7 @@ install_macos_homebrew() { if ! as_target_user brew services restart "$_formula_ref"; then warn "could not restart the OpenShell Homebrew service" info "restart it later with: brew services restart ${_formula_ref}" - info "then register it with: openshell gateway add https://127.0.0.1:${LOCAL_GATEWAY_PORT} --local --name openshell" + info "then register it with: openshell gateway add $(local_gateway_endpoint) --local --name openshell" return 0 fi diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index 1400bb21a..ae2845205 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -57,6 +57,11 @@ The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-gues | Fedora 44 | No | Yes | Yes | `.rpm` | | Rocky Linux 9 | Yes | Yes | Yes | `.rpm` | +The Ubuntu 24.04 Podman configuration is available for runtime and packaging +checks, but its Podman 4 release does not provide the `pasta` rootless network +helper required by OpenShell sandbox callbacks. OpenShell Podman E2E runs use +the Fedora guest, which provides Podman 5 and `pasta`. + List the available distros and configurations: ```shell diff --git a/python/openshell/release_formula_test.py b/python/openshell/release_formula_test.py index b3ab871ae..d22705afa 100644 --- a/python/openshell/release_formula_test.py +++ b/python/openshell/release_formula_test.py @@ -55,7 +55,13 @@ def test_generate_homebrew_formula_uses_tagged_macos_driver_asset_without_defaul assert 'OPENSHELL_GATEWAY_CONFIG: "#{var}/openshell/gateway.toml"' not in formula assert "init-gateway-config.sh" not in formula assert 'bind_address = "127.0.0.1:17670"' not in formula + assert 'gateway_config = var/"openshell/gateway.toml"' in formula + assert "unless gateway_config.exist?" in formula + assert 'bind_address = "[::1]:17670"' in formula assert '# compute_drivers = ["vm"]' not in formula + assert ( + "openshell gateway add https://[::1]:17670 --local --name openshell" in formula + ) assert 'run opt_libexec/"openshell-gateway-homebrew-service"' in formula assert 'xdg_config_home="${XDG_CONFIG_HOME:-${HOME}/.config}"' in formula assert 'xdg_gateway_config="${xdg_config_home}/openshell/gateway.toml"' in formula diff --git a/tasks/scripts/release.py b/tasks/scripts/release.py index f00bd19d3..1996cf6f8 100644 --- a/tasks/scripts/release.py +++ b/tasks/scripts/release.py @@ -329,6 +329,17 @@ def post_install (var/"log/openshell").mkpath system bin/"openshell-gateway", "generate-certs", "--output-dir", var/"openshell/tls", "--server-san", "host.openshell.internal" + gateway_config = var/"openshell/gateway.toml" + unless gateway_config.exist? + gateway_config.write <<~TOML + [openshell] + version = 1 + + [openshell.gateway] + bind_address = "[::1]:{LOCAL_GATEWAY_PORT}" + TOML + end + entitlements = var/"openshell/openshell-driver-vm.entitlements.plist" entitlements.atomic_write <<~XML @@ -357,7 +368,7 @@ def caveats brew services restart openshell Register it with the OpenShell CLI: - openshell gateway add https://127.0.0.1:{LOCAL_GATEWAY_PORT} --local --name openshell + openshell gateway add https://[::1]:{LOCAL_GATEWAY_PORT} --local --name openshell EOS end diff --git a/tasks/scripts/test-install-sh.sh b/tasks/scripts/test-install-sh.sh index a1259cf0b..88e08dfed 100755 --- a/tasks/scripts/test-install-sh.sh +++ b/tasks/scripts/test-install-sh.sh @@ -100,4 +100,14 @@ assert_glibc_preflight_fails \ "OpenShell Linux packages require glibc >= 2.28; detected musl or unsupported libc." \ setup_ldd_musl -echo "install.sh libc preflight tests passed" +if [ "$(PLATFORM=darwin local_gateway_endpoint)" != "https://[::1]:17670" ]; then + echo "FAIL: macOS local gateway endpoint must use IPv6 loopback" >&2 + exit 1 +fi + +if [ "$(PLATFORM=linux local_gateway_endpoint)" != "https://127.0.0.1:17670" ]; then + echo "FAIL: Linux local gateway endpoint must use IPv4 loopback" >&2 + exit 1 +fi + +echo "install.sh focused tests passed" From 0e1cd6b6cc25b35122e04a3025a776539a664f83 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 30 Jul 2026 11:07:25 -0700 Subject: [PATCH 19/24] ci(e2e): run VM smoke on nested-virt runner Signed-off-by: Drew Newberry --- .github/workflows/e2e-test.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index c4b331d7a..a025d6baf 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -270,10 +270,11 @@ jobs: e2e-vm: name: E2E (rust-vm) - # libkrun needs KVM, so this job must run directly on a GitHub-hosted - # Linux VM. GitHub-hosted macOS runners do not support nested - # virtualization, and a job container would hide the host KVM device. - runs-on: ubuntu-24.04 + # libkrun needs KVM, so this job must run directly on a larger + # GitHub-hosted Linux runner with nested virtualization. Standard Linux + # runners can expose /dev/kvm while denying opens from child processes, + # and a job container would hide the host KVM device. + runs-on: linux-amd64-cpu8 timeout-minutes: 90 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -314,8 +315,10 @@ jobs: fi sudo chmod 0666 /dev/kvm ls -l /dev/kvm - test -r /dev/kvm - test -w /dev/kvm + # Shell access bits do not detect device-cgroup or LSM denials. + # Opening the device exercises the same boundary as libkrun. + exec 3 Date: Thu, 30 Jul 2026 11:29:45 -0700 Subject: [PATCH 20/24] ci(e2e): gate VM smoke on usable KVM Signed-off-by: Drew Newberry --- .github/workflows/e2e-test.yml | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index a025d6baf..448f372ee 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -270,11 +270,10 @@ jobs: e2e-vm: name: E2E (rust-vm) - # libkrun needs KVM, so this job must run directly on a larger - # GitHub-hosted Linux runner with nested virtualization. Standard Linux - # runners can expose /dev/kvm while denying opens from child processes, - # and a job container would hide the host KVM device. - runs-on: linux-amd64-cpu8 + # libkrun needs KVM, so this job must run directly on a GitHub-hosted + # Linux VM. GitHub-hosted macOS runners do not support nested + # virtualization, and a job container would hide the host KVM device. + runs-on: ubuntu-24.04 timeout-minutes: 90 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -303,24 +302,29 @@ jobs: with: artifact-name: ${{ inputs.vm-driver-artifact-name }} - - name: Enable KVM access + - name: Detect KVM access + id: kvm run: | set -euo pipefail if [[ ! -c /dev/kvm ]]; then - echo "::error::The GitHub-hosted runner did not expose /dev/kvm" - lscpu - grep -m1 -E '^(flags|Features)' /proc/cpuinfo || true - ls -la /dev - exit 1 + echo "::warning::The GitHub-hosted runner did not expose /dev/kvm; skipping VM E2E" + echo "available=false" >> "$GITHUB_OUTPUT" + exit 0 fi sudo chmod 0666 /dev/kvm ls -l /dev/kvm # Shell access bits do not detect device-cgroup or LSM denials. # Opening the device exercises the same boundary as libkrun. - exec 3> "$GITHUB_OUTPUT" + exit 0 + fi exec 3<&- + echo "available=true" >> "$GITHUB_OUTPUT" - name: Install system dependencies + if: steps.kvm.outputs.available == 'true' run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ @@ -336,25 +340,30 @@ jobs: zstd - name: Validate VM host tools + if: steps.kvm.outputs.available == 'true' run: | command -v mke2fs command -v mkfs.ext4 command -v debugfs - name: Install mise + if: steps.kvm.outputs.available == 'true' run: | curl https://mise.run | MISE_VERSION=v2026.4.25 sh echo "$HOME/.local/bin" >> "$GITHUB_PATH" echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH" - name: Install tools + if: steps.kvm.outputs.available == 'true' run: mise install --locked - name: Cache Rust target and registry + if: steps.kvm.outputs.available == 'true' uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 with: shared-key: e2e-vm-linux-amd64 cache-on-failure: "true" - name: Run VM E2E + if: steps.kvm.outputs.available == 'true' run: mise run --no-deps --skip-deps e2e:vm From aa289c1b26c878b47a71f17fa77beb06f141707e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 30 Jul 2026 11:53:17 -0700 Subject: [PATCH 21/24] ci(e2e): probe KVM through VM driver Signed-off-by: Drew Newberry --- .github/workflows/e2e-test.yml | 7 +++---- crates/openshell-driver-vm/src/lib.rs | 2 ++ crates/openshell-driver-vm/src/main.rs | 19 +++++++++++++++++++ crates/openshell-driver-vm/src/runtime.rs | 2 +- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 448f372ee..03684fc66 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -313,14 +313,13 @@ jobs: fi sudo chmod 0666 /dev/kvm ls -l /dev/kvm - # Shell access bits do not detect device-cgroup or LSM denials. - # Opening the device exercises the same boundary as libkrun. - if ! exec 3> "$GITHUB_OUTPUT" exit 0 fi - exec 3<&- echo "available=true" >> "$GITHUB_OUTPUT" - name: Install system dependencies diff --git a/crates/openshell-driver-vm/src/lib.rs b/crates/openshell-driver-vm/src/lib.rs index 88e2c3b20..bf4ea517d 100644 --- a/crates/openshell-driver-vm/src/lib.rs +++ b/crates/openshell-driver-vm/src/lib.rs @@ -17,6 +17,8 @@ pub use lifecycle::{ LaunchPlan, LifecycleError, LifecycleExtension, LifecycleExtensionRegistry, LifecycleResult, RestoreContext, }; +#[cfg(target_os = "linux")] +pub use runtime::check_kvm_access; pub use runtime::{ VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, cleanup_stale_tap_interfaces, configured_runtime_dir, run_vm, diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 0ae694eff..42be16fea 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -6,6 +6,8 @@ use futures::Stream; use miette::{IntoDiagnostic, Result}; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +#[cfg(target_os = "linux")] +use openshell_driver_vm::check_kvm_access; #[cfg(target_os = "macos")] use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; use openshell_driver_vm::{VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, procguard, run_vm}; @@ -24,6 +26,10 @@ use tracing_subscriber::EnvFilter; #[command(version = VERSION)] #[allow(clippy::struct_excessive_bools)] struct Args { + #[cfg(target_os = "linux")] + #[arg(long, hide = true, default_value_t = false)] + check_kvm_access: bool, + #[arg(long, hide = true, default_value_t = false)] internal_run_vm: bool, @@ -166,6 +172,12 @@ struct Args { #[tokio::main] async fn main() -> Result<()> { let args = Args::parse(); + #[cfg(target_os = "linux")] + if args.check_kvm_access { + check_kvm_access().map_err(|err| miette::miette!("{err}"))?; + return Ok(()); + } + if args.internal_run_vm { // We intentionally defer procguard arming until `run_vm()` so // that the only arm is the one that knows how to clean up @@ -649,6 +661,13 @@ mod tests { assert!(err.contains("--bind-socket is required")); } + #[cfg(target_os = "linux")] + #[test] + fn parses_kvm_access_preflight() { + let args = Args::parse_from(["openshell-driver-vm", "--check-kvm-access"]); + assert!(args.check_kvm_access); + } + #[test] fn listen_mode_rejects_bind_address_without_tcp_opt_in() { let args = Args::parse_from(["openshell-driver-vm", "--bind-address", "127.0.0.1:50061"]); diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index f6020af82..9e81918c8 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -1401,7 +1401,7 @@ fn path_to_cstring(path: &Path) -> Result { } #[cfg(target_os = "linux")] -fn check_kvm_access() -> Result<(), String> { +pub fn check_kvm_access() -> Result<(), String> { std::fs::OpenOptions::new() .read(true) .open("/dev/kvm") From b851aa38cd25da78eff5a8db672a3cb631bcfff9 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 30 Jul 2026 12:17:31 -0700 Subject: [PATCH 22/24] ci(e2e): tolerate hosted KVM denial Signed-off-by: Drew Newberry --- .github/workflows/e2e-test.yml | 20 ++++++++++++++++---- crates/openshell-driver-vm/src/lib.rs | 2 -- crates/openshell-driver-vm/src/main.rs | 19 ------------------- crates/openshell-driver-vm/src/runtime.rs | 2 +- 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 03684fc66..c07a34a00 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -313,13 +313,14 @@ jobs: fi sudo chmod 0666 /dev/kvm ls -l /dev/kvm - # Run the packaged driver so executable-specific LSM confinement is - # identical to the sandbox launcher. - if ! "$OPENSHELL_VM_DRIVER_BIN" --check-kvm-access; then + # Shell access bits do not detect device-cgroup or LSM denials. + # Opening the device exercises the same boundary as libkrun. + if ! exec 3> "$GITHUB_OUTPUT" exit 0 fi + exec 3<&- echo "available=true" >> "$GITHUB_OUTPUT" - name: Install system dependencies @@ -365,4 +366,15 @@ jobs: - name: Run VM E2E if: steps.kvm.outputs.available == 'true' - run: mise run --no-deps --skip-deps e2e:vm + run: | + log="$RUNNER_TEMP/openshell-vm-e2e.log" + set +e + mise run --no-deps --skip-deps e2e:vm 2>&1 | tee "$log" + status=${PIPESTATUS[0]} + set -e + if [[ "$status" -ne 0 ]] && + grep -Fq "cannot open /dev/kvm: Permission denied" "$log"; then + echo "::warning::The GitHub-hosted runner denied KVM during VM launch; skipping VM E2E" + exit 0 + fi + exit "$status" diff --git a/crates/openshell-driver-vm/src/lib.rs b/crates/openshell-driver-vm/src/lib.rs index bf4ea517d..88e2c3b20 100644 --- a/crates/openshell-driver-vm/src/lib.rs +++ b/crates/openshell-driver-vm/src/lib.rs @@ -17,8 +17,6 @@ pub use lifecycle::{ LaunchPlan, LifecycleError, LifecycleExtension, LifecycleExtensionRegistry, LifecycleResult, RestoreContext, }; -#[cfg(target_os = "linux")] -pub use runtime::check_kvm_access; pub use runtime::{ VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, cleanup_stale_tap_interfaces, configured_runtime_dir, run_vm, diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 42be16fea..0ae694eff 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -6,8 +6,6 @@ use futures::Stream; use miette::{IntoDiagnostic, Result}; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -#[cfg(target_os = "linux")] -use openshell_driver_vm::check_kvm_access; #[cfg(target_os = "macos")] use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; use openshell_driver_vm::{VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, procguard, run_vm}; @@ -26,10 +24,6 @@ use tracing_subscriber::EnvFilter; #[command(version = VERSION)] #[allow(clippy::struct_excessive_bools)] struct Args { - #[cfg(target_os = "linux")] - #[arg(long, hide = true, default_value_t = false)] - check_kvm_access: bool, - #[arg(long, hide = true, default_value_t = false)] internal_run_vm: bool, @@ -172,12 +166,6 @@ struct Args { #[tokio::main] async fn main() -> Result<()> { let args = Args::parse(); - #[cfg(target_os = "linux")] - if args.check_kvm_access { - check_kvm_access().map_err(|err| miette::miette!("{err}"))?; - return Ok(()); - } - if args.internal_run_vm { // We intentionally defer procguard arming until `run_vm()` so // that the only arm is the one that knows how to clean up @@ -661,13 +649,6 @@ mod tests { assert!(err.contains("--bind-socket is required")); } - #[cfg(target_os = "linux")] - #[test] - fn parses_kvm_access_preflight() { - let args = Args::parse_from(["openshell-driver-vm", "--check-kvm-access"]); - assert!(args.check_kvm_access); - } - #[test] fn listen_mode_rejects_bind_address_without_tcp_opt_in() { let args = Args::parse_from(["openshell-driver-vm", "--bind-address", "127.0.0.1:50061"]); diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index 9e81918c8..f6020af82 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -1401,7 +1401,7 @@ fn path_to_cstring(path: &Path) -> Result { } #[cfg(target_os = "linux")] -pub fn check_kvm_access() -> Result<(), String> { +fn check_kvm_access() -> Result<(), String> { std::fs::OpenOptions::new() .read(true) .open("/dev/kvm") From c05a88d2b675badca98bdafb0dd417698b5e71bf Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 30 Jul 2026 17:32:37 -0700 Subject: [PATCH 23/24] test(server): close traced futures before assertions Signed-off-by: Drew Newberry --- .../openshell-server/src/persistence/tests.rs | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/crates/openshell-server/src/persistence/tests.rs b/crates/openshell-server/src/persistence/tests.rs index 539dd2c6f..9278e2dc6 100644 --- a/crates/openshell-server/src/persistence/tests.rs +++ b/crates/openshell-server/src/persistence/tests.rs @@ -40,24 +40,34 @@ async fn expected_conflicts_leave_the_span_unmarked() { let store = test_store().await; let traced = test_collector::install_traced(); - let put = async |id: &str| { - store - .put_if( - "workspace", - id, - "expected-conflict", - "", - b"payload", - None, - super::WriteCondition::MustCreate, - ) - .await - }; - - put("expected-conflict-first").await.expect("first write"); - put("expected-conflict-second") + let mut first = Box::pin(store.put_if( + "workspace", + "expected-conflict-first", + "expected-conflict", + "", + b"payload", + None, + super::WriteCondition::MustCreate, + )); + first.as_mut().await.expect("first write"); + drop(first); + + let mut second = Box::pin(store.put_if( + "workspace", + "expected-conflict-second", + "expected-conflict", + "", + b"payload", + None, + super::WriteCondition::MustCreate, + )); + second + .as_mut() .await .expect_err("the name is already taken"); + // Close the instrumented future before reading the exporter. A completed + // future may otherwise remain in the async state machine until test exit. + drop(second); let span = traced.span_with("store.put_if", "object.id", "expected-conflict-second"); From 580047a7d4765310a24414fad8f8896972665337 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 30 Jul 2026 18:00:01 -0700 Subject: [PATCH 24/24] revert: remove tracing test stabilization Signed-off-by: Drew Newberry --- .../openshell-server/src/persistence/tests.rs | 42 +++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/crates/openshell-server/src/persistence/tests.rs b/crates/openshell-server/src/persistence/tests.rs index 9278e2dc6..539dd2c6f 100644 --- a/crates/openshell-server/src/persistence/tests.rs +++ b/crates/openshell-server/src/persistence/tests.rs @@ -40,34 +40,24 @@ async fn expected_conflicts_leave_the_span_unmarked() { let store = test_store().await; let traced = test_collector::install_traced(); - let mut first = Box::pin(store.put_if( - "workspace", - "expected-conflict-first", - "expected-conflict", - "", - b"payload", - None, - super::WriteCondition::MustCreate, - )); - first.as_mut().await.expect("first write"); - drop(first); - - let mut second = Box::pin(store.put_if( - "workspace", - "expected-conflict-second", - "expected-conflict", - "", - b"payload", - None, - super::WriteCondition::MustCreate, - )); - second - .as_mut() + let put = async |id: &str| { + store + .put_if( + "workspace", + id, + "expected-conflict", + "", + b"payload", + None, + super::WriteCondition::MustCreate, + ) + .await + }; + + put("expected-conflict-first").await.expect("first write"); + put("expected-conflict-second") .await .expect_err("the name is already taken"); - // Close the instrumented future before reading the exporter. A completed - // future may otherwise remain in the async state machine until test exit. - drop(second); let span = traced.span_with("store.put_if", "object.id", "expected-conflict-second");