From a7e1ee75e3374a1bf0be2c4c6a2ee4019554b2ec Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 30 Jul 2026 13:19:21 -0700 Subject: [PATCH] feat(podman): honor OCI image working directories Signed-off-by: Matthew Grossman --- .../skills/debug-openshell-cluster/SKILL.md | 2 + .github/workflows/e2e-test.yml | 4 +- architecture/compute-runtimes.md | 23 +- crates/openshell-core/src/sandbox_env.rs | 7 + crates/openshell-driver-docker/src/lib.rs | 8 + crates/openshell-driver-docker/src/tests.rs | 9 + crates/openshell-driver-podman/README.md | 40 +- crates/openshell-driver-podman/src/client.rs | 128 +++++- .../openshell-driver-podman/src/container.rs | 419 +++++++++++++++++- crates/openshell-driver-podman/src/driver.rs | 287 ++++++++++-- crates/openshell-sandbox/src/lib.rs | 34 +- crates/openshell-sandbox/src/main.rs | 101 +++++ crates/openshell-server/src/compute/mod.rs | 42 ++ .../src/process.rs | 250 ++++++++++- .../openshell-supervisor-process/src/ssh.rs | 4 +- docs/reference/sandbox-compute-drivers.mdx | 36 +- e2e/rust/tests/custom_image.rs | 3 + e2e/rust/tests/driver_config_volume.rs | 12 +- e2e/rust/tests/podman_oci_identity.rs | 51 ++- proto/compute_driver.proto | 11 + 20 files changed, 1353 insertions(+), 118 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 9ef35c9425..d4bdd9b515 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -178,6 +178,8 @@ Common findings: - Rootless networking unavailable: inspect Podman network configuration. - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Root and missing identities are rejected. +- Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, or supplementary groups must already be able to traverse every parent and write and enter the directory. Podman checks the original image in a networkless temporary probe before attaching the workspace volume, so inspect the probe failure in gateway logs. +- If Podman reports probe cleanup or timeout failures, inspect temporary containers with `podman ps -a --filter name=workdir-probe` and gateway logs. The driver force-removes the probe on every normal success or failure path. - Supervisor cannot call back: check callback endpoint and gateway logs. ### Step 6: Check Kubernetes Helm Gateways diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index d671149999..f1cbd1097e 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -137,8 +137,8 @@ jobs: fail-fast: false matrix: include: - # Ubuntu 24.04 matches the environment reported in #2069 and ships - # Podman 4.x. The probe records whether AppArmor blocks the drop. + # Ubuntu 24.04 matches the AppArmor environment reported in #2069. + # GitHub's runner image now supplies the supported Podman 5.x line. - runner: ubuntu-24.04 podman_major: "4" podman_package_version: "4.9.3+ds1-1ubuntu0.2" diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 63083d8319..7eb34967c0 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -185,8 +185,8 @@ The gateway preserves whether each policy process field was omitted. The active driver then supplies one authoritative identity input to the supervisor: - Docker and Podman inspect the final sandbox image, pin container creation to - its immutable image ID, and pass its raw OCI `Config.User`. Docker also - resolves the workspace from OCI `Config.WorkingDir` during that inspection. + its immutable image ID, and pass its raw OCI `Config.User`. They also resolve + the workspace from OCI `Config.WorkingDir` during that inspection. - Kubernetes passes its platform-resolved numeric UID/GID, including OpenShift SCC-derived values. - VM keeps its existing guest identity behavior. @@ -199,7 +199,7 @@ and uses the same privilege-drop path for direct and SSH children. When a declaration omits the group, the supervisor fills it with the user's numeric primary GID. It does not rewrite the account files. -Docker uses an absolute OCI working directory as the workspace. An +Docker and Podman use an absolute OCI working directory as the workspace. An empty, root (`/`), or explicit `/sandbox` declaration uses `/sandbox`, which OpenShell creates and owns as a compatibility workspace. Any other workdir must already exist in the immutable image without symlink components. The completed @@ -208,11 +208,18 @@ every parent and write and enter the workdir; OpenShell does not change that directory's ownership or mode. Filesystem metadata identifies kernel-managed mounts, while collision checks are derived from actual OpenShell control paths. Docker performs the check in the final container before workload launch and -rejects image `VOLUME` declarations that would mask the workdir ancestry. The -resolved workspace is the child cwd and `HOME`; when -`filesystem.include_workdir` is enabled, it becomes the automatic writable -policy path. Podman, Kubernetes/OpenShift, and VM retain their existing -`/sandbox` workspace behavior. +rejects image `VOLUME` declarations that would mask the workdir ancestry. +Podman performs it in a minimal networkless container from the same pinned +image ID before its managed workspace volume covers the path. The probe retains +only the capabilities needed to adopt the completed process identity, drops to +that identity, and asks the kernel to validate access. It emits a normalized +identity attestation that the final supervisor must match, including when the +image supplies the default process policy. The driver captures a bounded, +sanitized diagnostic before removing a failed probe. The resolved workspace is +the child cwd and `HOME`; when `filesystem.include_workdir` is enabled, it +becomes the automatic writable policy path. Kubernetes/OpenShift keep their +`/sandbox` PVC and `fsGroup` behavior, and VM keeps its `/sandbox` guest +initialization path. Sandbox creation fails before the workload becomes ready when a required image identity is absent, malformed, unknown, ambiguous, or resolves to UID/GID 0. diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 1549258fa3..22a82e110c 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -126,6 +126,13 @@ pub const SANDBOX_GID: &str = "OPENSHELL_SANDBOX_GID"; /// OCI only for the former contract. pub const OCI_IMAGE_USER: &str = "OPENSHELL_OCI_IMAGE_USER"; +/// Normalized UID/GID/supplementary-group identity attested by the Podman +/// immutable-image workspace probe. +/// +/// A non-empty value also asserts that the +/// original workspace was validated before Podman's managed volume covered it. +pub const OCI_WORKSPACE_IDENTITY: &str = "OPENSHELL_OCI_WORKSPACE_IDENTITY"; + // The corporate upstream-proxy configuration deliberately has no reserved // environment variables: it travels on the supervisor's argv // (`--upstream-proxy` and friends), which a sandbox image cannot forge the diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index c7c3fab032..3734b67073 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -2183,6 +2183,7 @@ fn build_environment_for_oci_user( user_env.extend(template.environment.clone()); } user_env.extend(spec.environment.clone()); + user_env.remove(openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY); environment.extend(user_env.clone()); if !user_env.is_empty() && let Ok(json) = serde_json::to_string(&user_env) @@ -2238,6 +2239,13 @@ fn build_environment_for_oci_user( environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + // Docker never uses the Podman prevalidation contract. Emit an explicit + // driver-owned empty value so image-baked ENV entries cannot select the + // managed-workspace mutation path for an image-provided workdir. + environment.insert( + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY.to_string(), + String::new(), + ); environment.insert( openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(), oci_user.to_string(), diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 7d1ffbcf6b..0a37aea5df 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -44,6 +44,7 @@ fn test_sandbox() -> DriverSandbox { }), resource_requirements: None, sandbox_token: String::new(), + workspace_validation_identity: None, }), status: None, workspace: String::new(), @@ -540,6 +541,10 @@ fn build_environment_protects_oci_identity_metadata() { (openshell_core::sandbox_env::OCI_IMAGE_USER, "spoofed"), (openshell_core::sandbox_env::SANDBOX_UID, "9999"), (openshell_core::sandbox_env::SANDBOX_GID, "9999"), + ( + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY, + "9999:9999:", + ), ] { spec.environment.insert(key.to_string(), value.to_string()); } @@ -552,6 +557,10 @@ fn build_environment_protects_oci_identity_metadata() { ))); assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_UID))); assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_GID))); + assert!(env.contains(&format!( + "{}=", + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY + ))); assert!(!env.iter().any(|entry| entry.ends_with("=spoofed"))); assert!(!env.iter().any(|entry| entry.ends_with("=9999"))); } diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 965a295d19..4762d92480 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -8,14 +8,30 @@ isolation enforcement to the `openshell-sandbox` supervisor binary, which is sideloaded into each container via an OCI image volume mount. Before creating the container, the driver inspects the final sandbox image and -captures its immutable image ID and raw OCI `Config.User`. Container creation -uses that image ID with pulling disabled, preventing a mutable tag from changing -between inspection and launch. The supervisor runs as root, resolves omitted -policy identity fields from the image declaration, and drops only agent -children to the completed identity. Named OCI components remain names after -validation; a missing group is filled with the user's numeric primary GID. Explicit -`process.run_as_user` and `process.run_as_group` values take precedence -independently. +captures its immutable image ID, raw OCI `Config.User`, and OCI +`Config.WorkingDir`. Container creation uses that image ID with pulling +disabled, preventing a mutable tag from changing between inspection and launch. +The supervisor runs as root, resolves omitted policy identity fields from the +image declaration, and drops only agent children to the completed identity. +Named OCI components remain names after validation; a missing group is filled +with the user's numeric primary GID. Explicit `process.run_as_user` and +`process.run_as_group` values take precedence independently. + +An absolute OCI working directory becomes the agent workspace. An empty, +root (`/`), or explicit `/sandbox` declaration uses `/sandbox`, which OpenShell +creates when necessary and owns as a compatibility workspace. For any other workdir, Podman +first starts a minimal, networkless probe from the pinned image ID without the +workspace volume, tokens, or TLS secrets. The probe retains only `SETUID` and +`SETGID`, resolves and adopts the completed identity, including supplementary +groups, and uses the kernel to verify that it can traverse every parent and +write and enter the workdir. The path must be a real directory without symlink +components. The probe also rejects kernel-managed filesystems and overlaps with +concrete OpenShell control resources. It emits a normalized identity +attestation; the final supervisor must resolve to the same identity, including +when the image supplies the default process policy. On failure, the driver +captures a bounded, sanitized diagnostic before removing the probe. Only then +does Podman mount and prepare the managed workspace volume at that path; normal +copy-up preserves image content. The workspace is the child cwd and `HOME`. For a rootless networking deep dive, see [NETWORKING.md](NETWORKING.md). @@ -87,9 +103,11 @@ optional `selinux_label` of `shared` (applies `:z`) or `private` (applies read-only by default; set `read_only: false` to make them writable. Podman image and volume mounts do not support `subpath` in OpenShell driver config. Mount `source` and `target` values must not contain surrounding whitespace. -Mount targets must be absolute container paths and must not replace -the workspace root (`/sandbox`) or overlap OpenShell supervisor files, -`/etc/openshell`, `/etc/openshell-tls`, or `/run/netns`. +Mount targets must be absolute container paths and must not replace the +resolved workspace root or any of its parents. Nested workspace mounts remain +valid. Mounts also must not contain or be contained by concrete OpenShell +control targets such as the supervisor mount, TLS and token files, runtime +socket, or `/run/netns`. Example named-volume usage: diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 952938d47a..5d7a298178 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -25,6 +25,7 @@ const API_TIMEOUT: Duration = Duration::from_secs(30); /// Maximum allowed size for the event stream line buffer (1 MB). const MAX_EVENT_BUFFER: usize = 1_048_576; +const MAX_CONTAINER_LOG_BYTES: usize = 16 * 1024; #[derive(Debug, thiserror::Error)] pub enum PodmanApiError { @@ -117,6 +118,16 @@ pub struct ContainerState { pub finished_at: Option, } +#[derive(Debug, serde::Deserialize)] +#[serde(untagged)] +enum ContainerWaitResponse { + ExitCode(i64), + Status { + #[serde(rename = "StatusCode")] + status_code: i64, + }, +} + #[derive(Debug, Clone, serde::Deserialize)] #[serde(rename_all = "PascalCase")] pub struct HealthState { @@ -177,6 +188,8 @@ pub struct ImageInspect { pub struct ImageConfig { #[serde(default)] pub user: String, + #[serde(default)] + pub working_dir: String, } /// A container summary returned by the list API. @@ -350,6 +363,43 @@ impl PodmanClient { Ok((status, bytes)) } + /// Send a request while retaining at most `max_bytes` of the response. + async fn send_request_bounded( + &self, + req: Request>, + timeout: Duration, + max_bytes: usize, + ) -> Result<(hyper::StatusCode, Bytes), PodmanApiError> { + use hyper::body::Body; + + let mut sender = self.connect().await?; + let response = tokio::time::timeout(timeout, sender.send_request(req)) + .await + .map_err(|_| PodmanApiError::Timeout(timeout))? + .map_err(|error| PodmanApiError::Connection(error.to_string()))?; + let status = response.status(); + let deadline = tokio::time::Instant::now() + timeout; + let mut body = response.into_body(); + let mut bytes = Vec::with_capacity(max_bytes.min(4096)); + while bytes.len() < max_bytes { + let frame = tokio::time::timeout_at( + deadline, + std::future::poll_fn(|context| Pin::new(&mut body).poll_frame(context)), + ) + .await + .map_err(|_| PodmanApiError::Timeout(timeout))?; + let Some(frame) = frame else { + break; + }; + let frame = frame.map_err(|error| PodmanApiError::Connection(error.to_string()))?; + if let Some(data) = frame.data_ref() { + let remaining = max_bytes - bytes.len(); + bytes.extend_from_slice(&data[..data.len().min(remaining)]); + } + } + Ok((status, Bytes::from(bytes))) + } + /// Perform a versioned HTTP request and return status + body bytes. async fn request( &self, @@ -455,6 +505,22 @@ impl PodmanClient { .await } + /// Wait for a container to exit and return its exit code. + pub async fn wait_container(&self, name: &str) -> Result { + validate_name(name)?; + let response: ContainerWaitResponse = self + .request_json( + hyper::Method::POST, + &format!("/libpod/containers/{name}/wait?condition=exited"), + None, + ) + .await?; + Ok(match response { + ContainerWaitResponse::ExitCode(code) => code, + ContainerWaitResponse::Status { status_code } => status_code, + }) + } + /// Stop a container with a grace period in seconds. pub async fn stop_container( &self, @@ -501,6 +567,27 @@ impl PodmanClient { .await } + /// Fetch a bounded tail of stdout and stderr from a container. + pub async fn container_logs(&self, name: &str) -> Result { + validate_name(name)?; + let req = Self::build_request( + hyper::Method::GET, + &format!( + "/{API_VERSION}/libpod/containers/{name}/logs?stdout=true&stderr=true&tail=20×tamps=false" + ), + Full::new(Bytes::new()), + None, + ); + let (status, bytes) = self + .send_request_bounded(req, API_TIMEOUT, MAX_CONTAINER_LOG_BYTES) + .await?; + if status.is_success() { + Ok(String::from_utf8_lossy(&bytes).into_owned()) + } else { + Err(error_from_response(status.as_u16(), &bytes)) + } + } + /// List containers matching label filters (e.g. `&["openshell.managed=true"]`). pub async fn list_containers( &self, @@ -932,12 +1019,12 @@ mod tests { } #[tokio::test] - async fn inspect_image_reads_immutable_id_and_oci_user() { + async fn inspect_image_reads_immutable_id_and_oci_config() { let (socket_path, request_log, handle) = spawn_podman_stub( "inspect-image", vec![StubResponse::new( StatusCode::OK, - r#"{"Id":"sha256:immutable","Config":{"User":"app:staff"}}"#, + r#"{"Id":"sha256:immutable","Config":{"User":"app:staff","WorkingDir":"/workspace/project"}}"#, )], ); let client = PodmanClient::new(socket_path.clone()); @@ -952,6 +1039,13 @@ mod tests { image.config.as_ref().map(|config| config.user.as_str()), Some("app:staff") ); + assert_eq!( + image + .config + .as_ref() + .map(|config| config.working_dir.as_str()), + Some("/workspace/project") + ); handle.await.expect("stub task should finish"); assert_eq!( request_log @@ -962,4 +1056,34 @@ mod tests { ); let _ = std::fs::remove_file(socket_path); } + + #[tokio::test] + async fn container_logs_fetches_bounded_stdout_and_stderr_tail() { + let (socket_path, request_log, handle) = spawn_podman_stub( + "container-logs", + vec![StubResponse::new( + StatusCode::OK, + "workspace validation failed\n", + )], + ); + let client = PodmanClient::new(socket_path.clone()); + + let logs = client + .container_logs("workdir-probe") + .await + .expect("container logs should be returned"); + + assert_eq!(logs, "workspace validation failed\n"); + handle.await.expect("stub task should finish"); + assert_eq!( + request_log + .lock() + .expect("request log lock should not be poisoned") + .as_slice(), + [ + "GET /v5.0.0/libpod/containers/workdir-probe/logs?stdout=true&stderr=true&tail=20×tamps=false" + ] + ); + let _ = std::fs::remove_file(socket_path); + } } diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 223d2b43eb..12af404385 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -3,6 +3,7 @@ //! Container spec construction for the Podman driver. +use crate::client::ImageInspect; use crate::config::PodmanComputeConfig; use openshell_core::ComputeDriverError; use openshell_core::driver_mounts::SelinuxLabel; @@ -190,6 +191,9 @@ struct ContainerSpec { volumes: Vec, image_volumes: Vec, hostname: String, + /// Start the supervisor independently of the image workspace. The + /// supervisor validates and prepares that workspace before child launch. + work_dir: String, /// Overrides the image's ENTRYPOINT. In Podman's libpod API, `command` /// only overrides CMD (appended as args to the entrypoint). We must set /// `entrypoint` explicitly so the supervisor binary runs directly, @@ -423,6 +427,7 @@ fn build_env( user_env.insert(k.clone(), v.clone()); } } + user_env.remove(openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY); env.extend(user_env.clone()); if !user_env.is_empty() && let Ok(json) = serde_json::to_string(&user_env) @@ -483,6 +488,13 @@ fn build_env( env.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); env.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + // The final spec overwrites this only after a successful immutable-image + // probe. An explicit empty value prevents image ENV from forging the + // attestation contract on the /sandbox compatibility path. + env.insert( + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY.into(), + String::new(), + ); env.insert( openshell_core::sandbox_env::OCI_IMAGE_USER.into(), oci_user.to_string(), @@ -495,7 +507,6 @@ fn build_env( openshell_core::sandbox_env::SANDBOX_GID.into(), String::new(), ); - // 4. Gateway-minted sandbox JWT. Keep the raw bearer out of container // metadata; the supervisor reads it from a driver-owned bind mount. if let Some(s) = spec @@ -612,6 +623,7 @@ pub fn podman_driver_image_mount_sources( fn podman_user_mounts( sandbox: &DriverSandbox, enable_bind_mounts: bool, + workspace_root: &str, ) -> Result { let template = sandbox .spec @@ -623,6 +635,13 @@ fn podman_user_mounts( let config = podman_driver_config(template, enable_bind_mounts)?; let mut result = PodmanUserMounts::default(); for mount in config.mounts { + let target = match &mount { + PodmanDriverMountConfig::Bind { target, .. } + | PodmanDriverMountConfig::Volume { target, .. } + | PodmanDriverMountConfig::Tmpfs { target, .. } + | PodmanDriverMountConfig::Image { target, .. } => target, + }; + driver_mounts::validate_workspace_mount_target(target, workspace_root)?; match mount { PodmanDriverMountConfig::Bind { source, @@ -903,8 +922,11 @@ pub fn build_container_spec_with_token_and_gpu_devices( token_secret_name, gpu_device_ids, image, - image, - "", + &ImageInspect { + id: image.to_string(), + config: None, + }, + None, ) } @@ -914,16 +936,31 @@ pub fn build_container_spec_for_image( token_secret_name: Option<&str>, gpu_device_ids: Option<&[String]>, requested_image: &str, - image_id: &str, - oci_user: &str, + inspected_image: &ImageInspect, + workspace_identity: Option<&str>, ) -> Result { let name = container_name(&sandbox.workspace, &sandbox.name, &sandbox.id); let vol = volume_name(&sandbox.id); + let oci_user = inspected_image + .config + .as_ref() + .map_or("", |config| config.user.as_str()); + let oci_working_dir = inspected_image + .config + .as_ref() + .map_or("", |config| config.working_dir.as_str()); + let workspace_root = driver_mounts::resolve_oci_workspace_root(oci_working_dir) + .map_err(ComputeDriverError::Precondition)?; + driver_mounts::validate_workspace_control_path( + &workspace_root, + &config.sandbox_ssh_socket_path, + ) + .map_err(ComputeDriverError::Precondition)?; - let env = build_env(sandbox, config, requested_image, oci_user); + let mut env = build_env(sandbox, config, requested_image, oci_user); let labels = build_labels(sandbox); let resource_limits = build_resource_limits(sandbox, config); - let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) + let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts, &workspace_root) .map_err(ComputeDriverError::InvalidArgument)?; if sandbox .spec @@ -952,7 +989,7 @@ pub fn build_container_spec_for_image( let mut volumes = vec![NamedVolume { name: vol, - dest: "/sandbox".into(), + dest: workspace_root.clone(), options: vec!["rw".into()], }]; volumes.extend(user_mounts.volumes); @@ -963,15 +1000,19 @@ pub fn build_container_spec_for_image( rw: false, }]; image_volumes.extend(user_mounts.image_volumes); - let mut command = vec![ - "--workdir".to_string(), - driver_mounts::DEFAULT_WORKSPACE_ROOT.to_string(), - ]; + + let mut command = vec!["--workdir".to_string(), workspace_root]; command.extend(upstream_proxy_cli_args(config)); + if let Some(identity) = workspace_identity { + env.insert( + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY.into(), + identity.into(), + ); + } let container_spec = ContainerSpec { name, - image: image_id.to_string(), + image: inspected_image.id.clone(), labels, env, volumes, @@ -982,6 +1023,7 @@ pub fn build_container_spec_for_image( // /openshell-sandbox, so it appears at /opt/openshell/bin/openshell-sandbox. image_volumes, hostname: format!("sandbox-{}", sandbox.name), + work_dir: "/".to_string(), // Override the image's ENTRYPOINT so the supervisor binary runs // directly. Sandbox images (e.g. the community base image) set // ENTRYPOINT ["/bin/bash"], and Podman's `command` field only @@ -989,10 +1031,9 @@ pub fn build_container_spec_for_image( // Without this, the container would run the entrypoint binary with // the supervisor path as an argument instead of executing it directly. entrypoint: vec![SUPERVISOR_BINARY_PATH.into()], - // Keep Podman's existing /sandbox workspace contract explicit while - // the supervisor supports driver-selected workdirs. Operator-owned - // corporate proxy flags follow it; the workload command comes from - // the reserved environment variable. + // Operator-owned corporate proxy flags. The workload command is not + // part of argv (the supervisor takes it from the reserved command + // env var), so these flags are the whole command list. command, // Force the supervisor to run as root (UID 0). Sandbox images may // set a non-root USER directive (e.g. `USER sandbox`), but the @@ -1188,6 +1229,77 @@ pub fn build_container_spec_for_image( Ok(serde_json::to_value(container_spec).expect("ContainerSpec serialization cannot fail")) } +/// Build a minimal one-shot container that validates the image's original +/// workdir before the final Podman workspace volume covers it. +pub fn build_workspace_probe_spec( + sandbox: &DriverSandbox, + config: &PodmanComputeConfig, + inspected_image: &ImageInspect, + probe_name: &str, +) -> Result, ComputeDriverError> { + let image_config = inspected_image.config.as_ref(); + let oci_user = image_config.map_or("", |config| config.user.as_str()); + let oci_working_dir = image_config.map_or("", |config| config.working_dir.as_str()); + let workspace_root = driver_mounts::resolve_oci_workspace_root(oci_working_dir) + .map_err(ComputeDriverError::Precondition)?; + driver_mounts::validate_workspace_control_path( + &workspace_root, + &config.sandbox_ssh_socket_path, + ) + .map_err(ComputeDriverError::Precondition)?; + if workspace_root == driver_mounts::DEFAULT_WORKSPACE_ROOT { + return Ok(None); + } + + let spec = sandbox + .spec + .as_ref() + .ok_or_else(|| ComputeDriverError::Precondition("sandbox.spec is required".into()))?; + let mut command = vec![ + "validate-workspace".to_string(), + "--workdir".to_string(), + workspace_root, + "--oci-user".to_string(), + oci_user.to_string(), + ]; + if let Some(identity) = &spec.workspace_validation_identity { + for (flag, value) in [ + ("--run-as-user", identity.run_as_user.as_str()), + ("--run-as-group", identity.run_as_group.as_str()), + ] { + if !value.is_empty() { + command.push(flag.to_string()); + command.push(value.to_string()); + } + } + if identity.discover_from_image_policy { + command.push("--discover-policy-identity".to_string()); + } + } + + Ok(Some(serde_json::json!({ + "name": probe_name, + "image": inspected_image.id, + "entrypoint": [SUPERVISOR_BINARY_PATH], + "command": command, + "user": "0:0", + "work_dir": "/", + "image_volumes": [{ + "source": config.supervisor_image, + "destination": SUPERVISOR_MOUNT_DIR, + "rw": false + }], + // Do not materialize OCI VOLUME declarations: the probe must inspect + // the immutable image layer rather than a fresh anonymous volume. + "image_volume_mode": "ignore", + "netns": {"nsmode": "none"}, + "no_new_privileges": true, + "cap_drop": ["ALL"], + "cap_add": ["SETUID", "SETGID"], + "image_pull_policy": "never" + }))) +} + fn hostadd_entries(config: &PodmanComputeConfig) -> Vec { let host_gateway_ip = config.host_gateway_ip.trim(); if host_gateway_ip.is_empty() { @@ -1260,6 +1372,7 @@ fn parse_memory_to_bytes(quantity: &str) -> Option { #[cfg(test)] mod tests { use super::*; + use crate::client::ImageConfig; use openshell_core::proto::compute::v1::{GpuResourceRequirements, ResourceRequirements}; static ENV_LOCK: std::sync::LazyLock> = @@ -1279,6 +1392,16 @@ mod tests { } } + fn inspected_image(id: &str, user: &str, working_dir: &str) -> ImageInspect { + ImageInspect { + id: id.to_string(), + config: Some(ImageConfig { + user: user.to_string(), + working_dir: working_dir.to_string(), + }), + } + } + #[test] fn parse_cpu_millicore() { assert_eq!(parse_cpu_to_microseconds("500m"), Some(50_000)); @@ -1373,6 +1496,10 @@ mod tests { (openshell_core::sandbox_env::OCI_IMAGE_USER, "spoofed"), (openshell_core::sandbox_env::SANDBOX_UID, "9999"), (openshell_core::sandbox_env::SANDBOX_GID, "9999"), + ( + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY, + "9999:9999:", + ), ] { spec.environment.insert(key.to_string(), value.to_string()); } @@ -1383,8 +1510,8 @@ mod tests { None, None, "registry.example/app:latest", - "sha256:immutable", - "app:staff", + &inspected_image("sha256:immutable", "app:staff", "/workspace/project"), + Some("1000:1000:"), ) .unwrap(); @@ -1395,6 +1522,18 @@ mod tests { ); assert_eq!(container["user"].as_str(), Some("0:0")); assert_eq!(container["image_pull_policy"].as_str(), Some("never")); + assert_eq!( + container["command"], + serde_json::json!(["--workdir", "/workspace/project"]) + ); + assert_eq!(container["work_dir"].as_str(), Some("/")); + assert!(container["volumes"].as_array().is_some_and(|volumes| { + volumes.iter().any(|volume| { + volume["name"].as_str() == Some("openshell-sandbox-test-id-workspace") + && volume["dest"].as_str() == Some("/workspace/project") + && volume["options"] == serde_json::json!(["rw"]) + }) + })); assert_eq!( container["env"][openshell_core::sandbox_env::OCI_IMAGE_USER].as_str(), Some("app:staff") @@ -1408,11 +1547,249 @@ mod tests { Some("") ); assert_eq!( - container["command"], - serde_json::json!(["--workdir", "/sandbox"]) + container["env"][openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY].as_str(), + Some("1000:1000:") + ); + } + + #[test] + fn compatibility_workspace_emits_empty_attestation() { + let container = build_container_spec_for_image( + &test_sandbox("test-id", "test-name"), + &test_config(), + None, + None, + "registry.example/app:latest", + &inspected_image("sha256:immutable", "app:staff", ""), + None, + ) + .unwrap(); + + assert_eq!( + container["env"][openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY].as_str(), + Some("") + ); + } + + #[test] + fn container_spec_rejects_invalid_oci_working_dir() { + let err = build_container_spec_for_image( + &test_sandbox("test-id", "test-name"), + &test_config(), + None, + None, + "registry.example/app:latest", + &inspected_image("sha256:immutable", "app:staff", "relative/workspace"), + None, + ) + .unwrap_err(); + + assert!( + err.to_string() + .contains("must be an absolute container path") + ); + } + + #[test] + fn container_spec_rejects_openshell_control_path_working_dir() { + let err = build_container_spec_for_image( + &test_sandbox("test-id", "test-name"), + &test_config(), + None, + None, + "registry.example/app:latest", + &inspected_image( + "sha256:immutable", + "app:staff", + "/opt/openshell/bin/project", + ), + None, + ) + .unwrap_err(); + + assert!(err.to_string().contains("OpenShell control path")); + } + + #[test] + fn workspace_probe_uses_pinned_image_without_workspace_or_network() { + let mut sandbox = test_sandbox("test-id", "test-name"); + let spec = sandbox.spec.get_or_insert_default(); + spec.workspace_validation_identity = Some( + openshell_core::proto::compute::v1::WorkspaceValidationIdentity { + run_as_user: "policy-user".into(), + run_as_group: "policy-group".into(), + discover_from_image_policy: false, + }, + ); + let probe = build_workspace_probe_spec( + &sandbox, + &test_config(), + &inspected_image("sha256:immutable", "app:staff", "/workspace/project"), + "openshell-test-probe", + ) + .unwrap() + .unwrap(); + + assert_eq!(probe["image"], "sha256:immutable"); + assert_eq!(probe["netns"]["nsmode"], "none"); + assert_eq!(probe["image_volume_mode"], "ignore"); + assert_eq!(probe["cap_drop"], serde_json::json!(["ALL"])); + assert_eq!(probe["cap_add"], serde_json::json!(["SETUID", "SETGID"])); + assert!(probe.get("volumes").is_none()); + assert!(probe.get("secrets").is_none()); + assert!(probe.get("env").is_none()); + assert_eq!( + probe["command"], + serde_json::json!([ + "validate-workspace", + "--workdir", + "/workspace/project", + "--oci-user", + "app:staff", + "--run-as-user", + "policy-user", + "--run-as-group", + "policy-group" + ]) ); } + #[test] + fn workspace_probe_discovers_image_policy_identity_when_requested() { + let mut sandbox = test_sandbox("test-id", "test-name"); + sandbox + .spec + .get_or_insert_default() + .workspace_validation_identity = Some( + openshell_core::proto::compute::v1::WorkspaceValidationIdentity { + discover_from_image_policy: true, + ..Default::default() + }, + ); + + let probe = build_workspace_probe_spec( + &sandbox, + &test_config(), + &inspected_image("sha256:immutable", "app:staff", "/home/app/project"), + "openshell-test-probe", + ) + .unwrap() + .unwrap(); + + assert_eq!( + probe["command"], + serde_json::json!([ + "validate-workspace", + "--workdir", + "/home/app/project", + "--oci-user", + "app:staff", + "--discover-policy-identity" + ]) + ); + } + + #[test] + fn workspace_probe_skips_sandbox_compatibility_fallback() { + let probe = build_workspace_probe_spec( + &test_sandbox("test-id", "test-name"), + &test_config(), + &inspected_image("sha256:immutable", "sandbox:sandbox", "/"), + "openshell-test-probe", + ) + .unwrap(); + assert!(probe.is_none()); + } + + #[test] + fn container_spec_reserves_resolved_workspace_root_but_allows_nested_mounts() { + let mut sandbox = test_sandbox("test-id", "test-name"); + sandbox.spec = Some(openshell_core::proto::compute::v1::DriverSandboxSpec { + template: Some(DriverSandboxTemplate::default()), + ..Default::default() + }); + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some(json_struct(serde_json::json!({ + "mounts": [{ + "type": "tmpfs", + "target": "/workspace" + }] + }))); + + let err = build_container_spec_for_image( + &sandbox, + &test_config(), + None, + None, + "registry.example/app:latest", + &inspected_image("sha256:immutable", "app:staff", "/workspace"), + None, + ) + .unwrap_err(); + assert!( + err.to_string() + .contains("reserved for the OpenShell workspace") + ); + + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some(json_struct(serde_json::json!({ + "mounts": [{ + "type": "tmpfs", + "target": "/workspace" + }] + }))); + let err = build_container_spec_for_image( + &sandbox, + &test_config(), + None, + None, + "registry.example/app:latest", + &inspected_image("sha256:immutable", "app:staff", "/workspace/project"), + None, + ) + .unwrap_err(); + assert!( + err.to_string() + .contains("reserved for the OpenShell workspace") + ); + + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some(json_struct(serde_json::json!({ + "mounts": [{ + "type": "tmpfs", + "target": "/workspace/cache" + }] + }))); + build_container_spec_for_image( + &sandbox, + &test_config(), + None, + None, + "registry.example/app:latest", + &inspected_image("sha256:immutable", "app:staff", "/workspace"), + None, + ) + .expect("nested workspace mounts remain supported"); + } + #[test] fn volume_name_uses_id() { assert_eq!( diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index aa3df9cbd7..04b37f3427 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -42,6 +42,10 @@ pub struct PodmanComputeDriver { /// The host's IP on the bridge network. Sandbox containers use this to /// reach the gateway server when no explicit gRPC endpoint is configured. network_gateway_ip: Option, + /// Serializes container image-volume attachment and removal. Rootless + /// Podman can otherwise detach a shared type=image mount while another + /// sandbox container is starting. + container_lifecycle_lock: Arc>, gpu_selector: Arc, gpu_inventory_refresh: Arc (CdiGpuInventory, bool) + Send + Sync>, } @@ -73,6 +77,15 @@ fn validated_container_name(sandbox: &DriverSandbox) -> Result String { + const SUFFIX: &str = "-workdir-probe"; + let keep = 255usize.saturating_sub(SUFFIX.len()); + format!( + "{}{SUFFIX}", + &container_name[..container_name.len().min(keep)] + ) +} + fn podman_volume_is_bind_backed(volume: &VolumeInspect) -> bool { (volume.driver.is_empty() || volume.driver == "local") && volume.options.get("o").is_some_and(|options| { @@ -208,6 +221,36 @@ fn podman_gpu_selection_error(err: CdiGpuSelectionError) -> ComputeDriverError { ComputeDriverError::Precondition(err.to_string()) } +const WORKSPACE_IDENTITY_MARKER: &str = "OPENSHELL_WORKSPACE_IDENTITY="; + +fn parse_workspace_identity(logs: &str) -> Result { + logs.match_indices(WORKSPACE_IDENTITY_MARKER) + .filter_map(|(start, _)| { + logs[start + WORKSPACE_IDENTITY_MARKER.len()..] + .split(|character: char| character.is_control()) + .next() + }) + .find_map(|identity| serde_json::from_str::(identity).ok()) + .ok_or_else(|| { + ComputeDriverError::Precondition( + "OCI WorkingDir validation probe did not report its process identity".to_string(), + ) + }) +} + +fn sanitized_probe_diagnostic(logs: &str) -> String { + let sanitized = logs + .chars() + .filter(|character| !character.is_control() || matches!(character, '\n' | '\t')) + .collect::(); + let sanitized = sanitized.trim(); + if sanitized.is_empty() { + "no probe diagnostic was emitted".to_string() + } else { + sanitized.chars().take(2048).collect() + } +} + /// Resolve the socket to connect to: explicit configuration wins, otherwise /// fall back to `detect`. Returns an error if neither resolves. /// @@ -229,6 +272,78 @@ fn resolve_socket_path( } impl PodmanComputeDriver { + async fn validate_image_workspace( + &self, + sandbox: &DriverSandbox, + container_name: &str, + inspected_image: &crate::client::ImageInspect, + ) -> Result, ComputeDriverError> { + let probe_name = workspace_probe_name(container_name); + crate::client::validate_name(&probe_name) + .map_err(|error| ComputeDriverError::Precondition(error.to_string()))?; + let Some(spec) = container::build_workspace_probe_spec( + sandbox, + &self.config, + inspected_image, + &probe_name, + )? + else { + return Ok(None); + }; + + let start = { + let _lifecycle_guard = self.container_lifecycle_lock.lock().await; + self.client + .create_container(&spec) + .await + .map_err(ComputeDriverError::from)?; + self.client + .start_container(&probe_name) + .await + .map_err(ComputeDriverError::from) + }; + let validation = async { + start?; + let exit_code = self + .client + .wait_container(&probe_name) + .await + .map_err(ComputeDriverError::from)?; + let logs = self.client.container_logs(&probe_name).await; + if exit_code == 0 { + let logs = logs.map_err(ComputeDriverError::from)?; + parse_workspace_identity(&logs).map(Some) + } else { + let diagnostic = logs.map_or_else( + |error| format!("unable to read probe diagnostic: {error}"), + |logs| sanitized_probe_diagnostic(&logs), + ); + Err(ComputeDriverError::Precondition(format!( + "OCI WorkingDir validation failed for image '{}' (probe exited with code {exit_code}): {diagnostic}", + inspected_image.id, + ))) + } + } + .await; + let cleanup = { + let _lifecycle_guard = self.container_lifecycle_lock.lock().await; + self.client.remove_container(&probe_name).await + }; + match (validation, cleanup) { + (Ok(identity), Ok(())) => Ok(identity), + (Ok(_), Err(error)) => Err(ComputeDriverError::from(error)), + (Err(error), Ok(())) => Err(error), + (Err(error), Err(cleanup_error)) => { + warn!( + probe = %probe_name, + %cleanup_error, + "Failed to remove workspace validation probe" + ); + Err(error) + } + } + } + /// Create a new driver, verifying the Podman socket is reachable. pub async fn new(mut config: PodmanComputeConfig) -> Result { const MAX_PING_RETRIES: u32 = 5; @@ -368,6 +483,7 @@ impl PodmanComputeDriver { client, config, network_gateway_ip, + container_lifecycle_lock: Arc::new(tokio::sync::Mutex::new(())), gpu_selector: Arc::new(CdiGpuDefaultSelector::new( gpu_inventory, allow_all_default_gpu, @@ -581,11 +697,9 @@ impl PodmanComputeDriver { "podman image '{image}' inspection did not return an immutable image ID" ))); } - let image_user = inspected_image - .config - .as_ref() - .map_or("", |config| config.user.as_str()); - + let workspace_identity = self + .validate_image_workspace(sandbox, &name, &inspected_image) + .await?; for image in container::podman_driver_image_mount_sources(sandbox, self.config.enable_bind_mounts) .map_err(ComputeDriverError::Precondition)? @@ -650,8 +764,8 @@ impl PodmanComputeDriver { token_secret_name.as_deref(), gpu_devices.as_deref(), image, - &inspected_image.id, - image_user, + &inspected_image, + workspace_identity.as_deref(), ) { Ok(spec) => spec, Err(e) => { @@ -659,8 +773,28 @@ impl PodmanComputeDriver { return Err(e); } }; - match self.client.create_container(&spec).await { - Ok(_) => {} + // Podman implements the supervisor as a shared type=image mount. + // Keep attach/start atomic with respect to another sandbox's removal. + let lifecycle_result = { + let _lifecycle_guard = self.container_lifecycle_lock.lock().await; + match self.client.create_container(&spec).await { + Ok(_) => match self.client.start_container(&name).await { + Ok(()) => Ok(()), + Err(e) => { + warn!( + sandbox_name = %sandbox.name, + error = %e, + "Failed to start container; cleaning up" + ); + let _ = self.client.remove_container(&name).await; + Err(e) + } + }, + Err(e) => Err(e), + } + }; + match lifecycle_result { + Ok(()) => {} Err(PodmanApiError::Conflict(_)) => { // Clean up the volume we just created. It is keyed by *this* // sandbox's ID, not the conflicting container's ID (which @@ -675,18 +809,6 @@ impl PodmanComputeDriver { } } - // 5. Start container. - if let Err(e) = self.client.start_container(&name).await { - warn!( - sandbox_name = %sandbox.name, - error = %e, - "Failed to start container; cleaning up" - ); - let _ = self.client.remove_container(&name).await; - cleanup_created().await; - return Err(ComputeDriverError::from(e)); - } - info!( sandbox_id = %sandbox.id, sandbox_name = %sandbox.name, @@ -754,10 +876,15 @@ impl PodmanComputeDriver { .stop_container(&container_id, self.config.stop_timeout_secs) .await; - let container_existed = match self.client.remove_container(&container_id).await { - Ok(()) => true, - Err(PodmanApiError::NotFound(_)) => false, - Err(e) => return Err(ComputeDriverError::from(e)), + // Removing a container detaches its type=image mounts. Do not overlap + // that operation with another sandbox container's create/start window. + let container_existed = { + let _lifecycle_guard = self.container_lifecycle_lock.lock().await; + match self.client.remove_container(&container_id).await { + Ok(()) => true, + Err(PodmanApiError::NotFound(_)) => false, + Err(e) => return Err(ComputeDriverError::from(e)), + } }; // Remove workspace volume. @@ -892,6 +1019,7 @@ impl PodmanComputeDriver { client, config, network_gateway_ip: None, + container_lifecycle_lock: Arc::new(tokio::sync::Mutex::new(())), gpu_selector: Arc::new(CdiGpuDefaultSelector::new( gpu_inventory, allow_all_default_gpu, @@ -1649,6 +1777,90 @@ mod tests { } } + #[tokio::test] + async fn workspace_probe_waits_for_success_and_always_removes_container() { + let (socket_path, request_log, handle) = spawn_podman_stub( + "workspace-probe-success", + vec![ + StubResponse::new(StatusCode::CREATED, "{}"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new(StatusCode::OK, r#"{"StatusCode":0}"#), + StubResponse::new( + StatusCode::OK, + "OPENSHELL_WORKSPACE_IDENTITY=\"1234:1235:\"\n", + ), + StubResponse::new(StatusCode::NO_CONTENT, ""), + ], + ); + let driver = test_driver(socket_path.clone()); + let mut sandbox = plain_sandbox("sandbox-probe", "demo"); + sandbox.spec = Some(DriverSandboxSpec::default()); + let image = crate::client::ImageInspect { + id: "sha256:immutable".into(), + config: Some(crate::client::ImageConfig { + user: "1234:1235".into(), + working_dir: "/workspace".into(), + }), + }; + let name = validated_container_name(&sandbox).unwrap(); + + let identity = driver + .validate_image_workspace(&sandbox, &name, &image) + .await + .expect("successful probe should pass"); + assert_eq!(identity.as_deref(), Some("1234:1235:")); + + handle.await.expect("stub task should finish"); + let requests = request_log.lock().unwrap(); + assert_eq!(requests.len(), 5); + assert!(requests[0].contains("/libpod/containers/create")); + assert!(requests[1].contains("/start")); + assert!(requests[2].contains("/wait?condition=exited")); + assert!(requests[3].contains("/logs?")); + assert!(requests[4].starts_with("DELETE ")); + let _ = fs::remove_file(socket_path); + } + + #[tokio::test] + async fn workspace_probe_removes_container_after_validation_failure() { + let (socket_path, request_log, handle) = spawn_podman_stub( + "workspace-probe-failure", + vec![ + StubResponse::new(StatusCode::CREATED, "{}"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new(StatusCode::OK, r#"{"StatusCode":1}"#), + StubResponse::new( + StatusCode::OK, + "image workspace path component '/workspace' is not writable and traversable\n", + ), + StubResponse::new(StatusCode::NO_CONTENT, ""), + ], + ); + let driver = test_driver(socket_path.clone()); + let mut sandbox = plain_sandbox("sandbox-probe-fail", "demo"); + sandbox.spec = Some(DriverSandboxSpec::default()); + let image = crate::client::ImageInspect { + id: "sha256:immutable".into(), + config: Some(crate::client::ImageConfig { + user: "1234:1235".into(), + working_dir: "/workspace".into(), + }), + }; + let name = validated_container_name(&sandbox).unwrap(); + + let error = driver + .validate_image_workspace(&sandbox, &name, &image) + .await + .unwrap_err(); + assert!(error.to_string().contains("exited with code 1")); + assert!(error.to_string().contains("not writable and traversable")); + + handle.await.expect("stub task should finish"); + let requests = request_log.lock().unwrap(); + assert!(requests.last().unwrap().starts_with("DELETE ")); + let _ = fs::remove_file(socket_path); + } + fn secret_delete_request(sandbox_id: &str) -> String { format!( "DELETE {}", @@ -1831,4 +2043,29 @@ mod tests { ); let _ = fs::remove_file(socket_path); } + + #[test] + fn workspace_identity_parser_accepts_plain_and_multiplexed_logs() { + assert_eq!( + parse_workspace_identity( + "probe startup\nOPENSHELL_WORKSPACE_IDENTITY=\"1234:1235:7,8\"\n" + ) + .unwrap(), + "1234:1235:7,8" + ); + assert_eq!( + parse_workspace_identity( + "\u{1}\0\0\0\0\0\0.OPENSHELL_WORKSPACE_IDENTITY=\"1234:1235:\"\n" + ) + .unwrap(), + "1234:1235:" + ); + assert!( + parse_workspace_identity( + "OPENSHELL_WORKSPACE_IDENTITY=not-json\n\ + OPENSHELL_WORKSPACE_IDENTITY={\"uid\":1234}\n" + ) + .is_err() + ); + } } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index e30c9b8543..e75aada735 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -186,8 +186,8 @@ pub async fn run_sandbox( // Normalize the active driver's identity contract once, while both the // policy and launched image filesystem are available. Kubernetes and - // OpenShift retain their authoritative numeric pair; Docker fills only - // omitted policy fields from OCI Config.User. + // OpenShift retain their authoritative numeric pair; Docker and Podman + // fill only omitted policy fields from OCI Config.User. #[cfg(unix)] let (resolved_process_identity, workspace) = { let driver_identity = openshell_supervisor_process::identity::DriverIdentity::from_env()?; @@ -199,6 +199,24 @@ pub async fn run_sandbox( &mut policy, &driver_identity, )?; + if matches!( + &driver_identity, + openshell_supervisor_process::identity::DriverIdentity::OciUser { .. } + ) && std::env::var_os(openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY) + .is_some_and(|value| !value.is_empty()) + { + let expected = std::env::var(openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY) + .map_err(|_| miette::miette!("Podman workspace identity attestation is missing"))?; + let actual = + openshell_supervisor_process::process::resolved_workspace_identity_attestation( + &policy, resolved, + )?; + if expected != actual { + return Err(miette::miette!( + "process identity changed after OCI workspace validation" + )); + } + } ( resolved, openshell_supervisor_process::process::ResolvedWorkspace::new( @@ -2124,6 +2142,18 @@ fn discover_policy_from_disk_or_default() -> openshell_core::proto::SandboxPolic discover_policy_from_path(primary) } +/// Discover only the process identity portion of the immutable image policy. +/// +/// The Podman workspace probe uses this when no policy existed at create time, +/// matching the final supervisor's disk-policy discovery before the gateway +/// backfills that policy. +pub fn discover_process_policy_from_disk_or_default() -> openshell_core::policy::ProcessPolicy { + discover_policy_from_disk_or_default() + .process + .map(Into::into) + .unwrap_or_default() +} + /// Try to read a sandbox policy YAML from `path`, falling back to the /// hardcoded restrictive default if the file is missing or invalid. fn discover_policy_from_path(path: &std::path::Path) -> openshell_core::proto::SandboxPolicy { diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 62ae37b5a1..8711654ff2 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -32,6 +32,7 @@ const COPY_SELF_SUBCOMMAND: &str = "copy-self"; /// run `openshell-sandbox debug-rpc get-sandbox-config --sandbox-id ` /// to confirm the cross-sandbox IDOR guard fires. const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; +const VALIDATE_WORKSPACE_SUBCOMMAND: &str = "validate-workspace"; /// Default `--mode` value: run both supervisor leaves in a single binary. const DEFAULT_MODE: &str = "network,process"; @@ -231,6 +232,73 @@ struct Args { upstream_proxy_connect_by_hostname: bool, } +#[derive(Parser, Debug)] +#[command(name = "validate-workspace")] +struct ValidateWorkspaceArgs { + #[arg(long)] + workdir: String, + #[arg(long)] + oci_user: String, + #[arg(long)] + run_as_user: Option, + #[arg(long)] + run_as_group: Option, + #[arg(long)] + discover_policy_identity: bool, +} + +#[cfg(unix)] +fn validate_workspace(args: &[String]) -> Result<()> { + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, + }; + use openshell_supervisor_process::identity::{DriverIdentity, resolve_process_identity}; + + let args = ValidateWorkspaceArgs::try_parse_from( + std::iter::once(VALIDATE_WORKSPACE_SUBCOMMAND.to_string()).chain(args.iter().cloned()), + ) + .into_diagnostic()?; + let process = if args.discover_policy_identity { + openshell_sandbox::discover_process_policy_from_disk_or_default() + } else { + ProcessPolicy { + run_as_user: args.run_as_user, + run_as_group: args.run_as_group, + } + }; + let mut policy = SandboxPolicy { + version: 0, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process, + }; + let resolved_identity = resolve_process_identity( + &mut policy, + &DriverIdentity::OciUser { + declaration: args.oci_user, + }, + )?; + let identity = + openshell_supervisor_process::process::validate_oci_workspace_as_process_identity( + &policy, + resolved_identity, + Path::new(&args.workdir), + )?; + println!( + "OPENSHELL_WORKSPACE_IDENTITY={}", + serde_json::to_string(&identity).into_diagnostic()? + ); + Ok(()) +} + +#[cfg(not(unix))] +fn validate_workspace(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "workspace validation is only supported on Unix" + )) +} + /// Copy the running executable to `dest`, creating parent directories as /// needed and ensuring the result is executable (mode `0755`). /// @@ -479,6 +547,9 @@ fn main() -> Result<()> { std::process::exit(exit); }); } + if raw_args.get(1).map(String::as_str) == Some(VALIDATE_WORKSPACE_SUBCOMMAND) { + return validate_workspace(&raw_args[2..]); + } let args = Args::parse(); @@ -648,6 +719,36 @@ mod tests { use super::*; use std::os::unix::fs::PermissionsExt; + #[cfg(unix)] + #[test] + fn workspace_validation_subcommand_uses_final_policy_identity() { + let uid = nix::unistd::geteuid().as_raw(); + let gid = nix::unistd::getegid().as_raw(); + if uid < 1000 || gid < 1000 { + // Process policies intentionally reject system identities. Local + // macOS users commonly have IDs below 1000, so this test cannot + // exercise the credential transition for those accounts. + return; + } + let dir = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); + let root = dir.path().canonicalize().unwrap().join("workspace"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o777)).unwrap(); + let args = vec![ + "--workdir".to_string(), + root.display().to_string(), + "--oci-user".to_string(), + "unused".to_string(), + "--run-as-user".to_string(), + uid.to_string(), + "--run-as-group".to_string(), + gid.to_string(), + ]; + + validate_workspace(&args).expect("current identity should retain workspace authority"); + } + /// Drives `copy_self`'s file-copy logic against an arbitrary source path /// so tests don't depend on `current_exe()`. fn copy_executable(src: &Path, dest: &Path) -> Result<()> { diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 69bc390b46..94f906050a 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -2333,6 +2333,17 @@ fn driver_sandbox_spec_from_public( spec: &SandboxSpec, driver_name: &str, ) -> Result> { + let workspace_validation_identity = matches!(driver_name, "docker" | "podman").then(|| { + let process = spec + .policy + .as_ref() + .and_then(|policy| policy.process.as_ref()); + openshell_core::proto::compute::v1::WorkspaceValidationIdentity { + run_as_user: process.map_or_else(String::new, |process| process.run_as_user.clone()), + run_as_group: process.map_or_else(String::new, |process| process.run_as_group.clone()), + discover_from_image_policy: spec.policy.is_none(), + } + }); Ok(DriverSandboxSpec { log_level: spec.log_level.clone(), environment: spec.environment.clone(), @@ -2350,6 +2361,7 @@ fn driver_sandbox_spec_from_public( } }), sandbox_token: String::new(), + workspace_validation_identity, }) } @@ -3092,6 +3104,36 @@ mod tests { assert_eq!(gpu.count, Some(2)); } + #[test] + fn driver_sandbox_spec_projects_process_identity_for_local_image_probe() { + let public = SandboxSpec { + policy: Some(openshell_core::proto::SandboxPolicy { + process: Some(openshell_core::proto::ProcessPolicy { + run_as_user: "app".into(), + run_as_group: "staff".into(), + }), + ..Default::default() + }), + ..Default::default() + }; + + let driver = + driver_sandbox_spec_from_public(&public, "podman").expect("driver spec should map"); + let identity = driver.workspace_validation_identity.unwrap(); + assert_eq!(identity.run_as_user, "app"); + assert_eq!(identity.run_as_group, "staff"); + assert!(!identity.discover_from_image_policy); + + let without_policy = + driver_sandbox_spec_from_public(&SandboxSpec::default(), "podman").unwrap(); + assert!( + without_policy + .workspace_validation_identity + .unwrap() + .discover_from_image_policy + ); + } + #[test] fn select_driver_config_forwards_only_matching_driver_block() { let config = prost_types::Struct { diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index c14df9e2df..b150f77c08 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -142,6 +142,7 @@ pub(crate) fn prepare_child_sandbox( const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY, openshell_core::sandbox_env::SANDBOX_UID, openshell_core::sandbox_env::SANDBOX_GID, openshell_core::sandbox_env::SANDBOX_TOKEN, @@ -1329,6 +1330,101 @@ pub fn validate_oci_workspace( Ok(()) } +/// Drop a one-shot workspace probe to the completed sandbox identity, validate +/// using the kernel's real path-access checks, and return a normalized identity +/// attestation for the final supervisor. +#[cfg(unix)] +pub fn validate_oci_workspace_as_process_identity( + policy: &SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, + workdir: &Path, +) -> Result { + validate_sandbox_user_with_identity(policy, resolved_identity)?; + validate_sandbox_group_with_identity(policy, resolved_identity)?; + let (uid, gid, mut supplementary_gids) = + resolve_filesystem_identity(policy, resolved_identity)?; + let uid = uid.ok_or_else(|| miette::miette!("workspace probe UID is unresolved"))?; + let gid = gid.ok_or_else(|| miette::miette!("workspace probe GID is unresolved"))?; + supplementary_gids.sort_unstable_by_key(|group| group.as_raw()); + supplementary_gids.dedup(); + + if nix::unistd::geteuid().is_root() { + #[cfg(not(any( + target_os = "macos", + target_os = "ios", + target_os = "haiku", + target_os = "redox" + )))] + nix::unistd::setgroups(&supplementary_gids).into_diagnostic()?; + nix::unistd::setgid(gid).into_diagnostic()?; + nix::unistd::setuid(uid).into_diagnostic()?; + } + + let effective_identity = (nix::unistd::geteuid(), nix::unistd::getegid()); + if effective_identity != (uid, gid) { + return Err(miette::miette!( + "workspace probe privilege drop failed: expected {uid}:{gid}, got {}:{}", + effective_identity.0, + effective_identity.1 + )); + } + #[cfg(target_os = "linux")] + validate_oci_workspace_as_effective_identity(workdir)?; + #[cfg(not(target_os = "linux"))] + validate_oci_workspace( + workdir, + Some(effective_identity.0), + Some(effective_identity.1), + &supplementary_gids, + )?; + Ok(workspace_identity_attestation( + Some(uid), + Some(gid), + &supplementary_gids, + )) +} + +/// Normalize a completed process identity for comparison with the immutable +/// image probe. +#[cfg(unix)] +pub fn workspace_identity_attestation( + uid: Option, + gid: Option, + supplementary_gids: &[Gid], +) -> String { + let mut groups = supplementary_gids + .iter() + .map(|group| group.as_raw()) + .collect::>(); + groups.sort_unstable(); + groups.dedup(); + format!( + "{}:{}:{}", + uid.map_or_else(String::new, |value| value.as_raw().to_string()), + gid.map_or_else(String::new, |value| value.as_raw().to_string()), + groups + .iter() + .map(u32::to_string) + .collect::>() + .join(",") + ) +} + +/// Resolve and normalize the completed policy/driver identity without changing +/// process credentials. +#[cfg(unix)] +pub fn resolved_workspace_identity_attestation( + policy: &SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, +) -> Result { + let (uid, gid, supplementary_gids) = resolve_filesystem_identity(policy, resolved_identity)?; + Ok(workspace_identity_attestation( + uid, + gid, + &supplementary_gids, + )) +} + #[cfg(unix)] fn validate_workspace_component( path: &Path, @@ -1379,6 +1475,105 @@ fn validate_workspace_component( Ok(()) } +#[cfg(target_os = "linux")] +fn validate_oci_workspace_as_effective_identity(root: &Path) -> Result<()> { + use rustix::fs::{Access, AtFlags, FileType, Mode, OFlags}; + + let components = validated_workspace_components(root, false)?; + let open_flags = OFlags::PATH | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC; + let mut current_path = PathBuf::from("/"); + let mut current_fd = rustix::fs::open("/", open_flags, Mode::empty()).into_diagnostic()?; + reject_special_workspace_filesystem_fd(¤t_fd, ¤t_path)?; + rustix::fs::accessat( + ¤t_fd, + ".", + Access::EXEC_OK, + AtFlags::EACCESS | AtFlags::SYMLINK_NOFOLLOW, + ) + .map_err(|error| { + miette::miette!( + "workspace path component '{}' is not traversable by the sandbox identity in the image: {error}", + current_path.display() + ) + })?; + + let last_component = components.len().saturating_sub(1); + + for (index, component) in components.into_iter().enumerate() { + current_path.push(&component); + let stat = rustix::fs::statat(¤t_fd, &component, AtFlags::SYMLINK_NOFOLLOW).map_err( + |error| { + if error == rustix::io::Errno::NOENT { + miette::miette!( + "image workspace path component '{}' does not exist", + current_path.display() + ) + } else { + miette::miette!( + "failed to inspect image workspace path component '{}': {error}", + current_path.display() + ) + } + }, + )?; + let file_type = FileType::from_raw_mode(stat.st_mode); + if file_type.is_symlink() { + return Err(miette::miette!( + "workspace path component '{}' is a symlink — refusing to follow it", + current_path.display() + )); + } + if !file_type.is_dir() { + return Err(miette::miette!( + "workspace path component '{}' is not a directory", + current_path.display() + )); + } + + let is_workspace = index == last_component; + let access = if is_workspace { + Access::WRITE_OK | Access::EXEC_OK + } else { + Access::EXEC_OK + }; + rustix::fs::accessat( + ¤t_fd, + &component, + access, + AtFlags::EACCESS | AtFlags::SYMLINK_NOFOLLOW, + ) + .map_err(|error| { + let requirement = if is_workspace { + "writable and traversable" + } else { + "traversable" + }; + miette::miette!( + "workspace path component '{}' is not {requirement} by the sandbox identity in the image: {error}", + current_path.display() + ) + })?; + + current_fd = rustix::fs::openat(¤t_fd, &component, open_flags, Mode::empty()) + .map_err(|error| { + miette::miette!( + "failed to open image workspace path component '{}': {error}", + current_path.display() + ) + })?; + reject_special_workspace_filesystem_fd(¤t_fd, ¤t_path)?; + } + + Ok(()) +} + +#[cfg(target_os = "linux")] +fn reject_special_workspace_filesystem_fd(fd: &impl std::os::fd::AsFd, path: &Path) -> Result<()> { + let fs = rustix::fs::fstatfs(fd).into_diagnostic()?; + #[allow(clippy::cast_sign_loss)] + reject_special_workspace_filesystem_type(path, fs.f_type as u64) +} + #[cfg(target_os = "linux")] fn reject_special_workspace_filesystem(path: &Path) -> Result<()> { // Linux filesystem magic values for virtual/kernel-managed filesystems. @@ -1678,17 +1873,20 @@ pub fn prepare_filesystem_with_identity( let (uid, gid, supplementary_gids) = resolve_filesystem_identity(policy, resolved_identity)?; - // Docker owns workspace resolution and must make the selected root usable - // by the final effective identity, including when both policy identity - // fields were explicit. Validate it before processing any user-authored - // read-write paths so an unsafe image path fails first. Other drivers - // retain their preparation. + // Docker and Podman own workspace resolution and must make the selected + // root usable by the final effective identity, including when both policy + // identity fields were explicit. Validate it before processing any + // user-authored read-write paths so an unsafe image path fails first. + // Other drivers retain their preparation. if prepare_workspace { let workspace = workdir.ok_or_else(|| { miette::miette!("local container driver did not supply a workspace workdir") })?; let workspace = Path::new(workspace); - if workspace == Path::new(openshell_core::driver_mounts::DEFAULT_WORKSPACE_ROOT) { + if workspace == Path::new(openshell_core::driver_mounts::DEFAULT_WORKSPACE_ROOT) + || std::env::var_os(openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY) + .is_some_and(|value| !value.is_empty()) + { info!(path = %workspace.display(), ?uid, ?gid, "Preparing managed workspace"); prepare_oci_workspace(workspace, uid, gid, &supplementary_gids)?; } else { @@ -1711,8 +1909,8 @@ pub fn prepare_filesystem_with_identity( } // Retain the existing Kubernetes/OpenShift behavior for driver-injected - // numeric identities. Docker clears this variable and does not receive - // identity-specific workspace preparation. + // numeric identities. Docker and Podman clear this variable and do not + // receive identity-specific workspace preparation. if std::env::var(openshell_core::sandbox_env::SANDBOX_UID).is_ok_and(|uid| !uid.is_empty()) { let sandbox_home = Path::new("/sandbox"); if sandbox_home.exists() { @@ -2852,6 +3050,23 @@ mod tests { .expect("supplementary group already has write and traverse authority"); } + #[cfg(unix)] + #[test] + fn workspace_identity_attestation_sorts_and_deduplicates_groups() { + assert_eq!( + workspace_identity_attestation( + Some(Uid::from_raw(1000)), + Some(Gid::from_raw(1001)), + &[ + Gid::from_raw(1003), + Gid::from_raw(1002), + Gid::from_raw(1003), + ], + ), + "1000:1001:1002,1003" + ); + } + #[cfg(unix)] #[test] fn validate_oci_workspace_rejects_unwritable_directory() { @@ -2888,6 +3103,25 @@ mod tests { assert!(error.to_string().contains("does not exist")); } + #[cfg(target_os = "linux")] + #[test] + fn effective_identity_validation_uses_kernel_access_checks() { + if nix::unistd::geteuid().is_root() { + return; + } + + let dir = tempfile::tempdir_in("/tmp").unwrap(); + let root = dir.path().canonicalize().unwrap().join("project"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); + validate_oci_workspace_as_effective_identity(&root) + .expect("current identity can write and enter its directory"); + + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o500)).unwrap(); + let error = validate_oci_workspace_as_effective_identity(&root).unwrap_err(); + assert!(error.to_string().contains("not writable and traversable")); + } + #[cfg(unix)] #[test] fn validate_oci_workspace_rejects_restrictive_parent() { diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 03dd7517fb..650836c3f9 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -700,8 +700,8 @@ impl Default for PtyRequest { /// (or defaults to `/home/{user}`). /// /// For numeric UIDs, there is no passwd entry, so the default remains -/// `("{uid}", "/sandbox")`. Docker replaces that default with its resolved -/// image workspace. +/// `("{uid}", "/sandbox")`. Docker and Podman replace that default with their +/// resolved image workspace. fn session_user_and_home(policy: &SandboxPolicy, workdir_home: Option<&str>) -> (String, String) { let (user, default_home) = match policy.process.run_as_user.as_deref() { Some(user) if !user.is_empty() => { diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 32b552e108..55e6fecdb8 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -241,9 +241,10 @@ Podman mount schema: Podman `volume` and `image` mounts do not support `subpath` in OpenShell driver config, and OpenShell rejects `subpath` for those mount types. OpenShell rejects mount `source` and `target` values with surrounding whitespace. OpenShell also -rejects mount targets that replace the workspace root, container root, supervisor -files, `/etc/openshell`, `/etc/openshell-tls`, authentication material, or -network namespace paths. These checks do not make host bind mounts safe. +rejects mount targets that replace or contain the workspace root, target the +container root, or contain or are contained by concrete OpenShell control +targets such as the supervisor mount, TLS and token files, runtime socket, or +network namespace mount. These checks do not make host bind mounts safe. ## MicroVM Driver @@ -434,7 +435,7 @@ declared name or numeric components for both direct and SSH children. When `USER` omits the group, the supervisor uses the user's numeric primary GID. It does not modify `/etc/passwd` or `/etc/group`. -Docker also inspects OCI `WorkingDir`. An absolute value becomes the +Docker and Podman also inspect OCI `WorkingDir`. An absolute value becomes the agent workspace; an empty, root (`/`), or explicit `/sandbox` value uses the managed `/sandbox` compatibility workspace. OpenShell creates and owns that compatibility workspace. Any other workdir must @@ -445,10 +446,17 @@ ownership or mode. It rejects kernel-managed filesystems based on filesystem metadata and rejects overlap with actual OpenShell control paths. Docker checks the original image filesystem in the final supervisor and rejects image `VOLUME` declarations that would mask the workdir or one of its parents before -validation. The resolved workspace is the cwd and `HOME` for direct and SSH -children. The supervisor itself starts from `/`, so a missing or invalid -workspace is handled during readiness instead of preventing the container -runtime from starting it. +validation. Podman checks the same +pinned image ID in a temporary networkless probe without the workspace volume, +token, or TLS secrets. The probe retains only `SETUID` and `SETGID`, drops to +the completed process identity, and uses the kernel to validate access. The +final supervisor must match the probe's normalized identity attestation, +including when the image supplies the default process policy. Podman captures +a bounded, sanitized failure diagnostic, removes the probe, and only then +creates the final volume-backed sandbox. The resolved workspace is the cwd and +`HOME` for direct and SSH children. The supervisor itself starts from `/`, so a +missing or invalid workspace is handled during readiness instead of preventing +the container runtime from starting it. Sandbox creation fails before readiness if a required `USER` component is missing, malformed, unknown, ambiguous, or resolves to UID/GID 0. An image @@ -479,9 +487,9 @@ The VM driver injects the sandbox UID into the rootfs guest's `/etc/passwd`, `/e Docker and Podman custom images do not need a baked-in `"sandbox"` user. Declare a non-root OCI `USER`, or set both process identity fields explicitly in policy. Named image users require matching account entries; a numeric `UID:GID` pair -does not. For Docker, declare an absolute OCI `WORKDIR` to select the workspace. -Images with no working directory, `WORKDIR /`, or `WORKDIR /sandbox` use -OpenShell's managed `/sandbox` compatibility workspace. For any other Docker -path, create the directory in the image and grant the final process identity -write and execute permission in the Dockerfile. Podman, Kubernetes/OpenShift, -and VM sandboxes continue to use `/sandbox`. +does not. Declare an absolute OCI `WORKDIR` to select the workspace. Images with +no working directory, `WORKDIR /`, or `WORKDIR /sandbox` use OpenShell's +managed `/sandbox` compatibility workspace. +For any other path, create the directory in the image and grant the final +process identity write and execute permission in the Dockerfile. +Kubernetes/OpenShift and VM sandboxes continue to use `/sandbox`. diff --git a/e2e/rust/tests/custom_image.rs b/e2e/rust/tests/custom_image.rs index e43cdb0514..632fb3760b 100644 --- a/e2e/rust/tests/custom_image.rs +++ b/e2e/rust/tests/custom_image.rs @@ -55,6 +55,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ && useradd -m -u 3234 -g appstaff app WORKDIR /workspace/project +# Image metadata must not be able to forge the driver's successful-validation +# attestation. +ENV OPENSHELL_OCI_WORKSPACE_IDENTITY=3234:3235: USER app CMD ["sleep", "infinity"] "#; diff --git a/e2e/rust/tests/driver_config_volume.rs b/e2e/rust/tests/driver_config_volume.rs index 0702a4637d..2a91f9ce1d 100644 --- a/e2e/rust/tests/driver_config_volume.rs +++ b/e2e/rust/tests/driver_config_volume.rs @@ -25,9 +25,9 @@ use serde_json::{Map, Value}; const TEST_IMAGE: &str = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest"; const VOLUME_TARGET: &str = "/sandbox/e2e-volume"; const BIND_TARGET: &str = "/sandbox/e2e-bind"; -#[cfg(feature = "e2e-docker")] +#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] const OCI_VOLUME_TARGET: &str = "/workspace/project/e2e-volume"; -#[cfg(feature = "e2e-docker")] +#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] const OCI_USER_DOCKERFILE: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ @@ -169,12 +169,12 @@ async fn sandbox_mounts_existing_driver_config_volume() { } #[tokio::test] -#[cfg(feature = "e2e-docker")] +#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] async fn oci_workspace_preparation_skips_nested_volume_ownership() { let driver = e2e_driver().expect("OPENSHELL_E2E_DRIVER must be set by the e2e wrapper"); assert!( - driver == "docker", - "OCI workspace mount e2e requires docker, got {driver}" + matches!(driver.as_str(), "docker" | "podman"), + "OCI workspace mount e2e requires docker or podman, got {driver}" ); let volume = VolumeGuard::create(&driver) @@ -342,7 +342,7 @@ async fn verify_volume(volume: &VolumeGuard) -> Result<(), String> { Ok(()) } -#[cfg(feature = "e2e-docker")] +#[cfg(any(feature = "e2e-docker", feature = "e2e-podman"))] async fn verify_volume_ownership(volume: &VolumeGuard) -> Result<(), String> { let output = run_volume_container( volume, diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index e30516bf09..8dcc25155c 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -3,14 +3,14 @@ #![cfg(feature = "e2e-podman")] -//! Podman-specific E2E coverage for OCI identity inspection and immutable-image -//! launch. +//! Podman-specific E2E coverage for OCI identity/workspace inspection, +//! workspace-volume copy-up, and immutable-image launch. //! //! The test builds an image through the selected Podman engine, creates a -//! sandbox from its mutable tag, and verifies both the child identity and the -//! image ID recorded on the real sandbox container. This exercises the Podman -//! API inspect → protected metadata → create path rather than only its unit -//! serialization boundaries. +//! sandbox from its mutable tag, and verifies the child identity, workspace, +//! copied image content, and image ID recorded on the real sandbox container. +//! This exercises the Podman API inspect → protected metadata → create path +//! rather than only its unit serialization boundaries. use std::process::Stdio; @@ -54,7 +54,16 @@ impl ImageGuard { let containerfile = context.path().join("Containerfile"); std::fs::write( &containerfile, - format!("FROM {BASE_IMAGE}\nUSER {OCI_UID}:{OCI_GID}\n"), + format!( + "FROM {BASE_IMAGE}\n\ + USER 0:0\n\ + RUN mkdir -p /home/app/project && \ + chown {OCI_UID}:{OCI_GID} /home/app /home/app/project && \ + chmod 0700 /home/app\n\ + WORKDIR /home/app/project\n\ + RUN printf root-owned > root-owned.txt && chown {OCI_UID}:{OCI_GID} .\n\ + USER {OCI_UID}:{OCI_GID}\n" + ), ) .map_err(|err| format!("write Containerfile: {err}"))?; @@ -164,7 +173,7 @@ fn normalized_image_id(image_id: &str) -> &str { } #[tokio::test] -async fn podman_uses_oci_identity_and_inspected_image_id() { +async fn podman_uses_oci_identity_workspace_and_inspected_image_id() { if !is_e2e_driver("podman") { eprintln!("Skipping Podman OCI identity test: e2e driver is not podman"); return; @@ -178,17 +187,19 @@ async fn podman_uses_oci_identity_and_inspected_image_id() { std::fs::write(policy.path(), OCI_FALLBACK_POLICY).expect("write OCI fallback policy"); let policy_path = policy.path().to_str().expect("policy path is UTF-8"); let mut sandbox = SandboxGuard::create_keep_with_args( - &[ - "--from", - &image.tag, - "--policy", - policy_path, - "--no-tty", - ], + &["--from", &image.tag, "--policy", policy_path, "--no-tty"], &[ "sh", "-c", - "set -eu; printf 'direct-identity=%s:%s\n' \"$(id -u)\" \"$(id -g)\"; echo podman-oci-identity-ready; sleep infinity", + "set -eu; \ + test \"$(pwd -P)\" = /home/app/project; \ + test \"$HOME\" = /home/app/project; \ + test \"$(cat root-owned.txt)\" = root-owned; \ + test \"$(stat -c %u:%g .)\" = 2345:2346; \ + test \"$(stat -c %u:%g root-owned.txt)\" = 0:0; \ + touch direct-workspace-write; \ + printf 'direct-identity=%s:%s\n' \"$(id -u)\" \"$(id -g)\"; \ + echo podman-oci-identity-ready; sleep infinity", ], READY_MARKER, ) @@ -205,7 +216,13 @@ async fn podman_uses_oci_identity_and_inspected_image_id() { .exec(&[ "sh", "-c", - "test \"$(id -u):$(id -g)\" = 2345:2346; echo podman-ssh-identity-ok", + "set -eu; \ + test \"$(id -u):$(id -g)\" = 2345:2346; \ + test \"$(pwd -P)\" = /home/app/project; \ + test \"$HOME\" = /home/app/project; \ + test -f direct-workspace-write; \ + touch ssh-workspace-write; \ + echo podman-ssh-identity-ok", ]) .await .expect("SSH child should use Podman OCI identity"); diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index c99b7756b0..47f2695ef1 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -99,6 +99,17 @@ message DriverSandboxSpec { // ServiceAccount token bootstrap instead). Never echoed to the public // Sandbox proto. string sandbox_token = 11 [(openshell.options.v1.secret) = true]; + // Final process-identity inputs needed by local container drivers when they + // validate an OCI image workspace before launching the supervisor. + WorkspaceValidationIdentity workspace_validation_identity = 12; +} + +message WorkspaceValidationIdentity { + string run_as_user = 1; + string run_as_group = 2; + // No public policy was supplied, so the supervisor must discover an + // image-provided policy before resolving the final process identity. + bool discover_from_image_policy = 3; } message ResourceRequirements {