NVIDIA · matthewgrossman · Jul 30, 2026
@@ -178,6 +178,8 @@ Common findings:
 - Rootless networking unavailable: inspect Podman network configuration.
 - Sandbox image missing or pull denied: verify image reference and registry credentials.
 - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Root and missing identities are rejected.
+- Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, or supplementary groups must already be able to traverse every parent and write and enter the directory. Podman checks the original image in a networkless temporary probe before attaching the workspace volume, so inspect the probe failure in gateway logs.
+- If Podman reports probe cleanup or timeout failures, inspect temporary containers with `podman ps -a --filter name=workdir-probe` and gateway logs. The driver force-removes the probe on every normal success or failure path.
 - Supervisor cannot call back: check callback endpoint and gateway logs.
 
 ### Step 6: Check Kubernetes Helm Gateways

@@ -137,8 +137,8 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # Ubuntu 24.04 matches the environment reported in #2069 and ships
-          # Podman 4.x. The probe records whether AppArmor blocks the drop.
+          # Ubuntu 24.04 matches the AppArmor environment reported in #2069.
+          # GitHub's runner image now supplies the supported Podman 5.x line.
           - runner: ubuntu-24.04
             podman_major: "4"
             podman_package_version: "4.9.3+ds1-1ubuntu0.2"

@@ -185,8 +185,8 @@ The gateway preserves whether each policy process field was omitted. The active
 driver then supplies one authoritative identity input to the supervisor:
 
 - Docker and Podman inspect the final sandbox image, pin container creation to
-  its immutable image ID, and pass its raw OCI `Config.User`. Docker also
-  resolves the workspace from OCI `Config.WorkingDir` during that inspection.
+  its immutable image ID, and pass its raw OCI `Config.User`. They also resolve
+  the workspace from OCI `Config.WorkingDir` during that inspection.
 - Kubernetes passes its platform-resolved numeric UID/GID, including OpenShift
   SCC-derived values.
 - VM keeps its existing guest identity behavior.
@@ -199,7 +199,7 @@ and uses the same privilege-drop path for direct and SSH children. When a
 declaration omits the group, the supervisor fills it with the user's numeric
 primary GID. It does not rewrite the account files.
 
-Docker uses an absolute OCI working directory as the workspace. An
+Docker and Podman use an absolute OCI working directory as the workspace. An
 empty, root (`/`), or explicit `/sandbox` declaration uses `/sandbox`, which
 OpenShell creates and owns as a compatibility workspace. Any other workdir must already
 exist in the immutable image without symlink components. The completed
@@ -208,11 +208,18 @@ every parent and write and enter the workdir; OpenShell does not change that
 directory's ownership or mode. Filesystem metadata identifies kernel-managed
 mounts, while collision checks are derived from actual OpenShell control paths.
 Docker performs the check in the final container before workload launch and
-rejects image `VOLUME` declarations that would mask the workdir ancestry. The
-resolved workspace is the child cwd and `HOME`; when
-`filesystem.include_workdir` is enabled, it becomes the automatic writable
-policy path. Podman, Kubernetes/OpenShift, and VM retain their existing
-`/sandbox` workspace behavior.
+rejects image `VOLUME` declarations that would mask the workdir ancestry.
+Podman performs it in a minimal networkless container from the same pinned
+image ID before its managed workspace volume covers the path. The probe retains
+only the capabilities needed to adopt the completed process identity, drops to
+that identity, and asks the kernel to validate access. It emits a normalized
+identity attestation that the final supervisor must match, including when the
+image supplies the default process policy. The driver captures a bounded,
+sanitized diagnostic before removing a failed probe. The resolved workspace is
+the child cwd and `HOME`; when `filesystem.include_workdir` is enabled, it
+becomes the automatic writable policy path. Kubernetes/OpenShift keep their
+`/sandbox` PVC and `fsGroup` behavior, and VM keeps its `/sandbox` guest
+initialization path.
 
 Sandbox creation fails before the workload becomes ready when a required image
 identity is absent, malformed, unknown, ambiguous, or resolves to UID/GID 0.

@@ -126,6 +126,13 @@ pub const SANDBOX_GID: &str = "OPENSHELL_SANDBOX_GID";
 /// OCI only for the former contract.
 pub const OCI_IMAGE_USER: &str = "OPENSHELL_OCI_IMAGE_USER";
 
+/// Normalized UID/GID/supplementary-group identity attested by the Podman
+/// immutable-image workspace probe.
+///
+/// A non-empty value also asserts that the
+/// original workspace was validated before Podman's managed volume covered it.
+pub const OCI_WORKSPACE_IDENTITY: &str = "OPENSHELL_OCI_WORKSPACE_IDENTITY";
+
 // The corporate upstream-proxy configuration deliberately has no reserved
 // environment variables: it travels on the supervisor's argv
 // (`--upstream-proxy` and friends), which a sandbox image cannot forge the

@@ -2183,6 +2183,7 @@ fn build_environment_for_oci_user(
             user_env.extend(template.environment.clone());
         }
         user_env.extend(spec.environment.clone());
+        user_env.remove(openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY);
         environment.extend(user_env.clone());
         if !user_env.is_empty()
             && let Ok(json) = serde_json::to_string(&user_env)
@@ -2238,6 +2239,13 @@ fn build_environment_for_oci_user(
 
     environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN);
     environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE);
+    // Docker never uses the Podman prevalidation contract. Emit an explicit
+    // driver-owned empty value so image-baked ENV entries cannot select the
+    // managed-workspace mutation path for an image-provided workdir.
+    environment.insert(
+        openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY.to_string(),
+        String::new(),
+    );
     environment.insert(
         openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(),
         oci_user.to_string(),

@@ -44,6 +44,7 @@ fn test_sandbox() -> DriverSandbox {
             }),
             resource_requirements: None,
             sandbox_token: String::new(),
+            workspace_validation_identity: None,
         }),
         status: None,
         workspace: String::new(),
@@ -540,6 +541,10 @@ fn build_environment_protects_oci_identity_metadata() {
         (openshell_core::sandbox_env::OCI_IMAGE_USER, "spoofed"),
         (openshell_core::sandbox_env::SANDBOX_UID, "9999"),
         (openshell_core::sandbox_env::SANDBOX_GID, "9999"),
+        (
+            openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY,
+            "9999:9999:",
+        ),
     ] {
         spec.environment.insert(key.to_string(), value.to_string());
     }
@@ -552,6 +557,10 @@ fn build_environment_protects_oci_identity_metadata() {
     )));
     assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_UID)));
     assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_GID)));
+    assert!(env.contains(&format!(
+        "{}=",
+        openshell_core::sandbox_env::OCI_WORKSPACE_IDENTITY
+    )));
     assert!(!env.iter().any(|entry| entry.ends_with("=spoofed")));
     assert!(!env.iter().any(|entry| entry.ends_with("=9999")));
 }

@@ -8,14 +8,30 @@ isolation enforcement to the `openshell-sandbox` supervisor binary, which is
 sideloaded into each container via an OCI image volume mount.
 
 Before creating the container, the driver inspects the final sandbox image and
-captures its immutable image ID and raw OCI `Config.User`. Container creation
-uses that image ID with pulling disabled, preventing a mutable tag from changing
-between inspection and launch. The supervisor runs as root, resolves omitted
-policy identity fields from the image declaration, and drops only agent
-children to the completed identity. Named OCI components remain names after
-validation; a missing group is filled with the user's numeric primary GID. Explicit
-`process.run_as_user` and `process.run_as_group` values take precedence
-independently.
+captures its immutable image ID, raw OCI `Config.User`, and OCI
+`Config.WorkingDir`. Container creation uses that image ID with pulling
+disabled, preventing a mutable tag from changing between inspection and launch.
+The supervisor runs as root, resolves omitted policy identity fields from the
+image declaration, and drops only agent children to the completed identity.
+Named OCI components remain names after validation; a missing group is filled
+with the user's numeric primary GID. Explicit `process.run_as_user` and
+`process.run_as_group` values take precedence independently.
+
+An absolute OCI working directory becomes the agent workspace. An empty,
+root (`/`), or explicit `/sandbox` declaration uses `/sandbox`, which OpenShell
+creates when necessary and owns as a compatibility workspace. For any other workdir, Podman
+first starts a minimal, networkless probe from the pinned image ID without the
+workspace volume, tokens, or TLS secrets. The probe retains only `SETUID` and
+`SETGID`, resolves and adopts the completed identity, including supplementary
+groups, and uses the kernel to verify that it can traverse every parent and
+write and enter the workdir. The path must be a real directory without symlink
+components. The probe also rejects kernel-managed filesystems and overlaps with
+concrete OpenShell control resources. It emits a normalized identity
+attestation; the final supervisor must resolve to the same identity, including
+when the image supplies the default process policy. On failure, the driver
+captures a bounded, sanitized diagnostic before removing the probe. Only then
+does Podman mount and prepare the managed workspace volume at that path; normal
+copy-up preserves image content. The workspace is the child cwd and `HOME`.
 
 For a rootless networking deep dive, see [NETWORKING.md](NETWORKING.md).
 
@@ -87,9 +103,11 @@ optional `selinux_label` of `shared` (applies `:z`) or `private` (applies
 read-only by default; set `read_only: false` to make them writable. Podman
 image and volume mounts do not support `subpath` in OpenShell driver config.
 Mount `source` and `target` values must not contain surrounding whitespace.
-Mount targets must be absolute container paths and must not replace
-the workspace root (`/sandbox`) or overlap OpenShell supervisor files,
-`/etc/openshell`, `/etc/openshell-tls`, or `/run/netns`.
+Mount targets must be absolute container paths and must not replace the
+resolved workspace root or any of its parents. Nested workspace mounts remain
+valid. Mounts also must not contain or be contained by concrete OpenShell
+control targets such as the supervisor mount, TLS and token files, runtime
+socket, or `/run/netns`.
 
 Example named-volume usage:
 

@@ -25,6 +25,7 @@ const API_TIMEOUT: Duration = Duration::from_secs(30);
 
 /// Maximum allowed size for the event stream line buffer (1 MB).
 const MAX_EVENT_BUFFER: usize = 1_048_576;
+const MAX_CONTAINER_LOG_BYTES: usize = 16 * 1024;
 
 #[derive(Debug, thiserror::Error)]
 pub enum PodmanApiError {
@@ -117,6 +118,16 @@ pub struct ContainerState {
     pub finished_at: Option<String>,
 }
 
+#[derive(Debug, serde::Deserialize)]
+#[serde(untagged)]
+enum ContainerWaitResponse {
+    ExitCode(i64),
+    Status {
+        #[serde(rename = "StatusCode")]
+        status_code: i64,
+    },
+}
+
 #[derive(Debug, Clone, serde::Deserialize)]
 #[serde(rename_all = "PascalCase")]
 pub struct HealthState {
@@ -177,6 +188,8 @@ pub struct ImageInspect {
 pub struct ImageConfig {
     #[serde(default)]
     pub user: String,
+    #[serde(default)]
+    pub working_dir: String,
 }
 
 /// A container summary returned by the list API.
@@ -350,6 +363,43 @@ impl PodmanClient {
         Ok((status, bytes))
     }
 
+    /// Send a request while retaining at most `max_bytes` of the response.
+    async fn send_request_bounded(
+        &self,
+        req: Request<Full<Bytes>>,
+        timeout: Duration,
+        max_bytes: usize,
+    ) -> Result<(hyper::StatusCode, Bytes), PodmanApiError> {
+        use hyper::body::Body;
+
+        let mut sender = self.connect().await?;
+        let response = tokio::time::timeout(timeout, sender.send_request(req))
+            .await
+            .map_err(|_| PodmanApiError::Timeout(timeout))?
+            .map_err(|error| PodmanApiError::Connection(error.to_string()))?;
+        let status = response.status();
+        let deadline = tokio::time::Instant::now() + timeout;
+        let mut body = response.into_body();
+        let mut bytes = Vec::with_capacity(max_bytes.min(4096));
+        while bytes.len() < max_bytes {
+            let frame = tokio::time::timeout_at(
+                deadline,
+                std::future::poll_fn(|context| Pin::new(&mut body).poll_frame(context)),
+            )
+            .await
+            .map_err(|_| PodmanApiError::Timeout(timeout))?;
+            let Some(frame) = frame else {
+                break;
+            };
+            let frame = frame.map_err(|error| PodmanApiError::Connection(error.to_string()))?;
+            if let Some(data) = frame.data_ref() {
+                let remaining = max_bytes - bytes.len();
+                bytes.extend_from_slice(&data[..data.len().min(remaining)]);
+            }
+        }
+        Ok((status, Bytes::from(bytes)))
+    }
+
     /// Perform a versioned HTTP request and return status + body bytes.
     async fn request(
         &self,
@@ -455,6 +505,22 @@ impl PodmanClient {
         .await
     }
 
+    /// Wait for a container to exit and return its exit code.
+    pub async fn wait_container(&self, name: &str) -> Result<i64, PodmanApiError> {
+        validate_name(name)?;
+        let response: ContainerWaitResponse = self
+            .request_json(
+                hyper::Method::POST,
+                &format!("/libpod/containers/{name}/wait?condition=exited"),
+                None,
+            )
+            .await?;
+        Ok(match response {
+            ContainerWaitResponse::ExitCode(code) => code,
+            ContainerWaitResponse::Status { status_code } => status_code,
+        })
+    }
+
     /// Stop a container with a grace period in seconds.
     pub async fn stop_container(
         &self,
@@ -501,6 +567,27 @@ impl PodmanClient {
         .await
     }
 
+    /// Fetch a bounded tail of stdout and stderr from a container.
+    pub async fn container_logs(&self, name: &str) -> Result<String, PodmanApiError> {
+        validate_name(name)?;
+        let req = Self::build_request(
+            hyper::Method::GET,
+            &format!(
+                "/{API_VERSION}/libpod/containers/{name}/logs?stdout=true&stderr=true&tail=20&timestamps=false"
+            ),
+            Full::new(Bytes::new()),
+            None,
+        );
+        let (status, bytes) = self
+            .send_request_bounded(req, API_TIMEOUT, MAX_CONTAINER_LOG_BYTES)
+            .await?;
+        if status.is_success() {
+            Ok(String::from_utf8_lossy(&bytes).into_owned())
+        } else {
+            Err(error_from_response(status.as_u16(), &bytes))
+        }
+    }
+
     /// List containers matching label filters (e.g. `&["openshell.managed=true"]`).
     pub async fn list_containers(
         &self,
@@ -932,12 +1019,12 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn inspect_image_reads_immutable_id_and_oci_user() {
+    async fn inspect_image_reads_immutable_id_and_oci_config() {
         let (socket_path, request_log, handle) = spawn_podman_stub(
             "inspect-image",
             vec![StubResponse::new(
                 StatusCode::OK,
-                r#"{"Id":"sha256:immutable","Config":{"User":"app:staff"}}"#,
+                r#"{"Id":"sha256:immutable","Config":{"User":"app:staff","WorkingDir":"/workspace/project"}}"#,
             )],
         );
         let client = PodmanClient::new(socket_path.clone());
@@ -952,6 +1039,13 @@ mod tests {
             image.config.as_ref().map(|config| config.user.as_str()),
             Some("app:staff")
         );
+        assert_eq!(
+            image
+                .config
+                .as_ref()
+                .map(|config| config.working_dir.as_str()),
+            Some("/workspace/project")
+        );
         handle.await.expect("stub task should finish");
         assert_eq!(
             request_log
@@ -962,4 +1056,34 @@ mod tests {
         );
         let _ = std::fs::remove_file(socket_path);
     }
+
+    #[tokio::test]
+    async fn container_logs_fetches_bounded_stdout_and_stderr_tail() {
+        let (socket_path, request_log, handle) = spawn_podman_stub(
+            "container-logs",
+            vec![StubResponse::new(
+                StatusCode::OK,
+                "workspace validation failed\n",
+            )],
+        );
+        let client = PodmanClient::new(socket_path.clone());
+
+        let logs = client
+            .container_logs("workdir-probe")
+            .await
+            .expect("container logs should be returned");
+
+        assert_eq!(logs, "workspace validation failed\n");
+        handle.await.expect("stub task should finish");
+        assert_eq!(
+            request_log
+                .lock()
+                .expect("request log lock should not be poisoned")
+                .as_slice(),
+            [
+                "GET /v5.0.0/libpod/containers/workdir-probe/logs?stdout=true&stderr=true&tail=20&timestamps=false"
+            ]
+        );
+        let _ = std::fs::remove_file(socket_path);
+    }
 }