deploy: Pull bound images before staging via composefs mount

.github/workflows/ci.yml

@@ -236,9 +236,6 @@ jobs:
         seal_state: ["sealed", "unsealed"]
 
         exclude:
-          # https://github.com/bootc-dev/bootc/issues/1812
-          - test_os: centos-9
-            variant: composefs
           - seal_state: "sealed"
             boot_type: bls
           - seal_state: "sealed"

Makefile

@@ -29,7 +29,15 @@ prefix ?= /usr
 # We may in the future also want to include Fedora+derivatives as
 # the code is really tiny.
 # (Note we should also make installation of the units conditional on the rhsm feature)
-CARGO_FEATURES_DEFAULT ?= $(shell . /usr/lib/os-release; if echo "$$ID_LIKE" |grep -qF rhel; then echo rhsm; fi)
+#
+# Enable the rhel9 feature on RHEL/CentOS Stream 9, which runs kernel 5.14.
+# That kernel cannot mount an erofs image directly from a file descriptor;
+# composefs-ctl's rhel9 feature activates a loopback-device fallback instead.
+CARGO_FEATURES_DEFAULT ?= $(shell . /usr/lib/os-release; \
+  features=""; \
+  if echo "$$ID_LIKE" | grep -qF rhel; then features="$$features rhsm"; fi; \
+  if echo "$$ID_LIKE" | grep -qF rhel && [ "$$VERSION_ID" = "9" ]; then features="$$features rhel9"; fi; \
+  echo $$features)
 # You can set this to override all cargo features, including the defaults
 CARGO_FEATURES ?= $(CARGO_FEATURES_DEFAULT)
 

contrib/packaging/bootc.spec

@@ -12,6 +12,14 @@
     %bcond_with rhsm
 %endif
 
+# kernel 5.14 (RHEL/CentOS 9) cannot mount an erofs image directly from a file
+# descriptor; composefs-ctl's rhel9 feature enables a loopback-device fallback.
+%if 0%{?rhel} == 9
+    %bcond_without rhel9
+%else
+    %bcond_with rhel9
+%endif
+
 %global rust_minor %(rustc --version | cut -f2 -d" " | cut -f2 -d".")
 
 # https://github.com/bootc-dev/bootc/issues/1640
@@ -132,13 +140,16 @@ make manpages
 # Build all binaries
 %if 0%{?container_build}
 # Container build: use cargo directly with cached dependencies to avoid RPM macro overhead
-cargo build -j%{_smp_build_ncpus} --release %{?with_rhsm:--features rhsm} --bins
+cargo build -j%{_smp_build_ncpus} --release %{?with_rhsm:--features rhsm} %{?with_rhel9:--features rhel9} --bins
 %else
 # Non-container build: use RPM macros for proper dependency tracking
 %if %new_cargo_macros
-    %cargo_build %{?with_rhsm:-f rhsm} -- --bins
+    # Note: %%cargo_build's own -f option only accepts a single value, so a
+    # second -f would silently clobber the first; pass extra features as
+    # plain --features args after -- instead, which cargo unions correctly.
+    %cargo_build -- %{?with_rhsm:--features rhsm} %{?with_rhel9:--features rhel9} --bins
 %else
-    %cargo_build %{?with_rhsm:--features rhsm} -- --bins
+    %cargo_build %{?with_rhsm:--features rhsm} %{?with_rhel9:--features rhel9} -- --bins
 %endif
 %endif
 
@@ -152,7 +163,7 @@ sed -i -e '/https:\/\//d' cargo-vendor.txt
 
 %install
 # Pass CARGO_FEATURES explicitly to prevent auto-detection rebuild in install environment
-%make_install INSTALL="install -p -c" CARGO_FEATURES="%{?with_rhsm:rhsm}"
+%make_install INSTALL="install -p -c" CARGO_FEATURES="%{?with_rhsm:rhsm} %{?with_rhel9:rhel9}"
 %if %{with ostree_ext}
 make install-ostree-hooks DESTDIR=%{?buildroot}
 %endif

crates/lib/src/cli.rs

@@ -1245,6 +1245,7 @@ async fn upgrade(
             .ok_or_else(|| anyhow::anyhow!("No staged deployment found"))?;
 
         if staged_deployment.is_finalization_locked() {
+            crate::boundimage::pull_bound_images(storage, &staged_deployment).await?;
             ostree.change_finalization(&staged_deployment)?;
             println!("Staged deployment will now be applied on reboot");
         } else {

crates/lib/src/deploy.rs

@@ -1007,6 +1007,82 @@ impl MergeState {
     }
 }
 
+/// Mount an ostree commit as a composefs filesystem, returning a read-only
+/// directory handle.
+///
+/// This uses `checkout_composefs` to generate an erofs metadata image from
+/// the commit, then mounts it via composefs with the ostree repo objects
+/// as the backing data store.  The returned `Dir` is a detached mount —
+/// it stays alive as long as the fd is open.
+///
+/// The erofs image is written into the ostree repo's own `tmp/` directory
+/// (which lives on the real root filesystem) because erofs file-backed
+/// mounts require a non-stacked backing filesystem.
+#[context("Mounting ostree commit {commit}")]
+pub(crate) fn mount_ostree_commit(repo: &ostree::Repo, commit: &str) -> Result<Dir> {
+    use composefs_ctl::composefs::mount::{MountOptions, VerityRequirement, composefs_fsmount};
+    use std::os::fd::{AsFd, AsRawFd};
+
+    // Write the erofs image into the ostree repo's tmp/ directory so it
+    // lives on the real (non-stacked) filesystem.
+    let repo_dir = Dir::reopen_dir(&repo.dfd_borrow())?;
+    let repo_tmp = repo_dir
+        .open_dir("tmp")
+        .context("Opening ostree repo tmp/")?;
+    let td = cap_std_ext::cap_tempfile::TempDir::new_in(&repo_tmp)?;
+    let image_name = "image.cfs";
+
+    // Call checkout_composefs via FFI directly; the high-level ostree
+    // crate (0.20.x) has a broken feature ladder that omits v2024_7.
+    #[allow(unsafe_code)]
+    {
+        use glib::translate::*;
+        use std::ffi::CString;
+        let c_path = CString::new(image_name).unwrap();
+        let c_commit = CString::new(commit).context("Invalid commit string")?;
+        let mut error = std::ptr::null_mut();
+        // SAFETY: all pointers are valid; the repo, path, and commit are
+        // borrowed for the duration of the call.
+        let ok = unsafe {
+            ostree::ffi::ostree_repo_checkout_composefs(
+                repo.to_glib_none().0,
+                std::ptr::null_mut(),
+                td.as_raw_fd(),
+                c_path.as_ptr(),
+                c_commit.as_ptr(),
+                std::ptr::null_mut(),
+                &mut error,
+            )
+        };
+        if ok == glib::ffi::GFALSE {
+            // SAFETY: on failure, error is set by the C function.
+            return Err(unsafe { glib::Error::from_glib_full(error) })
+                .context("checkout_composefs");
+        }
+    }
+
+    // Open the erofs image and the ostree objects directory.
+    let image_fd = td
+        .open(image_name)
+        .context("Opening composefs image")?
+        .into();
+    let objects_fd = repo_dir
+        .open_dir("objects")
+        .context("Opening ostree objects dir")?;
+
+    // Mount via composefs: erofs metadata + ostree objects as data store.
+    let mount_fd = composefs_fsmount(
+        image_fd,
+        commit,
+        &[objects_fd.as_fd()],
+        VerityRequirement::Disabled, // no fsverity requirement for read-only inspection
+        &MountOptions::default(),
+    )
+    .context("composefs_fsmount")?;
+
+    Dir::reopen_dir(&mount_fd).context("Reopening composefs mount as Dir")
+}
+
 /// Stage (queue deployment of) a fetched container image.
 #[context("Staging")]
 pub(crate) async fn stage(
@@ -1054,9 +1130,9 @@ pub(crate) async fn stage(
 
     subtask.completed = true;
     subtasks.push(subtask.clone());
-    subtask.subtask = "deploying".into();
-    subtask.id = "deploying".into();
-    subtask.description = "Deploying Image".into();
+    subtask.subtask = "bound_images".into();
+    subtask.id = "bound_images".into();
+    subtask.description = "Pulling Bound Images".into();
     subtask.completed = false;
     prog.send(Event::ProgressSteps {
         task: "staging".into(),
@@ -1072,15 +1148,20 @@ pub(crate) async fn stage(
             .collect(),
     })
     .await;
-    let origin = origin_from_imageref(spec.image)?;
-    let deployment =
-        crate::deploy::deploy(sysroot, from, image, &origin, lock_finalization).await?;
+    // Pull bound images *before* staging by mounting the ostree commit
+    // as a composefs filesystem to read the image specs.  This way a pull
+    // failure never results in a staged deployment at all.
+    let repo = sysroot.get_ostree()?.repo();
+    let commit_root = mount_ostree_commit(&repo, &image.ostree_commit)?;
+    let bound_images = crate::boundimage::query_bound_images(&commit_root)?;
+    drop(commit_root);
+    crate::boundimage::pull_images(sysroot, bound_images).await?;
 
     subtask.completed = true;
     subtasks.push(subtask.clone());
-    subtask.subtask = "bound_images".into();
-    subtask.id = "bound_images".into();
-    subtask.description = "Pulling Bound Images".into();
+    subtask.subtask = "deploying".into();
+    subtask.id = "deploying".into();
+    subtask.description = "Deploying Image".into();
     subtask.completed = false;
     prog.send(Event::ProgressSteps {
         task: "staging".into(),
@@ -1096,7 +1177,9 @@ pub(crate) async fn stage(
             .collect(),
     })
     .await;
-    crate::boundimage::pull_bound_images(sysroot, &deployment).await?;
+    let origin = origin_from_imageref(spec.image)?;
+    let _deployment =
+        crate::deploy::deploy(sysroot, from, image, &origin, lock_finalization).await?;
 
     subtask.completed = true;
     subtasks.push(subtask.clone());

tmt/tests/booted/test-image-pushpull-upgrade.nu

@@ -130,9 +130,12 @@ def sanity_check_switch_progress_json [data] {
     assert equal $deploy.steps 3
     assert equal $deploy.stepsTotal 3
     let deploy_tasks = $deploy.subtasks
+    # Bound images are now pulled before staging (see deploy::stage), so
+    # the "bound_images" subtask now comes before "deploying" instead of
+    # after it.
     assert equal ($deploy_tasks | length) 5
     let deploy_names = $deploy_tasks | get subtask
-    assert equal $deploy_names ["merging", "deploying", "bound_images", "cleanup", "cleanup"]
+    assert equal $deploy_names ["merging", "bound_images", "deploying", "cleanup", "cleanup"]
 }
 
 # The second boot; verify we're in the derived image