Stage the definitions and run sysupdate once

cmd/quarry-sysupdate/transfers.go

@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2026 Amutable GmbH
+
+package sysupdate
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"maps"
+	"slices"
+
+	"go.amutable.dev/quarry/internal/systemdcmd"
+	"go.amutable.dev/quarry/internal/transferlayout"
+	"go.amutable.dev/quarry/internal/xsysupdate"
+)
+
+// sysupdateRunArgs returns the systemd-run arguments that run systemd-sysupdate
+// with the given arguments in a transient service that sees the staged
+// definitions through read-only bind mounts (see
+// [xsysupdate.StagedTransfers.BindPaths]).
+func sysupdateRunArgs(bindPaths []string, args ...string) []string {
+	runArgs := []string{"--collect", "--pipe"}
+	for _, bind := range bindPaths {
+		runArgs = append(runArgs, "-p", "BindReadOnlyPaths="+bind)
+	}
+	runArgs = append(runArgs, systemdcmd.FindCmd("sysupdate"))
+	return append(runArgs, args...)
+}
+
+// stageAndUpdate stages the definitions of every component with transfer
+// definitions for the given machine tags and updates them in one
+// systemd-sysupdate run (see [updateStaged]).
+func stageAndUpdate(ctx context.Context, ext *xsysupdate.TransferFileExtension, machineTags []string) error {
+	// Only components with transfer definitions are updated. A component
+	// without any is not managed by us.
+	transfers := ext.Transfers()
+	if len(transfers) == 0 {
+		slog.Info("No transfer definitions found -- nothing to update.")
+		return nil
+	}
+
+	var errs []error
+	components := slices.Sorted(maps.Keys(transfers))
+	slog.Info("Staging transfer definitions for 'systemd-sysupdate update'...",
+		"components", len(components), "machineTags", machineTags)
+	var staged []*xsysupdate.StagedTransfers
+	for _, component := range components {
+		componentDir := transferlayout.ComponentDir(component) // for log messages
+		sc, err := ext.Stage(ctx, transfers[component], xsysupdate.StageOptions{MachineTags: machineTags})
+		if err != nil {
+			slog.Error("Staging the component's transfer definitions failed.",
+				"err", err.Error(), "component", componentDir)
+			// FIXME: If we hit an error, continue on because we don't have
+			// a decent revert story at the moment.
+			errs = append(errs, err)
+			continue
+		}
+		if sc == nil {
+			continue // not enabled on this machine
+		}
+		slog.Info("Staged transfer definitions.",
+			"component", componentDir, "path", sc.Path(), "enabledFeatures", sc.EnabledFeatures)
+		staged = append(staged, sc)
+	}
+	if len(staged) > 0 {
+		// There is one run for everything staged. The hook units merge
+		// the extensions and relink the UKI after each component.
+		// TODO: Make sure we control this better to only do it at the end
+		// after everything was successfully updated.
+		slog.Info("Spawning 'systemd-sysupdate update' with the staged transfer definitions...",
+			"components", len(staged))
+		if err := updateStaged(ctx, staged); err != nil {
+			slog.Error("sysupdate failed.", "err", err.Error())
+			errs = append(errs, err)
+		}
+	}
+	if err := errors.Join(errs...); err != nil {
+		return fmt.Errorf("sysupdate failed: %w", err)
+	}
+	slog.Info("'systemd-sysupdate update' completed.")
+	return nil
+}
+
+// updateStaged runs systemd-sysupdate once over all staged components. Their
+// definitions are bind-mounted into a transient service, where
+// "--component-all update" updates the default component and each enabled
+// component.
+func updateStaged(ctx context.Context, staged []*xsysupdate.StagedTransfers) (Err error) {
+	var bindPaths []string
+	for _, sc := range staged {
+		bindPaths = append(bindPaths, sc.BindPaths()...)
+	}
+	defer func() {
+		failed := Err != nil
+		for _, sc := range staged {
+			if failed {
+				// Keep the staged definitions for debugging. They are pruned
+				// at the start of the next run.
+				slog.Warn("Keeping staged transfer definitions of failed update.",
+					"component", transferlayout.ComponentDir(sc.Component), "path", sc.Path())
+				Err = errors.Join(Err, sc.Close())
+			} else {
+				Err = errors.Join(Err, sc.Remove())
+			}
+		}
+	}()
+
+	// TODO: Pass --cleanup=yes to garbage-collect files that are no longer
+	// covered by this component's transfers, and also explicitly run
+	// "systemd-sysupdate -C <c> cleanup" for components that are no longer
+	// enabled (or have no transfer definitions anymore) on the bare host,
+	// which deletes everything recorded in their installdb. Both are blocked
+	// on the sysupdate garbage collector taking "bootctl link" references
+	// into account, as otherwise it would remove files still referenced from
+	// the ESP.
+	//
+	// systemd-run --collect --pipe -p BindReadOnlyPaths=... systemd-sysupdate --component-all update
+	return systemdcmd.Call(ctx, "run", sysupdateRunArgs(bindPaths, "--component-all", "update")...)
+}

cmd/quarry-sysupdate/transfers_test.go

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2026 Amutable GmbH
+
+package sysupdate
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"go.amutable.dev/quarry/internal/systemdcmd"
+)
+
+func TestSysupdateRunArgs(t *testing.T) {
+	binds := []string{
+		"/var/lib/quarry-sysupdate/xsysupdate/transfers/sysupdate.foo.d/sysupdate.foo.d:/run/sysupdate.foo.d",
+		"/var/lib/quarry-sysupdate/xsysupdate/transfers/sysupdate.foo.d/sysupdate.foo.component:/run/sysupdate.foo.component",
+	}
+	sysupdate := systemdcmd.FindCmd("sysupdate")
+	assert.Equal(t, []string{
+		"--collect", "--pipe",
+		"-p", "BindReadOnlyPaths=" + binds[0],
+		"-p", "BindReadOnlyPaths=" + binds[1],
+		sysupdate, "--component-all", "update",
+	}, sysupdateRunArgs(binds, "--component-all", "update"))
+
+	// There are no -p arguments at all without bind paths.
+	assert.Equal(t, []string{"--collect", "--pipe", sysupdate, "update"}, sysupdateRunArgs(nil, "update"))
+}

cmd/quarry-sysupdate/update.go

@@ -5,6 +5,7 @@ package sysupdate
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"slices"
@@ -24,17 +25,38 @@ var updateCommand = withExtensionDirFlag(withQuarryUserFlag(withQuarryProxyFlag(
 	Name:  "update",
 	Usage: "extended 'systemd-sysupdate update' wrapper",
 	Action: func(ctx context.Context, _ *cli.Command) (Err error) {
+		transfersExt := &xsysupdate.TransferFileExtension{
+			OverrideSourcePathURL: ctxQuarryProxyURL(ctx),
+		}
 		exts := xsysupdate.ExtensionSet{
-			&xsysupdate.TransferFileExtension{
-				OverrideSourcePathURL: ctxQuarryProxyURL(ctx),
-			},
+			transfersExt,
 		}
 		ctx, err := exts.DoInit(ctx)
 		if err != nil {
 			return fmt.Errorf("init extensions: %w", err)
 		}
 		slog.Info("Initialised extensions.",
 			"extensions", exts.String())
+		// A failure before the components are updated leaves nothing
+		// installed, so the extensions are aborted (removing the staging
+		// directory). Afterwards the state is kept (see the FIXME below) and
+		// the extensions are only closed.
+		updating, closed := false, false
+		defer func() {
+			switch {
+			case Err == nil || closed:
+			case !updating:
+				// A failed Do* call has aborted already, and aborting again
+				// does nothing.
+				if err := exts.DoAbort(context.WithoutCancel(ctx), Err); err != nil {
+					Err = errors.Join(Err, fmt.Errorf("abort extensions: %w", err))
+				}
+			default:
+				if err := exts.Close(); err != nil {
+					Err = errors.Join(Err, fmt.Errorf("post-update extension cleanup: %w", err))
+				}
+			}
+		}()
 		// Legacy-prefixed targets are only applied after all other targets so
 		// we can allow them to be overridden by the non-legacy prefix.
 		// TODO: Drop this.
@@ -95,40 +117,36 @@ var updateCommand = withExtensionDirFlag(withQuarryUserFlag(withQuarryProxyFlag(
 		// FIXME FIXME FIXME
 		//
 		// If these updates error out we are kind of in a messy position.
-		// Because we need to call sysupdate multiple times to update each
-		// component, it is possible that one component installation fails. We
+		// sysupdate updates the components one after the other, so one
+		// component installation can fail after another succeeded. We
 		// would ideally never want to allow execution with mixed repo states,
 		// but fixing this requires having logic for cleanup so we can abort
 		// the extension application, clean up the successfully installed
 		// components, and force-downgrade to the old versions.
 
-		// A refreshed confext may ship machine-tags.d(5) files, and features gated on
-		// those tags were evaluated against the old tags. Loop until the tags
-		// settle.
+		// The machine tags decide which components and features are enabled
+		// while staging the definitions, because sysupdate does not enable
+		// suggested ones by itself (see xsysupdate.TransferFileExtension.Stage).
+		tags, err := hostnamed.CurrentTags(ctx, "")
+		if err != nil {
+			return fmt.Errorf("fetch machine tags before update: %w", err)
+		}
+
+		// A refreshed confext may ship machine-tags.d(5) files, and features gated
+		// on those tags were staged with the old tags. Loop until the tags
+		// settle, staging the definitions again with the new tags for each pass.
 		// TODO: The real signal to stop looping is whether the set of
 		// installable features stopped changing (rather than the tags), but
 		// there isn't a really nice way for us to detect that so we use tags
 		// as a proxy. In future, sysupdate will handle the loop for us (and
 		// thus be responsible for this detection) so this can all be removed
 		// then.
-		tags, err := hostnamed.CurrentTags(ctx, "")
-		if err != nil {
-			return fmt.Errorf("fetch machine tags before update: %w", err)
-		}
-
-		// Pick up tags the user set by hand, not only ones a confext ships.
-		if err := enableSuggested(ctx); err != nil {
-			return err
-		}
-		if err := sysupdateAll(ctx); err != nil {
-			return err
-		}
-
-		if err := exts.Close(); err != nil {
-			return fmt.Errorf("post-update extension cleanup: %w", err)
-		}
-
+		updating = true
 		for iteration := 1; ; iteration++ {
+			if err := stageAndUpdate(ctx, transfersExt, tags); err != nil {
+				return err
+			}
+
 			// Do online-reload of sysexts and confexts.
 			slog.Info("Trigger online-reload of new components.")
 			if err := sysupdateRefresh(ctx); err != nil {
@@ -140,22 +158,22 @@ var updateCommand = withExtensionDirFlag(withQuarryUserFlag(withQuarryProxyFlag(
 				return fmt.Errorf("fetch machine tags after refresh: %w", err)
 			}
 			if slices.Equal(tags, newTags) {
-				return nil
+				break
 			}
 			if iteration >= maxUpdateIterations {
 				slog.Warn("Maximum number of update iterations reached, not looking for further updates.",
 					"max", maxUpdateIterations, "before", tags, "after", newTags)
-				return nil
+				break
 			}
 			slog.Info("Machine tags changed by refresh, running another update pass.",
 				"iteration", iteration, "before", tags, "after", newTags)
 			tags = newTags
-			if err := enableSuggested(ctx); err != nil {
-				return err
-			}
-			if err := sysupdateAll(ctx); err != nil {
-				return err
-			}
 		}
+
+		closed = true
+		if err := exts.Close(); err != nil {
+			return fmt.Errorf("post-update extension cleanup: %w", err)
+		}
+		return nil
 	},
 })))

cmd/quarry-sysupdate/utils_sysupdate.go

@@ -6,87 +6,10 @@ package sysupdate
 import (
 	"context"
 	"errors"
-	"fmt"
-	"log/slog"
-	"os/exec"
 
-	"go.amutable.dev/quarry/internal/jsonutils"
 	"go.amutable.dev/quarry/internal/systemdcmd"
 )
 
-const defaultComponent = "<default>" // copied from `systemd-sysupdate components`
-
-func listComponents(ctx context.Context) ([]string, error) {
-	sysupdatePath := systemdcmd.FindCmd("sysupdate")
-	// systemd-sysupdate --json=short components
-	cmd := exec.CommandContext(ctx,
-		sysupdatePath, "--json=short", "components")
-
-	slog.Info(fmt.Sprintf("[exec] %s", cmd))
-	data, err := cmd.Output()
-	if err != nil {
-		return nil, fmt.Errorf("get list of sysupdate components (%s): %w", cmd, err)
-	}
-
-	type outputType struct {
-		Default    bool     `json:"default"`
-		Components []string `json:"components"`
-	}
-	out, err := jsonutils.Parse[outputType](data)
-	if err != nil {
-		return nil, fmt.Errorf("(%s) produced invalid json: %w", cmd, err)
-	}
-	components := out.Components
-	if out.Default {
-		components = append(components, defaultComponent)
-	}
-	return components, nil
-}
-
-func sysupdate(ctx context.Context, component string) error {
-	// systemd-sysupdate [--component=$component] update
-	args := make([]string, 0, 2)
-	if component != defaultComponent {
-		args = append(args, "--component="+component)
-	}
-	args = append(args, "update")
-	return systemdcmd.Call(ctx, "sysupdate", args...)
-}
-
-// sysupdateAll runs "systemd-sysupdate update" for every component.
-func sysupdateAll(ctx context.Context) error {
-	components, err := listComponents(ctx)
-	if err != nil {
-		return fmt.Errorf("get list of sysupdate components: %w", err)
-	}
-	slog.Info("Spawning 'systemd-sysupdate update'...",
-		"components", components)
-	var errs []error
-	for _, component := range components {
-		if err := sysupdate(ctx, component); err != nil {
-			slog.Error("sysupdate for component failed.",
-				"err", err.Error(), "component", component)
-			// FIXME: If we hit an error, continue on because we don't have
-			// a decent revert story at the moment.
-			errs = append(errs, err)
-		}
-	}
-	if err := errors.Join(errs...); err != nil {
-		return fmt.Errorf("sysupdate failed: %w", err)
-	}
-	slog.Info("'systemd-sysupdate update' completed.")
-	return nil
-}
-
-// enableSuggested enables the components and features whose SuggestOn*
-// conditions now hold, e.g. after the machine tags changed.
-func enableSuggested(ctx context.Context) error {
-	if err := systemdcmd.Call(ctx, "systemctl", "start", "systemd-sysupdate-auto-enable.service"); err != nil {
-		return fmt.Errorf("failed to trigger systemd-sysupdate-auto-enable service: %w", err)
-	}
-	return nil
-}
-
 func sysupdateRefresh(ctx context.Context) error {
 	errs := []error{
 		// systemd-confext refresh