Taskfile.yml

version: "3"

# Recipes in recipes/ build with melange into packages/, a local repository signed with
# melange.rsa (`task keygen`), on top of Wolfi, for x86-64-v3 (build/x86-64-v3.env, the level
# the kernel targets too; the optimisation level stays each build system's). melange runs in
# Docker with bubblewrap inside, which needs --privileged.

# What each build fetches from outside the repository, pinned by content.
# .env, not committed: this machine's own settings - NEEDLES, the needles checkout.
dotenv: [build/versions.env, .env]

vars:
  MELANGE: cgr.dev/chainguard/melange@sha256:3285b57ff3d6f241993f181c1cb154a590ec3ade8c3f6d987686b42693d27afe
  APKO: cgr.dev/chainguard/apko@sha256:730ae486fc77b0cc7326cdd66fb2bb3e093f9fd14b101af603fe714786752ab3
  WOLFI: --repository-append https://packages.wolfi.dev/os --keyring-append https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
  LOCAL: --repository-append /work/packages --keyring-append /work/melange.rsa.pub
  # The kernel's banner dates it by the last commit, so the same sources make the same kernel.
  KERNEL_ARGS: >-
    -f kernel/Dockerfile --build-arg WOLFI_BASE_IMAGE={{.WOLFI_BASE_IMAGE}}
    --build-arg KERNEL_VERSION={{.KERNEL_VERSION}} --build-arg KERNEL_SHA256={{.KERNEL_SHA256}}
    --build-arg "KBUILD_BUILD_TIMESTAMP=$(TZ=UTC git log -1 --date=format-local:'%a %b %e %H:%M:%S UTC %Y' --format=%cd)"
  UID:
    sh: id -u
  # The signing keys as files, outside the repository and never in a container (task usb): a
  # person's own builds and the harness's throwaway keys. A release's are in AWS KMS instead, and
  # the builder names them (CONSTRUCT_KMS_*, infra builder/user-data.sh).
  SIGNING_HOME: '{{.SIGNING_HOME | default (print .HOME "/.local/share/construct/keys")}}'

tasks:
  keygen:
    desc: "The key that signs packages/, melange.rsa, and its public half melange.rsa.pub, which apk knows the signatures by: a new pair, or the public half of a key that is here (the builder's, from Secrets Manager). Neither is in the repository: a person's builds are signed by their own."
    status: [test -f melange.rsa.pub]
    cmds:
      - test -f melange.rsa || docker run --rm -u $(id -u):$(id -g) -v {{.ROOT_DIR}}:/work -w /work {{.MELANGE}} keygen melange.rsa
      - test -f melange.rsa.pub || openssl rsa -in melange.rsa -pubout -out melange.rsa.pub

  build:
    desc: "Build one recipe: task build -- json-glib"
    deps: [keygen]
    vars:
      NAME: "{{.CLI_ARGS}}"
    cmds:
      - mkdir -p packages/x86_64 .cache/apk .cache/logs .cache/out/{{.NAME}}
      # Each build writes its own out-dir, so builds run side by side; only the merge into
      # packages/ takes the lock.
      - >-
        docker run --rm --privileged -v {{.ROOT_DIR}}:/work -w /work {{.MELANGE}}
        build recipes/{{.NAME}}.yaml --arch x86_64 --runner bubblewrap
        --signing-key melange.rsa {{.WOLFI}} {{.LOCAL}} --env-file build/x86-64-v3.env
        --out-dir .cache/out/{{.NAME}} --apk-cache-dir .cache/apk
        > .cache/logs/{{.NAME}}.log 2>&1
        || { tail -40 .cache/logs/{{.NAME}}.log; exit 1; }
      - >-
        flock .cache/packages.lock sh -c '
        cp .cache/out/{{.NAME}}/x86_64/*.apk packages/x86_64/
        && docker run --rm -v {{.ROOT_DIR}}:/work -w /work {{.MELANGE}}
        index --signing-key melange.rsa -o packages/x86_64/APKINDEX.new.tar.gz packages/x86_64/*.apk
        && docker run --rm -v {{.ROOT_DIR}}:/work -w /work cgr.dev/chainguard/wolfi-base sh -c
        "mv packages/x86_64/APKINDEX.new.tar.gz packages/x86_64/APKINDEX.tar.gz && chown -R {{.UID}}:{{.UID}} packages .cache/out/{{.NAME}} .cache/logs"'

  check:
    desc: "What a pull request must pass, with no Docker and no keys: gofmt, go vet and the Go tests, which compare the recipes with origin/main."
    cmds:
      - test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; }
      - go vet ./...
      - go test ./internal/... ./cmd/...

  kernel:config:
    desc: Merge and assert the desktop kernel's configuration, without compiling it.
    cmds:
      - docker build --target build-base {{.KERNEL_ARGS}} .

  kernel:
    desc: Build the desktop kernel and its signed modules into _output/kernel and _output/modules, with kernel/profiles/autofdo.afdo when there is one.
    cmds:
      # The profile's directory, which git keeps only once a profile is in it.
      - mkdir -p _output kernel/profiles
      - docker build --target kernel --output type=local,dest=_output {{.KERNEL_ARGS}} --build-arg KERNEL_NPROC={{.NPROC | default "8"}} .

  kernel:profile:
    desc: "Turn what construct kernel-profile recorded on a machine (task kernel:profile PERF=kernel.perf) into kernel/profiles/autofdo.afdo, against _output/kernel/vmlinux - the kernel that machine ran, built from the same commit."
    requires:
      vars: [PERF]
    cmds:
      - mkdir -p kernel/profiles
      - docker build -q -t construct-kernel-base --target build-base {{.KERNEL_ARGS}} .
      - >-
        docker run --rm -u {{.UID}} -v {{.ROOT_DIR}}:/work -w /work construct-kernel-base
        llvm-profgen --kernel --binary=_output/kernel/vmlinux --perfdata={{.PERF}} -o kernel/profiles/autofdo.afdo

  firmware:
    desc: Take the firmware a PC can need out of the pinned linux-firmware and Intel's microcode into _output/firmware.
    cmds:
      - mkdir -p _output
      - >-
        docker build -f firmware/Dockerfile --target firmware --output type=local,dest=_output
        --build-arg WOLFI_BASE_IMAGE={{.WOLFI_BASE_IMAGE}}
        --build-arg LINUX_FIRMWARE_VERSION={{.LINUX_FIRMWARE_VERSION}} --build-arg LINUX_FIRMWARE_SHA256={{.LINUX_FIRMWARE_SHA256}}
        --build-arg INTEL_MICROCODE_VERSION={{.INTEL_MICROCODE_VERSION}} --build-arg INTEL_MICROCODE_SHA256={{.INTEL_MICROCODE_SHA256}} .

  rootfs:
    desc: Build the desktop's root filesystem from image/apko.yaml (Wolfi + packages/) into _output/rootfs.tar.
    deps: [keygen]
    cmds:
      - mkdir -p _output
      # packages/ holds what every recipe makes before apko reads it: for a recipe whose apk is
      # not there, apko would take Wolfi's package instead, without a word - at-spi2-core's
      # deletes the systemd user units the screen reader needs. The builder builds them first
      # (release:packages); a checkout builds each by hand.
      - go run ./cmd/image packages missing > _output/unbuilt.txt
      - >-
        test ! -s _output/unbuilt.txt
        || { echo "rootfs: packages/ lacks what these recipes make, so apko would take Wolfi's instead; task build -- each of:"; cat _output/unbuilt.txt; exit 1; }
      # The lock, which includes image/apko.yaml and pins every package at the version task lock
      # resolved: the same root on every machine, and Wolfi's bumps only through a commit.
      - docker run --rm -v {{.ROOT_DIR}}:/work -w /work {{.APKO}} build-minirootfs --build-arch amd64 image/apko.lock.yaml _output/rootfs.tar
      - docker run --rm -v {{.ROOT_DIR}}:/work cgr.dev/chainguard/wolfi-base chown {{.UID}}:{{.UID}} /work/_output/rootfs.tar
      - go run ./cmd/image check-lock rootfs.tar
      # Every package's translations are in the root (its -lang package, which apko installs
      # only when named).
      - go run ./cmd/image check-langs
      # Every command's manual (its -doc package), the same way.
      - go run ./cmd/image check-docs

  initrd:
    desc: Build the initrd systemd runs in before the root, from image/initrd/apko.yaml, into _output/initrd.tar.
    deps: [keygen]
    cmds:
      - mkdir -p _output
      - docker run --rm -v {{.ROOT_DIR}}:/work -w /work {{.APKO}} build-minirootfs --build-arch amd64 image/initrd/apko.lock.yaml _output/initrd.tar
      - docker run --rm -v {{.ROOT_DIR}}:/work cgr.dev/chainguard/wolfi-base chown {{.UID}}:{{.UID}} /work/_output/initrd.tar
      - go run ./cmd/image check-lock initrd.tar

  lock:
    desc: "Pin every package the root and the initrd install, in image/apko.lock.yaml and image/initrd/apko.lock.yaml, as apko resolves them: recipes/' packages at their build in packages/, Wolfi's as pinned - or at Wolfi's newest with -- -all. The lock workflow bumps Wolfi's every Monday in a pull request."
    deps: [keygen]
    cmds:
      - mkdir -p _output/lock
      - docker run --rm -v {{.ROOT_DIR}}:/work -w /work {{.APKO}} lock image/apko.yaml --arch x86_64 --output _output/lock/rootfs.lock.json
      - docker run --rm -v {{.ROOT_DIR}}:/work -w /work {{.APKO}} lock image/initrd/apko.yaml --arch x86_64 --output _output/lock/initrd.lock.json
      - docker run --rm -v {{.ROOT_DIR}}:/work cgr.dev/chainguard/wolfi-base chown -R {{.UID}}:{{.UID}} /work/_output/lock
      - go run ./cmd/image lock write {{.CLI_ARGS}}

  tools:
    desc: The container the root is laid in, and this repository's programs for it (cmd/image) and for the root (cmd/construct).
    deps: [keygen]
    cmds:
      # packages/ and its public key, for systemd (build/tools/Dockerfile).
      - mkdir -p _output/tools-keys && cp melange.rsa.pub _output/tools-keys/
      - >-
        docker build -q -t construct-tools -f build/tools/Dockerfile --build-arg WOLFI_BASE_IMAGE={{.WOLFI_BASE_IMAGE}}
        --build-context packages=packages --build-context keys=_output/tools-keys build/tools
      - CGO_ENABLED=0 go build -o _output/bin/image ./cmd/image
      - CGO_ENABLED=0 GOAMD64=v3 go build -trimpath "-ldflags=-s -w" -o _output/construct/usr/bin/construct ./cmd/construct

  harness-qemu:
    desc: The harness's QEMU, built with CanoKey (build/qemu), which cmd/image runs its machines in when it is there.
    cmds:
      - >-
        docker build -q -t construct-qemu -f build/qemu/Dockerfile --build-arg HARNESS_DEBIAN_IMAGE={{.HARNESS_DEBIAN_IMAGE}}
        --build-arg QEMU_VERSION={{.QEMU_VERSION}} --build-arg QEMU_SHA256={{.QEMU_SHA256}}
        --build-arg CANOKEY_QEMU_COMMIT={{.CANOKEY_QEMU_COMMIT}} build/qemu

  disk:
    desc: "The ext4 test disk, _output/disk.raw: the same baked root as the USB disk's, booted by its kernel directly (task disk -- -test adds image/test/rootfs)."
    deps: [tools]
    cmds:
      - >-
        docker run --rm --privileged -v {{.ROOT_DIR}}:/work -w /work construct-tools -c
        '_output/bin/image disk {{.CLI_ARGS}}; rc=$?; chown -R {{.UID}}:{{.UID}} _output/disk _output/disk.raw _output/root.tar; exit $rc'

  test:
    desc: "Build the test disk and drive the session: every app and Settings panel, failed units, core dumps (task test -- -only org.gnome.Ptyxis.desktop)."
    cmds:
      - task: disk
        vars: {CLI_ARGS: -test}
      - go run ./cmd/image check
      - go run ./cmd/image test {{.CLI_ARGS}}

  boot:
    desc: "Boot _output/disk.raw in QEMU (VNC :7); task boot -- -shot _output/shot.ppm takes a screenshot and stops."
    cmds:
      - go run ./cmd/image boot {{.CLI_ARGS}}

  keygen:test:
    desc: Throwaway release, PCR and Secure Boot keys under .cache/test-keys, for the harness's disks.
    cmds:
      - task: keygen:release
        vars: {SIGNING_HOME: .cache/test-keys, UID_NAME: Construct test release}
      - task: keygen:pcr
        vars: {SIGNING_HOME: .cache/test-keys}
      - task: keygen:secureboot
        vars: {SIGNING_HOME: .cache/test-keys}

  keygen:secureboot:
    desc: "The Secure Boot keys as files, {{.SIGNING_HOME}}/secureboot: PK's, KEK's and db's keys and certificates - db's signs systemd-boot and the kernel images - and the PK, KEK and db variables systemd-boot enrolls (cmd/image/secureboot.go). RSA-2048: UEFI verifies nothing else. Once; --force signs the variables again, under the same certificates, when what they hold changes."
    status: ['test -f {{.SIGNING_HOME}}/secureboot/db.crt']
    vars:
      SB: '{{.SIGNING_HOME}}/secureboot'
    cmds:
      - mkdir -p -m 700 {{.SB}}
      # A key that has one is kept: its certificate is what the machines that enrolled it hold.
      - for k in PK KEK db; do test -f {{.SB}}/$k.pem || openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out {{.SB}}/$k.pem; done
      - chmod 600 {{.SB}}/*.pem
      - go run ./cmd/image secureboot-keys -dir {{.SB}} -pk {{.SB}}/PK.pem -kek {{.SB}}/KEK.pem -db {{.SB}}/db.pem

  keygen:pcr:
    desc: "The PCR key, {{.SIGNING_HOME}}/pcr.pem, beside the release key: it signs each kernel image's PCR 11 policy, which an installed machine's TPM releases /var's key to (cmd/image/pcrsig.go). RSA-2048, the one kind of key besides Ed25519: a TPM cannot verify Ed25519, and systemd verifies PCR policies as RSA."
    status: ['test -f {{.SIGNING_HOME}}/pcr.pem']
    cmds:
      # In SIGNING_HOME, which is the owner's alone (0700).
      - mkdir -p -m 700 {{.SIGNING_HOME}}
      - openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out {{.SIGNING_HOME}}/pcr.pem
      - chmod 600 {{.SIGNING_HOME}}/pcr.pem

  keygen:release:
    desc: "The release key as a file, {{.SIGNING_HOME}}/release.pem (Ed25519), and its OpenPGP public key release.pgp, which machines trust: for a person's own builds; a release's is in KMS (task keys:kms)."
    status: ['test -f {{.SIGNING_HOME}}/release.pgp']
    vars:
      UID_NAME: '{{.UID_NAME | default "Construct release"}}'
    cmds:
      - mkdir -p -m 700 {{.SIGNING_HOME}}
      - test -f {{.SIGNING_HOME}}/release.pem || openssl genpkey -algorithm ed25519 -out {{.SIGNING_HOME}}/release.pem
      - chmod 600 {{.SIGNING_HOME}}/release.pem
      - go run ./cmd/image release-key -key {{.SIGNING_HOME}}/release.pem -uid "{{.UID_NAME}}" -o {{.SIGNING_HOME}}/release.pgp

  keys:kms:
    desc: "The public half of a release's KMS keys into keys/, to commit: release.pgp and the Secure Boot key set - each key's certificate, once, and the PK, KEK and db variables, which PK and KEK sign here and nowhere else. Once; --force signs the variables again, under the same certificates, when what they hold changes (go test ./cmd/image says when). Needs AWS credentials that may sign with all five keys (infra kms.tf)."
    status: ['test -f keys/release.pgp && test -f keys/secureboot/db.crt && test -f keys/secureboot/db.auth']
    cmds:
      # The release key's creation time is part of its fingerprint, so release.pgp is made once
      # and kept; the certificates are what machines hold, so secureboot-keys keeps each one it
      # finds and signs the variables under it.
      - test -f keys/release.pgp || go run ./cmd/image release-key -key kms:alias/construct-release -uid "CONSTRUCT release <release@constructlinux.org>" -o keys/release.pgp
      - >-
        go run ./cmd/image secureboot-keys -dir keys/secureboot -pk kms:alias/construct-secureboot-pk
        -kek kms:alias/construct-secureboot-kek -db kms:alias/construct-secureboot-db

  usb:
    desc: "The USB stick, _output/usb/disk.raw: live, installer and update source - a signed release on its construct-update partition. Signed by the keys in SIGNING_HOME, or in KMS on the builder (task usb -- -test for the harness's, -version for a release's)."
    deps: [tools]
    vars:
      # Each key a reference (internal/sign): KMS for a release (KMS=true, release:build alone sets
      # it, with the keys the builder names), files in SIGNING_HOME otherwise. Their public halves
      # are in the repository for KMS (keys/, task keys:kms), beside the files otherwise.
      RELEASE_KEY: '{{if .KMS}}kms:{{.CONSTRUCT_KMS_RELEASE}}{{else}}{{.SIGNING_HOME}}/release.pem{{end}}'
      PCR_KEY: '{{if .KMS}}kms:{{.CONSTRUCT_KMS_PCR}}{{else}}{{.SIGNING_HOME}}/pcr.pem{{end}}'
      DB_KEY: '{{if .KMS}}kms:{{.CONSTRUCT_KMS_SECUREBOOT_DB}}{{else}}{{.SIGNING_HOME}}/secureboot/db.pem{{end}}'
      PUBLIC: '{{if .KMS}}keys{{else}}{{.SIGNING_HOME}}{{end}}'
    preconditions:
      - sh: '{{if .KMS}}test -n "{{.CONSTRUCT_KMS_RELEASE}}" -a -n "{{.CONSTRUCT_KMS_PCR}}" -a -n "{{.CONSTRUCT_KMS_SECUREBOOT_DB}}"{{else}}true{{end}}'
        msg: KMS=true needs CONSTRUCT_KMS_RELEASE, CONSTRUCT_KMS_PCR and CONSTRUCT_KMS_SECUREBOOT_DB (the builder sets them)
    cmds:
      - mkdir -p _output/release-key/usr/share/construct
      - cp {{.PUBLIC}}/release.pgp _output/release-key/usr/share/construct/release.pgp
      # Releases are Ed25519 alone: gpg checks them on the machine (systemd-sysupdate), so the
      # keyring it is given is refused here if it holds any other kind of key.
      - go run ./cmd/image check-key _output/release-key/usr/share/construct/release.pgp
      - mkdir -p _output/release-key/usr/lib/systemd
      - cp _output/release-key/usr/share/construct/release.pgp _output/release-key/usr/lib/systemd/import-pubring.pgp
      - go run ./cmd/image public-key -key {{.PCR_KEY}} -o _output/release-key/pcr-public.pem
      # The Secure Boot variables the ESP carries for systemd-boot to enroll: signed, public.
      - mkdir -p _output/release-key/secureboot
      - cp {{.PUBLIC}}/secureboot/PK.auth {{.PUBLIC}}/secureboot/KEK.auth {{.PUBLIC}}/secureboot/db.auth _output/release-key/secureboot/
      - >-
        docker run --rm --privileged -e CONSTRUCT_UPDATE_URL -v /dev:/dev -v {{.ROOT_DIR}}:/work -w /work construct-tools -c
        '_output/bin/image usb -pcr-public _output/release-key/pcr-public.pem -secureboot _output/release-key/secureboot {{.CLI_ARGS}};
        rc=$?; chown -R {{.UID}}:{{.UID}} _output/usb _output/root.tar; exit $rc'
      # The keys sign here, on the host: they never enter a container. The PCR policy first, which
      # ukify joins to the kernel image; then Secure Boot's signatures over the kernel image and
      # systemd-boot, and the release.
      - go run ./cmd/image sign-pcr -key {{.PCR_KEY}}
      - >-
        docker run --rm --privileged -v {{.ROOT_DIR}}:/work -w /work construct-tools -c
        '_output/bin/image usb-join; rc=$?; chown -R {{.UID}}:{{.UID}} _output/usb; exit $rc'
      - go run ./cmd/image usb-sign -key {{.DB_KEY}} -cert {{.PUBLIC}}/secureboot/db.crt
      - go run ./cmd/image sign-release -key {{.RELEASE_KEY}} -pubring {{.PUBLIC}}/release.pgp
      - >-
        docker run --rm --privileged -v /dev:/dev -v {{.ROOT_DIR}}:/work -w /work construct-tools -c
        '_output/bin/image usb-disk; rc=$?; chown -R {{.UID}}:{{.UID}} _output/usb; exit $rc'

  play:
    desc: "Use Construct in a window on this desktop: the stick plugged in, an empty 64 GB disk to install it on, sound and 3D. What it holds stays in _output/play between runs (task play -- -fresh starts over; task play -- -stick=false boots what is installed). The stick comes from task play:stick, or task usb."
    cmds:
      - go run ./cmd/image play {{.CLI_ARGS}}

  play:stick:
    desc: "Build the stick task play copies: the image as a machine gets it - no test layer - signed with the throwaway test key."
    deps: [keygen:test]
    cmds:
      - task: usb
        vars: {SIGNING_HOME: .cache/test-keys}
      - rm -rf _output/play

  scan:
    desc: "Scan the stick's baked root (_output/usb/root, task usb) for vulnerabilities with Grype; fails on a Critical or High a version already fixes (build/grype.yaml). The report by severity is _output/scan/report.md, Grype's own _output/scan/grype.json."
    cmds:
      - mkdir -p _output/scan/grype-db _output/scan/tmp
      # The report is written whether the scan passed or not; the scan's status is the task's.
      - >-
        docker run --rm -u {{.UID}} -v {{.ROOT_DIR}}/_output/usb/root:/root:ro -v {{.ROOT_DIR}}/_output/scan:/scan
        -v {{.ROOT_DIR}}/build/grype.yaml:/grype.yaml:ro -e GRYPE_DB_CACHE_DIR=/scan/grype-db -e TMPDIR=/scan/tmp
        {{.GRYPE_IMAGE}} -c /grype.yaml -q dir:/root -o json=/scan/grype.json; rc=$?;
        go run ./cmd/image scan-report || exit 1; exit $rc

  test:usb:
    desc: Build the USB disk with the test layer, signed by a throwaway key, and run the harness on it, booted by UEFI from a USB stick.
    deps: [keygen:test]
    cmds:
      - task: usb
        vars: {CLI_ARGS: -test, SIGNING_HOME: .cache/test-keys}
      - go run ./cmd/image test -usb -needles "{{.NEEDLES}}" {{.CLI_ARGS}}

  dev:up:
    desc: "Boot the test stick (task usb -- -test) and leave it running for task theme:dev; it stays up until task dev:down or Ctrl-C."
    cmds:
      - go run ./cmd/image dev-up {{.CLI_ARGS}}

  dev:down:
    desc: Stop the machine task dev:up left running.
    cmds:
      - go run ./cmd/image dev-down

  theme:dev:
    desc: "Build the local Orchis checkout (ORCHIS, in .env) into the machine task dev:up runs - GNOME Shell loads it, construct theme copies GTK's - and take the shell's views against the needles: _output/test/look.html (task theme:dev -- -type adds the type matrix). About a minute; the image is still the source of truth (pin, rootfs, stick)."
    cmds:
      - go run ./cmd/image theme-dev -orchis "{{.ORCHIS}}" -needles "{{.NEEDLES}}" {{.CLI_ARGS}}

  type:dev:
    desc: "Take the type specimen and its rendering matrix in the machine task dev:up runs - GTK 4's text under each hinting, antialiasing, metrics and stem darkening setting, measured: _output/test/type.html and type.json. About a minute."
    cmds:
      - go run ./cmd/image type-dev

  needles:propose:
    desc: "Open a pull request in github.com/Construct-Linux/needles with the views the last test:usb drew differently from their needles, or that have none (_output/test/look.json): merging it approves the new look. Needs NEEDLES, a checkout of it."
    cmds:
      - go run ./cmd/image needles-propose -needles "{{.NEEDLES}}"

  test:lifecycle:
    desc: "Install from a stick onto an NVMe disk, open it with a security key, refuse forged sticks, update from a URL, roll back from a broken release by stick: three test releases, in QEMU."
    deps: [keygen:test]
    cmds:
      # .cache/logs is made by task build, which a run whose packages all came from the cache never
      # ran.
      - mkdir -p _output/lifecycle .cache/logs
      # The three releases, the forged sticks and the installed disk are ~50 GB a run, of no use
      # once it ends: gone whatever its outcome, the report, screens and journal kept.
      - defer: rm -rf _output/lifecycle/*.raw _output/lifecycle/*.stick _output/lifecycle/*.release
      - for: [v1, v2, v3]
        cmd: >-
          task usb SIGNING_HOME=.cache/test-keys -- -test {{if eq .ITEM "v3"}}-broken{{end}} > .cache/logs/usb-{{.ITEM}}.log 2>&1
          && cp --sparse=always _output/usb/disk.raw _output/lifecycle/{{.ITEM}}.raw
          && rel=$(ls -d _output/release/*/ | sort | tail -1)
          && basename $rel > _output/lifecycle/{{.ITEM}}.version
          && rm -rf _output/lifecycle/{{.ITEM}}.release && cp -r $rel _output/lifecycle/{{.ITEM}}.release
      # The sticks an update must refuse, from v2's (cmd/image/forge.go; mtools is the container's).
      - >-
        docker run --rm -v {{.ROOT_DIR}}:/work -w /work construct-tools -c
        '_output/bin/image forge -stick _output/lifecycle/v2.raw; rc=$?; chown -R {{.UID}}:{{.UID}} _output/lifecycle; exit $rc'
      - go run ./cmd/image lifecycle

  gap:
    desc: How far Wolfi is from GNOME's desktop, in recipes CONSTRUCT would own.
    cmds:
      - go run ./cmd/gap

  labels:
    desc: "Create or update the repository's issue labels from .github/labels.json, never deleting one it does not list: GITHUB_REPOSITORY and GITHUB_TOKEN; task labels -- -dry-run prints what would change."
    cmds:
      - go run ./cmd/watch labels {{.CLI_ARGS}}

  watch:
    desc: "Open, update or close the issues of what needs a person: a Critical or High a version fixes in the latest release (-grype FILE, Grype's report of its SBOM), a new kernel series or point release, a new GNOME major, Intel's microcode, the pins in build/versions.env and build/host.env behind upstream. GITHUB_REPOSITORY and GITHUB_TOKEN; task watch -- -dry-run changes nothing."
    cmds:
      - go run ./cmd/watch run {{.CLI_ARGS}}

  # A release. It is built on an EC2 machine that exists for this one job (infra builder.tf): the
  # keys are KMS's and never leave it, and nothing is left running after. task release launches
  # it; the builder runs release:build; a person promotes what testing proved (release:promote).

  release:
    desc: "Build and publish a release of this commit on an EC2 builder (infra): it runs task release:build and terminates, at 2 hours at the latest. NEW=true for the first release, which starts dl's and testing's indexes. The commit must be pushed; needs AWS credentials. Follow it with task release:log."
    vars:
      COMMIT:
        sh: git rev-parse HEAD
    preconditions:
      - sh: git diff --quiet HEAD
        msg: commit first - the builder builds the commit, not this checkout
      - sh: test -n "$(git branch -r --contains {{.COMMIT}})"
        msg: push first - the builder clones the commit from GitHub
      # The builder runs the same harness, two hours and a machine later: a commit it would fail
      # on is found here first.
      - sh: go run ./cmd/image tested -usb
        msg: "task test:usb must have passed on this commit, on this machine (_output/test/tested.json)"
    cmds:
      - >-
        aws ec2 run-instances --launch-template LaunchTemplateName=construct-builder
        --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=construct-builder},{Key=construct:role,Value=builder},{Key=construct:commit,Value={{.COMMIT}}},{Key=construct:job,Value=release:build}{{if eq .NEW "true"}},{Key=construct:args,Value=NEW=true}{{end}}]'
        --query 'Instances[0].InstanceId' --output text

  release:log:
    desc: "Follow the newest builder's log from the logs bucket, as tail -f: task release:log, or -- -file recipes/<recipe>.log for one recipe's build, -id i-... for another builder. Needs AWS credentials."
    cmds:
      - go run ./cmd/image release-log {{.CLI_ARGS}}

  release:build:
    desc: "On the builder (CONSTRUCT_BUILDER): the cache, the recipes it lacks, the kernel, firmware and QEMU it lacks, the harness on a test build, then the release - built with the next vYYYYMMDD.N, signed with KMS - scanned, published, offered to testing, and the cache back."
    preconditions:
      - sh: test -n "$CONSTRUCT_BUILDER"
        msg: release:build runs on the builder (task release launches one)
    vars:
      VERSION:
        sh: go run ./cmd/image release-version
    cmds:
      - echo "release {{.VERSION}} of $(git rev-parse --short HEAD)"
      - aws secretsmanager get-secret-value --secret-id construct/melange-key --query SecretString --output text > melange.rsa
      - chmod 600 melange.rsa
      - go run ./cmd/image cache pull
      # The cache goes back however the job ends: what a failed release built is not built again.
      - defer: go run ./cmd/image cache push
      - task: release:inputs
      - task: rootfs
      - task: initrd
      # The harness on a test build of the same commit: a release whose checks fail is not built.
      - task: test:usb
      - task: test:lifecycle
      - task: usb
        vars: {CLI_ARGS: '-version {{.VERSION}}', KMS: 'true'}
      - task: scan
      - task: release:publish
        vars: {VERSION: '{{.VERSION}}', NEW: '{{.NEW}}'}
      # Every build no recipe makes goes before the cache takes packages/ back (the deferred push).
      - task: packages:prune

  packages:prune:
    desc: "Remove every apk in packages/ that no recipe makes now, and index again: old versions, and a build of a recipe never committed, which apko would take over the recipe's own."
    deps: [keygen]
    cmds:
      - go run ./cmd/image packages prune
      - >-
        docker run --rm -v {{.ROOT_DIR}}:/work -w /work {{.MELANGE}}
        index --signing-key melange.rsa -o packages/x86_64/APKINDEX.tar.gz packages/x86_64/*.apk
      - docker run --rm -v {{.ROOT_DIR}}:/work cgr.dev/chainguard/wolfi-base chown {{.UID}}:{{.UID}} /work/packages/x86_64/APKINDEX.tar.gz

  release:inputs:
    desc: "What the cache lacked, side by side: the packages, the kernel, the firmware and the harness's QEMU, none of which needs another. On the builder, after cache pull."
    internal: true
    deps: [release:packages, release:kernel, release:firmware, release:harness-qemu]

  release:packages:
    internal: true
    vars:
      # Recipes built at once in a wave: each melange build compiles with every core itself, so a
      # few at once fill the machine while the small recipes' configure steps, single-threaded,
      # overlap.
      JOBS: '{{.JOBS | default 4}}'
    cmds:
      - mkdir -p .cache
      - go run ./cmd/image packages missing > .cache/waves.txt
      - >-
        while read -r wave; do
        echo "packages: $wave";
        printf '%s\n' $wave | xargs -P {{.JOBS}} -I{} task build -- {} || exit 1;
        done < .cache/waves.txt

  release:kernel:
    internal: true
    status: [test -f _output/kernel/bzImage]
    cmds:
      - task kernel NPROC=$(nproc)

  release:firmware:
    internal: true
    status: [test -f _output/firmware/firmware.tar]
    cmds:
      - task: firmware

  release:harness-qemu:
    internal: true
    status: [docker image inspect construct-qemu >/dev/null 2>&1]
    cmds:
      - task: harness-qemu

  release:publish:
    desc: "Publish the release task usb just built (VERSION): its files - the root, its verity tree and the kernel image - at updates.<domain>/releases/<VERSION>/; its notes, SHA256SUMS, packages.txt, the SBOM and Grype's report as GitHub Release v<VERSION>; the USB stick at dl.<domain> with the sticks' signed SHA256SUMS; the public keys at keys.<domain>; and the release in the testing channel's index at updates.<domain>. On the builder; NEW=true for the first release, when dl and testing have no index yet."
    requires: {vars: [VERSION]}
    vars:
      REL: _output/release/{{.VERSION}}
      PUB: _output/publish
      DIST: s3://{{.CONSTRUCT_DIST_BUCKET}}
    cmds:
      - rm -rf {{.PUB}} && mkdir -p {{.PUB}}
      - cp {{.REL}}/* keys/release.pgp {{.PUB}}/
      - go run ./cmd/image packages-list -o {{.PUB}}/packages.txt
      - >-
        docker run --rm -u {{.UID}} -v {{.ROOT_DIR}}/_output/usb/root:/root:ro -v {{.ROOT_DIR}}/{{.PUB}}:/out
        {{.SYFT_IMAGE}} -q dir:/root -o spdx-json=/out/sbom.spdx.json
      - cp _output/scan/grype.json {{.PUB}}/grype.json
      - cp _output/scan/report.md {{.PUB}}/grype.md
      # The files machines download, on the update CDN, before any index names them: the root is
      # past a GitHub asset's 2 GiB, and the repository is private. The CDN redirects a channel's
      # file to them (infra updates/redirect.js).
      - >-
        aws s3 cp --only-show-errors --recursive {{.REL}}/ {{.DIST}}/updates/releases/{{.VERSION}}/
        --exclude '*' --include '*.raw.xz' --include '*.efi'
      - >-
        prev=$(GH_TOKEN=$(construct-github-token) gh release list --exclude-drafts --limit 1 --json tagName --jq '.[0].tagName // ""');
        prevpkgs="";
        if [ -n "$prev" ]; then GH_TOKEN=$(construct-github-token) gh release download "$prev" -p packages.txt -D _output/prev --clobber && prevpkgs=_output/prev/packages.txt; fi;
        go run ./cmd/image release-notes -version {{.VERSION}} -prev "$prev" -packages {{.PUB}}/packages.txt -prev-packages "$prevpkgs" -dl-url "$CONSTRUCT_DL_URL" -o {{.PUB}}/notes.md
      - >-
        GH_TOKEN=$(construct-github-token) gh release create v{{.VERSION}} --target $(git rev-parse HEAD)
        --title "CONSTRUCT {{.VERSION}}" --notes-file {{.PUB}}/notes.md
        $(find {{.PUB}} -maxdepth 1 -type f ! -name notes.md ! -name '*.raw.xz' ! -name '*.efi' | sort)
      # The stick, compressed, for a new machine at dl.<domain>: larger than a GitHub asset may
      # be. The three newest are kept, and dl's SHA256SUMS - signed with the release key, as a
      # release's, and kept by channel-index as a channel's index - lists them. NEW=true, on the
      # first release, starts it as it starts the testing channel.
      - xz -T0 -6 -c _output/usb/disk.raw > _output/construct_{{.VERSION}}.raw.xz
      - aws s3 cp --only-show-errors _output/construct_{{.VERSION}}.raw.xz {{.DIST}}/dl/
      - (cd _output && sha256sum construct_{{.VERSION}}.raw.xz) > _output/stick.sums
      - task: release:index
        vars: {VERSION: '{{.VERSION}}', CHANNEL: dl, ADD: _output/stick.sums, NEW: '{{.NEW}}'}
      # Only once the signed index no longer names them: before, a download of one would fail.
      - for old in $(cat _output/channel/dl/dropped); do aws s3 rm --only-show-errors {{.DIST}}/dl/$old || exit 1; done
      # The public keys at keys.<domain>, whose addresses do not move: the same files every release.
      - aws s3 sync --only-show-errors keys/ {{.DIST}}/keys/
      - aws s3 cp --only-show-errors _output/release-key/pcr-public.pem {{.DIST}}/keys/pcr-public.pem
      - task: release:index
        vars: {VERSION: '{{.VERSION}}', CHANNEL: testing, ADD: '{{.REL}}/SHA256SUMS', NEW: '{{.NEW}}'}

  release:index:
    internal: true
    desc: "CHANNEL's signed index with VERSION added from the SHA256SUMS ADD, or taken out (WITHDRAW=true): the current index fetched, the new one made, signed with the release key and uploaded. CHANNEL is a channel under updates/, or dl, the sticks'. NEW=true starts an index that is not there yet; otherwise one that cannot be fetched stops it."
    requires: {vars: [VERSION, CHANNEL]}
    vars:
      OUT: _output/channel/{{.CHANNEL}}
      DIST: s3://{{.CONSTRUCT_DIST_BUCKET}}
      AT: '{{if eq .CHANNEL "dl"}}dl{{else}}updates/{{.CHANNEL}}{{end}}'
    cmds:
      - rm -rf {{.OUT}} && mkdir -p {{.OUT}}
      - cmd: aws s3 cp --only-show-errors {{.DIST}}/{{.AT}}/SHA256SUMS {{.OUT}}/current
        if: '[ "{{.NEW}}" != true ]'
      - >-
        go run ./cmd/image channel-index {{if eq .CHANNEL "dl"}}-kind dl {{end}}{{if eq .NEW "true"}}-new{{else}}-current {{.OUT}}/current{{end}}
        {{if eq .WITHDRAW "true"}}-withdraw{{else}}-add {{.ADD}}{{end}} -version {{.VERSION}} -o {{.OUT}}
      - go run ./cmd/image sign-release -key kms:{{.CONSTRUCT_KMS_RELEASE | default "alias/construct-release"}} -pubring keys/release.pgp -dir {{.OUT}}
      - aws s3 cp --only-show-errors {{.OUT}}/SHA256SUMS {{.DIST}}/{{.AT}}/SHA256SUMS
      - aws s3 cp --only-show-errors {{.OUT}}/SHA256SUMS.gpg {{.DIST}}/{{.AT}}/SHA256SUMS.gpg

  release:promote:
    desc: "Offer a release testing proved to the next channel: task release:promote VERSION=20261004.1 [FROM=testing] [TO=stable] [NEW=true, the first time TO gets a release]. Signs with the KMS release key: AWS credentials that may, and CONSTRUCT_DIST_BUCKET (infra output buckets.dist)."
    requires: {vars: [VERSION, CONSTRUCT_DIST_BUCKET]}
    vars:
      FROM: '{{.FROM | default "testing"}}'
      TO: '{{.TO | default "stable"}}'
    cmds:
      - mkdir -p _output/channel
      - aws s3 cp --only-show-errors s3://{{.CONSTRUCT_DIST_BUCKET}}/updates/{{.FROM}}/SHA256SUMS _output/channel/{{.FROM}}.from
      - task: release:index
        vars: {VERSION: '{{.VERSION}}', CHANNEL: '{{.TO}}', ADD: '_output/channel/{{.FROM}}.from', NEW: '{{.NEW}}'}

  release:withdraw:
    desc: "Take a release out of a channel, so machines that have not taken it never will: task release:withdraw VERSION=20261004.1 CHANNEL=stable. Signs with the KMS release key; CONSTRUCT_DIST_BUCKET."
    requires: {vars: [VERSION, CHANNEL, CONSTRUCT_DIST_BUCKET]}
    cmds:
      - task: release:index
        vars: {VERSION: '{{.VERSION}}', CHANNEL: '{{.CHANNEL}}', WITHDRAW: 'true'}

  cache:seed:
    desc: "Once, from a machine that has built everything: put its packages/, kernel, firmware and QEMU in the builder's cache (CONSTRUCT_PACKAGES_BUCKET, infra output buckets.packages), so the first release does not build them all."
    requires: {vars: [CONSTRUCT_PACKAGES_BUCKET]}
    cmds:
      - go run ./cmd/image cache push