diff --git a/.github/workflows/dstack-ingress-release.yml b/.github/workflows/dstack-ingress-release.yml index d4a2025..154fc2e 100644 --- a/.github/workflows/dstack-ingress-release.yml +++ b/.github/workflows/dstack-ingress-release.yml @@ -14,6 +14,9 @@ permissions: jobs: build-and-attest: runs-on: ubuntu-latest + defaults: + run: + working-directory: custom-domain/dstack-ingress env: IMAGE_REGISTRY: docker.io IMAGE_REPOSITORY: ${{ vars.DOCKERHUB_ORG }}/dstack-ingress @@ -21,13 +24,31 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Parse version from tag + - name: Parse and check version run: | - VERSION=${GITHUB_REF#refs/tags/dstack-ingress-v} + # The image records its version from the committed VERSION file, so + # that a plain checkout reproduces the digest. The release tag only + # selects which commit to build and must agree with that file. + VERSION=$(tr -d '[:space:]' < VERSION) if [ -z "${VERSION}" ]; then - echo "Unable to parse version from ref: ${GITHUB_REF}" >&2 + echo "VERSION file is empty" >&2 exit 1 fi + case "${GITHUB_REF}" in + refs/tags/dstack-ingress-v*) + TAG_VERSION=${GITHUB_REF#refs/tags/dstack-ingress-v} + if [ "${TAG_VERSION}" != "${VERSION}" ]; then + echo "Tag dstack-ingress-v${TAG_VERSION} does not match the VERSION file (${VERSION})." >&2 + echo "Update VERSION and re-tag, so the image version matches the release." >&2 + exit 1 + fi + ;; + *) + echo "This workflow builds a release and must run on a dstack-ingress-v* tag." >&2 + echo "Got ref: ${GITHUB_REF}. Re-run it selecting the release tag." >&2 + exit 1 + ;; + esac echo "VERSION=${VERSION}" >> "$GITHUB_ENV" echo "IMAGE_REFERENCE=${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:${VERSION}" >> "$GITHUB_ENV" echo "Parsed version: ${VERSION}" @@ -45,15 +66,13 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build reproducible image and push - working-directory: custom-domain/dstack-ingress env: IMAGE_REFERENCE: ${{ env.IMAGE_REFERENCE }} run: | - ./build-image.sh --push "${IMAGE_REFERENCE}" + ./build-image.sh --require-clean --push "${IMAGE_REFERENCE}" - name: Capture image digest id: capture-digest - working-directory: custom-domain/dstack-ingress run: | DIGEST=$(skopeo inspect oci-archive:./oci.tar | jq -r '.Digest') if [ -z "${DIGEST}" ]; then @@ -79,6 +98,7 @@ jobs: echo "" echo "- Tag: \`${IMAGE_REFERENCE}\`" echo "- Digest: \`${IMAGE_DIGEST}\`" + echo "- Source: \`${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/${GITHUB_SHA}/custom-domain/dstack-ingress\`" echo "- Sigstore: https://search.sigstore.dev/?hash=${IMAGE_DIGEST}" } >> "$GITHUB_STEP_SUMMARY" @@ -90,4 +110,27 @@ jobs: | Image | Digest | Verification | |---|---|---| - | ${{ env.IMAGE_REFERENCE }} | ${{ steps.capture-digest.outputs.digest }} | [Verify on Sigstore](https://search.sigstore.dev/?hash=${{ steps.capture-digest.outputs.digest }}) | \ No newline at end of file + | ${{ env.IMAGE_REFERENCE }} | ${{ steps.capture-digest.outputs.digest }} | [Verify on Sigstore](https://search.sigstore.dev/?hash=${{ steps.capture-digest.outputs.digest }}) | + + ## Source + + Built from [`${{ github.sha }}`](${{ github.server_url }}/${{ github.repository }}/tree/${{ github.sha }}/custom-domain/dstack-ingress). The image records its source repository, commit and version as OCI labels and manifest annotations: + + ```bash + skopeo inspect docker://${{ env.IMAGE_REFERENCE }} | jq .Labels + skopeo inspect --raw docker://${{ env.IMAGE_REFERENCE }} | jq .annotations + ``` + + ## Reproducible Build + + Build on a native Linux amd64 host with Docker Buildx, Skopeo, jq and Git installed: + + ```bash + git clone ${{ github.server_url }}/${{ github.repository }}.git + cd dstack-examples/custom-domain/dstack-ingress + git checkout ${{ github.sha }} + ./build-image.sh + skopeo inspect oci-archive:./oci.tar | jq -r '.Digest' + ``` + + Expected digest: `${{ steps.capture-digest.outputs.digest }}` diff --git a/custom-domain/dstack-ingress/.dockerignore b/custom-domain/dstack-ingress/.dockerignore new file mode 100644 index 0000000..897832a --- /dev/null +++ b/custom-domain/dstack-ingress/.dockerignore @@ -0,0 +1,16 @@ +# The image needs exactly four things from this directory, so everything else +# is excluded by default: an untracked local file can then neither leak into +# the image nor change its digest. Enumerating junk instead would be a trap -- +# a tool's cache directory often carries its own .gitignore, so git reports a +# clean tree and the --require-clean check in build-image.sh cannot see it. +* +!.BUILD_INFO +!pinned-packages.txt +!requirements.txt +!scripts/ + +# scripts/ comes back wholesale, so keep the same caches out of it: the dot +# directories the Python tooling writes (pytest, mypy, ruff, coverage) and +# __pycache__, which is the one that does not start with a dot. +scripts/**/.* +scripts/**/__pycache__ diff --git a/custom-domain/dstack-ingress/.gitignore b/custom-domain/dstack-ingress/.gitignore index 0866584..1ce7c8f 100644 --- a/custom-domain/dstack-ingress/.gitignore +++ b/custom-domain/dstack-ingress/.gitignore @@ -2,4 +2,6 @@ /CLAUDE.md /test/ __pycache__ +.pytest_cache /oci.tar +/.BUILD_INFO diff --git a/custom-domain/dstack-ingress/Dockerfile b/custom-domain/dstack-ingress/Dockerfile index c3165e2..a28c101 100644 --- a/custom-domain/dstack-ingress/Dockerfile +++ b/custom-domain/dstack-ingress/Dockerfile @@ -97,7 +97,10 @@ RUN --mount=type=bind,source=scripts,target=/tmp/scripts,ro \ ENV PATH="/scripts:$PATH" ENV PYTHONPATH="/scripts" ENV PYTHONUNBUFFERED=1 -COPY --chmod=666 .GIT_REV /etc/ +# Source metadata generated by build-image.sh (same key=value set as the OCI +# labels and manifest annotations), so a running container can identify its +# own source revision. Printed by the entrypoint at startup. +COPY --chmod=644 .BUILD_INFO /etc/dstack-ingress/build-info ENTRYPOINT ["/scripts/entrypoint.sh"] CMD ["haproxy", "-W", "-f", "/etc/haproxy/haproxy.cfg"] diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index e80ce9f..90dc88f 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -299,7 +299,44 @@ To disable the built-in evidence endpoint and serve evidence files only through ./build-image.sh --push yourusername/dstack-ingress:tag ``` -The build script ensures reproducibility via pinned packages, deterministic timestamps, and specific buildkit version. +The build script ensures reproducibility via pinned packages, deterministic timestamps, and specific buildkit version. Building the same commit from a clean checkout produces the same image digest; CI runs the same script with `--require-clean`. + +### Image metadata + +Every image records where it came from, using the standard [OCI image annotation keys](https://github.com/opencontainers/image-spec/blob/main/annotations.md). The values are derived from the git checkout only (commit, the `VERSION` file, the Dockerfile base image), so they do not disturb reproducibility. The same key/value set is written to three places: + +| Location | How to read it | +|---|---| +| Image config labels | `skopeo inspect docker://dstacktee/dstack-ingress: \| jq .Labels` or `docker inspect --format '{{json .Config.Labels}}' ` | +| Image manifest annotations | `skopeo inspect --raw docker://dstacktee/dstack-ingress: \| jq .annotations` | +| `/etc/dstack-ingress/build-info` inside the image | `docker run --rm --entrypoint cat /etc/dstack-ingress/build-info`; also printed as the first line of the container log | + +| Key | Value | +|---|---| +| `org.opencontainers.image.source` | Repository URL (`SOURCE_URL` env when building from a fork) | +| `org.opencontainers.image.revision` | Git commit; suffixed with `-dirty` when built from an unclean tree | +| `org.opencontainers.image.version` | Contents of `VERSION`; the release tag `dstack-ingress-v` must match | +| `org.opencontainers.image.url` / `.documentation` | This directory / README at that exact commit | +| `org.opencontainers.image.base.name` / `.base.digest` | The pinned haproxy base image | + +To reproduce a published image, check out the commit from its `revision` label and run `./build-image.sh` on a native Linux amd64 host with Docker Buildx, Skopeo, jq and Git installed; the digest printed at the end must match the registry. Releases are additionally signed with SLSA provenance, verifiable with `gh attestation verify oci://docker.io/dstacktee/dstack-ingress: --owner Dstack-TEE`. + +### Releasing + +A release is not finished when the image is pushed. The compose files and the snippets above are what people deploy, so they have to point at the new image; 2.4 and 2.5 were tagged and published without that step, and every example kept deploying 2.3. + +1. Update `VERSION` and commit it. Bumping the version is a source change: the release workflow refuses to build unless the tag matches this file. + + If the base image or the installed packages changed since the last release, run `./build-image.sh` locally first and commit the regenerated `pinned-packages.txt` in the same batch. The build refuses to publish an image whose packages that file does not record, so a stale one fails the release after a full CI build. +2. Tag that commit `dstack-ingress-v` and push the tag. CI builds with `--require-clean`, pushes the image, and reports the digest in the run summary and the release notes. +3. Pin the published `@sha256:` in one commit, everywhere the examples name the image: + + ```bash + # from the repository root + grep -rn 'dstacktee/dstack-ingress:[0-9]' --include='*.yaml' --include='*.md' . + ``` + + Today that is `custom-domain/dstack-ingress/docker-compose.yaml`, `docker-compose.multi.yaml`, three snippets in this README, and `k3s/docker-compose.yaml`. ## License diff --git a/custom-domain/dstack-ingress/VERSION b/custom-domain/dstack-ingress/VERSION new file mode 100644 index 0000000..5154b3f --- /dev/null +++ b/custom-domain/dstack-ingress/VERSION @@ -0,0 +1 @@ +2.6 diff --git a/custom-domain/dstack-ingress/build-image.sh b/custom-domain/dstack-ingress/build-image.sh index 7ccad4d..f11c799 100755 --- a/custom-domain/dstack-ingress/build-image.sh +++ b/custom-domain/dstack-ingress/build-image.sh @@ -1,23 +1,53 @@ #!/bin/bash +# +# Reproducible image build. The same script runs in CI and on a developer +# machine, so everything that ends up in the image -- including the OCI +# labels/annotations that point back to the source -- is derived from the git +# checkout only. Nothing that ends up in the image may depend on wall-clock +# time, the CI run, or the build host, or the digest stops being reproducible. + +set -euo pipefail + +usage() { + echo "Usage: $0 [--push [:]] [--require-clean]" + echo "" + echo " --push Push the built image to the given registry reference." + echo " --require-clean Fail instead of warn when the working tree has" + echo " uncommitted or untracked changes (used by CI)." + echo "" + echo "Environment:" + echo " SOURCE_URL Repository URL recorded in the image metadata." + echo " Defaults to the canonical upstream repository; set it" + echo " when building from a fork." +} -# Parse command line arguments PUSH=false REPO="" +REQUIRE_CLEAN=false +PINS_REGENERATED=false while [[ $# -gt 0 ]]; do case $1 in --push) PUSH=true - REPO="$2" + REPO="${2:-}" if [ -z "$REPO" ]; then - echo "Error: --push requires a repository argument" - echo "Usage: $0 [--push [:]]" + echo "Error: --push requires a repository argument" >&2 + usage >&2 exit 1 fi shift 2 ;; + --require-clean) + REQUIRE_CLEAN=true + shift + ;; + -h|--help) + usage + exit 0 + ;; *) - echo "Usage: $0 [--push [:]]" + usage >&2 exit 1 ;; esac @@ -35,32 +65,154 @@ for required in docker skopeo jq git; do require_command "$required" done +cd "$(dirname "$0")" + +# --------------------------------------------------------------------------- +# Source metadata. Every value below is a function of the checked-out commit +# (plus SOURCE_URL for forks), so a rebuild of the same commit yields the same +# labels and therefore the same digest. +# --------------------------------------------------------------------------- +SOURCE_URL="${SOURCE_URL:-https://github.com/Dstack-TEE/dstack-examples}" +SOURCE_URL="${SOURCE_URL%/}" +SUBDIR="$(git rev-parse --show-prefix)" +SUBDIR="${SUBDIR%/}" +GIT_REV="$(git rev-parse HEAD)" +VERSION="$(tr -d '[:space:]' < VERSION)" +if [ -z "$VERSION" ]; then + echo "Error: VERSION file is empty" >&2 + exit 1 +fi + +# Untracked files count as dirty: scripts/ is copied wholesale into the image. +DIRTY="$(git status --porcelain --untracked-files=all -- .)" +if [ -n "$DIRTY" ]; then + if [ "$REQUIRE_CLEAN" = true ]; then + echo "Error: working tree is not clean; refusing to build a release image:" >&2 + echo "$DIRTY" >&2 + exit 1 + fi + echo "Warning: working tree is not clean; the image will be marked dirty and" >&2 + echo " its digest will not match a build of commit ${GIT_REV}." >&2 + GIT_REV="${GIT_REV}-dirty" +fi + +# Base image, kept in sync with the Dockerfile FROM line. The labels below are +# what someone else checks the supply chain against, so a second FROM -- a +# builder stage, say -- must not silently relabel the image after the wrong +# one. Fail instead, and whoever adds the stage picks the right base. +BASE_REFS=$(sed -n 's/^FROM[[:space:]][[:space:]]*\([^[:space:]][^[:space:]]*\).*/\1/p' Dockerfile) +BASE_REF_COUNT=$(printf '%s\n' "$BASE_REFS" | grep -c . || true) +if [ "$BASE_REF_COUNT" -ne 1 ]; then + echo "Error: expected exactly one FROM in the Dockerfile, found ${BASE_REF_COUNT}" >&2 + if [ -n "$BASE_REFS" ]; then + printf '%s\n' "$BASE_REFS" | sed 's/^/ /' >&2 + fi + echo "Teach this script which one the final image is built on." >&2 + exit 1 +fi +BASE_REF="$BASE_REFS" +BASE_NAME="${BASE_REF%%@*}" +BASE_DIGEST="${BASE_REF#*@}" +case "$BASE_NAME" in + */*) ;; + *) BASE_NAME="docker.io/library/${BASE_NAME}" ;; +esac +if [ "$BASE_DIGEST" = "$BASE_REF" ]; then + echo "Error: Dockerfile FROM must pin the base image by digest" >&2 + exit 1 +fi + +# OCI standard keys: https://github.com/opencontainers/image-spec/blob/main/annotations.md +METADATA=( + "org.opencontainers.image.title=dstack-ingress" + "org.opencontainers.image.description=TLS ingress for dstack TEE applications with ACME certificates and attestation evidence" + "org.opencontainers.image.source=${SOURCE_URL}" + "org.opencontainers.image.revision=${GIT_REV}" + "org.opencontainers.image.version=${VERSION}" + "org.opencontainers.image.url=${SOURCE_URL}/tree/${GIT_REV%-dirty}/${SUBDIR}" + "org.opencontainers.image.documentation=${SOURCE_URL}/blob/${GIT_REV%-dirty}/${SUBDIR}/README.md" + "org.opencontainers.image.licenses=MIT" + "org.opencontainers.image.base.name=${BASE_NAME}" + "org.opencontainers.image.base.digest=${BASE_DIGEST}" +) + +# The same key=value set goes to three places: image config labels (docker +# inspect, skopeo inspect), image manifest annotations (visible in the registry +# without fetching the config) and a file inside the image (readable from the +# running container, printed by the entrypoint). +METADATA_ARGS=() +for kv in "${METADATA[@]}"; do + METADATA_ARGS+=(--label "$kv" --annotation "manifest:$kv") +done + +BUILD_INFO=.BUILD_INFO +PACKAGES_BUILT=$(mktemp) +cleanup() { + rm -f "$BUILD_INFO" "$PACKAGES_BUILT" + docker rmi "$TEMP_TAG" >/dev/null 2>&1 || true +} +TEMP_TAG="dstack-ingress-temp:$(date +%s)" +trap cleanup EXIT + +printf '%s\n' "${METADATA[@]}" > "$BUILD_INFO" + +echo "Image metadata:" +sed 's/^/ /' "$BUILD_INFO" +echo "" + # Check if buildkit_20 already exists before creating it if ! docker buildx inspect buildkit_20 &>/dev/null; then docker buildx create --use --driver-opt image=moby/buildkit:v0.20.2 --name buildkit_20 fi touch pinned-packages.txt -git rev-parse HEAD > .GIT_REV -TEMP_TAG="dstack-ingress-temp:$(date +%s)" docker buildx build --builder buildkit_20 --no-cache --build-arg SOURCE_DATE_EPOCH="0" \ + "${METADATA_ARGS[@]}" \ --output type=oci,dest=./oci.tar,rewrite-timestamp=true \ --output type=docker,name="$TEMP_TAG" . -if [ "$?" -ne 0 ]; then - echo "Build failed" - rm .GIT_REV - exit 1 -fi - echo "Build completed, manifest digest:" echo "" skopeo inspect oci-archive:./oci.tar | jq .Digest echo "" +# pinned-packages.txt is an input to the build as well as its record: it is +# tracked, and bind-mounted in to pin apt. Overwriting it in place would make +# the next run of this script see a dirty tree and stamp the image -dirty, so +# compare first and say what happened. +echo "Checking pinned-packages.txt against the built image..." +docker run --rm --entrypoint bash "$TEMP_TAG" \ + -c "dpkg -l | grep '^ii' | awk '{print \$2\"=\"\$3}' | sort" > "$PACKAGES_BUILT" + +if cmp -s "$PACKAGES_BUILT" pinned-packages.txt; then + echo "pinned-packages.txt matches the image ($(wc -l < pinned-packages.txt) packages)" +else + # Nothing that does not match its own record may leave this machine, so a + # mismatch is fatal whenever the image is about to be published. + if [ "$REQUIRE_CLEAN" = true ] || [ "$PUSH" = true ]; then + echo "Error: the image installed a package set that pinned-packages.txt does not record:" >&2 + diff -u pinned-packages.txt "$PACKAGES_BUILT" | tail -n +3 >&2 || true + echo "Regenerate it with a local build, commit it, and build again." >&2 + exit 1 + fi + cp "$PACKAGES_BUILT" pinned-packages.txt + PINS_REGENERATED=true + echo "Warning: pinned-packages.txt was out of date and has been regenerated" >&2 + echo " ($(wc -l < pinned-packages.txt) packages). This image was built with" >&2 + echo " the old pins -- commit the file and build again." >&2 +fi +echo "" + if [ "$PUSH" = true ]; then echo "Pushing image to $REPO..." skopeo copy --insecure-policy oci-archive:./oci.tar docker://"$REPO" echo "Image pushed successfully to $REPO" +elif [ "$PINS_REGENERATED" = true ]; then + # Withhold the push instructions rather than hand over a command that + # publishes this image: it was built with the pins that were just replaced, + # and the dirty check ran before that, so nothing in its labels says so. + echo "Not printing push instructions: ./oci.tar was built with the pins that" + echo "were just regenerated, and its revision label does not say so. Commit" + echo "pinned-packages.txt and build again to get an image worth publishing." else echo "To push the image to a registry, run:" echo "" @@ -71,17 +223,6 @@ else echo " skopeo copy --insecure-policy oci-archive:./oci.tar docker://[:]" echo "" echo " Pushing image to dstacktee org:" - echo " skopeo copy --insecure-policy oci-archive:./oci.tar docker://dstacktee/dstack-ingress:$(date +%Y%m%d) --authfile ~/.docker/config.json" + echo " skopeo copy --insecure-policy oci-archive:./oci.tar docker://dstacktee/dstack-ingress:${VERSION} --authfile ~/.docker/config.json" fi echo "" - -# Extract package information from the built image -echo "Extracting package information from built image: $TEMP_TAG" -docker run --rm --entrypoint bash "$TEMP_TAG" -c "dpkg -l | grep '^ii' | awk '{print \$2\"=\"\$3}' | sort" > pinned-packages.txt - -echo "Package information extracted to pinned-packages.txt ($(wc -l < pinned-packages.txt) packages)" - -# Clean up the temporary image from Docker daemon -docker rmi "$TEMP_TAG" 2>/dev/null || true - -rm .GIT_REV diff --git a/custom-domain/dstack-ingress/scripts/entrypoint.sh b/custom-domain/dstack-ingress/scripts/entrypoint.sh index 1f564e7..2656c19 100644 --- a/custom-domain/dstack-ingress/scripts/entrypoint.sh +++ b/custom-domain/dstack-ingress/scripts/entrypoint.sh @@ -9,6 +9,16 @@ set -e +# Identify the build first, so the line is there even if validation below +# fails. The same values are in the image labels and manifest annotations. +if [ -r /etc/dstack-ingress/build-info ]; then + awk -F= '{ key = $1; sub(/^[^=]*=/, ""); info[key] = $0 } + END { print "dstack-ingress " info["org.opencontainers.image.version"] \ + " revision " info["org.opencontainers.image.revision"] \ + " source " info["org.opencontainers.image.source"] }' \ + /etc/dstack-ingress/build-info +fi + source /scripts/functions.sh PORT=${PORT:-443}