fix(export-dynamic): verify registry artifacts before skipping workspace (RHDHBUGS-3635)

.github/workflows/test-pack-dist-dynamic.yaml

@@ -23,3 +23,6 @@ jobs:
 
       - name: Run pack-dist-dynamic tests
         run: bash export-dynamic/pack-dist-dynamic.test.sh
+
+      - name: Run verify-registry-artifacts tests
+        run: bash export-dynamic/verify-registry-artifacts.test.sh

README.md

@@ -21,6 +21,36 @@ Exports plugins as dynamic plugin archives. This should be run **after** the `ov
 - Exports plugins as dynamic plugin packages
 - Handles both frontend and backend plugins
 - Optional container image packaging
+- Verifies registry artifacts before skipping an unchanged workspace (requires `skopeo`)
+
+**Environment requirements:**
+
+When `last-publish-commit` is provided together with `image-repository-prefix`, the action uses
+`skopeo` to verify that all expected container images exist in the registry with valid dynamic
+package metadata before deciding to skip an unchanged workspace. If `skopeo` is not available in
+that situation, the script exits non-zero and sets `WORKSPACE_SKIPPED_UNCHANGED_SINCE=false` so
+that downstream steps treat the workspace as requiring export rather than silently skipping
+verification.
+
+`skopeo` is **pre-installed** in both known production environments:
+
+| Environment | Tool availability |
+|---|---|
+| GitHub Actions `ubuntu-latest` (Ubuntu 24.04) | `skopeo` 1.13+ included by default |
+| RHDH Konflux builder image (`builder.Containerfile`) | `skopeo` installed via `dnf` |
+
+If you invoke `export-dynamic.sh` directly in another environment and intend to use the
+workspace-skip path with container publishing, install `skopeo` before running the script.
+
+**Tag prefix changes force a full re-export:**
+
+The skip check looks for existing images using the tag prefix from `image-tag-prefix` (set from
+`versions.json` in the overlay flow). Because `versions.json` lives outside any workspace folder,
+a bump to that file does not appear in the `git merge-base` ancestor check, so every workspace
+looks unchanged. However, skopeo then fails to find the images under the new prefix, which
+correctly causes every workspace to be re-exported and published on that one run. This is the
+intended self-healing behaviour — workspaces that were previously published under the old prefix
+now also get tags under the new prefix — but it means that run will be longer than usual.
 
 ### override-sources
 

export-dynamic/action.yaml

@@ -60,7 +60,10 @@ inputs:
     required: false
 
   last-publish-commit:
-    description: Optional commit ID of the last successful publishing of plugin container images
+    description: >
+      Optional commit ID of the last successful publishing of plugin container images.
+      When provided together with `image-repository-prefix`, registry artifacts are verified
+      before skipping an unchanged workspace. Requires `skopeo` in PATH.
     default: ""
     required: false
 

export-dynamic/export-dynamic.sh

@@ -40,6 +40,8 @@ INPUTS_CLI_CALLER=${INPUTS_CLI_CALLER:-"npx --yes ${INPUTS_CLI_PACKAGE}@${INPUTS
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=export-dynamic/pack-dist-dynamic.sh
 source "${SCRIPT_DIR}/pack-dist-dynamic.sh"
+# shellcheck source=export-dynamic/verify-registry-artifacts.sh
+source "${SCRIPT_DIR}/verify-registry-artifacts.sh"
 
 # Check local installation first, then fall back to npx --yes (requires network)
 run_cli() {
@@ -89,6 +91,46 @@ then
     popd  > /dev/null
 fi
 
+if [[ "${skipWorkspace}" == "true" && "${INPUTS_IMAGE_REPOSITORY_PREFIX}" != "" && "${INPUTS_PUSH_CONTAINER_IMAGE}" == "true" && -f "${INPUTS_PLUGINS_FILE}" ]]
+then
+    if ! command -v skopeo >/dev/null 2>&1; then
+        echo "Error: skopeo is required to verify registry artifacts before skipping a workspace, but it was not found in PATH." >&2
+        echo "Install skopeo before running this script when INPUTS_PUSH_CONTAINER_IMAGE=true and INPUTS_LAST_PUBLISH_COMMIT is set." >&2
+        echo "Note: GitHub Actions ubuntu-latest runners and the RHDH Konflux builder image both provide skopeo." >&2
+        # Write the output before exiting so the action step captures it even when the step uses `|| true`.
+        if [[ ! "$GITHUB_OUTPUT" ]]; then GITHUB_OUTPUT=/tmp/github_output.txt; fi
+        echo "WORKSPACE_SKIPPED_UNCHANGED_SINCE=false" | tee -a "$GITHUB_OUTPUT"
+        exit 1
+    fi
+    echo "Verifying published artifacts in registry for workspace before skipping..."
+    while IFS= read -r plugin || [[ -n "$plugin" ]]
+    do
+        # Skip empty lines
+        if [[ -z "${plugin// /}" ]]; then
+            continue
+        fi
+        # Skip commented lines
+        # shellcheck disable=SC2001
+        if [[ "$(echo "$plugin" | sed 's/^#.*//')" == "" ]]; then
+            continue
+        fi
+        # shellcheck disable=SC2001
+        pluginPath=$(echo "$plugin" | sed 's/^\(^[^:]*\): *\(.*\)$/\1/')
+        if [[ -d "$pluginPath" && -f "$pluginPath/package.json" ]]; then
+            PLUGIN_NAME=$(jq -r '.name | sub("^@"; "") | sub("[/@]"; "-")' "$pluginPath/package.json")
+            PLUGIN_VERSION="${INPUTS_IMAGE_TAG_PREFIX}$(jq -r '.version' "$pluginPath/package.json")"
+            PLUGIN_CONTAINER_TAG="${INPUTS_IMAGE_REPOSITORY_PREFIX}/${PLUGIN_NAME}:${PLUGIN_VERSION}"
+            echo "  Checking registry for ${PLUGIN_CONTAINER_TAG}..."
+            if ! verify_registry_artifact "${INPUTS_IMAGE_REPOSITORY_PREFIX}" "${PLUGIN_NAME}" "${PLUGIN_VERSION}"; then
+                echo "  Missing or invalid artifact in registry: ${PLUGIN_CONTAINER_TAG}. Workspace cannot be skipped."
+                skipWorkspace=false
+                break
+            fi
+            echo "  Verified valid artifact in registry: ${PLUGIN_CONTAINER_TAG}"
+        fi
+    done < "${INPUTS_PLUGINS_FILE}"
+fi
+
 if [[ "${skipWorkspace}" == "true" ]]
 then
     echo "Skipping workspace since it didn't change since last published commit (${INPUTS_LAST_PUBLISH_COMMIT})"

export-dynamic/verify-registry-artifacts.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Verifies that a container image exists in the registry and contains valid,
+# non-empty dynamic package metadata (io.backstage.dynamic-packages).
+#
+# Supports both single-architecture OCI manifests (e.g. GHCR) and multi-architecture
+# image indexes / manifest lists (e.g. Quay/Konflux) by dereferencing child manifests.
+
+verify_registry_artifact() {
+    local repo_prefix="$1"
+    local plugin_name="$2"
+    local version="$3"
+    local transport="${4:-docker}"
+
+    local target="${transport}://${repo_prefix}/${plugin_name}:${version}"
+    if [[ "$transport" == "dir" ]]; then
+        target="dir:${repo_prefix}/${plugin_name}"
+    fi
+
+    local raw_manifest
+    raw_manifest=$(skopeo inspect --raw "${target}" 2>/dev/null) || raw_manifest=""
+    # If the image is an index/manifest list without direct annotations, resolve the first child manifest
+    if [[ -n "$raw_manifest" ]] && ! echo "$raw_manifest" | jq -e '.annotations["io.backstage.dynamic-packages"]' >/dev/null 2>&1 && echo "$raw_manifest" | jq -e '.manifests != null' >/dev/null 2>&1; then
+        local child_digest
+        child_digest=$(echo "$raw_manifest" | jq -r '.manifests[0].digest // empty')
+        if [[ -n "$child_digest" ]]; then
+            local child_target="${transport}://${repo_prefix}/${plugin_name}@${child_digest}"
+            if [[ "$transport" == "dir" ]]; then
+                child_target="dir:${repo_prefix}/${plugin_name}_child"
+            fi
+            raw_manifest=$(skopeo inspect --raw "${child_target}" 2>/dev/null) || raw_manifest=""
+        fi
+    fi
+
+    if [[ -z "$raw_manifest" ]] || ! echo "$raw_manifest" | jq -e '.annotations["io.backstage.dynamic-packages"] | @base64d | fromjson | length > 0' >/dev/null 2>&1; then
+        return 1
+    fi
+    return 0
+}

export-dynamic/verify-registry-artifacts.test.sh

@@ -0,0 +1,213 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=export-dynamic/verify-registry-artifacts.sh
+source "${SCRIPT_DIR}/verify-registry-artifacts.sh"
+
+assert_eq() {
+    local description="$1"
+    local expected="$2"
+    local actual="$3"
+    if [[ "$expected" == "$actual" ]]; then
+        echo "PASS: ${description}"
+    else
+        echo "FAIL: ${description} (expected '${expected}', got '${actual}')" >&2
+        exit 1
+    fi
+}
+
+WORKDIR=$(mktemp -d /tmp/verify-artifacts-test-XXXXXX)
+trap 'rm -rf "$WORKDIR"' EXIT
+
+# --- Offline tests using dir: transport ---
+
+# 1. Valid single OCI manifest with dynamic-packages annotation
+mkdir -p "${WORKDIR}/valid-pkg"
+cat << 'JSON' > "${WORKDIR}/valid-pkg/manifest.json"
+{
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.oci.image.manifest.v1+json",
+  "config": {
+    "mediaType": "application/vnd.oci.image.config.v1+json",
+    "digest": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "size": 2
+  },
+  "layers": [],
+  "annotations": {
+    "io.backstage.dynamic-packages": "W3sidGVzdC1wbHVnaW4iOnsidmVyc2lvbiI6IjEuMC4wIn19XQ=="
+  }
+}
+JSON
+
+res=0
+verify_registry_artifact "${WORKDIR}" "valid-pkg" "1.0.0" "dir" || res=$?
+assert_eq "offline: valid manifest accepted" "0" "$res"
+
+# 2. Manifest with empty annotation
+mkdir -p "${WORKDIR}/empty-pkg"
+cat << 'JSON' > "${WORKDIR}/empty-pkg/manifest.json"
+{
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.oci.image.manifest.v1+json",
+  "config": {
+    "mediaType": "application/vnd.oci.image.config.v1+json",
+    "digest": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "size": 2
+  },
+  "layers": [],
+  "annotations": {
+    "io.backstage.dynamic-packages": ""
+  }
+}
+JSON
+
+res=0
+verify_registry_artifact "${WORKDIR}" "empty-pkg" "1.0.0" "dir" || res=$?
+assert_eq "offline: empty annotation rejected" "1" "$res"
+
+# 3. Manifest missing dynamic-packages annotation entirely
+mkdir -p "${WORKDIR}/missing-pkg"
+cat << 'JSON' > "${WORKDIR}/missing-pkg/manifest.json"
+{
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.oci.image.manifest.v1+json",
+  "config": {
+    "mediaType": "application/vnd.oci.image.config.v1+json",
+    "digest": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "size": 2
+  },
+  "layers": [],
+  "annotations": {}
+}
+JSON
+
+res=0
+verify_registry_artifact "${WORKDIR}" "missing-pkg" "1.0.0" "dir" || res=$?
+assert_eq "offline: missing annotation rejected" "1" "$res"
+
+# 4. Manifest with empty array annotation (W10= is base64 of "[]")
+mkdir -p "${WORKDIR}/empty-arr-pkg"
+cat << 'JSON' > "${WORKDIR}/empty-arr-pkg/manifest.json"
+{
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.oci.image.manifest.v1+json",
+  "config": {
+    "mediaType": "application/vnd.oci.image.config.v1+json",
+    "digest": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "size": 2
+  },
+  "layers": [],
+  "annotations": {
+    "io.backstage.dynamic-packages": "W10="
+  }
+}
+JSON
+
+res=0
+verify_registry_artifact "${WORKDIR}" "empty-arr-pkg" "1.0.0" "dir" || res=$?
+assert_eq "offline: empty array annotation rejected" "1" "$res"
+
+# 5. Non-existent image/directory
+res=0
+verify_registry_artifact "${WORKDIR}" "non-existent" "1.0.0" "dir" || res=$?
+assert_eq "offline: non-existent target rejected" "1" "$res"
+
+# 6. Image index dereferencing valid child manifest
+mkdir -p "${WORKDIR}/index-valid-pkg" "${WORKDIR}/index-valid-pkg_child"
+cat << 'JSON' > "${WORKDIR}/index-valid-pkg/manifest.json"
+{
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.oci.image.index.v1+json",
+  "manifests": [
+    {
+      "mediaType": "application/vnd.oci.image.manifest.v1+json",
+      "digest": "sha256:childvalid",
+      "size": 100
+    }
+  ]
+}
+JSON
+cat << 'JSON' > "${WORKDIR}/index-valid-pkg_child/manifest.json"
+{
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.oci.image.manifest.v1+json",
+  "config": {
+    "mediaType": "application/vnd.oci.image.config.v1+json",
+    "digest": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "size": 2
+  },
+  "layers": [],
+  "annotations": {
+    "io.backstage.dynamic-packages": "W3sidGVzdC1wbHVnaW4iOnsidmVyc2lvbiI6IjEuMC4wIn19XQ=="
+  }
+}
+JSON
+
+res=0
+verify_registry_artifact "${WORKDIR}" "index-valid-pkg" "1.0.0" "dir" || res=$?
+assert_eq "offline: image index with valid child accepted" "0" "$res"
+
+# 7. Image index dereferencing child manifest with empty annotation
+mkdir -p "${WORKDIR}/index-empty-pkg" "${WORKDIR}/index-empty-pkg_child"
+cat << 'JSON' > "${WORKDIR}/index-empty-pkg/manifest.json"
+{
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.oci.image.index.v1+json",
+  "manifests": [
+    {
+      "mediaType": "application/vnd.oci.image.manifest.v1+json",
+      "digest": "sha256:childempty",
+      "size": 100
+    }
+  ]
+}
+JSON
+cat << 'JSON' > "${WORKDIR}/index-empty-pkg_child/manifest.json"
+{
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.oci.image.manifest.v1+json",
+  "config": {
+    "mediaType": "application/vnd.oci.image.config.v1+json",
+    "digest": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "size": 2
+  },
+  "layers": [],
+  "annotations": {
+    "io.backstage.dynamic-packages": ""
+  }
+}
+JSON
+
+res=0
+verify_registry_artifact "${WORKDIR}" "index-empty-pkg" "1.0.0" "dir" || res=$?
+assert_eq "offline: image index with empty child annotation rejected" "1" "$res"
+
+# --- Online live tests against real registries (if reachable) ---
+if skopeo inspect --raw docker://quay.io/rhdh/red-hat-developer-hub-backstage-plugin-scorecard-backend-module-dependabot:2.0.0--1.1.2 >/dev/null 2>&1; then
+    echo "Running online live registry tests..."
+
+    # Broken Quay scorecard image (empty annotation on child manifest, from RHDHBUGS-3834)
+    res=0
+    verify_registry_artifact "quay.io/rhdh" "red-hat-developer-hub-backstage-plugin-scorecard-backend-module-dependabot" "2.0.0--1.1.1" || res=$?
+    assert_eq "online: broken scorecard image on Quay rejected" "1" "$res"
+
+    # Fixed Quay scorecard image (valid annotation on child manifest)
+    res=0
+    verify_registry_artifact "quay.io/rhdh" "red-hat-developer-hub-backstage-plugin-scorecard-backend-module-dependabot" "2.0.0--1.1.2" || res=$?
+    assert_eq "online: fixed scorecard image on Quay accepted" "0" "$res"
+
+    # Valid GHCR image (single OCI manifest)
+    res=0
+    verify_registry_artifact "ghcr.io/redhat-developer/rhdh-plugin-export-overlays" "backstage-community-plugin-github-actions" "bs_1.54.9__1.4.0" || res=$?
+    assert_eq "online: valid GHCR image accepted" "0" "$res"
+
+    # Non-existent image tag
+    res=0
+    verify_registry_artifact "ghcr.io/redhat-developer/rhdh-plugin-export-overlays" "backstage-community-plugin-github-actions" "bs_1.54.9__99.99.99" || res=$?
+    assert_eq "online: non-existent tag rejected" "1" "$res"
+else
+    echo "Skipping online tests (registry unreachable or network offline)"
+fi
+
+echo "All verify-registry-artifacts tests passed."