Sync upstream runtime, release, and Helm updates

.agents/skills/detect-flaky-tests/SKILL.md

@@ -74,7 +74,7 @@ The two sandbox-class lanes are NOT a matrix — each `e2e-test` job runs the st
 `hack/run-e2e-kind.sh` command, the second with `E2E_SANDBOX_CLASS: microvm`). Split the
 each `e2e-test` job log at the "Run E2E tests (micro-VM)" step boundary: PASS/FAIL lines
 before it belong to the gVisor lane, lines after it to the microVM lane. Because the
-steps are sequential, a gVisor-lane failure means the micro-VM step never ran — record
+steps are sequential, a gVisor-lane failure means the microVM step never ran — record
 no microVM results for that run rather than counting them as failures.
 
 Parse `go test -v` output from each log:

.agents/skills/triage-issues/SKILL.md

@@ -53,7 +53,7 @@ Apply all areas that are touched. Most issues need one or two; a few need more.
 | `area/network` | Networking: atenet-router, Envoy, ext_proc, DNS, xDS, ingress (`cmd/atenet/`) |
 | `area/node` | Node agent and worker lifecycle: atelet, sandbox launch, OCI, cgroups (`cmd/atelet/`) |
 | `area/gvisor` | gVisor sandbox specifics: runsc integration, GPU in gVisor, gVisor OCI (`cmd/ateworker-gvisor/`) |
-| `area/microVM` | Micro-VM sandbox specifics: cloud-hypervisor, kata, snapshot/restore (`cmd/ateworker-microvm/`) |
+| `area/microVM` | MicroVM sandbox specifics: cloud-hypervisor, kata, snapshot/restore (`cmd/ateworker-microvm/`) |
 | `area/storage` | Snapshot storage, image cache, GCS/S3 backends, retention (`internal/imagecache/`, ActorSnapshot) |
 | `area/scheduling` | WorkerPool sizing, actor placement, HPA, resource allocation |
 | `area/observability` | Metrics, tracing, logging, OTLP, OTel SDK integration |

.github/scripts/release/attach-binaries.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Attaches every file in <dir> to the draft release for TAG in
+# GITHUB_REPOSITORY, replacing files of the same name. A release that is
+# already published is left unchanged, so a rerun never changes its files.
+#
+#   GITHUB_REPOSITORY=<owner/repo> TAG=<tag> GH_TOKEN=<token> \
+#     .github/scripts/release/attach-binaries.sh <dir>
+#
+# Used by the draft job in .github/workflows/release.yaml.
+
+set -o errexit -o nounset -o pipefail
+
+[[ $# -eq 1 ]] || {
+  echo "usage: GITHUB_REPOSITORY=<owner/repo> TAG=<tag> $0 <dir>" >&2
+  exit 2
+}
+dir="$1"
+
+# An assignment, so a failed query (API error, rate limit) stops the script
+# under errexit instead of being read as "published" and skipping the upload.
+is_draft="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq .isDraft)"
+if [[ "${is_draft}" != true ]]; then
+  echo "${TAG} is published; its binaries are left unchanged."
+  exit 0
+fi
+gh release upload "${TAG}" "${dir}"/* --repo "${GITHUB_REPOSITORY}" --clobber

.github/scripts/release/ghcr-login.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Logs docker in to ghcr.io as GITHUB_ACTOR with the token in TOKEN. ko and
+# ate-setup's registry checks read the credentials docker login stores.
+#
+#   GITHUB_ACTOR=<user> TOKEN=<token> .github/scripts/release/ghcr-login.sh
+#
+# Used by the images job in .github/workflows/release.yaml.
+
+set -o errexit -o nounset -o pipefail
+
+docker login ghcr.io --username "${GITHUB_ACTOR}" --password-stdin <<<"${TOKEN}"

.github/scripts/release/refuse-published-release.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Fails if the tag VERSION has a published (non-draft) release in
+# GITHUB_REPOSITORY. Publishing a release's images again replaces the digests
+# that operators install by tag, so it is refused once the release is
+# published, whether or not the run asked to overwrite.
+#
+#   GITHUB_REPOSITORY=<owner/repo> VERSION=<tag> GH_TOKEN=<token> \
+#     .github/scripts/release/refuse-published-release.sh
+#
+# A draft is not returned by tag, so a 404 means there is no published
+# release. Any other failure stops the script. A release the API returns is
+# treated as published unless it says it is a draft.
+#
+# Used by the images job in .github/workflows/release.yaml.
+
+set -o errexit -o nounset -o pipefail
+
+errfile="$(mktemp)"
+trap 'rm -f "${errfile}"' EXIT
+
+if draft="$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${VERSION}" --jq .draft 2>"${errfile}")"; then
+  if [[ "${draft}" != true ]]; then
+    echo "::error::${VERSION} has a published release; its images are not published again"
+    exit 1
+  fi
+elif ! grep -q 'HTTP 404' "${errfile}"; then
+  echo "::error::checking for a published ${VERSION} release: $(<"${errfile}")"
+  exit 1
+fi

.github/workflows/pr-workflow.yaml

@@ -51,10 +51,6 @@ jobs:
       uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
       with:
         go-version-file: 'go.mod'
-    # Unconditional: a released migration becoming permanent is exactly what
-    # this guards, and that happens on the push to main, not on the PR.
-    - name: Verify immutable PostgreSQL migrations
-      run: hack/verify/postgresql-migrations.sh
     # we only test on amd64, make sure everything still builds for arm64
     - name: build for arm64
       run: GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build ./...
@@ -145,6 +141,20 @@ jobs:
         if-no-files-found: error
     - name: verify
       run: hack/verify-all.sh
+  verify-migrations:
+    if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+    - name: Checkout
+      uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
+      with:
+        fetch-depth: 0
+    # A merge group has no base_ref. It exposes the base branch as refs/heads/<branch>.
+    - name: Verify immutable PostgreSQL migrations
+      env:
+        BASE_REF: ${{ github.base_ref || github.event.merge_group.base_ref }}
+      run: hack/verify/postgresql-migrations.sh "origin/${BASE_REF#refs/heads/}"
   # One kind cluster exercises BOTH runtimes. Free x86-64 ubuntu-latest runners
   # expose /dev/kvm (with a udev rule), so create-kind-cluster.sh mounts it and the
   # micro-VM (kata + cloud-hypervisor) sandbox class works alongside gVisor.

.github/workflows/release-fork.yaml

@@ -0,0 +1,159 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: release-fork
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Image tag (e.g. v1.2.3-rc1). Leave blank to auto-generate from branch+SHA.'
+        required: false
+      create_release:
+        description: 'Create a GitHub release'
+        type: boolean
+        default: false
+
+permissions:
+  contents: write
+  packages: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+
+      - name: Validate and resolve tag
+        id: tag
+        run: |
+          TAG="${{ inputs.tag }}"
+          if [[ -z "${TAG}" ]]; then
+            BRANCH="${GITHUB_REF_NAME//\//-}"
+            SHA="$(git rev-parse --short HEAD)"
+            TAG="${BRANCH}-${SHA}"
+          fi
+          if [[ "${{ inputs.create_release }}" == "true" ]]; then
+            if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9._-]+)?$ ]]; then
+              echo "::error::Tag '${TAG}' must match vMAJOR.MINOR.PATCH[-prerelease] when creating a release (e.g. v1.2.3 or v1.2.3-rc1)"
+              exit 1
+            fi
+          fi
+          echo "value=${TAG}" >> "$GITHUB_OUTPUT"
+          if [[ "${{ inputs.create_release }}" == "true" ]]; then
+            echo "tags=${TAG},latest" >> "$GITHUB_OUTPUT"
+          else
+            echo "tags=${TAG}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Setup Go
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Install ko
+        uses: ko-build/setup-ko@3aebd0597dc1e9d1a26bcfdb7cbeb19c131d3037 # v0.7
+
+      - name: Install Helm
+        uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
+
+      - name: Log in to GHCR
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up QEMU (multi-arch)
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0
+
+      - name: Build and push images
+        env:
+          # ghcr.io/<owner>/<repo> — resolves correctly in forks
+          IMAGE_REPOSITORY: ghcr.io/${{ github.repository }}
+          IMAGE_TAGS: ${{ steps.tag.outputs.tags }}
+        run: |
+          set -o errexit -o nounset -o pipefail
+
+          for component in ateapi ate-idp-server atecontroller atelet ateworker-gvisor ateworker-microvm podcertcontroller atenet objectstore-plugin credential-provider/kubernetes-secrets; do
+            KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component##*/}" \
+            ./hack/run-tool.sh ko build \
+              --tags "${IMAGE_TAGS}" \
+              --platform linux/amd64,linux/arm64 \
+              --bare \
+              "./cmd/${component}"
+          done
+
+      - name: Package and push Helm charts
+        if: inputs.create_release
+        env:
+          HELM_EXPERIMENTAL_OCI: "1"
+          CHART_REPOSITORY: oci://ghcr.io/kagent-dev/substrate/helm
+        run: |
+          set -o errexit -o nounset -o pipefail
+
+          tag="${{ steps.tag.outputs.value }}"
+          chart_version="${tag#v}"
+          package_dir="${RUNNER_TEMP}/helm-packages"
+          mkdir -p "${package_dir}"
+
+          echo "${{ secrets.GITHUB_TOKEN }}" \
+            | helm registry login ghcr.io \
+              --username "${{ github.actor }}" \
+              --password-stdin
+
+          helm package charts/substrate-crds \
+            --destination "${package_dir}" \
+            --version "${chart_version}" \
+            --app-version "${tag}"
+          helm package charts/substrate \
+            --destination "${package_dir}" \
+            --version "${chart_version}" \
+            --app-version "${tag}"
+          helm package charts/substrate-podcert \
+            --destination "${package_dir}" \
+            --version "${chart_version}" \
+            --app-version "${tag}"
+
+          helm push "${package_dir}/substrate-crds-${chart_version}.tgz" "${CHART_REPOSITORY}"
+          helm push "${package_dir}/substrate-${chart_version}.tgz" "${CHART_REPOSITORY}"
+          helm push "${package_dir}/substrate-podcert-${chart_version}.tgz" "${CHART_REPOSITORY}"
+
+      - name: Build kubectl-ate release binaries
+        if: inputs.create_release
+        env:
+          VERSION: ${{ steps.tag.outputs.value }}
+        run: |
+          set -o errexit -o nounset -o pipefail
+
+          mkdir -p dist
+          for os in linux darwin; do
+            for arch in amd64 arm64; do
+              CGO_ENABLED=0 GOOS="${os}" GOARCH="${arch}" go build \
+                -trimpath \
+                -ldflags="-s -w -X=go.ate.dev/substrate/internal/version.Version=${VERSION}" \
+                -o "dist/kubectl-ate-${os}-${arch}" \
+                ./cmd/kubectl-ate
+            done
+          done
+
+      - name: Create GitHub Release
+        if: inputs.create_release
+        uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2
+        with:
+          tag_name: ${{ steps.tag.outputs.value }}
+          generate_release_notes: true
+          files: dist/kubectl-ate-*