feat(scale-set): add scale set orchestration in Terraform

.github/dependabot.yml

@@ -100,6 +100,22 @@ updates:
     allow:
       - dependency-name: "ghcr.io/ministackorg/ministack"
 
+  - package-ecosystem: "docker"
+    directory: "/lambdas/services/scale-set"
+    schedule:
+      interval: "weekly"
+    cooldown:
+      default-days: 7
+    labels:
+      - "dependencies"
+      - "docker"
+    commit-message:
+      prefix: "chore(scale-set)"
+    # Keep the service runtime aligned with the supported Node.js major.
+    ignore:
+      - dependency-name: "node"
+        update-types: ["version-update:semver-major"]
+
   - package-ecosystem: "pip"
     directory: "/.github/workflows/mkdocs"
     schedule:

.github/scripts/scale-set-container-smoke-test.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+container_name="scale-set-service-smoke-${GITHUB_RUN_ID:-$$}"
+response_file="$(mktemp)"
+# shellcheck disable=SC2329 # cleanup is invoked indirectly by the EXIT trap.
+cleanup() {
+  docker rm -f "$container_name" >/dev/null 2>&1 || true
+  rm -f "$response_file"
+}
+trap cleanup EXIT
+
+scale_set_controller_manifest='{"version":1,"groupName":"ci-smoke","revision":"image-test","reconcilers":[{"schemaVersion":1,"runnerConfigName":"smoke","scaleSetId":1,"scaleSetName":"ci-smoke","githubConfigUrl":"https://github.com/example-org","githubApp":{"appIdParameterName":"/ci/app-id","privateKeyParameterName":"/ci/private-key"},"computeProvider":{"type":"ec2","configuration":{"region":"us-east-1","environment":"ci","runnerNamePrefix":"ci","jitConfigParameterPath":"/ci/jit","subnets":["subnet-00000000"],"launchTemplateName":"ci","ec2instanceCriteria":{"instanceTypes":["t3.micro"],"targetCapacityType":"on-demand","instanceAllocationStrategy":"lowest-price"}}},"minRunners":0,"maxRunners":0}]}'
+
+docker run --detach \
+  --name "$container_name" \
+  --network none \
+  --read-only \
+  --cap-drop ALL \
+  --security-opt no-new-privileges \
+  --env AWS_REGION=us-east-1 \
+  --env AWS_EC2_METADATA_DISABLED=true \
+  --env SCALE_SET_HEALTH_PORT=8080 \
+  --env "SCALE_SET_CONTROLLER_MANIFEST=$scale_set_controller_manifest" \
+  scale-set-service:smoke-test
+
+attempt=0
+while (( attempt < 30 )); do
+  ((attempt += 1))
+  docker exec "$container_name" node --input-type=module -e \
+    'const response = await fetch("http://127.0.0.1:8080/healthz", { signal: AbortSignal.timeout(1000) }); process.stdout.write(JSON.stringify({ status: response.status, body: await response.json() }));' \
+    >"$response_file" 2>/dev/null || true
+  if jq --exit-status \
+    --arg group_name ci-smoke \
+    '(.status == 200 or .status == 503) and .body.groupName == $group_name and (.body.live | type == "boolean") and (.body.ready | type == "boolean") and .body.reconcilers.smoke != null' \
+    "$response_file" >/dev/null 2>&1; then
+    jq . "$response_file"
+    exit 0
+  fi
+  sleep 1
+done
+
+docker logs "$container_name"
+echo "scale-set service image did not return the expected health response" >&2
+exit 1

.github/workflows/lambda.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     paths:
       - 'lambdas/**'
+      - '.github/scripts/**'
       - '.github/workflows/lambda.yml'
 
 concurrency:
@@ -32,21 +33,69 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+
       - name: Install dependencies
         run: yarn install --frozen-lockfile
+
       - name: Run prettier
         run: yarn format-check
+
       - name: Run linter
         run: yarn lint
+
       - name: Run tests
         id: test
         run: yarn test
+
       - name: Build distribution
         run: yarn build
+
       - name: Upload coverage report
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: ${{ failure() }}
         with:
           name: coverage-reports
           path: ./**/coverage
           retention-days: 5
+
+  scale-set-container:
+    name: Build scale-set service container
+    runs-on: ubuntu-latest
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+
+      - name: Build scale-set service image
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: ./lambdas/services/scale-set/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: false
+          cache-from: type=gha,scope=scale-set-service
+          cache-to: type=gha,mode=max,scope=scale-set-service
+
+      - name: Build scale-set service image for smoke test
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: ./lambdas/services/scale-set/Dockerfile
+          platforms: linux/amd64
+          load: true
+          tags: scale-set-service:smoke-test
+          cache-from: type=gha,scope=scale-set-service
+
+      - name: Run scale-set service image smoke test
+        run: ./.github/scripts/scale-set-container-smoke-test.sh

.github/workflows/ministack.yml

@@ -72,6 +72,7 @@ jobs:
           - ephemeral
           - multi-runner
           - multi-runner-v2
+          - multi-runner-scale-set
           - migration-test
           - termination-watcher
     services:
@@ -195,3 +196,78 @@ jobs:
           MINISTACK_GITHUB_MOCK_PORT: "1080"
           MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }}
         run: sh tests/ministack/run-smoke.sh
+
+  integration_scaleset_smoke:
+    name: Run scale-set ECS smoke test against MiniStack and MockServer
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    services:
+      ministack:
+        image: ghcr.io/ministackorg/ministack:1.5.12@sha256:41fe1ce2e666c6cc410c6047a9db8bf1df69cd0028ebc0a6c6e5517c3a83d6e0
+        ports:
+          - 4566:4566
+        options: >-
+          --add-host=host.docker.internal:host-gateway
+          --volume /var/run/docker.sock:/var/run/docker.sock
+        env:
+          MINISTACK_ACCOUNT_ID: "000000000000"
+          MINISTACK_REGION: eu-west-1
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: lambdas/.nvmrc
+          package-manager-cache: false
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1
+        with:
+          terraform_version: latest
+          terraform_wrapper: false
+
+      - name: Install Lambda dependencies
+        working-directory: lambdas
+        run: yarn install --frozen-lockfile
+
+      - name: Build smoke-test Lambda distributions
+        working-directory: lambdas
+        run: |
+          yarn workspace @aws-github-runner/webhook dist
+          yarn workspace @aws-github-runner/control-plane dist
+
+      - name: Start MockServer
+        id: mockserver
+        uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0
+        with:
+          version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290'
+          port: '1080'
+          startup-timeout: '60'
+
+      - name: Connect MockServer to MiniStack network
+        shell: bash
+        run: |
+          set -euo pipefail
+          ministack_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 ~ /ministack/ {print $1; exit}')"
+          network="$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$ministack_container" | sed -n '1p')"
+          docker network connect --alias mockserver "$network" mockserver
+
+      - name: Mark repository as safe
+        shell: sh
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: Run scale-set ECS/MockServer smoke test
+        env:
+          MINISTACK_GITHUB_MOCK_HOST: mockserver
+          MINISTACK_GITHUB_MOCK_PORT: "1080"
+          MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }}
+        run: sh tests/ministack/run-scale-set-integration.sh        

.github/workflows/release.yml

@@ -1,4 +1,7 @@
 name: Release build
+env:
+  SCALE_SET_IMAGE: ghcr.io/${{ github.repository_owner }}/terraform-aws-github-runner-scale-set-service
+
 on:
   push:
     branches:
@@ -21,6 +24,8 @@ jobs:
       actions: write # for release-please-action to trigger other workflows
       id-token: write # for actions/attest-build-provenance to generate attestations
       attestations: write # for actions/attest-build-provenance to write attestations
+      artifact-metadata: write # for publishing linked container attestations
+      packages: write # for publishing the scale-set service image to GHCR
     environment: release
     steps:
       - name: Harden the runner (Audit all outbound calls)
@@ -32,53 +37,113 @@ jobs:
         with:
           node-version: 24
           package-manager-cache: false
+
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+
       - name: Build dist
         working-directory: lambdas
         run: yarn install --frozen-lockfile && yarn run test && yarn dist
+
       - name: Get installation token
         uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
         id: token
         with:
           app-id: ${{ vars.RELEASER_APP_ID }}
           private-key: ${{ secrets.RELEASER_APP_PRIVATE_KEY }}
+
       - name: Extract branch name
         id: branch
         shell: bash
         run: echo "name=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
+
       - name: Release
         id: release
         uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0
         with:
           target-branch: ${{ steps.branch.outputs.name }}
           release-type: terraform-module
           token: ${{ steps.token.outputs.token }}
+
+      - name: Set up QEMU
+        if: ${{ steps.release.outputs.releases_created == 'true' }}
+        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+
+      - name: Set up Docker Buildx
+        if: ${{ steps.release.outputs.releases_created == 'true' }}
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+
+      - name: Log in to the GitHub Container Registry
+        if: ${{ steps.release.outputs.releases_created == 'true' }}
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and publish scale-set service image
+        if: ${{ steps.release.outputs.releases_created == 'true' }}
+        id: scale-set-image
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: ./lambdas/services/scale-set/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ env.SCALE_SET_IMAGE }}:${{ steps.release.outputs.tag_name }}
+            ${{ env.SCALE_SET_IMAGE }}:latest
+          labels: |
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.version=${{ steps.release.outputs.tag_name }}
+          sbom: true
+          provenance: mode=max
+          cache-from: type=gha,scope=scale-set-service
+          cache-to: type=gha,mode=max,scope=scale-set-service
+
+      - name: Attest scale-set service image
+        if: ${{ steps.release.outputs.releases_created == 'true' }}
+        id: scale-set-image-attest
+        uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
+        with:
+          subject-name: ${{ env.SCALE_SET_IMAGE }}
+          subject-digest: ${{ steps.scale-set-image.outputs.digest }}
+          push-to-registry: true
+
       - name: Attest
         if: ${{ steps.release.outputs.releases_created == 'true' }}
         id: attest
         uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
         with:
           subject-path: '${{ github.workspace }}/lambdas/functions/**/*.zip'
+
       - name: Update release notes with attestation
         if: ${{ steps.release.outputs.releases_created == 'true' }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          VERSION: ${{ github.event.inputs.version }}
           TAG_NAME: ${{ steps.release.outputs.tag_name }}
           ATTESTATION_URL: ${{ steps.attest.outputs.attestation-url }}
+          CONTAINER_ATTESTATION_URL: ${{ steps.scale-set-image-attest.outputs.attestation-url }}
+          CONTAINER_IMAGE: ${{ env.SCALE_SET_IMAGE }}
+          CONTAINER_DIGEST: ${{ steps.scale-set-image.outputs.digest }}
           REPOSITORY: ${{ github.repository }}
         run: |
-          version="${VERSION}"
           tag_name="${TAG_NAME}"
           attestation_url="${ATTESTATION_URL}"
+          container_attestation_url="${CONTAINER_ATTESTATION_URL}"
+          container_image="${CONTAINER_IMAGE}"
+          container_digest="${CONTAINER_DIGEST}"
           repository="${REPOSITORY}"
-          gh release view $version --json body -q '.body' > new-release-notes.md
+          gh release view "$tag_name" --json body -q '.body' > new-release-notes.md
           echo "## Attestation" >> new-release-notes.md
           echo "Attestation url: $attestation_url" >> new-release-notes.md
           echo "Verify the artifacts by running \`gh attestation verify <name_of_artifact> --repo ${repository}\`" >> new-release-notes.md
-          gh release edit $tag_name -F new-release-notes.md -t $tag_name
+          echo "Scale-set service image: \`${container_image}@${container_digest}\`" >> new-release-notes.md
+          echo "Container attestation url: $container_attestation_url" >> new-release-notes.md
+          gh release edit "$tag_name" -F new-release-notes.md -t "$tag_name"
+
       - name: Upload release assets
         if: ${{ steps.release.outputs.releases_created == 'true' }}
         env:
@@ -89,6 +154,7 @@ jobs:
           for f in $(find . -name '*.zip'); do
             gh release upload $tag_name $f
           done
+
       - name: Attach attestation
         if: ${{ steps.release.outputs.releases_created == 'true' }}
         env: