feat(scale-set): add service controller

.github/dependabot.yml

@@ -100,6 +100,22 @@ updates:
     allow:
       - dependency-name: "ghcr.io/ministackorg/ministack"
 
+  - package-ecosystem: "docker"
+    directory: "/lambdas/services/scale-set"
+    schedule:
+      interval: "weekly"
+    cooldown:
+      default-days: 7
+    labels:
+      - "dependencies"
+      - "docker"
+    commit-message:
+      prefix: "chore(scale-set)"
+    # Keep the service runtime aligned with the supported Node.js major.
+    ignore:
+      - dependency-name: "node"
+        update-types: ["version-update:semver-major"]
+
   - package-ecosystem: "pip"
     directory: "/.github/workflows/mkdocs"
     schedule:

.github/scripts/scale-set-container-smoke-test.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+container_name="scale-set-service-smoke-${GITHUB_RUN_ID:-$$}"
+response_file="$(mktemp)"
+# shellcheck disable=SC2329 # cleanup is invoked indirectly by the EXIT trap.
+cleanup() {
+  docker rm -f "$container_name" >/dev/null 2>&1 || true
+  rm -f "$response_file"
+}
+trap cleanup EXIT
+
+scale_set_controller_manifest='{"version":1,"groupName":"ci-smoke","revision":"image-test","reconcilers":[{"schemaVersion":1,"runnerConfigName":"smoke","scaleSetId":1,"scaleSetName":"ci-smoke","githubConfigUrl":"https://github.com/example-org","githubApp":{"appIdParameterName":"/ci/app-id","privateKeyParameterName":"/ci/private-key"},"computeProvider":{"type":"ec2","configuration":{"region":"us-east-1","environment":"ci","runnerNamePrefix":"ci","jitConfigParameterPath":"/ci/jit","subnets":["subnet-00000000"],"launchTemplateName":"ci","ec2instanceCriteria":{"instanceTypes":["t3.micro"],"targetCapacityType":"on-demand","instanceAllocationStrategy":"lowest-price"}}},"minRunners":0,"maxRunners":0}]}'
+
+docker run --detach \
+  --name "$container_name" \
+  --network none \
+  --read-only \
+  --cap-drop ALL \
+  --security-opt no-new-privileges \
+  --env AWS_REGION=us-east-1 \
+  --env AWS_EC2_METADATA_DISABLED=true \
+  --env SCALE_SET_HEALTH_PORT=8080 \
+  --env "SCALE_SET_CONTROLLER_MANIFEST=$scale_set_controller_manifest" \
+  scale-set-service:smoke-test
+
+attempt=0
+while (( attempt < 30 )); do
+  ((attempt += 1))
+  docker exec "$container_name" node --input-type=module -e \
+    'const response = await fetch("http://127.0.0.1:8080/healthz", { signal: AbortSignal.timeout(1000) }); process.stdout.write(JSON.stringify({ status: response.status, body: await response.json() }));' \
+    >"$response_file" 2>/dev/null || true
+  if jq --exit-status \
+    --arg group_name ci-smoke \
+    '(.status == 200 or .status == 503) and .body.groupName == $group_name and (.body.live | type == "boolean") and (.body.ready | type == "boolean") and .body.reconcilers.smoke != null' \
+    "$response_file" >/dev/null 2>&1; then
+    jq . "$response_file"
+    exit 0
+  fi
+  sleep 1
+done
+
+docker logs "$container_name"
+echo "scale-set service image did not return the expected health response" >&2
+exit 1

.github/workflows/lambda.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     paths:
       - 'lambdas/**'
+      - '.github/scripts/**'
       - '.github/workflows/lambda.yml'
 
 concurrency:
@@ -32,21 +33,69 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+
       - name: Install dependencies
         run: yarn install --frozen-lockfile
+
       - name: Run prettier
         run: yarn format-check
+
       - name: Run linter
         run: yarn lint
+
       - name: Run tests
         id: test
         run: yarn test
+
       - name: Build distribution
         run: yarn build
+
       - name: Upload coverage report
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: ${{ failure() }}
         with:
           name: coverage-reports
           path: ./**/coverage
           retention-days: 5
+
+  scale-set-container:
+    name: Build scale-set service container
+    runs-on: ubuntu-latest
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+
+      - name: Build scale-set service image
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: ./lambdas/services/scale-set/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: false
+          cache-from: type=gha,scope=scale-set-service
+          cache-to: type=gha,mode=max,scope=scale-set-service
+
+      - name: Build scale-set service image for smoke test
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: ./lambdas/services/scale-set/Dockerfile
+          platforms: linux/amd64
+          load: true
+          tags: scale-set-service:smoke-test
+          cache-from: type=gha,scope=scale-set-service
+
+      - name: Run scale-set service image smoke test
+        run: ./.github/scripts/scale-set-container-smoke-test.sh

.github/workflows/ministack.yml

@@ -196,3 +196,78 @@ jobs:
           MINISTACK_GITHUB_MOCK_PORT: "1080"
           MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }}
         run: sh tests/ministack/run-smoke.sh
+
+  integration_scaleset_smoke:
+    name: Run scale-set ECS smoke test against MiniStack and MockServer
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    services:
+      ministack:
+        image: ghcr.io/ministackorg/ministack:1.5.12@sha256:41fe1ce2e666c6cc410c6047a9db8bf1df69cd0028ebc0a6c6e5517c3a83d6e0
+        ports:
+          - 4566:4566
+        options: >-
+          --add-host=host.docker.internal:host-gateway
+          --volume /var/run/docker.sock:/var/run/docker.sock
+        env:
+          MINISTACK_ACCOUNT_ID: "000000000000"
+          MINISTACK_REGION: eu-west-1
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: lambdas/.nvmrc
+          package-manager-cache: false
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1
+        with:
+          terraform_version: latest
+          terraform_wrapper: false
+
+      - name: Install Lambda dependencies
+        working-directory: lambdas
+        run: yarn install --frozen-lockfile
+
+      - name: Build smoke-test Lambda distributions
+        working-directory: lambdas
+        run: |
+          yarn workspace @aws-github-runner/webhook dist
+          yarn workspace @aws-github-runner/control-plane dist
+
+      - name: Start MockServer
+        id: mockserver
+        uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0
+        with:
+          version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290'
+          port: '1080'
+          startup-timeout: '60'
+
+      - name: Connect MockServer to MiniStack network
+        shell: bash
+        run: |
+          set -euo pipefail
+          ministack_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 ~ /ministack/ {print $1; exit}')"
+          network="$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$ministack_container" | sed -n '1p')"
+          docker network connect --alias mockserver "$network" mockserver
+
+      - name: Mark repository as safe
+        shell: sh
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: Run scale-set ECS/MockServer smoke test
+        env:
+          MINISTACK_GITHUB_MOCK_HOST: mockserver
+          MINISTACK_GITHUB_MOCK_PORT: "1080"
+          MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }}
+        run: sh tests/ministack/run-scale-set-integration.sh        

.github/workflows/release.yml

@@ -1,4 +1,7 @@
 name: Release build
+env:
+  SCALE_SET_IMAGE: ghcr.io/${{ github.repository_owner }}/terraform-aws-github-runner-scale-set-service
+
 on:
   push:
     branches:
@@ -21,6 +24,8 @@ jobs:
       actions: write # for release-please-action to trigger other workflows
       id-token: write # for actions/attest-build-provenance to generate attestations
       attestations: write # for actions/attest-build-provenance to write attestations
+      artifact-metadata: write # for publishing linked container attestations
+      packages: write # for publishing the scale-set service image to GHCR
     environment: release
     steps:
       - name: Harden the runner (Audit all outbound calls)
@@ -32,53 +37,113 @@ jobs:
         with:
           node-version: 24
           package-manager-cache: false
+
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+
       - name: Build dist
         working-directory: lambdas
         run: yarn install --frozen-lockfile && yarn run test && yarn dist
+
       - name: Get installation token
         uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
         id: token
         with:
           app-id: ${{ vars.RELEASER_APP_ID }}
           private-key: ${{ secrets.RELEASER_APP_PRIVATE_KEY }}
+
       - name: Extract branch name
         id: branch
         shell: bash
         run: echo "name=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
+
       - name: Release
         id: release
         uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0
         with:
           target-branch: ${{ steps.branch.outputs.name }}
           release-type: terraform-module
           token: ${{ steps.token.outputs.token }}
+
+      - name: Set up QEMU
+        if: ${{ steps.release.outputs.releases_created == 'true' }}
+        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+
+      - name: Set up Docker Buildx
+        if: ${{ steps.release.outputs.releases_created == 'true' }}
+        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+
+      - name: Log in to the GitHub Container Registry
+        if: ${{ steps.release.outputs.releases_created == 'true' }}
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and publish scale-set service image
+        if: ${{ steps.release.outputs.releases_created == 'true' }}
+        id: scale-set-image
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: ./lambdas/services/scale-set/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ env.SCALE_SET_IMAGE }}:${{ steps.release.outputs.tag_name }}
+            ${{ env.SCALE_SET_IMAGE }}:latest
+          labels: |
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.version=${{ steps.release.outputs.tag_name }}
+          sbom: true
+          provenance: mode=max
+          cache-from: type=gha,scope=scale-set-service
+          cache-to: type=gha,mode=max,scope=scale-set-service
+
+      - name: Attest scale-set service image
+        if: ${{ steps.release.outputs.releases_created == 'true' }}
+        id: scale-set-image-attest
+        uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
+        with:
+          subject-name: ${{ env.SCALE_SET_IMAGE }}
+          subject-digest: ${{ steps.scale-set-image.outputs.digest }}
+          push-to-registry: true
+
       - name: Attest
         if: ${{ steps.release.outputs.releases_created == 'true' }}
         id: attest
         uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
         with:
           subject-path: '${{ github.workspace }}/lambdas/functions/**/*.zip'
+
       - name: Update release notes with attestation
         if: ${{ steps.release.outputs.releases_created == 'true' }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          VERSION: ${{ github.event.inputs.version }}
           TAG_NAME: ${{ steps.release.outputs.tag_name }}
           ATTESTATION_URL: ${{ steps.attest.outputs.attestation-url }}
+          CONTAINER_ATTESTATION_URL: ${{ steps.scale-set-image-attest.outputs.attestation-url }}
+          CONTAINER_IMAGE: ${{ env.SCALE_SET_IMAGE }}
+          CONTAINER_DIGEST: ${{ steps.scale-set-image.outputs.digest }}
           REPOSITORY: ${{ github.repository }}
         run: |
-          version="${VERSION}"
           tag_name="${TAG_NAME}"
           attestation_url="${ATTESTATION_URL}"
+          container_attestation_url="${CONTAINER_ATTESTATION_URL}"
+          container_image="${CONTAINER_IMAGE}"
+          container_digest="${CONTAINER_DIGEST}"
           repository="${REPOSITORY}"
-          gh release view $version --json body -q '.body' > new-release-notes.md
+          gh release view "$tag_name" --json body -q '.body' > new-release-notes.md
           echo "## Attestation" >> new-release-notes.md
           echo "Attestation url: $attestation_url" >> new-release-notes.md
           echo "Verify the artifacts by running \`gh attestation verify <name_of_artifact> --repo ${repository}\`" >> new-release-notes.md
-          gh release edit $tag_name -F new-release-notes.md -t $tag_name
+          echo "Scale-set service image: \`${container_image}@${container_digest}\`" >> new-release-notes.md
+          echo "Container attestation url: $container_attestation_url" >> new-release-notes.md
+          gh release edit "$tag_name" -F new-release-notes.md -t "$tag_name"
+
       - name: Upload release assets
         if: ${{ steps.release.outputs.releases_created == 'true' }}
         env:
@@ -89,6 +154,7 @@ jobs:
           for f in $(find . -name '*.zip'); do
             gh release upload $tag_name $f
           done
+
       - name: Attach attestation
         if: ${{ steps.release.outputs.releases_created == 'true' }}
         env:

docs/security.md

@@ -14,6 +14,14 @@ The examples are using standard AMI's for different operating systems. Instances
 
 ## Attestation
 
-The module is released using GitHub actions and the lambda artifacts are attached to the release as attachment. During the release attestations are created. The attestations are created by the release pipeline. You find a link to the attestation in the GitHub release. The attestation only provides provenance information about the release. The attestations are not a security guarantee. We recommend you to verify the attestation after downloading the lambda artifacts.
+The module is released using GitHub Actions and the Lambda artifacts are attached to the release. The release pipeline creates provenance attestations for those artifacts. You can find a link to the attestation in the GitHub release. The attestation only provides provenance information about the release; it is not a security guarantee. We recommend verifying the attestation after downloading the Lambda artifacts.
+
+Releases also publish the multi-architecture scale-set service image to the GitHub Container Registry with an SBOM, build provenance, and a registry attestation. The convenience image default follows the latest module release. Production deployments should override it with the immutable image digest printed in the release notes, then verify that image with:
+
+```bash
+gh attestation verify \
+  oci://ghcr.io/github-aws-runners/terraform-aws-github-runner-scale-set-service@sha256:<digest> \
+  --repo github-aws-runners/terraform-aws-github-runner
+```
 
 --8<-- "SECURITY.md:mkdocsrunners"

examples/multi-runner-scale-set/README.md

@@ -71,8 +71,8 @@ The GitHub App must be installed for the configured GitHub account.
 |------|-------------|------|---------|:--------:|
 | <a name="input_ami"></a> [ami](#input\_ami) | Optional AMI configuration keyed by runner lane. | <pre>map(object({<br/>    filter = optional(map(list(string)), { state = ["available"] })<br/>    owners = optional(list(string), ["amazon"])<br/>    id_ssm_parameter = optional(object({<br/>      arn = string<br/>    }), null)<br/>    kms_key = optional(object({<br/>      arn = string<br/>    }), null)<br/>  }))</pre> | `{}` | no |
 | <a name="input_aws_region"></a> [aws\_region](#input\_aws\_region) | AWS region to deploy to. | `string` | `"eu-west-1"` | no |
-| <a name="input_environment"></a> [environment](#input\_environment) | Environment name, used as prefix. | `string` | `null` | no |
-| <a name="input_github"></a> [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | <pre>object({<br/>    config_url         = optional(string, null)<br/>    ssl_verify         = optional(bool, true)<br/>    runner_owner       = optional(string, null)<br/>    registration_level = optional(string, "organization")<br/>  })</pre> | `{}` | no |
+| <a name="input_environment"></a> [environment](#input\_environment) | Environment name, used as prefix. | `string` | n/a | yes |
+| <a name="input_github"></a> [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | <pre>object({<br/>    url                = optional(string, null)<br/>    ssl_verify         = optional(bool, true)<br/>    runner_owner       = optional(string, null)<br/>    registration_level = optional(string, "organization")<br/>  })</pre> | `{}` | no |
 | <a name="input_github_app"></a> [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. | <pre>object({<br/>    id              = string<br/>    key_base64      = string<br/>    installation_id = optional(string, null)<br/>  })</pre> | n/a | yes |
 | <a name="input_runner_binaries_enabled"></a> [runner\_binaries\_enabled](#input\_runner\_binaries\_enabled) | Whether runner binary synchronization is enabled. | `bool` | `true` | no |
 | <a name="input_scale_set"></a> [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | <pre>object({<br/>    name              = string<br/>    runner_group_name = optional(string, "Default")<br/>    min_runners       = optional(number, 0)<br/>    container = optional(object({<br/>      image = optional(string, null)<br/>    }), {})<br/>  })</pre> | n/a | yes |