From a8a89d40102fc3f9658b4d395c1ed6c09d6cf2ff Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 17 Sep 2026 00:22:30 +0200 Subject: [PATCH 1/7] chore(scale-set): isolate docs and workflow changes --- .github/dependabot.yml | 16 ++++++++ .github/workflows/lambda.yml | 35 +++++++++++++++++ .github/workflows/release.yml | 74 +++++++++++++++++++++++++++++++++-- docs/security.md | 10 ++++- 4 files changed, 130 insertions(+), 5 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b70462a9e1..0280daa315 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -100,6 +100,22 @@ updates: allow: - dependency-name: "ghcr.io/ministackorg/ministack" + - package-ecosystem: "docker" + directory: "/lambdas/services/scale-set" + schedule: + interval: "weekly" + cooldown: + default-days: 7 + labels: + - "dependencies" + - "docker" + commit-message: + prefix: "chore(scale-set)" + # Keep the service runtime aligned with the supported Node.js major. + ignore: + - dependency-name: "node" + update-types: ["version-update:semver-major"] + - package-ecosystem: "pip" directory: "/.github/workflows/mkdocs" schedule: diff --git a/.github/workflows/lambda.yml b/.github/workflows/lambda.yml index 09d96892a1..bda4a93818 100644 --- a/.github/workflows/lambda.yml +++ b/.github/workflows/lambda.yml @@ -32,17 +32,23 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Install dependencies run: yarn install --frozen-lockfile + - name: Run prettier run: yarn format-check + - name: Run linter run: yarn lint + - name: Run tests id: test run: yarn test + - name: Build distribution run: yarn build + - name: Upload coverage report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ failure() }} @@ -50,3 +56,32 @@ jobs: name: coverage-reports path: ./**/coverage retention-days: 5 + + scale-set-container: + name: Build scale-set service container + runs-on: ubuntu-latest + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up QEMU + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + + - name: Build scale-set service image + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ./lambdas/services/scale-set/Dockerfile + platforms: linux/amd64,linux/arm64 + push: false + cache-from: type=gha,scope=scale-set-service + cache-to: type=gha,mode=max,scope=scale-set-service diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 68f5a38341..59d52d08d8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,7 @@ name: Release build +env: + SCALE_SET_IMAGE: ghcr.io/${{ github.repository_owner }}/terraform-aws-github-runner-scale-set-service + on: push: branches: @@ -21,6 +24,8 @@ jobs: actions: write # for release-please-action to trigger other workflows id-token: write # for actions/attest-build-provenance to generate attestations attestations: write # for actions/attest-build-provenance to write attestations + artifact-metadata: write # for publishing linked container attestations + packages: write # for publishing the scale-set service image to GHCR environment: release steps: - name: Harden the runner (Audit all outbound calls) @@ -32,22 +37,27 @@ jobs: with: node-version: 24 package-manager-cache: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Build dist working-directory: lambdas run: yarn install --frozen-lockfile && yarn run test && yarn dist + - name: Get installation token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 id: token with: app-id: ${{ vars.RELEASER_APP_ID }} private-key: ${{ secrets.RELEASER_APP_PRIVATE_KEY }} + - name: Extract branch name id: branch shell: bash run: echo "name=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT + - name: Release id: release uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0 @@ -55,30 +65,85 @@ jobs: target-branch: ${{ steps.branch.outputs.name }} release-type: terraform-module token: ${{ steps.token.outputs.token }} + + - name: Set up QEMU + if: ${{ steps.release.outputs.releases_created == 'true' }} + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + + - name: Set up Docker Buildx + if: ${{ steps.release.outputs.releases_created == 'true' }} + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + + - name: Log in to the GitHub Container Registry + if: ${{ steps.release.outputs.releases_created == 'true' }} + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and publish scale-set service image + if: ${{ steps.release.outputs.releases_created == 'true' }} + id: scale-set-image + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ./lambdas/services/scale-set/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.SCALE_SET_IMAGE }}:${{ steps.release.outputs.tag_name }} + ${{ env.SCALE_SET_IMAGE }}:latest + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.version=${{ steps.release.outputs.tag_name }} + sbom: true + provenance: mode=max + cache-from: type=gha,scope=scale-set-service + cache-to: type=gha,mode=max,scope=scale-set-service + + - name: Attest scale-set service image + if: ${{ steps.release.outputs.releases_created == 'true' }} + id: scale-set-image-attest + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-name: ${{ env.SCALE_SET_IMAGE }} + subject-digest: ${{ steps.scale-set-image.outputs.digest }} + push-to-registry: true + - name: Attest if: ${{ steps.release.outputs.releases_created == 'true' }} id: attest uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: subject-path: '${{ github.workspace }}/lambdas/functions/**/*.zip' + - name: Update release notes with attestation if: ${{ steps.release.outputs.releases_created == 'true' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ github.event.inputs.version }} TAG_NAME: ${{ steps.release.outputs.tag_name }} ATTESTATION_URL: ${{ steps.attest.outputs.attestation-url }} + CONTAINER_ATTESTATION_URL: ${{ steps.scale-set-image-attest.outputs.attestation-url }} + CONTAINER_IMAGE: ${{ env.SCALE_SET_IMAGE }} + CONTAINER_DIGEST: ${{ steps.scale-set-image.outputs.digest }} REPOSITORY: ${{ github.repository }} run: | - version="${VERSION}" tag_name="${TAG_NAME}" attestation_url="${ATTESTATION_URL}" + container_attestation_url="${CONTAINER_ATTESTATION_URL}" + container_image="${CONTAINER_IMAGE}" + container_digest="${CONTAINER_DIGEST}" repository="${REPOSITORY}" - gh release view $version --json body -q '.body' > new-release-notes.md + gh release view "$tag_name" --json body -q '.body' > new-release-notes.md echo "## Attestation" >> new-release-notes.md echo "Attestation url: $attestation_url" >> new-release-notes.md echo "Verify the artifacts by running \`gh attestation verify --repo ${repository}\`" >> new-release-notes.md - gh release edit $tag_name -F new-release-notes.md -t $tag_name + echo "Scale-set service image: \`${container_image}@${container_digest}\`" >> new-release-notes.md + echo "Container attestation url: $container_attestation_url" >> new-release-notes.md + gh release edit "$tag_name" -F new-release-notes.md -t "$tag_name" + - name: Upload release assets if: ${{ steps.release.outputs.releases_created == 'true' }} env: @@ -89,6 +154,7 @@ jobs: for f in $(find . -name '*.zip'); do gh release upload $tag_name $f done + - name: Attach attestation if: ${{ steps.release.outputs.releases_created == 'true' }} env: diff --git a/docs/security.md b/docs/security.md index a94688b234..4ef4d17b94 100644 --- a/docs/security.md +++ b/docs/security.md @@ -14,6 +14,14 @@ The examples are using standard AMI's for different operating systems. Instances ## Attestation -The module is released using GitHub actions and the lambda artifacts are attached to the release as attachment. During the release attestations are created. The attestations are created by the release pipeline. You find a link to the attestation in the GitHub release. The attestation only provides provenance information about the release. The attestations are not a security guarantee. We recommend you to verify the attestation after downloading the lambda artifacts. +The module is released using GitHub Actions and the Lambda artifacts are attached to the release. The release pipeline creates provenance attestations for those artifacts. You can find a link to the attestation in the GitHub release. The attestation only provides provenance information about the release; it is not a security guarantee. We recommend verifying the attestation after downloading the Lambda artifacts. + +Releases also publish the multi-architecture scale-set service image to the GitHub Container Registry with an SBOM, build provenance, and a registry attestation. The convenience image default follows the latest module release. Production deployments should override it with the immutable image digest printed in the release notes, then verify that image with: + +```bash +gh attestation verify \ + oci://ghcr.io/github-aws-runners/terraform-aws-github-runner-scale-set-service@sha256: \ + --repo github-aws-runners/terraform-aws-github-runner +``` --8<-- "SECURITY.md:mkdocsrunners" From a692d936ac4d0b3487757f9e09c91dd0a094065d Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 17 Sep 2026 00:24:55 +0200 Subject: [PATCH 2/7] chore(scale-set): update ministack workflow --- .github/workflows/ministack.yml | 64 +-------------------------------- 1 file changed, 1 insertion(+), 63 deletions(-) diff --git a/.github/workflows/ministack.yml b/.github/workflows/ministack.yml index 80f8453732..6258148461 100644 --- a/.github/workflows/ministack.yml +++ b/.github/workflows/ministack.yml @@ -11,7 +11,6 @@ on: - "policies/**" - "examples/**" - "modules/**" - - "lambdas/**" pull_request: paths: - ".github/workflows/ministack.yml" @@ -20,7 +19,6 @@ on: - "policies/**" - "examples/**" - "modules/**" - - "lambdas/**" workflow_dispatch: concurrency: @@ -75,7 +73,7 @@ jobs: - termination-watcher services: ministack: - image: ghcr.io/ministackorg/ministack:1.5.10@sha256:706b2b83c6be7e4f4dbb6a0dc28ffdebb500c6c80b64cf7938f45040fb2158e8 + image: ghcr.io/ministackorg/ministack:1.5.7@sha256:37361b9ef886463d5632d5a4b2d114da4b7a5c5793f52f07dbc72579f2fd9207 ports: - 4566:4566 env: @@ -133,63 +131,3 @@ jobs: IAC_BINARY: ${{ matrix.iac.binary }} IAC_LOCK_FILE: ${{ matrix.iac.lockfile }} run: tests/ministack/run-example.sh destroy "$EXAMPLE" - integration_smoke: - name: Run webhook and pool lifecycle smoke test against MiniStack - runs-on: ubuntu-latest - timeout-minutes: 30 - services: - ministack: - image: ghcr.io/ministackorg/ministack:1.5.10@sha256:706b2b83c6be7e4f4dbb6a0dc28ffdebb500c6c80b64cf7938f45040fb2158e8 - ports: - - 4566:4566 - options: --add-host=host.docker.internal:host-gateway - env: - MINISTACK_ACCOUNT_ID: "000000000000" - MINISTACK_REGION: eu-west-1 - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version-file: lambdas/.nvmrc - package-manager-cache: false - - - name: Setup Terraform - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - with: - terraform_version: latest - terraform_wrapper: false - - - name: Install Lambda dependencies - working-directory: lambdas - run: yarn install --frozen-lockfile - - - name: Build smoke-test Lambda distributions - working-directory: lambdas - run: | - yarn workspace @aws-github-runner/webhook dist - yarn workspace @aws-github-runner/control-plane dist - - - name: Start MockServer - id: mockserver - uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0 - with: - version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290' - port: '1080' - startup-timeout: '60' - - - name: Run webhook and pool lifecycle smoke test - env: - MINISTACK_GITHUB_MOCK_HOST: host.docker.internal - MINISTACK_GITHUB_MOCK_PORT: "1080" - MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} - run: sh tests/ministack/run-smoke.sh From 00a5c4a8397b22537a536f342077fb00878bc9a6 Mon Sep 17 00:00:00 2001 From: Ederson Brilhante Date: Thu, 17 Sep 2026 20:32:26 +0200 Subject: [PATCH 3/7] Update ministack.yml --- .github/workflows/ministack.yml | 64 ++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ministack.yml b/.github/workflows/ministack.yml index 4aaa1372b6..80af872e42 100644 --- a/.github/workflows/ministack.yml +++ b/.github/workflows/ministack.yml @@ -11,6 +11,7 @@ on: - "policies/**" - "examples/**" - "modules/**" + - "lambdas/**" pull_request: paths: - ".github/workflows/ministack.yml" @@ -19,6 +20,7 @@ on: - "policies/**" - "examples/**" - "modules/**" + - "lambdas/**" workflow_dispatch: concurrency: @@ -70,7 +72,6 @@ jobs: - ephemeral - multi-runner - multi-runner-v2 - - multi-runner-scale-set - migration-test - termination-watcher services: @@ -133,3 +134,64 @@ jobs: IAC_BINARY: ${{ matrix.iac.binary }} IAC_LOCK_FILE: ${{ matrix.iac.lockfile }} run: tests/ministack/run-example.sh destroy "$EXAMPLE" + + integration_smoke: + name: Run webhook and pool lifecycle smoke test against MiniStack + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + ministack: + image: ghcr.io/ministackorg/ministack:1.5.10@sha256:706b2b83c6be7e4f4dbb6a0dc28ffdebb500c6c80b64cf7938f45040fb2158e8 + ports: + - 4566:4566 + options: --add-host=host.docker.internal:host-gateway + env: + MINISTACK_ACCOUNT_ID: "000000000000" + MINISTACK_REGION: eu-west-1 + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: lambdas/.nvmrc + package-manager-cache: false + + - name: Setup Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: latest + terraform_wrapper: false + + - name: Install Lambda dependencies + working-directory: lambdas + run: yarn install --frozen-lockfile + + - name: Build smoke-test Lambda distributions + working-directory: lambdas + run: | + yarn workspace @aws-github-runner/webhook dist + yarn workspace @aws-github-runner/control-plane dist + + - name: Start MockServer + id: mockserver + uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0 + with: + version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290' + port: '1080' + startup-timeout: '60' + + - name: Run webhook and pool lifecycle smoke test + env: + MINISTACK_GITHUB_MOCK_HOST: host.docker.internal + MINISTACK_GITHUB_MOCK_PORT: "1080" + MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} + run: sh tests/ministack/run-smoke.sh From d6d90e887a79162bf351f2ea1cdb4fa5873a03c3 Mon Sep 17 00:00:00 2001 From: Ederson Brilhante Date: Fri, 18 Sep 2026 12:12:32 +0200 Subject: [PATCH 4/7] Update ministack.yml --- .github/workflows/ministack.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ministack.yml b/.github/workflows/ministack.yml index 80af872e42..aa6e2b505d 100644 --- a/.github/workflows/ministack.yml +++ b/.github/workflows/ministack.yml @@ -72,6 +72,7 @@ jobs: - ephemeral - multi-runner - multi-runner-v2 + - multi-runner-scale-set - migration-test - termination-watcher services: From 1b93610645b2c8667b7eb4ce7fc8a7e9741eb1c4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:21:06 +0000 Subject: [PATCH 5/7] docs: auto update terraform docs --- examples/multi-runner-scale-set/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/multi-runner-scale-set/README.md b/examples/multi-runner-scale-set/README.md index 167ae7f275..21b502fe7b 100644 --- a/examples/multi-runner-scale-set/README.md +++ b/examples/multi-runner-scale-set/README.md @@ -71,8 +71,8 @@ The GitHub App must be installed for the configured GitHub account. |------|-------------|------|---------|:--------:| | [ami](#input\_ami) | Optional AMI configuration keyed by runner lane. | 
map(object({
    filter = optional(map(list(string)), { state = ["available"] })
    owners = optional(list(string), ["amazon"])
    id_ssm_parameter = optional(object({
      arn = string
    }), null)
    kms_key = optional(object({
      arn = string
    }), null)
  }))
| `{}` | no | | [aws\_region](#input\_aws\_region) | AWS region to deploy to. | `string` | `"eu-west-1"` | no | -| [environment](#input\_environment) | Environment name, used as prefix. | `string` | `null` | no | -| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | 
object({
    config_url         = optional(string, null)
    ssl_verify         = optional(bool, true)
    runner_owner       = optional(string, null)
    registration_level = optional(string, "organization")
  })
| `{}` | no | +| [environment](#input\_environment) | Environment name, used as prefix. | `string` | n/a | yes | +| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | 
object({
    url                = optional(string, null)
    ssl_verify         = optional(bool, true)
    runner_owner       = optional(string, null)
    registration_level = optional(string, "organization")
  })
| `{}` | no | | [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. | 
object({
    id              = string
    key_base64      = string
    installation_id = optional(string, null)
  })
| n/a | yes | | [runner\_binaries\_enabled](#input\_runner\_binaries\_enabled) | Whether runner binary synchronization is enabled. | `bool` | `true` | no | | [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | 
object({
    name              = string
    runner_group_name = optional(string, "Default")
    min_runners       = optional(number, 0)
    container = optional(object({
      image = optional(string, null)
    }), {})
  })
| n/a | yes | From 61c058e2fb1842b0199ca18c7251e9e550d19f09 Mon Sep 17 00:00:00 2001 From: Ederson Brilhante Date: Fri, 18 Sep 2026 12:26:19 +0200 Subject: [PATCH 6/7] test(scale-set): run service container smoke test (#5375) ## Description Add the scale-set service-container smoke test and the ECS/MockServer scale-set lifecycle test on top of the scale-set Terraform wiring. - Build the scale-set service image from `lambdas/services/scale-set/Dockerfile`, push it to a MiniStack ECR repository, and allow the MiniStack account to pull it. - Run the controller through its real ECS container entrypoint with a read-only filesystem, dropped capabilities, and `no-new-privileges`. - Configure the scale-set example with the MockServer GitHub App/Actions endpoints and the required SSM-backed GitHub App values. - Verify the controller resolves the runner group and scale set, updates labels, creates a session, and reaches a converged state. - Start at `minRunners = 1`, verify MiniStack creates and registers the EC2 runner, then register a new ECS task definition with `minRunners = 0` and verify the controller removes the mocked GitHub runner and terminates the EC2 instance. - Keep the MockServer protocol fixture deterministic, including dynamic agent-name matching and runner/session cleanup. The workflow uses MiniStack 1.5.12 and a pinned MockServer image. The scale-set lifecycle integration remains in this PR; PR #5416 provides only the reusable example-fixture support. ## Test Plan - Full local MiniStack 1.5.12 lifecycle test passed: image build/ECR push, Terraform apply of 272 resources, ECS controller startup, scale-up runner registration, ECS redeployment with `minRunners = 0`, EC2 runner termination, MockServer session cleanup, and Terraform destroy. - `sh -n tests/ministack/run-scale-set-integration.sh` - `shellcheck -S warning tests/ministack/run-scale-set-integration.sh` - `python3 -m json.tool mockserver/initializerJson.json` - `terraform fmt -check examples/multi-runner-scale-set/main.tf examples/multi-runner-scale-set/variables.tf` - `git diff --check` ## Related Issues Depends on #5347. The Terraform wiring is provided by the scale-set stack beginning at #5350; the example is restored in #5405. --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Guilherme Caulada --- .../scripts/scale-set-container-smoke-test.sh | 45 ++ .github/workflows/lambda.yml | 14 + .github/workflows/ministack.yml | 75 +++ mockserver/initializerJson.json | 138 ++++ tests/ministack/multi-runner-scale-set.tfvars | 2 +- tests/ministack/run-scale-set-integration.sh | 600 ++++++++++++++++++ 6 files changed, 873 insertions(+), 1 deletion(-) create mode 100755 .github/scripts/scale-set-container-smoke-test.sh create mode 100644 mockserver/initializerJson.json create mode 100755 tests/ministack/run-scale-set-integration.sh diff --git a/.github/scripts/scale-set-container-smoke-test.sh b/.github/scripts/scale-set-container-smoke-test.sh new file mode 100755 index 0000000000..eea46eea72 --- /dev/null +++ b/.github/scripts/scale-set-container-smoke-test.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +container_name="scale-set-service-smoke-${GITHUB_RUN_ID:-$$}" +response_file="$(mktemp)" +# shellcheck disable=SC2329 # cleanup is invoked indirectly by the EXIT trap. +cleanup() { + docker rm -f "$container_name" >/dev/null 2>&1 || true + rm -f "$response_file" +} +trap cleanup EXIT + +scale_set_controller_manifest='{"version":1,"groupName":"ci-smoke","revision":"image-test","reconcilers":[{"schemaVersion":1,"runnerConfigName":"smoke","scaleSetId":1,"scaleSetName":"ci-smoke","githubConfigUrl":"https://github.com/example-org","githubApp":{"appIdParameterName":"/ci/app-id","privateKeyParameterName":"/ci/private-key"},"computeProvider":{"type":"ec2","configuration":{"region":"us-east-1","environment":"ci","runnerNamePrefix":"ci","jitConfigParameterPath":"/ci/jit","subnets":["subnet-00000000"],"launchTemplateName":"ci","ec2instanceCriteria":{"instanceTypes":["t3.micro"],"targetCapacityType":"on-demand","instanceAllocationStrategy":"lowest-price"}}},"minRunners":0,"maxRunners":0}]}' + +docker run --detach \ + --name "$container_name" \ + --network none \ + --read-only \ + --cap-drop ALL \ + --security-opt no-new-privileges \ + --env AWS_REGION=us-east-1 \ + --env AWS_EC2_METADATA_DISABLED=true \ + --env SCALE_SET_HEALTH_PORT=8080 \ + --env "SCALE_SET_CONTROLLER_MANIFEST=$scale_set_controller_manifest" \ + scale-set-service:smoke-test + +attempt=0 +while (( attempt < 30 )); do + ((attempt += 1)) + docker exec "$container_name" node --input-type=module -e \ + 'const response = await fetch("http://127.0.0.1:8080/healthz", { signal: AbortSignal.timeout(1000) }); process.stdout.write(JSON.stringify({ status: response.status, body: await response.json() }));' \ + >"$response_file" 2>/dev/null || true + if jq --exit-status \ + --arg group_name ci-smoke \ + '(.status == 200 or .status == 503) and .body.groupName == $group_name and (.body.live | type == "boolean") and (.body.ready | type == "boolean") and .body.reconcilers.smoke != null' \ + "$response_file" >/dev/null 2>&1; then + jq . "$response_file" + exit 0 + fi + sleep 1 +done + +docker logs "$container_name" +echo "scale-set service image did not return the expected health response" >&2 +exit 1 diff --git a/.github/workflows/lambda.yml b/.github/workflows/lambda.yml index bda4a93818..9de8ab0a12 100644 --- a/.github/workflows/lambda.yml +++ b/.github/workflows/lambda.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - 'lambdas/**' + - '.github/scripts/**' - '.github/workflows/lambda.yml' concurrency: @@ -85,3 +86,16 @@ jobs: push: false cache-from: type=gha,scope=scale-set-service cache-to: type=gha,mode=max,scope=scale-set-service + + - name: Build scale-set service image for smoke test + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ./lambdas/services/scale-set/Dockerfile + platforms: linux/amd64 + load: true + tags: scale-set-service:smoke-test + cache-from: type=gha,scope=scale-set-service + + - name: Run scale-set service image smoke test + run: ./.github/scripts/scale-set-container-smoke-test.sh diff --git a/.github/workflows/ministack.yml b/.github/workflows/ministack.yml index aa6e2b505d..fb8e14b4a5 100644 --- a/.github/workflows/ministack.yml +++ b/.github/workflows/ministack.yml @@ -196,3 +196,78 @@ jobs: MINISTACK_GITHUB_MOCK_PORT: "1080" MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} run: sh tests/ministack/run-smoke.sh + + integration_scaleset_smoke: + name: Run scale-set ECS smoke test against MiniStack and MockServer + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + ministack: + image: ghcr.io/ministackorg/ministack:1.5.12@sha256:41fe1ce2e666c6cc410c6047a9db8bf1df69cd0028ebc0a6c6e5517c3a83d6e0 + ports: + - 4566:4566 + options: >- + --add-host=host.docker.internal:host-gateway + --volume /var/run/docker.sock:/var/run/docker.sock + env: + MINISTACK_ACCOUNT_ID: "000000000000" + MINISTACK_REGION: eu-west-1 + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: lambdas/.nvmrc + package-manager-cache: false + + - name: Setup Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: latest + terraform_wrapper: false + + - name: Install Lambda dependencies + working-directory: lambdas + run: yarn install --frozen-lockfile + + - name: Build smoke-test Lambda distributions + working-directory: lambdas + run: | + yarn workspace @aws-github-runner/webhook dist + yarn workspace @aws-github-runner/control-plane dist + + - name: Start MockServer + id: mockserver + uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0 + with: + version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290' + port: '1080' + startup-timeout: '60' + + - name: Connect MockServer to MiniStack network + shell: bash + run: | + set -euo pipefail + ministack_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 ~ /ministack/ {print $1; exit}')" + network="$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$ministack_container" | sed -n '1p')" + docker network connect --alias mockserver "$network" mockserver + + - name: Mark repository as safe + shell: sh + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Run scale-set ECS/MockServer smoke test + env: + MINISTACK_GITHUB_MOCK_HOST: mockserver + MINISTACK_GITHUB_MOCK_PORT: "1080" + MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} + run: sh tests/ministack/run-scale-set-integration.sh diff --git a/mockserver/initializerJson.json b/mockserver/initializerJson.json new file mode 100644 index 0000000000..443ddb6201 --- /dev/null +++ b/mockserver/initializerJson.json @@ -0,0 +1,138 @@ +[ + { + "httpRequest": { + "method": "POST", + "path": "/api/v3/app/installations/456/access_tokens" + }, + "httpResponse": { + "statusCode": 201, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"token\":\"fake-installation-token\",\"expires_at\":\"2099-01-01T00:00:00Z\"}" + } + }, + { + "httpRequest": { + "method": "POST", + "path": "/api/v3/orgs/example/actions/runners/registration-token" + }, + "httpResponse": { + "statusCode": 201, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"token\":\"fake-registration-token\"}" + } + }, + { + "httpRequest": { + "method": "POST", + "path": "/api/v3/actions/runner-registration" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"url\":\"https://mockserver:1080/tenant/123\",\"token\":\"header.eyJleHAiOjQwNzA5MDg4MDB9.signature\"}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/tenant/123/_apis/runtime/runnergroups/" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"count\":1,\"value\":[{\"id\":48,\"name\":\"experimental-euw1-sl-cicd-forge-emu\"}]}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/tenant/123/_apis/runtime/runnerscalesets" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"count\":1,\"value\":[{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}]}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}" + } + }, + { + "httpRequest": { + "method": "PATCH", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"},{\"name\":\"linux\",\"type\":\"user\"},{\"name\":\"scale-set\",\"type\":\"user\"},{\"name\":\"self-hosted\",\"type\":\"user\"},{\"name\":\"x64\",\"type\":\"user\"}],\"runnerSetting\":{}}" + } + }, + { + "httpRequest": { + "method": "POST", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223/generatejitconfig" + }, + "httpResponseTemplate": { + "templateType": "MUSTACHE", + "template": "{\"statusCode\":200,\"headers\":{\"Content-Type\":[\"application/json\"]},\"body\":\"{\\\"runner\\\":{\\\"id\\\":321,\\\"name\\\":\\\"{{#jsonPath}}$.name{{/jsonPath}}{{jsonPathResult}}\\\",\\\"runnerScaleSetId\\\":223},\\\"encodedJITConfig\\\":\\\"test-only-jit-configuration\\\"}\"}" + } + }, + { + "httpRequest": { + "method": "POST", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223/sessions" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"sessionId\":\"11111111-1111-1111-1111-111111111111\",\"ownerName\":\"local.medium\",\"runnerScaleSet\":{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48},\"messageQueueUrl\":\"https://mockserver:1080/messages?sessionId=11111111-1111-1111-1111-111111111111&api-version=6.0-preview\",\"messageQueueAccessToken\":\"fake-queue-token\",\"statistics\":{\"totalAvailableJobs\":0,\"totalAcquiredJobs\":0,\"totalAssignedJobs\":0,\"totalRunningJobs\":0,\"totalRegisteredRunners\":0,\"totalBusyRunners\":0,\"totalIdleRunners\":0}}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/messages" + }, + "httpResponse": { + "statusCode": 202, + "delay": { "timeUnit": "MILLISECONDS", "value": 1000 } + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/tenant/123/_apis/distributedtask/pools/0/agents" + }, + "httpResponseTemplate": { + "templateType": "MUSTACHE", + "template": "{\"statusCode\":200,\"headers\":{\"Content-Type\":[\"application/json\"]},\"body\":\"{\\\"count\\\":1,\\\"value\\\":[{\\\"id\\\":321,\\\"name\\\":\\\"{{ request.queryStringParameters.agentName.0 }}\\\",\\\"runnerScaleSetId\\\":223}]}\"}" + } + }, + { + "httpRequest": { + "method": "DELETE", + "path": "/tenant/123/_apis/distributedtask/pools/0/agents/321" + }, + "httpResponse": { + "statusCode": 204 + } + }, + { + "httpRequest": { + "method": "DELETE", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223/sessions/11111111-1111-1111-1111-111111111111" + }, + "httpResponse": { + "statusCode": 204 + } + } +] diff --git a/tests/ministack/multi-runner-scale-set.tfvars b/tests/ministack/multi-runner-scale-set.tfvars index 6a47126ad6..7dd1a0ad39 100644 --- a/tests/ministack/multi-runner-scale-set.tfvars +++ b/tests/ministack/multi-runner-scale-set.tfvars @@ -54,4 +54,4 @@ scale_set = { container = { image = "localhost:4566/scale-set-controller:smoke" } -} \ No newline at end of file +} diff --git a/tests/ministack/run-scale-set-integration.sh b/tests/ministack/run-scale-set-integration.sh new file mode 100755 index 0000000000..b71440ad9e --- /dev/null +++ b/tests/ministack/run-scale-set-integration.sh @@ -0,0 +1,600 @@ +#!/bin/sh + +set -eu + +export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-000000000000}" +export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-test-only}" +export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-eu-west-1}" +export AWS_REGION="${AWS_REGION:-eu-west-1}" +export AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://127.0.0.1:4566}" +export AWS_EC2_METADATA_DISABLED="${AWS_EC2_METADATA_DISABLED:-true}" + +script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +source_root=$(CDPATH='' cd -- "$script_dir/../.." && pwd) +example="multi-runner-scale-set" +base_tfvars="$script_dir/$example.tfvars" +tfvars_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set.XXXXXX") +app_key_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-key.XXXXXX") +log_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-logs.XXXXXX") +controller_log_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-controller-logs.XXXXXX") +config_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-config.XXXXXX") +task_definition_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-task-definition.XXXXXX") +instance_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-instances.XXXXXX") +scale_down_config_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-scale-down-config.XXXXXX") +initializer_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-mockserver.XXXXXX") +repository_name="scale-set-controller" +image_reference="localhost:4566/${repository_name}:smoke" +mockserver_host="${MINISTACK_GITHUB_MOCK_HOST:-127.0.0.1}" +mockserver_port="${MINISTACK_GITHUB_MOCK_PORT:-1080}" +mockserver_url="${MINISTACK_GITHUB_MOCK_URL:-http://127.0.0.1:${mockserver_port}}" +controller_mock_url="https://${mockserver_host}:${mockserver_port}" +terraform_state_exists=false + +cleanup() { + cleanup_status=$? + set +e + + if [ "$terraform_state_exists" = true ]; then + "$source_root/tests/ministack/run-example.sh" destroy "$example" "$tfvars_file" >/dev/null 2>&1 + fi + + rm -f "$tfvars_file" "$app_key_file" "$log_file" "$controller_log_file" "$config_file" "$task_definition_file" "$instance_file" "$scale_down_config_file" "$initializer_file" + exit "$cleanup_status" +} +trap cleanup EXIT INT TERM + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "$1 is required to run the scale-set MiniStack smoke test." >&2 + exit 69 + fi +} + +for command in aws curl docker openssl python3 terraform; do + require_command "$command" +done + +ministack_aws() { + aws --endpoint-url "$AWS_ENDPOINT_URL" --region "$AWS_DEFAULT_REGION" "$@" +} + +wait_for_http() { + url="$1" + attempts=90 + while ! curl -fsS --max-time 2 "$url" >/dev/null 2>&1; do + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for $url." >&2 + exit 70 + fi + sleep 1 + done +} + +wait_for_scale_set_runner() { + attempts=90 + while :; do + ministack_aws ec2 describe-instances --output json > "$instance_file" + instance_id=$(python3 - "$instance_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as instance_file: + response = json.load(instance_file) + +required_tags = { + "ghr:Application": "github-action-runner", + "ghr:created_by": "scale-set-service", + "ghr:environment": "ministack-scale-set-linux-scale-set", + "ghr:Type": "Org", + "ghr:Owner": "example", + "ghr:scale_set_state": "config-published", + "ghr:github_runner_id": "321", +} +active_states = {"pending", "running", "stopping", "stopped", "shutting-down"} +matches = [] +for reservation in response.get("Reservations", []): + for instance in reservation.get("Instances", []): + if instance.get("State", {}).get("Name") not in active_states: + continue + tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])} + if all(tags.get(key) == value for key, value in required_tags.items()) and tags.get("ghr:runner_name", "").startswith("scale-set-"): + matches.append(instance["InstanceId"]) + +if len(matches) == 1: + print(matches[0]) +PY +) + if [ -n "$instance_id" ]; then + printf ' [PASS] MiniStack created and registered scale-set EC2 runner %s\n' "$instance_id" + return 0 + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for the scale-set EC2 runner to reach config-published state." >&2 + cat "$instance_file" >&2 || true + exit 1 + fi + sleep 2 + done +} + +wait_for_no_scale_set_runners() { + attempts=90 + while :; do + ministack_aws ec2 describe-instances --output json > "$instance_file" + active_count=$(python3 - "$instance_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as instance_file: + response = json.load(instance_file) + +required_tags = { + "ghr:Application": "github-action-runner", + "ghr:created_by": "scale-set-service", + "ghr:environment": "ministack-scale-set-linux-scale-set", + "ghr:Type": "Org", + "ghr:Owner": "example", +} +active_states = {"pending", "running", "stopping", "stopped", "shutting-down"} +count = 0 +for reservation in response.get("Reservations", []): + for instance in reservation.get("Instances", []): + if instance.get("State", {}).get("Name") not in active_states: + continue + tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])} + if all(tags.get(key) == value for key, value in required_tags.items()): + count += 1 +print(count) +PY +) + if [ "$active_count" = "0" ]; then + printf '%s\n' ' [PASS] scale-set EC2 runner was terminated after the minimum changed to zero' + return 0 + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for the scale-set EC2 runner to terminate; active count is $active_count." >&2 + cat "$instance_file" >&2 || true + exit 1 + fi + sleep 2 + done +} + +wait_for_http "$AWS_ENDPOINT_URL/_ministack/health" +attempts=90 +while ! curl -fsS --max-time 2 -X PUT "$mockserver_url/mockserver/status" >/dev/null 2>&1; do + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for $mockserver_url/mockserver/status." >&2 + exit 70 + fi + sleep 1 +done +curl -fsS -X PUT "$mockserver_url/mockserver/reset" >/dev/null +CONTROLLER_MOCK_URL="$controller_mock_url" python3 - "$source_root/mockserver/initializerJson.json" "$initializer_file" <<'PY' +import os +import sys + +source, destination = sys.argv[1:] +with open(source, encoding="utf-8") as source_file: + fixture = source_file.read() +fixture = fixture.replace("https://mockserver:1080", os.environ["CONTROLLER_MOCK_URL"]) +with open(destination, "w", encoding="utf-8") as destination_file: + destination_file.write(fixture) +PY +curl -fsS -X PUT \ + "$mockserver_url/mockserver/expectation" \ + -H 'Content-Type: application/json' \ + --data-binary "@$initializer_file" \ + >/dev/null + +repository_policy=$(python3 - <<'PY' +import json + +print(json.dumps({ + "Version": "2012-10-17", + "Statement": [{ + "Sid": "AllowAccountPull", + "Effect": "Allow", + "Principal": {"AWS": "arn:aws:iam::000000000000:root"}, + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + ], + }], +})) +PY +) + +ministack_aws ecr create-repository \ + --repository-name "$repository_name" \ + --image-tag-mutability IMMUTABLE \ + --image-scanning-configuration scanOnPush=false \ + >/dev/null +ministack_aws ecr set-repository-policy \ + --repository-name "$repository_name" \ + --policy-text "$repository_policy" \ + >/dev/null + +docker build \ + --target runtime \ + --file "$source_root/lambdas/services/scale-set/Dockerfile" \ + --tag "$image_reference" \ + "$source_root" + +ministack_aws ecr get-login-password | docker login \ + --username AWS \ + --password-stdin localhost:4566 >/dev/null +docker push "$image_reference" + +ministack_aws ecr describe-images \ + --repository-name "$repository_name" \ + --image-ids imageTag=smoke \ + >/dev/null + +openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$app_key_file" 2>/dev/null +app_key_base64=$(base64 < "$app_key_file" | tr -d '\n') +APP_KEY_BASE64="$app_key_base64" GITHUB_CONFIG_URL="$controller_mock_url" python3 - "$base_tfvars" "$tfvars_file" <<'PY' +import os +import sys + +source, destination = sys.argv[1:] +replacement = os.environ["APP_KEY_BASE64"] +github_config_url = os.environ["GITHUB_CONFIG_URL"] +with open(source, encoding="utf-8") as source_file: + lines = source_file.readlines() +with open(destination, "w", encoding="utf-8") as destination_file: + for line in lines: + if line.lstrip().startswith("key_base64"): + destination_file.write(f' key_base64 = "{replacement}"\n') + elif line.lstrip().startswith("config_url"): + destination_file.write(f' config_url = "{github_config_url}"\n') + else: + destination_file.write(line) +PY +unset app_key_base64 APP_KEY_BASE64 + +terraform_state_exists=true +"$source_root/tests/ministack/run-example.sh" apply "$example" "$tfvars_file" + +config_path="/ministack-scale-set/scale-set-controller/linux-scale-set/linux-scale-set" +ministack_aws ssm get-parameter \ + --name "$config_path" \ + --query 'Parameter.Value' \ + --output text > "$config_file" +EXPECTED_GITHUB_CONFIG_URL="$controller_mock_url/example" python3 - "$config_file" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as config_file: + config = json.load(config_file) + +assert config["githubConfigUrl"] == os.environ["EXPECTED_GITHUB_CONFIG_URL"] +assert config["forceGhes"] is True +assert config["sslVerify"] is False +assert config["minRunners"] == 1 +assert config["githubApp"]["appIdParameterName"] +assert config["githubApp"]["installationIdParameterName"] +assert config["githubApp"]["privateKeyParameterName"] +print(" [PASS] SSM manifest has the expected MockServer and GitHub App settings") +PY + +task_definition=$(ministack_aws ecs list-task-definitions \ + --family-prefix ministack-scale-set-ss-linux-scale-se- \ + --sort DESC \ + --query 'taskDefinitionArns[0]' \ + --output text) +if [ -z "$task_definition" ] || [ "$task_definition" = "None" ]; then + echo "The scale-set task definition was not registered." >&2 + exit 1 +fi + +actual_image=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.containerDefinitions[?name==`scale-set-controller`].image | [0]' \ + --output text) +if [ "$actual_image" != "$image_reference" ]; then + echo "Expected the ECS task to use $image_reference, got $actual_image." >&2 + exit 1 +fi +printf '%s\n' ' [PASS] ECS task definition uses the image pushed to MiniStack ECR' + +log_driver=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.containerDefinitions[?name==`scale-set-controller`].logConfiguration.logDriver | [0]' \ + --output text) +if [ "$log_driver" != "awslogs" ]; then + echo "Expected the ECS task to request the awslogs driver, got $log_driver." >&2 + exit 1 +fi +printf '%s\n' ' [PASS] ECS task definition requests the awslogs driver' + +log_group=$(ministack_aws logs describe-log-groups \ + --log-group-name-prefix "/aws/ecs/ministack-scale-set-ss-linux-scale-se-" \ + --query 'logGroups[0].logGroupName' \ + --output text) +if [ -z "$log_group" ] || [ "$log_group" = "None" ]; then + echo "The scale-set CloudWatch log group was not created." >&2 + exit 1 +fi +printf '%s\n' ' [PASS] CloudWatch log group was created for the ECS controller' + +task_family=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.family' \ + --output text) + +# MiniStack exposes ECS credentials through the gateway's container IP. The +# Node.js AWS SDK intentionally rejects that address in +# AWS_CONTAINER_CREDENTIALS_FULL_URI because non-HTTPS full URIs are limited +# to loopback and the real ECS metadata address. Use only synthetic, +# test-scoped credentials in a temporary task-definition revision so the +# smoke test still exercises the pushed image, ECS service, and controller +# lifecycle without changing the production task definition or application. +ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition' \ + --output json > "$task_definition_file" +TASK_DEFINITION_FILE="$task_definition_file" python3 - <<'PY' +import json +import os + +path = os.environ["TASK_DEFINITION_FILE"] +with open(path, encoding="utf-8") as task_definition_file: + task_definition = json.load(task_definition_file) + +for field in ( + "taskDefinitionArn", + "revision", + "status", + "requiresAttributes", + "compatibilities", + "registeredAt", + "registeredBy", +): + task_definition.pop(field, None) + +for container in task_definition["containerDefinitions"]: + if container["name"] != "scale-set-controller": + continue + environment = container.setdefault("environment", []) + environment.extend([ + {"name": "AWS_ACCESS_KEY_ID", "value": "000000000000"}, + {"name": "AWS_SECRET_ACCESS_KEY", "value": "test-only"}, + ]) + +with open(path, "w", encoding="utf-8") as task_definition_file: + json.dump(task_definition, task_definition_file) +PY + +environment_name=$(sed -n 's/^environment[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$base_tfvars") +cluster_name="${environment_name}-scale-set" +task_definition=$(ministack_aws ecs register-task-definition \ + --cli-input-json "file://$task_definition_file" \ + --query 'taskDefinition.taskDefinitionArn' \ + --output text) +ministack_aws ecs update-service \ + --cluster "$cluster_name" \ + --service "$task_family" \ + --task-definition "$task_definition" \ + >/dev/null + +task_revision=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.revision' \ + --output text) +controller_container="" +attempts=90 +while [ -z "$controller_container" ]; do + controller_container=$(docker ps -a \ + --filter "label=com.amazonaws.ecs.task-definition-family=$task_family" \ + --filter "label=com.amazonaws.ecs.task-definition-version=$task_revision" \ + --filter 'name=scale-set-controller' \ + --format '{{.ID}}' | sed -n '1p') + if [ -n "$controller_container" ]; then + break + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for the MiniStack ECS controller container." >&2 + docker ps -a --format '{{.ID}} {{.Image}} {{.Status}} {{.Names}}' >&2 || true + exit 1 + fi + sleep 2 +done +printf ' [PASS] MiniStack started ECS controller container %s\n' "$controller_container" + +wait_for_controller_log_event() { + marker="$1" + required_text="${2:-}" + attempts=90 + while :; do + docker logs "$controller_container" > "$controller_log_file" 2>&1 || true + if python3 - "$controller_log_file" "$marker" "$required_text" <<'PY' +import sys + +marker, required = sys.argv[2:] +with open(sys.argv[1], encoding="utf-8") as log_file: + messages = log_file.read().splitlines() +if any(marker in message and (not required or required in message) for message in messages): + raise SystemExit(0) +raise SystemExit(1) +PY + then + printf ' [PASS] CloudWatch logs contain %s\n' "$marker" + return 0 + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for controller log marker '$marker'." >&2 + cat "$controller_log_file" >&2 || true + echo "Controller AWS/ECS metadata environment:" >&2 + docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" \ + | sed -E 's/^(AWS_CONTAINER_AUTHORIZATION_TOKEN|AWS_CONTAINER_CREDENTIALS_FULL_URI)=.*/\1=/' \ + | grep -E '^(AWS_|ECS_)' >&2 || true + echo "Controller network attachments:" >&2 + docker inspect --format '{{json .NetworkSettings.Networks}}' "$controller_container" >&2 || true + echo "Controller credential endpoint probe:" >&2 + docker exec "$controller_container" node -e \ + 'fetch(process.env.AWS_CONTAINER_CREDENTIALS_FULL_URI, {headers: {Authorization: process.env.AWS_CONTAINER_AUTHORIZATION_TOKEN}}).then((response) => { console.error(`status=${response.status}`); process.exit(response.ok ? 0 : 1); }).catch((error) => { console.error(`${error.name}:${error.message}`); process.exit(1); })' \ + >&2 || true + exit 1 + fi + sleep 2 + done +} + +printf '%s\n' ' [INFO] MiniStack 1.5.12 does not emit ECS awslogs streams; validating controller runtime logs instead' +wait_for_controller_log_event 'scale_set_controller_started' +wait_for_controller_log_event 'scale_set_session_created' +wait_for_controller_log_event 'scale_set_reconciled' '"desiredRunners":1' +wait_for_controller_log_event 'scale_set_reconciled' '"status":"converged"' +wait_for_scale_set_runner + +wait_for_mock_route() { + method="$1" + route="$2" + body=$(REQUEST_METHOD="$method" REQUEST_PATH="$route" python3 - <<'PY' +import json +import os + +print(json.dumps({ + "httpRequest": { + "method": os.environ["REQUEST_METHOD"], + "path": os.environ["REQUEST_PATH"], + }, + "times": {"atLeast": 1}, +})) +PY + ) + attempts=45 + while ! curl -fsS --max-time 5 -X PUT \ + http://127.0.0.1:1080/mockserver/verify \ + -H 'Content-Type: application/json' \ + --data "$body" >/dev/null 2>&1; do + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for MockServer route: $method $route" >&2 + curl -sS --max-time 5 \ + 'http://127.0.0.1:1080/mockserver/retrieve?type=REQUESTS&format=JSON' >&2 || true + exit 1 + fi + sleep 2 + done + printf ' [PASS] MockServer received %s %s\n' "$method" "$route" +} + +wait_for_mock_route POST '/api/v3/app/installations/456/access_tokens' +wait_for_mock_route POST '/api/v3/orgs/example/actions/runners/registration-token' +wait_for_mock_route POST '/api/v3/actions/runner-registration' +wait_for_mock_route GET '/tenant/123/_apis/runtime/runnergroups/' +wait_for_mock_route GET '/tenant/123/_apis/runtime/runnerscalesets' +wait_for_mock_route GET '/tenant/123/_apis/runtime/runnerscalesets/223' +wait_for_mock_route PATCH '/tenant/123/_apis/runtime/runnerscalesets/223' +wait_for_mock_route POST '/tenant/123/_apis/runtime/runnerscalesets/223/generatejitconfig' +wait_for_mock_route POST '/tenant/123/_apis/runtime/runnerscalesets/223/sessions' +wait_for_mock_route GET '/messages' + +python3 - "$config_file" "$scale_down_config_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as config_file: + reconciler = json.load(config_file) + +assert reconciler["minRunners"] == 1 +reconciler["minRunners"] = 0 +with open(sys.argv[2], "w", encoding="utf-8") as config_file: + json.dump(reconciler, config_file) +PY +ministack_aws ssm put-parameter \ + --name "$config_path" \ + --type String \ + --value "$(cat "$scale_down_config_file")" \ + --overwrite \ + >/dev/null +printf '%s\n' ' [PASS] SSM manifest minimum changed from one runner to zero' + +SCALE_DOWN_CONFIG_FILE="$scale_down_config_file" TASK_DEFINITION_FILE="$task_definition_file" python3 - <<'PY' +import json +import os + +with open(os.environ["SCALE_DOWN_CONFIG_FILE"], encoding="utf-8") as config_file: + reconciler = json.load(config_file) +with open(os.environ["TASK_DEFINITION_FILE"], encoding="utf-8") as task_definition_file: + task_definition = json.load(task_definition_file) + +for container in task_definition["containerDefinitions"]: + if container["name"] != "scale-set-controller": + continue + for environment in container.get("environment", []): + if environment["name"] == "SCALE_SET_CONTROLLER_MANIFEST": + manifest = json.loads(environment["value"]) + assert len(manifest.get("reconcilers", [])) == 1 + manifest["reconcilers"][0]["minRunners"] = reconciler["minRunners"] + environment["value"] = json.dumps(manifest, separators=(",", ":")) + break + else: + raise RuntimeError("scale-set controller task definition has no inline manifest") + +with open(os.environ["TASK_DEFINITION_FILE"], "w", encoding="utf-8") as task_definition_file: + json.dump(task_definition, task_definition_file) +PY +task_definition=$(ministack_aws ecs register-task-definition \ + --cli-input-json "file://$task_definition_file" \ + --query 'taskDefinition.taskDefinitionArn' \ + --output text) +task_revision=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.revision' \ + --output text) +ministack_aws ecs update-service \ + --cluster "$cluster_name" \ + --service "$task_family" \ + --task-definition "$task_definition" \ + --force-new-deployment \ + >/dev/null + +old_controller_container="$controller_container" +new_controller_container="" +attempts=90 +while [ -z "$new_controller_container" ]; do + for candidate in $(docker ps -a \ + --filter "label=com.amazonaws.ecs.task-definition-family=$task_family" \ + --filter "label=com.amazonaws.ecs.task-definition-version=$task_revision" \ + --filter 'name=scale-set-controller' \ + --format '{{.ID}}'); do + if [ "$candidate" != "$old_controller_container" ]; then + new_controller_container="$candidate" + break + fi + done + if [ -n "$new_controller_container" ]; then + break + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo 'Timed out waiting for the ECS service to deploy the scale-down task.' >&2 + docker ps -a --format '{{.ID}} {{.Image}} {{.Status}} {{.Names}}' >&2 || true + exit 1 + fi + sleep 2 +done +controller_container="$new_controller_container" +printf ' [PASS] ECS service deployed a fresh controller container %s for scale-down\n' "$controller_container" +wait_for_controller_log_event 'scale_set_controller_started' +wait_for_controller_log_event 'scale_set_session_created' +wait_for_controller_log_event 'scale_set_reconciled' '"desiredRunners":0' +wait_for_controller_log_event 'scale_set_reconciled' '"status":"converged"' +wait_for_no_scale_set_runners + +"$source_root/tests/ministack/run-example.sh" destroy "$example" "$tfvars_file" +terraform_state_exists=false +wait_for_mock_route DELETE '/tenant/123/_apis/runtime/runnerscalesets/223/sessions/11111111-1111-1111-1111-111111111111' + +echo 'Scale-set MiniStack ECS/MockServer smoke test passed.' From dd42b2ec1d164ebc7c50f933bc5396fefb248074 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 18 Sep 2026 12:30:17 +0200 Subject: [PATCH 7/7] fix(ministack): relocate scale-set MockServer fixture --- {mockserver => tests/ministack}/initializerJson.json | 0 tests/ministack/run-scale-set-integration.sh | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename {mockserver => tests/ministack}/initializerJson.json (100%) diff --git a/mockserver/initializerJson.json b/tests/ministack/initializerJson.json similarity index 100% rename from mockserver/initializerJson.json rename to tests/ministack/initializerJson.json diff --git a/tests/ministack/run-scale-set-integration.sh b/tests/ministack/run-scale-set-integration.sh index b71440ad9e..08e0d156de 100755 --- a/tests/ministack/run-scale-set-integration.sh +++ b/tests/ministack/run-scale-set-integration.sh @@ -174,7 +174,7 @@ while ! curl -fsS --max-time 2 -X PUT "$mockserver_url/mockserver/status" >/dev/ sleep 1 done curl -fsS -X PUT "$mockserver_url/mockserver/reset" >/dev/null -CONTROLLER_MOCK_URL="$controller_mock_url" python3 - "$source_root/mockserver/initializerJson.json" "$initializer_file" <<'PY' +CONTROLLER_MOCK_URL="$controller_mock_url" python3 - "$source_root/tests/ministack/initializerJson.json" "$initializer_file" <<'PY' import os import sys