From e31f6fcadb0db197443733f55ede1c912152391f Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 15 Sep 2026 16:12:35 +0200 Subject: [PATCH 01/13] test(scale-set): run service container smoke test --- .../scripts/scale-set-container-smoke-test.sh | 45 +++++++++++++++++++ .github/workflows/lambda.yml | 14 ++++++ 2 files changed, 59 insertions(+) create mode 100755 .github/scripts/scale-set-container-smoke-test.sh diff --git a/.github/scripts/scale-set-container-smoke-test.sh b/.github/scripts/scale-set-container-smoke-test.sh new file mode 100755 index 0000000000..eea46eea72 --- /dev/null +++ b/.github/scripts/scale-set-container-smoke-test.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +container_name="scale-set-service-smoke-${GITHUB_RUN_ID:-$$}" +response_file="$(mktemp)" +# shellcheck disable=SC2329 # cleanup is invoked indirectly by the EXIT trap. +cleanup() { + docker rm -f "$container_name" >/dev/null 2>&1 || true + rm -f "$response_file" +} +trap cleanup EXIT + +scale_set_controller_manifest='{"version":1,"groupName":"ci-smoke","revision":"image-test","reconcilers":[{"schemaVersion":1,"runnerConfigName":"smoke","scaleSetId":1,"scaleSetName":"ci-smoke","githubConfigUrl":"https://github.com/example-org","githubApp":{"appIdParameterName":"/ci/app-id","privateKeyParameterName":"/ci/private-key"},"computeProvider":{"type":"ec2","configuration":{"region":"us-east-1","environment":"ci","runnerNamePrefix":"ci","jitConfigParameterPath":"/ci/jit","subnets":["subnet-00000000"],"launchTemplateName":"ci","ec2instanceCriteria":{"instanceTypes":["t3.micro"],"targetCapacityType":"on-demand","instanceAllocationStrategy":"lowest-price"}}},"minRunners":0,"maxRunners":0}]}' + +docker run --detach \ + --name "$container_name" \ + --network none \ + --read-only \ + --cap-drop ALL \ + --security-opt no-new-privileges \ + --env AWS_REGION=us-east-1 \ + --env AWS_EC2_METADATA_DISABLED=true \ + --env SCALE_SET_HEALTH_PORT=8080 \ + --env "SCALE_SET_CONTROLLER_MANIFEST=$scale_set_controller_manifest" \ + scale-set-service:smoke-test + +attempt=0 +while (( attempt < 30 )); do + ((attempt += 1)) + docker exec "$container_name" node --input-type=module -e \ + 'const response = await fetch("http://127.0.0.1:8080/healthz", { signal: AbortSignal.timeout(1000) }); process.stdout.write(JSON.stringify({ status: response.status, body: await response.json() }));' \ + >"$response_file" 2>/dev/null || true + if jq --exit-status \ + --arg group_name ci-smoke \ + '(.status == 200 or .status == 503) and .body.groupName == $group_name and (.body.live | type == "boolean") and (.body.ready | type == "boolean") and .body.reconcilers.smoke != null' \ + "$response_file" >/dev/null 2>&1; then + jq . "$response_file" + exit 0 + fi + sleep 1 +done + +docker logs "$container_name" +echo "scale-set service image did not return the expected health response" >&2 +exit 1 diff --git a/.github/workflows/lambda.yml b/.github/workflows/lambda.yml index bda4a93818..9de8ab0a12 100644 --- a/.github/workflows/lambda.yml +++ b/.github/workflows/lambda.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - 'lambdas/**' + - '.github/scripts/**' - '.github/workflows/lambda.yml' concurrency: @@ -85,3 +86,16 @@ jobs: push: false cache-from: type=gha,scope=scale-set-service cache-to: type=gha,mode=max,scope=scale-set-service + + - name: Build scale-set service image for smoke test + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ./lambdas/services/scale-set/Dockerfile + platforms: linux/amd64 + load: true + tags: scale-set-service:smoke-test + cache-from: type=gha,scope=scale-set-service + + - name: Run scale-set service image smoke test + run: ./.github/scripts/scale-set-container-smoke-test.sh From ebb489e2ba8336daf92b28a8552411b07f30223a Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 16 Sep 2026 00:21:24 +0200 Subject: [PATCH 02/13] test(ministack): run scale-set ECS smoke test --- .github/workflows/ministack.yml | 80 +++ examples/multi-runner-scale-set/README.md | 4 +- examples/multi-runner-scale-set/main.tf | 7 + examples/multi-runner-scale-set/variables.tf | 16 +- mockserver/initializerJson.json | 98 +++ tests/ministack/multi-runner-scale-set.tfvars | 22 +- tests/ministack/run-scale-set-integration.sh | 599 ++++++++++++++++++ 7 files changed, 816 insertions(+), 10 deletions(-) create mode 100644 mockserver/initializerJson.json create mode 100755 tests/ministack/run-scale-set-integration.sh diff --git a/.github/workflows/ministack.yml b/.github/workflows/ministack.yml index 6258148461..72fda289be 100644 --- a/.github/workflows/ministack.yml +++ b/.github/workflows/ministack.yml @@ -7,18 +7,22 @@ on: paths: - ".github/workflows/ministack.yml" - "tests/ministack/**" + - "mockserver/**" - "*.tf" - "policies/**" - "examples/**" - "modules/**" + - "lambdas/services/scale-set/**" pull_request: paths: - ".github/workflows/ministack.yml" - "tests/ministack/**" + - "mockserver/**" - "*.tf" - "policies/**" - "examples/**" - "modules/**" + - "lambdas/services/scale-set/**" workflow_dispatch: concurrency: @@ -70,6 +74,7 @@ jobs: - ephemeral - multi-runner - multi-runner-v2 + - multi-runner-scale-set - termination-watcher services: ministack: @@ -131,3 +136,78 @@ jobs: IAC_BINARY: ${{ matrix.iac.binary }} IAC_LOCK_FILE: ${{ matrix.iac.lockfile }} run: tests/ministack/run-example.sh destroy "$EXAMPLE" + + integration_smoke: + name: Run scale-set ECS smoke test against MiniStack and MockServer + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + ministack: + image: ghcr.io/ministackorg/ministack:1.5.12@sha256:41fe1ce2e666c6cc410c6047a9db8bf1df69cd0028ebc0a6c6e5517c3a83d6e0 + ports: + - 4566:4566 + options: >- + --add-host=host.docker.internal:host-gateway + --volume /var/run/docker.sock:/var/run/docker.sock + env: + MINISTACK_ACCOUNT_ID: "000000000000" + MINISTACK_REGION: eu-west-1 + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: lambdas/.nvmrc + package-manager-cache: false + + - name: Setup Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: latest + terraform_wrapper: false + + - name: Install Lambda dependencies + working-directory: lambdas + run: yarn install --frozen-lockfile + + - name: Build smoke-test Lambda distributions + working-directory: lambdas + run: | + yarn workspace @aws-github-runner/webhook dist + yarn workspace @aws-github-runner/control-plane dist + + - name: Start MockServer + id: mockserver + uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0 + with: + version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290' + port: '1080' + startup-timeout: '60' + + - name: Connect MockServer to MiniStack network + shell: bash + run: | + set -euo pipefail + ministack_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 ~ /ministack/ {print $1; exit}')" + network="$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$ministack_container" | sed -n '1p')" + docker network connect --alias mockserver "$network" mockserver + + - name: Mark repository as safe + shell: sh + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Run scale-set ECS/MockServer smoke test + env: + MINISTACK_GITHUB_MOCK_HOST: mockserver + MINISTACK_GITHUB_MOCK_PORT: "1080" + MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} + run: sh tests/ministack/run-scale-set-integration.sh diff --git a/examples/multi-runner-scale-set/README.md b/examples/multi-runner-scale-set/README.md index d7119a30be..20a120a4fb 100644 --- a/examples/multi-runner-scale-set/README.md +++ b/examples/multi-runner-scale-set/README.md @@ -27,6 +27,7 @@ Configure the GitHub App variables before applying: terraform init terraform apply \ -var='github_app={id="123456",key_base64="...",installation_id="123456789"}' \ + -var='github={config_url="https://github.com",runner_owner="example",registration_level="organization"}' \ -var='scale_set={name="linux-scale-set",container={image="ghcr.io/github-aws-runners/terraform-aws-github-runner-scale-set-service@sha256:"}}' ``` @@ -71,9 +72,10 @@ The GitHub App must be installed for the configured GitHub account. | [ami](#input\_ami) | Optional AMI configuration keyed by runner lane. | 
map(object({
    filter = optional(map(list(string)), { state = ["available"] })
    owners = optional(list(string), ["amazon"])
    id_ssm_parameter = optional(object({
      arn = string
    }), null)
    kms_key = optional(object({
      arn = string
    }), null)
  }))
| `{}` | no | | [aws\_region](#input\_aws\_region) | AWS region to deploy to. | `string` | `"eu-west-1"` | no | | [environment](#input\_environment) | Environment name, used as prefix. | `string` | `null` | no | +| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | 
object({
    config_url         = optional(string, null)
    ssl_verify         = optional(bool, true)
    runner_owner       = optional(string, null)
    registration_level = optional(string, "enterprise")
  })
| `{}` | no | | [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. | 
object({
    id              = string
    key_base64      = string
    installation_id = optional(string, null)
  })
| n/a | yes | | [runner\_binaries\_enabled](#input\_runner\_binaries\_enabled) | Whether runner binary synchronization is enabled. | `bool` | `true` | no | -| [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | 
object({
    name = string
    container = optional(object({
      image = optional(string, null)
    }), {})
  })
| n/a | yes | +| [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | 
object({
    name = string
    runner_group_name = optional(string, "Default")
    container = optional(object({
      image = optional(string, null)
    }), {})
  })
| n/a | yes | ## Outputs diff --git a/examples/multi-runner-scale-set/main.tf b/examples/multi-runner-scale-set/main.tf index 3bf1fd02a4..62c4a69d06 100644 --- a/examples/multi-runner-scale-set/main.tf +++ b/examples/multi-runner-scale-set/main.tf @@ -41,6 +41,12 @@ module "runners" { installation_id = var.github_app.installation_id webhook_secret = random_id.random.hex } + enterprise_server = { + url = var.github.config_url + ssl_verify = var.github.ssl_verify + } + runner_owner = var.github.runner_owner + runner_registration_level = var.github.registration_level } global_config_lambda = { @@ -150,6 +156,7 @@ module "runners" { scale_set = { name = var.scale_set.name runner = { + group_name = var.scale_set.runner_group_name min_runners = 0 max_runners = 10 boot_time_in_minutes = 10 diff --git a/examples/multi-runner-scale-set/variables.tf b/examples/multi-runner-scale-set/variables.tf index c41fea1186..b04ec96034 100644 --- a/examples/multi-runner-scale-set/variables.tf +++ b/examples/multi-runner-scale-set/variables.tf @@ -9,11 +9,25 @@ variable "github_app" { sensitive = true } +variable "github" { + description = "Optional GitHub endpoint and scale-set ownership settings." + + type = object({ + config_url = optional(string, null) + ssl_verify = optional(bool, true) + runner_owner = optional(string, null) + registration_level = optional(string, "enterprise") + }) + + default = {} +} + variable "scale_set" { description = "GitHub Actions scale-set configuration." type = object({ - name = string + name = string + runner_group_name = optional(string, "Default") container = optional(object({ image = optional(string, null) }), {}) diff --git a/mockserver/initializerJson.json b/mockserver/initializerJson.json new file mode 100644 index 0000000000..d7726538d4 --- /dev/null +++ b/mockserver/initializerJson.json @@ -0,0 +1,98 @@ +[ + { + "httpRequest": { + "method": "POST", + "path": "/api/v3/app/installations/456/access_tokens" + }, + "httpResponse": { + "statusCode": 201, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"token\":\"fake-installation-token\",\"expires_at\":\"2099-01-01T00:00:00Z\"}" + } + }, + { + "httpRequest": { + "method": "POST", + "path": "/api/v3/orgs/example/actions/runners/registration-token" + }, + "httpResponse": { + "statusCode": 201, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"token\":\"fake-registration-token\"}" + } + }, + { + "httpRequest": { + "method": "POST", + "path": "/api/v3/actions/runner-registration" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"url\":\"https://mockserver:1080/tenant/123\",\"token\":\"header.eyJleHAiOjQwNzA5MDg4MDB9.signature\"}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/tenant/123/_apis/runtime/runnergroups/" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"count\":1,\"value\":[{\"id\":48,\"name\":\"experimental-euw1-sl-cicd-forge-emu\"}]}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/tenant/123/_apis/runtime/runnerscalesets" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"count\":1,\"value\":[{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}]}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}" + } + }, + { + "httpRequest": { + "method": "POST", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223/sessions" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"sessionId\":\"11111111-1111-1111-1111-111111111111\",\"ownerName\":\"local.medium\",\"runnerScaleSet\":{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48},\"messageQueueUrl\":\"https://mockserver:1080/messages?sessionId=11111111-1111-1111-1111-111111111111&api-version=6.0-preview\",\"messageQueueAccessToken\":\"fake-queue-token\",\"statistics\":{\"totalAvailableJobs\":0,\"totalAcquiredJobs\":0,\"totalAssignedJobs\":0,\"totalRunningJobs\":0,\"totalRegisteredRunners\":0,\"totalBusyRunners\":0,\"totalIdleRunners\":0}}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/messages" + }, + "httpResponse": { + "statusCode": 202, + "delay": { "timeUnit": "MILLISECONDS", "value": 1000 } + } + }, + { + "httpRequest": { + "method": "DELETE", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223/sessions/11111111-1111-1111-1111-111111111111" + }, + "httpResponse": { + "statusCode": 204 + } + } +] diff --git a/tests/ministack/multi-runner-scale-set.tfvars b/tests/ministack/multi-runner-scale-set.tfvars index 67bc39ea7a..0bcb00a512 100644 --- a/tests/ministack/multi-runner-scale-set.tfvars +++ b/tests/ministack/multi-runner-scale-set.tfvars @@ -1,10 +1,17 @@ environment = "ministack-scale-set" aws_region = "eu-west-1" +github = { + config_url = "https://mockserver:1080" + ssl_verify = false + runner_owner = "example" + registration_level = "organization" +} + github_app = { - id = "0" + id = "123" key_base64 = "ministack-invalid-key" - installation_id = "1" + installation_id = "456" } runner_binaries_enabled = false @@ -41,10 +48,9 @@ ami = { } scale_set = { - config_url = "https://github.com/example" - name = "ministack-scale-set" - id = 1 - runner_group_id = 1 - runner_owner = "example" - runner_registration_level = "organization" + name = "medium" + runner_group_name = "experimental-euw1-sl-cicd-forge-emu" + container = { + image = "localhost:4566/scale-set-controller:smoke" + } } diff --git a/tests/ministack/run-scale-set-integration.sh b/tests/ministack/run-scale-set-integration.sh new file mode 100755 index 0000000000..d3164e4861 --- /dev/null +++ b/tests/ministack/run-scale-set-integration.sh @@ -0,0 +1,599 @@ +#!/bin/sh + +set -eu + +export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-000000000000}" +export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-test-only}" +export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-eu-west-1}" +export AWS_REGION="${AWS_REGION:-eu-west-1}" +export AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://127.0.0.1:4566}" +export AWS_EC2_METADATA_DISABLED="${AWS_EC2_METADATA_DISABLED:-true}" + +script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +source_root=$(CDPATH='' cd -- "$script_dir/../.." && pwd) +example="multi-runner-scale-set" +base_tfvars="$script_dir/$example.tfvars" +tfvars_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set.XXXXXX") +app_key_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-key.XXXXXX") +log_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-logs.XXXXXX") +controller_log_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-controller-logs.XXXXXX") +config_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-config.XXXXXX") +task_definition_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-task-definition.XXXXXX") +instance_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-instances.XXXXXX") +scale_down_config_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-scale-down-config.XXXXXX") +initializer_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-mockserver.XXXXXX") +repository_name="scale-set-controller" +image_reference="localhost:4566/${repository_name}:smoke" +mockserver_host="${MINISTACK_GITHUB_MOCK_HOST:-127.0.0.1}" +mockserver_port="${MINISTACK_GITHUB_MOCK_PORT:-1080}" +mockserver_url="${MINISTACK_GITHUB_MOCK_URL:-http://127.0.0.1:${mockserver_port}}" +controller_mock_url="https://${mockserver_host}:${mockserver_port}" +terraform_state_exists=false + +cleanup() { + cleanup_status=$? + set +e + + if [ "$terraform_state_exists" = true ]; then + "$source_root/tests/ministack/run-example.sh" destroy "$example" "$tfvars_file" >/dev/null 2>&1 + fi + + rm -f "$tfvars_file" "$app_key_file" "$log_file" "$controller_log_file" "$config_file" "$task_definition_file" "$instance_file" "$scale_down_config_file" "$initializer_file" + exit "$cleanup_status" +} +trap cleanup EXIT INT TERM + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "$1 is required to run the scale-set MiniStack smoke test." >&2 + exit 69 + fi +} + +for command in aws curl docker openssl python3 terraform; do + require_command "$command" +done + +ministack_aws() { + aws --endpoint-url "$AWS_ENDPOINT_URL" --region "$AWS_DEFAULT_REGION" "$@" +} + +wait_for_http() { + url="$1" + attempts=90 + while ! curl -fsS --max-time 2 "$url" >/dev/null 2>&1; do + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for $url." >&2 + exit 70 + fi + sleep 1 + done +} + +wait_for_scale_set_runner() { + attempts=90 + while :; do + ministack_aws ec2 describe-instances --output json > "$instance_file" + instance_id=$(python3 - "$instance_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as instance_file: + response = json.load(instance_file) + +required_tags = { + "ghr:Application": "github-action-runner", + "ghr:created_by": "scale-set-service", + "ghr:environment": "ministack-scale-set-linux-scale-set", + "ghr:Type": "Org", + "ghr:Owner": "example", + "ghr:scale_set_state": "config-published", + "ghr:github_runner_id": "321", +} +active_states = {"pending", "running", "stopping", "stopped", "shutting-down"} +matches = [] +for reservation in response.get("Reservations", []): + for instance in reservation.get("Instances", []): + if instance.get("State", {}).get("Name") not in active_states: + continue + tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])} + if all(tags.get(key) == value for key, value in required_tags.items()) and tags.get("ghr:runner_name", "").startswith("scale-set-"): + matches.append(instance["InstanceId"]) + +if len(matches) == 1: + print(matches[0]) +PY +) + if [ -n "$instance_id" ]; then + printf ' [PASS] MiniStack created and registered scale-set EC2 runner %s\n' "$instance_id" + return 0 + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for the scale-set EC2 runner to reach config-published state." >&2 + cat "$instance_file" >&2 || true + exit 1 + fi + sleep 2 + done +} + +wait_for_no_scale_set_runners() { + attempts=90 + while :; do + ministack_aws ec2 describe-instances --output json > "$instance_file" + active_count=$(python3 - "$instance_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as instance_file: + response = json.load(instance_file) + +required_tags = { + "ghr:Application": "github-action-runner", + "ghr:created_by": "scale-set-service", + "ghr:environment": "ministack-scale-set-linux-scale-set", + "ghr:Type": "Org", + "ghr:Owner": "example", +} +active_states = {"pending", "running", "stopping", "stopped", "shutting-down"} +count = 0 +for reservation in response.get("Reservations", []): + for instance in reservation.get("Instances", []): + if instance.get("State", {}).get("Name") not in active_states: + continue + tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])} + if all(tags.get(key) == value for key, value in required_tags.items()): + count += 1 +print(count) +PY +) + if [ "$active_count" = "0" ]; then + printf '%s\n' ' [PASS] scale-set EC2 runner was terminated after the minimum changed to zero' + return 0 + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for the scale-set EC2 runner to terminate; active count is $active_count." >&2 + cat "$instance_file" >&2 || true + exit 1 + fi + sleep 2 + done +} + +wait_for_http "$AWS_ENDPOINT_URL/_ministack/health" +attempts=90 +while ! curl -fsS --max-time 2 -X PUT "$mockserver_url/mockserver/status" >/dev/null 2>&1; do + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for $mockserver_url/mockserver/status." >&2 + exit 70 + fi + sleep 1 +done +curl -fsS -X PUT "$mockserver_url/mockserver/reset" >/dev/null +CONTROLLER_MOCK_URL="$controller_mock_url" python3 - "$source_root/mockserver/initializerJson.json" "$initializer_file" <<'PY' +import os +import sys + +source, destination = sys.argv[1:] +with open(source, encoding="utf-8") as source_file: + fixture = source_file.read() +fixture = fixture.replace("https://mockserver:1080", os.environ["CONTROLLER_MOCK_URL"]) +with open(destination, "w", encoding="utf-8") as destination_file: + destination_file.write(fixture) +PY +curl -fsS -X PUT \ + "$mockserver_url/mockserver/expectation" \ + -H 'Content-Type: application/json' \ + --data-binary "@$initializer_file" \ + >/dev/null + +repository_policy=$(python3 - <<'PY' +import json + +print(json.dumps({ + "Version": "2012-10-17", + "Statement": [{ + "Sid": "AllowAccountPull", + "Effect": "Allow", + "Principal": {"AWS": "arn:aws:iam::000000000000:root"}, + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + ], + }], +})) +PY +) + +ministack_aws ecr create-repository \ + --repository-name "$repository_name" \ + --image-tag-mutability IMMUTABLE \ + --image-scanning-configuration scanOnPush=false \ + >/dev/null +ministack_aws ecr set-repository-policy \ + --repository-name "$repository_name" \ + --policy-text "$repository_policy" \ + >/dev/null + +docker build \ + --target runtime \ + --file "$source_root/lambdas/services/scale-set/Dockerfile" \ + --tag "$image_reference" \ + "$source_root" + +ministack_aws ecr get-login-password | docker login \ + --username AWS \ + --password-stdin localhost:4566 >/dev/null +docker push "$image_reference" + +ministack_aws ecr describe-images \ + --repository-name "$repository_name" \ + --image-ids imageTag=smoke \ + >/dev/null + +openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$app_key_file" 2>/dev/null +app_key_base64=$(base64 < "$app_key_file" | tr -d '\n') +APP_KEY_BASE64="$app_key_base64" GITHUB_CONFIG_URL="$controller_mock_url" python3 - "$base_tfvars" "$tfvars_file" <<'PY' +import os +import sys + +source, destination = sys.argv[1:] +replacement = os.environ["APP_KEY_BASE64"] +github_config_url = os.environ["GITHUB_CONFIG_URL"] +with open(source, encoding="utf-8") as source_file: + lines = source_file.readlines() +with open(destination, "w", encoding="utf-8") as destination_file: + for line in lines: + if line.lstrip().startswith("key_base64"): + destination_file.write(f' key_base64 = "{replacement}"\n') + elif line.lstrip().startswith("config_url"): + destination_file.write(f' config_url = "{github_config_url}"\n') + else: + destination_file.write(line) +PY +unset app_key_base64 APP_KEY_BASE64 + +terraform_state_exists=true +"$source_root/tests/ministack/run-example.sh" apply "$example" "$tfvars_file" + +config_path="/ministack-scale-set/scale-set-controller/linux-scale-set/linux-scale-set" +ministack_aws ssm get-parameter \ + --name "$config_path" \ + --query 'Parameter.Value' \ + --output text > "$config_file" +EXPECTED_GITHUB_CONFIG_URL="$controller_mock_url/example" python3 - "$config_file" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as config_file: + config = json.load(config_file) + +assert config["githubConfigUrl"] == os.environ["EXPECTED_GITHUB_CONFIG_URL"] +assert config["forceGhes"] is True +assert config["sslVerify"] is False +assert config["minRunners"] == 1 +assert config["githubApp"]["appIdParameterName"] +assert config["githubApp"]["installationIdParameterName"] +assert config["githubApp"]["privateKeyParameterName"] +print(" [PASS] SSM manifest has the expected MockServer and GitHub App settings") +PY + +task_definition=$(ministack_aws ecs list-task-definitions \ + --family-prefix ministack-scale-set-ss-linux-scale-set- \ + --sort DESC \ + --query 'taskDefinitionArns[0]' \ + --output text) +if [ -z "$task_definition" ] || [ "$task_definition" = "None" ]; then + echo "The scale-set task definition was not registered." >&2 + exit 1 +fi + +actual_image=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.containerDefinitions[?name==`scale-set-controller`].image | [0]' \ + --output text) +if [ "$actual_image" != "$image_reference" ]; then + echo "Expected the ECS task to use $image_reference, got $actual_image." >&2 + exit 1 +fi +printf '%s\n' ' [PASS] ECS task definition uses the image pushed to MiniStack ECR' + +log_driver=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.containerDefinitions[?name==`scale-set-controller`].logConfiguration.logDriver | [0]' \ + --output text) +if [ "$log_driver" != "awslogs" ]; then + echo "Expected the ECS task to request the awslogs driver, got $log_driver." >&2 + exit 1 +fi +printf '%s\n' ' [PASS] ECS task definition requests the awslogs driver' + +log_group=$(ministack_aws logs describe-log-groups \ + --log-group-name-prefix "/aws/ecs/ministack-scale-set-ss-linux-scale-set-" \ + --query 'logGroups[0].logGroupName' \ + --output text) +if [ -z "$log_group" ] || [ "$log_group" = "None" ]; then + echo "The scale-set CloudWatch log group was not created." >&2 + exit 1 +fi +printf '%s\n' ' [PASS] CloudWatch log group was created for the ECS controller' + +task_family=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.family' \ + --output text) + +# MiniStack exposes ECS credentials through the gateway's container IP. The +# Node.js AWS SDK intentionally rejects that address in +# AWS_CONTAINER_CREDENTIALS_FULL_URI because non-HTTPS full URIs are limited +# to loopback and the real ECS metadata address. Use only synthetic, +# test-scoped credentials in a temporary task-definition revision so the +# smoke test still exercises the pushed image, ECS service, and controller +# lifecycle without changing the production task definition or application. +ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition' \ + --output json > "$task_definition_file" +TASK_DEFINITION_FILE="$task_definition_file" python3 - <<'PY' +import json +import os + +path = os.environ["TASK_DEFINITION_FILE"] +with open(path, encoding="utf-8") as task_definition_file: + task_definition = json.load(task_definition_file) + +for field in ( + "taskDefinitionArn", + "revision", + "status", + "requiresAttributes", + "compatibilities", + "registeredAt", + "registeredBy", +): + task_definition.pop(field, None) + +for container in task_definition["containerDefinitions"]: + if container["name"] != "scale-set-controller": + continue + environment = container.setdefault("environment", []) + environment.extend([ + {"name": "AWS_ACCESS_KEY_ID", "value": "000000000000"}, + {"name": "AWS_SECRET_ACCESS_KEY", "value": "test-only"}, + ]) + +with open(path, "w", encoding="utf-8") as task_definition_file: + json.dump(task_definition, task_definition_file) +PY + +environment_name=$(sed -n 's/^environment[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$base_tfvars") +cluster_name="${environment_name}-scale-set" +task_definition=$(ministack_aws ecs register-task-definition \ + --cli-input-json "file://$task_definition_file" \ + --query 'taskDefinition.taskDefinitionArn' \ + --output text) +ministack_aws ecs update-service \ + --cluster "$cluster_name" \ + --service "$task_family" \ + --task-definition "$task_definition" \ + >/dev/null + +task_revision=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.revision' \ + --output text) +controller_container="" +attempts=90 +while [ -z "$controller_container" ]; do + controller_container=$(docker ps -a \ + --filter "label=com.amazonaws.ecs.task-definition-family=$task_family" \ + --filter "label=com.amazonaws.ecs.task-definition-version=$task_revision" \ + --filter 'name=scale-set-controller' \ + --format '{{.ID}}' | sed -n '1p') + if [ -n "$controller_container" ]; then + break + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for the MiniStack ECS controller container." >&2 + docker ps -a --format '{{.ID}} {{.Image}} {{.Status}} {{.Names}}' >&2 || true + exit 1 + fi + sleep 2 +done +printf ' [PASS] MiniStack started ECS controller container %s\n' "$controller_container" + +wait_for_controller_log_event() { + marker="$1" + required_text="${2:-}" + attempts=90 + while :; do + docker logs "$controller_container" > "$controller_log_file" 2>&1 || true + if python3 - "$controller_log_file" "$marker" "$required_text" <<'PY' +import sys + +marker, required = sys.argv[2:] +with open(sys.argv[1], encoding="utf-8") as log_file: + messages = log_file.read().splitlines() +if any(marker in message and (not required or required in message) for message in messages): + raise SystemExit(0) +raise SystemExit(1) +PY + then + printf ' [PASS] CloudWatch logs contain %s\n' "$marker" + return 0 + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for controller log marker '$marker'." >&2 + cat "$controller_log_file" >&2 || true + echo "Controller AWS/ECS metadata environment:" >&2 + docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" \ + | sed -E 's/^(AWS_CONTAINER_AUTHORIZATION_TOKEN|AWS_CONTAINER_CREDENTIALS_FULL_URI)=.*/\1=/' \ + | grep -E '^(AWS_|ECS_)' >&2 || true + echo "Controller network attachments:" >&2 + docker inspect --format '{{json .NetworkSettings.Networks}}' "$controller_container" >&2 || true + echo "Controller credential endpoint probe:" >&2 + docker exec "$controller_container" node -e \ + 'fetch(process.env.AWS_CONTAINER_CREDENTIALS_FULL_URI, {headers: {Authorization: process.env.AWS_CONTAINER_AUTHORIZATION_TOKEN}}).then((response) => { console.error(`status=${response.status}`); process.exit(response.ok ? 0 : 1); }).catch((error) => { console.error(`${error.name}:${error.message}`); process.exit(1); })' \ + >&2 || true + exit 1 + fi + sleep 2 + done +} + +printf '%s\n' ' [INFO] MiniStack 1.5.12 does not emit ECS awslogs streams; validating controller runtime logs instead' +wait_for_controller_log_event 'scale_set_controller_started' +wait_for_controller_log_event 'scale_set_session_created' +wait_for_controller_log_event 'scale_set_reconciled' '"desiredRunners":1' +wait_for_controller_log_event 'scale_set_reconciled' '"status":"converged"' +wait_for_scale_set_runner + +wait_for_mock_route() { + method="$1" + route="$2" + body=$(REQUEST_METHOD="$method" REQUEST_PATH="$route" python3 - <<'PY' +import json +import os + +print(json.dumps({ + "httpRequest": { + "method": os.environ["REQUEST_METHOD"], + "path": os.environ["REQUEST_PATH"], + }, + "times": {"atLeast": 1}, +})) +PY + ) + attempts=45 + while ! curl -fsS --max-time 5 -X PUT \ + http://127.0.0.1:1080/mockserver/verify \ + -H 'Content-Type: application/json' \ + --data "$body" >/dev/null 2>&1; do + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for MockServer route: $method $route" >&2 + curl -sS --max-time 5 \ + 'http://127.0.0.1:1080/mockserver/retrieve?type=REQUESTS&format=JSON' >&2 || true + exit 1 + fi + sleep 2 + done + printf ' [PASS] MockServer received %s %s\n' "$method" "$route" +} + +wait_for_mock_route POST '/api/v3/app/installations/456/access_tokens' +wait_for_mock_route POST '/api/v3/orgs/example/actions/runners/registration-token' +wait_for_mock_route POST '/api/v3/actions/runner-registration' +wait_for_mock_route GET '/tenant/123/_apis/runtime/runnergroups/' +wait_for_mock_route GET '/tenant/123/_apis/runtime/runnerscalesets' +wait_for_mock_route GET '/tenant/123/_apis/runtime/runnerscalesets/223' +wait_for_mock_route PATCH '/tenant/123/_apis/runtime/runnerscalesets/223' +wait_for_mock_route POST '/tenant/123/_apis/runtime/runnerscalesets/223/sessions' +wait_for_mock_route GET '/messages' + +python3 - "$config_file" "$scale_down_config_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as config_file: + reconciler = json.load(config_file) + +assert reconciler["minRunners"] == 1 +reconciler["minRunners"] = 0 +with open(sys.argv[2], "w", encoding="utf-8") as config_file: + json.dump(reconciler, config_file) +PY +ministack_aws ssm put-parameter \ + --name "$config_path" \ + --type String \ + --value "$(cat "$scale_down_config_file")" \ + --overwrite \ + >/dev/null +printf '%s\n' ' [PASS] SSM manifest minimum changed from one runner to zero' + +SCALE_DOWN_CONFIG_FILE="$scale_down_config_file" TASK_DEFINITION_FILE="$task_definition_file" python3 - <<'PY' +import json +import os + +with open(os.environ["SCALE_DOWN_CONFIG_FILE"], encoding="utf-8") as config_file: + reconciler = json.load(config_file) +with open(os.environ["TASK_DEFINITION_FILE"], encoding="utf-8") as task_definition_file: + task_definition = json.load(task_definition_file) + +for container in task_definition["containerDefinitions"]: + if container["name"] != "scale-set-controller": + continue + for environment in container.get("environment", []): + if environment["name"] == "SCALE_SET_CONTROLLER_MANIFEST": + manifest = json.loads(environment["value"]) + assert len(manifest.get("reconcilers", [])) == 1 + manifest["reconcilers"][0]["minRunners"] = reconciler["minRunners"] + environment["value"] = json.dumps(manifest, separators=(",", ":")) + break + else: + raise RuntimeError("scale-set controller task definition has no inline manifest") + +with open(os.environ["TASK_DEFINITION_FILE"], "w", encoding="utf-8") as task_definition_file: + json.dump(task_definition, task_definition_file) +PY +task_definition=$(ministack_aws ecs register-task-definition \ + --cli-input-json "file://$task_definition_file" \ + --query 'taskDefinition.taskDefinitionArn' \ + --output text) +task_revision=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.revision' \ + --output text) +ministack_aws ecs update-service \ + --cluster "$cluster_name" \ + --service "$task_family" \ + --task-definition "$task_definition" \ + --force-new-deployment \ + >/dev/null + +old_controller_container="$controller_container" +new_controller_container="" +attempts=90 +while [ -z "$new_controller_container" ]; do + for candidate in $(docker ps -a \ + --filter "label=com.amazonaws.ecs.task-definition-family=$task_family" \ + --filter "label=com.amazonaws.ecs.task-definition-version=$task_revision" \ + --filter 'name=scale-set-controller' \ + --format '{{.ID}}'); do + if [ "$candidate" != "$old_controller_container" ]; then + new_controller_container="$candidate" + break + fi + done + if [ -n "$new_controller_container" ]; then + break + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo 'Timed out waiting for the ECS service to deploy the scale-down task.' >&2 + docker ps -a --format '{{.ID}} {{.Image}} {{.Status}} {{.Names}}' >&2 || true + exit 1 + fi + sleep 2 +done +controller_container="$new_controller_container" +printf ' [PASS] ECS service deployed a fresh controller container %s for scale-down\n' "$controller_container" +wait_for_controller_log_event 'scale_set_controller_started' +wait_for_controller_log_event 'scale_set_session_created' +wait_for_controller_log_event 'scale_set_reconciled' '"desiredRunners":0' +wait_for_controller_log_event 'scale_set_reconciled' '"status":"converged"' +wait_for_no_scale_set_runners + +"$source_root/tests/ministack/run-example.sh" destroy "$example" "$tfvars_file" +terraform_state_exists=false +wait_for_mock_route DELETE '/tenant/123/_apis/runtime/runnerscalesets/223/sessions/11111111-1111-1111-1111-111111111111' + +echo 'Scale-set MiniStack ECS/MockServer smoke test passed.' From 78d334aed78aec40234bb4adb3c2ce78fd826ed3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:22:53 +0000 Subject: [PATCH 03/13] docs: auto update terraform docs --- examples/multi-runner-scale-set/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/multi-runner-scale-set/README.md b/examples/multi-runner-scale-set/README.md index 20a120a4fb..c2875030af 100644 --- a/examples/multi-runner-scale-set/README.md +++ b/examples/multi-runner-scale-set/README.md @@ -75,7 +75,7 @@ The GitHub App must be installed for the configured GitHub account. | [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | 
object({
    config_url         = optional(string, null)
    ssl_verify         = optional(bool, true)
    runner_owner       = optional(string, null)
    registration_level = optional(string, "enterprise")
  })
| `{}` | no | | [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. | 
object({
    id              = string
    key_base64      = string
    installation_id = optional(string, null)
  })
| n/a | yes | | [runner\_binaries\_enabled](#input\_runner\_binaries\_enabled) | Whether runner binary synchronization is enabled. | `bool` | `true` | no | -| [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | 
object({
    name = string
    runner_group_name = optional(string, "Default")
    container = optional(object({
      image = optional(string, null)
    }), {})
  })
| n/a | yes | +| [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | 
object({
    name              = string
    runner_group_name = optional(string, "Default")
    container = optional(object({
      image = optional(string, null)
    }), {})
  })
| n/a | yes | ## Outputs From 214dc770669004edfe3df014eee50840d6332b90 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 16 Sep 2026 12:37:31 +0200 Subject: [PATCH 04/13] fix(ministack): mock scale-set label update --- mockserver/initializerJson.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mockserver/initializerJson.json b/mockserver/initializerJson.json index d7726538d4..709f1c3f56 100644 --- a/mockserver/initializerJson.json +++ b/mockserver/initializerJson.json @@ -65,6 +65,17 @@ "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}" } }, + { + "httpRequest": { + "method": "PATCH", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"},{\"name\":\"linux\",\"type\":\"user\"},{\"name\":\"scale-set\",\"type\":\"user\"},{\"name\":\"self-hosted\",\"type\":\"user\"},{\"name\":\"x64\",\"type\":\"user\"}],\"runnerSetting\":{}}" + } + }, { "httpRequest": { "method": "POST", From 4bdb153eb26e8b5c20803859ec26b44febf37b91 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 16 Sep 2026 14:51:57 +0200 Subject: [PATCH 05/13] test(ministack): cover scale-set runner lifecycle --- examples/multi-runner-scale-set/README.md | 2 +- examples/multi-runner-scale-set/main.tf | 4 ++-- examples/multi-runner-scale-set/variables.tf | 1 + mockserver/initializerJson.json | 19 +++++++++++++++++++ tests/ministack/multi-runner-scale-set.tfvars | 1 + 5 files changed, 24 insertions(+), 3 deletions(-) diff --git a/examples/multi-runner-scale-set/README.md b/examples/multi-runner-scale-set/README.md index c2875030af..7e843d704b 100644 --- a/examples/multi-runner-scale-set/README.md +++ b/examples/multi-runner-scale-set/README.md @@ -75,7 +75,7 @@ The GitHub App must be installed for the configured GitHub account. | [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | 
object({
    config_url         = optional(string, null)
    ssl_verify         = optional(bool, true)
    runner_owner       = optional(string, null)
    registration_level = optional(string, "enterprise")
  })
| `{}` | no | | [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. | 
object({
    id              = string
    key_base64      = string
    installation_id = optional(string, null)
  })
| n/a | yes | | [runner\_binaries\_enabled](#input\_runner\_binaries\_enabled) | Whether runner binary synchronization is enabled. | `bool` | `true` | no | -| [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | 
object({
    name              = string
    runner_group_name = optional(string, "Default")
    container = optional(object({
      image = optional(string, null)
    }), {})
  })
| n/a | yes | +| [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | 
object({
    name              = string
    runner_group_name = optional(string, "Default")
    min_runners       = optional(number, 0)
    container = optional(object({
      image = optional(string, null)
    }), {})
  })
| n/a | yes | ## Outputs diff --git a/examples/multi-runner-scale-set/main.tf b/examples/multi-runner-scale-set/main.tf index 62c4a69d06..de7c6d897e 100644 --- a/examples/multi-runner-scale-set/main.tf +++ b/examples/multi-runner-scale-set/main.tf @@ -151,13 +151,13 @@ module "runners" { runner = { name_prefix = "scale-set-" extra_labels = ["scale-set"] + group_name = var.scale_set.runner_group_name } orchestration_provider = { scale_set = { name = var.scale_set.name runner = { - group_name = var.scale_set.runner_group_name - min_runners = 0 + min_runners = var.scale_set.min_runners max_runners = 10 boot_time_in_minutes = 10 } diff --git a/examples/multi-runner-scale-set/variables.tf b/examples/multi-runner-scale-set/variables.tf index b04ec96034..d053869737 100644 --- a/examples/multi-runner-scale-set/variables.tf +++ b/examples/multi-runner-scale-set/variables.tf @@ -28,6 +28,7 @@ variable "scale_set" { type = object({ name = string runner_group_name = optional(string, "Default") + min_runners = optional(number, 0) container = optional(object({ image = optional(string, null) }), {}) diff --git a/mockserver/initializerJson.json b/mockserver/initializerJson.json index 709f1c3f56..829b90e08e 100644 --- a/mockserver/initializerJson.json +++ b/mockserver/initializerJson.json @@ -97,6 +97,25 @@ "delay": { "timeUnit": "MILLISECONDS", "value": 1000 } } }, + { + "httpRequest": { + "method": "GET", + "path": "/tenant/123/_apis/distributedtask/pools/0/agents" + }, + "httpResponseTemplate": { + "templateType": "MUSTACHE", + "template": "{\"statusCode\":200,\"headers\":{\"Content-Type\":[\"application/json\"]},\"body\":\"{\\\"count\\\":1,\\\"value\\\":[{\\\"id\\\":321,\\\"name\\\":\\\"{{ request.queryStringParameters.agentName.0 }}\\\",\\\"runnerScaleSetId\\\":223}]}\"}" + } + }, + { + "httpRequest": { + "method": "DELETE", + "path": "/tenant/123/_apis/distributedtask/pools/0/agents/321" + }, + "httpResponse": { + "statusCode": 204 + } + }, { "httpRequest": { "method": "DELETE", diff --git a/tests/ministack/multi-runner-scale-set.tfvars b/tests/ministack/multi-runner-scale-set.tfvars index 0bcb00a512..2fc34f71c5 100644 --- a/tests/ministack/multi-runner-scale-set.tfvars +++ b/tests/ministack/multi-runner-scale-set.tfvars @@ -50,6 +50,7 @@ ami = { scale_set = { name = "medium" runner_group_name = "experimental-euw1-sl-cicd-forge-emu" + min_runners = 1 container = { image = "localhost:4566/scale-set-controller:smoke" } From eeda517d65d257cc0ceea2de0ca3dec58aac8677 Mon Sep 17 00:00:00 2001 From: Ederson Brilhante Date: Wed, 16 Sep 2026 15:58:58 +0200 Subject: [PATCH 06/13] Update examples/multi-runner-scale-set/README.md Co-authored-by: Guilherme Caulada --- examples/multi-runner-scale-set/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/multi-runner-scale-set/README.md b/examples/multi-runner-scale-set/README.md index 7e843d704b..f8f8e64b54 100644 --- a/examples/multi-runner-scale-set/README.md +++ b/examples/multi-runner-scale-set/README.md @@ -27,7 +27,7 @@ Configure the GitHub App variables before applying: terraform init terraform apply \ -var='github_app={id="123456",key_base64="...",installation_id="123456789"}' \ - -var='github={config_url="https://github.com",runner_owner="example",registration_level="organization"}' \ + -var='github={runner_owner="example",registration_level="organization"}' \ -var='scale_set={name="linux-scale-set",container={image="ghcr.io/github-aws-runners/terraform-aws-github-runner-scale-set-service@sha256:"}}' ``` From 4e96513eb70198e6e454837542f82348a95d9450 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 16 Sep 2026 20:23:43 +0200 Subject: [PATCH 07/13] test(ministack): mock scale-set JIT config endpoint --- mockserver/initializerJson.json | 10 ++++++++++ tests/ministack/run-scale-set-integration.sh | 1 + 2 files changed, 11 insertions(+) diff --git a/mockserver/initializerJson.json b/mockserver/initializerJson.json index 829b90e08e..443ddb6201 100644 --- a/mockserver/initializerJson.json +++ b/mockserver/initializerJson.json @@ -76,6 +76,16 @@ "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"},{\"name\":\"linux\",\"type\":\"user\"},{\"name\":\"scale-set\",\"type\":\"user\"},{\"name\":\"self-hosted\",\"type\":\"user\"},{\"name\":\"x64\",\"type\":\"user\"}],\"runnerSetting\":{}}" } }, + { + "httpRequest": { + "method": "POST", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223/generatejitconfig" + }, + "httpResponseTemplate": { + "templateType": "MUSTACHE", + "template": "{\"statusCode\":200,\"headers\":{\"Content-Type\":[\"application/json\"]},\"body\":\"{\\\"runner\\\":{\\\"id\\\":321,\\\"name\\\":\\\"{{#jsonPath}}$.name{{/jsonPath}}{{jsonPathResult}}\\\",\\\"runnerScaleSetId\\\":223},\\\"encodedJITConfig\\\":\\\"test-only-jit-configuration\\\"}\"}" + } + }, { "httpRequest": { "method": "POST", diff --git a/tests/ministack/run-scale-set-integration.sh b/tests/ministack/run-scale-set-integration.sh index d3164e4861..9ed5a445d5 100755 --- a/tests/ministack/run-scale-set-integration.sh +++ b/tests/ministack/run-scale-set-integration.sh @@ -496,6 +496,7 @@ wait_for_mock_route GET '/tenant/123/_apis/runtime/runnergroups/' wait_for_mock_route GET '/tenant/123/_apis/runtime/runnerscalesets' wait_for_mock_route GET '/tenant/123/_apis/runtime/runnerscalesets/223' wait_for_mock_route PATCH '/tenant/123/_apis/runtime/runnerscalesets/223' +wait_for_mock_route POST '/tenant/123/_apis/runtime/runnerscalesets/223/generatejitconfig' wait_for_mock_route POST '/tenant/123/_apis/runtime/runnerscalesets/223/sessions' wait_for_mock_route GET '/messages' From f6ecf7ec6fc7476ad4a23dace209c77bfae5dff4 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 17 Sep 2026 00:39:27 +0200 Subject: [PATCH 08/13] fix(scale-set): use supported registration default --- examples/multi-runner-scale-set/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/multi-runner-scale-set/variables.tf b/examples/multi-runner-scale-set/variables.tf index d053869737..c2684beee4 100644 --- a/examples/multi-runner-scale-set/variables.tf +++ b/examples/multi-runner-scale-set/variables.tf @@ -16,7 +16,7 @@ variable "github" { config_url = optional(string, null) ssl_verify = optional(bool, true) runner_owner = optional(string, null) - registration_level = optional(string, "enterprise") + registration_level = optional(string, "organization") }) default = {} From 903db9868c13cb7d60749c346fa15db53602d35c Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 17 Sep 2026 00:53:33 +0200 Subject: [PATCH 09/13] docs(scale-set): update example registration default --- examples/multi-runner-scale-set/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/multi-runner-scale-set/README.md b/examples/multi-runner-scale-set/README.md index f8f8e64b54..d3a3e5a3aa 100644 --- a/examples/multi-runner-scale-set/README.md +++ b/examples/multi-runner-scale-set/README.md @@ -72,7 +72,7 @@ The GitHub App must be installed for the configured GitHub account. | [ami](#input\_ami) | Optional AMI configuration keyed by runner lane. | 
map(object({
    filter = optional(map(list(string)), { state = ["available"] })
    owners = optional(list(string), ["amazon"])
    id_ssm_parameter = optional(object({
      arn = string
    }), null)
    kms_key = optional(object({
      arn = string
    }), null)
  }))
| `{}` | no | | [aws\_region](#input\_aws\_region) | AWS region to deploy to. | `string` | `"eu-west-1"` | no | | [environment](#input\_environment) | Environment name, used as prefix. | `string` | `null` | no | -| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | 
object({
    config_url         = optional(string, null)
    ssl_verify         = optional(bool, true)
    runner_owner       = optional(string, null)
    registration_level = optional(string, "enterprise")
  })
| `{}` | no | +| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | 
object({
    config_url         = optional(string, null)
    ssl_verify         = optional(bool, true)
    runner_owner       = optional(string, null)
    registration_level = optional(string, "organization")
  })
| `{}` | no | | [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. | 
object({
    id              = string
    key_base64      = string
    installation_id = optional(string, null)
  })
| n/a | yes | | [runner\_binaries\_enabled](#input\_runner\_binaries\_enabled) | Whether runner binary synchronization is enabled. | `bool` | `true` | no | | [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | 
object({
    name              = string
    runner_group_name = optional(string, "Default")
    min_runners       = optional(number, 0)
    container = optional(object({
      image = optional(string, null)
    }), {})
  })
| n/a | yes | From 71645270ad25c988086d113941fcc1c111c4048d Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 17 Sep 2026 12:16:41 +0200 Subject: [PATCH 10/13] fix(ministack): match shortened scale-set resource names --- tests/ministack/run-scale-set-integration.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ministack/run-scale-set-integration.sh b/tests/ministack/run-scale-set-integration.sh index 9ed5a445d5..b71440ad9e 100755 --- a/tests/ministack/run-scale-set-integration.sh +++ b/tests/ministack/run-scale-set-integration.sh @@ -285,7 +285,7 @@ print(" [PASS] SSM manifest has the expected MockServer and GitHub App settings PY task_definition=$(ministack_aws ecs list-task-definitions \ - --family-prefix ministack-scale-set-ss-linux-scale-set- \ + --family-prefix ministack-scale-set-ss-linux-scale-se- \ --sort DESC \ --query 'taskDefinitionArns[0]' \ --output text) @@ -315,7 +315,7 @@ fi printf '%s\n' ' [PASS] ECS task definition requests the awslogs driver' log_group=$(ministack_aws logs describe-log-groups \ - --log-group-name-prefix "/aws/ecs/ministack-scale-set-ss-linux-scale-set-" \ + --log-group-name-prefix "/aws/ecs/ministack-scale-set-ss-linux-scale-se-" \ --query 'logGroups[0].logGroupName' \ --output text) if [ -z "$log_group" ] || [ "$log_group" = "None" ]; then From bcee08a7e9367f29f9702807f8cde37c1a7b58eb Mon Sep 17 00:00:00 2001 From: Ederson Brilhante Date: Thu, 17 Sep 2026 20:59:59 +0200 Subject: [PATCH 11/13] Update ministack.yml --- .github/workflows/ministack.yml | 80 --------------------------------- 1 file changed, 80 deletions(-) diff --git a/.github/workflows/ministack.yml b/.github/workflows/ministack.yml index 72fda289be..6258148461 100644 --- a/.github/workflows/ministack.yml +++ b/.github/workflows/ministack.yml @@ -7,22 +7,18 @@ on: paths: - ".github/workflows/ministack.yml" - "tests/ministack/**" - - "mockserver/**" - "*.tf" - "policies/**" - "examples/**" - "modules/**" - - "lambdas/services/scale-set/**" pull_request: paths: - ".github/workflows/ministack.yml" - "tests/ministack/**" - - "mockserver/**" - "*.tf" - "policies/**" - "examples/**" - "modules/**" - - "lambdas/services/scale-set/**" workflow_dispatch: concurrency: @@ -74,7 +70,6 @@ jobs: - ephemeral - multi-runner - multi-runner-v2 - - multi-runner-scale-set - termination-watcher services: ministack: @@ -136,78 +131,3 @@ jobs: IAC_BINARY: ${{ matrix.iac.binary }} IAC_LOCK_FILE: ${{ matrix.iac.lockfile }} run: tests/ministack/run-example.sh destroy "$EXAMPLE" - - integration_smoke: - name: Run scale-set ECS smoke test against MiniStack and MockServer - runs-on: ubuntu-latest - timeout-minutes: 30 - services: - ministack: - image: ghcr.io/ministackorg/ministack:1.5.12@sha256:41fe1ce2e666c6cc410c6047a9db8bf1df69cd0028ebc0a6c6e5517c3a83d6e0 - ports: - - 4566:4566 - options: >- - --add-host=host.docker.internal:host-gateway - --volume /var/run/docker.sock:/var/run/docker.sock - env: - MINISTACK_ACCOUNT_ID: "000000000000" - MINISTACK_REGION: eu-west-1 - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version-file: lambdas/.nvmrc - package-manager-cache: false - - - name: Setup Terraform - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - with: - terraform_version: latest - terraform_wrapper: false - - - name: Install Lambda dependencies - working-directory: lambdas - run: yarn install --frozen-lockfile - - - name: Build smoke-test Lambda distributions - working-directory: lambdas - run: | - yarn workspace @aws-github-runner/webhook dist - yarn workspace @aws-github-runner/control-plane dist - - - name: Start MockServer - id: mockserver - uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0 - with: - version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290' - port: '1080' - startup-timeout: '60' - - - name: Connect MockServer to MiniStack network - shell: bash - run: | - set -euo pipefail - ministack_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 ~ /ministack/ {print $1; exit}')" - network="$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$ministack_container" | sed -n '1p')" - docker network connect --alias mockserver "$network" mockserver - - - name: Mark repository as safe - shell: sh - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Run scale-set ECS/MockServer smoke test - env: - MINISTACK_GITHUB_MOCK_HOST: mockserver - MINISTACK_GITHUB_MOCK_PORT: "1080" - MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} - run: sh tests/ministack/run-scale-set-integration.sh From cbdd25ffda27066538036ef4125fe3e0c6285740 Mon Sep 17 00:00:00 2001 From: Ederson Brilhante Date: Thu, 17 Sep 2026 21:03:01 +0200 Subject: [PATCH 12/13] Update ministack.yml --- .github/workflows/ministack.yml | 76 +++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/.github/workflows/ministack.yml b/.github/workflows/ministack.yml index 80af872e42..fb8e14b4a5 100644 --- a/.github/workflows/ministack.yml +++ b/.github/workflows/ministack.yml @@ -72,6 +72,7 @@ jobs: - ephemeral - multi-runner - multi-runner-v2 + - multi-runner-scale-set - migration-test - termination-watcher services: @@ -195,3 +196,78 @@ jobs: MINISTACK_GITHUB_MOCK_PORT: "1080" MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} run: sh tests/ministack/run-smoke.sh + + integration_scaleset_smoke: + name: Run scale-set ECS smoke test against MiniStack and MockServer + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + ministack: + image: ghcr.io/ministackorg/ministack:1.5.12@sha256:41fe1ce2e666c6cc410c6047a9db8bf1df69cd0028ebc0a6c6e5517c3a83d6e0 + ports: + - 4566:4566 + options: >- + --add-host=host.docker.internal:host-gateway + --volume /var/run/docker.sock:/var/run/docker.sock + env: + MINISTACK_ACCOUNT_ID: "000000000000" + MINISTACK_REGION: eu-west-1 + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: lambdas/.nvmrc + package-manager-cache: false + + - name: Setup Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: latest + terraform_wrapper: false + + - name: Install Lambda dependencies + working-directory: lambdas + run: yarn install --frozen-lockfile + + - name: Build smoke-test Lambda distributions + working-directory: lambdas + run: | + yarn workspace @aws-github-runner/webhook dist + yarn workspace @aws-github-runner/control-plane dist + + - name: Start MockServer + id: mockserver + uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0 + with: + version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290' + port: '1080' + startup-timeout: '60' + + - name: Connect MockServer to MiniStack network + shell: bash + run: | + set -euo pipefail + ministack_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 ~ /ministack/ {print $1; exit}')" + network="$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$ministack_container" | sed -n '1p')" + docker network connect --alias mockserver "$network" mockserver + + - name: Mark repository as safe + shell: sh + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Run scale-set ECS/MockServer smoke test + env: + MINISTACK_GITHUB_MOCK_HOST: mockserver + MINISTACK_GITHUB_MOCK_PORT: "1080" + MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} + run: sh tests/ministack/run-scale-set-integration.sh From bd8df2d97a7d2c8f177a0433b07fcbb436cb4527 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:21:54 +0000 Subject: [PATCH 13/13] docs: auto update terraform docs --- examples/multi-runner-scale-set/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/multi-runner-scale-set/README.md b/examples/multi-runner-scale-set/README.md index 167ae7f275..21b502fe7b 100644 --- a/examples/multi-runner-scale-set/README.md +++ b/examples/multi-runner-scale-set/README.md @@ -71,8 +71,8 @@ The GitHub App must be installed for the configured GitHub account. |------|-------------|------|---------|:--------:| | [ami](#input\_ami) | Optional AMI configuration keyed by runner lane. | 
map(object({
    filter = optional(map(list(string)), { state = ["available"] })
    owners = optional(list(string), ["amazon"])
    id_ssm_parameter = optional(object({
      arn = string
    }), null)
    kms_key = optional(object({
      arn = string
    }), null)
  }))
| `{}` | no | | [aws\_region](#input\_aws\_region) | AWS region to deploy to. | `string` | `"eu-west-1"` | no | -| [environment](#input\_environment) | Environment name, used as prefix. | `string` | `null` | no | -| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | 
object({
    config_url         = optional(string, null)
    ssl_verify         = optional(bool, true)
    runner_owner       = optional(string, null)
    registration_level = optional(string, "organization")
  })
| `{}` | no | +| [environment](#input\_environment) | Environment name, used as prefix. | `string` | n/a | yes | +| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | 
object({
    url                = optional(string, null)
    ssl_verify         = optional(bool, true)
    runner_owner       = optional(string, null)
    registration_level = optional(string, "organization")
  })
| `{}` | no | | [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. | 
object({
    id              = string
    key_base64      = string
    installation_id = optional(string, null)
  })
| n/a | yes | | [runner\_binaries\_enabled](#input\_runner\_binaries\_enabled) | Whether runner binary synchronization is enabled. | `bool` | `true` | no | | [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | 
object({
    name              = string
    runner_group_name = optional(string, "Default")
    min_runners       = optional(number, 0)
    container = optional(object({
      image = optional(string, null)
    }), {})
  })
| n/a | yes |