diff --git a/.github/scripts/scale-set-container-smoke-test.sh b/.github/scripts/scale-set-container-smoke-test.sh new file mode 100755 index 0000000000..eea46eea72 --- /dev/null +++ b/.github/scripts/scale-set-container-smoke-test.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +container_name="scale-set-service-smoke-${GITHUB_RUN_ID:-$$}" +response_file="$(mktemp)" +# shellcheck disable=SC2329 # cleanup is invoked indirectly by the EXIT trap. +cleanup() { + docker rm -f "$container_name" >/dev/null 2>&1 || true + rm -f "$response_file" +} +trap cleanup EXIT + +scale_set_controller_manifest='{"version":1,"groupName":"ci-smoke","revision":"image-test","reconcilers":[{"schemaVersion":1,"runnerConfigName":"smoke","scaleSetId":1,"scaleSetName":"ci-smoke","githubConfigUrl":"https://github.com/example-org","githubApp":{"appIdParameterName":"/ci/app-id","privateKeyParameterName":"/ci/private-key"},"computeProvider":{"type":"ec2","configuration":{"region":"us-east-1","environment":"ci","runnerNamePrefix":"ci","jitConfigParameterPath":"/ci/jit","subnets":["subnet-00000000"],"launchTemplateName":"ci","ec2instanceCriteria":{"instanceTypes":["t3.micro"],"targetCapacityType":"on-demand","instanceAllocationStrategy":"lowest-price"}}},"minRunners":0,"maxRunners":0}]}' + +docker run --detach \ + --name "$container_name" \ + --network none \ + --read-only \ + --cap-drop ALL \ + --security-opt no-new-privileges \ + --env AWS_REGION=us-east-1 \ + --env AWS_EC2_METADATA_DISABLED=true \ + --env SCALE_SET_HEALTH_PORT=8080 \ + --env "SCALE_SET_CONTROLLER_MANIFEST=$scale_set_controller_manifest" \ + scale-set-service:smoke-test + +attempt=0 +while (( attempt < 30 )); do + ((attempt += 1)) + docker exec "$container_name" node --input-type=module -e \ + 'const response = await fetch("http://127.0.0.1:8080/healthz", { signal: AbortSignal.timeout(1000) }); process.stdout.write(JSON.stringify({ status: response.status, body: await response.json() }));' \ + >"$response_file" 2>/dev/null || true + if jq --exit-status \ + --arg group_name ci-smoke \ + '(.status == 200 or .status == 503) and .body.groupName == $group_name and (.body.live | type == "boolean") and (.body.ready | type == "boolean") and .body.reconcilers.smoke != null' \ + "$response_file" >/dev/null 2>&1; then + jq . "$response_file" + exit 0 + fi + sleep 1 +done + +docker logs "$container_name" +echo "scale-set service image did not return the expected health response" >&2 +exit 1 diff --git a/.github/workflows/lambda.yml b/.github/workflows/lambda.yml index bda4a93818..9de8ab0a12 100644 --- a/.github/workflows/lambda.yml +++ b/.github/workflows/lambda.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - 'lambdas/**' + - '.github/scripts/**' - '.github/workflows/lambda.yml' concurrency: @@ -85,3 +86,16 @@ jobs: push: false cache-from: type=gha,scope=scale-set-service cache-to: type=gha,mode=max,scope=scale-set-service + + - name: Build scale-set service image for smoke test + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ./lambdas/services/scale-set/Dockerfile + platforms: linux/amd64 + load: true + tags: scale-set-service:smoke-test + cache-from: type=gha,scope=scale-set-service + + - name: Run scale-set service image smoke test + run: ./.github/scripts/scale-set-container-smoke-test.sh diff --git a/.github/workflows/ministack.yml b/.github/workflows/ministack.yml index aa6e2b505d..fb8e14b4a5 100644 --- a/.github/workflows/ministack.yml +++ b/.github/workflows/ministack.yml @@ -196,3 +196,78 @@ jobs: MINISTACK_GITHUB_MOCK_PORT: "1080" MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} run: sh tests/ministack/run-smoke.sh + + integration_scaleset_smoke: + name: Run scale-set ECS smoke test against MiniStack and MockServer + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + ministack: + image: ghcr.io/ministackorg/ministack:1.5.12@sha256:41fe1ce2e666c6cc410c6047a9db8bf1df69cd0028ebc0a6c6e5517c3a83d6e0 + ports: + - 4566:4566 + options: >- + --add-host=host.docker.internal:host-gateway + --volume /var/run/docker.sock:/var/run/docker.sock + env: + MINISTACK_ACCOUNT_ID: "000000000000" + MINISTACK_REGION: eu-west-1 + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: lambdas/.nvmrc + package-manager-cache: false + + - name: Setup Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: latest + terraform_wrapper: false + + - name: Install Lambda dependencies + working-directory: lambdas + run: yarn install --frozen-lockfile + + - name: Build smoke-test Lambda distributions + working-directory: lambdas + run: | + yarn workspace @aws-github-runner/webhook dist + yarn workspace @aws-github-runner/control-plane dist + + - name: Start MockServer + id: mockserver + uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0 + with: + version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290' + port: '1080' + startup-timeout: '60' + + - name: Connect MockServer to MiniStack network + shell: bash + run: | + set -euo pipefail + ministack_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 ~ /ministack/ {print $1; exit}')" + network="$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$ministack_container" | sed -n '1p')" + docker network connect --alias mockserver "$network" mockserver + + - name: Mark repository as safe + shell: sh + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Run scale-set ECS/MockServer smoke test + env: + MINISTACK_GITHUB_MOCK_HOST: mockserver + MINISTACK_GITHUB_MOCK_PORT: "1080" + MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} + run: sh tests/ministack/run-scale-set-integration.sh diff --git a/examples/multi-runner-scale-set/README.md b/examples/multi-runner-scale-set/README.md index 167ae7f275..21b502fe7b 100644 --- a/examples/multi-runner-scale-set/README.md +++ b/examples/multi-runner-scale-set/README.md @@ -71,8 +71,8 @@ The GitHub App must be installed for the configured GitHub account. |------|-------------|------|---------|:--------:| | [ami](#input\_ami) | Optional AMI configuration keyed by runner lane. |
map(object({
filter = optional(map(list(string)), { state = ["available"] })
owners = optional(list(string), ["amazon"])
id_ssm_parameter = optional(object({
arn = string
}), null)
kms_key = optional(object({
arn = string
}), null)
})) | `{}` | no |
| [aws\_region](#input\_aws\_region) | AWS region to deploy to. | `string` | `"eu-west-1"` | no |
-| [environment](#input\_environment) | Environment name, used as prefix. | `string` | `null` | no |
-| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | object({
config_url = optional(string, null)
ssl_verify = optional(bool, true)
runner_owner = optional(string, null)
registration_level = optional(string, "organization")
}) | `{}` | no |
+| [environment](#input\_environment) | Environment name, used as prefix. | `string` | n/a | yes |
+| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | object({
url = optional(string, null)
ssl_verify = optional(bool, true)
runner_owner = optional(string, null)
registration_level = optional(string, "organization")
}) | `{}` | no |
| [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. | object({
id = string
key_base64 = string
installation_id = optional(string, null)
}) | n/a | yes |
| [runner\_binaries\_enabled](#input\_runner\_binaries\_enabled) | Whether runner binary synchronization is enabled. | `bool` | `true` | no |
| [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | object({
name = string
runner_group_name = optional(string, "Default")
min_runners = optional(number, 0)
container = optional(object({
image = optional(string, null)
}), {})
}) | n/a | yes |
diff --git a/mockserver/initializerJson.json b/mockserver/initializerJson.json
new file mode 100644
index 0000000000..443ddb6201
--- /dev/null
+++ b/mockserver/initializerJson.json
@@ -0,0 +1,138 @@
+[
+ {
+ "httpRequest": {
+ "method": "POST",
+ "path": "/api/v3/app/installations/456/access_tokens"
+ },
+ "httpResponse": {
+ "statusCode": 201,
+ "headers": { "Content-Type": ["application/json"] },
+ "body": "{\"token\":\"fake-installation-token\",\"expires_at\":\"2099-01-01T00:00:00Z\"}"
+ }
+ },
+ {
+ "httpRequest": {
+ "method": "POST",
+ "path": "/api/v3/orgs/example/actions/runners/registration-token"
+ },
+ "httpResponse": {
+ "statusCode": 201,
+ "headers": { "Content-Type": ["application/json"] },
+ "body": "{\"token\":\"fake-registration-token\"}"
+ }
+ },
+ {
+ "httpRequest": {
+ "method": "POST",
+ "path": "/api/v3/actions/runner-registration"
+ },
+ "httpResponse": {
+ "statusCode": 200,
+ "headers": { "Content-Type": ["application/json"] },
+ "body": "{\"url\":\"https://mockserver:1080/tenant/123\",\"token\":\"header.eyJleHAiOjQwNzA5MDg4MDB9.signature\"}"
+ }
+ },
+ {
+ "httpRequest": {
+ "method": "GET",
+ "path": "/tenant/123/_apis/runtime/runnergroups/"
+ },
+ "httpResponse": {
+ "statusCode": 200,
+ "headers": { "Content-Type": ["application/json"] },
+ "body": "{\"count\":1,\"value\":[{\"id\":48,\"name\":\"experimental-euw1-sl-cicd-forge-emu\"}]}"
+ }
+ },
+ {
+ "httpRequest": {
+ "method": "GET",
+ "path": "/tenant/123/_apis/runtime/runnerscalesets"
+ },
+ "httpResponse": {
+ "statusCode": 200,
+ "headers": { "Content-Type": ["application/json"] },
+ "body": "{\"count\":1,\"value\":[{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}]}"
+ }
+ },
+ {
+ "httpRequest": {
+ "method": "GET",
+ "path": "/tenant/123/_apis/runtime/runnerscalesets/223"
+ },
+ "httpResponse": {
+ "statusCode": 200,
+ "headers": { "Content-Type": ["application/json"] },
+ "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}"
+ }
+ },
+ {
+ "httpRequest": {
+ "method": "PATCH",
+ "path": "/tenant/123/_apis/runtime/runnerscalesets/223"
+ },
+ "httpResponse": {
+ "statusCode": 200,
+ "headers": { "Content-Type": ["application/json"] },
+ "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"},{\"name\":\"linux\",\"type\":\"user\"},{\"name\":\"scale-set\",\"type\":\"user\"},{\"name\":\"self-hosted\",\"type\":\"user\"},{\"name\":\"x64\",\"type\":\"user\"}],\"runnerSetting\":{}}"
+ }
+ },
+ {
+ "httpRequest": {
+ "method": "POST",
+ "path": "/tenant/123/_apis/runtime/runnerscalesets/223/generatejitconfig"
+ },
+ "httpResponseTemplate": {
+ "templateType": "MUSTACHE",
+ "template": "{\"statusCode\":200,\"headers\":{\"Content-Type\":[\"application/json\"]},\"body\":\"{\\\"runner\\\":{\\\"id\\\":321,\\\"name\\\":\\\"{{#jsonPath}}$.name{{/jsonPath}}{{jsonPathResult}}\\\",\\\"runnerScaleSetId\\\":223},\\\"encodedJITConfig\\\":\\\"test-only-jit-configuration\\\"}\"}"
+ }
+ },
+ {
+ "httpRequest": {
+ "method": "POST",
+ "path": "/tenant/123/_apis/runtime/runnerscalesets/223/sessions"
+ },
+ "httpResponse": {
+ "statusCode": 200,
+ "headers": { "Content-Type": ["application/json"] },
+ "body": "{\"sessionId\":\"11111111-1111-1111-1111-111111111111\",\"ownerName\":\"local.medium\",\"runnerScaleSet\":{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48},\"messageQueueUrl\":\"https://mockserver:1080/messages?sessionId=11111111-1111-1111-1111-111111111111&api-version=6.0-preview\",\"messageQueueAccessToken\":\"fake-queue-token\",\"statistics\":{\"totalAvailableJobs\":0,\"totalAcquiredJobs\":0,\"totalAssignedJobs\":0,\"totalRunningJobs\":0,\"totalRegisteredRunners\":0,\"totalBusyRunners\":0,\"totalIdleRunners\":0}}"
+ }
+ },
+ {
+ "httpRequest": {
+ "method": "GET",
+ "path": "/messages"
+ },
+ "httpResponse": {
+ "statusCode": 202,
+ "delay": { "timeUnit": "MILLISECONDS", "value": 1000 }
+ }
+ },
+ {
+ "httpRequest": {
+ "method": "GET",
+ "path": "/tenant/123/_apis/distributedtask/pools/0/agents"
+ },
+ "httpResponseTemplate": {
+ "templateType": "MUSTACHE",
+ "template": "{\"statusCode\":200,\"headers\":{\"Content-Type\":[\"application/json\"]},\"body\":\"{\\\"count\\\":1,\\\"value\\\":[{\\\"id\\\":321,\\\"name\\\":\\\"{{ request.queryStringParameters.agentName.0 }}\\\",\\\"runnerScaleSetId\\\":223}]}\"}"
+ }
+ },
+ {
+ "httpRequest": {
+ "method": "DELETE",
+ "path": "/tenant/123/_apis/distributedtask/pools/0/agents/321"
+ },
+ "httpResponse": {
+ "statusCode": 204
+ }
+ },
+ {
+ "httpRequest": {
+ "method": "DELETE",
+ "path": "/tenant/123/_apis/runtime/runnerscalesets/223/sessions/11111111-1111-1111-1111-111111111111"
+ },
+ "httpResponse": {
+ "statusCode": 204
+ }
+ }
+]
diff --git a/tests/ministack/multi-runner-scale-set.tfvars b/tests/ministack/multi-runner-scale-set.tfvars
index 6a47126ad6..7dd1a0ad39 100644
--- a/tests/ministack/multi-runner-scale-set.tfvars
+++ b/tests/ministack/multi-runner-scale-set.tfvars
@@ -54,4 +54,4 @@ scale_set = {
container = {
image = "localhost:4566/scale-set-controller:smoke"
}
-}
\ No newline at end of file
+}
diff --git a/tests/ministack/run-scale-set-integration.sh b/tests/ministack/run-scale-set-integration.sh
new file mode 100755
index 0000000000..b71440ad9e
--- /dev/null
+++ b/tests/ministack/run-scale-set-integration.sh
@@ -0,0 +1,600 @@
+#!/bin/sh
+
+set -eu
+
+export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-000000000000}"
+export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-test-only}"
+export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-eu-west-1}"
+export AWS_REGION="${AWS_REGION:-eu-west-1}"
+export AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://127.0.0.1:4566}"
+export AWS_EC2_METADATA_DISABLED="${AWS_EC2_METADATA_DISABLED:-true}"
+
+script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+source_root=$(CDPATH='' cd -- "$script_dir/../.." && pwd)
+example="multi-runner-scale-set"
+base_tfvars="$script_dir/$example.tfvars"
+tfvars_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set.XXXXXX")
+app_key_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-key.XXXXXX")
+log_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-logs.XXXXXX")
+controller_log_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-controller-logs.XXXXXX")
+config_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-config.XXXXXX")
+task_definition_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-task-definition.XXXXXX")
+instance_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-instances.XXXXXX")
+scale_down_config_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-scale-down-config.XXXXXX")
+initializer_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-mockserver.XXXXXX")
+repository_name="scale-set-controller"
+image_reference="localhost:4566/${repository_name}:smoke"
+mockserver_host="${MINISTACK_GITHUB_MOCK_HOST:-127.0.0.1}"
+mockserver_port="${MINISTACK_GITHUB_MOCK_PORT:-1080}"
+mockserver_url="${MINISTACK_GITHUB_MOCK_URL:-http://127.0.0.1:${mockserver_port}}"
+controller_mock_url="https://${mockserver_host}:${mockserver_port}"
+terraform_state_exists=false
+
+cleanup() {
+ cleanup_status=$?
+ set +e
+
+ if [ "$terraform_state_exists" = true ]; then
+ "$source_root/tests/ministack/run-example.sh" destroy "$example" "$tfvars_file" >/dev/null 2>&1
+ fi
+
+ rm -f "$tfvars_file" "$app_key_file" "$log_file" "$controller_log_file" "$config_file" "$task_definition_file" "$instance_file" "$scale_down_config_file" "$initializer_file"
+ exit "$cleanup_status"
+}
+trap cleanup EXIT INT TERM
+
+require_command() {
+ if ! command -v "$1" >/dev/null 2>&1; then
+ echo "$1 is required to run the scale-set MiniStack smoke test." >&2
+ exit 69
+ fi
+}
+
+for command in aws curl docker openssl python3 terraform; do
+ require_command "$command"
+done
+
+ministack_aws() {
+ aws --endpoint-url "$AWS_ENDPOINT_URL" --region "$AWS_DEFAULT_REGION" "$@"
+}
+
+wait_for_http() {
+ url="$1"
+ attempts=90
+ while ! curl -fsS --max-time 2 "$url" >/dev/null 2>&1; do
+ attempts=$((attempts - 1))
+ if [ "$attempts" -le 0 ]; then
+ echo "Timed out waiting for $url." >&2
+ exit 70
+ fi
+ sleep 1
+ done
+}
+
+wait_for_scale_set_runner() {
+ attempts=90
+ while :; do
+ ministack_aws ec2 describe-instances --output json > "$instance_file"
+ instance_id=$(python3 - "$instance_file" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as instance_file:
+ response = json.load(instance_file)
+
+required_tags = {
+ "ghr:Application": "github-action-runner",
+ "ghr:created_by": "scale-set-service",
+ "ghr:environment": "ministack-scale-set-linux-scale-set",
+ "ghr:Type": "Org",
+ "ghr:Owner": "example",
+ "ghr:scale_set_state": "config-published",
+ "ghr:github_runner_id": "321",
+}
+active_states = {"pending", "running", "stopping", "stopped", "shutting-down"}
+matches = []
+for reservation in response.get("Reservations", []):
+ for instance in reservation.get("Instances", []):
+ if instance.get("State", {}).get("Name") not in active_states:
+ continue
+ tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])}
+ if all(tags.get(key) == value for key, value in required_tags.items()) and tags.get("ghr:runner_name", "").startswith("scale-set-"):
+ matches.append(instance["InstanceId"])
+
+if len(matches) == 1:
+ print(matches[0])
+PY
+)
+ if [ -n "$instance_id" ]; then
+ printf ' [PASS] MiniStack created and registered scale-set EC2 runner %s\n' "$instance_id"
+ return 0
+ fi
+ attempts=$((attempts - 1))
+ if [ "$attempts" -le 0 ]; then
+ echo "Timed out waiting for the scale-set EC2 runner to reach config-published state." >&2
+ cat "$instance_file" >&2 || true
+ exit 1
+ fi
+ sleep 2
+ done
+}
+
+wait_for_no_scale_set_runners() {
+ attempts=90
+ while :; do
+ ministack_aws ec2 describe-instances --output json > "$instance_file"
+ active_count=$(python3 - "$instance_file" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as instance_file:
+ response = json.load(instance_file)
+
+required_tags = {
+ "ghr:Application": "github-action-runner",
+ "ghr:created_by": "scale-set-service",
+ "ghr:environment": "ministack-scale-set-linux-scale-set",
+ "ghr:Type": "Org",
+ "ghr:Owner": "example",
+}
+active_states = {"pending", "running", "stopping", "stopped", "shutting-down"}
+count = 0
+for reservation in response.get("Reservations", []):
+ for instance in reservation.get("Instances", []):
+ if instance.get("State", {}).get("Name") not in active_states:
+ continue
+ tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])}
+ if all(tags.get(key) == value for key, value in required_tags.items()):
+ count += 1
+print(count)
+PY
+)
+ if [ "$active_count" = "0" ]; then
+ printf '%s\n' ' [PASS] scale-set EC2 runner was terminated after the minimum changed to zero'
+ return 0
+ fi
+ attempts=$((attempts - 1))
+ if [ "$attempts" -le 0 ]; then
+ echo "Timed out waiting for the scale-set EC2 runner to terminate; active count is $active_count." >&2
+ cat "$instance_file" >&2 || true
+ exit 1
+ fi
+ sleep 2
+ done
+}
+
+wait_for_http "$AWS_ENDPOINT_URL/_ministack/health"
+attempts=90
+while ! curl -fsS --max-time 2 -X PUT "$mockserver_url/mockserver/status" >/dev/null 2>&1; do
+ attempts=$((attempts - 1))
+ if [ "$attempts" -le 0 ]; then
+ echo "Timed out waiting for $mockserver_url/mockserver/status." >&2
+ exit 70
+ fi
+ sleep 1
+done
+curl -fsS -X PUT "$mockserver_url/mockserver/reset" >/dev/null
+CONTROLLER_MOCK_URL="$controller_mock_url" python3 - "$source_root/mockserver/initializerJson.json" "$initializer_file" <<'PY'
+import os
+import sys
+
+source, destination = sys.argv[1:]
+with open(source, encoding="utf-8") as source_file:
+ fixture = source_file.read()
+fixture = fixture.replace("https://mockserver:1080", os.environ["CONTROLLER_MOCK_URL"])
+with open(destination, "w", encoding="utf-8") as destination_file:
+ destination_file.write(fixture)
+PY
+curl -fsS -X PUT \
+ "$mockserver_url/mockserver/expectation" \
+ -H 'Content-Type: application/json' \
+ --data-binary "@$initializer_file" \
+ >/dev/null
+
+repository_policy=$(python3 - <<'PY'
+import json
+
+print(json.dumps({
+ "Version": "2012-10-17",
+ "Statement": [{
+ "Sid": "AllowAccountPull",
+ "Effect": "Allow",
+ "Principal": {"AWS": "arn:aws:iam::000000000000:root"},
+ "Action": [
+ "ecr:BatchCheckLayerAvailability",
+ "ecr:BatchGetImage",
+ "ecr:GetDownloadUrlForLayer",
+ ],
+ }],
+}))
+PY
+)
+
+ministack_aws ecr create-repository \
+ --repository-name "$repository_name" \
+ --image-tag-mutability IMMUTABLE \
+ --image-scanning-configuration scanOnPush=false \
+ >/dev/null
+ministack_aws ecr set-repository-policy \
+ --repository-name "$repository_name" \
+ --policy-text "$repository_policy" \
+ >/dev/null
+
+docker build \
+ --target runtime \
+ --file "$source_root/lambdas/services/scale-set/Dockerfile" \
+ --tag "$image_reference" \
+ "$source_root"
+
+ministack_aws ecr get-login-password | docker login \
+ --username AWS \
+ --password-stdin localhost:4566 >/dev/null
+docker push "$image_reference"
+
+ministack_aws ecr describe-images \
+ --repository-name "$repository_name" \
+ --image-ids imageTag=smoke \
+ >/dev/null
+
+openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$app_key_file" 2>/dev/null
+app_key_base64=$(base64 < "$app_key_file" | tr -d '\n')
+APP_KEY_BASE64="$app_key_base64" GITHUB_CONFIG_URL="$controller_mock_url" python3 - "$base_tfvars" "$tfvars_file" <<'PY'
+import os
+import sys
+
+source, destination = sys.argv[1:]
+replacement = os.environ["APP_KEY_BASE64"]
+github_config_url = os.environ["GITHUB_CONFIG_URL"]
+with open(source, encoding="utf-8") as source_file:
+ lines = source_file.readlines()
+with open(destination, "w", encoding="utf-8") as destination_file:
+ for line in lines:
+ if line.lstrip().startswith("key_base64"):
+ destination_file.write(f' key_base64 = "{replacement}"\n')
+ elif line.lstrip().startswith("config_url"):
+ destination_file.write(f' config_url = "{github_config_url}"\n')
+ else:
+ destination_file.write(line)
+PY
+unset app_key_base64 APP_KEY_BASE64
+
+terraform_state_exists=true
+"$source_root/tests/ministack/run-example.sh" apply "$example" "$tfvars_file"
+
+config_path="/ministack-scale-set/scale-set-controller/linux-scale-set/linux-scale-set"
+ministack_aws ssm get-parameter \
+ --name "$config_path" \
+ --query 'Parameter.Value' \
+ --output text > "$config_file"
+EXPECTED_GITHUB_CONFIG_URL="$controller_mock_url/example" python3 - "$config_file" <<'PY'
+import json
+import os
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as config_file:
+ config = json.load(config_file)
+
+assert config["githubConfigUrl"] == os.environ["EXPECTED_GITHUB_CONFIG_URL"]
+assert config["forceGhes"] is True
+assert config["sslVerify"] is False
+assert config["minRunners"] == 1
+assert config["githubApp"]["appIdParameterName"]
+assert config["githubApp"]["installationIdParameterName"]
+assert config["githubApp"]["privateKeyParameterName"]
+print(" [PASS] SSM manifest has the expected MockServer and GitHub App settings")
+PY
+
+task_definition=$(ministack_aws ecs list-task-definitions \
+ --family-prefix ministack-scale-set-ss-linux-scale-se- \
+ --sort DESC \
+ --query 'taskDefinitionArns[0]' \
+ --output text)
+if [ -z "$task_definition" ] || [ "$task_definition" = "None" ]; then
+ echo "The scale-set task definition was not registered." >&2
+ exit 1
+fi
+
+actual_image=$(ministack_aws ecs describe-task-definition \
+ --task-definition "$task_definition" \
+ --query 'taskDefinition.containerDefinitions[?name==`scale-set-controller`].image | [0]' \
+ --output text)
+if [ "$actual_image" != "$image_reference" ]; then
+ echo "Expected the ECS task to use $image_reference, got $actual_image." >&2
+ exit 1
+fi
+printf '%s\n' ' [PASS] ECS task definition uses the image pushed to MiniStack ECR'
+
+log_driver=$(ministack_aws ecs describe-task-definition \
+ --task-definition "$task_definition" \
+ --query 'taskDefinition.containerDefinitions[?name==`scale-set-controller`].logConfiguration.logDriver | [0]' \
+ --output text)
+if [ "$log_driver" != "awslogs" ]; then
+ echo "Expected the ECS task to request the awslogs driver, got $log_driver." >&2
+ exit 1
+fi
+printf '%s\n' ' [PASS] ECS task definition requests the awslogs driver'
+
+log_group=$(ministack_aws logs describe-log-groups \
+ --log-group-name-prefix "/aws/ecs/ministack-scale-set-ss-linux-scale-se-" \
+ --query 'logGroups[0].logGroupName' \
+ --output text)
+if [ -z "$log_group" ] || [ "$log_group" = "None" ]; then
+ echo "The scale-set CloudWatch log group was not created." >&2
+ exit 1
+fi
+printf '%s\n' ' [PASS] CloudWatch log group was created for the ECS controller'
+
+task_family=$(ministack_aws ecs describe-task-definition \
+ --task-definition "$task_definition" \
+ --query 'taskDefinition.family' \
+ --output text)
+
+# MiniStack exposes ECS credentials through the gateway's container IP. The
+# Node.js AWS SDK intentionally rejects that address in
+# AWS_CONTAINER_CREDENTIALS_FULL_URI because non-HTTPS full URIs are limited
+# to loopback and the real ECS metadata address. Use only synthetic,
+# test-scoped credentials in a temporary task-definition revision so the
+# smoke test still exercises the pushed image, ECS service, and controller
+# lifecycle without changing the production task definition or application.
+ministack_aws ecs describe-task-definition \
+ --task-definition "$task_definition" \
+ --query 'taskDefinition' \
+ --output json > "$task_definition_file"
+TASK_DEFINITION_FILE="$task_definition_file" python3 - <<'PY'
+import json
+import os
+
+path = os.environ["TASK_DEFINITION_FILE"]
+with open(path, encoding="utf-8") as task_definition_file:
+ task_definition = json.load(task_definition_file)
+
+for field in (
+ "taskDefinitionArn",
+ "revision",
+ "status",
+ "requiresAttributes",
+ "compatibilities",
+ "registeredAt",
+ "registeredBy",
+):
+ task_definition.pop(field, None)
+
+for container in task_definition["containerDefinitions"]:
+ if container["name"] != "scale-set-controller":
+ continue
+ environment = container.setdefault("environment", [])
+ environment.extend([
+ {"name": "AWS_ACCESS_KEY_ID", "value": "000000000000"},
+ {"name": "AWS_SECRET_ACCESS_KEY", "value": "test-only"},
+ ])
+
+with open(path, "w", encoding="utf-8") as task_definition_file:
+ json.dump(task_definition, task_definition_file)
+PY
+
+environment_name=$(sed -n 's/^environment[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$base_tfvars")
+cluster_name="${environment_name}-scale-set"
+task_definition=$(ministack_aws ecs register-task-definition \
+ --cli-input-json "file://$task_definition_file" \
+ --query 'taskDefinition.taskDefinitionArn' \
+ --output text)
+ministack_aws ecs update-service \
+ --cluster "$cluster_name" \
+ --service "$task_family" \
+ --task-definition "$task_definition" \
+ >/dev/null
+
+task_revision=$(ministack_aws ecs describe-task-definition \
+ --task-definition "$task_definition" \
+ --query 'taskDefinition.revision' \
+ --output text)
+controller_container=""
+attempts=90
+while [ -z "$controller_container" ]; do
+ controller_container=$(docker ps -a \
+ --filter "label=com.amazonaws.ecs.task-definition-family=$task_family" \
+ --filter "label=com.amazonaws.ecs.task-definition-version=$task_revision" \
+ --filter 'name=scale-set-controller' \
+ --format '{{.ID}}' | sed -n '1p')
+ if [ -n "$controller_container" ]; then
+ break
+ fi
+ attempts=$((attempts - 1))
+ if [ "$attempts" -le 0 ]; then
+ echo "Timed out waiting for the MiniStack ECS controller container." >&2
+ docker ps -a --format '{{.ID}} {{.Image}} {{.Status}} {{.Names}}' >&2 || true
+ exit 1
+ fi
+ sleep 2
+done
+printf ' [PASS] MiniStack started ECS controller container %s\n' "$controller_container"
+
+wait_for_controller_log_event() {
+ marker="$1"
+ required_text="${2:-}"
+ attempts=90
+ while :; do
+ docker logs "$controller_container" > "$controller_log_file" 2>&1 || true
+ if python3 - "$controller_log_file" "$marker" "$required_text" <<'PY'
+import sys
+
+marker, required = sys.argv[2:]
+with open(sys.argv[1], encoding="utf-8") as log_file:
+ messages = log_file.read().splitlines()
+if any(marker in message and (not required or required in message) for message in messages):
+ raise SystemExit(0)
+raise SystemExit(1)
+PY
+ then
+ printf ' [PASS] CloudWatch logs contain %s\n' "$marker"
+ return 0
+ fi
+ attempts=$((attempts - 1))
+ if [ "$attempts" -le 0 ]; then
+ echo "Timed out waiting for controller log marker '$marker'." >&2
+ cat "$controller_log_file" >&2 || true
+ echo "Controller AWS/ECS metadata environment:" >&2
+ docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" \
+ | sed -E 's/^(AWS_CONTAINER_AUTHORIZATION_TOKEN|AWS_CONTAINER_CREDENTIALS_FULL_URI)=.*/\1=