diff --git a/.github/scripts/scale-set-container-smoke-test.sh b/.github/scripts/scale-set-container-smoke-test.sh new file mode 100755 index 0000000000..eea46eea72 --- /dev/null +++ b/.github/scripts/scale-set-container-smoke-test.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +container_name="scale-set-service-smoke-${GITHUB_RUN_ID:-$$}" +response_file="$(mktemp)" +# shellcheck disable=SC2329 # cleanup is invoked indirectly by the EXIT trap. +cleanup() { + docker rm -f "$container_name" >/dev/null 2>&1 || true + rm -f "$response_file" +} +trap cleanup EXIT + +scale_set_controller_manifest='{"version":1,"groupName":"ci-smoke","revision":"image-test","reconcilers":[{"schemaVersion":1,"runnerConfigName":"smoke","scaleSetId":1,"scaleSetName":"ci-smoke","githubConfigUrl":"https://github.com/example-org","githubApp":{"appIdParameterName":"/ci/app-id","privateKeyParameterName":"/ci/private-key"},"computeProvider":{"type":"ec2","configuration":{"region":"us-east-1","environment":"ci","runnerNamePrefix":"ci","jitConfigParameterPath":"/ci/jit","subnets":["subnet-00000000"],"launchTemplateName":"ci","ec2instanceCriteria":{"instanceTypes":["t3.micro"],"targetCapacityType":"on-demand","instanceAllocationStrategy":"lowest-price"}}},"minRunners":0,"maxRunners":0}]}' + +docker run --detach \ + --name "$container_name" \ + --network none \ + --read-only \ + --cap-drop ALL \ + --security-opt no-new-privileges \ + --env AWS_REGION=us-east-1 \ + --env AWS_EC2_METADATA_DISABLED=true \ + --env SCALE_SET_HEALTH_PORT=8080 \ + --env "SCALE_SET_CONTROLLER_MANIFEST=$scale_set_controller_manifest" \ + scale-set-service:smoke-test + +attempt=0 +while (( attempt < 30 )); do + ((attempt += 1)) + docker exec "$container_name" node --input-type=module -e \ + 'const response = await fetch("http://127.0.0.1:8080/healthz", { signal: AbortSignal.timeout(1000) }); process.stdout.write(JSON.stringify({ status: response.status, body: await response.json() }));' \ + >"$response_file" 2>/dev/null || true + if jq --exit-status \ + --arg group_name ci-smoke \ + '(.status == 200 or .status == 503) and .body.groupName == $group_name and (.body.live | type == "boolean") and (.body.ready | type == "boolean") and .body.reconcilers.smoke != null' \ + "$response_file" >/dev/null 2>&1; then + jq . "$response_file" + exit 0 + fi + sleep 1 +done + +docker logs "$container_name" +echo "scale-set service image did not return the expected health response" >&2 +exit 1 diff --git a/.github/workflows/lambda.yml b/.github/workflows/lambda.yml index bda4a93818..9de8ab0a12 100644 --- a/.github/workflows/lambda.yml +++ b/.github/workflows/lambda.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - 'lambdas/**' + - '.github/scripts/**' - '.github/workflows/lambda.yml' concurrency: @@ -85,3 +86,16 @@ jobs: push: false cache-from: type=gha,scope=scale-set-service cache-to: type=gha,mode=max,scope=scale-set-service + + - name: Build scale-set service image for smoke test + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ./lambdas/services/scale-set/Dockerfile + platforms: linux/amd64 + load: true + tags: scale-set-service:smoke-test + cache-from: type=gha,scope=scale-set-service + + - name: Run scale-set service image smoke test + run: ./.github/scripts/scale-set-container-smoke-test.sh diff --git a/.github/workflows/ministack.yml b/.github/workflows/ministack.yml index aa6e2b505d..fb8e14b4a5 100644 --- a/.github/workflows/ministack.yml +++ b/.github/workflows/ministack.yml @@ -196,3 +196,78 @@ jobs: MINISTACK_GITHUB_MOCK_PORT: "1080" MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} run: sh tests/ministack/run-smoke.sh + + integration_scaleset_smoke: + name: Run scale-set ECS smoke test against MiniStack and MockServer + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + ministack: + image: ghcr.io/ministackorg/ministack:1.5.12@sha256:41fe1ce2e666c6cc410c6047a9db8bf1df69cd0028ebc0a6c6e5517c3a83d6e0 + ports: + - 4566:4566 + options: >- + --add-host=host.docker.internal:host-gateway + --volume /var/run/docker.sock:/var/run/docker.sock + env: + MINISTACK_ACCOUNT_ID: "000000000000" + MINISTACK_REGION: eu-west-1 + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: lambdas/.nvmrc + package-manager-cache: false + + - name: Setup Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: latest + terraform_wrapper: false + + - name: Install Lambda dependencies + working-directory: lambdas + run: yarn install --frozen-lockfile + + - name: Build smoke-test Lambda distributions + working-directory: lambdas + run: | + yarn workspace @aws-github-runner/webhook dist + yarn workspace @aws-github-runner/control-plane dist + + - name: Start MockServer + id: mockserver + uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0 + with: + version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290' + port: '1080' + startup-timeout: '60' + + - name: Connect MockServer to MiniStack network + shell: bash + run: | + set -euo pipefail + ministack_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 ~ /ministack/ {print $1; exit}')" + network="$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$ministack_container" | sed -n '1p')" + docker network connect --alias mockserver "$network" mockserver + + - name: Mark repository as safe + shell: sh + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Run scale-set ECS/MockServer smoke test + env: + MINISTACK_GITHUB_MOCK_HOST: mockserver + MINISTACK_GITHUB_MOCK_PORT: "1080" + MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} + run: sh tests/ministack/run-scale-set-integration.sh diff --git a/examples/multi-runner-scale-set/README.md b/examples/multi-runner-scale-set/README.md index 167ae7f275..21b502fe7b 100644 --- a/examples/multi-runner-scale-set/README.md +++ b/examples/multi-runner-scale-set/README.md @@ -71,8 +71,8 @@ The GitHub App must be installed for the configured GitHub account. |------|-------------|------|---------|:--------:| | [ami](#input\_ami) | Optional AMI configuration keyed by runner lane. |
map(object({
filter = optional(map(list(string)), { state = ["available"] })
owners = optional(list(string), ["amazon"])
id_ssm_parameter = optional(object({
arn = string
}), null)
kms_key = optional(object({
arn = string
}), null)
}))
| `{}` | no | | [aws\_region](#input\_aws\_region) | AWS region to deploy to. | `string` | `"eu-west-1"` | no | -| [environment](#input\_environment) | Environment name, used as prefix. | `string` | `null` | no | -| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. |
object({
config_url = optional(string, null)
ssl_verify = optional(bool, true)
runner_owner = optional(string, null)
registration_level = optional(string, "organization")
})
| `{}` | no | +| [environment](#input\_environment) | Environment name, used as prefix. | `string` | n/a | yes | +| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. |
object({
url = optional(string, null)
ssl_verify = optional(bool, true)
runner_owner = optional(string, null)
registration_level = optional(string, "organization")
})
| `{}` | no | | [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. |
object({
id = string
key_base64 = string
installation_id = optional(string, null)
})
| n/a | yes | | [runner\_binaries\_enabled](#input\_runner\_binaries\_enabled) | Whether runner binary synchronization is enabled. | `bool` | `true` | no | | [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. |
object({
name = string
runner_group_name = optional(string, "Default")
min_runners = optional(number, 0)
container = optional(object({
image = optional(string, null)
}), {})
})
| n/a | yes | diff --git a/mockserver/initializerJson.json b/mockserver/initializerJson.json new file mode 100644 index 0000000000..443ddb6201 --- /dev/null +++ b/mockserver/initializerJson.json @@ -0,0 +1,138 @@ +[ + { + "httpRequest": { + "method": "POST", + "path": "/api/v3/app/installations/456/access_tokens" + }, + "httpResponse": { + "statusCode": 201, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"token\":\"fake-installation-token\",\"expires_at\":\"2099-01-01T00:00:00Z\"}" + } + }, + { + "httpRequest": { + "method": "POST", + "path": "/api/v3/orgs/example/actions/runners/registration-token" + }, + "httpResponse": { + "statusCode": 201, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"token\":\"fake-registration-token\"}" + } + }, + { + "httpRequest": { + "method": "POST", + "path": "/api/v3/actions/runner-registration" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"url\":\"https://mockserver:1080/tenant/123\",\"token\":\"header.eyJleHAiOjQwNzA5MDg4MDB9.signature\"}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/tenant/123/_apis/runtime/runnergroups/" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"count\":1,\"value\":[{\"id\":48,\"name\":\"experimental-euw1-sl-cicd-forge-emu\"}]}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/tenant/123/_apis/runtime/runnerscalesets" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"count\":1,\"value\":[{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}]}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}" + } + }, + { + "httpRequest": { + "method": "PATCH", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"},{\"name\":\"linux\",\"type\":\"user\"},{\"name\":\"scale-set\",\"type\":\"user\"},{\"name\":\"self-hosted\",\"type\":\"user\"},{\"name\":\"x64\",\"type\":\"user\"}],\"runnerSetting\":{}}" + } + }, + { + "httpRequest": { + "method": "POST", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223/generatejitconfig" + }, + "httpResponseTemplate": { + "templateType": "MUSTACHE", + "template": "{\"statusCode\":200,\"headers\":{\"Content-Type\":[\"application/json\"]},\"body\":\"{\\\"runner\\\":{\\\"id\\\":321,\\\"name\\\":\\\"{{#jsonPath}}$.name{{/jsonPath}}{{jsonPathResult}}\\\",\\\"runnerScaleSetId\\\":223},\\\"encodedJITConfig\\\":\\\"test-only-jit-configuration\\\"}\"}" + } + }, + { + "httpRequest": { + "method": "POST", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223/sessions" + }, + "httpResponse": { + "statusCode": 200, + "headers": { "Content-Type": ["application/json"] }, + "body": "{\"sessionId\":\"11111111-1111-1111-1111-111111111111\",\"ownerName\":\"local.medium\",\"runnerScaleSet\":{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48},\"messageQueueUrl\":\"https://mockserver:1080/messages?sessionId=11111111-1111-1111-1111-111111111111&api-version=6.0-preview\",\"messageQueueAccessToken\":\"fake-queue-token\",\"statistics\":{\"totalAvailableJobs\":0,\"totalAcquiredJobs\":0,\"totalAssignedJobs\":0,\"totalRunningJobs\":0,\"totalRegisteredRunners\":0,\"totalBusyRunners\":0,\"totalIdleRunners\":0}}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/messages" + }, + "httpResponse": { + "statusCode": 202, + "delay": { "timeUnit": "MILLISECONDS", "value": 1000 } + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/tenant/123/_apis/distributedtask/pools/0/agents" + }, + "httpResponseTemplate": { + "templateType": "MUSTACHE", + "template": "{\"statusCode\":200,\"headers\":{\"Content-Type\":[\"application/json\"]},\"body\":\"{\\\"count\\\":1,\\\"value\\\":[{\\\"id\\\":321,\\\"name\\\":\\\"{{ request.queryStringParameters.agentName.0 }}\\\",\\\"runnerScaleSetId\\\":223}]}\"}" + } + }, + { + "httpRequest": { + "method": "DELETE", + "path": "/tenant/123/_apis/distributedtask/pools/0/agents/321" + }, + "httpResponse": { + "statusCode": 204 + } + }, + { + "httpRequest": { + "method": "DELETE", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223/sessions/11111111-1111-1111-1111-111111111111" + }, + "httpResponse": { + "statusCode": 204 + } + } +] diff --git a/tests/ministack/multi-runner-scale-set.tfvars b/tests/ministack/multi-runner-scale-set.tfvars index 6a47126ad6..7dd1a0ad39 100644 --- a/tests/ministack/multi-runner-scale-set.tfvars +++ b/tests/ministack/multi-runner-scale-set.tfvars @@ -54,4 +54,4 @@ scale_set = { container = { image = "localhost:4566/scale-set-controller:smoke" } -} \ No newline at end of file +} diff --git a/tests/ministack/run-scale-set-integration.sh b/tests/ministack/run-scale-set-integration.sh new file mode 100755 index 0000000000..b71440ad9e --- /dev/null +++ b/tests/ministack/run-scale-set-integration.sh @@ -0,0 +1,600 @@ +#!/bin/sh + +set -eu + +export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-000000000000}" +export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-test-only}" +export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-eu-west-1}" +export AWS_REGION="${AWS_REGION:-eu-west-1}" +export AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://127.0.0.1:4566}" +export AWS_EC2_METADATA_DISABLED="${AWS_EC2_METADATA_DISABLED:-true}" + +script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +source_root=$(CDPATH='' cd -- "$script_dir/../.." && pwd) +example="multi-runner-scale-set" +base_tfvars="$script_dir/$example.tfvars" +tfvars_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set.XXXXXX") +app_key_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-key.XXXXXX") +log_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-logs.XXXXXX") +controller_log_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-controller-logs.XXXXXX") +config_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-config.XXXXXX") +task_definition_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-task-definition.XXXXXX") +instance_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-instances.XXXXXX") +scale_down_config_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-scale-down-config.XXXXXX") +initializer_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-mockserver.XXXXXX") +repository_name="scale-set-controller" +image_reference="localhost:4566/${repository_name}:smoke" +mockserver_host="${MINISTACK_GITHUB_MOCK_HOST:-127.0.0.1}" +mockserver_port="${MINISTACK_GITHUB_MOCK_PORT:-1080}" +mockserver_url="${MINISTACK_GITHUB_MOCK_URL:-http://127.0.0.1:${mockserver_port}}" +controller_mock_url="https://${mockserver_host}:${mockserver_port}" +terraform_state_exists=false + +cleanup() { + cleanup_status=$? + set +e + + if [ "$terraform_state_exists" = true ]; then + "$source_root/tests/ministack/run-example.sh" destroy "$example" "$tfvars_file" >/dev/null 2>&1 + fi + + rm -f "$tfvars_file" "$app_key_file" "$log_file" "$controller_log_file" "$config_file" "$task_definition_file" "$instance_file" "$scale_down_config_file" "$initializer_file" + exit "$cleanup_status" +} +trap cleanup EXIT INT TERM + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "$1 is required to run the scale-set MiniStack smoke test." >&2 + exit 69 + fi +} + +for command in aws curl docker openssl python3 terraform; do + require_command "$command" +done + +ministack_aws() { + aws --endpoint-url "$AWS_ENDPOINT_URL" --region "$AWS_DEFAULT_REGION" "$@" +} + +wait_for_http() { + url="$1" + attempts=90 + while ! curl -fsS --max-time 2 "$url" >/dev/null 2>&1; do + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for $url." >&2 + exit 70 + fi + sleep 1 + done +} + +wait_for_scale_set_runner() { + attempts=90 + while :; do + ministack_aws ec2 describe-instances --output json > "$instance_file" + instance_id=$(python3 - "$instance_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as instance_file: + response = json.load(instance_file) + +required_tags = { + "ghr:Application": "github-action-runner", + "ghr:created_by": "scale-set-service", + "ghr:environment": "ministack-scale-set-linux-scale-set", + "ghr:Type": "Org", + "ghr:Owner": "example", + "ghr:scale_set_state": "config-published", + "ghr:github_runner_id": "321", +} +active_states = {"pending", "running", "stopping", "stopped", "shutting-down"} +matches = [] +for reservation in response.get("Reservations", []): + for instance in reservation.get("Instances", []): + if instance.get("State", {}).get("Name") not in active_states: + continue + tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])} + if all(tags.get(key) == value for key, value in required_tags.items()) and tags.get("ghr:runner_name", "").startswith("scale-set-"): + matches.append(instance["InstanceId"]) + +if len(matches) == 1: + print(matches[0]) +PY +) + if [ -n "$instance_id" ]; then + printf ' [PASS] MiniStack created and registered scale-set EC2 runner %s\n' "$instance_id" + return 0 + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for the scale-set EC2 runner to reach config-published state." >&2 + cat "$instance_file" >&2 || true + exit 1 + fi + sleep 2 + done +} + +wait_for_no_scale_set_runners() { + attempts=90 + while :; do + ministack_aws ec2 describe-instances --output json > "$instance_file" + active_count=$(python3 - "$instance_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as instance_file: + response = json.load(instance_file) + +required_tags = { + "ghr:Application": "github-action-runner", + "ghr:created_by": "scale-set-service", + "ghr:environment": "ministack-scale-set-linux-scale-set", + "ghr:Type": "Org", + "ghr:Owner": "example", +} +active_states = {"pending", "running", "stopping", "stopped", "shutting-down"} +count = 0 +for reservation in response.get("Reservations", []): + for instance in reservation.get("Instances", []): + if instance.get("State", {}).get("Name") not in active_states: + continue + tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])} + if all(tags.get(key) == value for key, value in required_tags.items()): + count += 1 +print(count) +PY +) + if [ "$active_count" = "0" ]; then + printf '%s\n' ' [PASS] scale-set EC2 runner was terminated after the minimum changed to zero' + return 0 + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for the scale-set EC2 runner to terminate; active count is $active_count." >&2 + cat "$instance_file" >&2 || true + exit 1 + fi + sleep 2 + done +} + +wait_for_http "$AWS_ENDPOINT_URL/_ministack/health" +attempts=90 +while ! curl -fsS --max-time 2 -X PUT "$mockserver_url/mockserver/status" >/dev/null 2>&1; do + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for $mockserver_url/mockserver/status." >&2 + exit 70 + fi + sleep 1 +done +curl -fsS -X PUT "$mockserver_url/mockserver/reset" >/dev/null +CONTROLLER_MOCK_URL="$controller_mock_url" python3 - "$source_root/mockserver/initializerJson.json" "$initializer_file" <<'PY' +import os +import sys + +source, destination = sys.argv[1:] +with open(source, encoding="utf-8") as source_file: + fixture = source_file.read() +fixture = fixture.replace("https://mockserver:1080", os.environ["CONTROLLER_MOCK_URL"]) +with open(destination, "w", encoding="utf-8") as destination_file: + destination_file.write(fixture) +PY +curl -fsS -X PUT \ + "$mockserver_url/mockserver/expectation" \ + -H 'Content-Type: application/json' \ + --data-binary "@$initializer_file" \ + >/dev/null + +repository_policy=$(python3 - <<'PY' +import json + +print(json.dumps({ + "Version": "2012-10-17", + "Statement": [{ + "Sid": "AllowAccountPull", + "Effect": "Allow", + "Principal": {"AWS": "arn:aws:iam::000000000000:root"}, + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + ], + }], +})) +PY +) + +ministack_aws ecr create-repository \ + --repository-name "$repository_name" \ + --image-tag-mutability IMMUTABLE \ + --image-scanning-configuration scanOnPush=false \ + >/dev/null +ministack_aws ecr set-repository-policy \ + --repository-name "$repository_name" \ + --policy-text "$repository_policy" \ + >/dev/null + +docker build \ + --target runtime \ + --file "$source_root/lambdas/services/scale-set/Dockerfile" \ + --tag "$image_reference" \ + "$source_root" + +ministack_aws ecr get-login-password | docker login \ + --username AWS \ + --password-stdin localhost:4566 >/dev/null +docker push "$image_reference" + +ministack_aws ecr describe-images \ + --repository-name "$repository_name" \ + --image-ids imageTag=smoke \ + >/dev/null + +openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$app_key_file" 2>/dev/null +app_key_base64=$(base64 < "$app_key_file" | tr -d '\n') +APP_KEY_BASE64="$app_key_base64" GITHUB_CONFIG_URL="$controller_mock_url" python3 - "$base_tfvars" "$tfvars_file" <<'PY' +import os +import sys + +source, destination = sys.argv[1:] +replacement = os.environ["APP_KEY_BASE64"] +github_config_url = os.environ["GITHUB_CONFIG_URL"] +with open(source, encoding="utf-8") as source_file: + lines = source_file.readlines() +with open(destination, "w", encoding="utf-8") as destination_file: + for line in lines: + if line.lstrip().startswith("key_base64"): + destination_file.write(f' key_base64 = "{replacement}"\n') + elif line.lstrip().startswith("config_url"): + destination_file.write(f' config_url = "{github_config_url}"\n') + else: + destination_file.write(line) +PY +unset app_key_base64 APP_KEY_BASE64 + +terraform_state_exists=true +"$source_root/tests/ministack/run-example.sh" apply "$example" "$tfvars_file" + +config_path="/ministack-scale-set/scale-set-controller/linux-scale-set/linux-scale-set" +ministack_aws ssm get-parameter \ + --name "$config_path" \ + --query 'Parameter.Value' \ + --output text > "$config_file" +EXPECTED_GITHUB_CONFIG_URL="$controller_mock_url/example" python3 - "$config_file" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as config_file: + config = json.load(config_file) + +assert config["githubConfigUrl"] == os.environ["EXPECTED_GITHUB_CONFIG_URL"] +assert config["forceGhes"] is True +assert config["sslVerify"] is False +assert config["minRunners"] == 1 +assert config["githubApp"]["appIdParameterName"] +assert config["githubApp"]["installationIdParameterName"] +assert config["githubApp"]["privateKeyParameterName"] +print(" [PASS] SSM manifest has the expected MockServer and GitHub App settings") +PY + +task_definition=$(ministack_aws ecs list-task-definitions \ + --family-prefix ministack-scale-set-ss-linux-scale-se- \ + --sort DESC \ + --query 'taskDefinitionArns[0]' \ + --output text) +if [ -z "$task_definition" ] || [ "$task_definition" = "None" ]; then + echo "The scale-set task definition was not registered." >&2 + exit 1 +fi + +actual_image=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.containerDefinitions[?name==`scale-set-controller`].image | [0]' \ + --output text) +if [ "$actual_image" != "$image_reference" ]; then + echo "Expected the ECS task to use $image_reference, got $actual_image." >&2 + exit 1 +fi +printf '%s\n' ' [PASS] ECS task definition uses the image pushed to MiniStack ECR' + +log_driver=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.containerDefinitions[?name==`scale-set-controller`].logConfiguration.logDriver | [0]' \ + --output text) +if [ "$log_driver" != "awslogs" ]; then + echo "Expected the ECS task to request the awslogs driver, got $log_driver." >&2 + exit 1 +fi +printf '%s\n' ' [PASS] ECS task definition requests the awslogs driver' + +log_group=$(ministack_aws logs describe-log-groups \ + --log-group-name-prefix "/aws/ecs/ministack-scale-set-ss-linux-scale-se-" \ + --query 'logGroups[0].logGroupName' \ + --output text) +if [ -z "$log_group" ] || [ "$log_group" = "None" ]; then + echo "The scale-set CloudWatch log group was not created." >&2 + exit 1 +fi +printf '%s\n' ' [PASS] CloudWatch log group was created for the ECS controller' + +task_family=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.family' \ + --output text) + +# MiniStack exposes ECS credentials through the gateway's container IP. The +# Node.js AWS SDK intentionally rejects that address in +# AWS_CONTAINER_CREDENTIALS_FULL_URI because non-HTTPS full URIs are limited +# to loopback and the real ECS metadata address. Use only synthetic, +# test-scoped credentials in a temporary task-definition revision so the +# smoke test still exercises the pushed image, ECS service, and controller +# lifecycle without changing the production task definition or application. +ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition' \ + --output json > "$task_definition_file" +TASK_DEFINITION_FILE="$task_definition_file" python3 - <<'PY' +import json +import os + +path = os.environ["TASK_DEFINITION_FILE"] +with open(path, encoding="utf-8") as task_definition_file: + task_definition = json.load(task_definition_file) + +for field in ( + "taskDefinitionArn", + "revision", + "status", + "requiresAttributes", + "compatibilities", + "registeredAt", + "registeredBy", +): + task_definition.pop(field, None) + +for container in task_definition["containerDefinitions"]: + if container["name"] != "scale-set-controller": + continue + environment = container.setdefault("environment", []) + environment.extend([ + {"name": "AWS_ACCESS_KEY_ID", "value": "000000000000"}, + {"name": "AWS_SECRET_ACCESS_KEY", "value": "test-only"}, + ]) + +with open(path, "w", encoding="utf-8") as task_definition_file: + json.dump(task_definition, task_definition_file) +PY + +environment_name=$(sed -n 's/^environment[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$base_tfvars") +cluster_name="${environment_name}-scale-set" +task_definition=$(ministack_aws ecs register-task-definition \ + --cli-input-json "file://$task_definition_file" \ + --query 'taskDefinition.taskDefinitionArn' \ + --output text) +ministack_aws ecs update-service \ + --cluster "$cluster_name" \ + --service "$task_family" \ + --task-definition "$task_definition" \ + >/dev/null + +task_revision=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.revision' \ + --output text) +controller_container="" +attempts=90 +while [ -z "$controller_container" ]; do + controller_container=$(docker ps -a \ + --filter "label=com.amazonaws.ecs.task-definition-family=$task_family" \ + --filter "label=com.amazonaws.ecs.task-definition-version=$task_revision" \ + --filter 'name=scale-set-controller' \ + --format '{{.ID}}' | sed -n '1p') + if [ -n "$controller_container" ]; then + break + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for the MiniStack ECS controller container." >&2 + docker ps -a --format '{{.ID}} {{.Image}} {{.Status}} {{.Names}}' >&2 || true + exit 1 + fi + sleep 2 +done +printf ' [PASS] MiniStack started ECS controller container %s\n' "$controller_container" + +wait_for_controller_log_event() { + marker="$1" + required_text="${2:-}" + attempts=90 + while :; do + docker logs "$controller_container" > "$controller_log_file" 2>&1 || true + if python3 - "$controller_log_file" "$marker" "$required_text" <<'PY' +import sys + +marker, required = sys.argv[2:] +with open(sys.argv[1], encoding="utf-8") as log_file: + messages = log_file.read().splitlines() +if any(marker in message and (not required or required in message) for message in messages): + raise SystemExit(0) +raise SystemExit(1) +PY + then + printf ' [PASS] CloudWatch logs contain %s\n' "$marker" + return 0 + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for controller log marker '$marker'." >&2 + cat "$controller_log_file" >&2 || true + echo "Controller AWS/ECS metadata environment:" >&2 + docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" \ + | sed -E 's/^(AWS_CONTAINER_AUTHORIZATION_TOKEN|AWS_CONTAINER_CREDENTIALS_FULL_URI)=.*/\1=/' \ + | grep -E '^(AWS_|ECS_)' >&2 || true + echo "Controller network attachments:" >&2 + docker inspect --format '{{json .NetworkSettings.Networks}}' "$controller_container" >&2 || true + echo "Controller credential endpoint probe:" >&2 + docker exec "$controller_container" node -e \ + 'fetch(process.env.AWS_CONTAINER_CREDENTIALS_FULL_URI, {headers: {Authorization: process.env.AWS_CONTAINER_AUTHORIZATION_TOKEN}}).then((response) => { console.error(`status=${response.status}`); process.exit(response.ok ? 0 : 1); }).catch((error) => { console.error(`${error.name}:${error.message}`); process.exit(1); })' \ + >&2 || true + exit 1 + fi + sleep 2 + done +} + +printf '%s\n' ' [INFO] MiniStack 1.5.12 does not emit ECS awslogs streams; validating controller runtime logs instead' +wait_for_controller_log_event 'scale_set_controller_started' +wait_for_controller_log_event 'scale_set_session_created' +wait_for_controller_log_event 'scale_set_reconciled' '"desiredRunners":1' +wait_for_controller_log_event 'scale_set_reconciled' '"status":"converged"' +wait_for_scale_set_runner + +wait_for_mock_route() { + method="$1" + route="$2" + body=$(REQUEST_METHOD="$method" REQUEST_PATH="$route" python3 - <<'PY' +import json +import os + +print(json.dumps({ + "httpRequest": { + "method": os.environ["REQUEST_METHOD"], + "path": os.environ["REQUEST_PATH"], + }, + "times": {"atLeast": 1}, +})) +PY + ) + attempts=45 + while ! curl -fsS --max-time 5 -X PUT \ + http://127.0.0.1:1080/mockserver/verify \ + -H 'Content-Type: application/json' \ + --data "$body" >/dev/null 2>&1; do + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for MockServer route: $method $route" >&2 + curl -sS --max-time 5 \ + 'http://127.0.0.1:1080/mockserver/retrieve?type=REQUESTS&format=JSON' >&2 || true + exit 1 + fi + sleep 2 + done + printf ' [PASS] MockServer received %s %s\n' "$method" "$route" +} + +wait_for_mock_route POST '/api/v3/app/installations/456/access_tokens' +wait_for_mock_route POST '/api/v3/orgs/example/actions/runners/registration-token' +wait_for_mock_route POST '/api/v3/actions/runner-registration' +wait_for_mock_route GET '/tenant/123/_apis/runtime/runnergroups/' +wait_for_mock_route GET '/tenant/123/_apis/runtime/runnerscalesets' +wait_for_mock_route GET '/tenant/123/_apis/runtime/runnerscalesets/223' +wait_for_mock_route PATCH '/tenant/123/_apis/runtime/runnerscalesets/223' +wait_for_mock_route POST '/tenant/123/_apis/runtime/runnerscalesets/223/generatejitconfig' +wait_for_mock_route POST '/tenant/123/_apis/runtime/runnerscalesets/223/sessions' +wait_for_mock_route GET '/messages' + +python3 - "$config_file" "$scale_down_config_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as config_file: + reconciler = json.load(config_file) + +assert reconciler["minRunners"] == 1 +reconciler["minRunners"] = 0 +with open(sys.argv[2], "w", encoding="utf-8") as config_file: + json.dump(reconciler, config_file) +PY +ministack_aws ssm put-parameter \ + --name "$config_path" \ + --type String \ + --value "$(cat "$scale_down_config_file")" \ + --overwrite \ + >/dev/null +printf '%s\n' ' [PASS] SSM manifest minimum changed from one runner to zero' + +SCALE_DOWN_CONFIG_FILE="$scale_down_config_file" TASK_DEFINITION_FILE="$task_definition_file" python3 - <<'PY' +import json +import os + +with open(os.environ["SCALE_DOWN_CONFIG_FILE"], encoding="utf-8") as config_file: + reconciler = json.load(config_file) +with open(os.environ["TASK_DEFINITION_FILE"], encoding="utf-8") as task_definition_file: + task_definition = json.load(task_definition_file) + +for container in task_definition["containerDefinitions"]: + if container["name"] != "scale-set-controller": + continue + for environment in container.get("environment", []): + if environment["name"] == "SCALE_SET_CONTROLLER_MANIFEST": + manifest = json.loads(environment["value"]) + assert len(manifest.get("reconcilers", [])) == 1 + manifest["reconcilers"][0]["minRunners"] = reconciler["minRunners"] + environment["value"] = json.dumps(manifest, separators=(",", ":")) + break + else: + raise RuntimeError("scale-set controller task definition has no inline manifest") + +with open(os.environ["TASK_DEFINITION_FILE"], "w", encoding="utf-8") as task_definition_file: + json.dump(task_definition, task_definition_file) +PY +task_definition=$(ministack_aws ecs register-task-definition \ + --cli-input-json "file://$task_definition_file" \ + --query 'taskDefinition.taskDefinitionArn' \ + --output text) +task_revision=$(ministack_aws ecs describe-task-definition \ + --task-definition "$task_definition" \ + --query 'taskDefinition.revision' \ + --output text) +ministack_aws ecs update-service \ + --cluster "$cluster_name" \ + --service "$task_family" \ + --task-definition "$task_definition" \ + --force-new-deployment \ + >/dev/null + +old_controller_container="$controller_container" +new_controller_container="" +attempts=90 +while [ -z "$new_controller_container" ]; do + for candidate in $(docker ps -a \ + --filter "label=com.amazonaws.ecs.task-definition-family=$task_family" \ + --filter "label=com.amazonaws.ecs.task-definition-version=$task_revision" \ + --filter 'name=scale-set-controller' \ + --format '{{.ID}}'); do + if [ "$candidate" != "$old_controller_container" ]; then + new_controller_container="$candidate" + break + fi + done + if [ -n "$new_controller_container" ]; then + break + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo 'Timed out waiting for the ECS service to deploy the scale-down task.' >&2 + docker ps -a --format '{{.ID}} {{.Image}} {{.Status}} {{.Names}}' >&2 || true + exit 1 + fi + sleep 2 +done +controller_container="$new_controller_container" +printf ' [PASS] ECS service deployed a fresh controller container %s for scale-down\n' "$controller_container" +wait_for_controller_log_event 'scale_set_controller_started' +wait_for_controller_log_event 'scale_set_session_created' +wait_for_controller_log_event 'scale_set_reconciled' '"desiredRunners":0' +wait_for_controller_log_event 'scale_set_reconciled' '"status":"converged"' +wait_for_no_scale_set_runners + +"$source_root/tests/ministack/run-example.sh" destroy "$example" "$tfvars_file" +terraform_state_exists=false +wait_for_mock_route DELETE '/tenant/123/_apis/runtime/runnerscalesets/223/sessions/11111111-1111-1111-1111-111111111111' + +echo 'Scale-set MiniStack ECS/MockServer smoke test passed.'