test(scale-set): run service container smoke test

.github/scripts/scale-set-container-smoke-test.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+container_name="scale-set-service-smoke-${GITHUB_RUN_ID:-$$}"
+response_file="$(mktemp)"
+# shellcheck disable=SC2329 # cleanup is invoked indirectly by the EXIT trap.
+cleanup() {
+  docker rm -f "$container_name" >/dev/null 2>&1 || true
+  rm -f "$response_file"
+}
+trap cleanup EXIT
+
+scale_set_controller_manifest='{"version":1,"groupName":"ci-smoke","revision":"image-test","reconcilers":[{"schemaVersion":1,"runnerConfigName":"smoke","scaleSetId":1,"scaleSetName":"ci-smoke","githubConfigUrl":"https://github.com/example-org","githubApp":{"appIdParameterName":"/ci/app-id","privateKeyParameterName":"/ci/private-key"},"computeProvider":{"type":"ec2","configuration":{"region":"us-east-1","environment":"ci","runnerNamePrefix":"ci","jitConfigParameterPath":"/ci/jit","subnets":["subnet-00000000"],"launchTemplateName":"ci","ec2instanceCriteria":{"instanceTypes":["t3.micro"],"targetCapacityType":"on-demand","instanceAllocationStrategy":"lowest-price"}}},"minRunners":0,"maxRunners":0}]}'
+
+docker run --detach \
+  --name "$container_name" \
+  --network none \
+  --read-only \
+  --cap-drop ALL \
+  --security-opt no-new-privileges \
+  --env AWS_REGION=us-east-1 \
+  --env AWS_EC2_METADATA_DISABLED=true \
+  --env SCALE_SET_HEALTH_PORT=8080 \
+  --env "SCALE_SET_CONTROLLER_MANIFEST=$scale_set_controller_manifest" \
+  scale-set-service:smoke-test
+
+attempt=0
+while (( attempt < 30 )); do
+  ((attempt += 1))
+  docker exec "$container_name" node --input-type=module -e \
+    'const response = await fetch("http://127.0.0.1:8080/healthz", { signal: AbortSignal.timeout(1000) }); process.stdout.write(JSON.stringify({ status: response.status, body: await response.json() }));' \
+    >"$response_file" 2>/dev/null || true
+  if jq --exit-status \
+    --arg group_name ci-smoke \
+    '(.status == 200 or .status == 503) and .body.groupName == $group_name and (.body.live | type == "boolean") and (.body.ready | type == "boolean") and .body.reconcilers.smoke != null' \
+    "$response_file" >/dev/null 2>&1; then
+    jq . "$response_file"
+    exit 0
+  fi
+  sleep 1
+done
+
+docker logs "$container_name"
+echo "scale-set service image did not return the expected health response" >&2
+exit 1

.github/workflows/lambda.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     paths:
       - 'lambdas/**'
+      - '.github/scripts/**'
       - '.github/workflows/lambda.yml'
 
 concurrency:
@@ -85,3 +86,16 @@ jobs:
           push: false
           cache-from: type=gha,scope=scale-set-service
           cache-to: type=gha,mode=max,scope=scale-set-service
+
+      - name: Build scale-set service image for smoke test
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: ./lambdas/services/scale-set/Dockerfile
+          platforms: linux/amd64
+          load: true
+          tags: scale-set-service:smoke-test
+          cache-from: type=gha,scope=scale-set-service
+
+      - name: Run scale-set service image smoke test
+        run: ./.github/scripts/scale-set-container-smoke-test.sh

.github/workflows/ministack.yml

@@ -196,3 +196,78 @@ jobs:
           MINISTACK_GITHUB_MOCK_PORT: "1080"
           MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }}
         run: sh tests/ministack/run-smoke.sh
+
+  integration_scaleset_smoke:
+    name: Run scale-set ECS smoke test against MiniStack and MockServer
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    services:
+      ministack:
+        image: ghcr.io/ministackorg/ministack:1.5.12@sha256:41fe1ce2e666c6cc410c6047a9db8bf1df69cd0028ebc0a6c6e5517c3a83d6e0
+        ports:
+          - 4566:4566
+        options: >-
+          --add-host=host.docker.internal:host-gateway
+          --volume /var/run/docker.sock:/var/run/docker.sock
+        env:
+          MINISTACK_ACCOUNT_ID: "000000000000"
+          MINISTACK_REGION: eu-west-1
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: lambdas/.nvmrc
+          package-manager-cache: false
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1
+        with:
+          terraform_version: latest
+          terraform_wrapper: false
+
+      - name: Install Lambda dependencies
+        working-directory: lambdas
+        run: yarn install --frozen-lockfile
+
+      - name: Build smoke-test Lambda distributions
+        working-directory: lambdas
+        run: |
+          yarn workspace @aws-github-runner/webhook dist
+          yarn workspace @aws-github-runner/control-plane dist
+
+      - name: Start MockServer
+        id: mockserver
+        uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0
+        with:
+          version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290'
+          port: '1080'
+          startup-timeout: '60'
+
+      - name: Connect MockServer to MiniStack network
+        shell: bash
+        run: |
+          set -euo pipefail
+          ministack_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 ~ /ministack/ {print $1; exit}')"
+          network="$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$ministack_container" | sed -n '1p')"
+          docker network connect --alias mockserver "$network" mockserver
+
+      - name: Mark repository as safe
+        shell: sh
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: Run scale-set ECS/MockServer smoke test
+        env:
+          MINISTACK_GITHUB_MOCK_HOST: mockserver
+          MINISTACK_GITHUB_MOCK_PORT: "1080"
+          MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }}
+        run: sh tests/ministack/run-scale-set-integration.sh        

examples/multi-runner-scale-set/README.md

@@ -71,8 +71,8 @@ The GitHub App must be installed for the configured GitHub account.
 |------|-------------|------|---------|:--------:|
 | <a name="input_ami"></a> [ami](#input\_ami) | Optional AMI configuration keyed by runner lane. | <pre>map(object({<br/>    filter = optional(map(list(string)), { state = ["available"] })<br/>    owners = optional(list(string), ["amazon"])<br/>    id_ssm_parameter = optional(object({<br/>      arn = string<br/>    }), null)<br/>    kms_key = optional(object({<br/>      arn = string<br/>    }), null)<br/>  }))</pre> | `{}` | no |
 | <a name="input_aws_region"></a> [aws\_region](#input\_aws\_region) | AWS region to deploy to. | `string` | `"eu-west-1"` | no |
-| <a name="input_environment"></a> [environment](#input\_environment) | Environment name, used as prefix. | `string` | `null` | no |
-| <a name="input_github"></a> [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | <pre>object({<br/>    config_url         = optional(string, null)<br/>    ssl_verify         = optional(bool, true)<br/>    runner_owner       = optional(string, null)<br/>    registration_level = optional(string, "organization")<br/>  })</pre> | `{}` | no |
+| <a name="input_environment"></a> [environment](#input\_environment) | Environment name, used as prefix. | `string` | n/a | yes |
+| <a name="input_github"></a> [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. | <pre>object({<br/>    url                = optional(string, null)<br/>    ssl_verify         = optional(bool, true)<br/>    runner_owner       = optional(string, null)<br/>    registration_level = optional(string, "organization")<br/>  })</pre> | `{}` | no |
 | <a name="input_github_app"></a> [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. | <pre>object({<br/>    id              = string<br/>    key_base64      = string<br/>    installation_id = optional(string, null)<br/>  })</pre> | n/a | yes |
 | <a name="input_runner_binaries_enabled"></a> [runner\_binaries\_enabled](#input\_runner\_binaries\_enabled) | Whether runner binary synchronization is enabled. | `bool` | `true` | no |
 | <a name="input_scale_set"></a> [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. | <pre>object({<br/>    name              = string<br/>    runner_group_name = optional(string, "Default")<br/>    min_runners       = optional(number, 0)<br/>    container = optional(object({<br/>      image = optional(string, null)<br/>    }), {})<br/>  })</pre> | n/a | yes |

mockserver/initializerJson.json

@@ -0,0 +1,138 @@
+[
+  {
+    "httpRequest": {
+      "method": "POST",
+      "path": "/api/v3/app/installations/456/access_tokens"
+    },
+    "httpResponse": {
+      "statusCode": 201,
+      "headers": { "Content-Type": ["application/json"] },
+      "body": "{\"token\":\"fake-installation-token\",\"expires_at\":\"2099-01-01T00:00:00Z\"}"
+    }
+  },
+  {
+    "httpRequest": {
+      "method": "POST",
+      "path": "/api/v3/orgs/example/actions/runners/registration-token"
+    },
+    "httpResponse": {
+      "statusCode": 201,
+      "headers": { "Content-Type": ["application/json"] },
+      "body": "{\"token\":\"fake-registration-token\"}"
+    }
+  },
+  {
+    "httpRequest": {
+      "method": "POST",
+      "path": "/api/v3/actions/runner-registration"
+    },
+    "httpResponse": {
+      "statusCode": 200,
+      "headers": { "Content-Type": ["application/json"] },
+      "body": "{\"url\":\"https://mockserver:1080/tenant/123\",\"token\":\"header.eyJleHAiOjQwNzA5MDg4MDB9.signature\"}"
+    }
+  },
+  {
+    "httpRequest": {
+      "method": "GET",
+      "path": "/tenant/123/_apis/runtime/runnergroups/"
+    },
+    "httpResponse": {
+      "statusCode": 200,
+      "headers": { "Content-Type": ["application/json"] },
+      "body": "{\"count\":1,\"value\":[{\"id\":48,\"name\":\"experimental-euw1-sl-cicd-forge-emu\"}]}"
+    }
+  },
+  {
+    "httpRequest": {
+      "method": "GET",
+      "path": "/tenant/123/_apis/runtime/runnerscalesets"
+    },
+    "httpResponse": {
+      "statusCode": 200,
+      "headers": { "Content-Type": ["application/json"] },
+      "body": "{\"count\":1,\"value\":[{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}]}"
+    }
+  },
+  {
+    "httpRequest": {
+      "method": "GET",
+      "path": "/tenant/123/_apis/runtime/runnerscalesets/223"
+    },
+    "httpResponse": {
+      "statusCode": 200,
+      "headers": { "Content-Type": ["application/json"] },
+      "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}"
+    }
+  },
+  {
+    "httpRequest": {
+      "method": "PATCH",
+      "path": "/tenant/123/_apis/runtime/runnerscalesets/223"
+    },
+    "httpResponse": {
+      "statusCode": 200,
+      "headers": { "Content-Type": ["application/json"] },
+      "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"},{\"name\":\"linux\",\"type\":\"user\"},{\"name\":\"scale-set\",\"type\":\"user\"},{\"name\":\"self-hosted\",\"type\":\"user\"},{\"name\":\"x64\",\"type\":\"user\"}],\"runnerSetting\":{}}"
+    }
+  },
+  {
+    "httpRequest": {
+      "method": "POST",
+      "path": "/tenant/123/_apis/runtime/runnerscalesets/223/generatejitconfig"
+    },
+    "httpResponseTemplate": {
+      "templateType": "MUSTACHE",
+      "template": "{\"statusCode\":200,\"headers\":{\"Content-Type\":[\"application/json\"]},\"body\":\"{\\\"runner\\\":{\\\"id\\\":321,\\\"name\\\":\\\"{{#jsonPath}}$.name{{/jsonPath}}{{jsonPathResult}}\\\",\\\"runnerScaleSetId\\\":223},\\\"encodedJITConfig\\\":\\\"test-only-jit-configuration\\\"}\"}"
+    }
+  },
+  {
+    "httpRequest": {
+      "method": "POST",
+      "path": "/tenant/123/_apis/runtime/runnerscalesets/223/sessions"
+    },
+    "httpResponse": {
+      "statusCode": 200,
+      "headers": { "Content-Type": ["application/json"] },
+      "body": "{\"sessionId\":\"11111111-1111-1111-1111-111111111111\",\"ownerName\":\"local.medium\",\"runnerScaleSet\":{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48},\"messageQueueUrl\":\"https://mockserver:1080/messages?sessionId=11111111-1111-1111-1111-111111111111&api-version=6.0-preview\",\"messageQueueAccessToken\":\"fake-queue-token\",\"statistics\":{\"totalAvailableJobs\":0,\"totalAcquiredJobs\":0,\"totalAssignedJobs\":0,\"totalRunningJobs\":0,\"totalRegisteredRunners\":0,\"totalBusyRunners\":0,\"totalIdleRunners\":0}}"
+    }
+  },
+  {
+    "httpRequest": {
+      "method": "GET",
+      "path": "/messages"
+    },
+    "httpResponse": {
+      "statusCode": 202,
+      "delay": { "timeUnit": "MILLISECONDS", "value": 1000 }
+    }
+  },
+  {
+    "httpRequest": {
+      "method": "GET",
+      "path": "/tenant/123/_apis/distributedtask/pools/0/agents"
+    },
+    "httpResponseTemplate": {
+      "templateType": "MUSTACHE",
+      "template": "{\"statusCode\":200,\"headers\":{\"Content-Type\":[\"application/json\"]},\"body\":\"{\\\"count\\\":1,\\\"value\\\":[{\\\"id\\\":321,\\\"name\\\":\\\"{{ request.queryStringParameters.agentName.0 }}\\\",\\\"runnerScaleSetId\\\":223}]}\"}"
+    }
+  },
+  {
+    "httpRequest": {
+      "method": "DELETE",
+      "path": "/tenant/123/_apis/distributedtask/pools/0/agents/321"
+    },
+    "httpResponse": {
+      "statusCode": 204
+    }
+  },
+  {
+    "httpRequest": {
+      "method": "DELETE",
+      "path": "/tenant/123/_apis/runtime/runnerscalesets/223/sessions/11111111-1111-1111-1111-111111111111"
+    },
+    "httpResponse": {
+      "statusCode": 204
+    }
+  }
+]

tests/ministack/multi-runner-scale-set.tfvars

@@ -54,4 +54,4 @@ scale_set = {
   container = {
     image = "localhost:4566/scale-set-controller:smoke"
   }
-}
+}