diff --git a/.circleci/config.yml b/.circleci/config.yml index 9faeca6..75c0c49 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,91 +1,123 @@ version: 2.1 - executors: - build: + reviewed: docker: - - image: cimg/python:3.13.2-browsers - resource_class: medium - + - image: cimg/aws@sha256:4cf3768db00ddf9bc439b8c84e8a60c11a14edf81c8baf76f74132144f8340d5 commands: - install-dependencies: + checkout-source: + steps: + - run: + name: Check out the exact public source commit over HTTPS + command: | + set -euo pipefail + [[ "$CIRCLE_SHA1" =~ ^[0-9a-f]{40}$ ]] + export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 + git init --quiet . + git remote add origin https://github.com/topcoder-platform/forms-api-v6.git + git -c credential.helper= fetch --quiet --depth 1 origin "$CIRCLE_SHA1" + git checkout --quiet --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$CIRCLE_SHA1" + checkout-suite: steps: - - run: - name: Install build and deployment dependencies - command: | - sudo apt-get update -qq - sudo apt-get install -y -qq jq python3-pip - pip install boto3 awscli - + - run: + name: Verify deployment suite v2.0.5 + command: | + set -euo pipefail + git init --quiet /tmp/tc-deploy-scripts + git -C /tmp/tc-deploy-scripts remote add origin https://github.com/topcoder-platform/tc-deploy-scripts.git + git -C /tmp/tc-deploy-scripts fetch --quiet --depth 1 origin refs/tags/v2.0.5 + git -C /tmp/tc-deploy-scripts checkout --quiet --detach FETCH_HEAD + test "$(git -C /tmp/tc-deploy-scripts rev-parse HEAD)" = bff574ab5c0b46773e25d2af1eff76675fba2ba5 jobs: + build: + executor: reviewed + steps: + - checkout-source + - setup_remote_docker + - checkout-suite + - run: + name: Build committed source without AWS or broker credentials + command: | + set -euo pipefail + python3 -I /tmp/tc-deploy-scripts/build_image.py --target runtime --image forms-api-v6:candidate + python3 -I /tmp/tc-deploy-scripts/build_image.py --target migrate --image forms-api-v6:migrate-candidate + mkdir -p /tmp/release + docker save --output /tmp/release/image.tar forms-api-v6:candidate forms-api-v6:migrate-candidate + no_output_timeout: 30m + - persist_to_workspace: + root: /tmp/release + paths: + - image.tar deploy: - executor: build + executor: reviewed parameters: - deploy_env: - type: enum - enum: [DEV, PROD] environment: type: enum - enum: [dev, production] + enum: + - dev + - prod environment: - AWS_REGION: us-east-1 - AWS_DEFAULT_REGION: us-east-1 - AWS_STS_REGIONAL_ENDPOINTS: regional - APPNAME: forms-api-v6 + DEPLOY_ENV: << parameters.environment >> + steps: + - checkout-source + - setup_remote_docker + - checkout-suite + - attach_workspace: + at: /tmp/release + - run: + name: Promote a scanned digest with repository-bound OIDC + command: | + set +x + set -euo pipefail + test -f "deploy/${DEPLOY_ENV}.json" + docker load --input /tmp/release/image.tar + python3 -I deploy/release.py "deploy/${DEPLOY_ENV}.json" + no_output_timeout: 45m + deployment-checks: + executor: reviewed steps: - - checkout - - setup_remote_docker - - install-dependencies - - run: - name: Verify first-release activation and rollback handling - command: python3 -m unittest discover -s deploy/tests - - run: - name: Build runtime and one-shot migration images - command: | - docker buildx build --load --target migrate -t forms-api-v6:migrate-candidate . - docker buildx build --load --target runtime -t forms-api-v6:latest . - - run: - name: Migrate and deploy using the Topcoder deployment suite - no_output_timeout: 35m - command: | - git clone --quiet --depth 1 --branch v1.4.20 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript - test "$(git -C ../buildscript rev-parse HEAD)" = 5f3745c35463ce0e2475c12269aa7a2b446456e5 - cp ../buildscript/{master_deploy,buildenv,awsconfiguration,psvar-processor}.sh . - ./awsconfiguration.sh << parameters.deploy_env >> - set +x - source awsenvconf - ./psvar-processor.sh -t appenv -p "/config/${APPNAME}/deployvar" - source deployvar_env - python3 -u deploy/release.py << parameters.environment >> "<< parameters.environment >>-${CIRCLE_SHA1}-${CIRCLE_BUILD_NUM}" - ./master_deploy.sh \ - -d ECS \ - -e << parameters.deploy_env >> \ - -t latest \ - -j "/config/${APPNAME}/appvar,/config/common/global-appvar" \ - -i "$APPNAME" \ - -p FARGATE - python3 -u deploy/activate.py - - store_artifacts: - path: deploy/release-<< parameters.environment >>.json - destination: release - + - checkout-source + - checkout-suite + - run: + name: Test migration, activation and rollback gates + command: python3 -m unittest discover -s deploy/tests -q workflows: - build-and-deploy: + release: jobs: - - deploy: - name: deploy-dev - context: org-global - deploy_env: DEV - environment: dev - serial-group: << pipeline.project.slug >>/deploy-dev - filters: - branches: - only: develop - - deploy: - name: deploy-production - context: org-global - deploy_env: PROD - environment: production - serial-group: << pipeline.project.slug >>/deploy-production - filters: - branches: - only: master + - deployment-checks + - build: + requires: + - deployment-checks + - deploy: + name: deploy-dev + environment: dev + requires: + - build + context: forms-api-v6-dev-deploy + serial-group: << pipeline.project.slug >>/deploy-dev + filters: + branches: + only: develop + tags: + ignore: /.*/ + - approve-prod: + type: approval + requires: + - build + filters: + branches: + only: master + tags: + ignore: /.*/ + - deploy: + name: deploy-prod + environment: prod + requires: + - approve-prod + context: forms-api-v6-prod-deploy + serial-group: << pipeline.project.slug >>/deploy-prod + filters: + branches: + only: master + tags: + ignore: /.*/ diff --git a/.gitignore b/.gitignore index 740fea4..cad6ff8 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ src/generated/ __pycache__/ *.pyc + +# Infrastructure definitions and detailed AWS inventories are maintained privately. +/deploy/iam/ +/deploy/*-baseline.json diff --git a/deploy/MIGRATION.md b/deploy/MIGRATION.md new file mode 100644 index 0000000..9918997 --- /dev/null +++ b/deploy/MIGRATION.md @@ -0,0 +1,44 @@ +# OIDC deployment migration + +This change replaces the Auth0 credential broker and SSM-to-shell deployment path with [tc-deploy-scripts v2.0.5](https://github.com/topcoder-platform/tc-deploy-scripts/releases/tag/v2.0.5), fetched by tag and verified against commit `bff574ab5c0b46773e25d2af1eff76675fba2ba5`. Build and deployment run in separate jobs. The deployer cannot read Parameter Store, Secrets Manager or decrypt KMS data, including through another assumed role. ECS continues resolving the existing runtime secret references using the execution role. + +## Production target + +The read-only inventory on 2026-11-03 verified account `409275337247`, cluster `topcoder-infrastructure`, service `forms-api-v6`, family `forms-api-v6`, and container `forms-api-v6`. `prod.json` preserves the observed task/execution roles and names the proposed service-specific deployment role. [prod-baseline.json](https://github.com/topcoder-platform/v6-services-iac/blob/main/services/forms-api-v6/deploy/prod-baseline.json) records references only; it contains no parameter values. Do not recreate the service or replace its complete task definition from this baseline. + +## Infrastructure prerequisites, applied separately by an administrator + +1. In each account, register the CircleCI OIDC issuer `https://oidc.circleci.com/org/947bef5e-1669-415d-b83c-4fc85155b7df` with client ID `947bef5e-1669-415d-b83c-4fc85155b7df`. The current production account has only the legacy Auth0 provider. Do not remove that provider until all its consumers have migrated. +2. Review and install [iam/prod-role.cloudformation.json](https://github.com/topcoder-platform/v6-services-iac/blob/main/services/forms-api-v6/deploy/iam/prod-role.cloudformation.json) with an administrator/IaC identity. It creates the proposed deployment role and applies the same least-privilege policy as a permissions boundary. The trust binds this CircleCI project, repository, checkout branch and configuration branch. It does not grant infrastructure creation to CircleCI. The role and matching boundary include `inspector2:ListFindings` and `inspector2:ListCoverage` restricted to the deployment region; these scan-metadata APIs require `Resource: "*"`. Secret-access denies remain explicit. +3. Configure ECR repository `forms-api-v6` with unconditional `IMMUTABLE` tags and supported scanning. Preserve existing enhanced scanning where enabled. Existing tags are not retagged. The release fails for MEDIUM/HIGH/CRITICAL/UNDEFINED findings, failed scans, or a mismatched digest. +4. In the infrastructure that owns the task definition, add the reviewed container check from `prod-healthcheck.json`, keeping all roles, runtime secret references, resources, mounts and sidecars. Verify the health endpoint against the running container in dev first. Preserve circuit-breaker rollback; the Forms-specific first-activation rule is described below. Remove any `RESET_DB` secret binding and plaintext secret environment entries before adoption. Keep the execution role's runtime read permissions scoped to the references that service needs. +5. Create restricted `forms-api-v6-dev-deploy` and `forms-api-v6-prod-deploy` CircleCI contexts without Auth0/static AWS credentials. Restrict production to the reviewed `master` checkout and configuration ref, authorized operators, and approval job. Remove legacy credentials from project settings too: removing `org-global` alone does not remove project-level variables. Confirm `CIRCLE_OIDC_TOKEN_V3` is issued. Disable secret access for forks and SSH debugging in deployment jobs. + +## Development acceptance before merge/release + +Read-only inventory on 2026-11-04 verified `dev.json` against the actual development service, task/execution roles and image repositories. This PR targets `develop`. Matching IAM templates and detailed baselines remain private. **Keep this PR in draft until the service-specific prerequisites below are complete.** No AWS or CircleCI resources were provisioned by this refresh. + +- Install the service deployment role and matching permissions boundary. +- Create the restricted empty CircleCI deployment context. +- Review and install the dev container health-check proposal; preserve all other task configuration. +- Verify one-shot migration secret/execution-role grants and migration-before-runtime promotion in dev. + +Run the shared offline suite, validate the CircleCI config and IAM policies, and test a dev release plus an unhealthy-image rollback. Verify CloudTrail contains no SSM/Secrets Manager calls from the deployment principal. Remove its old broker grants only after migration. Check the runtime secret references and roles stayed unchanged. Production requires the manual approval job and a passing digest/scan/health verification. + +The pinned executor is `cimg/aws@sha256:4cf3768db00ddf9bc439b8c84e8a60c11a14edf81c8baf76f74132144f8340d5`. Python/AWS CLI/Buildx/Git compatibility was tested with the shared offline suite. No package installation or service build runs in the deployment job. The Dockerfile retains the existing in-image quality gates. This PR does not deploy, change IAM, rotate credentials, or claim a successful live rollout. + +## Forms migration and first activation + +The adapter preserves the stable-stack/environment/bootstrap checks, runs the separate migration image as a private one-shot task, and requires exit code zero plus the exact scanned image digest before runtime promotion. Both images are built without credentials and scanned before either executes. The migration image runs as non-root. The only added permissions are the exact migration task family and read access to the existing Forms stack; the deployment role still cannot read parameter values. ECS resolves the migration database secret through the existing execution role. + +Unlike the default CLI, this reviewed adapter retains Forms first activation: an existing zero-capacity service is started at one replica only after the infrastructure bootstrap and migration gates pass. Existing positive capacity is preserved. Runtime read-back, concurrency, digest/health and rollback checks must pass. Migration waiting is bounded to 15 minutes and runtime observation to 20 minutes, within the one-hour STS session; a timeout fails without pretending the rollout succeeded. Test both first activation and failed migration in dev. + +## Infrastructure source + +IAM templates and detailed AWS baselines are maintained in the private [v6-services-iac service directory](https://github.com/topcoder-platform/v6-services-iac/tree/main/services/forms-api-v6). Provision infrastructure separately with an administrator identity; application CI does not fetch that repository or install IAM. Required application manifests and health-check proposals remain here. + +## Dev branch refresh + +This PR targets `develop` and preserves its current application/dependency fixes. The shared suite is v2.0.5 (82 offline tests); service-specific adapters and their tests remain part of review. Keep this PR in draft until its documented infrastructure and integration prerequisites are met. Updating these files does not provision AWS or CircleCI resources. + +Source checkout uses the public HTTPS repository and verifies the exact CircleCI commit, avoiding dependency on failing SSH checkout keys. No GitHub token is added to build or deployment contexts. diff --git a/deploy/activate.py b/deploy/activate.py index 4f53e29..bdfda6e 100644 --- a/deploy/activate.py +++ b/deploy/activate.py @@ -6,7 +6,6 @@ """ import os -import boto3 from release import wait_until @@ -65,14 +64,7 @@ def is_stable(): def main(): - """Read CircleCI/deployvar settings and activate the expected image; errors propagate.""" - session = boto3.Session(region_name=os.environ.get('AWS_REGION', 'us-east-1')) - account = session.client('sts').get_caller_identity()['Account'] - image = (f'{account}.dkr.ecr.{session.region_name}.amazonaws.com/' - f'{os.environ["AWS_REPOSITORY"]}:{os.environ["CIRCLE_BUILD_NUM"]}') - service = activate_service(session.client('ecs'), os.environ['AWS_ECS_CLUSTER'], - os.environ['AWS_ECS_SERVICE'], image) - print('Verified running release: ' + service['taskDefinition'], flush=True) + raise RuntimeError('Activation is now part of deploy/release.py after the scanned migration gate.') if __name__ == '__main__': diff --git a/deploy/dev-healthcheck.json b/deploy/dev-healthcheck.json new file mode 100644 index 0000000..caf0f82 --- /dev/null +++ b/deploy/dev-healthcheck.json @@ -0,0 +1,14 @@ +{ + "forms-api-v6": { + "command": [ + "CMD", + "node", + "-e", + "const r=require('node:http').get(\"http://127.0.0.1:3000/v6/forms/health/ready\",r=>{r.resume();process.exit(r.statusCode>=200&&r.statusCode<300?0:1)});r.setTimeout(4000,()=>{r.destroy();process.exit(1)});r.on('error',()=>process.exit(1))" + ], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 120 + } +} diff --git a/deploy/dev.json b/deploy/dev.json new file mode 100644 index 0000000..f4d6eb0 --- /dev/null +++ b/deploy/dev.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 2, + "environment": "dev", + "accountId": "811668436784", + "region": "us-east-1", + "deployRoleArn": "arn:aws:iam::811668436784:role/forms-api-v6-dev-deploy", + "taskRoleArn": "arn:aws:iam::811668436784:role/forms-api-v6-dev-TaskRole-14rblBbkqXuT", + "executionRoleArn": "arn:aws:iam::811668436784:role/ecsTaskExecutionRole", + "cluster": "topcoder-infrastructure", + "service": "forms-api-v6", + "taskFamily": "forms-api-v6", + "circleciOrgId": "947bef5e-1669-415d-b83c-4fc85155b7df", + "circleciProjectId": "ac910b17-c01b-494f-876b-43fddd16f82b", + "repository": "github.com/topcoder-platform/forms-api-v6", + "branch": "develop", + "images": { + "forms-api-v6": "forms-api-v6" + } +} diff --git a/deploy/prod-healthcheck.json b/deploy/prod-healthcheck.json new file mode 100644 index 0000000..77f215a --- /dev/null +++ b/deploy/prod-healthcheck.json @@ -0,0 +1,14 @@ +{ + "forms-api-v6": { + "command": [ + "CMD", + "node", + "-e", + "const r=require('node:http').get('http://127.0.0.1:3000/v6/forms/health/ready',r=>{r.resume();process.exit(r.statusCode>=200&&r.statusCode<300?0:1)});r.setTimeout(4000,()=>{r.destroy();process.exit(1)});r.on('error',()=>process.exit(1))" + ], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 120 + } +} diff --git a/deploy/prod.json b/deploy/prod.json new file mode 100644 index 0000000..eb8ab99 --- /dev/null +++ b/deploy/prod.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 2, + "environment": "prod", + "accountId": "409275337247", + "region": "us-east-1", + "deployRoleArn": "arn:aws:iam::409275337247:role/forms-api-v6-prod-deploy", + "taskRoleArn": "arn:aws:iam::409275337247:role/forms-api-v6-production-TaskRole-9CWeN3VGr09Z", + "executionRoleArn": "arn:aws:iam::409275337247:role/ecsTaskExecutionRole", + "cluster": "topcoder-infrastructure", + "service": "forms-api-v6", + "taskFamily": "forms-api-v6", + "circleciOrgId": "947bef5e-1669-415d-b83c-4fc85155b7df", + "circleciProjectId": "ac910b17-c01b-494f-876b-43fddd16f82b", + "repository": "github.com/topcoder-platform/forms-api-v6", + "branch": "master", + "images": { + "forms-api-v6": "forms-api-v6" + } +} diff --git a/deploy/release.py b/deploy/release.py index 7260cd5..edd0444 100644 --- a/deploy/release.py +++ b/deploy/release.py @@ -1,112 +1,163 @@ #!/usr/bin/env python3 -"""Run the migration gate before the shared Topcoder ECS deployment script. - -CLI: release.py dev|production IMAGE_TAG [MIGRATION_IMAGE]. -Requires inherited AWS credentials, boto3, and Docker. Pushes the prebuilt migration -image to ECR and runs it as a private one-shot ECS task. Raises on failures so -CircleCI stops before master_deploy.sh promotes the runtime image. -""" -import base64 -import copy +"""Scan both images, run the one-shot migration, then promote/activate Forms.""" +from copy import deepcopy import json import os from pathlib import Path -import re -import subprocess import sys +import tempfile import time -import boto3 +sys.path.insert(0, '/tmp/tc-deploy-scripts') +import tcdeploy as d -def wait_until(description, check, seconds=1800): - """Poll a live deployment check every 15 seconds, returning its truthy result. - - Inputs are a progress label, callable, and timeout. Check exceptions propagate; - timeout raises RuntimeError without restarting the observed deployment. - """ +def wait_until(description, check, seconds=900): deadline = time.monotonic() + seconds while time.monotonic() < deadline: result = check() if result: return result - print(f'Waiting for {description}...', flush=True) - time.sleep(15) - raise RuntimeError(f'Timed out observing {description}; inspect the existing AWS operation before retrying.') + time.sleep(10) + raise d.DeployError('Timed out: ' + description) -def main(): - """Validate CLI arguments and environment, run migrations, and save evidence. - - Returns None on success; AWS, Docker, and failed migration - errors propagate. Credentials are passed through stdin/environment, never argv. - """ - if len(sys.argv) not in (3, 4) or sys.argv[1] not in ('dev', 'production'): - raise RuntimeError(__doc__) - environment, tag = sys.argv[1:3] - if not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9_.-]{0,110}', tag): - raise RuntimeError('Invalid immutable release tag.') - local_migration = sys.argv[3] if len(sys.argv) == 4 else 'forms-api-v6:migrate-candidate' - session = boto3.Session(region_name=os.environ.get('AWS_REGION', 'us-east-1')) - account = session.client('sts').get_caller_identity()['Account'] - if (account == '811668436784') != (environment == 'dev'): - raise RuntimeError('AWS account does not match the selected deployment environment.') - cfn, ecs, ecr = [session.client(name) for name in ['cloudformation', 'ecs', 'ecr']] - stack_name = f'forms-api-v6-{environment}' - stack = cfn.describe_stacks(StackName=stack_name)['Stacks'][0] - if stack['StackStatus'] not in ('CREATE_COMPLETE', 'UPDATE_COMPLETE', 'UPDATE_ROLLBACK_COMPLETE'): - raise RuntimeError('Existing stack is not stable; inspect it before releasing.') - settings = {x['ParameterKey']: x['ParameterValue'] for x in stack['Parameters']} - if settings['Environment'] != environment or settings['BootstrapOnly'] != 'false': - raise RuntimeError('Stack environment or bootstrap state does not permit deployment.') - output = {x['OutputKey']: x['OutputValue'] for x in stack['Outputs']} - repository = output['RepositoryUri'] - authorization = ecr.get_authorization_token()['authorizationData'][0] - user, password = base64.b64decode(authorization['authorizationToken']).decode().split(':', 1) - subprocess.run(['docker', 'login', '--username', user, '--password-stdin', authorization['proxyEndpoint']], - input=password, text=True, check=True, stdout=subprocess.DEVNULL) - destination = f'{repository}:{tag}-migrate' - subprocess.run(['docker', 'tag', local_migration, destination], check=True) - subprocess.run(['docker', 'push', destination], check=True) - digest = ecr.describe_images(repositoryName='forms-api-v6', imageIds=[{'imageTag': tag + '-migrate'}])['imageDetails'][0]['imageDigest'] - migration_image = f'{repository}@{digest}' - service = ecs.describe_services(cluster=settings['ClusterName'], services=[output['ServiceName']])['services'][0] - current = ecs.describe_task_definition(taskDefinition=service['taskDefinition'])['taskDefinition'] - allowed = {'family', 'taskRoleArn', 'executionRoleArn', 'networkMode', 'containerDefinitions', - 'volumes', 'placementConstraints', 'requiresCompatibilities', 'cpu', 'memory', 'runtimePlatform', 'ephemeralStorage'} - migration = {k: copy.deepcopy(v) for k, v in current.items() if k in allowed} - migration['family'] = 'forms-api-v6-migrate' - container = migration['containerDefinitions'][0] - container['image'] = migration_image +def stack_settings(aws, m): + environment = 'production' if m['environment'] == 'prod' else 'dev' + stack_name = 'forms-api-v6-' + environment + stacks = aws.call('cloudformation', 'describe-stacks', StackName=stack_name).get('Stacks', []) + d.require(len(stacks) == 1, 'Expected the existing Forms stack') + stack = stacks[0] + d.require(stack.get('StackStatus') in ('CREATE_COMPLETE', 'UPDATE_COMPLETE', 'UPDATE_ROLLBACK_COMPLETE'), 'Forms stack is not stable') + settings = {v['ParameterKey']: v['ParameterValue'] for v in stack.get('Parameters', [])} + output = {v['OutputKey']: v['OutputValue'] for v in stack.get('Outputs', [])} + d.require(settings.get('Environment') == environment and settings.get('BootstrapOnly') == 'false', 'Complete Forms infrastructure bootstrap before releasing') + d.require(settings.get('ClusterName') == m['cluster'] and output.get('ServiceName') == m['service'], 'Forms stack target differs from manifest') + d.require(settings.get('ParameterPrefix') == '/config/forms-api-v6/appvar', 'Unexpected migration secret namespace') + d.require(output.get('RepositoryUri') == f"{m['accountId']}.dkr.ecr.{m['region']}.amazonaws.com/forms-api-v6", 'Unexpected Forms repository') + return settings + + +def read_service(aws, m): + result = aws.call('ecs', 'describe-services', cluster=m['cluster'], services=[m['service']]) + d.require(not result.get('failures') and len(result.get('services', [])) == 1, 'Expected the existing Forms service') + service = result['services'][0] + d.require(service.get('status') == 'ACTIVE' and service.get('desiredCount', -1) >= 0, 'Forms service is not active') + d.require(service.get('serviceArn') == d.arn(m, 'ecs', f"service/{m['cluster']}/{m['service']}"), 'Unexpected service ARN') + d.require(service.get('deploymentController', {}).get('type', 'ECS') == 'ECS', 'Forms requires ECS rolling deployment') + breaker = service.get('deploymentConfiguration', {}).get('deploymentCircuitBreaker') + d.require(isinstance(breaker, dict) and breaker.get('enable') is True and breaker.get('rollback') is True, 'Enable Forms circuit-breaker rollback') + return service + + +def initial_stable(service, definition): + d.require(service.get('taskDefinition') == definition, 'Concurrent Forms deployment detected') + deployments = service.get('deployments', []) + d.require(not any(x.get('rolloutState') == 'FAILED' for x in deployments), 'Forms rollout failed') + return (len(deployments) == 1 and deployments[0].get('rolloutState') == 'COMPLETED' + and deployments[0].get('taskDefinition') == definition + and service.get('runningCount') == service['desiredCount'] + and service.get('pendingCount') == 0) + + +def migration_request(m, task, tags, image): + # Validate exact roles, runtime settings, health checks and secret references + # before deriving a separate one-shot definition from this service. + request = d.task_request(m, task, tags, {'forms-api-v6': image}) + request['family'] = 'forms-api-v6-migrate' + containers = request['containerDefinitions'] + d.require(len(containers) == 1 and containers[0]['name'] == 'forms-api-v6', 'Unexpected Forms containers') + container = containers[0] container['readonlyRootFilesystem'] = False container.pop('healthCheck', None) container['portMappings'] = [] - container['secrets'] = [{'name': 'DATABASE_URL', 'valueFrom': settings['ParameterPrefix'] + '/MIGRATION_DATABASE_URL'}] + container['secrets'] = [{'name': 'DATABASE_URL', 'valueFrom': '/config/forms-api-v6/appvar/MIGRATION_DATABASE_URL'}] + container['environment'] = [v for v in container.get('environment', []) if v['name'] != 'DATABASE_URL'] container['command'] = ['node', 'node_modules/prisma/build/index.js', 'migrate', 'deploy'] - migration_definition = ecs.register_task_definition(**migration)['taskDefinition']['taskDefinitionArn'] - result = ecs.run_task(cluster=settings['ClusterName'], taskDefinition=migration_definition, - launchType='FARGATE', networkConfiguration=service['networkConfiguration'], - startedBy='forms-release') - if result.get('failures') or not result.get('tasks'): - raise RuntimeError('ECS could not start the migration task.') - migration_arn = result['tasks'][0]['taskArn'] - print('Migration task: ' + migration_arn, flush=True) - - def migration_finished(): - """Read this migration task; return the stopped task, or None while live.""" - tasks = ecs.describe_tasks(cluster=settings['ClusterName'], tasks=[migration_arn])['tasks'] - if not tasks: - raise RuntimeError('Migration task handle is missing; inspect ECS before retrying.') - return tasks[0] if tasks[0]['lastStatus'] == 'STOPPED' else None - - completed = wait_until('migration task ' + migration_arn, migration_finished) - if any(c.get('exitCode') != 0 for c in completed['containers']): - raise RuntimeError('Migration failed; inspect the migration task logs. Runtime was not promoted.') - evidence = {'environment': environment, 'tag': tag, 'migrationImage': migration_image, - 'migrationTask': migration_arn, 'stack': stack_name} - Path('deploy/release-' + environment + '.json').write_text(json.dumps(evidence, indent=2) + '\n') - print(json.dumps(evidence, indent=2)) + return request + + +def migrate(aws, m, service, task, tags, image): + request = migration_request(m, task, tags, image) + definition = aws.call('ecs', 'register-task-definition', **request)['taskDefinition']['taskDefinitionArn'] + d.require(definition.startswith(d.arn(m, 'ecs', 'task-definition/forms-api-v6-migrate:')), 'Unexpected migration family') + result = aws.call('ecs', 'run-task', cluster=m['cluster'], taskDefinition=definition, + launchType='FARGATE', networkConfiguration=service['networkConfiguration'], startedBy='forms-release') + d.require(not result.get('failures') and len(result.get('tasks', [])) == 1, 'Migration task could not start') + task_arn = result['tasks'][0]['taskArn'] + + def stopped(): + result = aws.call('ecs', 'describe-tasks', cluster=m['cluster'], tasks=[task_arn]) + d.require(not result.get('failures') and len(result.get('tasks', [])) == 1, 'Migration task is missing') + current = result['tasks'][0] + d.require(current.get('taskDefinitionArn') == definition, 'Migration task revision differs') + return current if current.get('lastStatus') == 'STOPPED' else None + + completed = wait_until('Forms migration', stopped) + containers = completed.get('containers', []) + d.require(len(containers) == 1 and containers[0].get('exitCode') == 0 + and containers[0].get('imageDigest') == image.rsplit('@', 1)[1], + 'Migration failed or executed an unexpected image; runtime was not promoted') + return task_arn + + +def promote(aws, m, previous, task, tags, image): + d.require(initial_stable(read_service(aws, m), previous['taskDefinition']), 'Service changed during migration') + request = d.task_request(m, task, tags, {'forms-api-v6': image}) + definition = aws.call('ecs', 'register-task-definition', **request)['taskDefinition']['taskDefinitionArn'] + d.require(definition.startswith(d.arn(m, 'ecs', 'task-definition/' + m['taskFamily'] + ':')), 'Unexpected runtime family') + saved = aws.call('ecs', 'describe-task-definition', taskDefinition=definition, include=['TAGS']) + actual = {k: v for k, v in saved['taskDefinition'].items() if k in d.TASK_FIELDS} + actual['tags'] = sorted(saved.get('tags', []), key=lambda x: x['key']) + expected = deepcopy(request) + expected['tags'] = sorted(expected.get('tags', []), key=lambda x: x['key']) + d.require(actual == expected, 'Registered Forms runtime differs from the requested definition') + d.require(initial_stable(read_service(aws, m), previous['taskDefinition']), 'Concurrent Forms release before activation') + updated = aws.call('ecs', 'update-service', cluster=m['cluster'], service=m['service'], + taskDefinition=definition, desiredCount=max(1, previous['desiredCount'])) + d.require(updated.get('service', {}).get('taskDefinition') == definition, 'ECS did not accept the Forms revision') + + def healthy(): + current = d.read_service(aws, m) + return (d.stable(current, definition) and + d.verify_running(aws, m, definition, {'forms-api-v6': image}, current['desiredCount'])) + + wait_until('healthy Forms runtime', healthy, seconds=1200) + return definition + + +def release(aws, m, publisher, release_tag): + stack_settings(aws, m) + previous = read_service(aws, m) + d.require(initial_stable(previous, previous['taskDefinition']), 'Forms must be stable before release') + response = aws.call('ecs', 'describe-task-definition', taskDefinition=previous['taskDefinition'], include=['TAGS']) + task, tags = response['taskDefinition'], response.get('tags', []) + migration_request(m, task, tags, 'preflight@sha256:' + '0' * 64) + deadline = time.monotonic() + 900 + runtime = publisher.publish('forms-api-v6', 'forms-api-v6:candidate', release_tag, deadline) + migration = publisher.publish('forms-api-v6', 'forms-api-v6:migrate-candidate', release_tag + '-migrate', deadline) + d.require(initial_stable(read_service(aws, m), previous['taskDefinition']), 'Service changed during image publication') + migration_task = migrate(aws, m, previous, task, tags, migration) + definition = promote(aws, m, previous, task, tags, runtime) + return {'service': m['service'], 'taskDefinition': definition, 'migrationTask': migration_task, 'image': runtime} + + +def main(): + m = d.read_manifest(sys.argv[1]) + d.require(m['service'] == 'forms-api-v6' and set(m['images']) == {'forms-api-v6'}, 'Expected Forms manifest') + d.require(os.environ.get('CIRCLE_BRANCH') == m['branch'] and not os.environ.get('CIRCLE_TAG'), 'Wrong deployment branch') + d.require(os.environ.get('CIRCLE_PROJECT_USERNAME', '') + '/' + os.environ.get('CIRCLE_PROJECT_REPONAME', '') == m['repository'].removeprefix('github.com/'), 'Wrong deployment repository') + tag = os.environ.get('CIRCLE_SHA1', '') + '-' + os.environ.get('CIRCLE_BUILD_NUM', '') + d.require(d.re.fullmatch(r'[a-f0-9]{40}-[0-9]{1,12}', tag), 'Invalid release identity') + aws = d.authenticate(m, os.environ.get('CIRCLE_OIDC_TOKEN_V3'), ['ecs:RunTask', 'cloudformation:DescribeStacks']) + with tempfile.TemporaryDirectory(prefix='forms-release-') as directory: + result = release(aws, m, d.Publisher(aws, m, directory), tag) + print(json.dumps(result)) if __name__ == '__main__': - main() + try: + main() + except (d.DeployError, ValueError, KeyError, TypeError, OSError, IndexError): + print('Forms release stopped; inspect the migration or ECS rollout. No secret values were fetched.', file=sys.stderr) + sys.exit(1) diff --git a/deploy/tests/test_oidc_release.py b/deploy/tests/test_oidc_release.py new file mode 100644 index 0000000..cfa816f --- /dev/null +++ b/deploy/tests/test_oidc_release.py @@ -0,0 +1,141 @@ +"""Migration/activation regressions without AWS credentials or network access.""" +from copy import deepcopy +import json +from pathlib import Path +import sys +import unittest +from unittest.mock import Mock, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import release as r + + +class ReleaseTests(unittest.TestCase): + def setUp(self): + self.m = json.loads((Path(__file__).resolve().parents[1] / 'prod.json').read_text()) + self.image = 'registry/forms-api-v6@sha256:' + 'a' * 64 + self.old = r.d.arn(self.m, 'ecs', 'task-definition/forms-api-v6:1') + self.new = r.d.arn(self.m, 'ecs', 'task-definition/forms-api-v6:2') + self.task = { + 'family': 'forms-api-v6', 'status': 'ACTIVE', 'networkMode': 'awsvpc', + 'requiresCompatibilities': ['FARGATE'], + 'taskRoleArn': self.m['taskRoleArn'], 'executionRoleArn': self.m['executionRoleArn'], + 'containerDefinitions': [{'name': 'forms-api-v6', 'image': 'old', + 'readonlyRootFilesystem': True, 'portMappings': [{'containerPort': 3000}], + 'healthCheck': {'command': ['CMD', 'healthcheck']}, + 'secrets': [{'name': 'DATABASE_URL', 'valueFrom': '/runtime/database'}, + {'name': 'AUTH_SECRET', 'valueFrom': '/runtime/auth'}]}], + } + self.previous = { + 'status': 'ACTIVE', 'serviceArn': r.d.arn(self.m, 'ecs', f"service/{self.m['cluster']}/{self.m['service']}"), + 'taskDefinition': self.old, 'desiredCount': 2, 'runningCount': 2, 'pendingCount': 0, + 'deploymentConfiguration': {'deploymentCircuitBreaker': {'enable': True, 'rollback': True}}, + 'deployments': [{'taskDefinition': self.old, 'rolloutState': 'COMPLETED'}], + 'networkConfiguration': {'awsvpcConfiguration': {'subnets': ['private'], 'assignPublicIp': 'DISABLED'}}, + } + + def test_migration_has_only_its_runtime_database_reference(self): + original = deepcopy(self.task) + request = r.migration_request(self.m, self.task, [], self.image) + container = request['containerDefinitions'][0] + self.assertEqual(request['family'], 'forms-api-v6-migrate') + self.assertEqual(request['executionRoleArn'], self.m['executionRoleArn']) + self.assertEqual(container['secrets'], [{'name': 'DATABASE_URL', 'valueFrom': '/config/forms-api-v6/appvar/MIGRATION_DATABASE_URL'}]) + self.assertNotIn('healthCheck', container) + self.assertEqual(container['command'], ['node', 'node_modules/prisma/build/index.js', 'migrate', 'deploy']) + self.assertNotIn('tags', request) + self.assertEqual(self.task, original) + + def test_wrong_role_stops_before_a_migration_can_run(self): + self.task['executionRoleArn'] = 'unexpected' + with self.assertRaises(r.d.DeployError): + r.migration_request(self.m, self.task, [], self.image) + + def test_zero_capacity_is_only_an_initial_state(self): + self.previous.update(desiredCount=0, runningCount=0) + self.assertTrue(r.initial_stable(self.previous, self.old)) + self.assertFalse(r.d.stable(self.previous, self.old)) + + def test_incomplete_stack_bootstrap_blocks_release(self): + aws = Mock() + aws.call.return_value = {'Stacks': [{'StackStatus': 'UPDATE_COMPLETE', 'Parameters': [ + {'ParameterKey': 'Environment', 'ParameterValue': 'production'}, + {'ParameterKey': 'BootstrapOnly', 'ParameterValue': 'true'}, + ]}]} + with self.assertRaisesRegex(r.d.DeployError, 'bootstrap'): + r.stack_settings(aws, self.m) + + def migration_aws(self, exit_code=0, digest=None): + aws = Mock() + definition = r.d.arn(self.m, 'ecs', 'task-definition/forms-api-v6-migrate:4') + def call(api_service, operation, **kwargs): + if operation == 'register-task-definition': + return {'taskDefinition': {'taskDefinitionArn': definition}} + if operation == 'run-task': + self.assertEqual(kwargs['networkConfiguration'], self.previous['networkConfiguration']) + return {'tasks': [{'taskArn': 'migration-task'}]} + if operation == 'describe-tasks': + return {'tasks': [{'taskDefinitionArn': definition, 'lastStatus': 'STOPPED', + 'containers': [{'exitCode': exit_code, 'imageDigest': digest or 'sha256:' + 'a' * 64}]}]} + self.fail('Unexpected AWS operation: ' + operation) + aws.call.side_effect = call + return aws + + def test_failed_or_wrong_image_migration_never_promotes(self): + for code, digest in ((1, None), (0, 'sha256:' + 'b' * 64)): + with self.subTest(code=code, digest=digest), self.assertRaises(r.d.DeployError): + r.migrate(self.migration_aws(code, digest), self.m, self.previous, self.task, [], self.image) + + def test_successful_migration_uses_the_private_service_network(self): + self.assertEqual(r.migrate(self.migration_aws(), self.m, self.previous, self.task, [], self.image), 'migration-task') + + def test_migration_failure_stops_release_before_runtime_registration(self): + aws = Mock() + aws.call.return_value = {'taskDefinition': self.task} + publisher = Mock() + publisher.publish.return_value = self.image + with patch.object(r, 'stack_settings'), patch.object(r, 'read_service', return_value=self.previous), \ + patch.object(r, 'migrate', side_effect=r.d.DeployError('migration failed')), patch.object(r, 'promote') as promote: + with self.assertRaises(r.d.DeployError): + r.release(aws, self.m, publisher, 'a' * 40 + '-1') + self.assertEqual(publisher.publish.call_count, 2) + promote.assert_not_called() + + def test_activation_preserves_positive_capacity_and_starts_zero_at_one(self): + for desired in (0, 3): + with self.subTest(desired=desired): + previous = deepcopy(self.previous) + previous.update(desiredCount=desired, runningCount=desired) + request = r.d.task_request(self.m, self.task, [], {'forms-api-v6': self.image}) + aws = Mock() + def call(api_service, operation, **kwargs): + if operation == 'register-task-definition': + self.assertNotIn('tags', kwargs) + return {'taskDefinition': {'taskDefinitionArn': self.new}} + if operation == 'describe-task-definition': + return {'taskDefinition': {k: v for k, v in request.items() if k != 'tags'}, 'tags': []} + if operation == 'update-service': + self.assertEqual(kwargs['desiredCount'], max(1, desired)) + return {'service': {'taskDefinition': self.new}} + self.fail('Unexpected AWS operation: ' + operation) + aws.call.side_effect = call + with patch.object(r, 'read_service', return_value=previous), \ + patch.object(r.d, 'read_service'), patch.object(r.d, 'stable', return_value=True), \ + patch.object(r.d, 'verify_running', return_value=True): + self.assertEqual(r.promote(aws, self.m, previous, self.task, [], self.image), self.new) + + def test_optional_circuit_breaker_fields_preserve_safety_flags(self): + self.previous['deploymentConfiguration']['deploymentCircuitBreaker'].update(resetOnHealthyTask=True, thresholdConfiguration={'minimumFailureThreshold': 3}) + aws = Mock() + aws.call.return_value = {'services': [self.previous]} + self.assertEqual(r.read_service(aws, self.m), self.previous) + for flag in ('enable', 'rollback'): + with self.subTest(flag=flag): + self.previous['deploymentConfiguration']['deploymentCircuitBreaker'][flag] = False + with self.assertRaises(r.d.DeployError): + r.read_service(aws, self.m) + self.previous['deploymentConfiguration']['deploymentCircuitBreaker'][flag] = True + + +if __name__ == '__main__': + unittest.main()