diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 41355f2..781e4e3 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -59,6 +59,20 @@ on: description: 'The run label to use for the actions' type: string default: 'ubuntu-latest' + enforce-fresh: + description: > + Re-check live app and infra commits before applying; skip when either is + newer or divergent. Defaults to false. + type: boolean + default: false + freshness-module-id: + description: 'Terraform module ID used to read the infra deploy marker.' + type: string + default: '' + freshness-allow-stale: + description: 'Bypass freshness checks for a deliberate rollback.' + type: boolean + default: false concurrency: cd-${{ inputs.stage }} @@ -83,6 +97,11 @@ jobs: tf-directory: ${{ inputs.tf-directory }} tf-variables: ${{ inputs.tf-variables }} run-label: ${{ inputs.run-label }} + enforce-fresh: ${{ inputs.enforce-fresh }} + freshness-target-commit: ${{ github.sha }} + freshness-module-id: ${{ inputs.freshness-module-id }} + freshness-aws-role-arn: ${{ inputs.aws-role-arn }} + freshness-allow-stale: ${{ inputs.freshness-allow-stale }} deploy-app: name: Deploy App @@ -99,3 +118,6 @@ jobs: aws-role-arn: ${{ inputs.aws-role-arn }} aws-region: ${{ inputs.aws-region }} run-label: ${{ inputs.run-label }} + enforce-fresh: ${{ inputs.enforce-fresh }} + freshness-target-commit: ${{ inputs.version }} + freshness-allow-stale: ${{ inputs.freshness-allow-stale }} diff --git a/.github/workflows/deploy-app.yml b/.github/workflows/deploy-app.yml index ca0c90a..93aa1ab 100644 --- a/.github/workflows/deploy-app.yml +++ b/.github/workflows/deploy-app.yml @@ -35,6 +35,18 @@ on: description: 'The run label to use for the actions' type: string default: 'ubuntu-latest' + enforce-fresh: + description: 'Skip when the live app commit is newer or divergent.' + type: boolean + default: false + freshness-target-commit: + description: 'Commit this run intends to deploy.' + type: string + default: '' + freshness-allow-stale: + description: 'Bypass freshness checks for a deliberate rollback.' + type: boolean + default: false concurrency: deploy-${{ inputs.stage }} @@ -74,8 +86,62 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 + # Re-check at the apply boundary so every re-run is guarded. ECS read + # failures block the deploy; allow-stale skips this step. + - name: Assert fresh + id: freshness + if: ${{ inputs.enforce-fresh && !inputs.freshness-allow-stale }} + shell: bash + env: + TARGET: ${{ inputs.freshness-target-commit }} + NAME: ${{ steps.build_task_name.outputs.name }} + AWS_REGION: ${{ inputs.aws-region }} + run: | + set -euo pipefail + git fetch --no-tags --prune --unshallow origin 2>/dev/null || git fetch --no-tags origin 2>/dev/null || true + if ! git cat-file -e "${TARGET}^{commit}" 2>/dev/null; then git fetch --no-tags origin "$TARGET" 2>/dev/null || true; fi + if ! git cat-file -e "${TARGET}^{commit}" 2>/dev/null; then + # Non-commit image tags cannot be ancestry-checked. + echo "::warning::assert-fresh(app): target '${TARGET}' is not a git commit — cannot evaluate freshness; proceeding UNGUARDED" + echo "skip=false" >> "$GITHUB_OUTPUT"; exit 0 + fi + set +e + td="$(aws ecs describe-services --cluster "${NAME}-cluster" --services "${NAME}-service" --query "services[0].deployments[?status=='PRIMARY'].taskDefinition | [0]" --output text --region "$AWS_REGION" 2>/tmp/ecs_err)" + ecs_rc=$? + set -e + if [ "$ecs_rc" -ne 0 ] || [ -z "$td" ] || [ "$td" = "None" ]; then + echo "::error::assert-fresh(app): could not read the PRIMARY ECS task definition ($(tr -d '\n' /tmp/ecs_err)" + image_rc=$? + set -e + if [ "$image_rc" -ne 0 ] || [ -z "$image" ] || [ "$image" = "None" ]; then + set +e + image="$(aws ecs describe-task-definition --task-definition "$td" --query "taskDefinition.containerDefinitions[0].image" --output text --region "$AWS_REGION" 2>/tmp/ecs_err)" + image_rc=$? + set -e + fi + if [ "$image_rc" -ne 0 ] || [ -z "$image" ] || [ "$image" = "None" ]; then + echo "::error::assert-fresh(app): could not read the PRIMARY ECS task definition ($(tr -d '\n' /dev/null; then git fetch --no-tags origin "$running" 2>/dev/null || true; fi + if ! git cat-file -e "${running}^{commit}" 2>/dev/null; then + echo "::warning::assert-fresh(app): live ${running:0:7} unresolvable in history — treating as stale"; skip=true + elif git merge-base --is-ancestor "$running" "$TARGET"; then + echo "assert-fresh(app): live ${running:0:7} is an ancestor of ${TARGET:0:7} — fresh" + else + echo "::warning::assert-fresh(app): live ${running:0:7} is NOT an ancestor of ${TARGET:0:7} — newer/divergent already live; skipping ECS deploy"; skip=true + fi + echo "skip=$skip" >> "$GITHUB_OUTPUT" + - name: Deploy image to ECS id: deploy + if: ${{ steps.freshness.outputs.skip != 'true' }} uses: WalletConnect/actions/aws/ecs/deploy-image/@2.5.4 with: aws-role-arn: ${{ inputs.aws-role-arn }} diff --git a/.github/workflows/deploy-infra.yml b/.github/workflows/deploy-infra.yml index 6703b75..908ac09 100644 --- a/.github/workflows/deploy-infra.yml +++ b/.github/workflows/deploy-infra.yml @@ -39,6 +39,26 @@ on: description: 'The run label to use for the actions' type: string default: 'ubuntu-latest' + enforce-fresh: + description: 'Skip when the live infra commit is newer or divergent.' + type: boolean + default: false + freshness-target-commit: + description: 'Commit this run intends to apply.' + type: string + default: '' + freshness-module-id: + description: 'Terraform module ID used to read the infra deploy marker.' + type: string + default: '' + freshness-aws-role-arn: + description: 'App-account AWS role used to read the infra deploy marker.' + type: string + default: '' + freshness-allow-stale: + description: 'Bypass freshness checks for a deliberate rollback.' + type: boolean + default: false secrets: TF_API_TOKEN: required: true @@ -67,6 +87,67 @@ jobs: submodules: recursive token: ${{ secrets.PRIVATE_SUBMODULE_ACCESS_TOKEN || github.token }} + # Use app-account credentials for the marker read, then restore monitoring + # credentials below. Credential and marker read failures block the apply. + - name: Configure AWS Credentials for freshness read + if: ${{ inputs.enforce-fresh && !inputs.freshness-allow-stale }} + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ inputs.freshness-aws-role-arn }} + aws-region: ${{ inputs.aws-region }} + + - name: Assert fresh + id: freshness + if: ${{ inputs.enforce-fresh && !inputs.freshness-allow-stale }} + shell: bash + env: + TARGET: ${{ inputs.freshness-target-commit }} + MODULE_ID: ${{ inputs.freshness-module-id }} + AWS_REGION: ${{ inputs.aws-region }} + run: | + set -euo pipefail + # Missing marker configuration would disable the guard. + if [ -z "$MODULE_ID" ]; then + echo "::error::assert-fresh(infra): enforce-fresh is set but freshness-module-id is empty — refusing to apply unguarded"; exit 1 + fi + git fetch --no-tags --prune --unshallow origin 2>/dev/null || git fetch --no-tags origin 2>/dev/null || true + if ! git cat-file -e "${TARGET}^{commit}" 2>/dev/null; then git fetch --no-tags origin "$TARGET" 2>/dev/null || true; fi + if ! git cat-file -e "${TARGET}^{commit}" 2>/dev/null; then + echo "::error::assert-fresh(infra): target ${TARGET} is not a resolvable commit"; exit 1 + fi + # Read the marker distinguishing "not created yet" (legit first apply → + # fail OPEN) from a genuine read ERROR — access denied, throttle, timeout + # (anomalous → fail CLOSED). SSM is a reliable read, so an error here is a + # real red flag and we will NOT apply blind and risk a rollback. The app + # guard likewise uses the reliable ECS control plane and fails closed. + # `freshness-allow-stale` overrides for a deliberate run. + set +e + running="$(aws ssm get-parameter --name "/${MODULE_ID}/deployed_infra_commit" --query 'Parameter.Value' --output text --region "$AWS_REGION" 2>/tmp/ssm_err)" + ssm_rc=$? + set -e + skip=false + if [ "$ssm_rc" -ne 0 ]; then + if grep -q "ParameterNotFound" /tmp/ssm_err; then + running="" # marker not created yet (first apply) — the open check below logs it + else + echo "::error::assert-fresh(infra): could not read the deployed_infra_commit marker ($(tr -d '\n' /dev/null; then git fetch --no-tags origin "$running" 2>/dev/null || true; fi + if ! git cat-file -e "${running}^{commit}" 2>/dev/null; then + echo "::warning::assert-fresh(infra): live ${running:0:7} unresolvable in history — treating as stale"; skip=true + elif git merge-base --is-ancestor "$running" "$TARGET"; then + echo "assert-fresh(infra): live ${running:0:7} is an ancestor of ${TARGET:0:7} — fresh" + else + echo "::warning::assert-fresh(infra): live ${running:0:7} is NOT an ancestor of ${TARGET:0:7} — newer/divergent already applied; skipping terraform apply"; skip=true + fi + fi + echo "skip=$skip" >> "$GITHUB_OUTPUT" + - name: Configure AWS Credentials for Monitoring account uses: aws-actions/configure-aws-credentials@v4 with: @@ -105,6 +186,7 @@ jobs: ${{ inputs.tf-variables }} - name: Apply on ${{ inputs.stage }} + if: ${{ steps.freshness.outputs.skip != 'true' }} working-directory: ${{ inputs.tf-directory }} run: terraform apply -auto-approve -no-color @@ -114,3 +196,52 @@ jobs: with: workspace-id: ${{ steps.grafana-get-key.outputs.workspace-id }} key-name: ${{ steps.grafana-get-key.outputs.key-name }} + + - name: Configure AWS Credentials for marker write + if: ${{ inputs.enforce-fresh && !inputs.freshness-allow-stale && steps.freshness.outputs.skip != 'true' }} + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ inputs.freshness-aws-role-arn }} + aws-region: ${{ inputs.aws-region }} + + - name: Advance infra marker + if: ${{ inputs.enforce-fresh && !inputs.freshness-allow-stale && steps.freshness.outputs.skip != 'true' }} + shell: bash + env: + SHA: ${{ inputs.freshness-target-commit }} + MODULE_ID: ${{ inputs.freshness-module-id }} + AWS_REGION: ${{ inputs.aws-region }} + run: | + set -euo pipefail + set +e + current="$(aws ssm get-parameter --name "/${MODULE_ID}/deployed_infra_commit" --query 'Parameter.Value' --output text --region "$AWS_REGION" 2>/tmp/ssm_err)" + ssm_rc=$? + set -e + if [ "$ssm_rc" -ne 0 ]; then + if grep -q "ParameterNotFound" /tmp/ssm_err; then + aws ssm put-parameter --name "/${MODULE_ID}/deployed_infra_commit" --type String --overwrite --value "$SHA" --region "$AWS_REGION" >/dev/null + exit 0 + fi + echo "::warning::advance-infra-marker: could not read the deployed_infra_commit marker ($(tr -d '\n' /dev/null + exit 0 + fi + if ! git cat-file -e "${SHA}^{commit}" 2>/dev/null; then git fetch --no-tags origin "$SHA" 2>/dev/null || true; fi + if ! git cat-file -e "${current}^{commit}" 2>/dev/null; then git fetch --no-tags origin "$current" 2>/dev/null || true; fi + if ! git cat-file -e "${SHA}^{commit}" 2>/dev/null; then + echo "::warning::advance-infra-marker: target ${SHA:0:7} is unresolvable in history — marker not advanced" + exit 0 + fi + if ! git cat-file -e "${current}^{commit}" 2>/dev/null; then + echo "::warning::advance-infra-marker: current marker ${current:0:7} is unresolvable in history — marker not advanced" + exit 0 + fi + if git merge-base --is-ancestor "$current" "$SHA"; then + aws ssm put-parameter --name "/${MODULE_ID}/deployed_infra_commit" --type String --overwrite --value "$SHA" --region "$AWS_REGION" >/dev/null + else + echo "::warning::advance-infra-marker: marker ${current:0:7} newer/divergent than ${SHA:0:7} — refusing to regress" + exit 0 + fi