Segment and cache Stage 12 US simulations

.github/scripts/stage12-cache-precompute.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Build the independent Stage 12 cache and export its manifest digest.
+# Cache files are immutable and addressed by identity, so there is no force
+# option: a run computes only missing files.
+
+set -euo pipefail
+
+modal_environment="${1:?Modal environment is required}"
+: "${STAGE12_CACHE_BUCKET:?STAGE12_CACHE_BUCKET is required}"
+: "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}"
+
+output_file="$(mktemp)"
+trap 'rm -f "${output_file}"' EXIT
+uv run modal run --env="${modal_environment}" src/modal/stage12_precompute_app.py 2>&1 | tee "${output_file}"
+digest_line="$(tr -d '\r' <"${output_file}" | grep '^STAGE12_CACHE_MANIFEST_DIGEST=' | tail -n 1 || true)"
+if [[ -z "${digest_line}" ]]; then
+  echo "Stage 12 precompute returned no cache manifest digest" >&2
+  exit 1
+fi
+digest="${digest_line#STAGE12_CACHE_MANIFEST_DIGEST=}"
+if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then
+  echo "Stage 12 precompute returned an invalid cache manifest digest" >&2
+  exit 1
+fi
+echo "manifest_digest=${digest}" >>"${GITHUB_OUTPUT}"

.github/scripts/stage12-sync-modal-secrets.sh

@@ -9,6 +9,7 @@ modal_environment="${1:?Modal environment is required}"
 : "${STAGE12_DATABASE_URL_SECRET_NAME:?STAGE12_DATABASE_URL_SECRET_NAME is required}"
 : "${STAGE12_GCP_CREDENTIALS_SECRET_NAME:?STAGE12_GCP_CREDENTIALS_SECRET_NAME is required}"
 : "${STAGE12_ARTIFACT_BUCKET:?STAGE12_ARTIFACT_BUCKET is required}"
+: "${STAGE12_CACHE_BUCKET:?STAGE12_CACHE_BUCKET is required}"
 : "${MODAL_TOKEN_ID:?MODAL_TOKEN_ID is required}"
 : "${MODAL_TOKEN_SECRET:?MODAL_TOKEN_SECRET is required}"
 
@@ -34,7 +35,8 @@ gcloud secrets versions access latest \
 jq -n \
   --rawfile database_url "${database_url_file}" \
   --arg artifact_bucket "${STAGE12_ARTIFACT_BUCKET}" \
-  '{STAGE12_DATABASE_URL: ($database_url | rtrimstr("\n")), STAGE12_ARTIFACT_BUCKET: $artifact_bucket}' \
+  --arg cache_bucket "${STAGE12_CACHE_BUCKET}" \
+  '{STAGE12_DATABASE_URL: ($database_url | rtrimstr("\n")), STAGE12_ARTIFACT_BUCKET: $artifact_bucket, STAGE12_CACHE_BUCKET: $cache_bucket}' \
   >"${runtime_modal_file}"
 jq -n \
   --rawfile credentials "${credentials_file}" \

.github/scripts/stage12-validate-infrastructure.sh

@@ -7,6 +7,7 @@ set +x
 : "${STAGE12_ENVIRONMENT:?STAGE12_ENVIRONMENT is required}"
 : "${STAGE12_GCP_PROJECT_ID:?STAGE12_GCP_PROJECT_ID is required}"
 : "${STAGE12_ARTIFACT_BUCKET:?STAGE12_ARTIFACT_BUCKET is required}"
+: "${STAGE12_CACHE_BUCKET:?STAGE12_CACHE_BUCKET is required}"
 : "${STAGE12_ENTRYPOINT_SERVICE_ACCOUNT:?STAGE12_ENTRYPOINT_SERVICE_ACCOUNT is required}"
 : "${STAGE12_DEPLOY_SERVICE_ACCOUNT:?STAGE12_DEPLOY_SERVICE_ACCOUNT is required}"
 : "${STAGE12_MODAL_SERVICE_ACCOUNT:?STAGE12_MODAL_SERVICE_ACCOUNT is required}"
@@ -28,6 +29,11 @@ if [[ ! "${STAGE12_ARTIFACT_BUCKET}" =~ ^[a-z0-9][a-z0-9._-]{2,61}[a-z0-9]$ ]] |
   echo "STAGE12_ARTIFACT_BUCKET must be valid and identify the environment" >&2
   exit 1
 fi
+if [[ ! "${STAGE12_CACHE_BUCKET}" =~ ^[a-z0-9][a-z0-9._-]{2,61}[a-z0-9]$ ]] ||
+  [[ ! "${STAGE12_CACHE_BUCKET}" =~ ${STAGE12_ENVIRONMENT} ]]; then
+  echo "STAGE12_CACHE_BUCKET must be valid and identify the environment" >&2
+  exit 1
+fi
 for account in \
   "${STAGE12_ENTRYPOINT_SERVICE_ACCOUNT}" \
   "${STAGE12_DEPLOY_SERVICE_ACCOUNT}" \
@@ -68,6 +74,7 @@ canary_source_file="$(mktemp)"
 canary_download_file="$(mktemp)"
 runtime_gcloud_config="$(mktemp -d)"
 canary_object=""
+cache_canary_object=""
 runtime_gcloud() {
   # The GitHub auth action exports credentials for the deployment identity.
   # Remove them only here so the activated Modal worker account controls this check.
@@ -83,6 +90,10 @@ cleanup() {
     runtime_gcloud --account="${STAGE12_MODAL_SERVICE_ACCOUNT}" storage rm \
       "${canary_object}" --quiet >/dev/null 2>&1 || true
   fi
+  if [[ -n "${cache_canary_object}" ]]; then
+    runtime_gcloud --account="${STAGE12_MODAL_SERVICE_ACCOUNT}" storage rm \
+      "${cache_canary_object}" --quiet >/dev/null 2>&1 || true
+  fi
   rm -f \
     "${database_url_file}" \
     "${credentials_file}" \
@@ -164,6 +175,19 @@ runtime_gcloud --account="${STAGE12_MODAL_SERVICE_ACCOUNT}" storage rm \
   "${canary_object}" --quiet
 canary_object=""
 
+cache_canary_object="gs://${STAGE12_CACHE_BUCKET}/_deployment-validation/${STAGE12_ENVIRONMENT}/${canary_id}.txt"
+runtime_gcloud --account="${STAGE12_MODAL_SERVICE_ACCOUNT}" storage cp \
+  "${canary_source_file}" "${cache_canary_object}" --quiet
+runtime_gcloud --account="${STAGE12_MODAL_SERVICE_ACCOUNT}" storage cp \
+  "${cache_canary_object}" "${canary_download_file}" --quiet
+cmp "${canary_source_file}" "${canary_download_file}"
+gcloud storage cp \
+  "${cache_canary_object}" "${canary_download_file}" --quiet
+cmp "${canary_source_file}" "${canary_download_file}"
+runtime_gcloud --account="${STAGE12_MODAL_SERVICE_ACCOUNT}" storage rm \
+  "${cache_canary_object}" --quiet
+cache_canary_object=""
+
 for lookup_object in "${uk_geography_lookup_objects[@]}"; do
   runtime_gcloud --account="${STAGE12_MODAL_SERVICE_ACCOUNT}" storage cp \
     "${lookup_object}" "${canary_download_file}" --quiet
@@ -173,4 +197,4 @@ for lookup_object in "${uk_geography_lookup_objects[@]}"; do
   fi
 done
 
-echo "Pre-provisioned Stage 12 database, secret, artifact storage, and UK geography lookup access is verified."
+echo "Pre-provisioned Stage 12 database, secret, report storage, cache storage, and UK geography lookup access is verified."

.github/scripts/stage12-validate-v2-workers.sh

@@ -1,17 +1,20 @@
 #!/usr/bin/env bash
-# Validate the installed bundles, secrets, datasets and calculations of the
-# deployed v2 workers, and keep the result for manifest publication.
-# Usage: stage12-validate-v2-workers.sh <modal-environment>
+# Validate the installed bundles, secrets, datasets, calculations and Stage 12
+# cache of the deployed v2 workers, and keep the result for manifest
+# publication.
+# Usage: stage12-validate-v2-workers.sh <modal-environment> <cache-manifest-sha256>
 # Run from projects/policyengine-simulation-executor.
 # Required env vars: RUNNER_TEMP, GITHUB_OUTPUT.
 # Outputs: file (the validation result path) in GITHUB_OUTPUT.
 
 set -euo pipefail
 
 MODAL_ENV="${1:?Modal environment required}"
+CACHE_MANIFEST_SHA256="${2:?Stage 12 cache manifest digest required}"
 
 output="${RUNNER_TEMP}/stage12-worker-validation.json"
 uv run python -m src.modal.utils.validate_v2_worker_release \
   --environment "${MODAL_ENV}" \
+  --cache-manifest-sha256 "${CACHE_MANIFEST_SHA256}" \
   --output "${output}"
 echo "file=${output}" >> "$GITHUB_OUTPUT"

.github/workflows/simulation-deploy.reusable.yml

@@ -547,6 +547,7 @@ jobs:
           STAGE12_ENVIRONMENT: ${{ inputs.deployment_environment }}
           STAGE12_GCP_PROJECT_ID: ${{ vars.SIMULATION_ENTRYPOINT_GCP_PROJECT_ID }}
           STAGE12_ARTIFACT_BUCKET: ${{ vars.STAGE12_ARTIFACT_BUCKET }}
+          STAGE12_CACHE_BUCKET: ${{ vars.STAGE12_CACHE_BUCKET }}
           STAGE12_ENTRYPOINT_SERVICE_ACCOUNT: ${{ vars.SIMULATION_ENTRYPOINT_RUNTIME_SERVICE_ACCOUNT }}
           STAGE12_DEPLOY_SERVICE_ACCOUNT: ${{ secrets.SIMULATION_ENTRYPOINT_GCP_DEPLOY_SERVICE_ACCOUNT }}
           STAGE12_MODAL_SERVICE_ACCOUNT: ${{ vars.STAGE12_MODAL_SERVICE_ACCOUNT }}
@@ -568,6 +569,7 @@ jobs:
     outputs:
       application_name: ${{ steps.application.outputs.name }}
       validation_file: ${{ steps.validation.outputs.file }}
+      cache_manifest_digest: ${{ steps.stage12_cache.outputs.manifest_digest }}
     steps:
       - uses: actions/checkout@v6
 
@@ -603,11 +605,21 @@ jobs:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
           STAGE12_ARTIFACT_BUCKET: ${{ vars.STAGE12_ARTIFACT_BUCKET }}
+          STAGE12_CACHE_BUCKET: ${{ vars.STAGE12_CACHE_BUCKET }}
           STAGE12_DATABASE_URL_SECRET_NAME: ${{ vars.STAGE12_DATABASE_URL_SECRET_NAME }}
           STAGE12_GCP_PROJECT_ID: ${{ vars.SIMULATION_ENTRYPOINT_GCP_PROJECT_ID }}
           STAGE12_GCP_CREDENTIALS_SECRET_NAME: ${{ vars.STAGE12_GCP_CREDENTIALS_SECRET_NAME }}
         run: ../../.github/scripts/stage12-sync-modal-secrets.sh "${{ inputs.modal_environment }}"
 
+      - name: Build the independent Stage 12 cache
+        id: stage12_cache
+        working-directory: projects/policyengine-simulation-executor
+        env:
+          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
+          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
+          STAGE12_CACHE_BUCKET: ${{ vars.STAGE12_CACHE_BUCKET }}
+        run: ../../.github/scripts/stage12-cache-precompute.sh "${{ inputs.modal_environment }}"
+
       - name: Deploy the additional versioned v2 Modal application
         working-directory: projects/policyengine-simulation-executor
         env:
@@ -618,6 +630,8 @@ jobs:
           STAGE12_EXPECT_US_VERSION: ${{ needs.prepare.outputs.us_version }}
           STAGE12_EXPECT_UK_VERSION: ${{ needs.prepare.outputs.uk_version }}
           STAGE12_ARTIFACT_BUCKET: ${{ vars.STAGE12_ARTIFACT_BUCKET }}
+          STAGE12_CACHE_BUCKET: ${{ vars.STAGE12_CACHE_BUCKET }}
+          STAGE12_CACHE_MANIFEST_DIGEST: ${{ steps.stage12_cache.outputs.manifest_digest }}
           OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}
           OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}
           OBSERVABILITY_LOGGING_PROJECT_ID: ${{ vars.OBSERVABILITY_LOGGING_PROJECT_ID }}
@@ -634,7 +648,7 @@ jobs:
         env:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
-        run: ../../.github/scripts/stage12-validate-v2-workers.sh "${{ inputs.modal_environment }}"
+        run: ../../.github/scripts/stage12-validate-v2-workers.sh "${{ inputs.modal_environment }}" "${{ steps.stage12_cache.outputs.manifest_digest }}"
 
       - name: Retain worker validation for manifest publication
         uses: actions/upload-artifact@v7

.github/workflows/stage12-deploy.yml

@@ -54,7 +54,9 @@ jobs:
       entrypoint_min_instances: 0
       entrypoint_max_instances: 2
       promote_entrypoint: false
-      run_full_integration: true
+      # This workflow deploys and validates only the independent Stage 12 path.
+      # The main API deployment continues to run the existing-stack suite.
+      run_full_integration: false
       deploy_existing_stack: false
       deploy_stage12_v2: true
     secrets: inherit

AGENTS.md

@@ -93,7 +93,7 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`.
 
 ### Stage 12 persistence
 
-- `sim.stage12.deployment-scope`: Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources; the Stage 12 worker can request a Google OIDC token. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`)
+- `sim.stage12.deployment-scope`: Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources or runs the existing stack's complete integration suite; the Stage 12 worker can request a Google OIDC token. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`)
 - `sim.stage12.deployment-uses-api-schema`: Stage 12 deployment uses the API-owned schema directly, and the Stage 12 runtime it deploys uses typed DML without raw SQL or schema changes. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`)
 - `sim.stage12.no-migration-chain`: This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py`)
 - `sim.stage12.no-production-user-associations`: Stage 12 runtime modules never create production user associations (user_simulations or user_reports). (blocking: pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py`)

changelog_entry.yaml

@@ -3,6 +3,7 @@
     added:
       - Made CLAUDE.md and the Copilot instructions thin pointers to AGENTS.md, required workflow logic longer than one line to live in tested and shellchecked scripts, and registered the tracking-issue rule
       - Added a rules registry (rules/registry.toml) with a standard-library tool that validates it, checks every rule is enforced where it claims, and renders docs/generated/rules.md and the Rules block in AGENTS.md
+      - Ran Stage 12 US national simulations as 20 region-group calls each, and built an independent, immutable Stage 12 cache of three annual US datasets and 60 current-law baselines before each deployment, publishing it only after a recorded determinism check and recording its release in the v2 version manifest under a new key, while the existing key keeps a copy that earlier readers accept
     changed:
       - Ran ruff check in CI with the first group of rule families (imports, pyupgrade and other safe auto-fixes) and fixed every finding
       - Moved the multi-line `run` blocks of the simulation deployment workflows into single-line steps and tested scripts without changing their behaviour or outputs, leaving only the release-sentinel and routing-rollback blocks in the one-line `run` rule's baseline
@@ -23,3 +24,4 @@
       - Released each Stage 12 database session after its operation instead of holding pooled sessions through calculations, retried opening each Stage 12 coordinator and worker database connection that the pooler refused without ever retrying a statement or commit, and reported Stage 12 failures to telemetry before recording them in the database so a failed write cannot hide the original error
       - Accepted a null SPM provenance in Stage 12 aggregate artifacts, so UK and other reports without SPM calculations are saved, while every supplied provenance record is still validated
       - Recorded a redacted, bounded Stage 12 failure cause naming the exception, failing stage, simulation role and message instead of only the exception class, and kept a failed worker's recorded cause on its simulation and report
+      - Gave Stage 12 baseline and reform simulations one shared wait window that opens when the coordinator starts and fits inside the coordinator's Modal timeout, and cancelled every unfinished simulation, recording it as failed with simulation_cancelled, when a simulation fails, the window closes, or the coordinator fails after starting a simulation

docs/generated/rules.md

@@ -89,7 +89,7 @@ flowchart LR
 
 | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof |
 | --- | --- | --- | --- | --- | --- | --- | --- |
-| `sim.stage12.deployment-scope` | Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources; the Stage 12 worker can request a Google OIDC token. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** |
+| `sim.stage12.deployment-scope` | Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources or runs the existing stack's complete integration suite; the Stage 12 worker can request a Google OIDC token. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** |
 | `sim.stage12.deployment-uses-api-schema` | Stage 12 deployment uses the API-owned schema directly, and the Stage 12 runtime it deploys uses typed DML without raw SQL or schema changes. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** |
 | `sim.stage12.no-migration-chain` | This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** |
 | `sim.stage12.no-production-user-associations` | Stage 12 runtime modules never create production user associations (user_simulations or user_reports). | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** |