diff --git a/.env.example b/.env.example index b43e2dc22..dfd01ccb3 100644 --- a/.env.example +++ b/.env.example @@ -17,7 +17,9 @@ POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN=policyengine_github_token OPENAI_API_KEY=policyengine_openai_api_key # Token for Hugging Face models -HUGGING_FACE_TOKEN=policyengine_huggingface_token +# PolicyEngine Core compatibility variable. Managed deployments bind the +# PE_UK_PRIVATE_HF_READ_TOKEN credential to this process-level name. +HUGGING_FACE_TOKEN=pe_uk_private_hf_read_token_value # Redis is required for budget-window economy requests and other API cache # paths. Configure local development explicitly as described in README.md; diff --git a/.github/check-policyengine-bundle-supported.sh b/.github/check-policyengine-bundle-supported.sh index f48d1f076..20e78734d 100755 --- a/.github/check-policyengine-bundle-supported.sh +++ b/.github/check-policyengine-bundle-supported.sh @@ -51,10 +51,16 @@ if [ "$CHECK_ONLY_IF_CHANGED" = "1" ]; then || true )" - if [ "$current_version" = "$base_version" ]; then - echo "PolicyEngine .py bundle pin is unchanged; skipping simulation API support check." + if [ "$current_version" = "$base_version" ] \ + && git diff --quiet "origin/${BASE_REF}" -- \ + policyengine_api/spm.py policyengine_api/worker_spm.py \ + policyengine_api/worker_spm_release.py policyengine_api/constants.py \ + policyengine_api/country.py \ + .github/check-policyengine-bundle-supported.sh \ + .github/request-simulation-model-versions.sh; then + echo "Bundle pin and SPM integration are unchanged; skipping simulation API support check." exit 0 fi fi -bash "$VERSION_GUARD_SCRIPT" -py "$current_version" +bash "$VERSION_GUARD_SCRIPT" -py "$current_version" --check-installed-spm diff --git a/.github/find-api-model-versions.py b/.github/find-api-model-versions.py index 7a9a9f09b..1e5a262a4 100644 --- a/.github/find-api-model-versions.py +++ b/.github/find-api-model-versions.py @@ -1,38 +1,89 @@ import argparse import os +import shlex import sys +from importlib.metadata import version as distribution_version +from policyengine.bundle import get_current_bundle from policyengine_api.constants import ( COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_CORE_VERSION, POLICYENGINE_VERSION, ) +REQUIRED_PACKAGES = ( + "policyengine", + "policyengine-core", + "policyengine-us", + "policyengine-uk", + "spm-calculator", +) + + +def _manifest_versions(bundle: dict) -> dict[str, str]: + packages = bundle.get("packages") + if not isinstance(packages, dict): + raise RuntimeError("PolicyEngine bundle manifest has no package mapping.") + + versions = {} + for package_name in REQUIRED_PACKAGES: + package = packages.get(package_name) + if not isinstance(package, dict) or not package.get("version"): + raise RuntimeError( + f"PolicyEngine bundle manifest has no version for {package_name}." + ) + versions[package_name] = str(package["version"]) + return versions + + +def _data_release_version(bundle: dict, country_id: str) -> str: + data_releases = bundle.get("data_releases") + release = data_releases.get(country_id) if isinstance(data_releases, dict) else None + if not isinstance(release, dict) or not release.get("version"): + raise RuntimeError( + f"PolicyEngine bundle manifest has no {country_id.upper()} data release." + ) + return str(release["version"]) + def find_api_model_versions() -> dict[str, str]: """ - Find the API model versions from the installed PolicyEngine bundle. + Find and validate the package and data versions in the installed bundle. """ - us_version = COUNTRY_PACKAGE_VERSIONS.get("us") - uk_version = COUNTRY_PACKAGE_VERSIONS.get("uk") + bundle = get_current_bundle() + if not isinstance(bundle, dict): + raise RuntimeError("Installed PolicyEngine bundle manifest is not an object.") - if not us_version: - print("Error: US package version not found.", file=sys.stderr) - sys.exit(1) + manifest_versions = _manifest_versions(bundle) + for package_name, manifest_version in manifest_versions.items(): + installed_version = distribution_version(package_name) + if installed_version != manifest_version: + raise RuntimeError( + f"Installed {package_name} version {installed_version} does not match " + f"bundle manifest version {manifest_version}." + ) - if not uk_version: - print("Error: UK package version not found.", file=sys.stderr) - sys.exit(1) - - if not POLICYENGINE_VERSION: - print("Error: PolicyEngine package version not found.", file=sys.stderr) - sys.exit(1) + expected_constants = { + "policyengine": POLICYENGINE_VERSION, + "policyengine-core": POLICYENGINE_CORE_VERSION, + "policyengine-us": COUNTRY_PACKAGE_VERSIONS.get("us"), + "policyengine-uk": COUNTRY_PACKAGE_VERSIONS.get("uk"), + } + for package_name, constant_version in expected_constants.items(): + if constant_version != manifest_versions[package_name]: + raise RuntimeError( + f"API version for {package_name} is {constant_version}, but the bundle " + f"manifest specifies {manifest_versions[package_name]}." + ) return { - "POLICYENGINE_VERSION": POLICYENGINE_VERSION, - "POLICYENGINE_CORE_VERSION": POLICYENGINE_CORE_VERSION, - "US_VERSION": us_version, - "UK_VERSION": uk_version, + "POLICYENGINE_VERSION": manifest_versions["policyengine"], + "POLICYENGINE_CORE_VERSION": manifest_versions["policyengine-core"], + "US_VERSION": manifest_versions["policyengine-us"], + "UK_VERSION": manifest_versions["policyengine-uk"], + "SPM_CALCULATOR_VERSION": manifest_versions["spm-calculator"], + "US_DATA_VERSION": _data_release_version(bundle, "us"), + "UK_DATA_VERSION": _data_release_version(bundle, "uk"), } @@ -57,7 +108,7 @@ def find_api_model_versions_and_output_to_github(): if args.shell: for key, value in find_api_model_versions().items(): - print(f"{key}={value}") + print(f"{key}={shlex.quote(value)}") else: find_api_model_versions_and_output_to_github() print("API model versions found and written to GitHub environment.") diff --git a/.github/request-simulation-model-versions.sh b/.github/request-simulation-model-versions.sh index 27a68062c..b2b38240d 100755 --- a/.github/request-simulation-model-versions.sh +++ b/.github/request-simulation-model-versions.sh @@ -23,12 +23,14 @@ usage() { echo "Optional compatibility checks:" echo " -us us_version Expected bundled policyengine-us version" echo " -uk uk_version Expected bundled policyengine-uk version" + echo " --check-installed-spm Compare the installed bundle with worker capability" exit 1 } POLICYENGINE_VERSION="" US_VERSION="" UK_VERSION="" +CHECK_INSTALLED_SPM=0 while [ $# -gt 0 ]; do case "$1" in @@ -44,6 +46,10 @@ while [ $# -gt 0 ]; do UK_VERSION="$2" shift 2 ;; + --check-installed-spm) + CHECK_INSTALLED_SPM=1 + shift + ;; -h|--help) usage ;; @@ -70,7 +76,7 @@ if [ -n "$UK_VERSION" ]; then fi echo "" -VERSIONS_RESPONSE=$(curl -s "${GATEWAY_URL}/versions") +VERSIONS_RESPONSE=$(curl --fail --silent --show-error --connect-timeout 10 --max-time 60 "${GATEWAY_URL}/versions") if [ -z "$VERSIONS_RESPONSE" ]; then echo "ERROR: Failed to fetch versions from gateway" @@ -114,6 +120,11 @@ check_country_route() { check_country_route "us" "$US_VERSION" check_country_route "uk" "$UK_VERSION" +if [ "$CHECK_INSTALLED_SPM" = "1" ]; then + printf '%s' "$VERSIONS_RESPONSE" \ + | uv run --frozen python -m policyengine_api.worker_spm_release "$POLICYENGINE_VERSION" +fi + echo "" echo "SUCCESS: PolicyEngine bundle route is deployed and ready" exit 0 diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index c97672628..3a0743fec 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -39,7 +39,7 @@ cloud_run_set_defaults() { CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-}" CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}" CLOUD_RUN_OPENAI_API_KEY_SECRET="${CLOUD_RUN_OPENAI_API_KEY_SECRET:-policyengine-api-prod-openai-api-key:latest}" - CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET="${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET:-policyengine-api-prod-hugging-face-token:latest}" + CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET:-pe-uk-private-hf-read-token:latest}" CLOUD_RUN_RUNTIME_CACHE_URL_SECRET="${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET:-policyengine-api-prod-runtime-cache-url:latest}" CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET="${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET:-policyengine-api-prod-runtime-cache-ca:latest}" CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT="${CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT:-production}" @@ -76,7 +76,7 @@ cloud_run_set_defaults() { export CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET export CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET export CLOUD_RUN_OPENAI_API_KEY_SECRET - export CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET + export CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET export CLOUD_RUN_RUNTIME_CACHE_URL_SECRET export CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET export CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT diff --git a/.github/scripts/deploy_cloud_run_candidate.sh b/.github/scripts/deploy_cloud_run_candidate.sh index 2e5d80773..827403ce0 100755 --- a/.github/scripts/deploy_cloud_run_candidate.sh +++ b/.github/scripts/deploy_cloud_run_candidate.sh @@ -62,7 +62,9 @@ secret_vars=( "POLICYENGINE_DB_PASSWORD=${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}" "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN=${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET}" "OPENAI_API_KEY=${CLOUD_RUN_OPENAI_API_KEY_SECRET}" - "HUGGING_FACE_TOKEN=${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET}" + # PolicyEngine Core reads HUGGING_FACE_TOKEN. The stored credential and every + # deployment input use the purpose-specific PE_UK_PRIVATE_HF_READ_TOKEN name. + "HUGGING_FACE_TOKEN=${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET}" "RUNTIME_CACHE_URL=${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET}" "RUNTIME_CACHE_CA_CERT=${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET}" ) diff --git a/.github/scripts/resolve_cloud_run_candidate_state.sh b/.github/scripts/resolve_cloud_run_candidate_state.sh index 334a42b66..60f3b49c7 100755 --- a/.github/scripts/resolve_cloud_run_candidate_state.sh +++ b/.github/scripts/resolve_cloud_run_candidate_state.sh @@ -78,6 +78,32 @@ image="$(jq -er ' | select(type == "string" and contains("@sha256:")) ' <<<"${revision_json}")" +expected_hf_secret_resource="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET%:*}" +expected_hf_secret_name="${expected_hf_secret_resource##*/}" +expected_hf_secret_version="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET##*:}" +actual_hf_secret_binding="$(jq -cer ' + [ + .spec.containers[0].env[]? + | select(.name == "HUGGING_FACE_TOKEN") + | { + name: .valueFrom.secretKeyRef.name, + version: .valueFrom.secretKeyRef.key + } + ] + | if length == 1 then .[0] + else error("HUGGING_FACE_TOKEN must have exactly one secret binding") + end +' <<<"${revision_json}")" +actual_hf_secret_name="$(jq -er '.name' <<<"${actual_hf_secret_binding}")" +actual_hf_secret_version="$(jq -er '.version' <<<"${actual_hf_secret_binding}")" +if [[ "${actual_hf_secret_name}" != "${expected_hf_secret_name}" \ + || "${actual_hf_secret_version}" != "${expected_hf_secret_version}" ]]; then + printf 'Revision %s binds HUGGING_FACE_TOKEN to %s:%s; expected %s:%s\n' \ + "${revision}" "${actual_hf_secret_name}" "${actual_hf_secret_version}" \ + "${expected_hf_secret_name}" "${expected_hf_secret_version}" >&2 + exit 2 +fi + deployment_selector_count=0 for selector in \ ROUTE_IMPL_HEALTH \ diff --git a/.github/scripts/sync_cloud_run_secrets.sh b/.github/scripts/sync_cloud_run_secrets.sh index a83b3599c..3ee45fa82 100644 --- a/.github/scripts/sync_cloud_run_secrets.sh +++ b/.github/scripts/sync_cloud_run_secrets.sh @@ -4,6 +4,21 @@ set -euo pipefail set +x CLOUD_RUN_PROJECT="${CLOUD_RUN_PROJECT:-policyengine-api}" +CLOUD_RUN_REGION="${CLOUD_RUN_REGION:-us-central1}" +GCLOUD_BIN="${GCLOUD_BIN:-gcloud}" +include_database_password=0 +case "${1:-}" in + "") ;; + --include-database-password) include_database_password=1 ;; + *) + echo "::error::Unknown argument: $1" >&2 + exit 2 + ;; +esac +if [[ "$#" -gt 1 ]]; then + echo "::error::Expected at most one argument." >&2 + exit 2 +fi require_env() { local env_name="$1" @@ -17,35 +32,119 @@ sync_secret() { local env_name="$1" local secret_name="$2" local secret_value="${!env_name:-}" + local expected_hash + local stored_hash="" if [[ -z "${secret_value}" ]]; then echo "::error::Missing required GitHub secret ${env_name}." exit 1 fi - if ! gcloud secrets describe "${secret_name}" \ + if ! "${GCLOUD_BIN}" secrets describe "${secret_name}" \ --project "${CLOUD_RUN_PROJECT}" >/dev/null 2>&1; then - gcloud secrets create "${secret_name}" \ + "${GCLOUD_BIN}" secrets create "${secret_name}" \ --project "${CLOUD_RUN_PROJECT}" \ --replication-policy automatic fi - printf '%s' "${secret_value}" | gcloud secrets versions add \ - "${secret_name}" \ - --project "${CLOUD_RUN_PROJECT}" \ - --data-file=- >/dev/null + expected_hash="$(printf '%s' "${secret_value}" | sha256sum | cut -d ' ' -f 1)" + if stored_hash="$( + "${GCLOUD_BIN}" secrets versions access latest \ + --secret "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}" 2>/dev/null \ + | sha256sum \ + | cut -d ' ' -f 1 + )" && [[ "${stored_hash}" == "${expected_hash}" ]]; then + echo "Secret Manager already matches ${env_name}; no version added." + else + printf '%s' "${secret_value}" | "${GCLOUD_BIN}" secrets versions add \ + "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}" \ + --data-file=- >/dev/null + echo "Synchronized ${env_name} to Secret Manager secret ${secret_name}." + fi + + verify_secret_value "${env_name}" "${secret_name}" + grant_secret_access "${secret_name}" + unset secret_value expected_hash stored_hash +} + +verify_secret_value() { + local env_name="$1" + local secret_name="$2" + local expected_value="${!env_name:-}" + local expected_hash + local stored_hash + + expected_hash="$(printf '%s' "${expected_value}" | sha256sum | cut -d ' ' -f 1)" + stored_hash="$( + "${GCLOUD_BIN}" secrets versions access latest \ + --secret "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}" \ + | sha256sum \ + | cut -d ' ' -f 1 + )" + if [[ "${stored_hash}" != "${expected_hash}" ]]; then + echo "::error::Secret Manager value verification failed for ${secret_name}." >&2 + return 1 + fi + unset stored_hash expected_hash expected_value + echo "Verified ${env_name} in Secret Manager without printing its value." +} + +grant_secret_access() { + local secret_name="$1" + local member - gcloud secrets add-iam-policy-binding "${secret_name}" \ + member="serviceAccount:${runtime_service_account}" + + "${GCLOUD_BIN}" secrets add-iam-policy-binding "${secret_name}" \ --project "${CLOUD_RUN_PROJECT}" \ - --member "serviceAccount:${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" \ + --member "${member}" \ --role roles/secretmanager.secretAccessor >/dev/null - echo "Synced ${env_name} to Secret Manager secret ${secret_name}." + if ! "${GCLOUD_BIN}" secrets get-iam-policy "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}" \ + --format=json \ + | jq -e --arg member "${member}" ' + any( + .bindings[]?; + .role == "roles/secretmanager.secretAccessor" + and ((.members // []) | index($member) != null) + ) + ' >/dev/null; then + echo "::error::Secret Manager access verification failed for ${CLOUD_RUN_SERVICE}." >&2 + return 1 + fi + echo "Verified ${CLOUD_RUN_SERVICE} can read ${secret_name}." } -require_env CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT +require_env CLOUD_RUN_SERVICE + +runtime_service_account="$( + "${GCLOUD_BIN}" run services describe "${CLOUD_RUN_SERVICE}" \ + --project "${CLOUD_RUN_PROJECT}" \ + --region "${CLOUD_RUN_REGION}" \ + --platform managed \ + --format='value(spec.template.spec.serviceAccountName)' +)" +if [[ -z "${runtime_service_account}" ]]; then + echo "::error::Cloud Run service ${CLOUD_RUN_SERVICE} has no runtime service account." >&2 + exit 1 +fi -sync_secret POLICYENGINE_DB_PASSWORD policyengine-api-prod-db-password +if [[ "${include_database_password}" -eq 1 ]]; then + require_env CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET + db_password_secret_version="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET##*:}" + db_password_secret_resource="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET%:*}" + db_password_secret_name="${db_password_secret_resource##*/}" + if [[ "${db_password_secret_version}" != "latest" \ + || -z "${db_password_secret_name}" ]]; then + echo "::error::CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET must name a :latest secret version." >&2 + exit 1 + fi + sync_secret POLICYENGINE_DB_PASSWORD "${db_password_secret_name}" +fi sync_secret POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN policyengine-api-prod-github-microdata-token sync_secret OPENAI_API_KEY policyengine-api-prod-openai-api-key -sync_secret HUGGING_FACE_TOKEN policyengine-api-prod-hugging-face-token +sync_secret PE_UK_PRIVATE_HF_READ_TOKEN pe-uk-private-hf-read-token diff --git a/.github/scripts/update-policyengine-package.sh b/.github/scripts/update-policyengine-package.sh index ac8345fec..12a47403e 100755 --- a/.github/scripts/update-policyengine-package.sh +++ b/.github/scripts/update-policyengine-package.sh @@ -11,6 +11,7 @@ # repository_dispatch trigger passes the just-released version here); # otherwise the latest version on PyPI is used. # FORCE=1 allows targeting a version that is not newer than the current pin. +# LOCK_RETRY_SECONDS controls the pause between lock retries (default: 30). set -euo pipefail DRY_RUN=0 @@ -30,6 +31,12 @@ Update PolicyEngine .py bundle from ${CURRENT} to ${LATEST}. - policyengine-core: ${POLICYENGINE_CORE_VERSION:-resolved during update} - policyengine-us: ${US_VERSION:-resolved during update} - policyengine-uk: ${UK_VERSION:-resolved during update} +- spm-calculator: ${SPM_CALCULATOR_VERSION:-resolved during update} + +## Certified data releases + +- US: ${US_DATA_VERSION:-resolved during update} +- UK: ${UK_DATA_VERSION:-resolved during update} --- Generated automatically by GitHub Actions @@ -84,11 +91,9 @@ if [[ "$DRY_RUN" == "1" ]]; then exit 0 fi -EXISTING_PR=$(gh pr list \ - --head "$BRANCH" \ - --state open \ - --json number \ - --jq '.[0].number' 2>/dev/null || true) +EXISTING_PR=$(gh pr view "$BRANCH" \ + --json number,state \ + --jq 'select(.state == "OPEN") | .number' 2>/dev/null || true) if [[ -n "$EXISTING_PR" ]]; then echo "PR #${EXISTING_PR} already exists for ${BRANCH}. Skipping." exit 0 @@ -138,13 +143,20 @@ for attempt in 1 2 3; do echo "ERROR: uv lock failed after ${attempt} attempts." >&2 exit 1 fi - echo "uv lock attempt ${attempt} failed; retrying in 30s..." - sleep 30 + echo "uv lock attempt ${attempt} failed; retrying in ${LOCK_RETRY_SECONDS:-30}s..." + sleep "${LOCK_RETRY_SECONDS:-30}" done -VERSIONS_OUTPUT=$(uv run python .github/find-api-model-versions.py --shell) +uv lock --check + +VERSIONS_OUTPUT=$(uv run --frozen python .github/find-api-model-versions.py --shell) eval "$VERSIONS_OUTPUT" +if [[ "$POLICYENGINE_VERSION" != "$LATEST" ]]; then + echo "Installed PolicyEngine version ${POLICYENGINE_VERSION} does not match requested version ${LATEST}." >&2 + exit 1 +fi + FRAGMENT="changelog.d/update-policyengine-bundle-${LATEST}.changed.md" echo "Update the PolicyEngine bundle to ${LATEST}." > "$FRAGMENT" diff --git a/.github/scripts/validate_cloud_run_deploy_env.sh b/.github/scripts/validate_cloud_run_deploy_env.sh index 8647f1235..1f1e32451 100755 --- a/.github/scripts/validate_cloud_run_deploy_env.sh +++ b/.github/scripts/validate_cloud_run_deploy_env.sh @@ -29,7 +29,7 @@ cloud_run_require_env \ CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET \ CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET \ CLOUD_RUN_OPENAI_API_KEY_SECRET \ - CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET \ + CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET \ CLOUD_RUN_RUNTIME_CACHE_URL_SECRET \ CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET \ CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT \ diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index dee90cc71..b2bfb28ed 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -27,6 +27,8 @@ jobs: fetch-depth: 0 - name: Install jq run: sudo apt-get install -y jq + - name: Set up uv for installed bundle validation + uses: astral-sh/setup-uv@v6 - name: Check simulation API supports updated PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh --if-changed-from-base "${{ github.base_ref }}" @@ -145,7 +147,9 @@ jobs: run: pytest tests/env_variables/test_environment_variables.py env: POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + # Compatibility name required by PolicyEngine Core. The GitHub + # credential name describes its provider, data, and read-only scope. + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} test: name: Test @@ -184,7 +188,7 @@ jobs: POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 with: diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 6b817ab14..856b4f570 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -60,6 +60,8 @@ jobs: uses: actions/checkout@v4 - name: Install jq run: sudo apt-get install -y jq + - name: Set up uv for installed bundle validation + uses: astral-sh/setup-uv@v6 - name: Check simulation API supports PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh @@ -240,7 +242,8 @@ jobs: POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + # Compatibility name required by PolicyEngine Core. + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} @@ -254,6 +257,14 @@ jobs: uses: "google-github-actions/setup-gcloud@v2" - name: Install jq run: sudo apt-get install -y jq + # The staging database password is managed independently in Secret Manager + # and differs from the repository-level production database password. + - name: Synchronize shared Cloud Run runtime secrets + env: + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + run: bash .github/scripts/sync_cloud_run_secrets.sh - name: Capture current Cloud Run staging state id: previous run: bash .github/scripts/capture_cloud_run_service_state.sh >> "$GITHUB_OUTPUT" @@ -402,6 +413,8 @@ jobs: uses: actions/checkout@v4 - name: Install jq run: sudo apt-get install -y jq + - name: Set up uv for installed bundle validation + uses: astral-sh/setup-uv@v6 - name: Check simulation API supports PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh @@ -488,6 +501,13 @@ jobs: uses: "google-github-actions/setup-gcloud@v2" - name: Install jq run: sudo apt-get install -y jq + - name: Synchronize complete Cloud Run runtime secret batch + env: + POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + run: bash .github/scripts/sync_cloud_run_secrets.sh --include-database-password - name: Capture current Cloud Run production state id: previous run: bash .github/scripts/capture_cloud_run_service_state.sh >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/sync-cloud-run-secrets.yml b/.github/workflows/sync-cloud-run-secrets.yml index 77b766777..c6b46f80a 100644 --- a/.github/workflows/sync-cloud-run-secrets.yml +++ b/.github/workflows/sync-cloud-run-secrets.yml @@ -2,6 +2,15 @@ name: Sync Cloud Run secrets on: workflow_dispatch: + inputs: + deployment_environment: + description: GitHub and Cloud Run environment to synchronize + required: true + default: production + type: choice + options: + - staging + - production concurrency: group: cloud-run-secret-sync @@ -10,7 +19,7 @@ jobs: sync-cloud-run-secrets: name: Sync GitHub secrets to Secret Manager runs-on: ubuntu-latest - environment: production + environment: ${{ inputs.deployment_environment }} permissions: contents: read id-token: write @@ -29,12 +38,25 @@ jobs: service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}" - name: Set up GCloud uses: "google-github-actions/setup-gcloud@v2" - - name: Sync runtime secrets + - name: Install jq + run: sudo apt-get install -y jq + - name: Sync shared staging runtime secrets + if: inputs.deployment_environment == 'staging' env: CLOUD_RUN_PROJECT: policyengine-api - CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT: ${{ secrets.GCP_CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT }} - POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} + CLOUD_RUN_SERVICE: policyengine-api-staging POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} run: bash .github/scripts/sync_cloud_run_secrets.sh + - name: Sync complete production runtime secret batch + if: inputs.deployment_environment == 'production' + env: + CLOUD_RUN_PROJECT: policyengine-api + CLOUD_RUN_SERVICE: policyengine-api + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} + POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + run: bash .github/scripts/sync_cloud_run_secrets.sh --include-database-password diff --git a/README.md b/README.md index 9f0a4470d..f572bcdf9 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,9 @@ make setup-env - `POLICYENGINE_DB_INSTANCE_CONNECTION_NAME` - `POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN` - `OPENAI_API_KEY` -- `HUGGING_FACE_TOKEN` +- `HUGGING_FACE_TOKEN` (the local PolicyEngine Core compatibility variable; + managed deployments source it from the purpose-specific GitHub secret + `PE_UK_PRIVATE_HF_READ_TOKEN`) The database settings must resolve to an explicit durable development MySQL database (or an authorized Cloud SQL development target). `FLASK_DEBUG` does diff --git a/changelog.d/3862.breaking.md b/changelog.d/3862.breaking.md new file mode 100644 index 000000000..4a719cf50 --- /dev/null +++ b/changelog.d/3862.breaking.md @@ -0,0 +1 @@ +Replace large SPM diagnostic provenance payloads and duplicate completed-result SPM configuration with strict compact v2 receipts for household, society-wide, report, and Stage 12 artifact results. Older cached or stored SPM output is recalculated instead of translated, and runtime version values are required. diff --git a/changelog.d/policyengine-6-2-1.changed.md b/changelog.d/policyengine-6-2-1.changed.md new file mode 100644 index 000000000..1f716b484 --- /dev/null +++ b/changelog.d/policyengine-6-2-1.changed.md @@ -0,0 +1 @@ +Update the API to PolicyEngine.py 6.2.1, including PolicyEngine Core 3.32.10, PolicyEngine US 2.2.1, PolicyEngine UK 2.102.3, and spm-calculator 1.0.0. Validate that simulation workers report the same bundle and SPM measurement settings before submitting US economy calculations. diff --git a/changelog.d/uk-private-hf-read-token.fixed.md b/changelog.d/uk-private-hf-read-token.fixed.md new file mode 100644 index 000000000..50f27be5a --- /dev/null +++ b/changelog.d/uk-private-hf-read-token.fixed.md @@ -0,0 +1 @@ +Use a dedicated read-only Hugging Face credential for private UK runtime data, synchronize GitHub-owned runtime secrets before Cloud Run deployment, and verify their values and access bindings. diff --git a/docker/Dockerfile b/docker/Dockerfile index 7b08cd132..83b3cabe4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ FROM python:3.12 # Match the API bundle in pyproject.toml; the bundle updater changes both pins. -# Exact bundled model requirements prevent pip from silently backtracking. -RUN pip install "policyengine[models]==5.2.0" spm-calculator==0.3.1 ipython +# The models extra pins the complete certified runtime package set. +RUN pip install "policyengine[models]==6.2.1" ipython diff --git a/docs/canonical-spm.md b/docs/canonical-spm.md index 042e3be71..9794be4bb 100644 --- a/docs/canonical-spm.md +++ b/docs/canonical-spm.md @@ -1,8 +1,8 @@ # Canonical SPM household API contract -This contract is enabled only by an installed, certified US bundle that pins its -SPM forecast hash/scenario and supports the country `spm` constructor. This change -does not select a new released model or promote a deployment. +This contract is enabled only by an installed, certified US bundle that identifies +one SPM forecast file and scenario and supports the country `spm` constructor. +Installing a package version does not deploy the API or its simulation worker. Whether a bundle predates this contract is decided by the installed country model's capability, never by a bundle version string; the automated bundle update @@ -13,10 +13,48 @@ its existing behavior when settings are omitted and rejects explicit SPM setting with `SPM_SETTINGS_UNSUPPORTED`, whatever its version. A bundle whose US model does implement the constructor but ships no certified `measurements.spm` configuration fails closed with `SPM_CONFIGURATION_UNAVAILABLE`: such a -deployment also fails `/readiness-check`, so the condition is reported where the -release is gated rather than only on each request. Other countries retain their +deployment also fails `/readiness-check`, so the release checks report the +condition rather than leaving it to individual requests. Other countries retain their behavior and reject US-only SPM settings. +## PolicyEngine 6 compatibility + +This API pins one exact `policyengine[models]` release in `pyproject.toml` and +the generic Docker image. That requirement installs the five package versions +recorded in that PolicyEngine.py release's bundle manifest: + +- `policyengine` +- `policyengine-core` +- `policyengine-us` +- `policyengine-uk` +- `spm-calculator` + +The same manifest identifies the certified US and UK data releases. The API +does not declare separate country-model or SPM calculator requirements. This +prevents an independently selected package version from disagreeing with the +tested combination in the manifest. + +PolicyEngine.py 6 also records the exact SPM forecast file and default scenario +used for US calculations. The API resolves those values before it submits an +economy calculation. It then checks that the worker reports the same values and +that every completed result includes a receipt describing the forecast, scenario, +years, and geography it used. This is the concrete behavior referred to elsewhere +as SPM selection and provenance. + +The API and simulation worker can be reviewed independently. Economy calculations +using this API version require a worker whose `/versions` response maps the +installed PolicyEngine.py, US, and UK versions to the same worker application and +reports the matching `canonical-spm-v1` settings. Until such a worker is deployed, +the API returns `SPM_CONFIGURATION_UNAVAILABLE` instead of submitting a calculation +to an incompatible worker. [Simulation API PR 703](https://github.com/PolicyEngine/policyengine-sim-api/pull/703) +implements the corresponding worker package update. + +The automated dependency updater changes the single `policyengine[models]` +requirement in `pyproject.toml` and the generic Docker image, refreshes `uv.lock`, +and checks all five installed distributions against the new manifest. Its pull +request description lists the five package versions and both certified data +release identifiers. + ## Selecting a measurement `GET /us/metadata` exposes `result.spm.available`. When true, `settings_schema` @@ -155,16 +193,44 @@ storage; no additional schema migration is required. selected policy. No independent simulation-level override is supported. Top-level `spm` on simulation POST/PATCH is rejected instead of silently ignored. -Successful canonical calculations add `spm_config` and `spm_provenance` beside -`result`. A receipt describes the measurement the simulation was constructed -with, not a guarantee that every SPM-dependent variable produced a value: a +Successful canonical calculations add only `spm_provenance` beside `result`. +Its forecast, scenario, geography, county-vintage and as-of fields are the +complete resolved calculation configuration; completed results do not repeat +that configuration in a sibling `spm_config` field. A receipt describes the +measurement the simulation was constructed with, not a guarantee that every +SPM-dependent variable produced a value: a calculation that never chose a measurement still carries the inherited one's receipt beside its null cells. Read the values to learn which of them a -measurement produced. Provenance comes from the actual simulation and includes artifact, -scenario, years, geography, composition/storage methods and runtime versions. -It remains in JSON form through stored replay and cache hits. A tax-only receipt -may have empty `years` and `geographies` because no SPM measurement was requested. -Clients saving simulation outputs must retain this full response envelope. +measurement produced. Provenance comes from the actual simulation and uses the +required compact `canonical-spm-provenance-v2` schema: + +```json +{ + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "canonical-spm-forecast", + "forecast_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "scenario": "baseline", + "geography_kind": "national", + "geography_id": null, + "county_vintage": "2020", + "as_of": null, + "years": ["2026"], + "runtime_versions": { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0" + } +} +``` + +All fields are required. The four runtime-version values are non-null strings. +`years` is sorted and unique, and it is empty only when no SPM dependency ran. +Large diagnostic, mapping, request, effective-setting and source +objects from the country package are intentionally excluded from HTTP and +cache payloads. The compact receipt remains in JSON form through stored replay +and cache hits. Clients saving simulation outputs must retain this response +envelope. `POST /us/simulation` takes `population_id`, `population_type` (`household` or `geography`) and integer `policy_id`. It creates a pending record (HTTP 201) or @@ -181,12 +247,13 @@ country, certification, or whether the household has saved `spm`. `PATCH /us/simulation` identifies the record with body `id` and accepts `status` (`pending`, `complete` or `error`), `output` and `error_message`. At least one update field must be non-null, and `complete` requires non-null `output`. Store -the full calculation envelope inside `output`, including its `spm_config` and +the full calculation envelope inside `output`, including its `spm_provenance`. A JSON-encoded output string is also accepted. This endpoint -stores the supplied output; it does not validate SPM receipt integrity. Null -update fields are ignored. The legacy `api_version` input is ignored; writes -record the installed country model version. Top-level `spm`, even null, returns -HTTP 400 `SPM_SETTINGS_UNSUPPORTED` on POST and PATCH. +validates the one compact SPM receipt strictly before storage. Completed output +that includes the former sibling `spm_config`, an old receipt or a rich receipt +is rejected rather than translated. Null update fields are ignored. The legacy `api_version` input +is ignored; writes record the installed country model version. Top-level `spm`, +even null, returns HTTP 400 `SPM_SETTINGS_UNSUPPORTED` on POST and PATCH. POST, PATCH and `GET /us/simulation/{id}` return the simulation record inside `result`. Non-string JSON `output` and `simulation_spec_json` are returned as @@ -201,9 +268,10 @@ Flask and the native specification route. HTTP response cache identity includes the normalized selection and model/bundle versions, and validates certification before reading the cache. The stored -calculated-household cache uses schema version 2, includes the selection in -identity, and stores settings/provenance atomically with the result and its -warnings. Missing or mismatched canonical receipts are cache misses. +calculated-household cache uses schema version 4, includes the selection in +identity, and stores provenance atomically with the result and its +warnings. Missing, old, rich or mismatched canonical receipts are cache misses +and are recalculated rather than translated. ## Errors @@ -219,7 +287,8 @@ SPM input failures return HTTP 400 in the existing validation envelope: ``` `SPM_GEOGRAPHY_REQUIRED` indicates missing explicit geography for an SPM -dependency; `SPM_GEOGRAPHY_UNAVAILABLE` indicates malformed/unknown county or area; +dependency, including a county value that is not a five-digit FIPS code; +`SPM_GEOGRAPHY_UNAVAILABLE` indicates a syntactically valid but unknown county or area; `SPM_COMPOSITION_REQUIRED` indicates no classified SPM adult. Country error text is retained. `SPM_YEAR_UNAVAILABLE` indicates an unsupported measurement year. Settings errors use `SPM_SETTINGS_INVALID`, @@ -261,22 +330,37 @@ from `/versions`, including agreement with the bundle's independently pinned artifact hash. An old or uncertified worker returns `SPM_CONFIGURATION_UNAVAILABLE`. Normalized settings are sent to both annual and budget-window jobs and participate -in economy cache identity. Results carry `spm_config` and `spm_provenance` with -`baseline` and `reform` receipt lists (one per executed regional segment). Each -budget-window annual row carries these fields; annual results expose them inside -`result`, and budget-window results inside `result.annualImpacts`. Stored results require matching -settings and valid receipts. Typed SPM input errors remain structured 400s +in economy cache identity. Results carry only `spm_provenance` using +`canonical-spm-comparison-v2`. `baseline` and `reform` each contain one compact +`receipt` plus an `execution_count`; the receipt collapses identical child +execution receipts without transporting a repeated list. Baseline and reform +receipts must be identical, and their receipt fields are the sole resolved SPM +configuration. Each budget-window annual row carries this field; +annual results expose it inside `result`, and budget-window results inside +`result.annualImpacts`. Stored results require matching settings and valid +receipts. Economy cache identity includes the comparison schema version, so old +entries are not selected and are recomputed rather than translated. Typed SPM input errors remain structured 400s through asynchronous submission and polling. Poll-time typed failures retain their code and message for the existing cache lifetime. Later reads replay that failure after API service restarts without polling or resubmitting the failed job. Canonical cache identity and runtime-bundle refresh rules still apply. +Completed US economy report outputs pass the same compact comparison validation +before `report_outputs` or `report_output_runs` persistence. Their report cache +identity uses schema version 3. Reads of earlier report records resolve +to a new pending record with no copied output, preventing another client from +receiving a previously stored rich receipt. Household and non-US report output +shapes retain their existing behavior. + See the [canonical SPM worker PR](https://github.com/PolicyEngine/policyengine-sim-api/pull/677) -for the implemented worker paths and remaining coordinated release gates. +for the implemented worker paths and remaining coordinated release validation +checks. See [Simulation API PR 703](https://github.com/PolicyEngine/policyengine-sim-api/pull/703) +for the corresponding PolicyEngine.py 6.2.1 worker package update and compatibility +checks. Partial selections preserve omitted fields in JSON; omissions inherit the certified bundle defaults. The selection schema deliberately supplies no wire defaults, so generated clients preserve that distinction. Resolved settings freeze all six -fields. Responses may omit null geography_id/as_of values, but must include every -non-null resolved field; receipt replay never substitutes current defaults. +fields. Responses include all six fields, including explicit null values for +`geography_id` or `as_of`; receipt replay never substitutes current defaults. diff --git a/docs/engineering/uk-private-hugging-face-credential.md b/docs/engineering/uk-private-hugging-face-credential.md new file mode 100644 index 000000000..1bc1c6134 --- /dev/null +++ b/docs/engineering/uk-private-hugging-face-credential.md @@ -0,0 +1,65 @@ +# UK private-data Hugging Face credential + +API and simulation deployments use one read-only Hugging Face credential for +the private UK runtime data. Its managed name is +`PE_UK_PRIVATE_HF_READ_TOKEN`. + +## Required access + +The credential must be a fine-grained token named +`pe-uk-private-hf-read-token` with read-only access to: + +- the model repository `policyengine/policyengine-uk-data-private`; +- the dataset repository `policyengine/populace-uk-private`. + +It must not have repository write access. The deployed API and simulation +paths do not use `policyengine/populace-uk-staging`; build and publication +credentials for that repository are separate. + +## API v1 path + +The selected-repository GitHub organization secret +`PE_UK_PRIVATE_HF_READ_TOKEN` is copied to the Google Secret Manager resource +`pe-uk-private-hf-read-token`. It is one member of the GitHub-owned runtime +secret batch, alongside the environment-specific database password and the +shared GitHub microdata and OpenAI credentials. + +Each staging and production deployment synchronizes the shared three-secret +batch before deploying its Cloud Run candidate. Production additionally +synchronizes the existing GitHub-owned database password to +`policyengine-api-prod-db-password`. Staging deliberately leaves +`policyengine-api-staging-db-password` untouched: its value differs from the +production password and its canonical value is managed directly in Secret +Manager rather than duplicated in GitHub. Synchronization compares SHA-256 +digests without logging either value and creates a new secret version only when +the value changed. It then reads back and verifies every synchronized value. + +The synchronization script resolves the runtime identity from the deployed +`policyengine-api-staging` or `policyengine-api` service. It grants that identity +`roles/secretmanager.secretAccessor` on every resource synchronized for that +environment and verifies each IAM binding. A manually dispatched workflow +performs the corresponding staging or production operation when credentials +need to be rotated independently of a deployment. + +Cloud Run exposes the resource as `HUGGING_FACE_TOKEN` because PolicyEngine +Core reads that compatibility environment variable. `HUGGING_FACE_TOKEN` is +not the name of the stored GitHub or Google Cloud credential. + +The pull-request environment check verifies the token identity, confirms that +it has no write permission, and checks both required private repositories. The +Cloud Run candidate resolver verifies that the exact deployed revision binds +`HUGGING_FACE_TOKEN` to `pe-uk-private-hf-read-token:latest`, without printing +the credential. + +The simulation-entry Cloud Run service does not receive this credential. It +submits requests to Modal and does not download model data. + +## Rotation and retirement + +Provision the replacement credential before merging workflow changes. Deploy +and verify staging before production. Keep the previous Google Secret Manager +resource during the rollback period; then disable its version before deleting +the resource. + +Do not remove the organization secret named `HUGGING_FACE_TOKEN` as part of +this migration. Other PolicyEngine repositories still consume it. diff --git a/docs/migration/stage-12-worker-contract.md b/docs/migration/stage-12-worker-contract.md index dbb7a8aa2..1e830a360 100644 --- a/docs/migration/stage-12-worker-contract.md +++ b/docs/migration/stage-12-worker-contract.md @@ -171,6 +171,14 @@ SHA-256 content digests, schema versions, row-identity metadata, and bounded provenance. When an option such as SPM produces a detached calculation receipt, the simulation descriptor and Parquet metadata retain it so the coordinator can validate the receipt before reproducing the existing aggregate response. +`SimulationArtifactDescriptor.calculation_provenance` is either null or the +exact typed `canonical-spm-provenance-v2` receipt. The receipt's forecast, +scenario, geography, county-vintage and as-of fields are the complete resolved +configuration; the descriptor does not transport a duplicate `spm_config`. +It rejects partial, rich, legacy and dual-field receipt shapes rather than +preserving or normalizing them. +`AggregateReportArtifactPayload.result` applies the same rule to comparison +provenance and rejects a sibling `spm_config` before artifact storage. Only the Simulation Entrypoint, the versioned worker application, the bucket's object-lifecycle process, and explicitly authorized operators may access the diff --git a/policyengine_api/constants.py b/policyengine_api/constants.py index 683fb4aa9..7056574cb 100644 --- a/policyengine_api/constants.py +++ b/policyengine_api/constants.py @@ -176,8 +176,8 @@ def get_bundle_default_dataset_option(country_id: str) -> dict: ) or _resolve_distribution_version(_dist_versions, "policyengine-core", "policyengine") RUNTIME_CACHE_SCHEMA_VERSIONS = { - "economy_impact": 2, - "report_output": 1, + "economy_impact": 3, + "report_output": 3, } diff --git a/policyengine_api/openapi_spec.yaml b/policyengine_api/openapi_spec.yaml index 4e0d0f8b2..3155fe341 100644 --- a/policyengine_api/openapi_spec.yaml +++ b/policyengine_api/openapi_spec.yaml @@ -439,8 +439,6 @@ paths: nullable: true result: type: object - spm_config: - $ref: '#/components/schemas/SPMSelection' spm_provenance: $ref: '#/components/schemas/SPMProvenance' 404: @@ -531,8 +529,6 @@ paths: nullable: true result: type: object - spm_config: - $ref: '#/components/schemas/SPMSelection' spm_provenance: $ref: '#/components/schemas/SPMProvenance' 500: @@ -623,9 +619,9 @@ paths: non-null output. Null update fields are ignored, and omitted fields retain their values. The installed country model version and current run state are updated with the record. Preserve the full household calculation - envelope, including spm_config and spm_provenance, inside output. Its SPM - selection belongs to the linked household; this operation does not select - a measurement or validate stored receipt integrity. Top-level spm is + envelope, including spm_provenance, inside output. Its SPM selection is + derived from that receipt and belongs to the linked household; this + operation does not select a measurement. Top-level spm is rejected, including null, with HTTP 400 SPM_SETTINGS_UNSUPPORTED. parameters: - name: country_id @@ -658,7 +654,7 @@ paths: Object and array outputs and simulation_spec_json use the legacy JSON-encoded string representation, or null when absent. Stored scalar strings are returned unchanged. Decode a saved household output envelope - to read its spm_config and spm_provenance. Run identifiers and + to read its spm_provenance. Run identifiers and specification metadata may be null on historical records. parameters: - name: country_id @@ -760,7 +756,10 @@ paths: patch: summary: Update a report output operationId: update_report_output - description: Update a report output status, result, or error message. + description: >- + Update a report output status, result, or error message. Completed US + economy outputs require compact v2 comparison provenance. Each receipt + contains the complete resolved configuration; a sibling spm_config is rejected. parameters: - name: country_id in: path @@ -880,7 +879,7 @@ paths: # Query parameters are generated from policyengine_api.query_parameters. responses: 200: - description: Annual economic impact, progress, or persisted error state. Canonical completed results include the selected SPM configuration and actual baseline/reform receipts. + description: Annual economic impact, progress, or persisted error state. Canonical completed results include actual baseline/reform receipts containing the selected SPM configuration. content: application/json: schema: @@ -896,10 +895,8 @@ paths: type: object nullable: true properties: - spm_config: - $ref: '#/components/schemas/SPMSelection' spm_provenance: - $ref: '#/components/schemas/SPMWorkerProvenance' + $ref: '#/components/schemas/SPMComparisonProvenance' 400: $ref: '#/components/responses/SPMValidationError' 404: @@ -969,10 +966,8 @@ paths: properties: year: type: string - spm_config: - $ref: '#/components/schemas/SPMSelection' spm_provenance: - $ref: '#/components/schemas/SPMWorkerProvenance' + $ref: '#/components/schemas/SPMComparisonProvenance' progress: type: integer nullable: true @@ -1188,7 +1183,7 @@ components: description: >- Non-string JSON output is JSON-encoded, or null when absent. Stored scalar strings are returned unchanged. Decode a saved household - calculation envelope once to recover spm_config and spm_provenance + calculation envelope once to recover spm_provenance beside its result, not beside this record. x-household-envelope-schema: $ref: '#/components/schemas/StoredHouseholdOutputEnvelope' @@ -1217,7 +1212,7 @@ components: type: object description: >- Household calculation envelope saved as simulation output. Keep result - and both SPM fields together. Legacy calculations omit SPM fields; this + and SPM provenance together. Legacy calculations omit SPM provenance; this descriptive schema does not add validation to the persistence endpoint. properties: status: @@ -1228,8 +1223,6 @@ components: result: type: object nullable: true - spm_config: - $ref: '#/components/schemas/SPMSelection' spm_provenance: $ref: '#/components/schemas/SPMProvenance' SimulationError: @@ -1270,25 +1263,16 @@ components: description: JSON Schema for SPMSelection, preserving omitted fields. defaults: $ref: '#/components/schemas/SPMSelection' - SPMWorkerProvenance: - type: object - additionalProperties: false - required: [baseline, reform] - properties: - baseline: - type: array - minItems: 1 - items: - $ref: '#/components/schemas/SPMProvenance' - reform: - type: array - minItems: 1 - items: - $ref: '#/components/schemas/SPMProvenance' SPMSelection: description: Generated from policyengine_api.spm.SPMSelection. + SPMRuntimeVersions: + description: Generated from policyengine_api.spm.SPMRuntimeVersions. SPMProvenance: description: Generated from policyengine_api.spm.SPMProvenance. + SPMExecutionProvenance: + description: Generated from policyengine_api.spm.SPMExecutionProvenance. + SPMComparisonProvenance: + description: Generated from policyengine_api.spm.SPMComparisonProvenance. responses: SPMValidationError: description: >- diff --git a/policyengine_api/routes/household_routes.py b/policyengine_api/routes/household_routes.py index 5f2c65dd9..300f2d9d3 100644 --- a/policyengine_api/routes/household_routes.py +++ b/policyengine_api/routes/household_routes.py @@ -137,10 +137,8 @@ def _calculation_response(calculation) -> dict: result = dict(status="ok", message=None, result=calculation.household) if calculation.warnings: result["warnings"] = list(calculation.warnings) - for field in ("spm_config", "spm_provenance"): - value = getattr(calculation, field, None) - if value is not None: - result[field] = value + if calculation.spm_provenance is not None: + result["spm_provenance"] = calculation.spm_provenance return result @@ -231,7 +229,7 @@ def _calculation_cache_key(*args, **kwargs): country_id = request.view_args["country_id"] return hash_object( { - "schema": 2, + "schema": 3, "path": request.full_path, "payload": request.get_json(), "spm": g.spm, diff --git a/policyengine_api/runtime_cache/household_calculations.py b/policyengine_api/runtime_cache/household_calculations.py index 363f9d49e..4ab7ceaf7 100644 --- a/policyengine_api/runtime_cache/household_calculations.py +++ b/policyengine_api/runtime_cache/household_calculations.py @@ -10,10 +10,13 @@ CacheNamespace, RecoverableJSONCache, ) -from policyengine_api.spm import SPMProvenance, resolved_spm_settings +from policyengine_api.spm import ( + resolved_spm_configuration, + validate_spm_calculation_provenance, +) -HOUSEHOLD_CALCULATION_SCHEMA_VERSION = 2 +HOUSEHOLD_CALCULATION_SCHEMA_VERSION = 4 HOUSEHOLD_CALCULATION_TTL_SECONDS = 86_400 @@ -33,7 +36,6 @@ class HouseholdCalculationIdentity: class CachedHouseholdCalculation: household: dict[str, Any] warnings: tuple[str, ...] = () - spm_config: dict[str, Any] | None = None spm_provenance: dict[str, Any] | None = None @@ -65,33 +67,28 @@ def get( return None if not all(isinstance(warning, str) for warning in warnings): return None - spm_config = payload.get("spm_config") + if "spm_config" in payload: + return None spm_provenance = payload.get("spm_provenance") - if any( - value is not None and not isinstance(value, dict) - for value in (spm_config, spm_provenance) - ): + if spm_provenance is not None and not isinstance(spm_provenance, dict): return None - if identity.spm is not None: - # A stored receipt may omit its null values, so compare resolved - # settings rather than raw JSON. Requiring exact equality would miss - # the cache on every replay of a canonical calculation. - if resolved_spm_settings(spm_config) != identity.spm: + if identity.spm is None: + if spm_provenance is not None: return None + else: try: - receipt = SPMProvenance.model_validate(spm_provenance) - except ValidationError: + receipt = validate_spm_calculation_provenance(spm_provenance) + except (ValidationError, ValueError): return None if ( - receipt.forecast_sha256 != identity.spm.get("forecast_content_sha256") - or receipt.scenario != identity.spm.get("scenario") - or receipt.geography_kind != identity.spm.get("geography_kind") + resolved_spm_configuration(receipt).model_dump(mode="json") + != identity.spm ): return None + spm_provenance = receipt.model_dump(mode="json", by_alias=True) return CachedHouseholdCalculation( household=household, warnings=tuple(warnings), - spm_config=spm_config, spm_provenance=spm_provenance, ) @@ -100,4 +97,16 @@ def set( identity: HouseholdCalculationIdentity, value: CachedHouseholdCalculation, ) -> bool: - return self._cache.set(asdict(identity), asdict(value)) + if identity.spm is None: + if value.spm_provenance is not None: + return False + return self._cache.set(asdict(identity), asdict(value)) + try: + receipt = validate_spm_calculation_provenance(value.spm_provenance) + except (ValidationError, ValueError): + return False + if resolved_spm_configuration(receipt).model_dump(mode="json") != identity.spm: + return False + payload = asdict(value) + payload["spm_provenance"] = receipt.model_dump(mode="json", by_alias=True) + return self._cache.set(asdict(identity), payload) diff --git a/policyengine_api/runtime_cache/reform_impacts.py b/policyengine_api/runtime_cache/reform_impacts.py index 571e5b24c..8a7ed5dc8 100644 --- a/policyengine_api/runtime_cache/reform_impacts.py +++ b/policyengine_api/runtime_cache/reform_impacts.py @@ -19,8 +19,7 @@ record_cache_event, ) - -REFORM_IMPACT_SCHEMA_VERSION = 1 +REFORM_IMPACT_SCHEMA_VERSION = 3 REFORM_IMPACT_TTL_SECONDS = 2_592_000 REFORM_IMPACT_INDEX_LIMIT = 1_000 REFORM_IMPACT_START_CLAIM_TTL_SECONDS = 300 diff --git a/policyengine_api/services/budget_window_cache.py b/policyengine_api/services/budget_window_cache.py index 9402bc952..b6190341a 100644 --- a/policyengine_api/services/budget_window_cache.py +++ b/policyengine_api/services/budget_window_cache.py @@ -19,9 +19,8 @@ ) from policyengine_api.runtime_cache.dependencies import get_runtime_cache_context - BUDGET_WINDOW_CACHE_FAMILY = "budget-window" -BUDGET_WINDOW_CACHE_SCHEMA_VERSION = 2 +BUDGET_WINDOW_CACHE_SCHEMA_VERSION = 4 BUDGET_WINDOW_STARTING_TTL_SECONDS = 300 BUDGET_WINDOW_BATCH_TTL_SECONDS = 86_400 BUDGET_WINDOW_RESULT_TTL_SECONDS = 2_592_000 diff --git a/policyengine_api/services/economy_service.py b/policyengine_api/services/economy_service.py index 9cec6e90c..ce7112851 100644 --- a/policyengine_api/services/economy_service.py +++ b/policyengine_api/services/economy_service.py @@ -1530,6 +1530,8 @@ def _build_options_hash( bundle_parts.append(f"data_version={data_version}") if policyengine_version: bundle_parts.append(f"policyengine_version={policyengine_version}") + if options.get("spm") is not None: + bundle_parts.append("spm_provenance_schema=canonical-spm-comparison-v2") if runtime_app_name: bundle_parts.append(f"runtime_app_name={runtime_app_name}") return "[" + "&".join([option_pairs, *bundle_parts]).strip("&") + "]" diff --git a/policyengine_api/services/household_calculation_service.py b/policyengine_api/services/household_calculation_service.py index 3d6284567..bce5743c7 100644 --- a/policyengine_api/services/household_calculation_service.py +++ b/policyengine_api/services/household_calculation_service.py @@ -26,14 +26,17 @@ ) from policyengine_api.utils.deprecated_inputs import drop_deprecated_inputs from policyengine_api.utils.input_validation import find_unrecognized_inputs -from policyengine_api.spm import normalize_spm_selection +from policyengine_api.spm import ( + normalize_spm_selection, + resolved_spm_configuration, + validate_spm_calculation_provenance, +) @dataclass(frozen=True) class CalculationResult: household: dict warnings: tuple[str, ...] = () - spm_config: dict | None = None spm_provenance: dict | None = None @@ -42,7 +45,6 @@ class HouseholdCalculationResult: household: dict warnings: tuple[str, ...] = () cached: bool = False - spm_config: dict | None = None spm_provenance: dict | None = None @@ -73,6 +75,21 @@ def __init__(self, invalid_inputs: list[Any]) -> None: super().__init__("Household or policy contains unrecognized inputs") +def _validated_spm_provenance( + spm_provenance: object, + expected_selection: dict | None, +) -> dict | None: + if expected_selection is None: + if spm_provenance is None: + return None + raise ValueError("Unexpected SPM provenance for a calculation without SPM") + receipt = validate_spm_calculation_provenance(spm_provenance) + canonical_config = resolved_spm_configuration(receipt).model_dump(mode="json") + if canonical_config != expected_selection: + raise ValueError("SPM calculation configuration differs from the request") + return receipt.model_dump(mode="json", by_alias=True) + + def get_household_year(household: dict) -> int | str: household_year: int | str = date.today().year household_age_list = list( @@ -192,7 +209,6 @@ def _store_result( CachedHouseholdCalculation( household=calculation.household, warnings=warnings, - spm_config=calculation.spm_config, spm_provenance=calculation.spm_provenance, ), ) @@ -244,7 +260,6 @@ def calculate_stored_household( household=cached.household, warnings=cached.warnings, cached=True, - spm_config=cached.spm_config, spm_provenance=cached.spm_provenance, ) @@ -296,7 +311,6 @@ def calculate_stored_household( calculation = CalculationResult( household=raw_calculation.household, warnings=tuple(getattr(raw_calculation, "warnings", ())), - spm_config=getattr(raw_calculation, "spm_config", None), spm_provenance=getattr(raw_calculation, "spm_provenance", None), ) else: @@ -305,6 +319,14 @@ def calculate_stored_household( calculation = CalculationResult( household=raw_calculation, ) + spm_provenance = _validated_spm_provenance( + calculation.spm_provenance, + spm, + ) + calculation = replace( + calculation, + spm_provenance=spm_provenance, + ) record_cache_event( family="household-calculation", event="recompute", @@ -322,7 +344,6 @@ def calculate_stored_household( return HouseholdCalculationResult( household=calculation.household, warnings=response_warnings, - spm_config=calculation.spm_config, spm_provenance=calculation.spm_provenance, ) @@ -398,11 +419,14 @@ def calculate_prepared_household( else: household = raw_calculation.household calculation_warnings = tuple(getattr(raw_calculation, "warnings", ())) + spm_provenance = _validated_spm_provenance( + getattr(raw_calculation, "spm_provenance", None), + prepared.spm, + ) return HouseholdCalculationResult( household=household, warnings=(prepared.warnings + calculation_warnings), - spm_config=getattr(raw_calculation, "spm_config", None), - spm_provenance=getattr(raw_calculation, "spm_provenance", None), + spm_provenance=spm_provenance, ) @observability_runtime.span( diff --git a/policyengine_api/services/report_output_service.py b/policyengine_api/services/report_output_service.py index 28bdd3e9e..69246e827 100644 --- a/policyengine_api/services/report_output_service.py +++ b/policyengine_api/services/report_output_service.py @@ -1,6 +1,7 @@ import json from dataclasses import dataclass from datetime import datetime, timezone +from typing import cast from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker @@ -8,6 +9,7 @@ from policyengine_api.constants import get_report_output_cache_version from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import ReportOutput, ReportOutputRun, Simulation +from policyengine_api.json_types import JSONObject, JSONValue from policyengine_api.services.report_run_service import ReportRunService from policyengine_api.services.report_spec_service import ( ECONOMY_REPORT_KINDS, @@ -15,6 +17,72 @@ ReportSpecService, ) from policyengine_api.services.simulation_service import SimulationService +from policyengine_api.spm import ( + SPMComparisonProvenance, + error_message, +) + + +def _canonicalize_annual_spm_output( + output: JSONObject, + *, + expected_year: str | None, +) -> JSONObject: + """Validate and serialize compact SPM provenance on one economy result.""" + + if "spm_config" in output: + raise ValueError( + "Completed US economy output must not include legacy spm_config" + ) + if "spm_provenance" not in output: + raise ValueError("Completed US economy output requires SPM provenance") + + comparison = SPMComparisonProvenance.model_validate(output["spm_provenance"]) + if expected_year is not None and any( + expected_year not in execution.receipt.years + for execution in (comparison.baseline, comparison.reform) + ): + raise ValueError("SPM receipt does not cover the report year") + return { + **output, + "spm_provenance": cast( + JSONValue, + comparison.model_dump(mode="json", by_alias=True), + ), + } + + +def _canonicalize_society_wide_spm_output( + output: object, + *, + expected_year: str | None, +) -> object: + """Validate compact annual or budget-window society-wide SPM output.""" + + if not isinstance(output, dict): + return output + typed_output = cast(JSONObject, output) + if output.get("kind") != "budgetWindow": + return _canonicalize_annual_spm_output( + typed_output, + expected_year=expected_year, + ) + + annual_impacts = output.get("annualImpacts") + if not isinstance(annual_impacts, list): + raise ValueError("SPM budget-window output requires annual impact objects") + canonical_rows: list[JSONValue] = [] + for row in annual_impacts: + if not isinstance(row, dict): + raise ValueError("SPM budget-window output requires annual impact objects") + row_year = row.get("year") + canonical_rows.append( + _canonicalize_annual_spm_output( + cast(JSONObject, row), + expected_year=str(row_year) if row_year is not None else None, + ) + ) + return {**typed_output, "annualImpacts": canonical_rows} @dataclass(frozen=True) @@ -648,6 +716,23 @@ def _update_report_output( "Cannot mark report output running without an active pending " "or running report run" ) + effective_status = values.get("status", report_output.status) + effective_output = values.get("output", report_output.output) + if ( + effective_status == "complete" + and report_output.country_id == "us" + and report_output.report_kind in ECONOMY_REPORT_KINDS + ): + try: + values["output"] = _canonicalize_society_wide_spm_output( + effective_output, + expected_year=report_output.year, + ) + except (ValueError, TypeError, KeyError) as error: + raise ValueError( + "Completed US economy report has invalid SPM provenance: " + f"{error_message(error)}" + ) from error for field, value in values.items(): setattr(report_output, field, value) self._ensure_report_output_dual_write_state( diff --git a/policyengine_api/services/simulation_service.py b/policyengine_api/services/simulation_service.py index dfbe828ce..3ef805673 100644 --- a/policyengine_api/services/simulation_service.py +++ b/policyengine_api/services/simulation_service.py @@ -8,6 +8,7 @@ from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import Simulation, SimulationRun +from policyengine_api.spm import validate_spm_calculation_provenance from policyengine_api.utils.population_identity import ( canonical_numeric_household_id, population_id_matches, @@ -52,6 +53,28 @@ def _select_simulation( def _latest_successful_run_id(runs: list[SimulationRun]) -> str | None: return next((run.id for run in runs if run.status == "complete"), None) + @staticmethod + def _has_stale_spm_output(output: object) -> bool: + """Identify stored results that predate the compact SPM receipt contract. + + Legacy SPM output cannot be translated safely: it may omit the exact + forecast or runtime identity now required to verify a result. Treating + it as stale makes the existing simulation pending and clears its output + so the calculation is run again with the installed certified bundle. + Output without any SPM fields remains valid non-SPM output. + """ + if not isinstance(output, dict): + return False + if "spm_config" in output: + return True + if "spm_provenance" not in output: + return False + try: + validate_spm_calculation_provenance(output["spm_provenance"]) + except ValueError: + return True + return False + def _ensure_simulation_dual_write_state( self, session: Session, @@ -67,6 +90,11 @@ def _ensure_simulation_dual_write_state( if simulation is None: raise ValueError(f"Simulation #{simulation_id} not found") + if self._has_stale_spm_output(simulation.output): + simulation.status = "pending" + simulation.output = None + simulation.error_message = None + spec = { "country_id": simulation.country_id, "population_id": simulation.population_id, @@ -252,6 +280,19 @@ def update_simulation( return None if isinstance(values.get("output"), str): values["output"] = json.loads(values["output"]) + output_value = values.get("output") + if isinstance(output_value, dict) and "spm_config" in output_value: + raise ValueError( + "Completed simulation output must not include legacy spm_config" + ) + if isinstance(output_value, dict) and "spm_provenance" in output_value: + receipt = validate_spm_calculation_provenance( + output_value["spm_provenance"] + ) + values["output"] = { + **output_value, + "spm_provenance": receipt.model_dump(mode="json", by_alias=True), + } with self._sessions.begin() as session: simulation = self._select_simulation( session, diff --git a/policyengine_api/services/v2/reports/types.py b/policyengine_api/services/v2/reports/types.py index 9d0298f43..ef4d997f4 100644 --- a/policyengine_api/services/v2/reports/types.py +++ b/policyengine_api/services/v2/reports/types.py @@ -16,6 +16,7 @@ SimulationRole, StrictContractModel, ) +from policyengine_api.spm import SPMComparisonProvenance ContractVersion = Literal[1] @@ -102,3 +103,13 @@ def require_unique_aggregates( if len(value) != len(set(value)): raise ValueError("requested aggregates must be unique") return value + + @model_validator(mode="after") + def validate_spm_provenance(self) -> AggregateReportArtifactPayload: + """Reject duplicate or noncanonical SPM data in aggregate artifacts.""" + + if "spm_config" in self.result: + raise ValueError("aggregate result must not include legacy spm_config") + if "spm_provenance" in self.result: + SPMComparisonProvenance.model_validate(self.result["spm_provenance"]) + return self diff --git a/policyengine_api/services/v2/simulations/types.py b/policyengine_api/services/v2/simulations/types.py index cc9b0a723..d1e466b13 100644 --- a/policyengine_api/services/v2/simulations/types.py +++ b/policyengine_api/services/v2/simulations/types.py @@ -16,6 +16,7 @@ ) from policyengine_api.query_parameters import CountryId +from policyengine_api.spm import SPMProvenance ContractText = Annotated[str, Field(min_length=1, max_length=255)] ContractVersion = Literal[1] @@ -205,4 +206,4 @@ class SimulationArtifactDescriptor(StrictContractModel): output_schema_version: ContractVersion = 1 row_identity: RowIdentity bundle: BundleProvenance - calculation_provenance: dict[str, JsonValue] | None = None + calculation_provenance: SPMProvenance | None = None diff --git a/policyengine_api/specification.py b/policyengine_api/specification.py index 11b4b3c8f..e2c9296b6 100644 --- a/policyengine_api/specification.py +++ b/policyengine_api/specification.py @@ -13,7 +13,13 @@ AnnualEconomyQuery, BudgetWindowEconomyQuery, ) -from policyengine_api.spm import SPMProvenance, SPMSelection +from policyengine_api.spm import ( + SPMComparisonProvenance, + SPMExecutionProvenance, + SPMProvenance, + SPMRuntimeVersions, + SPMSelection, +) DEFAULT_SPECIFICATION_PATH = Path(__file__).with_name("openapi_spec.yaml") @@ -62,6 +68,15 @@ def _economy_query_parameters(model): return result +def _component_schema(model): + """Generate one OpenAPI component and publish nested models separately.""" + schema = model.model_json_schema( + ref_template="#/components/schemas/{model}", + ) + schema.pop("$defs", None) + return _openapi_30_schema(schema) + + def load_specification( path: Path = DEFAULT_SPECIFICATION_PATH, version: str = VERSION, @@ -74,10 +89,13 @@ def load_specification( schemas = document.get("components", {}).get("schemas", {}) for name, model in { "SPMSelection": SPMSelection, + "SPMRuntimeVersions": SPMRuntimeVersions, "SPMProvenance": SPMProvenance, + "SPMExecutionProvenance": SPMExecutionProvenance, + "SPMComparisonProvenance": SPMComparisonProvenance, }.items(): if name in schemas: - schemas[name] = _openapi_30_schema(model.model_json_schema()) + schemas[name] = _component_schema(model) economy_path = "/{country_id}/economy/{policy_id}/over/{baseline_policy_id}" for suffix, query_model in ( ("", AnnualEconomyQuery), diff --git a/policyengine_api/spm.py b/policyengine_api/spm.py index c3c627ea9..4f92a3b96 100644 --- a/policyengine_api/spm.py +++ b/policyengine_api/spm.py @@ -8,9 +8,10 @@ import importlib import inspect -from functools import lru_cache +from collections.abc import Mapping, Sequence from datetime import date -from typing import Any, Literal +from functools import lru_cache +from typing import Literal from pydantic import ( BaseModel, @@ -96,19 +97,218 @@ def validate_location(self): return self +class SPMResolvedConfiguration(BaseModel): + """An internal complete SPM selection derived from a compact receipt.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + forecast_content_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + scenario: str = Field(min_length=1, pattern=r"^\S+$") + geography_kind: Literal["county", "national", "metro"] + geography_id: str | None = Field(min_length=1) + county_vintage: str = Field(pattern=r"^[0-9]{4}$") + as_of: str | None + + @field_validator("as_of") + @classmethod + def validate_as_of(cls, value: str | None) -> str | None: + if value is not None and date.fromisoformat(value).isoformat() != value: + raise ValueError("as_of must be an ISO calendar date (YYYY-MM-DD)") + return value + + @model_validator(mode="after") + def validate_location(self) -> "SPMResolvedConfiguration": + if self.geography_kind == "metro": + if self.geography_id is None or not self.geography_id.strip(): + raise ValueError("An SPM area selection requires geography_id") + elif self.geography_id is not None: + raise ValueError("Only an SPM area selection accepts geography_id") + return self + + class SPMProvenance(BaseModel): - """JSON receipt matching the country's public provenance contract.""" + """Compact public receipt for one canonical SPM execution.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + schema_version: Literal["canonical-spm-provenance-v2"] + forecast_id: str = Field(min_length=1, pattern=r"^\S+$") + forecast_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + scenario: str = Field(min_length=1, pattern=r"^\S+$") + geography_kind: Literal["county", "national", "metro"] + geography_id: str | None = Field(min_length=1) + county_vintage: str = Field(pattern=r"^[0-9]{4}$") + as_of: date | None + years: tuple[str, ...] + runtime_versions: "SPMRuntimeVersions" + + @field_validator("years") + @classmethod + def validate_years(cls, years: tuple[str, ...]) -> tuple[str, ...]: + if any( + len(year) != 4 or not year.isascii() or not year.isdigit() for year in years + ): + raise ValueError("years must contain four-digit calendar years") + if tuple(sorted(set(years))) != years: + raise ValueError("years must be sorted and unique") + return years + + @model_validator(mode="after") + def validate_location(self) -> "SPMProvenance": + if self.geography_kind == "metro": + if self.geography_id is None or not self.geography_id.strip(): + raise ValueError("A metro SPM receipt requires geography_id") + elif self.geography_id is not None: + raise ValueError("Only a metro SPM receipt accepts geography_id") + return self + + +class SPMRuntimeVersions(BaseModel): + """Package versions required to reproduce one certified SPM execution.""" + + model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True) + policyengine: str = Field(min_length=1) + policyengine_core: str = Field(min_length=1, alias="policyengine-core") + policyengine_us: str = Field(min_length=1, alias="policyengine-us") + spm_calculator: str = Field(min_length=1, alias="spm-calculator") + + +SPMProvenance.model_rebuild() + + +class SPMExecutionProvenance(BaseModel): + """One shared compact receipt and the executions that produced it.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + receipt: SPMProvenance + execution_count: int = Field(ge=1) + - model_config = ConfigDict(extra="forbid") - forecast_id: str - forecast_sha256: str - scenario: str - geography_kind: str - runtime_versions: dict[str, str | None] - years: dict[str, dict[str, Any]] - geographies: list[dict[str, Any]] - composition_method: str - storage_method: str +class SPMComparisonProvenance(BaseModel): + """Compact baseline and reform receipts for a society-wide result.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + schema_version: Literal["canonical-spm-comparison-v2"] + baseline: SPMExecutionProvenance + reform: SPMExecutionProvenance + + @model_validator(mode="after") + def validate_compatible_receipts(self) -> "SPMComparisonProvenance": + if self.baseline.receipt != self.reform.receipt: + raise ValueError("baseline and reform SPM receipts differ") + return self + + +def build_spm_provenance( + *, + forecast_id: str, + forecast_sha256: str, + selection: SPMSelection, + years: Sequence[str], + runtime_versions: SPMRuntimeVersions, +) -> SPMProvenance: + """Build one compact receipt from typed calculation inputs.""" + + if selection.forecast_content_sha256 != forecast_sha256: + raise ValueError("SPM receipt differs from the SPM configuration") + if selection.scenario is None: + raise ValueError("Resolved SPM configuration must include scenario") + return SPMProvenance( + schema_version="canonical-spm-provenance-v2", + forecast_id=forecast_id, + forecast_sha256=forecast_sha256, + scenario=selection.scenario, + geography_kind=selection.geography_kind, + geography_id=selection.geography_id, + county_vintage=selection.county_vintage, + as_of=selection.as_of, + years=tuple(years), + runtime_versions=runtime_versions, + ) + + +def _required_string(source: Mapping[str, object], field: str) -> str: + value = source.get(field) + if not isinstance(value, str) or not value: + raise ValueError(f"Country SPM receipt requires {field}") + return value + + +def build_spm_calculation_provenance( + config: SPMSelection, + receipt: Mapping[str, object], +) -> SPMProvenance: + """Reduce the country-owned rich receipt to the public compact contract.""" + + forecast_sha256 = _required_string(receipt, "forecast_sha256") + scenario = _required_string(receipt, "scenario") + geography_kind = _required_string(receipt, "geography_kind") + if ( + forecast_sha256 != config.forecast_content_sha256 + or scenario != config.scenario + or geography_kind != config.geography_kind + ): + raise ValueError("Country SPM receipt differs from the SPM configuration") + + source_years = receipt.get("years") + if not isinstance(source_years, Mapping): + raise ValueError("Country SPM receipt requires a years mapping") + years: list[str] = [] + for year in source_years: + if not isinstance(year, str): + raise ValueError("Country SPM receipt years must be strings") + years.append(year) + + source_versions = receipt.get("runtime_versions") + if not isinstance(source_versions, Mapping): + raise ValueError("Country SPM receipt requires runtime_versions") + runtime_versions = SPMRuntimeVersions.model_validate(source_versions) + return build_spm_provenance( + forecast_id=_required_string(receipt, "forecast_id"), + forecast_sha256=forecast_sha256, + selection=config, + years=tuple(sorted(years)), + runtime_versions=runtime_versions, + ) + + +def resolved_spm_configuration( + provenance: SPMProvenance, +) -> SPMResolvedConfiguration: + """Derive the complete resolved selection from its compact receipt.""" + + return SPMResolvedConfiguration( + forecast_content_sha256=provenance.forecast_sha256, + scenario=provenance.scenario, + geography_kind=provenance.geography_kind, + geography_id=provenance.geography_id, + county_vintage=provenance.county_vintage, + as_of=provenance.as_of.isoformat() if provenance.as_of is not None else None, + ) + + +def _collapse_execution_receipts( + side: Literal["baseline", "reform"], + receipts: Sequence[SPMProvenance], +) -> SPMExecutionProvenance: + if not receipts: + raise ValueError(f"{side} SPM receipts are empty") + first = receipts[0] + if any(receipt != first for receipt in receipts[1:]): + raise ValueError(f"{side} SPM receipts differ") + return SPMExecutionProvenance(receipt=first, execution_count=len(receipts)) + + +def build_spm_comparison_provenance( + *, + baseline_receipts: Sequence[SPMProvenance], + reform_receipts: Sequence[SPMProvenance], +) -> SPMComparisonProvenance: + """Collapse matching child receipts into one comparison receipt.""" + + return SPMComparisonProvenance( + schema_version="canonical-spm-comparison-v2", + baseline=_collapse_execution_receipts("baseline", baseline_receipts), + reform=_collapse_execution_receipts("reform", reform_receipts), + ) class SPMValidationError(ValueError): @@ -190,12 +390,9 @@ def simulation_supports_spm(simulation_type) -> bool: ) -# A resolved selection freezes all six fields, but output transports may omit -# null values. Only those omissions are safe: a receipt must never inherit a -# non-null setting from today's bundle defaults. -REQUIRED_RESOLVED_SPM_FIELDS = frozenset( - {"forecast_content_sha256", "scenario", "geography_kind", "county_vintage"} -) +# A transported resolved selection freezes all six fields, including explicit +# null values. Receipt validation never fills output from current defaults. +REQUIRED_RESOLVED_SPM_FIELDS = frozenset(SPMSelection.model_fields) def resolved_spm_settings(settings: object) -> dict | None: @@ -214,6 +411,14 @@ def resolved_spm_settings(settings: object) -> dict | None: return {name: getattr(config, name) for name in SPMSelection.model_fields} +def validate_spm_calculation_provenance( + provenance: object, +) -> SPMProvenance: + """Validate the sole transported compact receipt for a completed result.""" + + return SPMProvenance.model_validate(provenance) + + def _current_bundle() -> dict: from policyengine_api.constants import _policyengine_bundle @@ -413,25 +618,24 @@ def spm_metadata(country_id: str) -> dict: def calculation_spm_receipt(simulation) -> dict: - """Read existing calculation receipts; never request SPM calculations here.""" + """Reduce a country's existing receipt; never request calculations here.""" if not hasattr(simulation, "spm_config"): return {} - # A receipt this API cannot read in full is an uncertified country contract, - # not a caller error and not an internal failure. Report it as a typed - # configuration failure rather than letting pydantic surface a 500, and - # never publish a receipt with unrecognized fields dropped. try: - return { - "spm_config": SPMSelection.model_validate(simulation.spm_config).model_dump( - mode="json" - ), - "spm_provenance": SPMProvenance.model_validate( - simulation.spm_provenance() - ).model_dump(mode="json"), - } - except ValidationError as error: + resolved = resolved_spm_settings(simulation.spm_config) + if resolved is None: + raise ValueError("Country SPM configuration is incomplete") + source = simulation.spm_provenance() + if not isinstance(source, Mapping): + raise ValueError("Country SPM receipt must be an object") + calculation = build_spm_calculation_provenance( + SPMSelection.model_validate(resolved), + source, + ) + return {"spm_provenance": calculation.model_dump(mode="json", by_alias=True)} + except (ValidationError, ValueError) as error: raise SPMValidationError( "SPM_CONFIGURATION_UNAVAILABLE", "The installed country model reported an SPM receipt this API cannot " - f"certify: {readable_validation_error(error)}", + f"certify: {error_message(error)}", ) from error diff --git a/policyengine_api/worker_spm.py b/policyengine_api/worker_spm.py index 28d8566c8..2d9977d90 100644 --- a/policyengine_api/worker_spm.py +++ b/policyengine_api/worker_spm.py @@ -1,12 +1,13 @@ """Canonical economy execution requires the selected worker's certified capability.""" from policyengine_api.spm import ( + SPMComparisonProvenance, SPMSelection, - SPMProvenance, SPMValidationError, error_message, normalize_spm_selection, - resolved_spm_settings, + resolved_spm_configuration, + validate_spm_calculation_provenance, ) @@ -95,32 +96,19 @@ def validate_worker_result( return if expected_years is not None: raise ValueError("Expected a budget window SPM result") - resolved = resolved_spm_settings(result.get("spm_config")) - if resolved is None: - raise ValueError("Worker result has incomplete resolved SPM settings") - if resolved != selection: - raise ValueError("Worker result SPM settings differ from the request") - receipts = result.get("spm_provenance") - if not isinstance(receipts, dict) or set(receipts) != {"baseline", "reform"}: - raise ValueError("Worker result has no baseline/reform SPM receipts") + if "spm_config" in result: + raise ValueError("Worker result must not include legacy spm_config") + comparison = SPMComparisonProvenance.model_validate( + result.get("spm_provenance") + ) for side in ("baseline", "reform"): - if not isinstance(receipts[side], list) or not receipts[side]: - raise ValueError("Worker result has an empty SPM receipt") - for item in receipts[side]: - receipt = SPMProvenance.model_validate(item) - if ( - expected_year is not None - and str(expected_year) not in receipt.years - ): - raise ValueError("Worker SPM receipt does not cover requested year") - if ( - receipt.forecast_sha256 != selection["forecast_content_sha256"] - or receipt.scenario != selection["scenario"] - or receipt.geography_kind != selection["geography_kind"] - ): - raise ValueError( - "Worker result SPM provenance differs from the request" - ) + execution = getattr(comparison, side) + receipt = validate_spm_calculation_provenance(execution.receipt) + resolved = resolved_spm_configuration(receipt).model_dump(mode="json") + if resolved != selection: + raise ValueError("Worker result SPM settings differ from the request") + if expected_year is not None and str(expected_year) not in receipt.years: + raise ValueError("Worker SPM receipt does not cover requested year") except (ValueError, TypeError, KeyError) as exc: raise SPMValidationError( "SPM_CONFIGURATION_UNAVAILABLE", error_message(exc) diff --git a/policyengine_api/worker_spm_release.py b/policyengine_api/worker_spm_release.py new file mode 100644 index 000000000..8f876e0d2 --- /dev/null +++ b/policyengine_api/worker_spm_release.py @@ -0,0 +1,122 @@ +"""Validate the installed API bundle against a simulation worker registry.""" + +from __future__ import annotations + +from importlib.metadata import version as distribution_version +import json +import sys + +from policyengine_api import constants +from policyengine_api.worker_spm import validate_worker_spm + + +REQUIRED_BUNDLE_PACKAGES = ( + "policyengine", + "policyengine-core", + "policyengine-us", + "policyengine-uk", + "spm-calculator", +) + + +class RegistryCapability: + """Expose one registry document through the request validator interface.""" + + def __init__(self, registry: dict): + self.registry = registry + + def get_spm_capability( + self, + country, + model_version, + *, + policyengine_version, + ): + capabilities = self.registry.get("spm_capabilities") + if not isinstance(capabilities, dict): + return None + return capabilities.get(policyengine_version) + + +def _bundle_package_versions() -> dict[str, str]: + bundle = constants._policyengine_bundle + packages = bundle.get("packages") if isinstance(bundle, dict) else None + if not isinstance(packages, dict): + raise ValueError("The installed PolicyEngine bundle manifest has no packages") + + versions: dict[str, str] = {} + for package_name in REQUIRED_BUNDLE_PACKAGES: + package = packages.get(package_name) + package_version = package.get("version") if isinstance(package, dict) else None + if not isinstance(package_version, str) or not package_version: + raise ValueError(f"The installed bundle does not identify {package_name}") + versions[package_name] = package_version + return versions + + +def _registered_application( + registry: dict, + route_kind: str, + route_version: str, +) -> str: + routes = registry.get(route_kind) if isinstance(registry, dict) else None + app = routes.get(route_version) if isinstance(routes, dict) else None + if not isinstance(app, str) or not app: + raise ValueError( + f"The worker registry has no {route_kind} {route_version} application" + ) + return app + + +def validate_installed_worker(registry: dict, expected_bundle: str) -> dict | None: + """Validate local package versions, worker routes, and SPM capability.""" + versions = _bundle_package_versions() + if versions["policyengine"] != expected_bundle: + raise ValueError("The bundle manifest differs from the release requirement") + if constants.POLICYENGINE_VERSION != expected_bundle: + raise ValueError("The API runtime differs from the release requirement") + + for package_name, expected_version in versions.items(): + if distribution_version(package_name) != expected_version: + raise ValueError( + f"Installed {package_name} does not match the bundle manifest" + ) + + bundle_app = _registered_application(registry, "policyengine", expected_bundle) + for country_id, package_name in ( + ("us", "policyengine-us"), + ("uk", "policyengine-uk"), + ): + country_app = _registered_application( + registry, + country_id, + versions[package_name], + ) + if country_app != bundle_app: + raise ValueError( + f"The {country_id} route resolves to a different worker application" + ) + + return validate_worker_spm( + "us", + gateway=RegistryCapability(registry), + policyengine_version=expected_bundle, + model_version=versions["policyengine-us"], + ) + + +def main() -> None: + """Read a registry document from stdin and validate one bundle version.""" + if len(sys.argv) != 2: + raise SystemExit( + "Usage: python -m policyengine_api.worker_spm_release BUNDLE_VERSION" + ) + selection = validate_installed_worker(json.load(sys.stdin), sys.argv[1]) + if selection is None: + print("The installed legacy bundle and registered worker agree") + else: + print("The installed bundle and registered worker capability agree") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 779e7d7c6..d4bf88535 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,10 +43,11 @@ dependencies = [ "policyengine_canada==0.96.3", "policyengine-ng==0.5.1", "policyengine-il==0.1.0", - "policyengine[models]==5.2.0", - # Keep the calculator compatible with the country model in this bundle. - # The Cloud Run image installs this exact selection from uv.lock. - "spm-calculator==0.3.1", + # The models extra pins Core, US, UK, and the SPM calculator to the exact + # versions certified by this PolicyEngine bundle. Cloud Run installs the + # complete frozen environment from uv.lock; the generic image installs + # this requirement directly. + "policyengine[models]==6.2.1", "pydantic", "pymysql", "python-dotenv", diff --git a/tests/contract/test_simulation_gateway_contract.py b/tests/contract/test_simulation_gateway_contract.py index 12ccc0e8e..2452e3e38 100644 --- a/tests/contract/test_simulation_gateway_contract.py +++ b/tests/contract/test_simulation_gateway_contract.py @@ -15,6 +15,7 @@ MOCK_RESOLVED_APP_NAME, MOCK_SIMULATION_PAYLOAD_WITH_TELEMETRY, MOCK_SUBMIT_RESPONSE_SUCCESS, + MOCK_WORKER_VERSIONS, ) @@ -56,6 +57,10 @@ def test_gateway_comparison_submit_and_poll_contract(monkeypatch): monkeypatch.setenv("OLD_SIMULATION_GATEWAY_URL", "https://simulation.test") client = _client_for( { + ("GET", "/versions"): _response( + status_code=200, + json_data=MOCK_WORKER_VERSIONS, + ), ("POST", "/simulate/economy/comparison"): _response( status_code=202, json_data=MOCK_SUBMIT_RESPONSE_SUCCESS, @@ -87,6 +92,10 @@ def test_gateway_budget_window_submit_and_poll_contract(monkeypatch): monkeypatch.setenv("OLD_SIMULATION_GATEWAY_URL", "https://simulation.test") client = _client_for( { + ("GET", "/versions"): _response( + status_code=200, + json_data=MOCK_WORKER_VERSIONS, + ), ( "POST", "/simulate/economy/budget-window", diff --git a/tests/env_variables/test_environment_variables.py b/tests/env_variables/test_environment_variables.py index 64cc2176b..5dde7a2d2 100644 --- a/tests/env_variables/test_environment_variables.py +++ b/tests/env_variables/test_environment_variables.py @@ -3,6 +3,13 @@ import requests HUGGING_FACE_API_URL = "https://huggingface.co/api/whoami-v2" +HUGGING_FACE_UK_DATA_MODEL_URL = ( + "https://huggingface.co/api/models/policyengine/policyengine-uk-data-private" +) +HUGGING_FACE_UK_POPULATION_DATASET_URL = ( + "https://huggingface.co/api/datasets/policyengine/populace-uk-private" +) +EXPECTED_HUGGING_FACE_TOKEN_NAME = "pe-uk-private-hf-read-token" GITHUB_API_URL = "https://api.github.com/user" @@ -32,6 +39,48 @@ def test_hugging_face_token(self): assert token_validation_response.status_code == 200, ( f"Invalid HUGGING_FACE_TOKEN: {token_validation_response.text}" ) + token_details = token_validation_response.json()["auth"]["accessToken"] + assert token_details["displayName"] == EXPECTED_HUGGING_FACE_TOKEN_NAME + assert token_details["role"] == "fineGrained" + fine_grained = token_details["fineGrained"] + granted_permissions = set(fine_grained.get("global", [])) + granted_permissions.update( + permission + for scope in fine_grained["scoped"] + for permission in scope["permissions"] + ) + assert "repo.content.read" in granted_permissions + assert not any("write" in permission for permission in granted_permissions) + + headers = {"Authorization": f"Bearer {token}"} + uk_data_response = requests.get( + HUGGING_FACE_UK_DATA_MODEL_URL, + headers=headers, + timeout=10, + ) + assert uk_data_response.status_code == 200, ( + "The API runtime token cannot read " + "policyengine/policyengine-uk-data-private: " + f"{uk_data_response.text}" + ) + uk_data_files = { + sibling["rfilename"] for sibling in uk_data_response.json()["siblings"] + } + assert "local_authority_weights.h5" in uk_data_files + + populace_response = requests.get( + HUGGING_FACE_UK_POPULATION_DATASET_URL, + headers=headers, + timeout=10, + ) + assert populace_response.status_code == 200, ( + "The shared API and simulation runtime token cannot read " + f"policyengine/populace-uk-private: {populace_response.text}" + ) + populace_files = { + sibling["rfilename"] for sibling in populace_response.json()["siblings"] + } + assert "populace_uk_2023.h5" in populace_files @pytest.mark.skipif( do_not_run_in_debug(), diff --git a/tests/fixtures/libs/simulation_entrypoint.py b/tests/fixtures/libs/simulation_entrypoint.py index 47263c121..02a444be8 100644 --- a/tests/fixtures/libs/simulation_entrypoint.py +++ b/tests/fixtures/libs/simulation_entrypoint.py @@ -15,6 +15,11 @@ MODAL_EXECUTION_STATUS_COMPLETE, MODAL_EXECUTION_STATUS_FAILED, ) +from tests.fixtures.spm import ( + worker_spm_capability, + worker_result_fields, + worker_versions_document, +) # Mock data constants MOCK_MODAL_JOB_ID = "fc-abc123xyz" @@ -44,6 +49,7 @@ "poverty_impact": {"baseline": 0.12, "reform": 0.10}, "budget_impact": {"baseline": 1000, "reform": 1200}, "inequality_impact": {"baseline": 0.45, "reform": 0.42}, + **worker_result_fields(years=["2025"]), } MOCK_POLICYENGINE_BUNDLE = { "model_version": "1.459.0", @@ -55,6 +61,13 @@ ), } MOCK_RESOLVED_APP_NAME = "policyengine-us-1-459-0" +MOCK_WORKER_VERSIONS = worker_versions_document( + app_name=MOCK_RESOLVED_APP_NAME, + country_version="1.459.0", +) +for bundle_version in ("3.4.0", "4.18.3"): + MOCK_WORKER_VERSIONS["policyengine"][bundle_version] = MOCK_RESOLVED_APP_NAME + MOCK_WORKER_VERSIONS["spm_capabilities"][bundle_version] = worker_spm_capability() MOCK_SUBMIT_RESPONSE_SUCCESS = { "job_id": MOCK_MODAL_JOB_ID, @@ -119,7 +132,14 @@ "startYear": "2026", "endYear": "2028", "windowSize": 3, - "annualImpacts": [], + "annualImpacts": [ + { + "year": year, + **MOCK_SIMULATION_RESULT, + **worker_result_fields(years=[year]), + } + for year in ("2026", "2027", "2028") + ], "totals": {}, }, "error": None, @@ -200,6 +220,10 @@ def mock_httpx_client(): ) as mock_client_class: mock_client = MagicMock() mock_client_class.return_value = mock_client + mock_client.get.return_value = create_mock_httpx_response( + status_code=200, + json_data=MOCK_WORKER_VERSIONS, + ) yield mock_client diff --git a/tests/fixtures/services/economy_service.py b/tests/fixtures/services/economy_service.py index 666f93880..e1ae2d190 100644 --- a/tests/fixtures/services/economy_service.py +++ b/tests/fixtures/services/economy_service.py @@ -9,6 +9,12 @@ MODAL_EXECUTION_STATUS_SUBMITTED, ) from policyengine_api.data.v1_models import ReformImpact +from tests.fixtures.spm import ( + options_hash_segment, + provenance_schema_hash_segment, + worker_result_fields, + worker_spm_capability, +) # Mock data constants MOCK_COUNTRY_ID = "us" @@ -26,9 +32,12 @@ MOCK_DATA_VERSION = "faux-populace-us-2099-test-release" MOCK_LOOKUP_OPTIONS_HASH = ( "[option1=value1&option2=value2" - "&dataset=default" - "&model_version=1.2.3" - "&policyengine_version=3.4.0]" + + options_hash_segment() + + "&dataset=default" + + "&model_version=1.2.3" + + "&policyengine_version=3.4.0" + + provenance_schema_hash_segment() + + "]" ) MOCK_OPTIONS_HASH = ( MOCK_LOOKUP_OPTIONS_HASH[:-1] @@ -56,6 +65,7 @@ "poverty_impact": {"baseline": 0.12, "reform": 0.10}, "budget_impact": {"baseline": 1000, "reform": 1200}, "inequality_impact": {"baseline": 0.45, "reform": 0.42}, + **worker_result_fields(years=[MOCK_TIME_PERIOD]), } MOCK_SIM_CONFIG = { @@ -136,6 +146,7 @@ def mock_simulation_entrypoint(): mock_batch_execution = create_mock_budget_window_batch_execution() mock_api._setup_sim_options.return_value = MOCK_SIM_CONFIG + mock_api.get_spm_capability.return_value = worker_spm_capability() mock_api.run.return_value = mock_execution mock_api.resolve_app_name.side_effect = ( lambda country_id, version=None, policyengine_version=None: ( @@ -316,6 +327,7 @@ def mock_simulation_entrypoint_legacy(): mock_execution = create_mock_modal_execution() mock_api.run.return_value = mock_execution + mock_api.get_spm_capability.return_value = None mock_api.resolve_app_name.side_effect = ( lambda country_id, version=None, policyengine_version=None: ( MOCK_RESOLVED_APP_NAME, diff --git a/tests/fixtures/spm.py b/tests/fixtures/spm.py new file mode 100644 index 000000000..c295633ec --- /dev/null +++ b/tests/fixtures/spm.py @@ -0,0 +1,121 @@ +"""Shared result shapes for the SPM behavior selected by the installed bundle.""" + +from __future__ import annotations + +from policyengine_api import spm +from policyengine_api.constants import POLICYENGINE_VERSION + + +SPM_CONTRACT_VERSION = "canonical-spm-v1" +INSTALLED_SPM_SELECTION = spm.normalize_spm_selection("us", None) + + +def worker_spm_capability(selection: dict | None = None) -> dict | None: + """Return the capability advertised by a worker using this bundle.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + if resolved is None: + return None + return { + "contract_version": SPM_CONTRACT_VERSION, + "defaults": dict(resolved), + } + + +def worker_versions_document( + *, + bundle_version: str = POLICYENGINE_VERSION, + app_name: str = "test-worker-app", + country: str = "us", + country_version: str | None = None, + selection: dict | None = None, +) -> dict: + """Return the gateway registry document for a matching test worker.""" + document: dict = { + "policyengine": {bundle_version: app_name, "latest": bundle_version}, + } + if country_version is not None: + document[country] = { + country_version: app_name, + "latest": country_version, + } + capability = worker_spm_capability(selection) + if capability is not None: + document["spm_capabilities"] = {bundle_version: capability} + return document + + +def spm_receipt(*, years, selection: dict | None = None) -> dict: + """Return one calculation receipt covering the supplied years.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + if resolved is None: + raise ValueError("An SPM receipt requires a resolved selection") + return { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "test-only", + "forecast_sha256": resolved["forecast_content_sha256"], + "scenario": resolved["scenario"], + "geography_kind": resolved["geography_kind"], + "geography_id": resolved["geography_id"], + "county_vintage": resolved["county_vintage"], + "as_of": resolved["as_of"], + "years": sorted({str(year) for year in years}), + "runtime_versions": { + "policyengine": "test-only", + "policyengine-core": "test-only", + "policyengine-us": "test-only", + "spm-calculator": "test-only", + }, + } + + +def worker_result_fields(*, years, selection: dict | None = None) -> dict: + """Return the SPM fields required on an economy worker result.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + if resolved is None: + return {} + receipt = spm_receipt(years=years, selection=resolved) + return { + "spm_provenance": { + "schema_version": "canonical-spm-comparison-v2", + "baseline": { + "receipt": dict(receipt), + "execution_count": 1, + }, + "reform": { + "receipt": dict(receipt), + "execution_count": 1, + }, + }, + } + + +def household_result_fields(*, years, selection: dict | None = None) -> dict: + """Return the SPM fields required on a household calculation result.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + if resolved is None: + return {} + return { + "spm_provenance": spm_receipt(years=years, selection=resolved), + } + + +def resolved_options(options: dict, selection: dict | None = None) -> dict: + """Return request options after applying the installed bundle default.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + if resolved is None: + return dict(options) + return {**options, "spm": dict(resolved)} + + +def options_hash_segment(selection: dict | None = None) -> str: + """Return the part of a sorted options hash contributed by SPM.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + return "" if resolved is None else f"&spm={resolved}" + + +def provenance_schema_hash_segment(selection: dict | None = None) -> str: + """Return the cache-identity segment for compact society-wide receipts.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + if resolved is None: + return "" + return "&spm_provenance_schema=canonical-spm-comparison-v2" diff --git a/tests/integration/test_budget_window_in_flight_dedupe.py b/tests/integration/test_budget_window_in_flight_dedupe.py index 4c3c45d8e..826154ca9 100644 --- a/tests/integration/test_budget_window_in_flight_dedupe.py +++ b/tests/integration/test_budget_window_in_flight_dedupe.py @@ -2,6 +2,7 @@ from flask import Flask from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from tests.fixtures.spm import worker_spm_capability class FakeRedis(InMemoryCacheBackend): @@ -31,6 +32,7 @@ def test_budget_window_in_flight_dedupe_uses_existing_batch_without_live_db( fake_cache = BudgetWindowCache(client=FakeRedis()) simulation_entrypoint = MagicMock() + simulation_entrypoint.get_spm_capability.return_value = worker_spm_capability() reform_impacts_service = MagicMock() simulation_entrypoint.run_budget_window_batch.return_value = ( diff --git a/tests/integration/test_runtime_cache_redis.py b/tests/integration/test_runtime_cache_redis.py index 913e9193d..e12b04575 100644 --- a/tests/integration/test_runtime_cache_redis.py +++ b/tests/integration/test_runtime_cache_redis.py @@ -120,25 +120,29 @@ def test_cached_calculation_receipts_survive_a_real_round_trip(redis_pair) -> No value = CachedHouseholdCalculation( household={"people": {"you": {}}}, warnings=(), - # A canonical receipt may omit its null settings; that is the same choice. - spm_config={ - name: setting for name, setting in selection.items() if setting is not None - }, spm_provenance={ + "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-artifact", "forecast_sha256": selection["forecast_content_sha256"], "scenario": "baseline", "geography_kind": "national", - "runtime_versions": {"policyengine-us": "test-only"}, - "years": {"2026": {"status": "forecast"}}, - "geographies": [], - "composition_method": "classified-inputs", - "storage_method": "formula", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "runtime_versions": { + "policyengine": "test-only", + "policyengine-core": "test-only", + "policyengine-us": "test-only", + "spm-calculator": "test-only", + }, + "years": ["2026"], }, ) assert HouseholdCalculationCache(first, namespace).set(identity, value) - assert HouseholdCalculationCache(second, namespace).get(identity) == value + cached = HouseholdCalculationCache(second, namespace).get(identity) + assert cached is not None + assert cached.spm_provenance == value.spm_provenance other = HouseholdCalculationIdentity( **{**vars(identity), "spm": {**selection, "geography_kind": "county"}} ) diff --git a/tests/integration/test_v2_catalog_installed.py b/tests/integration/test_v2_catalog_installed.py index b09ce746a..88718b122 100644 --- a/tests/integration/test_v2_catalog_installed.py +++ b/tests/integration/test_v2_catalog_installed.py @@ -39,10 +39,10 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: assert catalog.entity_counts() == { "models": 2, "model_versions": 2, - "variables": 6_649, - "parameter_nodes": 27_813, - "parameters": 99_006, - "parameter_values": 1_172_130, + "variables": 7_104, + "parameter_nodes": 29_170, + "parameters": 103_853, + "parameter_values": 1_193_400, "datasets": 2, "regions": 826, } diff --git a/tests/unit/libs/test_simulation_entrypoint.py b/tests/unit/libs/test_simulation_entrypoint.py index 8e124ec4f..833c85f68 100644 --- a/tests/unit/libs/test_simulation_entrypoint.py +++ b/tests/unit/libs/test_simulation_entrypoint.py @@ -15,6 +15,7 @@ os.environ.setdefault("FLASK_DEBUG", "1") from policyengine_api.constants import ( # noqa: E402 + POLICYENGINE_VERSION, MODAL_EXECUTION_STATUS_COMPLETE, MODAL_EXECUTION_STATUS_FAILED, MODAL_EXECUTION_STATUS_RUNNING, @@ -57,8 +58,10 @@ MOCK_SIMULATION_PAYLOAD_WITH_TELEMETRY, MOCK_SIMULATION_RESULT, MOCK_SUBMIT_RESPONSE_SUCCESS, + MOCK_WORKER_VERSIONS, create_mock_httpx_response, ) +from tests.fixtures.spm import INSTALLED_SPM_SELECTION # noqa: E402 pytest_plugins = ("tests.fixtures.libs.simulation_entrypoint",) @@ -107,6 +110,9 @@ def _response(self, method, url, json=None): elif "/jobs/" in path: payload = MOCK_POLL_RESPONSE_RUNNING status_code = 202 + elif path == "/versions": + payload = MOCK_WORKER_VERSIONS + status_code = 200 elif "/versions/" in path: payload = { "latest": "1.459.0", @@ -500,6 +506,7 @@ def test__given_request_context__then_all_calls_forward_request_id( "/simulate/economy/budget-window", f"/jobs/{MOCK_MODAL_JOB_ID}", f"/budget-window-jobs/{MOCK_BATCH_JOB_ID}", + "/versions", "/versions/us", "/health", } @@ -601,7 +608,11 @@ def test__given_valid_payload__then_posts_to_correct_endpoint( # Then call_args = mock_httpx_client.post.call_args assert "/simulate/economy/comparison" in call_args[0][0] - assert call_args[1]["json"] == MOCK_SIMULATION_PAYLOAD + assert call_args[1]["json"] == { + **MOCK_SIMULATION_PAYLOAD, + "spm": INSTALLED_SPM_SELECTION, + "policyengine_version": POLICYENGINE_VERSION, + } def test__given_telemetry_payload__then_preserves_non_identity_fields( self, @@ -688,6 +699,7 @@ def test__given_api_v1_default_bundle_payload__then_posts_gateway_contract_body( "include_cliffs", "version", "policyengine_version", + "spm", "_metadata", "_telemetry", } diff --git a/tests/unit/routes/test_calculate_deprecated_inputs.py b/tests/unit/routes/test_calculate_deprecated_inputs.py index 67ebe09f5..3c52df5b4 100644 --- a/tests/unit/routes/test_calculate_deprecated_inputs.py +++ b/tests/unit/routes/test_calculate_deprecated_inputs.py @@ -5,8 +5,10 @@ from policyengine_api.routes import household_routes from policyengine_api.extensions import cache from policyengine_api.services.household_calculation_service import ( + CalculationResult, HouseholdCalculationService, ) +from tests.fixtures.spm import household_result_fields class DummyCountry: @@ -38,10 +40,13 @@ def __init__(self): }, } - def calculate(self, household, policy): + def calculate(self, household, policy, **_kwargs): self.household = household self.policy = policy - return {"household": household, "policy": policy} + return CalculationResult( + household={"household": household, "policy": policy}, + **household_result_fields(years=["2025"]), + ) @pytest.fixture diff --git a/tests/unit/routes/test_calculate_error_statuses.py b/tests/unit/routes/test_calculate_error_statuses.py index 206d4dddd..4af690247 100644 --- a/tests/unit/routes/test_calculate_error_statuses.py +++ b/tests/unit/routes/test_calculate_error_statuses.py @@ -21,7 +21,7 @@ class ParsingErrorCountry(DummyCountry): - def prepare_calculation(self, household, policy): + def prepare_calculation(self, household, policy, **_kwargs): raise SituationParsingError( ["people", "you", "employment_income", "2026"], "Can't deal with value: expected type number, received '{}'.", @@ -29,7 +29,7 @@ def prepare_calculation(self, household, policy): class CrashingCountry(DummyCountry): - def calculate(self, household, policy): + def calculate(self, household, policy, **_kwargs): raise RuntimeError("engine exploded") diff --git a/tests/unit/routes/test_canonical_spm.py b/tests/unit/routes/test_canonical_spm.py index 2dd51f9d3..7e8c678e8 100644 --- a/tests/unit/routes/test_canonical_spm.py +++ b/tests/unit/routes/test_canonical_spm.py @@ -69,18 +69,24 @@ def calculate(self, household, policy, **kwargs): None if config is None else { + "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-artifact", "forecast_sha256": config["forecast_content_sha256"], "scenario": config["scenario"], "geography_kind": config["geography_kind"], - "runtime_versions": {"policyengine-us": "test-only"}, - "years": {"2026": {"status": "forecast"}}, - "geographies": [], - "composition_method": "classified-inputs", - "storage_method": "formula", + "geography_id": config["geography_id"], + "county_vintage": config["county_vintage"], + "as_of": config["as_of"], + "runtime_versions": { + "policyengine": "test-only", + "policyengine-core": "test-only", + "policyengine-us": "test-only", + "spm-calculator": "test-only", + }, + "years": ["2026"], } ) - return CalculationResult(household, (), config, receipt) + return CalculationResult(household, (), receipt) @pytest.fixture @@ -215,7 +221,7 @@ def test_settings_and_provenance_survive_storage_reform_replay_and_replacement( cached = client.get(url + "/policy/2") assert first.status_code == cached.status_code == 200 assert first.json == cached.json - assert first.json["spm_config"] == stored["spm"] + assert "spm_config" not in first.json assert first.json["spm_provenance"]["forecast_sha256"] == FORECAST_HASH assert len(country.calls) == 1 assert "spm" not in country.calls[0][0] @@ -251,7 +257,7 @@ def test_settings_and_provenance_survive_storage_reform_replay_and_replacement( latest = client.get(latest_url).json["result"] assert latest["household_hash"] != edited["household_hash"] assert ( - client.get(latest_url + "/policy/2").json["spm_config"]["geography_kind"] + client.get(latest_url + "/policy/2").json["spm_provenance"]["geography_kind"] == "county" ) assert len(country.calls) == 4 @@ -280,7 +286,7 @@ def test_http_cache_varies_with_measurement_settings(certified, harness, selecti second = client.post("/us/calculate", json=payload) assert second.status_code == 200 assert len(country.calls) == 2 - assert first.json["spm_config"] != second.json["spm_config"] + assert first.json["spm_provenance"] != second.json["spm_provenance"] def test_successful_http_cache_hit_still_starts_observability(certified, harness): @@ -313,7 +319,9 @@ def test_certification_checked_before_cached_response(certified, harness): @pytest.mark.parametrize("country_id", ["us", "uk"]) -def test_legacy_country_requests_do_not_receive_spm(harness, country_id): +def test_legacy_country_requests_do_not_receive_spm(harness, country_id, monkeypatch): + monkeypatch.setattr(spm, "_current_bundle", lambda: {}) + monkeypatch.setattr(spm, "simulation_supports_spm", lambda _: False) client, country = harness response = client.post(f"/{country_id}/calculate", json={"household": HOUSEHOLD}) assert response.status_code == 200 @@ -330,7 +338,7 @@ def test_http_cache_default_artifact_hash_is_part_of_identity(certified, harness second = client.post("/us/calculate", json=payload) assert first.status_code == second.status_code == 200 assert len(country.calls) == 2 - assert first.json["spm_config"] != second.json["spm_config"] + assert first.json["spm_provenance"] != second.json["spm_provenance"] def test_linked_household_measurement_is_immutable_for_saved_reports( @@ -538,14 +546,8 @@ def test_valid_spm_database_timeout_keeps_stage11_safe_persistence_response( copy_event.assert_not_called() -def test_stored_replay_hits_the_calculation_cache_when_a_receipt_omits_nulls( - certified, harness -): - """A canonical country may omit null receipt settings; replay must still hit. - - Requiring exact JSON equality between the receipt and the resolved identity - made every stored replay recompute. - """ +def test_stored_replay_rejects_a_receipt_that_omits_null_settings(certified, harness): + """The sole receipt must state every field in its resolved configuration.""" client, country = harness calculate = country.calculate @@ -554,12 +556,11 @@ def omitting_null_settings(household, policy, **kwargs): return CalculationResult( household=result.household, warnings=result.warnings, - spm_config={ + spm_provenance={ key: value - for key, value in result.spm_config.items() + for key, value in result.spm_provenance.items() if value is not None }, - spm_provenance=result.spm_provenance, ) country.calculate = omitting_null_settings @@ -568,18 +569,9 @@ def omitting_null_settings(household, policy, **kwargs): ) assert created.status_code == 201, created.json url = f"/us/household/{created.json['result']['household_id']}/policy/2" - first = client.get(url) - assert first.status_code == 200, first.json - assert set(first.json["spm_config"]) == { - "forecast_content_sha256", - "scenario", - "geography_kind", - "county_vintage", - } - cached = client.get(url) - assert cached.status_code == 200 - assert cached.json == first.json - assert len(country.calls) == 1 + assert client.get(url).status_code == 500 + assert client.get(url).status_code == 500 + assert len(country.calls) == 2 @pytest.mark.parametrize("version", ["5.2.1", "5.4.0", "6.0.0"]) diff --git a/tests/unit/routes/test_household_and_user_policy_orm_routes.py b/tests/unit/routes/test_household_and_user_policy_orm_routes.py index ee38bea40..43c782c52 100644 --- a/tests/unit/routes/test_household_and_user_policy_orm_routes.py +++ b/tests/unit/routes/test_household_and_user_policy_orm_routes.py @@ -25,8 +25,10 @@ update_user_policy, ) from policyengine_api.services.household_calculation_service import ( + CalculationResult, HouseholdCalculationService, ) +from tests.fixtures.spm import INSTALLED_SPM_SELECTION, household_result_fields def test_household_under_policy_returns_cached_json_object(orm_session_factory): @@ -65,8 +67,12 @@ def test_household_under_policy_returns_cached_json_object(orm_session_factory): policy_hash="policy-hash", country_package_version=COUNTRY_PACKAGE_VERSIONS["us"], policyengine_version=POLICYENGINE_VERSION, + spm=INSTALLED_SPM_SELECTION, + ), + CachedHouseholdCalculation( + household=stored_result, + **household_result_fields(years=["2026"]), ), - CachedHouseholdCalculation(household=stored_result), ) service = HouseholdCalculationService( primary_session_factory=orm_session_factory, @@ -108,7 +114,12 @@ def test_household_under_policy_calculates_and_caches_json_as_an_object( ) calculated = {"people": {"you": {"net_income": {"2026": 42}}}} country = SimpleNamespace( - calculate=Mock(return_value=calculated), + calculate=Mock( + return_value=CalculationResult( + household=calculated, + **household_result_fields(years=["2026"]), + ) + ), metadata={ "variables": {}, "entities": {"person": {"plural": "people", "roles": {}}}, @@ -134,6 +145,8 @@ def test_household_under_policy_calculates_and_caches_json_as_an_object( country.calculate.assert_called_once_with( {"people": {"you": {}}}, {"gov.example.parameter": 1}, + spm=INSTALLED_SPM_SELECTION, + spm_requested=False, ) diff --git a/tests/unit/routes/test_household_numeric_identity.py b/tests/unit/routes/test_household_numeric_identity.py index 524cf4e5b..e65e4b13b 100644 --- a/tests/unit/routes/test_household_numeric_identity.py +++ b/tests/unit/routes/test_household_numeric_identity.py @@ -219,9 +219,12 @@ def test_numeric_link_preserves_immutable_household_and_saved_simulation( assert saved["population_id"] == saved_spelling assert json.loads(saved["simulation_spec_json"])["population_id"] == saved_spelling if reference == "historical-padded": - assert saved["status"] == "complete" - assert json.loads(saved["output"]) == output + assert saved["status"] == "pending" + assert saved["output"] is None + assert saved["latest_successful_run_id"] is None + assert saved["active_run_id"] is not None with orm_session_factory() as session: - run = session.get(SimulationRun, saved["latest_successful_run_id"]) + run = session.get(SimulationRun, saved["active_run_id"]) assert run.simulation_spec_snapshot_json["population_id"] == saved_spelling - assert run.output == output + assert run.status == "pending" + assert run.output is None diff --git a/tests/unit/routes/test_simulation_spm.py b/tests/unit/routes/test_simulation_spm.py index 59b591a5a..22a151696 100644 --- a/tests/unit/routes/test_simulation_spm.py +++ b/tests/unit/routes/test_simulation_spm.py @@ -46,7 +46,7 @@ def test_simulation_records_reject_spm_overrides_before_read_or_write( @pytest.mark.parametrize("population_id", ["00001", "1", "0000001", 1]) -def test_repost_historical_numeric_household_alias_replays_completed_output( +def test_repost_historical_numeric_household_alias_recalculates_old_spm_output( orm_session_factory, population_id ): output = {"household": {"people": {}}, "spm_config": {"geography_kind": "national"}} @@ -81,8 +81,8 @@ def test_repost_historical_numeric_household_alias_replays_completed_output( assert response.status_code == 200 result = response.get_json()["result"] assert result["id"] == historical_id - assert result["status"] == "complete" - assert json.loads(result["output"]) == output + assert result["status"] == "pending" + assert result["output"] is None assert result["population_id"] == "00001" assert ( json.loads(result["simulation_spec_json"])["population_id"] == "00001" @@ -91,6 +91,6 @@ def test_repost_historical_numeric_household_alias_replays_completed_output( with orm_session_factory() as session: assert session.scalar(select(func.count()).select_from(Simulation)) == 1 run = session.scalar(select(SimulationRun)) - assert run.status == "complete" - assert run.output == output + assert run.status == "pending" + assert run.output is None assert run.simulation_spec_snapshot_json["population_id"] == "00001" diff --git a/tests/unit/routes/test_spm_year_worker_polling.py b/tests/unit/routes/test_spm_year_worker_polling.py index c79f023f4..3faa0ab02 100644 --- a/tests/unit/routes/test_spm_year_worker_polling.py +++ b/tests/unit/routes/test_spm_year_worker_polling.py @@ -15,7 +15,6 @@ from policyengine_api.services.economy_service import EconomyService from policyengine_api.services.reform_impacts_service import ReformImpactsService - SELECTION = { "forecast_content_sha256": "a" * 64, "scenario": "baseline", @@ -32,26 +31,32 @@ def segmented_result(year, invalid_side=None): receipt = { + "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-only", "forecast_sha256": SELECTION["forecast_content_sha256"], "scenario": SELECTION["scenario"], "geography_kind": SELECTION["geography_kind"], - "runtime_versions": {}, - "years": {year: {}}, - "geographies": [], - "composition_method": "classified", - "storage_method": "formula", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "runtime_versions": { + "policyengine": "test", + "policyengine-core": "test", + "policyengine-us": "test", + "spm-calculator": "test", + }, + "years": [year], } result = { "year": year, - "spm_config": SELECTION, "spm_provenance": { - "baseline": [dict(receipt), dict(receipt)], - "reform": [dict(receipt), dict(receipt)], + "schema_version": "canonical-spm-comparison-v2", + "baseline": {"receipt": dict(receipt), "execution_count": 2}, + "reform": {"receipt": dict(receipt), "execution_count": 2}, }, } if invalid_side: - result["spm_provenance"][invalid_side][1]["years"] = {"2000": {}} + result["spm_provenance"][invalid_side]["receipt"]["years"] = ["2000"] return result @@ -75,7 +80,7 @@ def test_typed_worker_error_is_terminal_and_replays_after_service_recreation( if isinstance(failure, str): typed_error = { "code": "SPM_CONFIGURATION_UNAVAILABLE", - "message": "Worker SPM receipt does not cover requested year", + "message": "baseline and reform SPM receipts differ", } side = failure.removeprefix("segmented-") worker_result = ( diff --git a/tests/unit/runtime_cache/test_household_calculations.py b/tests/unit/runtime_cache/test_household_calculations.py index 27b23a044..cfe422efe 100644 --- a/tests/unit/runtime_cache/test_household_calculations.py +++ b/tests/unit/runtime_cache/test_household_calculations.py @@ -10,6 +10,7 @@ HouseholdCalculationCache, HouseholdCalculationIdentity, ) +from policyengine_api.spm import SPMSelection, build_spm_calculation_provenance def _namespace() -> CacheNamespace: @@ -93,15 +94,21 @@ def test_receipt_bearing_payload_supersedes_the_receiptless_schema() -> None: "as_of": None, } SPM_RECEIPT = { + "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-artifact", "forecast_sha256": "a" * 64, "scenario": "baseline", "geography_kind": "national", - "runtime_versions": {"policyengine-us": "test"}, - "years": {}, - "geographies": [], - "composition_method": "test composition", - "storage_method": "test storage", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "runtime_versions": { + "policyengine": "test", + "policyengine-core": "test", + "policyengine-us": "test", + "spm-calculator": "test", + }, + "years": [], } @@ -112,7 +119,7 @@ def test_receipt_bearing_payload_supersedes_the_receiptless_schema() -> None: {**SPM_RECEIPT, "forecast_sha256": "b" * 64}, {**SPM_RECEIPT, "scenario": "other"}, {**SPM_RECEIPT, "geography_kind": "county"}, - {**SPM_RECEIPT, "years": []}, + {**SPM_RECEIPT, "years": ["2027", "2026"]}, {**SPM_RECEIPT, "unexpected": True}, ], ids=["empty", "hash", "scenario", "geography", "years-shape", "unknown-field"], @@ -122,24 +129,53 @@ def test_household_cache_rejects_invalid_or_mismatched_spm_receipts(receipt): identity = _identity(spm=SPM_CONFIG) value = CachedHouseholdCalculation( household={"people": {}}, - spm_config=SPM_CONFIG, spm_provenance=receipt, ) - assert cache.set(identity, value) is True + assert cache.set(identity, value) is False assert cache.get(identity) is None -@pytest.mark.parametrize("years", [{}, {"2024": {"source": "test"}}]) +def test_household_cache_rejects_previous_diagnostic_receipt_before_write(): + backend = InMemoryCacheBackend() + cache = HouseholdCalculationCache(backend, _namespace()) + identity = _identity(spm=SPM_CONFIG) + previous_receipt = { + "forecast_id": "test-artifact", + "forecast_sha256": "a" * 64, + "scenario": "baseline", + "geography_kind": "national", + "runtime_versions": {}, + "years": {}, + "geographies": [], + "composition_method": "test", + "storage_method": "test", + } + + assert ( + cache.set( + identity, + CachedHouseholdCalculation( + household={"people": {}}, + spm_provenance=previous_receipt, + ), + ) + is False + ) + assert backend._values == {} + + +@pytest.mark.parametrize("years", [[], ["2024"]]) def test_household_cache_keeps_valid_receipts_including_lazy_tax_only(years): cache = HouseholdCalculationCache(InMemoryCacheBackend(), _namespace()) identity = _identity(spm=SPM_CONFIG) value = CachedHouseholdCalculation( household={"people": {}}, - spm_config=SPM_CONFIG, spm_provenance={**SPM_RECEIPT, "years": years}, ) assert cache.set(identity, value) is True - assert cache.get(identity) == value + cached = cache.get(identity) + assert cached is not None + assert cached.spm_provenance == {**SPM_RECEIPT, "years": years} @pytest.mark.parametrize("calculate_spm", [False, True], ids=["tax-only", "measured"]) @@ -154,51 +190,45 @@ def test_real_country_provider_receipt_round_trips_household_cache(calculate_spm tenure="renter", ) config = country_spm.spm_config(provider) - receipt = provider.provenance() + receipt = build_spm_calculation_provenance( + SPMSelection.model_validate(config), provider.provenance() + ).model_dump(mode="json", by_alias=True) assert bool(receipt["years"]) is calculate_spm cache = HouseholdCalculationCache(InMemoryCacheBackend(), _namespace()) identity = _identity(spm=config) value = CachedHouseholdCalculation( household={"people": {}}, - spm_config=config, spm_provenance=receipt, ) assert cache.set(identity, value) is True - assert cache.get(identity) == value - - -OMITTED_NULL_CONFIG = { - key: value for key, value in SPM_CONFIG.items() if value is not None -} + cached = cache.get(identity) + assert cached is not None + assert cached.spm_provenance == receipt -def test_household_cache_hits_when_receipt_omits_null_settings(): - """A canonical receipt may omit its null values; that is still the same selection.""" - cache = HouseholdCalculationCache(InMemoryCacheBackend(), _namespace()) - identity = _identity(spm=SPM_CONFIG) - value = CachedHouseholdCalculation( - household={"people": {}}, - spm_config=OMITTED_NULL_CONFIG, - spm_provenance=SPM_RECEIPT, - ) - assert cache.set(identity, value) is True - assert cache.get(identity) == value - - -@pytest.mark.parametrize("omitted", sorted(OMITTED_NULL_CONFIG)) -def test_household_cache_rejects_receipt_omitting_a_nonnull_setting(omitted): - """An omitted non-null setting must never inherit today's resolved default.""" +@pytest.mark.parametrize( + "omitted", + [ + "forecast_sha256", + "scenario", + "geography_kind", + "geography_id", + "county_vintage", + "as_of", + ], +) +def test_household_cache_rejects_receipt_omitting_resolved_setting(omitted): + """Every resolved setting is required on the sole completed-result receipt.""" cache = HouseholdCalculationCache(InMemoryCacheBackend(), _namespace()) identity = _identity(spm=SPM_CONFIG) value = CachedHouseholdCalculation( household={"people": {}}, - spm_config={ - key: item for key, item in OMITTED_NULL_CONFIG.items() if key != omitted + spm_provenance={ + key: item for key, item in SPM_RECEIPT.items() if key != omitted }, - spm_provenance=SPM_RECEIPT, ) - assert cache.set(identity, value) is True + assert cache.set(identity, value) is False assert cache.get(identity) is None @@ -218,8 +248,28 @@ def test_household_cache_rejects_receipt_with_different_settings(changed): identity = _identity(spm=SPM_CONFIG) value = CachedHouseholdCalculation( household={"people": {}}, - spm_config={**OMITTED_NULL_CONFIG, **changed}, - spm_provenance=SPM_RECEIPT, + spm_provenance={**SPM_RECEIPT, **changed}, ) - assert cache.set(identity, value) is True + assert cache.set(identity, value) is False + assert cache.get(identity) is None + + +def test_household_cache_rejects_legacy_dual_field_payload() -> None: + backend = InMemoryCacheBackend() + cache = HouseholdCalculationCache(backend, _namespace()) + identity = _identity(spm=SPM_CONFIG) + backend.set( + cache.cache_key(identity), + encode_envelope( + "household-calculation", + HOUSEHOLD_CALCULATION_SCHEMA_VERSION, + { + "household": {"people": {}}, + "warnings": [], + "spm_config": SPM_CONFIG, + "spm_provenance": SPM_RECEIPT, + }, + ), + ) + assert cache.get(identity) is None diff --git a/tests/unit/runtime_cache/test_reform_impacts.py b/tests/unit/runtime_cache/test_reform_impacts.py index 43445b86a..26a02cca7 100644 --- a/tests/unit/runtime_cache/test_reform_impacts.py +++ b/tests/unit/runtime_cache/test_reform_impacts.py @@ -4,9 +4,14 @@ import pytest -from policyengine_api.runtime_cache.core import CacheCoordinationError, CacheNamespace +from policyengine_api.runtime_cache.core import ( + CacheCoordinationError, + CacheNamespace, + encode_envelope, +) from policyengine_api.runtime_cache.fake import InMemoryCacheBackend from policyengine_api.runtime_cache.reform_impacts import ( + REFORM_IMPACT_SCHEMA_VERSION, REFORM_IMPACT_START_CLAIM_TTL_SECONDS, CachedReformImpact, ReformImpactCache, @@ -189,6 +194,31 @@ def test_reform_impact_record_and_indexes_share_one_jittered_ttl( assert set(backend._expires.values()) == {123} +def test_previous_schema_reform_impact_is_not_read() -> None: + backend = InMemoryCacheBackend() + namespace = _namespace() + cache = ReformImpactCache(backend, namespace) + legacy = _impact("legacy", "[spm=canonical]", 1) + previous_version = REFORM_IMPACT_SCHEMA_VERSION - 1 + legacy_key = namespace.key( + "reform-impact", previous_version, {"execution_id": "legacy"} + ) + backend.set( + legacy_key, + encode_envelope( + "reform-impact", + previous_version, + { + **legacy.__dict__, + "start_time": legacy.start_time.isoformat(), + "end_time": None, + }, + ), + ) + + assert cache.get_by_execution_id("legacy") is None + + def test_reform_impact_updates_and_deletes_only_matching_computing_values() -> None: backend = InMemoryCacheBackend() cache = ReformImpactCache(backend, _namespace()) diff --git a/tests/unit/services/test_budget_window_cache.py b/tests/unit/services/test_budget_window_cache.py index 1c9009c5e..ff344e924 100644 --- a/tests/unit/services/test_budget_window_cache.py +++ b/tests/unit/services/test_budget_window_cache.py @@ -3,9 +3,11 @@ import pytest from policyengine_api.runtime_cache.core import CacheCoordinationError +from policyengine_api.runtime_cache.core import CacheNamespace, encode_envelope from policyengine_api.runtime_cache.fake import InMemoryCacheBackend from policyengine_api.services.budget_window_cache import ( BUDGET_WINDOW_BATCH_TTL_SECONDS, + BUDGET_WINDOW_CACHE_SCHEMA_VERSION, BUDGET_WINDOW_STARTING_TTL_SECONDS, BudgetWindowCache, BudgetWindowCacheState, @@ -80,7 +82,37 @@ def test_build_key_is_stable_for_request_identity(): ) assert first == second - assert first.startswith("policyengine:test:api:budget-window:v2:") + assert first.startswith("policyengine:test:api:budget-window:v4:") + + +def test_previous_schema_cache_identity_is_not_read(): + backend = FakeRedis() + namespace = CacheNamespace("test", "api") + cache = BudgetWindowCache(client=backend, namespace=namespace) + identity = { + "api_version": "v1", + "baseline_policy_id": 1, + "country_id": "us", + "dataset": "default", + "options_hash": "[spm=canonical]", + "reform_policy_id": 2, + "region": "us", + "time_period": "budget_window:2026:2", + } + previous_version = BUDGET_WINDOW_CACHE_SCHEMA_VERSION - 1 + old_key = namespace.key("budget-window", previous_version, identity) + backend.set( + f"{old_key}:state", + encode_envelope( + "budget-window", + previous_version, + {"status": "completed", "result": {"legacy": True}}, + ), + ) + current_key = cache.build_key(**identity) + + assert current_key != old_key + assert cache.get_state(current_key) is None def test_claim_batch_start_allows_one_starter_and_preserves_identity(): diff --git a/tests/unit/services/test_economy_service.py b/tests/unit/services/test_economy_service.py index 60f1a42e1..e9c30264f 100644 --- a/tests/unit/services/test_economy_service.py +++ b/tests/unit/services/test_economy_service.py @@ -45,6 +45,11 @@ create_mock_budget_window_batch_execution, create_mock_reform_impact, ) +from tests.fixtures.spm import ( + resolved_options, + worker_result_fields, + worker_spm_capability, +) pytest_plugins = ("tests.fixtures.services.economy_service",) @@ -106,6 +111,28 @@ def make_http_status_error( ) +def complete_budget_window_result( + *, annual_values: dict[str, dict] | None = None, totals: dict | None = None +) -> dict: + years = ("2026", "2027", "2028") + annual_values = annual_values or {} + return { + "kind": "budgetWindow", + "startYear": years[0], + "endYear": years[-1], + "windowSize": len(years), + "annualImpacts": [ + { + "year": year, + **annual_values.get(year, {}), + **worker_result_fields(years=[year]), + } + for year in years + ], + "totals": totals or {}, + } + + class TestEconomyService: class TestGetEconomicImpact: @pytest.fixture @@ -394,7 +421,7 @@ def test__given_no_previous_impact__creates_new_simulation( write_values = ( mock_reform_impacts_service.set_reform_impact.call_args.kwargs ) - assert write_values["options"] == MOCK_OPTIONS + assert write_values["options"] == resolved_options(MOCK_OPTIONS) assert write_values["reform_impact_json"] == {} assert write_values["observability_id"] == MOCK_OBSERVABILITY_ID @@ -685,6 +712,7 @@ def test__given_policies_created_through_orm__submits_decoded_json( "policyengine-simulation-test", MOCK_MODEL_VERSION, ) + simulation_gateway.get_spm_capability.return_value = worker_spm_capability() simulation_gateway.get_execution_id.return_value = "execution-1" monkeypatch.setattr( "policyengine_api.services.economy_service.logger", @@ -1207,22 +1235,18 @@ def test__given_completed_cached_result__returns_completed_batch_result( mock_simulation_entrypoint, mock_budget_window_cache, ): - completed_result = { - "kind": "budgetWindow", - "startYear": "2026", - "endYear": "2028", - "windowSize": 3, - "annualImpacts": [ - { + completed_result = complete_budget_window_result( + annual_values={ + "2026": { "year": "2026", "taxRevenueImpact": 100, "federalTaxRevenueImpact": 80, "stateTaxRevenueImpact": 20, "benefitSpendingImpact": -10, "budgetaryImpact": 90, - } - ], - "totals": { + }, + }, + totals={ "year": "Total", "taxRevenueImpact": 100, "federalTaxRevenueImpact": 80, @@ -1230,7 +1254,7 @@ def test__given_completed_cached_result__returns_completed_batch_result( "benefitSpendingImpact": -10, "budgetaryImpact": 90, }, - } + ) mock_budget_window_cache.get_state.return_value = BudgetWindowCacheState( status="completed", result=completed_result, @@ -1289,14 +1313,7 @@ def test__given_completed_batch_poll__caches_result_and_returns_completed( mock_simulation_entrypoint, mock_budget_window_cache, ): - completed_result = { - "kind": "budgetWindow", - "startYear": "2026", - "endYear": "2028", - "windowSize": 3, - "annualImpacts": [], - "totals": {}, - } + completed_result = complete_budget_window_result() mock_budget_window_cache.get_state.return_value = BudgetWindowCacheState( status="submitted", batch_job_id="fc-budget-123", @@ -1367,14 +1384,7 @@ def test__given_completed_batch_cache_write_fails__does_not_clear_batch_id( mock_simulation_entrypoint, mock_budget_window_cache, ): - completed_result = { - "kind": "budgetWindow", - "startYear": "2026", - "endYear": "2028", - "windowSize": 3, - "annualImpacts": [], - "totals": {}, - } + completed_result = complete_budget_window_result() mock_budget_window_cache.get_state.return_value = BudgetWindowCacheState( status="submitted", batch_job_id="fc-budget-123", @@ -1814,7 +1824,7 @@ def test__given_reordered_options__uses_same_budget_window_cache_identity( ): mock_budget_window_cache.get_state.return_value = BudgetWindowCacheState( status="completed", - result={"kind": "budgetWindow"}, + result=complete_budget_window_result(), observability_id=MOCK_OBSERVABILITY_ID, ) diff --git a/tests/unit/services/test_household_calculation_service.py b/tests/unit/services/test_household_calculation_service.py index 5259abad7..5199da998 100644 --- a/tests/unit/services/test_household_calculation_service.py +++ b/tests/unit/services/test_household_calculation_service.py @@ -22,6 +22,7 @@ HouseholdNotFoundError, PolicyNotFoundError, ) +from tests.fixtures.spm import INSTALLED_SPM_SELECTION, household_result_fields PACKAGE_ROOT = Path(__file__).parents[3] / "policyengine_api" @@ -91,6 +92,7 @@ def _identity() -> HouseholdCalculationIdentity: policy_hash="policy-hash", country_package_version=COUNTRY_PACKAGE_VERSIONS["us"], policyengine_version=POLICYENGINE_VERSION, + spm=INSTALLED_SPM_SELECTION, ) @@ -115,10 +117,11 @@ class Country: "parameters": {}, } - def calculate(self, household, policy): + def calculate(self, household, policy, **_kwargs): return CalculationResult( household=household, warnings=("employment_income could not be calculated",), + **household_result_fields(years=["2026"]), ) service = HouseholdCalculationService( @@ -144,10 +147,13 @@ class Country: def __init__(self): self.prepare = Mock(return_value=parsed_calculation) self.calculate = Mock( - return_value=CalculationResult(household={"people": {}}) + return_value=CalculationResult( + household={"people": {}}, + **household_result_fields(years=[]), + ) ) - def prepare_calculation(self, household, policy): + def prepare_calculation(self, household, policy, **_kwargs): return self.prepare(household, policy) country = Country() @@ -168,6 +174,8 @@ def prepare_calculation(self, household, policy): country.calculate.assert_called_once_with( {"people": {}}, {}, + spm=INSTALLED_SPM_SELECTION, + spm_requested=False, prepared=parsed_calculation, ) assert result.household == {"people": {}} @@ -183,7 +191,7 @@ class Country: "parameters": {}, } - def prepare_calculation(self, household, policy): + def prepare_calculation(self, household, policy, **_kwargs): return object() service = HouseholdCalculationService( @@ -211,7 +219,7 @@ class Country: "parameters": {}, } - def prepare_calculation(self, household, policy): + def prepare_calculation(self, household, policy, **_kwargs): raise RuntimeError("invalid situation") service = HouseholdCalculationService( @@ -245,11 +253,12 @@ class Country: "entities": {"person": {"plural": "people", "roles": {}}}, } - def calculate(self, household, policy): + def calculate(self, household, policy, **_kwargs): assert primary.active_scopes == 0 return CalculationResult( household={"people": {"you": {"net_income": {"2026": 42}}}}, warnings=("net_income could not be calculated",), + **household_result_fields(years=["2026"]), ) service = HouseholdCalculationService( @@ -326,6 +335,7 @@ def test_calculation_uses_local_cache_without_recomputing(orm_session_factory): CachedHouseholdCalculation( household=calculated, warnings=("net_income could not be calculated",), + **household_result_fields(years=["2026"]), ), ) country = SimpleNamespace( @@ -363,8 +373,9 @@ def test_failed_cache_write_does_not_invalidate_successful_calculation( "variables": {}, "entities": {"person": {"plural": "people", "roles": {}}}, }, - calculate=lambda *_: SimpleNamespace( + calculate=lambda *_, **__: SimpleNamespace( household={"people": {"you": {}}}, + **household_result_fields(years=["2026"]), ), ) diff --git a/tests/unit/services/test_report_output_service.py b/tests/unit/services/test_report_output_service.py index f4e6d67ea..92344b68e 100644 --- a/tests/unit/services/test_report_output_service.py +++ b/tests/unit/services/test_report_output_service.py @@ -1,6 +1,7 @@ import pytest from sqlalchemy import func, select +from policyengine_api import constants from policyengine_api.constants import get_report_output_cache_version from policyengine_api.data.v1_models import ReportOutput, ReportOutputRun from policyengine_api.services.report_output_service import ReportOutputService @@ -9,6 +10,42 @@ run_service = ReportRunService() +SPM_CONFIG = { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} +SPM_RECEIPT = { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "canonical-forecast", + "forecast_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": ["2025"], + "runtime_versions": { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0", + }, +} + + +def compact_economy_output() -> dict: + return { + "spm_provenance": { + "schema_version": "canonical-spm-comparison-v2", + "baseline": {"receipt": SPM_RECEIPT, "execution_count": 2}, + "reform": {"receipt": SPM_RECEIPT, "execution_count": 2}, + }, + "budgetary_impact": 42, + } @pytest.fixture @@ -19,15 +56,17 @@ def service(orm_session_factory): def create_simulation( orm_session_factory, *, + country_id="us", policy_id=1, population_id="household-1", + population_type="household", ): return ( SimulationService(orm_session_factory) .get_or_create_simulation( - country_id="us", + country_id=country_id, population_id=population_id, - population_type="household", + population_type=population_type, policy_id=policy_id, ) .simulation @@ -182,6 +221,201 @@ def test_update_accepts_existing_v1_json_string_boundary( assert view.report_output.output == {"ok": True} +def test_update_complete_persists_only_compact_society_wide_spm_output( + service, + orm_session_factory, +): + baseline = create_simulation( + orm_session_factory, + policy_id=1, + population_id="us", + population_type="geography", + ) + reform = create_simulation( + orm_session_factory, + policy_id=2, + population_id="us", + population_type="geography", + ) + report = service.create_or_reuse_report_output( + "us", baseline.id, reform.id, "2025" + ).view.report_output + + view = service.update_report_output( + "us", + report.id, + status="complete", + output=compact_economy_output(), + ) + + assert view.report_output.output == compact_economy_output() + assert view.display_run.output == compact_economy_output() + + +@pytest.mark.parametrize( + ("output", "error"), + [ + ({"budgetary_impact": 42}, "requires SPM provenance"), + ( + {"budgetary_impact": 42, "spm_config": SPM_CONFIG}, + "legacy spm_config", + ), + ( + {**compact_economy_output(), "spm_config": SPM_CONFIG}, + "legacy spm_config", + ), + ], + ids=["absent", "config-only", "dual-field"], +) +def test_update_complete_requires_only_compact_society_wide_spm_provenance( + service, + orm_session_factory, + output, + error, +): + baseline = create_simulation( + orm_session_factory, + policy_id=1, + population_id="us", + population_type="geography", + ) + reform = create_simulation( + orm_session_factory, + policy_id=2, + population_id="us", + population_type="geography", + ) + report = service.create_or_reuse_report_output( + "us", baseline.id, reform.id, "2025" + ).view.report_output + + with pytest.raises(ValueError, match=error): + service.update_report_output("us", report.id, status="complete", output=output) + + with orm_session_factory() as session: + stored = session.get(ReportOutput, report.id) + assert stored.status == "pending" + assert stored.output is None + + +@pytest.mark.parametrize( + ("country_id", "population_type"), + [("us", "household"), ("uk", "geography")], + ids=["us-household", "non-us-economy"], +) +def test_update_complete_preserves_optional_spm_for_other_report_kinds( + service, + orm_session_factory, + country_id, + population_type, +): + simulation = create_simulation( + orm_session_factory, + country_id=country_id, + population_id="household-1" if population_type == "household" else country_id, + population_type=population_type, + ) + report = service.create_or_reuse_report_output( + country_id, simulation.id, year="2025" + ).view.report_output + + view = service.update_report_output( + country_id, + report.id, + status="complete", + output={"ok": True}, + ) + + assert view.report_output.output == {"ok": True} + + +def test_update_complete_rejects_legacy_society_wide_spm_output( + service, + orm_session_factory, +): + baseline = create_simulation( + orm_session_factory, + policy_id=1, + population_id="us", + population_type="geography", + ) + reform = create_simulation( + orm_session_factory, + policy_id=2, + population_id="us", + population_type="geography", + ) + report = service.create_or_reuse_report_output( + "us", baseline.id, reform.id, "2025" + ).view.report_output + legacy = compact_economy_output() + legacy["spm_provenance"] = { + "baseline": [SPM_RECEIPT], + "reform": [SPM_RECEIPT], + } + + with pytest.raises(ValueError, match="SPM"): + service.update_report_output("us", report.id, status="complete", output=legacy) + + with orm_session_factory() as session: + stored = session.get(ReportOutput, report.id) + assert stored.status == "pending" + assert stored.output is None + run = session.get(ReportOutputRun, stored.active_run_id) + assert run.status == "pending" + assert run.output is None + + +def test_resolve_legacy_spm_report_returns_new_empty_cache_record( + service, + orm_session_factory, + monkeypatch, +): + baseline = create_simulation( + orm_session_factory, + policy_id=1, + population_id="us", + population_type="geography", + ) + reform = create_simulation( + orm_session_factory, + policy_id=2, + population_id="us", + population_type="geography", + ) + monkeypatch.setitem(constants.RUNTIME_CACHE_SCHEMA_VERSIONS, "report_output", 2) + legacy_cache_version = get_report_output_cache_version("us") + monkeypatch.setitem(constants.RUNTIME_CACHE_SCHEMA_VERSIONS, "report_output", 3) + assert legacy_cache_version != get_report_output_cache_version("us") + with orm_session_factory.begin() as session: + legacy = ReportOutput( + country_id="us", + simulation_1_id=baseline.id, + simulation_2_id=reform.id, + api_version=legacy_cache_version, + status="complete", + output={ + "spm_config": SPM_CONFIG, + "spm_provenance": { + "baseline": [SPM_RECEIPT], + "reform": [SPM_RECEIPT], + }, + }, + year="2025", + ) + session.add(legacy) + session.flush() + legacy_id = legacy.id + + view = service.resolve_report_output("us", legacy_id) + + assert view.response_id == legacy_id + assert view.report_output.id != legacy_id + assert view.report_output.api_version == get_report_output_cache_version("us") + assert view.report_output.status == "pending" + assert view.report_output.output is None + + def test_update_running_requires_mutable_run(service, orm_session_factory): simulation = create_simulation(orm_session_factory) report = create_report(service, simulation.id) diff --git a/tests/unit/services/test_simulation_service.py b/tests/unit/services/test_simulation_service.py index bc5d4ef2e..464ae50b9 100644 --- a/tests/unit/services/test_simulation_service.py +++ b/tests/unit/services/test_simulation_service.py @@ -162,6 +162,95 @@ def test_update_simulation_accepts_legacy_json_text_at_wire_boundary(service): assert updated.output == {"result": 42} +def test_update_simulation_rejects_legacy_spm_config(service): + simulation = service.get_or_create_simulation( + "us", "household-1", "household", 1 + ).simulation + + with pytest.raises(ValueError, match="legacy spm_config"): + service.update_simulation( + "us", + simulation.id, + status="complete", + output={"spm_config": {"geography_kind": "national"}}, + ) + + +def test_update_simulation_rejects_incomplete_spm_provenance(service): + simulation = service.get_or_create_simulation( + "us", "household-1", "household", 1 + ).simulation + + with pytest.raises(ValueError, match="Field required"): + service.update_simulation( + "us", + simulation.id, + status="complete", + output={"spm_provenance": {"geography_kind": "national"}}, + ) + + +def test_update_simulation_persists_only_compact_spm_provenance(service): + simulation = service.get_or_create_simulation( + "us", "household-1", "household", 1 + ).simulation + provenance = { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "spm-rolling-2026-09-09", + "forecast_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": ["2026"], + "runtime_versions": { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0", + }, + } + + updated = service.update_simulation( + "us", + simulation.id, + status="complete", + output={"result": 42, "spm_provenance": provenance}, + ) + + assert updated.output == { + "result": 42, + "spm_provenance": provenance, + } + + +def test_get_or_create_marks_previous_spm_output_for_recalculation( + service, + orm_session_factory, +): + simulation = service.get_or_create_simulation( + "us", "household-1", "household", 1 + ).simulation + with orm_session_factory.begin() as session: + stored = session.get(Simulation, simulation.id) + stored.status = "complete" + stored.output = { + "result": 42, + "spm_config": {"geography_kind": "national"}, + "spm_provenance": { + "years": {"2026": {"large": "diagnostics"}}, + }, + } + + refreshed = service.get_or_create_simulation( + "us", "household-1", "household", 1 + ).simulation + + assert refreshed.status == "pending" + assert refreshed.output is None + + def test_update_simulation_without_values_is_a_noop(service): simulation = service.get_or_create_simulation( "us", "household-1", "household", 1 diff --git a/tests/unit/services/test_spm_household_identity.py b/tests/unit/services/test_spm_household_identity.py index 2ed2572f1..661d3ec03 100644 --- a/tests/unit/services/test_spm_household_identity.py +++ b/tests/unit/services/test_spm_household_identity.py @@ -158,7 +158,8 @@ def test_replacement_household_preserves_historical_zero_padded_reference( ) assert replay.created is False assert replay.simulation.population_id == str(household.id).zfill(5) - assert replay.simulation.output == {"spm_config": NATIONAL} + assert replay.simulation.status == "pending" + assert replay.simulation.output is None saved = service.get_household("us", household.id) assert saved.household_json == original diff --git a/tests/unit/services/test_worker_spm.py b/tests/unit/services/test_worker_spm.py index 1bbcc17c7..b09a614d2 100644 --- a/tests/unit/services/test_worker_spm.py +++ b/tests/unit/services/test_worker_spm.py @@ -9,7 +9,7 @@ from policyengine_api.routes.economy_routes import economy_bp from policyengine_api.services.economy_service import EconomyService from policyengine_api.spm import SPMValidationError -from policyengine_api.worker_spm import validate_worker_spm +from policyengine_api.worker_spm import validate_worker_result, validate_worker_spm @pytest.mark.parametrize("selection", [None, {"geography_kind": "national"}]) @@ -43,9 +43,10 @@ def test_actual_settings_validator_rejects_explicit_settings_on_legacy_worker( "packages": {"policyengine-us": {"version": "1.764.6"}}, }, ): - assert validate_worker_spm(country_id) is None - with pytest.raises(SPMValidationError) as error: - validate_worker_spm(country_id, {"geography_kind": "national"}) + with patch("policyengine_api.spm.simulation_supports_spm", return_value=False): + assert validate_worker_spm(country_id) is None + with pytest.raises(SPMValidationError) as error: + validate_worker_spm(country_id, {"geography_kind": "national"}) assert error.value.code == "SPM_SETTINGS_UNSUPPORTED" @@ -174,6 +175,37 @@ def test_canonical_economy_errors_use_existing_validation_envelope(path): "as_of": None, } CAPABILITY = {"contract_version": "canonical-spm-v1", "defaults": SELECTION} +RUNTIME_VERSIONS = { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0", +} + + +def worker_receipt(*, years=("2026",)): + return { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "test-only", + "forecast_sha256": SELECTION["forecast_content_sha256"], + "scenario": SELECTION["scenario"], + "geography_kind": SELECTION["geography_kind"], + "geography_id": SELECTION["geography_id"], + "county_vintage": SELECTION["county_vintage"], + "as_of": SELECTION["as_of"], + "years": list(years), + "runtime_versions": RUNTIME_VERSIONS, + } + + +def worker_comparison(*, baseline=None, reform=None, execution_count=1): + baseline = baseline or worker_receipt() + reform = reform or worker_receipt() + return { + "schema_version": "canonical-spm-comparison-v2", + "baseline": {"receipt": baseline, "execution_count": execution_count}, + "reform": {"receipt": reform, "execution_count": execution_count}, + } def test_certified_worker_selection_enters_identity_and_submission(): @@ -200,6 +232,15 @@ def test_certified_worker_selection_enters_identity_and_submission(): ) != service._build_options_hash( options={"spm": changed}, model_version="test", dataset="default" ) + assert ( + "spm_provenance_schema=canonical-spm-comparison-v2" + in service._build_options_hash( + options={"spm": SELECTION}, model_version="test", dataset="default" + ) + ) + assert "spm_provenance_schema" not in service._build_options_hash( + options={}, model_version="test", dataset="default" + ) options = service._setup_sim_options("us", {}, {}, "us", "2026", spm=SELECTION) assert options.model_dump(mode="json")["spm"] == SELECTION @@ -403,31 +444,10 @@ def test_worker_cache_receipts_are_required_and_json_roundtrip(): import json from policyengine_api.worker_spm import validate_worker_result - receipt = dict( - forecast_id="test-only", - forecast_sha256=SELECTION["forecast_content_sha256"], - scenario="ce_trend", - geography_kind="national", - runtime_versions={}, - years={"2026": {}}, - geographies=[], - composition_method="classified-inputs", - storage_method="formula", - ) output = { - "spm_config": SELECTION, - "spm_provenance": {"baseline": [receipt], "reform": [receipt]}, + "spm_provenance": worker_comparison(), } validate_worker_result(json.loads(json.dumps(output)), SELECTION) - validate_worker_result( - { - **output, - "spm_config": { - key: value for key, value in SELECTION.items() if value is not None - }, - }, - SELECTION, - ) validate_worker_result( { "kind": "budgetWindow", @@ -439,7 +459,19 @@ def test_worker_cache_receipts_are_required_and_json_roundtrip(): for incomplete in ( {}, {"spm_config": SELECTION}, - {**output, "spm_config": {**SELECTION, "scenario": "zero_real"}}, + { + **output, + "spm_config": SELECTION, + }, + { + "spm_provenance": worker_comparison( + baseline={ + key: value + for key, value in worker_receipt().items() + if key != "scenario" + } + ) + }, ): with pytest.raises(SPMValidationError): validate_worker_result(incomplete, SELECTION) @@ -452,37 +484,55 @@ def test_worker_cache_receipts_are_required_and_json_roundtrip(): def test_worker_receipt_cannot_inherit_nonnull_settings(missing): from policyengine_api.worker_spm import validate_worker_result - config = {key: value for key, value in SELECTION.items() if key != missing} - with pytest.raises(SPMValidationError, match="incomplete resolved"): - validate_worker_result({"spm_config": config}, SELECTION) + receipt_field = ( + "forecast_sha256" if missing == "forecast_content_sha256" else missing + ) + receipt = { + key: value for key, value in worker_receipt().items() if key != receipt_field + } + with pytest.raises(SPMValidationError): + validate_worker_result( + {"spm_provenance": worker_comparison(baseline=receipt)}, SELECTION + ) -@pytest.mark.parametrize("side", ["baseline", "reform"]) -def test_worker_receipt_must_cover_requested_year(side): +def test_worker_receipt_must_cover_requested_year(): from policyengine_api.worker_spm import validate_worker_result - receipt = { - "forecast_id": "test-only", - "forecast_sha256": SELECTION["forecast_content_sha256"], - "scenario": "ce_trend", - "geography_kind": "national", - "runtime_versions": {}, - "years": {"2026": {}}, - "geographies": [], - "composition_method": "classified", - "storage_method": "formula", + result = { + "spm_provenance": worker_comparison(), } + validate_worker_result(result, SELECTION, expected_year="2026") + result["spm_provenance"] = worker_comparison( + baseline=worker_receipt(years=("2025",)), + reform=worker_receipt(years=("2025",)), + ) + with pytest.raises(SPMValidationError, match="requested year"): + validate_worker_result(result, SELECTION, expected_year="2026") + + +def test_worker_receipt_rejects_baseline_reform_mismatch(): + result = { + "spm_provenance": worker_comparison( + reform=worker_receipt(years=("2025",)), + ), + } + + with pytest.raises(SPMValidationError, match="baseline and reform"): + validate_worker_result(result, SELECTION, expected_year="2026") + + +def test_worker_receipt_rejects_removed_segment_list_shape(): result = { - "spm_config": SELECTION, "spm_provenance": { - "baseline": [dict(receipt)], - "reform": [dict(receipt)], + "baseline": [worker_receipt()], + "reform": [worker_receipt()], }, } - validate_worker_result(result, SELECTION, expected_year="2026") - result["spm_provenance"][side][0]["years"] = {"2025": {}} - with pytest.raises(SPMValidationError, match="requested year"): + + with pytest.raises(SPMValidationError) as caught: validate_worker_result(result, SELECTION, expected_year="2026") + assert caught.value.code == "SPM_CONFIGURATION_UNAVAILABLE" @pytest.mark.parametrize( @@ -565,21 +615,9 @@ def test_budget_result_cannot_nest_another_budget_window(): def annual_shape_result(year="2026"): - receipt = { - "forecast_id": "test-only", - "forecast_sha256": SELECTION["forecast_content_sha256"], - "scenario": SELECTION["scenario"], - "geography_kind": SELECTION["geography_kind"], - "runtime_versions": {}, - "years": {"2026": {}}, - "geographies": [], - "composition_method": "classified", - "storage_method": "formula", - } return { "year": year, - "spm_config": SELECTION, - "spm_provenance": {"baseline": [receipt], "reform": [receipt]}, + "spm_provenance": worker_comparison(), } @@ -613,17 +651,7 @@ def test_worker_valid_annual_and_window_shapes_cover_requested_year(year): PYDANTIC_INTERNALS = ("errors.pydantic.dev", "input_value", "validation error for") -WORKER_RECEIPT = { - "forecast_id": "test-artifact", - "forecast_sha256": SELECTION["forecast_content_sha256"], - "scenario": SELECTION["scenario"], - "geography_kind": SELECTION["geography_kind"], - "runtime_versions": {"policyengine-us": "test"}, - "years": {"2026": {}}, - "geographies": [], - "composition_method": "test composition", - "storage_method": "test storage", -} +WORKER_RECEIPT = worker_receipt() @pytest.mark.parametrize( @@ -667,11 +695,7 @@ def test_a_worker_receipt_failure_quotes_no_validator_internals(receipt): with pytest.raises(SPMValidationError) as caught: validate_worker_result( { - "spm_config": SELECTION, - "spm_provenance": { - "baseline": [WORKER_RECEIPT], - "reform": [receipt], - }, + "spm_provenance": worker_comparison(reform=receipt), }, SELECTION, expected_year="2026", diff --git a/tests/unit/test_bundle_update_pins.py b/tests/unit/test_bundle_update_pins.py index c13abb99b..2ccfb19e6 100644 --- a/tests/unit/test_bundle_update_pins.py +++ b/tests/unit/test_bundle_update_pins.py @@ -9,6 +9,8 @@ REPO = Path(__file__).resolve().parents[2] SCRIPT = REPO / ".github/scripts/update-policyengine-package.sh" +SOURCE_VERSION = "4.0.0" +TARGET_VERSION = "4.1.0" @pytest.fixture @@ -16,12 +18,11 @@ def update_checkout(tmp_path): (tmp_path / "docker").mkdir() (tmp_path / "changelog.d").mkdir() (tmp_path / "pyproject.toml").write_text( - '[project]\ndependencies = ["policyengine[models]==5.2.0", ' - '"spm-calculator==0.3.1"]\n' + f'[project]\ndependencies = ["policyengine[models]=={SOURCE_VERSION}"]\n' ) (tmp_path / "docker/Dockerfile").write_text( - 'FROM python:3.12\nRUN pip install "policyengine[models]==5.2.0" ' - "spm-calculator==0.3.1 ipython\n" + "FROM python:3.12\n" + f'RUN pip install "policyengine[models]=={SOURCE_VERSION}" ipython\n' ) (tmp_path / "uv.lock").write_text("# registry resolution is stubbed\n") binaries = tmp_path / "bin" @@ -38,8 +39,23 @@ def update_checkout(tmp_path): stream.write(json.dumps([name, *args]) + "\\n") if name == "git" and args[:1] == ["ls-remote"]: sys.exit(2) +if name == "gh" and args[:2] == ["pr", "view"]: + if os.environ.get("BUNDLE_TEST_OPEN_PR"): + print(os.environ["BUNDLE_TEST_OPEN_PR"]) + sys.exit(0) + sys.exit(1) +if name == "uv" and args[:2] == ["lock", "--upgrade-package"]: + if os.environ.get("BUNDLE_TEST_LOCK_FAIL"): + sys.exit(1) if name == "uv" and args[:1] == ["run"]: - print("POLICYENGINE_VERSION=5.3.0") + reported = os.environ.get("BUNDLE_TEST_REPORTED_VERSION", "4.1.0") + print(f"POLICYENGINE_VERSION={reported}") + print("POLICYENGINE_CORE_VERSION=3.0.0") + print("US_VERSION=2.0.0") + print("UK_VERSION=2.1.0") + print("SPM_CALCULATOR_VERSION=1.0.0") + print("US_DATA_VERSION=us-data-release") + print("UK_DATA_VERSION=uk-data-release") """ for name in ("git", "gh", "uv"): path = binaries / name @@ -48,16 +64,18 @@ def update_checkout(tmp_path): return tmp_path -def run_update(root): +def run_update(root, *arguments, **environment): log = root / "calls.jsonl" result = subprocess.run( - ["bash", str(SCRIPT)], + ["bash", str(SCRIPT), *arguments], cwd=root, env={ **os.environ, "PATH": f"{root / 'bin'}{os.pathsep}{os.environ['PATH']}", - "LATEST_OVERRIDE": "5.3.0", + "LATEST_OVERRIDE": TARGET_VERSION, "BUNDLE_TEST_CALLS": str(log), + "LOCK_RETRY_SECONDS": "0", + **environment, }, capture_output=True, text=True, @@ -72,24 +90,33 @@ def test_automatic_update_changes_and_stages_both_image_pins(update_checkout): assert result.returncode == 0, result.stderr for name in ("pyproject.toml", "docker/Dockerfile"): text = (update_checkout / name).read_text() - assert "policyengine[models]==5.3.0" in text - assert "policyengine[models]==5.2.0" not in text - assert "spm-calculator==0.3.1" in text + assert f"policyengine[models]=={TARGET_VERSION}" in text + assert f"policyengine[models]=={SOURCE_VERSION}" not in text + assert "spm-calculator==" not in text assert [ "git", "add", "pyproject.toml", "docker/Dockerfile", "uv.lock", - "changelog.d/update-policyengine-bundle-5.3.0.changed.md", + f"changelog.d/update-policyengine-bundle-{TARGET_VERSION}.changed.md", ] in calls assert ["uv", "lock", "--upgrade-package", "policyengine"] in calls + assert ["uv", "lock", "--check"] in calls + assert [ + "uv", + "run", + "--frozen", + "python", + ".github/find-api-model-versions.py", + "--shell", + ] in calls -@pytest.mark.parametrize("pin", ["5.1.0", "5.2.01", "5.2.0rc1"]) +@pytest.mark.parametrize("pin", ["3.9.0", "4.0.01", "4.0.0rc1"]) def test_mismatched_image_pin_fails_before_file_changes(update_checkout, pin): image = update_checkout / "docker/Dockerfile" - image.write_text(image.read_text().replace("5.2.0", pin)) + image.write_text(image.read_text().replace(SOURCE_VERSION, pin)) before = { name: (update_checkout / name).read_bytes() for name in ("pyproject.toml", "docker/Dockerfile") @@ -101,3 +128,54 @@ def test_mismatched_image_pin_fails_before_file_changes(update_checkout, pin): assert (update_checkout / name).read_bytes() == data assert not any(call[:1] == ["uv"] for call in calls) assert not any(call[:2] == ["git", "commit"] for call in calls) + + +def test_dry_run_does_not_change_files(update_checkout): + before = { + name: (update_checkout / name).read_bytes() + for name in ("pyproject.toml", "docker/Dockerfile", "uv.lock") + } + result, calls = run_update(update_checkout, "--dry-run") + assert result.returncode == 0, result.stderr + assert "Dry run complete" in result.stdout + for name, data in before.items(): + assert (update_checkout / name).read_bytes() == data + assert not any(call[:1] == ["uv"] for call in calls) + + +def test_lock_failure_stops_before_commit(update_checkout): + result, calls = run_update(update_checkout, BUNDLE_TEST_LOCK_FAIL="1") + assert result.returncode != 0 + assert "uv lock failed after 3 attempts" in result.stderr + assert ( + sum( + call == ["uv", "lock", "--upgrade-package", "policyengine"] + for call in calls + ) + == 3 + ) + assert not any(call[:2] == ["git", "commit"] for call in calls) + + +def test_manifest_version_mismatch_stops_before_commit(update_checkout): + result, calls = run_update(update_checkout, BUNDLE_TEST_REPORTED_VERSION="4.0.9") + assert result.returncode != 0 + assert "does not match requested version" in result.stderr + assert not any(call[:2] == ["git", "commit"] for call in calls) + + +def test_existing_pull_request_is_found_by_exact_branch(update_checkout): + result, calls = run_update(update_checkout, BUNDLE_TEST_OPEN_PR="321") + assert result.returncode == 0, result.stderr + assert "PR #321 already exists" in result.stdout + assert [ + "gh", + "pr", + "view", + f"auto/update-policyengine-bundle-{TARGET_VERSION}", + "--json", + "number,state", + "--jq", + 'select(.state == "OPEN") | .number', + ] in calls + assert not any(call[:1] == ["uv"] for call in calls) diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index 653e4ea98..dd6842606 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -35,7 +35,14 @@ "policyengine-api-prod-github-microdata-token:latest" ), "OPENAI_API_KEY": "policyengine-api-prod-openai-api-key:latest", - "HUGGING_FACE_TOKEN": "policyengine-api-prod-hugging-face-token:latest", + "HUGGING_FACE_TOKEN": "pe-uk-private-hf-read-token:latest", +} +SYNCED_SHARED_CLOUD_RUN_SECRETS = { + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN": ( + "policyengine-api-prod-github-microdata-token" + ), + "OPENAI_API_KEY": "policyengine-api-prod-openai-api-key", + "PE_UK_PRIVATE_HF_READ_TOKEN": "pe-uk-private-hf-read-token", } RAW_CLOUD_RUN_SECRET_VALUES = ( "raw-db-secret-value", @@ -103,7 +110,7 @@ def _required_runtime_env() -> dict[str, str]: "POLICYENGINE_DB_PASSWORD": "raw-db-secret-value", "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN": ("raw-github-secret-value"), "OPENAI_API_KEY": "raw-openai-secret-value", - "HUGGING_FACE_TOKEN": "raw-hf-secret-value", + "PE_UK_PRIVATE_HF_READ_TOKEN": "raw-hf-secret-value", "SIMULATION_ENTRYPOINT_URL": "https://simulation.example.test", "OLD_SIMULATION_GATEWAY_URL": "https://old-gateway.example.test", "SIM_ENTRYPOINT": "cloud_run_simulation_entrypoint", @@ -122,9 +129,13 @@ def _required_runtime_env() -> dict[str, str]: } -def _run_script(path: str, env: dict[str, str]) -> subprocess.CompletedProcess[str]: +def _run_script( + path: str, + env: dict[str, str], + *args: str, +) -> subprocess.CompletedProcess[str]: return subprocess.run( - ["bash", path], + ["bash", path, *args], cwd=REPO, env=env, text=True, @@ -164,6 +175,12 @@ def _fake_gcloud(tmp_path: Path) -> tuple[Path, Path]: ), }, "candidate_cloud_sql": PRODUCTION_CLOUD_SQL_INSTANCE, + "candidate_secret_env": { + "HUGGING_FACE_TOKEN": { + "name": "pe-uk-private-hf-read-token", + "key": "latest", + } + }, "updates": [], } ), @@ -221,6 +238,15 @@ def _fake_gcloud(tmp_path: Path) -> tuple[Path, Path]: for name, value in state.get( "candidate_env", {} ).items() + ] + + [ + { + "name": name, + "valueFrom": {"secretKeyRef": reference}, + } + for name, reference in state.get( + "candidate_secret_env", {} + ).items() ], } ] @@ -267,6 +293,91 @@ def _fake_gcloud_env(gcloud_path: Path, state_path: Path) -> dict[str, str]: ) +def _fake_secret_sync_gcloud(tmp_path: Path) -> tuple[Path, Path]: + state_path = tmp_path / "secret-sync-state.json" + state_path.write_text( + json.dumps( + { + "services": { + STAGING_CLOUD_RUN_SERVICE: ( + "policyengine-api-cr-staging@policyengine-api.iam." + "gserviceaccount.com" + ), + PRODUCTION_CLOUD_RUN_SERVICE: ( + "policyengine-api-cr-runtime@policyengine-api.iam." + "gserviceaccount.com" + ), + }, + "secrets": { + "policyengine-api-staging-db-password": "db-value", + "policyengine-api-prod-db-password": "db-value", + "policyengine-api-prod-openai-api-key": "old-openai-value", + "pe-uk-private-hf-read-token": "hf-value", + }, + "iam": {}, + "version_adds": [], + } + ), + encoding="utf-8", + ) + gcloud_path = tmp_path / "gcloud-secret-sync" + gcloud_path.write_text( + """#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path + +state_path = Path(os.environ["FAKE_SECRET_SYNC_STATE"]) +state = json.loads(state_path.read_text(encoding="utf-8")) +args = sys.argv[1:] + +if args[:3] == ["run", "services", "describe"]: + print(state["services"][args[3]]) +elif args[:2] == ["secrets", "describe"]: + raise SystemExit(0 if args[2] in state["secrets"] else 1) +elif args[:2] == ["secrets", "create"]: + state["secrets"][args[2]] = "" +elif args[:3] == ["secrets", "versions", "access"]: + secret_name = args[args.index("--secret") + 1] + if secret_name not in state["secrets"]: + raise SystemExit(1) + print(state["secrets"][secret_name], end="") +elif args[:3] == ["secrets", "versions", "add"]: + secret_name = args[3] + state["secrets"][secret_name] = sys.stdin.read() + state["version_adds"].append(secret_name) +elif args[:2] == ["secrets", "add-iam-policy-binding"]: + secret_name = args[2] + member = args[args.index("--member") + 1] + members = state["iam"].setdefault(secret_name, []) + if member not in members: + members.append(member) +elif args[:2] == ["secrets", "get-iam-policy"]: + secret_name = args[2] + print( + json.dumps( + { + "bindings": [ + { + "role": "roles/secretmanager.secretAccessor", + "members": state["iam"].get(secret_name, []), + } + ] + } + ) + ) +else: + raise SystemExit(f"unexpected gcloud arguments: {args}") + +state_path.write_text(json.dumps(state), encoding="utf-8") +""", + encoding="utf-8", + ) + gcloud_path.chmod(0o755) + return gcloud_path, state_path + + def _run_simulation_version_guard( versions_response: dict, *args: str, @@ -493,6 +604,7 @@ def test_policyengine_bundle_support_check_passes_pyproject_pin_to_guard(tmp_pat assert capture_path.read_text(encoding="utf-8").splitlines() == [ "-py", current_version, + "--check-installed-spm", ] @@ -541,24 +653,24 @@ def test_cloud_run_dockerfile_installs_the_frozen_uv_environment(): assert "!uv.lock" in dockerignore -def test_active_images_pin_spm_calculator_for_the_legacy_country_bundle(): - """Both deployed images must protect the current bundle's SPM behavior.""" +def test_active_images_install_the_same_policyengine_models_bundle(): + """Both deployed images use the wrapper's exact certified package set.""" import tomllib project = tomllib.loads((REPO / "pyproject.toml").read_text()) - assert "spm-calculator==0.3.1" in project["project"]["dependencies"] - # The independently published GHCR image does not install this project. + dependencies = project["project"]["dependencies"] generic_image = (REPO / "docker/Dockerfile").read_text() - assert "spm-calculator==0.3.1" in generic_image bundle_pin = next( requirement - for requirement in project["project"]["dependencies"] + for requirement in dependencies if requirement.startswith("policyengine[models]==") ) assert bundle_pin in generic_image - assert "pip install policyengine-core policyengine-uk policyengine-us" not in ( - generic_image - ) + assert not any(item.startswith("spm-calculator") for item in dependencies) + assert "spm-calculator==" not in generic_image + for package in ("policyengine-core", "policyengine-us", "policyengine-uk"): + assert not any(item.startswith(package) for item in dependencies) + assert f"{package}==" not in generic_image def test_deployed_startup_execs_only_the_api_server(): @@ -1220,6 +1332,24 @@ def test_resolve_cloud_run_candidate_records_exact_ready_revision_and_image(tmp_ ] +def test_resolve_cloud_run_candidate_rejects_wrong_hugging_face_secret(tmp_path): + gcloud_path, state_path = _fake_gcloud(tmp_path) + state = json.loads(state_path.read_text(encoding="utf-8")) + state["candidate_secret_env"]["HUGGING_FACE_TOKEN"]["name"] = ( + "policyengine-api-prod-hugging-face-token" + ) + state_path.write_text(json.dumps(state), encoding="utf-8") + + result = _run_script( + ".github/scripts/resolve_cloud_run_candidate_state.sh", + _fake_gcloud_env(gcloud_path, state_path), + ) + + assert result.returncode == 2 + assert "binds HUGGING_FACE_TOKEN" in result.stderr + assert "expected pe-uk-private-hf-read-token:latest" in result.stderr + + def test_resolve_cloud_run_candidate_rejects_changed_revision(tmp_path): gcloud_path, state_path = _fake_gcloud(tmp_path) @@ -1880,6 +2010,20 @@ def test_workflows_scope_simulation_routing_config_to_github_environments(): assert secret_env in job +def test_pr_build_and_test_jobs_require_simulation_bundle_compatibility(): + workflow = _pr_workflow() + compatibility_job = "ensure-policyengine-bundle-supported-by-simulation-api" + + for job_name in ( + "test_container_builds", + "test_cloud_run_container_builds", + "test_env_vars", + "test", + ): + job = _workflow_job_block(workflow, job_name) + assert compatibility_job in job + + def test_cloud_run_candidate_jobs_use_environment_scoped_migration_selectors(): workflow = _push_workflow() selectors = ( @@ -1979,6 +2123,16 @@ def test_push_workflow_does_not_pass_raw_secrets_to_cloud_run_deploy_commands(): staging_deploy_start, ) staging_deploy = cloud_run_staging[staging_deploy_start:staging_deploy_end] + production_deploy_start = cloud_run_production.index( + "- name: Deploy tagged Cloud Run candidate" + ) + production_deploy_end = cloud_run_production.index( + "- name: Resolve exact Cloud Run production candidate", + production_deploy_start, + ) + production_deploy = cloud_run_production[ + production_deploy_start:production_deploy_end + ] raw_secret_envs = ( "POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}", ( @@ -1986,15 +2140,15 @@ def test_push_workflow_does_not_pass_raw_secrets_to_cloud_run_deploy_commands(): "${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}" ), "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}", - "HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}", + "HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}", ) for raw_secret_env in raw_secret_envs: assert raw_secret_env not in staging_deploy - assert raw_secret_env not in cloud_run_production + assert raw_secret_env not in production_deploy -def test_push_workflow_release_test_step_is_the_only_raw_secret_consumer(): +def test_push_workflow_syncs_environment_specific_secrets_before_deployment(): workflow = _push_workflow() cloud_run_staging = _workflow_job_block(workflow, "deploy-cloud-run-staging") cloud_run_production = _workflow_job_block(workflow, "deploy-cloud-run-candidate") @@ -2004,20 +2158,58 @@ def test_push_workflow_release_test_step_is_the_only_raw_secret_consumer(): test_step_start, ) release_test_step = cloud_run_staging[test_step_start:test_step_end] + staging_sync_start = cloud_run_staging.index( + "- name: Synchronize shared Cloud Run runtime secrets" + ) + staging_sync_end = cloud_run_staging.index( + "- name: Capture current Cloud Run staging state", + staging_sync_start, + ) + staging_sync_step = cloud_run_staging[staging_sync_start:staging_sync_end] + production_sync_start = cloud_run_production.index( + "- name: Synchronize complete Cloud Run runtime secret batch" + ) + production_sync_end = cloud_run_production.index( + "- name: Capture current Cloud Run production state", + production_sync_start, + ) + production_sync_step = cloud_run_production[ + production_sync_start:production_sync_end + ] - raw_secret_envs = ( + release_test_secret_envs = ( "POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}", ( "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: " "${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}" ), "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}", - "HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}", + "HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}", ) - for raw_secret_env in raw_secret_envs: + shared_sync_secret_envs = ( + ( + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: " + "${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}" + ), + "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}", + ("PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}"), + ) + for raw_secret_env in release_test_secret_envs: assert release_test_step.count(raw_secret_env) == 1 - assert cloud_run_staging.count(raw_secret_env) == 1 - assert raw_secret_env not in cloud_run_production + for raw_secret_env in shared_sync_secret_envs: + assert staging_sync_step.count(raw_secret_env) == 1 + assert production_sync_step.count(raw_secret_env) == 1 + database_secret_env = ( + "POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}" + ) + assert database_secret_env not in staging_sync_step + assert production_sync_step.count(database_secret_env) == 1 + assert "run: bash .github/scripts/sync_cloud_run_secrets.sh\n" in staging_sync_step + assert "--include-database-password" not in staging_sync_step + assert ( + "run: bash .github/scripts/sync_cloud_run_secrets.sh " + "--include-database-password" in production_sync_step + ) def test_sync_cloud_run_secrets_workflow_is_manual_and_environment_gated(): @@ -2026,7 +2218,10 @@ def test_sync_cloud_run_secrets_workflow_is_manual_and_environment_gated(): assert "workflow_dispatch:" in workflow assert "pull_request:" not in workflow assert "push:" not in workflow - assert "environment: production" in workflow + assert "deployment_environment:" in workflow + assert "- staging" in workflow + assert "- production" in workflow + assert "environment: ${{ inputs.deployment_environment }}" in workflow assert "id-token: write" in workflow assert "github.ref != 'refs/heads/master'" in workflow assert "google-github-actions/auth@v2" in workflow @@ -2044,12 +2239,129 @@ def test_sync_cloud_run_secrets_workflow_writes_expected_secret_versions(): assert "run: bash .github/scripts/sync_cloud_run_secrets.sh" in workflow assert "set +x" in script assert "--data-file=-" in script - assert "gcloud secrets add-iam-policy-binding" in script + assert "Secret Manager already matches" in script + assert "secrets versions access latest" in script + assert "secrets add-iam-policy-binding" in script + assert "secrets get-iam-policy" in script + assert "run services describe" in script assert "roles/secretmanager.secretAccessor" in script - for env_name, secret_ref in CLOUD_RUN_SECRET_MAPPINGS.items(): - secret_name = secret_ref.removesuffix(":latest") + assert ( + "CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: " + "${{ vars.CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }}" in workflow + ) + assert 'sync_secret POLICYENGINE_DB_PASSWORD "${db_password_secret_name}"' in script + for env_name, secret_name in SYNCED_SHARED_CLOUD_RUN_SECRETS.items(): assert f"{env_name}: ${{{{ secrets.{env_name} }}}}" in workflow assert f"sync_secret {env_name} {secret_name}" in script + assert "verify_secret_value" in script + assert "grant_secret_access" in script + + +def test_sync_cloud_run_secrets_is_idempotent_and_grants_complete_batch(tmp_path): + gcloud_path, state_path = _fake_secret_sync_gcloud(tmp_path) + secret_values = { + "POLICYENGINE_DB_PASSWORD": "db-value", + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN": "microdata-value", + "OPENAI_API_KEY": "openai-value", + "PE_UK_PRIVATE_HF_READ_TOKEN": "hf-value", + } + env = _script_env( + GCLOUD_BIN=str(gcloud_path), + FAKE_SECRET_SYNC_STATE=str(state_path), + CLOUD_RUN_PROJECT="policyengine-api", + CLOUD_RUN_REGION="us-central1", + CLOUD_RUN_SERVICE=PRODUCTION_CLOUD_RUN_SERVICE, + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET=( + "policyengine-api-prod-db-password:latest" + ), + **secret_values, + ) + + first_result = _run_script( + ".github/scripts/sync_cloud_run_secrets.sh", + env, + "--include-database-password", + ) + assert first_result.returncode == 0, first_result.stderr + first_state = json.loads(state_path.read_text(encoding="utf-8")) + assert first_state["version_adds"] == [ + "policyengine-api-prod-github-microdata-token", + "policyengine-api-prod-openai-api-key", + ] + assert first_state["secrets"] == { + "policyengine-api-staging-db-password": "db-value", + "policyengine-api-prod-db-password": "db-value", + "policyengine-api-prod-github-microdata-token": "microdata-value", + "policyengine-api-prod-openai-api-key": "openai-value", + "pe-uk-private-hf-read-token": "hf-value", + } + expected_member = ( + "serviceAccount:policyengine-api-cr-runtime@policyengine-api.iam." + "gserviceaccount.com" + ) + synchronized_secret_names = { + "policyengine-api-prod-db-password", + "policyengine-api-prod-github-microdata-token", + "policyengine-api-prod-openai-api-key", + "pe-uk-private-hf-read-token", + } + assert set(first_state["iam"]) == synchronized_secret_names + assert all(members == [expected_member] for members in first_state["iam"].values()) + assert all(value not in first_result.stdout for value in secret_values.values()) + + second_result = _run_script( + ".github/scripts/sync_cloud_run_secrets.sh", + env, + "--include-database-password", + ) + assert second_result.returncode == 0, second_result.stderr + second_state = json.loads(state_path.read_text(encoding="utf-8")) + assert second_state["version_adds"] == first_state["version_adds"] + + +def test_sync_cloud_run_staging_batch_preserves_distinct_database_secret(tmp_path): + gcloud_path, state_path = _fake_secret_sync_gcloud(tmp_path) + env = _script_env( + GCLOUD_BIN=str(gcloud_path), + FAKE_SECRET_SYNC_STATE=str(state_path), + CLOUD_RUN_PROJECT="policyengine-api", + CLOUD_RUN_REGION="us-central1", + CLOUD_RUN_SERVICE=STAGING_CLOUD_RUN_SERVICE, + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN="microdata-value", + OPENAI_API_KEY="openai-value", + PE_UK_PRIVATE_HF_READ_TOKEN="hf-value", + ) + + result = _run_script( + ".github/scripts/sync_cloud_run_secrets.sh", + env, + ) + assert result.returncode == 0, result.stderr + state = json.loads(state_path.read_text(encoding="utf-8")) + assert state["secrets"]["policyengine-api-staging-db-password"] == "db-value" + assert "policyengine-api-staging-db-password" not in state["iam"] + assert set(state["iam"]) == { + "policyengine-api-prod-github-microdata-token", + "policyengine-api-prod-openai-api-key", + "pe-uk-private-hf-read-token", + } + + +def test_hugging_face_credential_uses_purpose_specific_managed_names(): + workflow_sources = "\n".join( + ( + _pr_workflow(), + _push_workflow(), + _sync_secrets_workflow(), + _sync_secrets_script(), + (REPO / ".github/scripts/cloud_run_env.sh").read_text(encoding="utf-8"), + ) + ) + + assert "secrets.HUGGING_FACE_TOKEN" not in workflow_sources + assert "secrets.PE_UK_PRIVATE_HF_READ_TOKEN" in workflow_sources + assert "policyengine-api-prod-hugging-face-token" not in workflow_sources + assert "pe-uk-private-hf-read-token" in workflow_sources def test_sync_cloud_run_secrets_script_is_shell_syntax_valid(): diff --git a/tests/unit/test_country_spm.py b/tests/unit/test_country_spm.py index a36d9f8a0..937c0e537 100644 --- a/tests/unit/test_country_spm.py +++ b/tests/unit/test_country_spm.py @@ -191,13 +191,25 @@ def __init__(self, *, tax_benefit_system, situation, spm): def test_tax_only_result_reads_provenance_without_calculating_spm(monkeypatch): calculations = [] - selection = spm.SPMSelection(geography_kind="national").model_dump() + selection = spm.SPMSelection( + forecast_content_sha256="a" * 64, + scenario="baseline", + geography_kind="national", + geography_id=None, + county_vintage="2020", + as_of=None, + ).model_dump() receipt = { "forecast_id": "test-artifact", "forecast_sha256": "a" * 64, "scenario": "baseline", "geography_kind": "national", - "runtime_versions": {"policyengine-us": "test"}, + "runtime_versions": { + "policyengine": "test", + "policyengine-core": "test", + "policyengine-us": "test", + "spm-calculator": "test", + }, "years": {}, "geographies": [], "composition_method": "test adult classification", @@ -224,12 +236,22 @@ def calculate(variable, period): ) result = country.calculate(requested_household("spm_unit_federal_tax"), None) assert calculations == ["spm_unit_federal_tax"] - assert result.spm_provenance["years"] == {} + assert result.spm_provenance["years"] == [] assert ( result.household["spm_units"]["spm_unit"]["spm_unit_federal_tax"]["2024"] == 123 ) - assert json.loads(json.dumps(result.spm_config)) == selection - assert json.loads(json.dumps(result.spm_provenance)) == receipt + assert result.spm_provenance == { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "test-artifact", + "forecast_sha256": "a" * 64, + "scenario": "baseline", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": [], + "runtime_versions": receipt["runtime_versions"], + } @pytest.fixture @@ -263,7 +285,7 @@ def test_real_country_state_only_tax_succeeds_and_receipt_stays_empty( result.household["spm_units"]["spm_unit"]["spm_unit_federal_tax"]["2024"] is not None ) - assert result.spm_provenance["years"] == {} + assert result.spm_provenance["years"] == [] json.dumps(result.spm_provenance) @@ -271,8 +293,6 @@ def test_real_country_state_only_tax_succeeds_and_receipt_stays_empty( "variable", [ "spm_unit_spm_threshold", - "spm_unit_capped_housing_subsidy", - "spm_unit_net_income", "spm_unit_is_in_spm_poverty", ], ) @@ -291,8 +311,6 @@ def test_real_country_state_only_spm_dependency_requires_geography( "variable", [ "spm_unit_spm_threshold", - "spm_unit_capped_housing_subsidy", - "spm_unit_net_income", "spm_unit_is_in_spm_poverty", ], ) @@ -307,16 +325,21 @@ def test_real_country_state_only_dependency_is_null_when_nothing_was_chosen( """ result = real_canonical_country.calculate(requested_household(variable), None) assert result.household["spm_units"]["spm_unit"][variable]["2024"] is None - assert result.spm_config["geography_kind"] == "county" + assert result.spm_provenance["geography_kind"] == "county" -@pytest.mark.parametrize("county", ["99999", "malformed"]) -def test_real_country_unknown_county_is_structured(real_canonical_country, county): +@pytest.mark.parametrize( + ("county", "expected_code"), + [("99999", "SPM_GEOGRAPHY_UNAVAILABLE"), ("malformed", "SPM_GEOGRAPHY_REQUIRED")], +) +def test_real_country_unknown_county_is_structured( + real_canonical_country, county, expected_code +): household = requested_household() household["households"]["household"]["county_fips"] = {"2024": county} with pytest.raises(ValueError) as caught: real_canonical_country.calculate(household, None, spm_requested=True) - assert spm.spm_error_detail(caught.value)["code"] == "SPM_GEOGRAPHY_UNAVAILABLE" + assert spm.spm_error_detail(caught.value)["code"] == expected_code def test_real_country_unknown_area_is_structured(real_canonical_country): @@ -352,6 +375,7 @@ def test_real_country_explicit_geography_yields_json_receipt( resolved = spm.normalize_spm_selection("us", None) forecast = spm._selected_forecast(resolved["forecast_content_sha256"]) selection["geography_id"] = forecast.resolve_county(2024, "06037")["area_id"] + resolved_selection = spm.normalize_spm_selection("us", selection) original = deepcopy(household) result = real_canonical_country.calculate( household, None, spm=selection, spm_requested=True @@ -360,10 +384,10 @@ def test_real_country_explicit_geography_yields_json_receipt( assert ( result.household["spm_units"]["spm_unit"]["spm_unit_spm_threshold"]["2024"] > 0 ) - assert result.spm_config["geography_kind"] == geography + assert result.spm_provenance["geography_kind"] == geography assert ( result.spm_provenance["forecast_sha256"] - == result.spm_config["forecast_content_sha256"] + == resolved_selection["forecast_content_sha256"] ) assert "2024" in result.spm_provenance["years"] json.dumps(result.spm_provenance) @@ -422,7 +446,7 @@ def test_real_http_tax_only_succeeds_without_geography(real_http_client): json={"household": requested_household("spm_unit_federal_tax")}, ) assert response.status_code == 200, response.json - assert response.json["spm_provenance"]["years"] == {} + assert response.json["spm_provenance"]["years"] == [] @pytest.mark.parametrize( @@ -472,7 +496,7 @@ def test_real_http_missing_primitives_are_null_when_nothing_was_chosen( response = real_http_client.post("/us/calculate", json={"household": household}) assert response.status_code == 200, response.json assert response.json["status"] == "ok" - assert response.json["spm_config"]["geography_kind"] == "county" + assert response.json["spm_provenance"]["geography_kind"] == "county" if kind != "axes": threshold = response.json["result"]["spm_units"]["spm_unit"][ "spm_unit_spm_threshold" @@ -490,7 +514,7 @@ def test_real_http_national_result_and_receipt_survive_response_cache(real_http_ assert response.status_code == 200, response.json assert cached.status_code == 200 assert cached.json == response.json - assert response.json["spm_config"]["geography_kind"] == "national" + assert response.json["spm_provenance"]["geography_kind"] == "national" assert "2024" in response.json["spm_provenance"]["years"] @@ -511,7 +535,7 @@ def test_real_http_full_national_calculation_keeps_threshold_and_provenance( ] > 0 ) - assert response.json["spm_config"]["geography_kind"] == "national" + assert response.json["spm_provenance"]["geography_kind"] == "national" assert "2024" in response.json["spm_provenance"]["years"] @@ -592,7 +616,7 @@ def test_real_http_tax_only_outside_artifact_years_is_lazy( ] is not None ) - assert response.json["spm_provenance"]["years"] == {} + assert response.json["spm_provenance"]["years"] == [] @pytest.mark.parametrize("geography", ["national", "metro"]) @@ -601,10 +625,13 @@ def test_real_http_tax_only_outside_artifact_years_is_lazy( def test_real_http_unsupported_spm_year_is_structured_only_when_calculated( real_http_client, geography, variable, axes ): + household = household_in_year(variable, 2036, axes=axes) + if variable == "spm_unit_net_income": + household["spm_units"]["spm_unit"]["housing_assistance"] = {"2036": 1_000} response = real_http_client.post( "/us/calculate", json={ - "household": household_in_year(variable, 2036, axes=axes), + "household": household, "spm": selection_for_geography(geography), }, ) diff --git a/tests/unit/test_openapi_simulation.py b/tests/unit/test_openapi_simulation.py index ff4fd6c38..3d9ca1f76 100644 --- a/tests/unit/test_openapi_simulation.py +++ b/tests/unit/test_openapi_simulation.py @@ -147,17 +147,22 @@ def test_simulation_persistence_responses_preserve_spm_output_and_run_fields( "status": "ok", "message": None, "result": {"people": {}}, - "spm_config": {"geography_kind": "national"}, "spm_provenance": { + "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-forecast", "forecast_sha256": "a" * 64, "scenario": "baseline", "geography_kind": "national", - "runtime_versions": {"policyengine-us": "test"}, - "years": {}, - "geographies": [], - "composition_method": "test", - "storage_method": "test", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "runtime_versions": { + "policyengine": "test", + "policyengine-core": "test", + "policyengine-us": "test", + "spm-calculator": "test", + }, + "years": ["2026"], }, } update_payload = { diff --git a/tests/unit/test_openapi_spm.py b/tests/unit/test_openapi_spm.py index 75a6e6d46..a7d709a60 100644 --- a/tests/unit/test_openapi_spm.py +++ b/tests/unit/test_openapi_spm.py @@ -5,7 +5,13 @@ from flask import Flask from policyengine_api.routes.system_routes import system_bp -from policyengine_api.spm import SPMProvenance, SPMSelection +from policyengine_api.spm import ( + SPMComparisonProvenance, + SPMExecutionProvenance, + SPMProvenance, + SPMRuntimeVersions, + SPMSelection, +) from policyengine_api.query_parameters import ( AnnualEconomyQuery, BudgetWindowEconomyQuery, @@ -46,19 +52,25 @@ def test_economy_query_and_receipt_schemas_match_public_http_contract(): ]["result"] if suffix: result = result["properties"]["annualImpacts"]["items"] - assert result["properties"]["spm_config"] == { - "$ref": "#/components/schemas/SPMSelection" - } + assert "spm_config" not in result["properties"] assert result["properties"]["spm_provenance"] == { - "$ref": "#/components/schemas/SPMWorkerProvenance" + "$ref": "#/components/schemas/SPMComparisonProvenance" } assert operation["responses"]["400"]["$ref"].endswith("SPMValidationError") - receipt = spec["components"]["schemas"]["SPMWorkerProvenance"] - assert receipt["required"] == ["baseline", "reform"] - for side in receipt["required"]: - assert receipt["properties"][side]["items"] == { - "$ref": "#/components/schemas/SPMProvenance" + comparison = spec["components"]["schemas"]["SPMComparisonProvenance"] + assert comparison["required"] == ["schema_version", "baseline", "reform"] + assert comparison["properties"]["schema_version"]["enum"] == [ + "canonical-spm-comparison-v2" + ] + for side in ("baseline", "reform"): + assert comparison["properties"][side] == { + "$ref": "#/components/schemas/SPMExecutionProvenance" } + execution = spec["components"]["schemas"]["SPMExecutionProvenance"] + assert execution["required"] == ["receipt", "execution_count"] + assert execution["properties"]["receipt"] == { + "$ref": "#/components/schemas/SPMProvenance" + } def test_metadata_discovery_and_selection_constraints_are_published(): @@ -106,10 +118,17 @@ def test_spm_documentation_uses_public_models_and_actual_route_envelopes(): for name, model in { "SPMSelection": SPMSelection, + "SPMRuntimeVersions": SPMRuntimeVersions, "SPMProvenance": SPMProvenance, + "SPMExecutionProvenance": SPMExecutionProvenance, + "SPMComparisonProvenance": SPMComparisonProvenance, }.items(): schema = schemas[name] - assert set(schema["properties"]) == set(model.model_fields) + expected_fields = { + field.alias or field_name + for field_name, field in model.model_fields.items() + } + assert set(schema["properties"]) == expected_fields assert schema["additionalProperties"] is False assert '"type": "null"' not in json.dumps(schema) @@ -119,12 +138,49 @@ def test_spm_documentation_uses_public_models_and_actual_route_envelopes(): for field in ("forecast_content_sha256", "scenario", "geography_id", "as_of"): assert selection[field]["type"] == "string" assert selection[field]["nullable"] is True - assert schemas["SPMProvenance"]["properties"]["runtime_versions"][ - "additionalProperties" - ] == { - "type": "string", - "nullable": True, + provenance = schemas["SPMProvenance"] + assert provenance["required"] == [ + "schema_version", + "forecast_id", + "forecast_sha256", + "scenario", + "geography_kind", + "geography_id", + "county_vintage", + "as_of", + "years", + "runtime_versions", + ] + assert provenance["properties"]["runtime_versions"] == { + "$ref": "#/components/schemas/SPMRuntimeVersions" + } + assert set(schemas["SPMRuntimeVersions"]["properties"]) == { + "policyengine", + "policyengine-core", + "policyengine-us", + "spm-calculator", + } + assert schemas["SPMRuntimeVersions"]["required"] == [ + "policyengine", + "policyengine-core", + "policyengine-us", + "spm-calculator", + ] + assert all( + "nullable" not in field + for field in schemas["SPMRuntimeVersions"]["properties"].values() + ) + assert "SPMResolvedConfiguration" not in schemas + removed_rich_fields = { + "geographies", + "requested", + "effective", + "execution_count", + "sources", + "diagnostics", + "mappings", } + assert not removed_rich_fields.intersection(provenance["properties"]) for route in ("calculate", "calculate-full"): operation = paths[f"/{{country_id}}/{route}"]["post"] @@ -146,7 +202,7 @@ def test_spm_documentation_uses_public_models_and_actual_route_envelopes(): "properties" ] assert {"status", "message", "result"} <= set(result) - assert result["spm_config"] == selection_ref + assert "spm_config" not in result assert result["spm_provenance"] == provenance_ref assert ( operation["responses"]["400"]["$ref"] diff --git a/tests/unit/test_simulation_and_report_routes.py b/tests/unit/test_simulation_and_report_routes.py index 2d2502f7c..23aa8c76f 100644 --- a/tests/unit/test_simulation_and_report_routes.py +++ b/tests/unit/test_simulation_and_report_routes.py @@ -27,15 +27,48 @@ def create_test_client() -> Flask: return app.test_client() -def create_simulation(factory, *, population_id="household-1", policy_id=1): +def create_simulation( + factory, + *, + population_id="household-1", + policy_id=1, + population_type="household", +): simulation = ( SimulationService(factory) - .get_or_create_simulation("us", population_id, "household", policy_id) + .get_or_create_simulation("us", population_id, population_type, policy_id) .simulation ) return simulation.id +def compact_spm_report_output() -> dict: + receipt = { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "canonical-forecast", + "forecast_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": ["2025"], + "runtime_versions": { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0", + }, + } + return { + "spm_provenance": { + "schema_version": "canonical-spm-comparison-v2", + "baseline": {"receipt": receipt, "execution_count": 1}, + "reform": {"receipt": receipt, "execution_count": 1}, + }, + } + + def create_report(factory, simulation_id): report = ( ReportOutputService(factory) @@ -261,6 +294,45 @@ def test_report_patch_complete_promotes_active_rerun(orm_session_factory): assert report.latest_successful_run_id == rerun_id +def test_report_patch_rejects_legacy_spm_output_before_persistence( + orm_session_factory, +): + baseline_id = create_simulation( + orm_session_factory, + population_id="us", + policy_id=50, + population_type="geography", + ) + reform_id = create_simulation( + orm_session_factory, + population_id="us", + policy_id=51, + population_type="geography", + ) + report_id = ( + ReportOutputService(orm_session_factory) + .create_or_reuse_report_output("us", baseline_id, reform_id, "2025") + .view.report_output.id + ) + legacy = compact_spm_report_output() + receipt = legacy["spm_provenance"]["baseline"]["receipt"] + legacy["spm_provenance"] = { + "baseline": [receipt], + "reform": [receipt], + } + + response = create_test_client().patch( + "/us/report", + json={"id": report_id, "status": "complete", "output": legacy}, + ) + + assert response.status_code == 400 + with orm_session_factory() as session: + report = session.get(ReportOutput, report_id) + assert report.status == "pending" + assert report.output is None + + def test_simulation_v1_routes_keep_json_fields_as_strings(orm_session_factory): simulation_id = create_simulation(orm_session_factory, policy_id=49) output = {"result": "ok", "values": [1, 2, 3]} diff --git a/tests/unit/test_spm_provenance.py b/tests/unit/test_spm_provenance.py new file mode 100644 index 000000000..08521a1cf --- /dev/null +++ b/tests/unit/test_spm_provenance.py @@ -0,0 +1,232 @@ +"""Compact canonical SPM provenance contracts.""" + +import json + +import pytest +from pydantic import ValidationError + +from policyengine_api.spm import ( + SPMComparisonProvenance, + SPMProvenance, + SPMRuntimeVersions, + SPMSelection, + build_spm_calculation_provenance, + build_spm_comparison_provenance, + build_spm_provenance, + resolved_spm_configuration, + validate_spm_calculation_provenance, +) + + +SHA256 = "a" * 64 +SELECTION = SPMSelection.model_validate( + { + "forecast_content_sha256": SHA256, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + } +) +# Synthetic fixture values used only to exercise the public receipt contract. +# Deployed workers obtain real package versions from the PolicyEngine.py bundle. +TEST_RUNTIME_VERSIONS = SPMRuntimeVersions.model_validate( + { + "policyengine": "0.0.0-test-policyengine", + "policyengine-core": "0.0.0-test-policyengine-core", + "policyengine-us": "0.0.0-test-policyengine-us", + "spm-calculator": "0.0.0-test-spm-calculator", + } +) + + +def _receipt(**changes: object) -> SPMProvenance: + values: dict[str, object] = { + "forecast_id": "spm-rolling-2026-09-09", + "forecast_sha256": SHA256, + "selection": SELECTION, + "years": ("2026",), + "runtime_versions": TEST_RUNTIME_VERSIONS, + } + values.update(changes) + return build_spm_provenance(**values) + + +def test_compact_receipt_has_only_the_public_v2_fields() -> None: + receipt = _receipt() + + assert receipt.model_dump(mode="json", by_alias=True) == { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "spm-rolling-2026-09-09", + "forecast_sha256": SHA256, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": ["2026"], + "runtime_versions": { + "policyengine": "0.0.0-test-policyengine", + "policyengine-core": "0.0.0-test-policyengine-core", + "policyengine-us": "0.0.0-test-policyengine-us", + "spm-calculator": "0.0.0-test-spm-calculator", + }, + } + assert len(receipt.model_dump_json(by_alias=True).encode()) < 1_024 + + +@pytest.mark.parametrize( + "changes", + [ + {"forecast_sha256": "A" * 64}, + {"years": ("26",)}, + {"years": ("2027", "2026")}, + {"years": ("2026", "2026")}, + ], +) +def test_compact_receipt_rejects_noncanonical_values( + changes: dict[str, object], +) -> None: + with pytest.raises((ValidationError, ValueError)): + _receipt(**changes) + + +@pytest.mark.parametrize( + "invalid_version", + ["policyengine", "policyengine-core", "policyengine-us", "spm-calculator"], +) +@pytest.mark.parametrize("value", [None, ""]) +def test_runtime_versions_require_four_nonempty_strings( + invalid_version: str, + value: object, +) -> None: + versions = TEST_RUNTIME_VERSIONS.model_dump(mode="json", by_alias=True) + versions[invalid_version] = value + + with pytest.raises(ValidationError): + SPMRuntimeVersions.model_validate(versions) + + +def test_completed_calculation_contract_rejects_legacy_dual_field_shape() -> None: + receipt = _receipt().model_dump(mode="json", by_alias=True) + + with pytest.raises(ValidationError): + validate_spm_calculation_provenance( + { + "spm_config": SELECTION.model_dump(mode="json"), + "spm_provenance": receipt, + } + ) + + +def test_compact_receipt_rejects_the_previous_diagnostic_shape() -> None: + previous = { + "forecast_id": "spm-rolling-2026-09-09", + "forecast_sha256": SHA256, + "scenario": "ce_trend", + "geography_kind": "national", + "runtime_versions": {}, + "years": {"2026": {}}, + "geographies": [], + "composition_method": "classified inputs", + "storage_method": "formula", + } + + with pytest.raises(ValidationError): + SPMProvenance.model_validate(previous) + + +def test_country_receipt_adapter_extracts_only_compact_scalars() -> None: + country_receipt: dict[str, object] = { + "forecast_id": "spm-rolling-2026-09-09", + "forecast_sha256": SHA256, + "scenario": "ce_trend", + "geography_kind": "national", + "runtime_versions": TEST_RUNTIME_VERSIONS.model_dump( + mode="json", by_alias=True + ), + "years": {"2027": {"large": "diagnostic"}, "2026": {}}, + "geographies": [{"large": "mapping"}], + "composition_method": "classified inputs", + "storage_method": "formula", + } + + calculation = build_spm_calculation_provenance(SELECTION, country_receipt) + + assert resolved_spm_configuration(calculation).model_dump( + mode="json" + ) == SELECTION.model_dump(mode="json") + assert calculation.years == ("2026", "2027") + encoded = calculation.model_dump_json(by_alias=True) + assert "diagnostic" not in encoded + assert "mapping" not in encoded + assert "composition_method" not in encoded + + +def test_country_receipt_must_agree_with_adjacent_configuration() -> None: + country_receipt: dict[str, object] = { + "forecast_id": "spm-rolling-2026-09-09", + "forecast_sha256": SHA256, + "scenario": "different", + "geography_kind": "national", + "runtime_versions": TEST_RUNTIME_VERSIONS.model_dump( + mode="json", by_alias=True + ), + "years": {}, + } + + with pytest.raises(ValueError, match="differs from the SPM configuration"): + build_spm_calculation_provenance(SELECTION, country_receipt) + + +def test_comparison_collapses_identical_child_receipts() -> None: + receipt = _receipt() + + comparison = build_spm_comparison_provenance( + baseline_receipts=(receipt,) * 20, + reform_receipts=(receipt,) * 20, + ) + + assert comparison.baseline.receipt == receipt + assert comparison.baseline.execution_count == 20 + assert comparison.reform.receipt == receipt + assert comparison.reform.execution_count == 20 + assert len(comparison.model_dump_json(by_alias=True).encode()) < 2_048 + + +@pytest.mark.parametrize("side", ["baseline", "reform"]) +def test_comparison_rejects_inconsistent_child_receipts(side: str) -> None: + receipt = _receipt() + different = _receipt(years=("2027",)) + inputs = { + "baseline_receipts": (receipt,), + "reform_receipts": (receipt,), + } + inputs[f"{side}_receipts"] = (receipt, different) + + with pytest.raises(ValueError, match=f"{side} SPM receipts differ"): + build_spm_comparison_provenance(**inputs) + + +def test_comparison_rejects_previous_list_shape() -> None: + receipt = _receipt().model_dump(mode="json", by_alias=True) + + with pytest.raises(ValidationError): + SPMComparisonProvenance.model_validate( + {"baseline": [receipt], "reform": [receipt]} + ) + + +def test_compact_models_survive_json_round_trip() -> None: + comparison = build_spm_comparison_provenance( + baseline_receipts=(_receipt(),), + reform_receipts=(_receipt(),), + ) + + assert ( + SPMComparisonProvenance.model_validate_json( + json.dumps(comparison.model_dump(mode="json", by_alias=True)) + ) + == comparison + ) diff --git a/tests/unit/test_spm_settings.py b/tests/unit/test_spm_settings.py index 615fba9d2..6ed564f1f 100644 --- a/tests/unit/test_spm_settings.py +++ b/tests/unit/test_spm_settings.py @@ -403,13 +403,13 @@ def test_canonical_country_cannot_inherit_the_legacy_exception(monkeypatch, vers } -def test_resolved_settings_recover_receipt_omitted_null_values(certified_bundle): - """A receipt that omits its null values still names the same selection.""" +def test_resolved_settings_require_explicit_null_values(certified_bundle): + """A transported resolved config includes all six selection fields.""" assert spm.resolved_spm_settings(RESOLVED_SELECTION) == RESOLVED_SELECTION omitted = { key: value for key, value in RESOLVED_SELECTION.items() if value is not None } - assert spm.resolved_spm_settings(omitted) == RESOLVED_SELECTION + assert spm.resolved_spm_settings(omitted) is None assert spm.resolved_spm_settings(spm.normalize_spm_selection("us", None)) == ( RESOLVED_SELECTION ) @@ -432,17 +432,28 @@ def test_resolved_settings_reject_unreadable_receipt_settings(settings): assert spm.resolved_spm_settings(settings) is None -def test_receipt_keeps_the_countrys_own_omissions(certified_bundle): - """A published receipt repeats the country's wire shape, nulls and all.""" - omitted = { - key: value for key, value in RESOLVED_SELECTION.items() if value is not None - } +def test_receipt_reduces_country_diagnostics_to_one_compact_provenance( + certified_bundle, +): + """The public receipt itself contains every resolved configuration field.""" simulation = SimpleNamespace( - spm_config=omitted, spm_provenance=lambda: deepcopy(COUNTRY_RECEIPT) + spm_config=RESOLVED_SELECTION, + spm_provenance=lambda: deepcopy(COUNTRY_RECEIPT), ) receipt = spm.calculation_spm_receipt(simulation) - assert receipt["spm_config"] == omitted - assert receipt["spm_provenance"] == COUNTRY_RECEIPT + assert set(receipt) == {"spm_provenance"} + assert receipt["spm_provenance"] == { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "test-artifact", + "forecast_sha256": ARTIFACT_HASH, + "scenario": "baseline", + "geography_kind": "county", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": [], + "runtime_versions": COUNTRY_RECEIPT["runtime_versions"], + } COUNTRY_RECEIPT = { @@ -450,7 +461,12 @@ def test_receipt_keeps_the_countrys_own_omissions(certified_bundle): "forecast_sha256": ARTIFACT_HASH, "scenario": "baseline", "geography_kind": "county", - "runtime_versions": {"policyengine-us": "test"}, + "runtime_versions": { + "policyengine": "test", + "policyengine-core": "test", + "policyengine-us": "test", + "spm-calculator": "test", + }, "years": {}, "geographies": [], "composition_method": "test composition", @@ -465,13 +481,9 @@ def test_receipt_keeps_the_countrys_own_omissions(certified_bundle): spm_config={**RESOLVED_SELECTION, "threshold_method": "canonical"}, spm_provenance=lambda: COUNTRY_RECEIPT, ), - SimpleNamespace( - spm_config=RESOLVED_SELECTION, - spm_provenance=lambda: {**COUNTRY_RECEIPT, "unreviewed_field": True}, - ), SimpleNamespace(spm_config=RESOLVED_SELECTION, spm_provenance=lambda: {}), ], - ids=["extra-setting", "extra-receipt-field", "incomplete-receipt"], + ids=["extra-setting", "incomplete-receipt"], ) def test_uncertifiable_country_receipt_is_typed_not_an_internal_failure(simulation): """An unreadable receipt is an uncertified country contract, never a 500.""" @@ -556,6 +568,7 @@ def test_a_model_this_build_cannot_load_reads_as_no_canonical_model( def refuse(name): raise failure + monkeypatch.setattr(spm, "_current_bundle", lambda: deepcopy(LEGACY_BUNDLE)) monkeypatch.setattr(spm.importlib, "import_module", refuse) assert spm._installed_country_implements_spm("us") is False diff --git a/tests/unit/test_worker_spm_release.py b/tests/unit/test_worker_spm_release.py new file mode 100644 index 000000000..9df37be59 --- /dev/null +++ b/tests/unit/test_worker_spm_release.py @@ -0,0 +1,127 @@ +"""Release checks compare installed packages with the selected worker.""" + +from copy import deepcopy + +import pytest + +from policyengine_api import constants, worker_spm, worker_spm_release +from policyengine_api.spm import SPMValidationError + + +FIXTURE_BUNDLE_VERSION = "fixture-policyengine" +FIXTURE_PACKAGE_VERSIONS = { + "policyengine": FIXTURE_BUNDLE_VERSION, + "policyengine-core": "fixture-core", + "policyengine-us": "fixture-us", + "policyengine-uk": "fixture-uk", + "spm-calculator": "fixture-spm", +} +SELECTION = { + "forecast_content_sha256": "a" * 64, + "scenario": "fixture", + "geography_kind": "county", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} +WORKER_APP = "fixture-worker" + + +@pytest.fixture +def registry(monkeypatch): + monkeypatch.setattr(constants, "POLICYENGINE_VERSION", FIXTURE_BUNDLE_VERSION) + monkeypatch.setattr( + constants, + "_policyengine_bundle", + { + "packages": { + name: {"name": name, "version": package_version} + for name, package_version in FIXTURE_PACKAGE_VERSIONS.items() + } + }, + ) + monkeypatch.setattr( + worker_spm_release, + "distribution_version", + FIXTURE_PACKAGE_VERSIONS.__getitem__, + ) + monkeypatch.setattr(worker_spm, "normalize_spm_selection", lambda *_: SELECTION) + return { + "policyengine": {FIXTURE_BUNDLE_VERSION: WORKER_APP}, + "us": {FIXTURE_PACKAGE_VERSIONS["policyengine-us"]: WORKER_APP}, + "uk": {FIXTURE_PACKAGE_VERSIONS["policyengine-uk"]: WORKER_APP}, + "spm_capabilities": { + FIXTURE_BUNDLE_VERSION: { + "contract_version": "canonical-spm-v1", + "defaults": dict(SELECTION), + } + }, + } + + +def test_release_accepts_matching_installed_bundle_and_worker(registry): + assert ( + worker_spm_release.validate_installed_worker( + registry, + FIXTURE_BUNDLE_VERSION, + ) + == SELECTION + ) + + +@pytest.mark.parametrize( + "mutation", + ["missing-capability", "wrong-contract", "wrong-forecast"], +) +def test_release_rejects_worker_without_matching_spm_capability(registry, mutation): + registry = deepcopy(registry) + capability = registry["spm_capabilities"][FIXTURE_BUNDLE_VERSION] + if mutation == "missing-capability": + registry["spm_capabilities"] = {} + elif mutation == "wrong-contract": + capability["contract_version"] = "different-contract" + else: + capability["defaults"]["forecast_content_sha256"] = "b" * 64 + + with pytest.raises(SPMValidationError): + worker_spm_release.validate_installed_worker(registry, FIXTURE_BUNDLE_VERSION) + + +@pytest.mark.parametrize("route", ["policyengine", "us", "uk"]) +def test_release_rejects_missing_worker_route(registry, route): + registry[route] = {} + with pytest.raises(ValueError, match="no .* application"): + worker_spm_release.validate_installed_worker(registry, FIXTURE_BUNDLE_VERSION) + + +@pytest.mark.parametrize("route", ["us", "uk"]) +def test_release_rejects_country_route_to_different_worker(registry, route): + version = next(iter(registry[route])) + registry[route][version] = "different-worker" + with pytest.raises(ValueError, match="different worker"): + worker_spm_release.validate_installed_worker(registry, FIXTURE_BUNDLE_VERSION) + + +def test_release_rejects_installed_distribution_outside_manifest( + registry, + monkeypatch, +): + installed = {**FIXTURE_PACKAGE_VERSIONS, "policyengine-core": "different-core"} + monkeypatch.setattr( + worker_spm_release, + "distribution_version", + installed.__getitem__, + ) + with pytest.raises(ValueError, match="policyengine-core"): + worker_spm_release.validate_installed_worker(registry, FIXTURE_BUNDLE_VERSION) + + +def test_release_rejects_requirement_outside_manifest(registry): + with pytest.raises(ValueError, match="manifest"): + worker_spm_release.validate_installed_worker(registry, "different-bundle") + + +def test_release_rejects_manifest_without_complete_package_set(registry): + constants._policyengine_bundle["packages"].pop("spm-calculator") + with pytest.raises(ValueError, match="spm-calculator"): + worker_spm_release.validate_installed_worker(registry, FIXTURE_BUNDLE_VERSION) diff --git a/tests/unit/v2/test_stage12_worker_contracts.py b/tests/unit/v2/test_stage12_worker_contracts.py index df177f792..330af0483 100644 --- a/tests/unit/v2/test_stage12_worker_contracts.py +++ b/tests/unit/v2/test_stage12_worker_contracts.py @@ -27,6 +27,35 @@ DIGEST_C = "c" * 64 +def _calculation_provenance() -> dict[str, object]: + return { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "canonical-forecast", + "forecast_sha256": DIGEST_A, + "scenario": "official", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": ["2026"], + "runtime_versions": { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0", + }, + } + + +def _comparison_provenance() -> dict[str, object]: + receipt = _calculation_provenance() + return { + "schema_version": "canonical-spm-comparison-v2", + "baseline": {"receipt": receipt, "execution_count": 1}, + "reform": {"receipt": receipt, "execution_count": 1}, + } + + def _bundle() -> dict[str, object]: return { "policyengine_version": "5.2.0", @@ -173,10 +202,7 @@ def test_artifact_descriptors_retain_identity_and_bundle_provenance() -> None: "identity_sha256": DIGEST_B, }, "bundle": _bundle(), - "calculation_provenance": { - "spm_config": {"scenario": "official"}, - "spm_provenance": {"forecast_sha256": DIGEST_A}, - }, + "calculation_provenance": _calculation_provenance(), } ) report_artifact = AggregateReportArtifactDescriptor.model_validate( @@ -197,16 +223,58 @@ def test_artifact_descriptors_retain_identity_and_bundle_provenance() -> None: ) assert simulation_artifact.row_identity.identifier_columns == ("household_id",) - assert simulation_artifact.calculation_provenance == { - "spm_config": {"scenario": "official"}, - "spm_provenance": {"forecast_sha256": DIGEST_A}, - } + assert simulation_artifact.calculation_provenance is not None + assert ( + simulation_artifact.calculation_provenance.model_dump( + mode="json", by_alias=True + ) + == _calculation_provenance() + ) assert ( report_artifact.baseline_artifact_sha256 == simulation_artifact.artifact.content_sha256 ) +@pytest.mark.parametrize( + "provenance", + [ + {"scenario": "official"}, + { + "spm_config": {"scenario": "official"}, + "spm_provenance": _calculation_provenance(), + }, + { + **_calculation_provenance(), + "geographies": [], + }, + ], + ids=["partial", "legacy-dual-field", "legacy-rich"], +) +def test_simulation_artifact_rejects_noncompact_calculation_provenance( + provenance: dict[str, object], +) -> None: + descriptor = { + "contract_version": 1, + "evaluation_id": str(EVALUATION_ID), + "simulation_execution_id": str(BASELINE_ID), + "role": "baseline", + "artifact": {**_population_artifact(), "content_sha256": DIGEST_C}, + "output_schema_version": 1, + "row_identity": { + "schema_version": 1, + "identifier_columns": ["household_id"], + "row_count": 100, + "identity_sha256": DIGEST_B, + }, + "bundle": _bundle(), + "calculation_provenance": provenance, + } + + with pytest.raises(ValidationError): + SimulationArtifactDescriptor.model_validate(descriptor) + + def test_artifact_payload_contracts_cover_parquet_and_aggregate_json() -> None: assert SIMULATION_PARQUET_PAYLOAD_CONTRACT.model_dump(mode="json") == { "payload_schema_version": 1, @@ -230,7 +298,10 @@ def test_artifact_payload_contracts_cover_parquet_and_aggregate_json() -> None: "evaluation_id": str(EVALUATION_ID), "requested_aggregates": ["budget"], "bundle": _bundle(), - "result": {"budget": {"change": 100.0}}, + "result": { + "budget": {"change": 100.0}, + "spm_provenance": _comparison_provenance(), + }, } ) @@ -243,6 +314,19 @@ def test_artifact_payload_contracts_cover_parquet_and_aggregate_json() -> None: } ) + with pytest.raises(ValidationError, match="legacy spm_config"): + AggregateReportArtifactPayload.model_validate( + { + **payload.model_dump(mode="json"), + "result": { + **payload.result, + "spm_config": { + "forecast_content_sha256": DIGEST_A, + }, + }, + } + ) + def test_contract_models_are_immutable_and_reject_duplicate_lists() -> None: report = ReportExecutionInput.model_validate(_report()) diff --git a/uv.lock b/uv.lock index 9cdc2e332..c5ecf2412 100644 --- a/uv.lock +++ b/uv.lock @@ -1,12 +1,12 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", - "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version >= '3.15' and sys_platform == 'emscripten'", - "python_full_version == '3.14.*' and sys_platform == 'emscripten'", "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", @@ -2743,7 +2743,7 @@ wheels = [ [[package]] name = "policyengine" -version = "5.2.0" +version = "6.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "h5py" }, @@ -2755,9 +2755,9 @@ dependencies = [ { name = "pydantic" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/45/2d/5b09f414e26573748f9ae2e448d6f9f94196b7733e8400d727db9ed17868/policyengine-5.2.0.tar.gz", hash = "sha256:c1e7a3a8d7fb23401aedbe8a9fa2d6ed69e24ae9983b6a4a8a0d5112394b0bc4", size = 732330, upload-time = "2026-08-29T17:24:52.814Z" } +sdist = { url = "https://files.pythonhosted.org/packages/67/f2/3a75eb571fa3eda2e9f043c145cadae9eab4df6959fc447e767f92885d88/policyengine-6.2.1.tar.gz", hash = "sha256:0f2e51be23229ab5d8a8f80fb7b716fe4d3780806e2e16dfcda0362983beac56", size = 772927, upload-time = "2026-09-30T22:19:57.846Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/c5/677144df3800ae41dd587b292bd633c09edd5c998695ca15987c05730bc6/policyengine-5.2.0-py3-none-any.whl", hash = "sha256:e73307f787cf366dc16c080b708c39c28a58b0adf34abe51de5ecd97ecfab154", size = 226102, upload-time = "2026-08-29T17:24:51.377Z" }, + { url = "https://files.pythonhosted.org/packages/5a/8b/4b216e42d150852e51c01ea654455496e1e1197c008e72e70f29392dec8c/policyengine-6.2.1-py3-none-any.whl", hash = "sha256:d07ff400da096be4b43843bd333ea0b7f42b06f5eea584e72c78f75a16fe3cdb", size = 252406, upload-time = "2026-09-30T22:19:56.61Z" }, ] [package.optional-dependencies] @@ -2765,6 +2765,7 @@ models = [ { name = "policyengine-core" }, { name = "policyengine-uk" }, { name = "policyengine-us" }, + { name = "spm-calculator" }, ] [[package]] @@ -2801,7 +2802,6 @@ dependencies = [ { name = "python-dotenv" }, { name = "redis" }, { name = "rq" }, - { name = "spm-calculator" }, { name = "sqlalchemy" }, { name = "sqlmodel" }, { name = "streamlit" }, @@ -2844,7 +2844,7 @@ requires-dist = [ { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.15,<2" }, { name = "openai" }, { name = "packaging", specifier = ">=24,<27" }, - { name = "policyengine", extras = ["models"], specifier = "==5.2.0" }, + { name = "policyengine", extras = ["models"], specifier = "==6.2.1" }, { name = "policyengine-canada", specifier = "==0.96.3" }, { name = "policyengine-il", specifier = "==0.1.0" }, { name = "policyengine-ng", specifier = "==0.5.1" }, @@ -2859,7 +2859,6 @@ requires-dist = [ { name = "redis" }, { name = "rq" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9.0" }, - { name = "spm-calculator", specifier = "==0.3.1" }, { name = "sqlalchemy", specifier = ">=2,<3" }, { name = "sqlmodel", specifier = ">=0.0.39,<0.1" }, { name = "streamlit" }, @@ -2899,7 +2898,7 @@ wheels = [ [[package]] name = "policyengine-core" -version = "3.30.1" +version = "3.32.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dpath" }, @@ -2919,9 +2918,9 @@ dependencies = [ { name = "standard-imghdr" }, { name = "wheel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a1/9d/f71da3e348ddc52e058c76b72bbc4b2f20abcf0e7389ecd5aca5d550fd14/policyengine_core-3.30.1.tar.gz", hash = "sha256:a16f29fe51ec01a7be2171386f8bd26b9ba55ffc338adb70adcadafe33d9c55f", size = 500419, upload-time = "2026-07-19T15:50:41.288Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/cd/9fc967c5b787baa5f9d83461773ba0982a8e22ae83c40155a7ee55edd11a/policyengine_core-3.32.10.tar.gz", hash = "sha256:0f51e8abfa8f172c69d03ba8f4304632a348fc6dca780e41021b2fd0cec04fde", size = 427479, upload-time = "2026-09-29T17:56:43.582Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/7e/07624d8add8889af1b4af796d36c38b2e2af494b1d78a064fab29e9bea00/policyengine_core-3.30.1-py3-none-any.whl", hash = "sha256:2dbcf5f590a0199a7b7c77fcbbda2ff6bc289f6c6169ca0af3beace35d8b3e63", size = 244846, upload-time = "2026-07-19T15:50:39.651Z" }, + { url = "https://files.pythonhosted.org/packages/58/50/291d73da16f39a3c4f05dffb6d40b1142fc400ec39b5a2ca7481e6fe9d90/policyengine_core-3.32.10-py3-none-any.whl", hash = "sha256:124afc555fe7c848ae8237b820fed0b6ea0ea1280e8850ad8bf356937ee10e5b", size = 247410, upload-time = "2026-09-29T17:56:42.218Z" }, ] [[package]] @@ -2978,7 +2977,7 @@ otlp-grpc = [ [[package]] name = "policyengine-uk" -version = "2.90.2" +version = "2.102.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -2986,14 +2985,14 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/e1/6652fd4ff8806d81b7c846ce2baeef4b9a8457ed380bb0853419d3cf36d9/policyengine_uk-2.90.2.tar.gz", hash = "sha256:c9baeaded1bb6d6f5ca5fcd962b80a198509513737c194aa867b0ef6d13d6a09", size = 1237478, upload-time = "2026-08-13T05:30:05.961Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/49/1db19a439463f1304e93521debf40794f7b3483d37356c6d99c346112d0f/policyengine_uk-2.102.3.tar.gz", hash = "sha256:cdb0900e78818b46fdb79b2a8299bce6c7d6c14692e26cf38367781beea607be", size = 1314347, upload-time = "2026-09-28T19:50:44.714Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/da/658bff2df6094bd7bf2b0f561ff60fd41508adbd0a3af29b4553c83bc678/policyengine_uk-2.90.2-py3-none-any.whl", hash = "sha256:f100ef97e20b29b84138078f1dbc5b83f8857e6bc74b745fc6abef7ea1eeadf2", size = 2034933, upload-time = "2026-08-13T05:30:03.884Z" }, + { url = "https://files.pythonhosted.org/packages/37/e6/e8d9069d498f1510a1380f24a78e957902d01f00b10248c146d6710d5b37/policyengine_uk-2.102.3-py3-none-any.whl", hash = "sha256:f02a76712a3edf9983c713c1db42e9653a6e974e4ce335dd4b5676bbc2bcd636", size = 2189605, upload-time = "2026-09-28T19:50:42.472Z" }, ] [[package]] name = "policyengine-us" -version = "1.764.6" +version = "2.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -3003,9 +3002,9 @@ dependencies = [ { name = "tables" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6e/cc/9e65a9586069beab1f02102a23f7a0e6332f1a9c2d590cb9f0e68f9ac987/policyengine_us-1.764.6.tar.gz", hash = "sha256:9817d48abc5b7d690b8abe33047005e5cf40b71654a85a2534bb5e3bcf921f79", size = 11009555, upload-time = "2026-07-06T13:13:07.254Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/09/763b21371a62635e15e62b53fb696e299f01460907c27313d40f63031eb2/policyengine_us-2.2.1.tar.gz", hash = "sha256:d5300c91f3eae2e47a61f485fdfde91a892b80e95db3e46ff2c4bbd417d76501", size = 11946510, upload-time = "2026-09-15T04:45:50.33Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/5a/0566666f30ad06e416f0cf25b24bdee73acd71897fb2de3c0ac43b59e3e3/policyengine_us-1.764.6-py3-none-any.whl", hash = "sha256:2b2c5c02c26ab212270948db1f30129728b625fc6109fc9ca0c604b714546909", size = 13117921, upload-time = "2026-07-06T13:13:03.476Z" }, + { url = "https://files.pythonhosted.org/packages/5e/09/86ac3968d2a18e28fce3ba1186fc21728e9f299b96a806d6f204f1e0aaed/policyengine_us-2.2.1-py3-none-any.whl", hash = "sha256:0993a6c73fcdfbe171a796aca9d302741bd8e00ab83388ad81c15a08740318c6", size = 14885537, upload-time = "2026-09-15T04:45:46.649Z" }, ] [[package]] @@ -3924,7 +3923,7 @@ wheels = [ [[package]] name = "spm-calculator" -version = "0.3.1" +version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "census" }, @@ -3934,9 +3933,9 @@ dependencies = [ { name = "requests" }, { name = "us" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/54/3b/b805c7e3e18c5b5c00f61b60112f9690d084c910e2481bc020f35390d8fd/spm_calculator-0.3.1.tar.gz", hash = "sha256:41f2f4d00d8c03422a7d57b800052e7760b88e463a5884802f83ed58d35c18c1", size = 75945, upload-time = "2026-04-17T19:52:39.707Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c2/5e/3f28b212c401795990250bb3daec741f699a8c7dc2a23a3db7005df6feb6/spm_calculator-1.0.0.tar.gz", hash = "sha256:a99aac8c2c0bf81a9455105bbf872366f1ea9066a99cd793714881cd3db846dc", size = 7150825, upload-time = "2026-09-11T15:45:12.732Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/1b/29f705f8a96fc7f55f2c07dfcddbbae78efdc6f174d25d4a0560fc3f5cf9/spm_calculator-0.3.1-py3-none-any.whl", hash = "sha256:52c57ecc5a240ec941b0f2b0d93bc4fa437ef6250e233baed8e11916fa9c1150", size = 57826, upload-time = "2026-04-17T19:52:38.444Z" }, + { url = "https://files.pythonhosted.org/packages/19/a0/c484f69a0ebf88a9b9fd0ac28176f8df66550f46714f1b0c5ec0b360824e/spm_calculator-1.0.0-py3-none-any.whl", hash = "sha256:e354937a5e1a4045d4966ed594a528d8b02866fabaac9bb5672017004b627305", size = 7395384, upload-time = "2026-09-11T15:45:10.492Z" }, ] [[package]]