From 60905063e84e54ee5c4669bb44c9657f4c4b4bcd Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:18:28 +0400 Subject: [PATCH 01/22] Use dedicated UK private data credential --- .env.example | 4 +- .github/scripts/cloud_run_env.sh | 4 +- .github/scripts/deploy_cloud_run_candidate.sh | 4 +- .../resolve_cloud_run_candidate_state.sh | 26 +++++++ .github/scripts/sync_cloud_run_secrets.sh | 63 ++++++++++++++- .../scripts/validate_cloud_run_deploy_env.sh | 2 +- .github/workflows/pr.yml | 6 +- .github/workflows/push.yml | 3 +- .github/workflows/sync-cloud-run-secrets.yml | 2 +- README.md | 4 +- changelog.d/uk-private-hf-read-token.fixed.md | 1 + .../uk-private-hugging-face-credential.md | 53 +++++++++++++ .../test_environment_variables.py | 49 ++++++++++++ tests/unit/test_cloud_run_deploy_scripts.py | 78 +++++++++++++++++-- 14 files changed, 282 insertions(+), 17 deletions(-) create mode 100644 changelog.d/uk-private-hf-read-token.fixed.md create mode 100644 docs/engineering/uk-private-hugging-face-credential.md diff --git a/.env.example b/.env.example index b43e2dc22..dfd01ccb3 100644 --- a/.env.example +++ b/.env.example @@ -17,7 +17,9 @@ POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN=policyengine_github_token OPENAI_API_KEY=policyengine_openai_api_key # Token for Hugging Face models -HUGGING_FACE_TOKEN=policyengine_huggingface_token +# PolicyEngine Core compatibility variable. Managed deployments bind the +# PE_UK_PRIVATE_HF_READ_TOKEN credential to this process-level name. +HUGGING_FACE_TOKEN=pe_uk_private_hf_read_token_value # Redis is required for budget-window economy requests and other API cache # paths. Configure local development explicitly as described in README.md; diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index c97672628..3a0743fec 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -39,7 +39,7 @@ cloud_run_set_defaults() { CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-}" CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}" CLOUD_RUN_OPENAI_API_KEY_SECRET="${CLOUD_RUN_OPENAI_API_KEY_SECRET:-policyengine-api-prod-openai-api-key:latest}" - CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET="${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET:-policyengine-api-prod-hugging-face-token:latest}" + CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET:-pe-uk-private-hf-read-token:latest}" CLOUD_RUN_RUNTIME_CACHE_URL_SECRET="${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET:-policyengine-api-prod-runtime-cache-url:latest}" CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET="${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET:-policyengine-api-prod-runtime-cache-ca:latest}" CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT="${CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT:-production}" @@ -76,7 +76,7 @@ cloud_run_set_defaults() { export CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET export CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET export CLOUD_RUN_OPENAI_API_KEY_SECRET - export CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET + export CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET export CLOUD_RUN_RUNTIME_CACHE_URL_SECRET export CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET export CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT diff --git a/.github/scripts/deploy_cloud_run_candidate.sh b/.github/scripts/deploy_cloud_run_candidate.sh index 2e5d80773..827403ce0 100755 --- a/.github/scripts/deploy_cloud_run_candidate.sh +++ b/.github/scripts/deploy_cloud_run_candidate.sh @@ -62,7 +62,9 @@ secret_vars=( "POLICYENGINE_DB_PASSWORD=${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}" "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN=${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET}" "OPENAI_API_KEY=${CLOUD_RUN_OPENAI_API_KEY_SECRET}" - "HUGGING_FACE_TOKEN=${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET}" + # PolicyEngine Core reads HUGGING_FACE_TOKEN. The stored credential and every + # deployment input use the purpose-specific PE_UK_PRIVATE_HF_READ_TOKEN name. + "HUGGING_FACE_TOKEN=${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET}" "RUNTIME_CACHE_URL=${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET}" "RUNTIME_CACHE_CA_CERT=${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET}" ) diff --git a/.github/scripts/resolve_cloud_run_candidate_state.sh b/.github/scripts/resolve_cloud_run_candidate_state.sh index 334a42b66..60f3b49c7 100755 --- a/.github/scripts/resolve_cloud_run_candidate_state.sh +++ b/.github/scripts/resolve_cloud_run_candidate_state.sh @@ -78,6 +78,32 @@ image="$(jq -er ' | select(type == "string" and contains("@sha256:")) ' <<<"${revision_json}")" +expected_hf_secret_resource="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET%:*}" +expected_hf_secret_name="${expected_hf_secret_resource##*/}" +expected_hf_secret_version="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET##*:}" +actual_hf_secret_binding="$(jq -cer ' + [ + .spec.containers[0].env[]? + | select(.name == "HUGGING_FACE_TOKEN") + | { + name: .valueFrom.secretKeyRef.name, + version: .valueFrom.secretKeyRef.key + } + ] + | if length == 1 then .[0] + else error("HUGGING_FACE_TOKEN must have exactly one secret binding") + end +' <<<"${revision_json}")" +actual_hf_secret_name="$(jq -er '.name' <<<"${actual_hf_secret_binding}")" +actual_hf_secret_version="$(jq -er '.version' <<<"${actual_hf_secret_binding}")" +if [[ "${actual_hf_secret_name}" != "${expected_hf_secret_name}" \ + || "${actual_hf_secret_version}" != "${expected_hf_secret_version}" ]]; then + printf 'Revision %s binds HUGGING_FACE_TOKEN to %s:%s; expected %s:%s\n' \ + "${revision}" "${actual_hf_secret_name}" "${actual_hf_secret_version}" \ + "${expected_hf_secret_name}" "${expected_hf_secret_version}" >&2 + exit 2 +fi + deployment_selector_count=0 for selector in \ ROUTE_IMPL_HEALTH \ diff --git a/.github/scripts/sync_cloud_run_secrets.sh b/.github/scripts/sync_cloud_run_secrets.sh index a83b3599c..2a1626bb6 100644 --- a/.github/scripts/sync_cloud_run_secrets.sh +++ b/.github/scripts/sync_cloud_run_secrets.sh @@ -4,6 +4,7 @@ set -euo pipefail set +x CLOUD_RUN_PROJECT="${CLOUD_RUN_PROJECT:-policyengine-api}" +CLOUD_RUN_REGION="${CLOUD_RUN_REGION:-us-central1}" require_env() { local env_name="$1" @@ -43,9 +44,69 @@ sync_secret() { echo "Synced ${env_name} to Secret Manager secret ${secret_name}." } +verify_secret_value() { + local env_name="$1" + local secret_name="$2" + local expected_value="${!env_name:-}" + local stored_value + + stored_value="$(gcloud secrets versions access latest \ + --secret "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}")" + if [[ "${stored_value}" != "${expected_value}" ]]; then + echo "::error::Secret Manager value verification failed for ${secret_name}." >&2 + return 1 + fi + unset stored_value expected_value + echo "Verified ${env_name} in Secret Manager without printing its value." +} + +grant_secret_access_to_cloud_run_service() { + local secret_name="$1" + local service_name="$2" + local runtime_service_account + local member + + runtime_service_account="$(gcloud run services describe "${service_name}" \ + --project "${CLOUD_RUN_PROJECT}" \ + --region "${CLOUD_RUN_REGION}" \ + --platform managed \ + --format='value(spec.template.spec.serviceAccountName)')" + if [[ -z "${runtime_service_account}" ]]; then + echo "::error::Cloud Run service ${service_name} has no runtime service account." >&2 + return 1 + fi + member="serviceAccount:${runtime_service_account}" + + gcloud secrets add-iam-policy-binding "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}" \ + --member "${member}" \ + --role roles/secretmanager.secretAccessor >/dev/null + + if ! gcloud secrets get-iam-policy "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}" \ + --format=json \ + | jq -e --arg member "${member}" ' + any( + .bindings[]?; + .role == "roles/secretmanager.secretAccessor" + and ((.members // []) | index($member) != null) + ) + ' >/dev/null; then + echo "::error::Secret Manager access verification failed for ${service_name}." >&2 + return 1 + fi + echo "Verified ${service_name} can read ${secret_name}." +} + require_env CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT sync_secret POLICYENGINE_DB_PASSWORD policyengine-api-prod-db-password sync_secret POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN policyengine-api-prod-github-microdata-token sync_secret OPENAI_API_KEY policyengine-api-prod-openai-api-key -sync_secret HUGGING_FACE_TOKEN policyengine-api-prod-hugging-face-token +sync_secret PE_UK_PRIVATE_HF_READ_TOKEN pe-uk-private-hf-read-token +verify_secret_value PE_UK_PRIVATE_HF_READ_TOKEN pe-uk-private-hf-read-token +grant_secret_access_to_cloud_run_service \ + pe-uk-private-hf-read-token policyengine-api-staging +grant_secret_access_to_cloud_run_service \ + pe-uk-private-hf-read-token policyengine-api diff --git a/.github/scripts/validate_cloud_run_deploy_env.sh b/.github/scripts/validate_cloud_run_deploy_env.sh index 8647f1235..1f1e32451 100755 --- a/.github/scripts/validate_cloud_run_deploy_env.sh +++ b/.github/scripts/validate_cloud_run_deploy_env.sh @@ -29,7 +29,7 @@ cloud_run_require_env \ CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET \ CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET \ CLOUD_RUN_OPENAI_API_KEY_SECRET \ - CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET \ + CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET \ CLOUD_RUN_RUNTIME_CACHE_URL_SECRET \ CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET \ CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT \ diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index dee90cc71..9171eba74 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -145,7 +145,9 @@ jobs: run: pytest tests/env_variables/test_environment_variables.py env: POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + # Compatibility name required by PolicyEngine Core. The GitHub + # credential name describes its provider, data, and read-only scope. + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} test: name: Test @@ -184,7 +186,7 @@ jobs: POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 with: diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 6b817ab14..7c0dd22c9 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -240,7 +240,8 @@ jobs: POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + # Compatibility name required by PolicyEngine Core. + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} diff --git a/.github/workflows/sync-cloud-run-secrets.yml b/.github/workflows/sync-cloud-run-secrets.yml index 77b766777..5a1bbc372 100644 --- a/.github/workflows/sync-cloud-run-secrets.yml +++ b/.github/workflows/sync-cloud-run-secrets.yml @@ -36,5 +36,5 @@ jobs: POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} run: bash .github/scripts/sync_cloud_run_secrets.sh diff --git a/README.md b/README.md index 9f0a4470d..f572bcdf9 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,9 @@ make setup-env - `POLICYENGINE_DB_INSTANCE_CONNECTION_NAME` - `POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN` - `OPENAI_API_KEY` -- `HUGGING_FACE_TOKEN` +- `HUGGING_FACE_TOKEN` (the local PolicyEngine Core compatibility variable; + managed deployments source it from the purpose-specific GitHub secret + `PE_UK_PRIVATE_HF_READ_TOKEN`) The database settings must resolve to an explicit durable development MySQL database (or an authorized Cloud SQL development target). `FLASK_DEBUG` does diff --git a/changelog.d/uk-private-hf-read-token.fixed.md b/changelog.d/uk-private-hf-read-token.fixed.md new file mode 100644 index 000000000..dd46feb49 --- /dev/null +++ b/changelog.d/uk-private-hf-read-token.fixed.md @@ -0,0 +1 @@ +Use a dedicated read-only Hugging Face credential for private UK runtime data and verify its Cloud Run secret binding during deployment. diff --git a/docs/engineering/uk-private-hugging-face-credential.md b/docs/engineering/uk-private-hugging-face-credential.md new file mode 100644 index 000000000..123cfdc5d --- /dev/null +++ b/docs/engineering/uk-private-hugging-face-credential.md @@ -0,0 +1,53 @@ +# UK private-data Hugging Face credential + +API and simulation deployments use one read-only Hugging Face credential for +the private UK runtime data. Its managed name is +`PE_UK_PRIVATE_HF_READ_TOKEN`. + +## Required access + +The credential must be a fine-grained token named +`pe-uk-private-hf-read-token` with read-only access to: + +- the model repository `policyengine/policyengine-uk-data-private`; +- the dataset repository `policyengine/populace-uk-private`. + +It must not have repository write access. The deployed API and simulation +paths do not use `policyengine/populace-uk-staging`; build and publication +credentials for that repository are separate. + +## API v1 path + +The selected-repository GitHub organization secret +`PE_UK_PRIVATE_HF_READ_TOKEN` is copied to the Google Secret Manager resource +`pe-uk-private-hf-read-token`. Both API v1 Cloud Run runtime service accounts +receive `roles/secretmanager.secretAccessor` for that resource. The +synchronization script resolves the runtime identities from the deployed +`policyengine-api-staging` and `policyengine-api` services rather than relying +on a duplicated account-name configuration value, then verifies both IAM +bindings. +The synchronization workflow reads the newly stored version back and compares +it in memory with the GitHub value without logging either value. + +Cloud Run exposes the resource as `HUGGING_FACE_TOKEN` because PolicyEngine +Core reads that compatibility environment variable. `HUGGING_FACE_TOKEN` is +not the name of the stored GitHub or Google Cloud credential. + +The pull-request environment check verifies the token identity, confirms that +it has no write permission, and checks both required private repositories. The +Cloud Run candidate resolver verifies that the exact deployed revision binds +`HUGGING_FACE_TOKEN` to `pe-uk-private-hf-read-token:latest`, without printing +the credential. + +The simulation-entry Cloud Run service does not receive this credential. It +submits requests to Modal and does not download model data. + +## Rotation and retirement + +Provision the replacement credential before merging workflow changes. Deploy +and verify staging before production. Keep the previous Google Secret Manager +resource during the rollback period; then disable its version before deleting +the resource. + +Do not remove the organization secret named `HUGGING_FACE_TOKEN` as part of +this migration. Other PolicyEngine repositories still consume it. diff --git a/tests/env_variables/test_environment_variables.py b/tests/env_variables/test_environment_variables.py index 64cc2176b..5dde7a2d2 100644 --- a/tests/env_variables/test_environment_variables.py +++ b/tests/env_variables/test_environment_variables.py @@ -3,6 +3,13 @@ import requests HUGGING_FACE_API_URL = "https://huggingface.co/api/whoami-v2" +HUGGING_FACE_UK_DATA_MODEL_URL = ( + "https://huggingface.co/api/models/policyengine/policyengine-uk-data-private" +) +HUGGING_FACE_UK_POPULATION_DATASET_URL = ( + "https://huggingface.co/api/datasets/policyengine/populace-uk-private" +) +EXPECTED_HUGGING_FACE_TOKEN_NAME = "pe-uk-private-hf-read-token" GITHUB_API_URL = "https://api.github.com/user" @@ -32,6 +39,48 @@ def test_hugging_face_token(self): assert token_validation_response.status_code == 200, ( f"Invalid HUGGING_FACE_TOKEN: {token_validation_response.text}" ) + token_details = token_validation_response.json()["auth"]["accessToken"] + assert token_details["displayName"] == EXPECTED_HUGGING_FACE_TOKEN_NAME + assert token_details["role"] == "fineGrained" + fine_grained = token_details["fineGrained"] + granted_permissions = set(fine_grained.get("global", [])) + granted_permissions.update( + permission + for scope in fine_grained["scoped"] + for permission in scope["permissions"] + ) + assert "repo.content.read" in granted_permissions + assert not any("write" in permission for permission in granted_permissions) + + headers = {"Authorization": f"Bearer {token}"} + uk_data_response = requests.get( + HUGGING_FACE_UK_DATA_MODEL_URL, + headers=headers, + timeout=10, + ) + assert uk_data_response.status_code == 200, ( + "The API runtime token cannot read " + "policyengine/policyengine-uk-data-private: " + f"{uk_data_response.text}" + ) + uk_data_files = { + sibling["rfilename"] for sibling in uk_data_response.json()["siblings"] + } + assert "local_authority_weights.h5" in uk_data_files + + populace_response = requests.get( + HUGGING_FACE_UK_POPULATION_DATASET_URL, + headers=headers, + timeout=10, + ) + assert populace_response.status_code == 200, ( + "The shared API and simulation runtime token cannot read " + f"policyengine/populace-uk-private: {populace_response.text}" + ) + populace_files = { + sibling["rfilename"] for sibling in populace_response.json()["siblings"] + } + assert "populace_uk_2023.h5" in populace_files @pytest.mark.skipif( do_not_run_in_debug(), diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index 653e4ea98..a7f33cb81 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -35,7 +35,15 @@ "policyengine-api-prod-github-microdata-token:latest" ), "OPENAI_API_KEY": "policyengine-api-prod-openai-api-key:latest", - "HUGGING_FACE_TOKEN": "policyengine-api-prod-hugging-face-token:latest", + "HUGGING_FACE_TOKEN": "pe-uk-private-hf-read-token:latest", +} +SYNCED_CLOUD_RUN_SECRETS = { + "POLICYENGINE_DB_PASSWORD": "policyengine-api-prod-db-password", + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN": ( + "policyengine-api-prod-github-microdata-token" + ), + "OPENAI_API_KEY": "policyengine-api-prod-openai-api-key", + "PE_UK_PRIVATE_HF_READ_TOKEN": "pe-uk-private-hf-read-token", } RAW_CLOUD_RUN_SECRET_VALUES = ( "raw-db-secret-value", @@ -103,7 +111,7 @@ def _required_runtime_env() -> dict[str, str]: "POLICYENGINE_DB_PASSWORD": "raw-db-secret-value", "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN": ("raw-github-secret-value"), "OPENAI_API_KEY": "raw-openai-secret-value", - "HUGGING_FACE_TOKEN": "raw-hf-secret-value", + "PE_UK_PRIVATE_HF_READ_TOKEN": "raw-hf-secret-value", "SIMULATION_ENTRYPOINT_URL": "https://simulation.example.test", "OLD_SIMULATION_GATEWAY_URL": "https://old-gateway.example.test", "SIM_ENTRYPOINT": "cloud_run_simulation_entrypoint", @@ -164,6 +172,12 @@ def _fake_gcloud(tmp_path: Path) -> tuple[Path, Path]: ), }, "candidate_cloud_sql": PRODUCTION_CLOUD_SQL_INSTANCE, + "candidate_secret_env": { + "HUGGING_FACE_TOKEN": { + "name": "pe-uk-private-hf-read-token", + "key": "latest", + } + }, "updates": [], } ), @@ -221,6 +235,15 @@ def _fake_gcloud(tmp_path: Path) -> tuple[Path, Path]: for name, value in state.get( "candidate_env", {} ).items() + ] + + [ + { + "name": name, + "valueFrom": {"secretKeyRef": reference}, + } + for name, reference in state.get( + "candidate_secret_env", {} + ).items() ], } ] @@ -1220,6 +1243,24 @@ def test_resolve_cloud_run_candidate_records_exact_ready_revision_and_image(tmp_ ] +def test_resolve_cloud_run_candidate_rejects_wrong_hugging_face_secret(tmp_path): + gcloud_path, state_path = _fake_gcloud(tmp_path) + state = json.loads(state_path.read_text(encoding="utf-8")) + state["candidate_secret_env"]["HUGGING_FACE_TOKEN"]["name"] = ( + "policyengine-api-prod-hugging-face-token" + ) + state_path.write_text(json.dumps(state), encoding="utf-8") + + result = _run_script( + ".github/scripts/resolve_cloud_run_candidate_state.sh", + _fake_gcloud_env(gcloud_path, state_path), + ) + + assert result.returncode == 2 + assert "binds HUGGING_FACE_TOKEN" in result.stderr + assert "expected pe-uk-private-hf-read-token:latest" in result.stderr + + def test_resolve_cloud_run_candidate_rejects_changed_revision(tmp_path): gcloud_path, state_path = _fake_gcloud(tmp_path) @@ -1986,7 +2027,7 @@ def test_push_workflow_does_not_pass_raw_secrets_to_cloud_run_deploy_commands(): "${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}" ), "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}", - "HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}", + "HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}", ) for raw_secret_env in raw_secret_envs: @@ -2012,7 +2053,7 @@ def test_push_workflow_release_test_step_is_the_only_raw_secret_consumer(): "${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}" ), "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}", - "HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}", + "HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}", ) for raw_secret_env in raw_secret_envs: assert release_test_step.count(raw_secret_env) == 1 @@ -2044,12 +2085,37 @@ def test_sync_cloud_run_secrets_workflow_writes_expected_secret_versions(): assert "run: bash .github/scripts/sync_cloud_run_secrets.sh" in workflow assert "set +x" in script assert "--data-file=-" in script + assert "gcloud secrets versions access latest" in script assert "gcloud secrets add-iam-policy-binding" in script + assert "gcloud secrets get-iam-policy" in script + assert "gcloud run services describe" in script assert "roles/secretmanager.secretAccessor" in script - for env_name, secret_ref in CLOUD_RUN_SECRET_MAPPINGS.items(): - secret_name = secret_ref.removesuffix(":latest") + for env_name, secret_name in SYNCED_CLOUD_RUN_SECRETS.items(): assert f"{env_name}: ${{{{ secrets.{env_name} }}}}" in workflow assert f"sync_secret {env_name} {secret_name}" in script + assert ( + "verify_secret_value PE_UK_PRIVATE_HF_READ_TOKEN " + "pe-uk-private-hf-read-token" in script + ) + assert "pe-uk-private-hf-read-token policyengine-api-staging" in script + assert "pe-uk-private-hf-read-token policyengine-api" in script + + +def test_hugging_face_credential_uses_purpose_specific_managed_names(): + workflow_sources = "\n".join( + ( + _pr_workflow(), + _push_workflow(), + _sync_secrets_workflow(), + _sync_secrets_script(), + (REPO / ".github/scripts/cloud_run_env.sh").read_text(encoding="utf-8"), + ) + ) + + assert "secrets.HUGGING_FACE_TOKEN" not in workflow_sources + assert "secrets.PE_UK_PRIVATE_HF_READ_TOKEN" in workflow_sources + assert "policyengine-api-prod-hugging-face-token" not in workflow_sources + assert "pe-uk-private-hf-read-token" in workflow_sources def test_sync_cloud_run_secrets_script_is_shell_syntax_valid(): From 08c833887efc6fbcb8d11cc0ef44058b7cd34b2c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:03:09 +0400 Subject: [PATCH 02/22] Synchronize Cloud Run secret batch before deployment --- .github/scripts/sync_cloud_run_secrets.sh | 120 +++++--- .github/workflows/push.yml | 15 + .github/workflows/sync-cloud-run-secrets.yml | 30 +- changelog.d/uk-private-hf-read-token.fixed.md | 2 +- .../uk-private-hugging-face-credential.md | 28 +- tests/unit/test_cloud_run_deploy_scripts.py | 273 ++++++++++++++++-- 6 files changed, 393 insertions(+), 75 deletions(-) diff --git a/.github/scripts/sync_cloud_run_secrets.sh b/.github/scripts/sync_cloud_run_secrets.sh index 2a1626bb6..3ee45fa82 100644 --- a/.github/scripts/sync_cloud_run_secrets.sh +++ b/.github/scripts/sync_cloud_run_secrets.sh @@ -5,6 +5,20 @@ set +x CLOUD_RUN_PROJECT="${CLOUD_RUN_PROJECT:-policyengine-api}" CLOUD_RUN_REGION="${CLOUD_RUN_REGION:-us-central1}" +GCLOUD_BIN="${GCLOUD_BIN:-gcloud}" +include_database_password=0 +case "${1:-}" in + "") ;; + --include-database-password) include_database_password=1 ;; + *) + echo "::error::Unknown argument: $1" >&2 + exit 2 + ;; +esac +if [[ "$#" -gt 1 ]]; then + echo "::error::Expected at most one argument." >&2 + exit 2 +fi require_env() { local env_name="$1" @@ -18,72 +32,78 @@ sync_secret() { local env_name="$1" local secret_name="$2" local secret_value="${!env_name:-}" + local expected_hash + local stored_hash="" if [[ -z "${secret_value}" ]]; then echo "::error::Missing required GitHub secret ${env_name}." exit 1 fi - if ! gcloud secrets describe "${secret_name}" \ + if ! "${GCLOUD_BIN}" secrets describe "${secret_name}" \ --project "${CLOUD_RUN_PROJECT}" >/dev/null 2>&1; then - gcloud secrets create "${secret_name}" \ + "${GCLOUD_BIN}" secrets create "${secret_name}" \ --project "${CLOUD_RUN_PROJECT}" \ --replication-policy automatic fi - printf '%s' "${secret_value}" | gcloud secrets versions add \ - "${secret_name}" \ - --project "${CLOUD_RUN_PROJECT}" \ - --data-file=- >/dev/null - - gcloud secrets add-iam-policy-binding "${secret_name}" \ - --project "${CLOUD_RUN_PROJECT}" \ - --member "serviceAccount:${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" \ - --role roles/secretmanager.secretAccessor >/dev/null + expected_hash="$(printf '%s' "${secret_value}" | sha256sum | cut -d ' ' -f 1)" + if stored_hash="$( + "${GCLOUD_BIN}" secrets versions access latest \ + --secret "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}" 2>/dev/null \ + | sha256sum \ + | cut -d ' ' -f 1 + )" && [[ "${stored_hash}" == "${expected_hash}" ]]; then + echo "Secret Manager already matches ${env_name}; no version added." + else + printf '%s' "${secret_value}" | "${GCLOUD_BIN}" secrets versions add \ + "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}" \ + --data-file=- >/dev/null + echo "Synchronized ${env_name} to Secret Manager secret ${secret_name}." + fi - echo "Synced ${env_name} to Secret Manager secret ${secret_name}." + verify_secret_value "${env_name}" "${secret_name}" + grant_secret_access "${secret_name}" + unset secret_value expected_hash stored_hash } verify_secret_value() { local env_name="$1" local secret_name="$2" local expected_value="${!env_name:-}" - local stored_value + local expected_hash + local stored_hash - stored_value="$(gcloud secrets versions access latest \ - --secret "${secret_name}" \ - --project "${CLOUD_RUN_PROJECT}")" - if [[ "${stored_value}" != "${expected_value}" ]]; then + expected_hash="$(printf '%s' "${expected_value}" | sha256sum | cut -d ' ' -f 1)" + stored_hash="$( + "${GCLOUD_BIN}" secrets versions access latest \ + --secret "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}" \ + | sha256sum \ + | cut -d ' ' -f 1 + )" + if [[ "${stored_hash}" != "${expected_hash}" ]]; then echo "::error::Secret Manager value verification failed for ${secret_name}." >&2 return 1 fi - unset stored_value expected_value + unset stored_hash expected_hash expected_value echo "Verified ${env_name} in Secret Manager without printing its value." } -grant_secret_access_to_cloud_run_service() { +grant_secret_access() { local secret_name="$1" - local service_name="$2" - local runtime_service_account local member - runtime_service_account="$(gcloud run services describe "${service_name}" \ - --project "${CLOUD_RUN_PROJECT}" \ - --region "${CLOUD_RUN_REGION}" \ - --platform managed \ - --format='value(spec.template.spec.serviceAccountName)')" - if [[ -z "${runtime_service_account}" ]]; then - echo "::error::Cloud Run service ${service_name} has no runtime service account." >&2 - return 1 - fi member="serviceAccount:${runtime_service_account}" - gcloud secrets add-iam-policy-binding "${secret_name}" \ + "${GCLOUD_BIN}" secrets add-iam-policy-binding "${secret_name}" \ --project "${CLOUD_RUN_PROJECT}" \ --member "${member}" \ --role roles/secretmanager.secretAccessor >/dev/null - if ! gcloud secrets get-iam-policy "${secret_name}" \ + if ! "${GCLOUD_BIN}" secrets get-iam-policy "${secret_name}" \ --project "${CLOUD_RUN_PROJECT}" \ --format=json \ | jq -e --arg member "${member}" ' @@ -93,20 +113,38 @@ grant_secret_access_to_cloud_run_service() { and ((.members // []) | index($member) != null) ) ' >/dev/null; then - echo "::error::Secret Manager access verification failed for ${service_name}." >&2 + echo "::error::Secret Manager access verification failed for ${CLOUD_RUN_SERVICE}." >&2 return 1 fi - echo "Verified ${service_name} can read ${secret_name}." + echo "Verified ${CLOUD_RUN_SERVICE} can read ${secret_name}." } -require_env CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT +require_env CLOUD_RUN_SERVICE -sync_secret POLICYENGINE_DB_PASSWORD policyengine-api-prod-db-password +runtime_service_account="$( + "${GCLOUD_BIN}" run services describe "${CLOUD_RUN_SERVICE}" \ + --project "${CLOUD_RUN_PROJECT}" \ + --region "${CLOUD_RUN_REGION}" \ + --platform managed \ + --format='value(spec.template.spec.serviceAccountName)' +)" +if [[ -z "${runtime_service_account}" ]]; then + echo "::error::Cloud Run service ${CLOUD_RUN_SERVICE} has no runtime service account." >&2 + exit 1 +fi + +if [[ "${include_database_password}" -eq 1 ]]; then + require_env CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET + db_password_secret_version="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET##*:}" + db_password_secret_resource="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET%:*}" + db_password_secret_name="${db_password_secret_resource##*/}" + if [[ "${db_password_secret_version}" != "latest" \ + || -z "${db_password_secret_name}" ]]; then + echo "::error::CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET must name a :latest secret version." >&2 + exit 1 + fi + sync_secret POLICYENGINE_DB_PASSWORD "${db_password_secret_name}" +fi sync_secret POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN policyengine-api-prod-github-microdata-token sync_secret OPENAI_API_KEY policyengine-api-prod-openai-api-key sync_secret PE_UK_PRIVATE_HF_READ_TOKEN pe-uk-private-hf-read-token -verify_secret_value PE_UK_PRIVATE_HF_READ_TOKEN pe-uk-private-hf-read-token -grant_secret_access_to_cloud_run_service \ - pe-uk-private-hf-read-token policyengine-api-staging -grant_secret_access_to_cloud_run_service \ - pe-uk-private-hf-read-token policyengine-api diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 7c0dd22c9..a602c0138 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -255,6 +255,14 @@ jobs: uses: "google-github-actions/setup-gcloud@v2" - name: Install jq run: sudo apt-get install -y jq + # The staging database password is managed independently in Secret Manager + # and differs from the repository-level production database password. + - name: Synchronize shared Cloud Run runtime secrets + env: + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + run: bash .github/scripts/sync_cloud_run_secrets.sh - name: Capture current Cloud Run staging state id: previous run: bash .github/scripts/capture_cloud_run_service_state.sh >> "$GITHUB_OUTPUT" @@ -489,6 +497,13 @@ jobs: uses: "google-github-actions/setup-gcloud@v2" - name: Install jq run: sudo apt-get install -y jq + - name: Synchronize complete Cloud Run runtime secret batch + env: + POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + run: bash .github/scripts/sync_cloud_run_secrets.sh --include-database-password - name: Capture current Cloud Run production state id: previous run: bash .github/scripts/capture_cloud_run_service_state.sh >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/sync-cloud-run-secrets.yml b/.github/workflows/sync-cloud-run-secrets.yml index 5a1bbc372..c6b46f80a 100644 --- a/.github/workflows/sync-cloud-run-secrets.yml +++ b/.github/workflows/sync-cloud-run-secrets.yml @@ -2,6 +2,15 @@ name: Sync Cloud Run secrets on: workflow_dispatch: + inputs: + deployment_environment: + description: GitHub and Cloud Run environment to synchronize + required: true + default: production + type: choice + options: + - staging + - production concurrency: group: cloud-run-secret-sync @@ -10,7 +19,7 @@ jobs: sync-cloud-run-secrets: name: Sync GitHub secrets to Secret Manager runs-on: ubuntu-latest - environment: production + environment: ${{ inputs.deployment_environment }} permissions: contents: read id-token: write @@ -29,12 +38,25 @@ jobs: service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}" - name: Set up GCloud uses: "google-github-actions/setup-gcloud@v2" - - name: Sync runtime secrets + - name: Install jq + run: sudo apt-get install -y jq + - name: Sync shared staging runtime secrets + if: inputs.deployment_environment == 'staging' env: CLOUD_RUN_PROJECT: policyengine-api - CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT: ${{ secrets.GCP_CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT }} - POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} + CLOUD_RUN_SERVICE: policyengine-api-staging POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} run: bash .github/scripts/sync_cloud_run_secrets.sh + - name: Sync complete production runtime secret batch + if: inputs.deployment_environment == 'production' + env: + CLOUD_RUN_PROJECT: policyengine-api + CLOUD_RUN_SERVICE: policyengine-api + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} + POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + run: bash .github/scripts/sync_cloud_run_secrets.sh --include-database-password diff --git a/changelog.d/uk-private-hf-read-token.fixed.md b/changelog.d/uk-private-hf-read-token.fixed.md index dd46feb49..50f27be5a 100644 --- a/changelog.d/uk-private-hf-read-token.fixed.md +++ b/changelog.d/uk-private-hf-read-token.fixed.md @@ -1 +1 @@ -Use a dedicated read-only Hugging Face credential for private UK runtime data and verify its Cloud Run secret binding during deployment. +Use a dedicated read-only Hugging Face credential for private UK runtime data, synchronize GitHub-owned runtime secrets before Cloud Run deployment, and verify their values and access bindings. diff --git a/docs/engineering/uk-private-hugging-face-credential.md b/docs/engineering/uk-private-hugging-face-credential.md index 123cfdc5d..1bc1c6134 100644 --- a/docs/engineering/uk-private-hugging-face-credential.md +++ b/docs/engineering/uk-private-hugging-face-credential.md @@ -20,14 +20,26 @@ credentials for that repository are separate. The selected-repository GitHub organization secret `PE_UK_PRIVATE_HF_READ_TOKEN` is copied to the Google Secret Manager resource -`pe-uk-private-hf-read-token`. Both API v1 Cloud Run runtime service accounts -receive `roles/secretmanager.secretAccessor` for that resource. The -synchronization script resolves the runtime identities from the deployed -`policyengine-api-staging` and `policyengine-api` services rather than relying -on a duplicated account-name configuration value, then verifies both IAM -bindings. -The synchronization workflow reads the newly stored version back and compares -it in memory with the GitHub value without logging either value. +`pe-uk-private-hf-read-token`. It is one member of the GitHub-owned runtime +secret batch, alongside the environment-specific database password and the +shared GitHub microdata and OpenAI credentials. + +Each staging and production deployment synchronizes the shared three-secret +batch before deploying its Cloud Run candidate. Production additionally +synchronizes the existing GitHub-owned database password to +`policyengine-api-prod-db-password`. Staging deliberately leaves +`policyengine-api-staging-db-password` untouched: its value differs from the +production password and its canonical value is managed directly in Secret +Manager rather than duplicated in GitHub. Synchronization compares SHA-256 +digests without logging either value and creates a new secret version only when +the value changed. It then reads back and verifies every synchronized value. + +The synchronization script resolves the runtime identity from the deployed +`policyengine-api-staging` or `policyengine-api` service. It grants that identity +`roles/secretmanager.secretAccessor` on every resource synchronized for that +environment and verifies each IAM binding. A manually dispatched workflow +performs the corresponding staging or production operation when credentials +need to be rotated independently of a deployment. Cloud Run exposes the resource as `HUGGING_FACE_TOKEN` because PolicyEngine Core reads that compatibility environment variable. `HUGGING_FACE_TOKEN` is diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index a7f33cb81..ec1b334f4 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -37,8 +37,7 @@ "OPENAI_API_KEY": "policyengine-api-prod-openai-api-key:latest", "HUGGING_FACE_TOKEN": "pe-uk-private-hf-read-token:latest", } -SYNCED_CLOUD_RUN_SECRETS = { - "POLICYENGINE_DB_PASSWORD": "policyengine-api-prod-db-password", +SYNCED_SHARED_CLOUD_RUN_SECRETS = { "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN": ( "policyengine-api-prod-github-microdata-token" ), @@ -130,9 +129,13 @@ def _required_runtime_env() -> dict[str, str]: } -def _run_script(path: str, env: dict[str, str]) -> subprocess.CompletedProcess[str]: +def _run_script( + path: str, + env: dict[str, str], + *args: str, +) -> subprocess.CompletedProcess[str]: return subprocess.run( - ["bash", path], + ["bash", path, *args], cwd=REPO, env=env, text=True, @@ -290,6 +293,91 @@ def _fake_gcloud_env(gcloud_path: Path, state_path: Path) -> dict[str, str]: ) +def _fake_secret_sync_gcloud(tmp_path: Path) -> tuple[Path, Path]: + state_path = tmp_path / "secret-sync-state.json" + state_path.write_text( + json.dumps( + { + "services": { + STAGING_CLOUD_RUN_SERVICE: ( + "policyengine-api-cr-staging@policyengine-api.iam." + "gserviceaccount.com" + ), + PRODUCTION_CLOUD_RUN_SERVICE: ( + "policyengine-api-cr-runtime@policyengine-api.iam." + "gserviceaccount.com" + ), + }, + "secrets": { + "policyengine-api-staging-db-password": "db-value", + "policyengine-api-prod-db-password": "db-value", + "policyengine-api-prod-openai-api-key": "old-openai-value", + "pe-uk-private-hf-read-token": "hf-value", + }, + "iam": {}, + "version_adds": [], + } + ), + encoding="utf-8", + ) + gcloud_path = tmp_path / "gcloud-secret-sync" + gcloud_path.write_text( + """#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path + +state_path = Path(os.environ["FAKE_SECRET_SYNC_STATE"]) +state = json.loads(state_path.read_text(encoding="utf-8")) +args = sys.argv[1:] + +if args[:3] == ["run", "services", "describe"]: + print(state["services"][args[3]]) +elif args[:2] == ["secrets", "describe"]: + raise SystemExit(0 if args[2] in state["secrets"] else 1) +elif args[:2] == ["secrets", "create"]: + state["secrets"][args[2]] = "" +elif args[:3] == ["secrets", "versions", "access"]: + secret_name = args[args.index("--secret") + 1] + if secret_name not in state["secrets"]: + raise SystemExit(1) + print(state["secrets"][secret_name], end="") +elif args[:3] == ["secrets", "versions", "add"]: + secret_name = args[3] + state["secrets"][secret_name] = sys.stdin.read() + state["version_adds"].append(secret_name) +elif args[:2] == ["secrets", "add-iam-policy-binding"]: + secret_name = args[2] + member = args[args.index("--member") + 1] + members = state["iam"].setdefault(secret_name, []) + if member not in members: + members.append(member) +elif args[:2] == ["secrets", "get-iam-policy"]: + secret_name = args[2] + print( + json.dumps( + { + "bindings": [ + { + "role": "roles/secretmanager.secretAccessor", + "members": state["iam"].get(secret_name, []), + } + ] + } + ) + ) +else: + raise SystemExit(f"unexpected gcloud arguments: {args}") + +state_path.write_text(json.dumps(state), encoding="utf-8") +""", + encoding="utf-8", + ) + gcloud_path.chmod(0o755) + return gcloud_path, state_path + + def _run_simulation_version_guard( versions_response: dict, *args: str, @@ -2020,6 +2108,16 @@ def test_push_workflow_does_not_pass_raw_secrets_to_cloud_run_deploy_commands(): staging_deploy_start, ) staging_deploy = cloud_run_staging[staging_deploy_start:staging_deploy_end] + production_deploy_start = cloud_run_production.index( + "- name: Deploy tagged Cloud Run candidate" + ) + production_deploy_end = cloud_run_production.index( + "- name: Resolve exact Cloud Run production candidate", + production_deploy_start, + ) + production_deploy = cloud_run_production[ + production_deploy_start:production_deploy_end + ] raw_secret_envs = ( "POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}", ( @@ -2032,10 +2130,10 @@ def test_push_workflow_does_not_pass_raw_secrets_to_cloud_run_deploy_commands(): for raw_secret_env in raw_secret_envs: assert raw_secret_env not in staging_deploy - assert raw_secret_env not in cloud_run_production + assert raw_secret_env not in production_deploy -def test_push_workflow_release_test_step_is_the_only_raw_secret_consumer(): +def test_push_workflow_syncs_environment_specific_secrets_before_deployment(): workflow = _push_workflow() cloud_run_staging = _workflow_job_block(workflow, "deploy-cloud-run-staging") cloud_run_production = _workflow_job_block(workflow, "deploy-cloud-run-candidate") @@ -2045,8 +2143,26 @@ def test_push_workflow_release_test_step_is_the_only_raw_secret_consumer(): test_step_start, ) release_test_step = cloud_run_staging[test_step_start:test_step_end] + staging_sync_start = cloud_run_staging.index( + "- name: Synchronize shared Cloud Run runtime secrets" + ) + staging_sync_end = cloud_run_staging.index( + "- name: Capture current Cloud Run staging state", + staging_sync_start, + ) + staging_sync_step = cloud_run_staging[staging_sync_start:staging_sync_end] + production_sync_start = cloud_run_production.index( + "- name: Synchronize complete Cloud Run runtime secret batch" + ) + production_sync_end = cloud_run_production.index( + "- name: Capture current Cloud Run production state", + production_sync_start, + ) + production_sync_step = cloud_run_production[ + production_sync_start:production_sync_end + ] - raw_secret_envs = ( + release_test_secret_envs = ( "POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}", ( "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: " @@ -2055,10 +2171,30 @@ def test_push_workflow_release_test_step_is_the_only_raw_secret_consumer(): "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}", "HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}", ) - for raw_secret_env in raw_secret_envs: + shared_sync_secret_envs = ( + ( + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: " + "${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}" + ), + "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}", + ("PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}"), + ) + for raw_secret_env in release_test_secret_envs: assert release_test_step.count(raw_secret_env) == 1 - assert cloud_run_staging.count(raw_secret_env) == 1 - assert raw_secret_env not in cloud_run_production + for raw_secret_env in shared_sync_secret_envs: + assert staging_sync_step.count(raw_secret_env) == 1 + assert production_sync_step.count(raw_secret_env) == 1 + database_secret_env = ( + "POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}" + ) + assert database_secret_env not in staging_sync_step + assert production_sync_step.count(database_secret_env) == 1 + assert "run: bash .github/scripts/sync_cloud_run_secrets.sh\n" in staging_sync_step + assert "--include-database-password" not in staging_sync_step + assert ( + "run: bash .github/scripts/sync_cloud_run_secrets.sh " + "--include-database-password" in production_sync_step + ) def test_sync_cloud_run_secrets_workflow_is_manual_and_environment_gated(): @@ -2067,7 +2203,10 @@ def test_sync_cloud_run_secrets_workflow_is_manual_and_environment_gated(): assert "workflow_dispatch:" in workflow assert "pull_request:" not in workflow assert "push:" not in workflow - assert "environment: production" in workflow + assert "deployment_environment:" in workflow + assert "- staging" in workflow + assert "- production" in workflow + assert "environment: ${{ inputs.deployment_environment }}" in workflow assert "id-token: write" in workflow assert "github.ref != 'refs/heads/master'" in workflow assert "google-github-actions/auth@v2" in workflow @@ -2085,20 +2224,112 @@ def test_sync_cloud_run_secrets_workflow_writes_expected_secret_versions(): assert "run: bash .github/scripts/sync_cloud_run_secrets.sh" in workflow assert "set +x" in script assert "--data-file=-" in script - assert "gcloud secrets versions access latest" in script - assert "gcloud secrets add-iam-policy-binding" in script - assert "gcloud secrets get-iam-policy" in script - assert "gcloud run services describe" in script + assert "Secret Manager already matches" in script + assert "secrets versions access latest" in script + assert "secrets add-iam-policy-binding" in script + assert "secrets get-iam-policy" in script + assert "run services describe" in script assert "roles/secretmanager.secretAccessor" in script - for env_name, secret_name in SYNCED_CLOUD_RUN_SECRETS.items(): + assert ( + "CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: " + "${{ vars.CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }}" in workflow + ) + assert 'sync_secret POLICYENGINE_DB_PASSWORD "${db_password_secret_name}"' in script + for env_name, secret_name in SYNCED_SHARED_CLOUD_RUN_SECRETS.items(): assert f"{env_name}: ${{{{ secrets.{env_name} }}}}" in workflow assert f"sync_secret {env_name} {secret_name}" in script - assert ( - "verify_secret_value PE_UK_PRIVATE_HF_READ_TOKEN " - "pe-uk-private-hf-read-token" in script + assert "verify_secret_value" in script + assert "grant_secret_access" in script + + +def test_sync_cloud_run_secrets_is_idempotent_and_grants_complete_batch(tmp_path): + gcloud_path, state_path = _fake_secret_sync_gcloud(tmp_path) + secret_values = { + "POLICYENGINE_DB_PASSWORD": "db-value", + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN": "microdata-value", + "OPENAI_API_KEY": "openai-value", + "PE_UK_PRIVATE_HF_READ_TOKEN": "hf-value", + } + env = _script_env( + GCLOUD_BIN=str(gcloud_path), + FAKE_SECRET_SYNC_STATE=str(state_path), + CLOUD_RUN_PROJECT="policyengine-api", + CLOUD_RUN_REGION="us-central1", + CLOUD_RUN_SERVICE=PRODUCTION_CLOUD_RUN_SERVICE, + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET=( + "policyengine-api-prod-db-password:latest" + ), + **secret_values, + ) + + first_result = _run_script( + ".github/scripts/sync_cloud_run_secrets.sh", + env, + "--include-database-password", + ) + assert first_result.returncode == 0, first_result.stderr + first_state = json.loads(state_path.read_text(encoding="utf-8")) + assert first_state["version_adds"] == [ + "policyengine-api-prod-github-microdata-token", + "policyengine-api-prod-openai-api-key", + ] + assert first_state["secrets"] == { + "policyengine-api-staging-db-password": "db-value", + "policyengine-api-prod-db-password": "db-value", + "policyengine-api-prod-github-microdata-token": "microdata-value", + "policyengine-api-prod-openai-api-key": "openai-value", + "pe-uk-private-hf-read-token": "hf-value", + } + expected_member = ( + "serviceAccount:policyengine-api-cr-runtime@policyengine-api.iam." + "gserviceaccount.com" + ) + synchronized_secret_names = { + "policyengine-api-prod-db-password", + "policyengine-api-prod-github-microdata-token", + "policyengine-api-prod-openai-api-key", + "pe-uk-private-hf-read-token", + } + assert set(first_state["iam"]) == synchronized_secret_names + assert all(members == [expected_member] for members in first_state["iam"].values()) + assert all(value not in first_result.stdout for value in secret_values.values()) + + second_result = _run_script( + ".github/scripts/sync_cloud_run_secrets.sh", + env, + "--include-database-password", ) - assert "pe-uk-private-hf-read-token policyengine-api-staging" in script - assert "pe-uk-private-hf-read-token policyengine-api" in script + assert second_result.returncode == 0, second_result.stderr + second_state = json.loads(state_path.read_text(encoding="utf-8")) + assert second_state["version_adds"] == first_state["version_adds"] + + +def test_sync_cloud_run_staging_batch_preserves_distinct_database_secret(tmp_path): + gcloud_path, state_path = _fake_secret_sync_gcloud(tmp_path) + env = _script_env( + GCLOUD_BIN=str(gcloud_path), + FAKE_SECRET_SYNC_STATE=str(state_path), + CLOUD_RUN_PROJECT="policyengine-api", + CLOUD_RUN_REGION="us-central1", + CLOUD_RUN_SERVICE=STAGING_CLOUD_RUN_SERVICE, + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN="microdata-value", + OPENAI_API_KEY="openai-value", + PE_UK_PRIVATE_HF_READ_TOKEN="hf-value", + ) + + result = _run_script( + ".github/scripts/sync_cloud_run_secrets.sh", + env, + ) + assert result.returncode == 0, result.stderr + state = json.loads(state_path.read_text(encoding="utf-8")) + assert state["secrets"]["policyengine-api-staging-db-password"] == "db-value" + assert "policyengine-api-staging-db-password" not in state["iam"] + assert set(state["iam"]) == { + "policyengine-api-prod-github-microdata-token", + "policyengine-api-prod-openai-api-key", + "pe-uk-private-hf-read-token", + } def test_hugging_face_credential_uses_purpose_specific_managed_names(): From 16840afbfd4c88d865e8ccfe19fd75d367b18a14 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:59:33 +0400 Subject: [PATCH 03/22] Define compact SPM provenance contracts --- policyengine_api/spm.py | 233 ++++++++++++++++++++++++++---- tests/unit/test_spm_provenance.py | 194 +++++++++++++++++++++++++ tests/unit/test_spm_settings.py | 32 ++-- 3 files changed, 422 insertions(+), 37 deletions(-) create mode 100644 tests/unit/test_spm_provenance.py diff --git a/policyengine_api/spm.py b/policyengine_api/spm.py index c3c627ea9..f4970ad55 100644 --- a/policyengine_api/spm.py +++ b/policyengine_api/spm.py @@ -8,9 +8,10 @@ import importlib import inspect +from collections.abc import Mapping, Sequence from functools import lru_cache from datetime import date -from typing import Any, Literal +from typing import Literal from pydantic import ( BaseModel, @@ -97,18 +98,197 @@ def validate_location(self): class SPMProvenance(BaseModel): - """JSON receipt matching the country's public provenance contract.""" + """Compact public receipt for one canonical SPM execution.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + schema_version: Literal["canonical-spm-provenance-v2"] + forecast_id: str = Field(min_length=1, pattern=r"^\S+$") + forecast_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + scenario: str = Field(min_length=1, pattern=r"^\S+$") + geography_kind: Literal["county", "national", "metro"] + geography_id: str | None = Field(min_length=1) + county_vintage: str = Field(pattern=r"^[0-9]{4}$") + as_of: date | None + years: tuple[str, ...] + runtime_versions: "SPMRuntimeVersions" + + @field_validator("years") + @classmethod + def validate_years(cls, years: tuple[str, ...]) -> tuple[str, ...]: + if any( + len(year) != 4 or not year.isascii() or not year.isdigit() for year in years + ): + raise ValueError("years must contain four-digit calendar years") + if tuple(sorted(set(years))) != years: + raise ValueError("years must be sorted and unique") + return years + + @model_validator(mode="after") + def validate_location(self) -> "SPMProvenance": + if self.geography_kind == "metro": + if self.geography_id is None or not self.geography_id.strip(): + raise ValueError("A metro SPM receipt requires geography_id") + elif self.geography_id is not None: + raise ValueError("Only a metro SPM receipt accepts geography_id") + return self + + +class SPMRuntimeVersions(BaseModel): + """Package versions required to reproduce one certified SPM execution.""" + + model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True) + policyengine: str | None + policyengine_core: str | None = Field(alias="policyengine-core") + policyengine_us: str | None = Field(alias="policyengine-us") + spm_calculator: str | None = Field(alias="spm-calculator") + + +SPMProvenance.model_rebuild() + + +class SPMCalculationProvenance(BaseModel): + """A resolved SPM configuration and its matching compact receipt.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + spm_config: SPMSelection + spm_provenance: SPMProvenance + + +class SPMExecutionProvenance(BaseModel): + """One shared compact receipt and the executions that produced it.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + receipt: SPMProvenance + execution_count: int = Field(ge=1) + + +class SPMComparisonProvenance(BaseModel): + """Compact baseline and reform receipts for a society-wide result.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + schema_version: Literal["canonical-spm-comparison-v2"] + baseline: SPMExecutionProvenance + reform: SPMExecutionProvenance + + @model_validator(mode="after") + def validate_compatible_receipts(self) -> "SPMComparisonProvenance": + if self.baseline.receipt != self.reform.receipt: + raise ValueError("baseline and reform SPM receipts differ") + return self - model_config = ConfigDict(extra="forbid") - forecast_id: str - forecast_sha256: str - scenario: str - geography_kind: str - runtime_versions: dict[str, str | None] - years: dict[str, dict[str, Any]] - geographies: list[dict[str, Any]] - composition_method: str - storage_method: str + +def build_spm_provenance( + *, + forecast_id: str, + forecast_sha256: str, + selection: SPMSelection, + years: Sequence[str], + runtime_versions: SPMRuntimeVersions, +) -> SPMProvenance: + """Build one compact receipt from typed calculation inputs.""" + + if selection.forecast_content_sha256 != forecast_sha256: + raise ValueError("SPM receipt differs from the SPM configuration") + if selection.scenario is None: + raise ValueError("Resolved SPM configuration must include scenario") + return SPMProvenance( + schema_version="canonical-spm-provenance-v2", + forecast_id=forecast_id, + forecast_sha256=forecast_sha256, + scenario=selection.scenario, + geography_kind=selection.geography_kind, + geography_id=selection.geography_id, + county_vintage=selection.county_vintage, + as_of=selection.as_of, + years=tuple(years), + runtime_versions=runtime_versions, + ) + + +def _required_string(source: Mapping[str, object], field: str) -> str: + value = source.get(field) + if not isinstance(value, str) or not value: + raise ValueError(f"Country SPM receipt requires {field}") + return value + + +def build_spm_calculation_provenance( + config: SPMSelection, + receipt: Mapping[str, object], +) -> SPMCalculationProvenance: + """Reduce the country-owned rich receipt to the public compact contract.""" + + forecast_sha256 = _required_string(receipt, "forecast_sha256") + scenario = _required_string(receipt, "scenario") + geography_kind = _required_string(receipt, "geography_kind") + if ( + forecast_sha256 != config.forecast_content_sha256 + or scenario != config.scenario + or geography_kind != config.geography_kind + ): + raise ValueError("Country SPM receipt differs from the SPM configuration") + + source_years = receipt.get("years") + if not isinstance(source_years, Mapping): + raise ValueError("Country SPM receipt requires a years mapping") + years: list[str] = [] + for year in source_years: + if not isinstance(year, str): + raise ValueError("Country SPM receipt years must be strings") + years.append(year) + + source_versions = receipt.get("runtime_versions") + if not isinstance(source_versions, Mapping): + raise ValueError("Country SPM receipt requires runtime_versions") + runtime_versions = SPMRuntimeVersions.model_validate(source_versions) + if any( + version is None + for version in ( + runtime_versions.policyengine, + runtime_versions.policyengine_core, + runtime_versions.policyengine_us, + runtime_versions.spm_calculator, + ) + ): + raise ValueError("Certified SPM receipts require every runtime version") + + compact = build_spm_provenance( + forecast_id=_required_string(receipt, "forecast_id"), + forecast_sha256=forecast_sha256, + selection=config, + years=tuple(sorted(years)), + runtime_versions=runtime_versions, + ) + return SPMCalculationProvenance( + spm_config=config, + spm_provenance=compact, + ) + + +def _collapse_execution_receipts( + side: Literal["baseline", "reform"], + receipts: Sequence[SPMProvenance], +) -> SPMExecutionProvenance: + if not receipts: + raise ValueError(f"{side} SPM receipts are empty") + first = receipts[0] + if any(receipt != first for receipt in receipts[1:]): + raise ValueError(f"{side} SPM receipts differ") + return SPMExecutionProvenance(receipt=first, execution_count=len(receipts)) + + +def build_spm_comparison_provenance( + *, + baseline_receipts: Sequence[SPMProvenance], + reform_receipts: Sequence[SPMProvenance], +) -> SPMComparisonProvenance: + """Collapse matching child receipts into one comparison receipt.""" + + return SPMComparisonProvenance( + schema_version="canonical-spm-comparison-v2", + baseline=_collapse_execution_receipts("baseline", baseline_receipts), + reform=_collapse_execution_receipts("reform", reform_receipts), + ) class SPMValidationError(ValueError): @@ -413,25 +593,24 @@ def spm_metadata(country_id: str) -> dict: def calculation_spm_receipt(simulation) -> dict: - """Read existing calculation receipts; never request SPM calculations here.""" + """Reduce a country's existing receipt; never request calculations here.""" if not hasattr(simulation, "spm_config"): return {} - # A receipt this API cannot read in full is an uncertified country contract, - # not a caller error and not an internal failure. Report it as a typed - # configuration failure rather than letting pydantic surface a 500, and - # never publish a receipt with unrecognized fields dropped. try: - return { - "spm_config": SPMSelection.model_validate(simulation.spm_config).model_dump( - mode="json" - ), - "spm_provenance": SPMProvenance.model_validate( - simulation.spm_provenance() - ).model_dump(mode="json"), - } - except ValidationError as error: + resolved = resolved_spm_settings(simulation.spm_config) + if resolved is None: + raise ValueError("Country SPM configuration is incomplete") + source = simulation.spm_provenance() + if not isinstance(source, Mapping): + raise ValueError("Country SPM receipt must be an object") + calculation = build_spm_calculation_provenance( + SPMSelection.model_validate(resolved), + source, + ) + return calculation.model_dump(mode="json", by_alias=True) + except (ValidationError, ValueError) as error: raise SPMValidationError( "SPM_CONFIGURATION_UNAVAILABLE", "The installed country model reported an SPM receipt this API cannot " - f"certify: {readable_validation_error(error)}", + f"certify: {error_message(error)}", ) from error diff --git a/tests/unit/test_spm_provenance.py b/tests/unit/test_spm_provenance.py new file mode 100644 index 000000000..2a7147df2 --- /dev/null +++ b/tests/unit/test_spm_provenance.py @@ -0,0 +1,194 @@ +"""Compact canonical SPM provenance contracts.""" + +import json + +import pytest +from pydantic import ValidationError + +from policyengine_api.spm import ( + SPMComparisonProvenance, + SPMProvenance, + SPMRuntimeVersions, + SPMSelection, + build_spm_calculation_provenance, + build_spm_comparison_provenance, + build_spm_provenance, +) + + +SHA256 = "a" * 64 +SELECTION = SPMSelection.model_validate( + { + "forecast_content_sha256": SHA256, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + } +) +VERSIONS = SPMRuntimeVersions.model_validate( + { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0", + } +) + + +def _receipt(**changes: object) -> SPMProvenance: + values: dict[str, object] = { + "forecast_id": "spm-rolling-2026-09-09", + "forecast_sha256": SHA256, + "selection": SELECTION, + "years": ("2026",), + "runtime_versions": VERSIONS, + } + values.update(changes) + return build_spm_provenance(**values) + + +def test_compact_receipt_has_only_the_public_v2_fields() -> None: + receipt = _receipt() + + assert receipt.model_dump(mode="json", by_alias=True) == { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "spm-rolling-2026-09-09", + "forecast_sha256": SHA256, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": ["2026"], + "runtime_versions": { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0", + }, + } + assert len(receipt.model_dump_json(by_alias=True).encode()) < 1_024 + + +@pytest.mark.parametrize( + "changes", + [ + {"forecast_sha256": "A" * 64}, + {"years": ("26",)}, + {"years": ("2027", "2026")}, + {"years": ("2026", "2026")}, + ], +) +def test_compact_receipt_rejects_noncanonical_values( + changes: dict[str, object], +) -> None: + with pytest.raises((ValidationError, ValueError)): + _receipt(**changes) + + +def test_compact_receipt_rejects_the_previous_diagnostic_shape() -> None: + previous = { + "forecast_id": "spm-rolling-2026-09-09", + "forecast_sha256": SHA256, + "scenario": "ce_trend", + "geography_kind": "national", + "runtime_versions": {}, + "years": {"2026": {}}, + "geographies": [], + "composition_method": "classified inputs", + "storage_method": "formula", + } + + with pytest.raises(ValidationError): + SPMProvenance.model_validate(previous) + + +def test_country_receipt_adapter_extracts_only_compact_scalars() -> None: + country_receipt: dict[str, object] = { + "forecast_id": "spm-rolling-2026-09-09", + "forecast_sha256": SHA256, + "scenario": "ce_trend", + "geography_kind": "national", + "runtime_versions": VERSIONS.model_dump(mode="json", by_alias=True), + "years": {"2027": {"large": "diagnostic"}, "2026": {}}, + "geographies": [{"large": "mapping"}], + "composition_method": "classified inputs", + "storage_method": "formula", + } + + calculation = build_spm_calculation_provenance(SELECTION, country_receipt) + + assert calculation.spm_config == SELECTION + assert calculation.spm_provenance.years == ("2026", "2027") + encoded = calculation.model_dump_json(by_alias=True) + assert "diagnostic" not in encoded + assert "mapping" not in encoded + assert "composition_method" not in encoded + + +def test_country_receipt_must_agree_with_adjacent_configuration() -> None: + country_receipt: dict[str, object] = { + "forecast_id": "spm-rolling-2026-09-09", + "forecast_sha256": SHA256, + "scenario": "different", + "geography_kind": "national", + "runtime_versions": VERSIONS.model_dump(mode="json", by_alias=True), + "years": {}, + } + + with pytest.raises(ValueError, match="differs from the SPM configuration"): + build_spm_calculation_provenance(SELECTION, country_receipt) + + +def test_comparison_collapses_identical_child_receipts() -> None: + receipt = _receipt() + + comparison = build_spm_comparison_provenance( + baseline_receipts=(receipt,) * 20, + reform_receipts=(receipt,) * 20, + ) + + assert comparison.baseline.receipt == receipt + assert comparison.baseline.execution_count == 20 + assert comparison.reform.receipt == receipt + assert comparison.reform.execution_count == 20 + assert len(comparison.model_dump_json(by_alias=True).encode()) < 2_048 + + +@pytest.mark.parametrize("side", ["baseline", "reform"]) +def test_comparison_rejects_inconsistent_child_receipts(side: str) -> None: + receipt = _receipt() + different = _receipt(years=("2027",)) + inputs = { + "baseline_receipts": (receipt,), + "reform_receipts": (receipt,), + } + inputs[f"{side}_receipts"] = (receipt, different) + + with pytest.raises(ValueError, match=f"{side} SPM receipts differ"): + build_spm_comparison_provenance(**inputs) + + +def test_comparison_rejects_previous_list_shape() -> None: + receipt = _receipt().model_dump(mode="json", by_alias=True) + + with pytest.raises(ValidationError): + SPMComparisonProvenance.model_validate( + {"baseline": [receipt], "reform": [receipt]} + ) + + +def test_compact_models_survive_json_round_trip() -> None: + comparison = build_spm_comparison_provenance( + baseline_receipts=(_receipt(),), + reform_receipts=(_receipt(),), + ) + + assert ( + SPMComparisonProvenance.model_validate_json( + json.dumps(comparison.model_dump(mode="json", by_alias=True)) + ) + == comparison + ) diff --git a/tests/unit/test_spm_settings.py b/tests/unit/test_spm_settings.py index 615fba9d2..f22b3b75e 100644 --- a/tests/unit/test_spm_settings.py +++ b/tests/unit/test_spm_settings.py @@ -432,8 +432,8 @@ def test_resolved_settings_reject_unreadable_receipt_settings(settings): assert spm.resolved_spm_settings(settings) is None -def test_receipt_keeps_the_countrys_own_omissions(certified_bundle): - """A published receipt repeats the country's wire shape, nulls and all.""" +def test_receipt_expands_config_and_reduces_country_diagnostics(certified_bundle): + """The public receipt contains one resolved config and compact provenance.""" omitted = { key: value for key, value in RESOLVED_SELECTION.items() if value is not None } @@ -441,8 +441,19 @@ def test_receipt_keeps_the_countrys_own_omissions(certified_bundle): spm_config=omitted, spm_provenance=lambda: deepcopy(COUNTRY_RECEIPT) ) receipt = spm.calculation_spm_receipt(simulation) - assert receipt["spm_config"] == omitted - assert receipt["spm_provenance"] == COUNTRY_RECEIPT + assert receipt["spm_config"] == RESOLVED_SELECTION + assert receipt["spm_provenance"] == { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "test-artifact", + "forecast_sha256": ARTIFACT_HASH, + "scenario": "baseline", + "geography_kind": "county", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": [], + "runtime_versions": COUNTRY_RECEIPT["runtime_versions"], + } COUNTRY_RECEIPT = { @@ -450,7 +461,12 @@ def test_receipt_keeps_the_countrys_own_omissions(certified_bundle): "forecast_sha256": ARTIFACT_HASH, "scenario": "baseline", "geography_kind": "county", - "runtime_versions": {"policyengine-us": "test"}, + "runtime_versions": { + "policyengine": "test", + "policyengine-core": "test", + "policyengine-us": "test", + "spm-calculator": "test", + }, "years": {}, "geographies": [], "composition_method": "test composition", @@ -465,13 +481,9 @@ def test_receipt_keeps_the_countrys_own_omissions(certified_bundle): spm_config={**RESOLVED_SELECTION, "threshold_method": "canonical"}, spm_provenance=lambda: COUNTRY_RECEIPT, ), - SimpleNamespace( - spm_config=RESOLVED_SELECTION, - spm_provenance=lambda: {**COUNTRY_RECEIPT, "unreviewed_field": True}, - ), SimpleNamespace(spm_config=RESOLVED_SELECTION, spm_provenance=lambda: {}), ], - ids=["extra-setting", "extra-receipt-field", "incomplete-receipt"], + ids=["extra-setting", "incomplete-receipt"], ) def test_uncertifiable_country_receipt_is_typed_not_an_internal_failure(simulation): """An unreadable receipt is an uncertified country contract, never a 500.""" From db6da5e8872705f84932215bc5b386d75ca7556c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:10:41 +0400 Subject: [PATCH 04/22] Persist only compact household SPM receipts --- policyengine_api/routes/household_routes.py | 2 +- .../runtime_cache/household_calculations.py | 45 ++++++---- .../services/household_calculation_service.py | 46 +++++++++- .../services/simulation_service.py | 33 ++++++++ policyengine_api/spm.py | 31 +++++++ tests/integration/test_runtime_cache_redis.py | 21 +++-- tests/unit/routes/test_canonical_spm.py | 28 ++++--- .../test_household_calculations.py | 76 +++++++++++++---- .../unit/services/test_simulation_service.py | 84 +++++++++++++++++++ tests/unit/test_country_spm.py | 37 ++++++-- 10 files changed, 347 insertions(+), 56 deletions(-) diff --git a/policyengine_api/routes/household_routes.py b/policyengine_api/routes/household_routes.py index 5f2c65dd9..071102b4c 100644 --- a/policyengine_api/routes/household_routes.py +++ b/policyengine_api/routes/household_routes.py @@ -231,7 +231,7 @@ def _calculation_cache_key(*args, **kwargs): country_id = request.view_args["country_id"] return hash_object( { - "schema": 2, + "schema": 3, "path": request.full_path, "payload": request.get_json(), "spm": g.spm, diff --git a/policyengine_api/runtime_cache/household_calculations.py b/policyengine_api/runtime_cache/household_calculations.py index 363f9d49e..966afa329 100644 --- a/policyengine_api/runtime_cache/household_calculations.py +++ b/policyengine_api/runtime_cache/household_calculations.py @@ -10,10 +10,10 @@ CacheNamespace, RecoverableJSONCache, ) -from policyengine_api.spm import SPMProvenance, resolved_spm_settings +from policyengine_api.spm import validate_spm_calculation_provenance -HOUSEHOLD_CALCULATION_SCHEMA_VERSION = 2 +HOUSEHOLD_CALCULATION_SCHEMA_VERSION = 3 HOUSEHOLD_CALCULATION_TTL_SECONDS = 86_400 @@ -72,21 +72,18 @@ def get( for value in (spm_config, spm_provenance) ): return None - if identity.spm is not None: - # A stored receipt may omit its null values, so compare resolved - # settings rather than raw JSON. Requiring exact equality would miss - # the cache on every replay of a canonical calculation. - if resolved_spm_settings(spm_config) != identity.spm: + if identity.spm is None: + if spm_config is not None or spm_provenance is not None: return None + else: try: - receipt = SPMProvenance.model_validate(spm_provenance) - except ValidationError: + calculation = validate_spm_calculation_provenance( + spm_config, + spm_provenance, + ) + except (ValidationError, ValueError): return None - if ( - receipt.forecast_sha256 != identity.spm.get("forecast_content_sha256") - or receipt.scenario != identity.spm.get("scenario") - or receipt.geography_kind != identity.spm.get("geography_kind") - ): + if calculation.spm_config.model_dump(mode="json") != identity.spm: return None return CachedHouseholdCalculation( household=household, @@ -100,4 +97,22 @@ def set( identity: HouseholdCalculationIdentity, value: CachedHouseholdCalculation, ) -> bool: - return self._cache.set(asdict(identity), asdict(value)) + if identity.spm is None: + if value.spm_config is not None or value.spm_provenance is not None: + return False + return self._cache.set(asdict(identity), asdict(value)) + try: + calculation = validate_spm_calculation_provenance( + value.spm_config, + value.spm_provenance, + ) + except (ValidationError, ValueError): + return False + if calculation.spm_config.model_dump(mode="json") != identity.spm: + return False + payload = asdict(value) + payload["spm_config"] = calculation.spm_config.model_dump(mode="json") + payload["spm_provenance"] = calculation.spm_provenance.model_dump( + mode="json", by_alias=True + ) + return self._cache.set(asdict(identity), payload) diff --git a/policyengine_api/services/household_calculation_service.py b/policyengine_api/services/household_calculation_service.py index 3d6284567..ff8f90845 100644 --- a/policyengine_api/services/household_calculation_service.py +++ b/policyengine_api/services/household_calculation_service.py @@ -26,7 +26,10 @@ ) from policyengine_api.utils.deprecated_inputs import drop_deprecated_inputs from policyengine_api.utils.input_validation import find_unrecognized_inputs -from policyengine_api.spm import normalize_spm_selection +from policyengine_api.spm import ( + normalize_spm_selection, + validate_spm_calculation_provenance, +) @dataclass(frozen=True) @@ -73,6 +76,28 @@ def __init__(self, invalid_inputs: list[Any]) -> None: super().__init__("Household or policy contains unrecognized inputs") +def _validated_spm_fields( + spm_config: object, + spm_provenance: object, + expected_selection: dict | None, +) -> tuple[dict | None, dict | None]: + if expected_selection is None: + if spm_config is None and spm_provenance is None: + return None, None + raise ValueError("Unexpected SPM provenance for a calculation without SPM") + calculation = validate_spm_calculation_provenance( + spm_config, + spm_provenance, + ) + canonical_config = calculation.spm_config.model_dump(mode="json") + if canonical_config != expected_selection: + raise ValueError("SPM calculation configuration differs from the request") + return ( + canonical_config, + calculation.spm_provenance.model_dump(mode="json", by_alias=True), + ) + + def get_household_year(household: dict) -> int | str: household_year: int | str = date.today().year household_age_list = list( @@ -305,6 +330,16 @@ def calculate_stored_household( calculation = CalculationResult( household=raw_calculation, ) + spm_config, spm_provenance = _validated_spm_fields( + calculation.spm_config, + calculation.spm_provenance, + spm, + ) + calculation = replace( + calculation, + spm_config=spm_config, + spm_provenance=spm_provenance, + ) record_cache_event( family="household-calculation", event="recompute", @@ -398,11 +433,16 @@ def calculate_prepared_household( else: household = raw_calculation.household calculation_warnings = tuple(getattr(raw_calculation, "warnings", ())) + spm_config, spm_provenance = _validated_spm_fields( + getattr(raw_calculation, "spm_config", None), + getattr(raw_calculation, "spm_provenance", None), + prepared.spm, + ) return HouseholdCalculationResult( household=household, warnings=(prepared.warnings + calculation_warnings), - spm_config=getattr(raw_calculation, "spm_config", None), - spm_provenance=getattr(raw_calculation, "spm_provenance", None), + spm_config=spm_config, + spm_provenance=spm_provenance, ) @observability_runtime.span( diff --git a/policyengine_api/services/simulation_service.py b/policyengine_api/services/simulation_service.py index dfbe828ce..64ac114eb 100644 --- a/policyengine_api/services/simulation_service.py +++ b/policyengine_api/services/simulation_service.py @@ -8,6 +8,7 @@ from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import Simulation, SimulationRun +from policyengine_api.spm import validate_spm_calculation_provenance from policyengine_api.utils.population_identity import ( canonical_numeric_household_id, population_id_matches, @@ -52,6 +53,21 @@ def _select_simulation( def _latest_successful_run_id(runs: list[SimulationRun]) -> str | None: return next((run.id for run in runs if run.status == "complete"), None) + @staticmethod + def _has_stale_spm_output(output: object) -> bool: + if not isinstance(output, dict) or not ( + "spm_config" in output or "spm_provenance" in output + ): + return False + try: + validate_spm_calculation_provenance( + output.get("spm_config"), + output.get("spm_provenance"), + ) + except ValueError: + return True + return False + def _ensure_simulation_dual_write_state( self, session: Session, @@ -67,6 +83,11 @@ def _ensure_simulation_dual_write_state( if simulation is None: raise ValueError(f"Simulation #{simulation_id} not found") + if self._has_stale_spm_output(simulation.output): + simulation.status = "pending" + simulation.output = None + simulation.error_message = None + spec = { "country_id": simulation.country_id, "population_id": simulation.population_id, @@ -252,6 +273,18 @@ def update_simulation( return None if isinstance(values.get("output"), str): values["output"] = json.loads(values["output"]) + output_value = values.get("output") + if isinstance(output_value, dict) and ( + "spm_config" in output_value or "spm_provenance" in output_value + ): + calculation = validate_spm_calculation_provenance( + output_value.get("spm_config"), + output_value.get("spm_provenance"), + ) + values["output"] = { + **output_value, + **calculation.model_dump(mode="json", by_alias=True), + } with self._sessions.begin() as session: simulation = self._select_simulation( session, diff --git a/policyengine_api/spm.py b/policyengine_api/spm.py index f4970ad55..419a565c7 100644 --- a/policyengine_api/spm.py +++ b/policyengine_api/spm.py @@ -153,6 +153,22 @@ class SPMCalculationProvenance(BaseModel): spm_config: SPMSelection spm_provenance: SPMProvenance + @model_validator(mode="after") + def validate_matching_receipt(self) -> "SPMCalculationProvenance": + receipt = self.spm_provenance + config = self.spm_config + receipt_as_of = receipt.as_of.isoformat() if receipt.as_of is not None else None + if ( + receipt.forecast_sha256 != config.forecast_content_sha256 + or receipt.scenario != config.scenario + or receipt.geography_kind != config.geography_kind + or receipt.geography_id != config.geography_id + or receipt.county_vintage != config.county_vintage + or receipt_as_of != config.as_of + ): + raise ValueError("SPM receipt differs from the SPM configuration") + return self + class SPMExecutionProvenance(BaseModel): """One shared compact receipt and the executions that produced it.""" @@ -394,6 +410,21 @@ def resolved_spm_settings(settings: object) -> dict | None: return {name: getattr(config, name) for name in SPMSelection.model_fields} +def validate_spm_calculation_provenance( + config: object, + provenance: object, +) -> SPMCalculationProvenance: + """Validate a transported compact receipt against its resolved selection.""" + + resolved = resolved_spm_settings(config) + if resolved is None: + raise ValueError("SPM configuration is incomplete") + return SPMCalculationProvenance( + spm_config=SPMSelection.model_validate(resolved), + spm_provenance=SPMProvenance.model_validate(provenance), + ) + + def _current_bundle() -> dict: from policyengine_api.constants import _policyengine_bundle diff --git a/tests/integration/test_runtime_cache_redis.py b/tests/integration/test_runtime_cache_redis.py index 913e9193d..9c107da22 100644 --- a/tests/integration/test_runtime_cache_redis.py +++ b/tests/integration/test_runtime_cache_redis.py @@ -125,20 +125,29 @@ def test_cached_calculation_receipts_survive_a_real_round_trip(redis_pair) -> No name: setting for name, setting in selection.items() if setting is not None }, spm_provenance={ + "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-artifact", "forecast_sha256": selection["forecast_content_sha256"], "scenario": "baseline", "geography_kind": "national", - "runtime_versions": {"policyengine-us": "test-only"}, - "years": {"2026": {"status": "forecast"}}, - "geographies": [], - "composition_method": "classified-inputs", - "storage_method": "formula", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "runtime_versions": { + "policyengine": "test-only", + "policyengine-core": "test-only", + "policyengine-us": "test-only", + "spm-calculator": "test-only", + }, + "years": ["2026"], }, ) assert HouseholdCalculationCache(first, namespace).set(identity, value) - assert HouseholdCalculationCache(second, namespace).get(identity) == value + cached = HouseholdCalculationCache(second, namespace).get(identity) + assert cached is not None + assert cached.spm_config == selection + assert cached.spm_provenance == value.spm_provenance other = HouseholdCalculationIdentity( **{**vars(identity), "spm": {**selection, "geography_kind": "county"}} ) diff --git a/tests/unit/routes/test_canonical_spm.py b/tests/unit/routes/test_canonical_spm.py index 2dd51f9d3..c90e744d6 100644 --- a/tests/unit/routes/test_canonical_spm.py +++ b/tests/unit/routes/test_canonical_spm.py @@ -69,15 +69,21 @@ def calculate(self, household, policy, **kwargs): None if config is None else { + "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-artifact", "forecast_sha256": config["forecast_content_sha256"], "scenario": config["scenario"], "geography_kind": config["geography_kind"], - "runtime_versions": {"policyengine-us": "test-only"}, - "years": {"2026": {"status": "forecast"}}, - "geographies": [], - "composition_method": "classified-inputs", - "storage_method": "formula", + "geography_id": config["geography_id"], + "county_vintage": config["county_vintage"], + "as_of": config["as_of"], + "runtime_versions": { + "policyengine": "test-only", + "policyengine-core": "test-only", + "policyengine-us": "test-only", + "spm-calculator": "test-only", + }, + "years": ["2026"], } ) return CalculationResult(household, (), config, receipt) @@ -570,11 +576,13 @@ def omitting_null_settings(household, policy, **kwargs): url = f"/us/household/{created.json['result']['household_id']}/policy/2" first = client.get(url) assert first.status_code == 200, first.json - assert set(first.json["spm_config"]) == { - "forecast_content_sha256", - "scenario", - "geography_kind", - "county_vintage", + assert first.json["spm_config"] == { + "forecast_content_sha256": FORECAST_HASH, + "scenario": "baseline", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, } cached = client.get(url) assert cached.status_code == 200 diff --git a/tests/unit/runtime_cache/test_household_calculations.py b/tests/unit/runtime_cache/test_household_calculations.py index 27b23a044..d14f645c2 100644 --- a/tests/unit/runtime_cache/test_household_calculations.py +++ b/tests/unit/runtime_cache/test_household_calculations.py @@ -10,6 +10,7 @@ HouseholdCalculationCache, HouseholdCalculationIdentity, ) +from policyengine_api.spm import SPMSelection, build_spm_calculation_provenance def _namespace() -> CacheNamespace: @@ -93,15 +94,21 @@ def test_receipt_bearing_payload_supersedes_the_receiptless_schema() -> None: "as_of": None, } SPM_RECEIPT = { + "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-artifact", "forecast_sha256": "a" * 64, "scenario": "baseline", "geography_kind": "national", - "runtime_versions": {"policyengine-us": "test"}, - "years": {}, - "geographies": [], - "composition_method": "test composition", - "storage_method": "test storage", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "runtime_versions": { + "policyengine": "test", + "policyengine-core": "test", + "policyengine-us": "test", + "spm-calculator": "test", + }, + "years": [], } @@ -112,7 +119,7 @@ def test_receipt_bearing_payload_supersedes_the_receiptless_schema() -> None: {**SPM_RECEIPT, "forecast_sha256": "b" * 64}, {**SPM_RECEIPT, "scenario": "other"}, {**SPM_RECEIPT, "geography_kind": "county"}, - {**SPM_RECEIPT, "years": []}, + {**SPM_RECEIPT, "years": ["2027", "2026"]}, {**SPM_RECEIPT, "unexpected": True}, ], ids=["empty", "hash", "scenario", "geography", "years-shape", "unknown-field"], @@ -125,11 +132,41 @@ def test_household_cache_rejects_invalid_or_mismatched_spm_receipts(receipt): spm_config=SPM_CONFIG, spm_provenance=receipt, ) - assert cache.set(identity, value) is True + assert cache.set(identity, value) is False assert cache.get(identity) is None -@pytest.mark.parametrize("years", [{}, {"2024": {"source": "test"}}]) +def test_household_cache_rejects_previous_diagnostic_receipt_before_write(): + backend = InMemoryCacheBackend() + cache = HouseholdCalculationCache(backend, _namespace()) + identity = _identity(spm=SPM_CONFIG) + previous_receipt = { + "forecast_id": "test-artifact", + "forecast_sha256": "a" * 64, + "scenario": "baseline", + "geography_kind": "national", + "runtime_versions": {}, + "years": {}, + "geographies": [], + "composition_method": "test", + "storage_method": "test", + } + + assert ( + cache.set( + identity, + CachedHouseholdCalculation( + household={"people": {}}, + spm_config=SPM_CONFIG, + spm_provenance=previous_receipt, + ), + ) + is False + ) + assert backend._values == {} + + +@pytest.mark.parametrize("years", [[], ["2024"]]) def test_household_cache_keeps_valid_receipts_including_lazy_tax_only(years): cache = HouseholdCalculationCache(InMemoryCacheBackend(), _namespace()) identity = _identity(spm=SPM_CONFIG) @@ -139,7 +176,10 @@ def test_household_cache_keeps_valid_receipts_including_lazy_tax_only(years): spm_provenance={**SPM_RECEIPT, "years": years}, ) assert cache.set(identity, value) is True - assert cache.get(identity) == value + cached = cache.get(identity) + assert cached is not None + assert cached.spm_config == SPM_CONFIG + assert cached.spm_provenance == {**SPM_RECEIPT, "years": years} @pytest.mark.parametrize("calculate_spm", [False, True], ids=["tax-only", "measured"]) @@ -154,7 +194,9 @@ def test_real_country_provider_receipt_round_trips_household_cache(calculate_spm tenure="renter", ) config = country_spm.spm_config(provider) - receipt = provider.provenance() + receipt = build_spm_calculation_provenance( + SPMSelection.model_validate(config), provider.provenance() + ).spm_provenance.model_dump(mode="json", by_alias=True) assert bool(receipt["years"]) is calculate_spm cache = HouseholdCalculationCache(InMemoryCacheBackend(), _namespace()) @@ -165,7 +207,10 @@ def test_real_country_provider_receipt_round_trips_household_cache(calculate_spm spm_provenance=receipt, ) assert cache.set(identity, value) is True - assert cache.get(identity) == value + cached = cache.get(identity) + assert cached is not None + assert cached.spm_config == config + assert cached.spm_provenance == receipt OMITTED_NULL_CONFIG = { @@ -183,7 +228,10 @@ def test_household_cache_hits_when_receipt_omits_null_settings(): spm_provenance=SPM_RECEIPT, ) assert cache.set(identity, value) is True - assert cache.get(identity) == value + cached = cache.get(identity) + assert cached is not None + assert cached.spm_config == SPM_CONFIG + assert cached.spm_provenance == SPM_RECEIPT @pytest.mark.parametrize("omitted", sorted(OMITTED_NULL_CONFIG)) @@ -198,7 +246,7 @@ def test_household_cache_rejects_receipt_omitting_a_nonnull_setting(omitted): }, spm_provenance=SPM_RECEIPT, ) - assert cache.set(identity, value) is True + assert cache.set(identity, value) is False assert cache.get(identity) is None @@ -221,5 +269,5 @@ def test_household_cache_rejects_receipt_with_different_settings(changed): spm_config={**OMITTED_NULL_CONFIG, **changed}, spm_provenance=SPM_RECEIPT, ) - assert cache.set(identity, value) is True + assert cache.set(identity, value) is False assert cache.get(identity) is None diff --git a/tests/unit/services/test_simulation_service.py b/tests/unit/services/test_simulation_service.py index bc5d4ef2e..5e7b224bb 100644 --- a/tests/unit/services/test_simulation_service.py +++ b/tests/unit/services/test_simulation_service.py @@ -162,6 +162,90 @@ def test_update_simulation_accepts_legacy_json_text_at_wire_boundary(service): assert updated.output == {"result": 42} +def test_update_simulation_rejects_incomplete_spm_provenance(service): + simulation = service.get_or_create_simulation( + "us", "household-1", "household", 1 + ).simulation + + with pytest.raises(ValueError, match="SPM"): + service.update_simulation( + "us", + simulation.id, + status="complete", + output={"spm_config": {"geography_kind": "national"}}, + ) + + +def test_update_simulation_persists_only_compact_spm_provenance(service): + simulation = service.get_or_create_simulation( + "us", "household-1", "household", 1 + ).simulation + config = { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + } + provenance = { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "spm-rolling-2026-09-09", + "forecast_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": ["2026"], + "runtime_versions": { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0", + }, + } + + updated = service.update_simulation( + "us", + simulation.id, + status="complete", + output={"result": 42, "spm_config": config, "spm_provenance": provenance}, + ) + + assert updated.output == { + "result": 42, + "spm_config": config, + "spm_provenance": provenance, + } + + +def test_get_or_create_marks_previous_spm_output_for_recalculation( + service, + orm_session_factory, +): + simulation = service.get_or_create_simulation( + "us", "household-1", "household", 1 + ).simulation + with orm_session_factory.begin() as session: + stored = session.get(Simulation, simulation.id) + stored.status = "complete" + stored.output = { + "result": 42, + "spm_config": {"geography_kind": "national"}, + "spm_provenance": { + "years": {"2026": {"large": "diagnostics"}}, + }, + } + + refreshed = service.get_or_create_simulation( + "us", "household-1", "household", 1 + ).simulation + + assert refreshed.status == "pending" + assert refreshed.output is None + + def test_update_simulation_without_values_is_a_noop(service): simulation = service.get_or_create_simulation( "us", "household-1", "household", 1 diff --git a/tests/unit/test_country_spm.py b/tests/unit/test_country_spm.py index a36d9f8a0..f392df43e 100644 --- a/tests/unit/test_country_spm.py +++ b/tests/unit/test_country_spm.py @@ -191,13 +191,25 @@ def __init__(self, *, tax_benefit_system, situation, spm): def test_tax_only_result_reads_provenance_without_calculating_spm(monkeypatch): calculations = [] - selection = spm.SPMSelection(geography_kind="national").model_dump() + selection = spm.SPMSelection( + forecast_content_sha256="a" * 64, + scenario="baseline", + geography_kind="national", + geography_id=None, + county_vintage="2020", + as_of=None, + ).model_dump() receipt = { "forecast_id": "test-artifact", "forecast_sha256": "a" * 64, "scenario": "baseline", "geography_kind": "national", - "runtime_versions": {"policyengine-us": "test"}, + "runtime_versions": { + "policyengine": "test", + "policyengine-core": "test", + "policyengine-us": "test", + "spm-calculator": "test", + }, "years": {}, "geographies": [], "composition_method": "test adult classification", @@ -224,12 +236,23 @@ def calculate(variable, period): ) result = country.calculate(requested_household("spm_unit_federal_tax"), None) assert calculations == ["spm_unit_federal_tax"] - assert result.spm_provenance["years"] == {} + assert result.spm_provenance["years"] == [] assert ( result.household["spm_units"]["spm_unit"]["spm_unit_federal_tax"]["2024"] == 123 ) assert json.loads(json.dumps(result.spm_config)) == selection - assert json.loads(json.dumps(result.spm_provenance)) == receipt + assert result.spm_provenance == { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "test-artifact", + "forecast_sha256": "a" * 64, + "scenario": "baseline", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": [], + "runtime_versions": receipt["runtime_versions"], + } @pytest.fixture @@ -263,7 +286,7 @@ def test_real_country_state_only_tax_succeeds_and_receipt_stays_empty( result.household["spm_units"]["spm_unit"]["spm_unit_federal_tax"]["2024"] is not None ) - assert result.spm_provenance["years"] == {} + assert result.spm_provenance["years"] == [] json.dumps(result.spm_provenance) @@ -422,7 +445,7 @@ def test_real_http_tax_only_succeeds_without_geography(real_http_client): json={"household": requested_household("spm_unit_federal_tax")}, ) assert response.status_code == 200, response.json - assert response.json["spm_provenance"]["years"] == {} + assert response.json["spm_provenance"]["years"] == [] @pytest.mark.parametrize( @@ -592,7 +615,7 @@ def test_real_http_tax_only_outside_artifact_years_is_lazy( ] is not None ) - assert response.json["spm_provenance"]["years"] == {} + assert response.json["spm_provenance"]["years"] == [] @pytest.mark.parametrize("geography", ["national", "metro"]) From 8eefd17b8e7b065eacf678dd67c8c95435eb0570 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:14:52 +0400 Subject: [PATCH 05/22] Require compact society-wide SPM receipts --- .../runtime_cache/reform_impacts.py | 3 +- .../services/budget_window_cache.py | 3 +- policyengine_api/services/economy_service.py | 2 + policyengine_api/worker_spm.py | 43 +++--- .../routes/test_spm_year_worker_polling.py | 26 ++-- .../unit/runtime_cache/test_reform_impacts.py | 28 +++- .../unit/services/test_budget_window_cache.py | 32 ++++- tests/unit/services/test_worker_spm.py | 133 ++++++++++-------- 8 files changed, 176 insertions(+), 94 deletions(-) diff --git a/policyengine_api/runtime_cache/reform_impacts.py b/policyengine_api/runtime_cache/reform_impacts.py index 571e5b24c..09c96c0f0 100644 --- a/policyengine_api/runtime_cache/reform_impacts.py +++ b/policyengine_api/runtime_cache/reform_impacts.py @@ -19,8 +19,7 @@ record_cache_event, ) - -REFORM_IMPACT_SCHEMA_VERSION = 1 +REFORM_IMPACT_SCHEMA_VERSION = 2 REFORM_IMPACT_TTL_SECONDS = 2_592_000 REFORM_IMPACT_INDEX_LIMIT = 1_000 REFORM_IMPACT_START_CLAIM_TTL_SECONDS = 300 diff --git a/policyengine_api/services/budget_window_cache.py b/policyengine_api/services/budget_window_cache.py index 9402bc952..9da6475ac 100644 --- a/policyengine_api/services/budget_window_cache.py +++ b/policyengine_api/services/budget_window_cache.py @@ -19,9 +19,8 @@ ) from policyengine_api.runtime_cache.dependencies import get_runtime_cache_context - BUDGET_WINDOW_CACHE_FAMILY = "budget-window" -BUDGET_WINDOW_CACHE_SCHEMA_VERSION = 2 +BUDGET_WINDOW_CACHE_SCHEMA_VERSION = 3 BUDGET_WINDOW_STARTING_TTL_SECONDS = 300 BUDGET_WINDOW_BATCH_TTL_SECONDS = 86_400 BUDGET_WINDOW_RESULT_TTL_SECONDS = 2_592_000 diff --git a/policyengine_api/services/economy_service.py b/policyengine_api/services/economy_service.py index 9cec6e90c..ce7112851 100644 --- a/policyengine_api/services/economy_service.py +++ b/policyengine_api/services/economy_service.py @@ -1530,6 +1530,8 @@ def _build_options_hash( bundle_parts.append(f"data_version={data_version}") if policyengine_version: bundle_parts.append(f"policyengine_version={policyengine_version}") + if options.get("spm") is not None: + bundle_parts.append("spm_provenance_schema=canonical-spm-comparison-v2") if runtime_app_name: bundle_parts.append(f"runtime_app_name={runtime_app_name}") return "[" + "&".join([option_pairs, *bundle_parts]).strip("&") + "]" diff --git a/policyengine_api/worker_spm.py b/policyengine_api/worker_spm.py index 28d8566c8..bcf059c01 100644 --- a/policyengine_api/worker_spm.py +++ b/policyengine_api/worker_spm.py @@ -1,12 +1,13 @@ """Canonical economy execution requires the selected worker's certified capability.""" from policyengine_api.spm import ( + SPMComparisonProvenance, SPMSelection, - SPMProvenance, SPMValidationError, error_message, normalize_spm_selection, resolved_spm_settings, + validate_spm_calculation_provenance, ) @@ -100,27 +101,27 @@ def validate_worker_result( raise ValueError("Worker result has incomplete resolved SPM settings") if resolved != selection: raise ValueError("Worker result SPM settings differ from the request") - receipts = result.get("spm_provenance") - if not isinstance(receipts, dict) or set(receipts) != {"baseline", "reform"}: - raise ValueError("Worker result has no baseline/reform SPM receipts") + comparison = SPMComparisonProvenance.model_validate( + result.get("spm_provenance") + ) for side in ("baseline", "reform"): - if not isinstance(receipts[side], list) or not receipts[side]: - raise ValueError("Worker result has an empty SPM receipt") - for item in receipts[side]: - receipt = SPMProvenance.model_validate(item) - if ( - expected_year is not None - and str(expected_year) not in receipt.years - ): - raise ValueError("Worker SPM receipt does not cover requested year") - if ( - receipt.forecast_sha256 != selection["forecast_content_sha256"] - or receipt.scenario != selection["scenario"] - or receipt.geography_kind != selection["geography_kind"] - ): - raise ValueError( - "Worker result SPM provenance differs from the request" - ) + execution = getattr(comparison, side) + receipt = validate_spm_calculation_provenance( + resolved, + execution.receipt, + ).spm_provenance + if expected_year is not None and str(expected_year) not in receipt.years: + raise ValueError("Worker SPM receipt does not cover requested year") + if any( + version is None + for version in ( + receipt.runtime_versions.policyengine, + receipt.runtime_versions.policyengine_core, + receipt.runtime_versions.policyengine_us, + receipt.runtime_versions.spm_calculator, + ) + ): + raise ValueError("Worker SPM receipt has incomplete runtime versions") except (ValueError, TypeError, KeyError) as exc: raise SPMValidationError( "SPM_CONFIGURATION_UNAVAILABLE", error_message(exc) diff --git a/tests/unit/routes/test_spm_year_worker_polling.py b/tests/unit/routes/test_spm_year_worker_polling.py index c79f023f4..d68b32d38 100644 --- a/tests/unit/routes/test_spm_year_worker_polling.py +++ b/tests/unit/routes/test_spm_year_worker_polling.py @@ -15,7 +15,6 @@ from policyengine_api.services.economy_service import EconomyService from policyengine_api.services.reform_impacts_service import ReformImpactsService - SELECTION = { "forecast_content_sha256": "a" * 64, "scenario": "baseline", @@ -32,26 +31,33 @@ def segmented_result(year, invalid_side=None): receipt = { + "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-only", "forecast_sha256": SELECTION["forecast_content_sha256"], "scenario": SELECTION["scenario"], "geography_kind": SELECTION["geography_kind"], - "runtime_versions": {}, - "years": {year: {}}, - "geographies": [], - "composition_method": "classified", - "storage_method": "formula", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "runtime_versions": { + "policyengine": "test", + "policyengine-core": "test", + "policyengine-us": "test", + "spm-calculator": "test", + }, + "years": [year], } result = { "year": year, "spm_config": SELECTION, "spm_provenance": { - "baseline": [dict(receipt), dict(receipt)], - "reform": [dict(receipt), dict(receipt)], + "schema_version": "canonical-spm-comparison-v2", + "baseline": {"receipt": dict(receipt), "execution_count": 2}, + "reform": {"receipt": dict(receipt), "execution_count": 2}, }, } if invalid_side: - result["spm_provenance"][invalid_side][1]["years"] = {"2000": {}} + result["spm_provenance"][invalid_side]["receipt"]["years"] = ["2000"] return result @@ -75,7 +81,7 @@ def test_typed_worker_error_is_terminal_and_replays_after_service_recreation( if isinstance(failure, str): typed_error = { "code": "SPM_CONFIGURATION_UNAVAILABLE", - "message": "Worker SPM receipt does not cover requested year", + "message": "baseline and reform SPM receipts differ", } side = failure.removeprefix("segmented-") worker_result = ( diff --git a/tests/unit/runtime_cache/test_reform_impacts.py b/tests/unit/runtime_cache/test_reform_impacts.py index 43445b86a..c7ef0877d 100644 --- a/tests/unit/runtime_cache/test_reform_impacts.py +++ b/tests/unit/runtime_cache/test_reform_impacts.py @@ -4,7 +4,11 @@ import pytest -from policyengine_api.runtime_cache.core import CacheCoordinationError, CacheNamespace +from policyengine_api.runtime_cache.core import ( + CacheCoordinationError, + CacheNamespace, + encode_envelope, +) from policyengine_api.runtime_cache.fake import InMemoryCacheBackend from policyengine_api.runtime_cache.reform_impacts import ( REFORM_IMPACT_START_CLAIM_TTL_SECONDS, @@ -189,6 +193,28 @@ def test_reform_impact_record_and_indexes_share_one_jittered_ttl( assert set(backend._expires.values()) == {123} +def test_previous_schema_reform_impact_is_not_read() -> None: + backend = InMemoryCacheBackend() + namespace = _namespace() + cache = ReformImpactCache(backend, namespace) + legacy = _impact("legacy", "[spm=canonical]", 1) + legacy_key = namespace.key("reform-impact", 1, {"execution_id": "legacy"}) + backend.set( + legacy_key, + encode_envelope( + "reform-impact", + 1, + { + **legacy.__dict__, + "start_time": legacy.start_time.isoformat(), + "end_time": None, + }, + ), + ) + + assert cache.get_by_execution_id("legacy") is None + + def test_reform_impact_updates_and_deletes_only_matching_computing_values() -> None: backend = InMemoryCacheBackend() cache = ReformImpactCache(backend, _namespace()) diff --git a/tests/unit/services/test_budget_window_cache.py b/tests/unit/services/test_budget_window_cache.py index 1c9009c5e..24e3896c8 100644 --- a/tests/unit/services/test_budget_window_cache.py +++ b/tests/unit/services/test_budget_window_cache.py @@ -3,6 +3,7 @@ import pytest from policyengine_api.runtime_cache.core import CacheCoordinationError +from policyengine_api.runtime_cache.core import CacheNamespace, encode_envelope from policyengine_api.runtime_cache.fake import InMemoryCacheBackend from policyengine_api.services.budget_window_cache import ( BUDGET_WINDOW_BATCH_TTL_SECONDS, @@ -80,7 +81,36 @@ def test_build_key_is_stable_for_request_identity(): ) assert first == second - assert first.startswith("policyengine:test:api:budget-window:v2:") + assert first.startswith("policyengine:test:api:budget-window:v3:") + + +def test_previous_schema_cache_identity_is_not_read(): + backend = FakeRedis() + namespace = CacheNamespace("test", "api") + cache = BudgetWindowCache(client=backend, namespace=namespace) + identity = { + "api_version": "v1", + "baseline_policy_id": 1, + "country_id": "us", + "dataset": "default", + "options_hash": "[spm=canonical]", + "reform_policy_id": 2, + "region": "us", + "time_period": "budget_window:2026:2", + } + old_key = namespace.key("budget-window", 2, identity) + backend.set( + f"{old_key}:state", + encode_envelope( + "budget-window", + 2, + {"status": "completed", "result": {"legacy": True}}, + ), + ) + current_key = cache.build_key(**identity) + + assert current_key != old_key + assert cache.get_state(current_key) is None def test_claim_batch_start_allows_one_starter_and_preserves_identity(): diff --git a/tests/unit/services/test_worker_spm.py b/tests/unit/services/test_worker_spm.py index 1bbcc17c7..9c406cb62 100644 --- a/tests/unit/services/test_worker_spm.py +++ b/tests/unit/services/test_worker_spm.py @@ -9,7 +9,7 @@ from policyengine_api.routes.economy_routes import economy_bp from policyengine_api.services.economy_service import EconomyService from policyengine_api.spm import SPMValidationError -from policyengine_api.worker_spm import validate_worker_spm +from policyengine_api.worker_spm import validate_worker_result, validate_worker_spm @pytest.mark.parametrize("selection", [None, {"geography_kind": "national"}]) @@ -174,6 +174,37 @@ def test_canonical_economy_errors_use_existing_validation_envelope(path): "as_of": None, } CAPABILITY = {"contract_version": "canonical-spm-v1", "defaults": SELECTION} +RUNTIME_VERSIONS = { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0", +} + + +def worker_receipt(*, years=("2026",)): + return { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "test-only", + "forecast_sha256": SELECTION["forecast_content_sha256"], + "scenario": SELECTION["scenario"], + "geography_kind": SELECTION["geography_kind"], + "geography_id": SELECTION["geography_id"], + "county_vintage": SELECTION["county_vintage"], + "as_of": SELECTION["as_of"], + "years": list(years), + "runtime_versions": RUNTIME_VERSIONS, + } + + +def worker_comparison(*, baseline=None, reform=None, execution_count=1): + baseline = baseline or worker_receipt() + reform = reform or worker_receipt() + return { + "schema_version": "canonical-spm-comparison-v2", + "baseline": {"receipt": baseline, "execution_count": execution_count}, + "reform": {"receipt": reform, "execution_count": execution_count}, + } def test_certified_worker_selection_enters_identity_and_submission(): @@ -200,6 +231,15 @@ def test_certified_worker_selection_enters_identity_and_submission(): ) != service._build_options_hash( options={"spm": changed}, model_version="test", dataset="default" ) + assert ( + "spm_provenance_schema=canonical-spm-comparison-v2" + in service._build_options_hash( + options={"spm": SELECTION}, model_version="test", dataset="default" + ) + ) + assert "spm_provenance_schema" not in service._build_options_hash( + options={}, model_version="test", dataset="default" + ) options = service._setup_sim_options("us", {}, {}, "us", "2026", spm=SELECTION) assert options.model_dump(mode="json")["spm"] == SELECTION @@ -403,20 +443,9 @@ def test_worker_cache_receipts_are_required_and_json_roundtrip(): import json from policyengine_api.worker_spm import validate_worker_result - receipt = dict( - forecast_id="test-only", - forecast_sha256=SELECTION["forecast_content_sha256"], - scenario="ce_trend", - geography_kind="national", - runtime_versions={}, - years={"2026": {}}, - geographies=[], - composition_method="classified-inputs", - storage_method="formula", - ) output = { "spm_config": SELECTION, - "spm_provenance": {"baseline": [receipt], "reform": [receipt]}, + "spm_provenance": worker_comparison(), } validate_worker_result(json.loads(json.dumps(output)), SELECTION) validate_worker_result( @@ -457,32 +486,46 @@ def test_worker_receipt_cannot_inherit_nonnull_settings(missing): validate_worker_result({"spm_config": config}, SELECTION) -@pytest.mark.parametrize("side", ["baseline", "reform"]) -def test_worker_receipt_must_cover_requested_year(side): +def test_worker_receipt_must_cover_requested_year(): from policyengine_api.worker_spm import validate_worker_result - receipt = { - "forecast_id": "test-only", - "forecast_sha256": SELECTION["forecast_content_sha256"], - "scenario": "ce_trend", - "geography_kind": "national", - "runtime_versions": {}, - "years": {"2026": {}}, - "geographies": [], - "composition_method": "classified", - "storage_method": "formula", + result = { + "spm_config": SELECTION, + "spm_provenance": worker_comparison(), + } + validate_worker_result(result, SELECTION, expected_year="2026") + result["spm_provenance"] = worker_comparison( + baseline=worker_receipt(years=("2025",)), + reform=worker_receipt(years=("2025",)), + ) + with pytest.raises(SPMValidationError, match="requested year"): + validate_worker_result(result, SELECTION, expected_year="2026") + + +def test_worker_receipt_rejects_baseline_reform_mismatch(): + result = { + "spm_config": SELECTION, + "spm_provenance": worker_comparison( + reform=worker_receipt(years=("2025",)), + ), } + + with pytest.raises(SPMValidationError, match="baseline and reform"): + validate_worker_result(result, SELECTION, expected_year="2026") + + +def test_worker_receipt_rejects_removed_segment_list_shape(): result = { "spm_config": SELECTION, "spm_provenance": { - "baseline": [dict(receipt)], - "reform": [dict(receipt)], + "baseline": [worker_receipt()], + "reform": [worker_receipt()], }, } - validate_worker_result(result, SELECTION, expected_year="2026") - result["spm_provenance"][side][0]["years"] = {"2025": {}} - with pytest.raises(SPMValidationError, match="requested year"): + + with pytest.raises(SPMValidationError) as caught: validate_worker_result(result, SELECTION, expected_year="2026") + assert caught.value.code == "SPM_CONFIGURATION_UNAVAILABLE" @pytest.mark.parametrize( @@ -565,21 +608,10 @@ def test_budget_result_cannot_nest_another_budget_window(): def annual_shape_result(year="2026"): - receipt = { - "forecast_id": "test-only", - "forecast_sha256": SELECTION["forecast_content_sha256"], - "scenario": SELECTION["scenario"], - "geography_kind": SELECTION["geography_kind"], - "runtime_versions": {}, - "years": {"2026": {}}, - "geographies": [], - "composition_method": "classified", - "storage_method": "formula", - } return { "year": year, "spm_config": SELECTION, - "spm_provenance": {"baseline": [receipt], "reform": [receipt]}, + "spm_provenance": worker_comparison(), } @@ -613,17 +645,7 @@ def test_worker_valid_annual_and_window_shapes_cover_requested_year(year): PYDANTIC_INTERNALS = ("errors.pydantic.dev", "input_value", "validation error for") -WORKER_RECEIPT = { - "forecast_id": "test-artifact", - "forecast_sha256": SELECTION["forecast_content_sha256"], - "scenario": SELECTION["scenario"], - "geography_kind": SELECTION["geography_kind"], - "runtime_versions": {"policyengine-us": "test"}, - "years": {"2026": {}}, - "geographies": [], - "composition_method": "test composition", - "storage_method": "test storage", -} +WORKER_RECEIPT = worker_receipt() @pytest.mark.parametrize( @@ -668,10 +690,7 @@ def test_a_worker_receipt_failure_quotes_no_validator_internals(receipt): validate_worker_result( { "spm_config": SELECTION, - "spm_provenance": { - "baseline": [WORKER_RECEIPT], - "reform": [receipt], - }, + "spm_provenance": worker_comparison(reform=receipt), }, SELECTION, expected_year="2026", From c228f672791cdade84bf46c874706bf729b7494d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:22:41 +0400 Subject: [PATCH 06/22] Publish compact SPM provenance schemas --- changelog.d/3862.breaking.md | 1 + docs/canonical-spm.md | 67 ++++++++++++----- policyengine_api/openapi_spec.yaml | 25 ++----- policyengine_api/specification.py | 22 +++++- policyengine_api/spm.py | 9 +-- tests/integration/test_runtime_cache_redis.py | 5 +- tests/unit/routes/test_canonical_spm.py | 27 ++----- .../routes/test_household_numeric_identity.py | 11 ++- tests/unit/routes/test_simulation_spm.py | 10 +-- .../test_household_calculations.py | 11 +-- .../services/test_spm_household_identity.py | 3 +- tests/unit/services/test_worker_spm.py | 15 ++-- tests/unit/test_openapi_simulation.py | 25 +++++-- tests/unit/test_openapi_spm.py | 73 +++++++++++++++---- tests/unit/test_spm_settings.py | 12 ++- 15 files changed, 195 insertions(+), 121 deletions(-) create mode 100644 changelog.d/3862.breaking.md diff --git a/changelog.d/3862.breaking.md b/changelog.d/3862.breaking.md new file mode 100644 index 000000000..9c9b795a8 --- /dev/null +++ b/changelog.d/3862.breaking.md @@ -0,0 +1 @@ +Replace large SPM diagnostic provenance payloads with strict compact v2 receipts for household and society-wide results. Older cached or stored SPM output is recalculated instead of translated. diff --git a/docs/canonical-spm.md b/docs/canonical-spm.md index 042e3be71..01b2ce313 100644 --- a/docs/canonical-spm.md +++ b/docs/canonical-spm.md @@ -160,11 +160,36 @@ Successful canonical calculations add `spm_config` and `spm_provenance` beside with, not a guarantee that every SPM-dependent variable produced a value: a calculation that never chose a measurement still carries the inherited one's receipt beside its null cells. Read the values to learn which of them a -measurement produced. Provenance comes from the actual simulation and includes artifact, -scenario, years, geography, composition/storage methods and runtime versions. -It remains in JSON form through stored replay and cache hits. A tax-only receipt -may have empty `years` and `geographies` because no SPM measurement was requested. -Clients saving simulation outputs must retain this full response envelope. +measurement produced. Provenance comes from the actual simulation and uses the +required compact `canonical-spm-provenance-v2` schema: + +```json +{ + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "canonical-spm-forecast", + "forecast_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "scenario": "baseline", + "geography_kind": "national", + "geography_id": null, + "county_vintage": "2020", + "as_of": null, + "years": ["2026"], + "runtime_versions": { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0" + } +} +``` + +All fields are required. `years` is sorted and unique, and it is empty only +when no SPM dependency ran. Certified deployments populate every runtime +version. Large diagnostic, mapping, request, effective-setting and source +objects from the country package are intentionally excluded from HTTP and +cache payloads. The compact receipt remains in JSON form through stored replay +and cache hits. Clients saving simulation outputs must retain this response +envelope. `POST /us/simulation` takes `population_id`, `population_type` (`household` or `geography`) and integer `policy_id`. It creates a pending record (HTTP 201) or @@ -183,10 +208,11 @@ country, certification, or whether the household has saved `spm`. update field must be non-null, and `complete` requires non-null `output`. Store the full calculation envelope inside `output`, including its `spm_config` and `spm_provenance`. A JSON-encoded output string is also accepted. This endpoint -stores the supplied output; it does not validate SPM receipt integrity. Null -update fields are ignored. The legacy `api_version` input is ignored; writes -record the installed country model version. Top-level `spm`, even null, returns -HTTP 400 `SPM_SETTINGS_UNSUPPORTED` on POST and PATCH. +validates compact SPM receipt integrity and agreement with the adjacent, +fully-resolved six-field `spm_config` before storage. Old or rich receipts are +not translated. Null update fields are ignored. The legacy `api_version` input +is ignored; writes record the installed country model version. Top-level `spm`, +even null, returns HTTP 400 `SPM_SETTINGS_UNSUPPORTED` on POST and PATCH. POST, PATCH and `GET /us/simulation/{id}` return the simulation record inside `result`. Non-string JSON `output` and `simulation_spec_json` are returned as @@ -201,9 +227,10 @@ Flask and the native specification route. HTTP response cache identity includes the normalized selection and model/bundle versions, and validates certification before reading the cache. The stored -calculated-household cache uses schema version 2, includes the selection in +calculated-household cache uses schema version 3, includes the selection in identity, and stores settings/provenance atomically with the result and its -warnings. Missing or mismatched canonical receipts are cache misses. +warnings. Missing, old, rich or mismatched canonical receipts are cache misses +and are recalculated rather than translated. ## Errors @@ -261,11 +288,15 @@ from `/versions`, including agreement with the bundle's independently pinned artifact hash. An old or uncertified worker returns `SPM_CONFIGURATION_UNAVAILABLE`. Normalized settings are sent to both annual and budget-window jobs and participate -in economy cache identity. Results carry `spm_config` and `spm_provenance` with -`baseline` and `reform` receipt lists (one per executed regional segment). Each -budget-window annual row carries these fields; annual results expose them inside -`result`, and budget-window results inside `result.annualImpacts`. Stored results require matching -settings and valid receipts. Typed SPM input errors remain structured 400s +in economy cache identity. Results carry `spm_config` and `spm_provenance` using +`canonical-spm-comparison-v2`. `baseline` and `reform` each contain one compact +`receipt` plus an `execution_count`; the receipt collapses identical child +execution receipts without transporting a repeated list. Baseline and reform +receipts must be identical. Each budget-window annual row carries these fields; +annual results expose them inside `result`, and budget-window results inside +`result.annualImpacts`. Stored results require matching settings and valid +receipts. Economy cache identity includes the comparison schema version, so old +entries are not selected and are recomputed rather than translated. Typed SPM input errors remain structured 400s through asynchronous submission and polling. Poll-time typed failures retain their code and message for the existing cache lifetime. Later reads replay that failure after API service restarts without polling or resubmitting the failed @@ -278,5 +309,5 @@ for the implemented worker paths and remaining coordinated release gates. Partial selections preserve omitted fields in JSON; omissions inherit the certified bundle defaults. The selection schema deliberately supplies no wire defaults, so generated clients preserve that distinction. Resolved settings freeze all six -fields. Responses may omit null geography_id/as_of values, but must include every -non-null resolved field; receipt replay never substitutes current defaults. +fields. Responses include all six fields, including explicit null values for +`geography_id` or `as_of`; receipt replay never substitutes current defaults. diff --git a/policyengine_api/openapi_spec.yaml b/policyengine_api/openapi_spec.yaml index 4e0d0f8b2..b03698ab0 100644 --- a/policyengine_api/openapi_spec.yaml +++ b/policyengine_api/openapi_spec.yaml @@ -899,7 +899,7 @@ paths: spm_config: $ref: '#/components/schemas/SPMSelection' spm_provenance: - $ref: '#/components/schemas/SPMWorkerProvenance' + $ref: '#/components/schemas/SPMComparisonProvenance' 400: $ref: '#/components/responses/SPMValidationError' 404: @@ -972,7 +972,7 @@ paths: spm_config: $ref: '#/components/schemas/SPMSelection' spm_provenance: - $ref: '#/components/schemas/SPMWorkerProvenance' + $ref: '#/components/schemas/SPMComparisonProvenance' progress: type: integer nullable: true @@ -1270,25 +1270,16 @@ components: description: JSON Schema for SPMSelection, preserving omitted fields. defaults: $ref: '#/components/schemas/SPMSelection' - SPMWorkerProvenance: - type: object - additionalProperties: false - required: [baseline, reform] - properties: - baseline: - type: array - minItems: 1 - items: - $ref: '#/components/schemas/SPMProvenance' - reform: - type: array - minItems: 1 - items: - $ref: '#/components/schemas/SPMProvenance' SPMSelection: description: Generated from policyengine_api.spm.SPMSelection. + SPMRuntimeVersions: + description: Generated from policyengine_api.spm.SPMRuntimeVersions. SPMProvenance: description: Generated from policyengine_api.spm.SPMProvenance. + SPMExecutionProvenance: + description: Generated from policyengine_api.spm.SPMExecutionProvenance. + SPMComparisonProvenance: + description: Generated from policyengine_api.spm.SPMComparisonProvenance. responses: SPMValidationError: description: >- diff --git a/policyengine_api/specification.py b/policyengine_api/specification.py index 11b4b3c8f..e2c9296b6 100644 --- a/policyengine_api/specification.py +++ b/policyengine_api/specification.py @@ -13,7 +13,13 @@ AnnualEconomyQuery, BudgetWindowEconomyQuery, ) -from policyengine_api.spm import SPMProvenance, SPMSelection +from policyengine_api.spm import ( + SPMComparisonProvenance, + SPMExecutionProvenance, + SPMProvenance, + SPMRuntimeVersions, + SPMSelection, +) DEFAULT_SPECIFICATION_PATH = Path(__file__).with_name("openapi_spec.yaml") @@ -62,6 +68,15 @@ def _economy_query_parameters(model): return result +def _component_schema(model): + """Generate one OpenAPI component and publish nested models separately.""" + schema = model.model_json_schema( + ref_template="#/components/schemas/{model}", + ) + schema.pop("$defs", None) + return _openapi_30_schema(schema) + + def load_specification( path: Path = DEFAULT_SPECIFICATION_PATH, version: str = VERSION, @@ -74,10 +89,13 @@ def load_specification( schemas = document.get("components", {}).get("schemas", {}) for name, model in { "SPMSelection": SPMSelection, + "SPMRuntimeVersions": SPMRuntimeVersions, "SPMProvenance": SPMProvenance, + "SPMExecutionProvenance": SPMExecutionProvenance, + "SPMComparisonProvenance": SPMComparisonProvenance, }.items(): if name in schemas: - schemas[name] = _openapi_30_schema(model.model_json_schema()) + schemas[name] = _component_schema(model) economy_path = "/{country_id}/economy/{policy_id}/over/{baseline_policy_id}" for suffix, query_model in ( ("", AnnualEconomyQuery), diff --git a/policyengine_api/spm.py b/policyengine_api/spm.py index 419a565c7..9cc181f08 100644 --- a/policyengine_api/spm.py +++ b/policyengine_api/spm.py @@ -386,12 +386,9 @@ def simulation_supports_spm(simulation_type) -> bool: ) -# A resolved selection freezes all six fields, but output transports may omit -# null values. Only those omissions are safe: a receipt must never inherit a -# non-null setting from today's bundle defaults. -REQUIRED_RESOLVED_SPM_FIELDS = frozenset( - {"forecast_content_sha256", "scenario", "geography_kind", "county_vintage"} -) +# A transported resolved selection freezes all six fields, including explicit +# null values. Receipt validation never fills output from current defaults. +REQUIRED_RESOLVED_SPM_FIELDS = frozenset(SPMSelection.model_fields) def resolved_spm_settings(settings: object) -> dict | None: diff --git a/tests/integration/test_runtime_cache_redis.py b/tests/integration/test_runtime_cache_redis.py index 9c107da22..2d7c571cc 100644 --- a/tests/integration/test_runtime_cache_redis.py +++ b/tests/integration/test_runtime_cache_redis.py @@ -120,10 +120,7 @@ def test_cached_calculation_receipts_survive_a_real_round_trip(redis_pair) -> No value = CachedHouseholdCalculation( household={"people": {"you": {}}}, warnings=(), - # A canonical receipt may omit its null settings; that is the same choice. - spm_config={ - name: setting for name, setting in selection.items() if setting is not None - }, + spm_config=selection, spm_provenance={ "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-artifact", diff --git a/tests/unit/routes/test_canonical_spm.py b/tests/unit/routes/test_canonical_spm.py index c90e744d6..95f51cb83 100644 --- a/tests/unit/routes/test_canonical_spm.py +++ b/tests/unit/routes/test_canonical_spm.py @@ -544,14 +544,8 @@ def test_valid_spm_database_timeout_keeps_stage11_safe_persistence_response( copy_event.assert_not_called() -def test_stored_replay_hits_the_calculation_cache_when_a_receipt_omits_nulls( - certified, harness -): - """A canonical country may omit null receipt settings; replay must still hit. - - Requiring exact JSON equality between the receipt and the resolved identity - made every stored replay recompute. - """ +def test_stored_replay_rejects_a_receipt_config_that_omits_nulls(certified, harness): + """Country output must state every field in its resolved configuration.""" client, country = harness calculate = country.calculate @@ -574,20 +568,9 @@ def omitting_null_settings(household, policy, **kwargs): ) assert created.status_code == 201, created.json url = f"/us/household/{created.json['result']['household_id']}/policy/2" - first = client.get(url) - assert first.status_code == 200, first.json - assert first.json["spm_config"] == { - "forecast_content_sha256": FORECAST_HASH, - "scenario": "baseline", - "geography_kind": "national", - "geography_id": None, - "county_vintage": "2020", - "as_of": None, - } - cached = client.get(url) - assert cached.status_code == 200 - assert cached.json == first.json - assert len(country.calls) == 1 + assert client.get(url).status_code == 500 + assert client.get(url).status_code == 500 + assert len(country.calls) == 2 @pytest.mark.parametrize("version", ["5.2.1", "5.4.0", "6.0.0"]) diff --git a/tests/unit/routes/test_household_numeric_identity.py b/tests/unit/routes/test_household_numeric_identity.py index 524cf4e5b..e65e4b13b 100644 --- a/tests/unit/routes/test_household_numeric_identity.py +++ b/tests/unit/routes/test_household_numeric_identity.py @@ -219,9 +219,12 @@ def test_numeric_link_preserves_immutable_household_and_saved_simulation( assert saved["population_id"] == saved_spelling assert json.loads(saved["simulation_spec_json"])["population_id"] == saved_spelling if reference == "historical-padded": - assert saved["status"] == "complete" - assert json.loads(saved["output"]) == output + assert saved["status"] == "pending" + assert saved["output"] is None + assert saved["latest_successful_run_id"] is None + assert saved["active_run_id"] is not None with orm_session_factory() as session: - run = session.get(SimulationRun, saved["latest_successful_run_id"]) + run = session.get(SimulationRun, saved["active_run_id"]) assert run.simulation_spec_snapshot_json["population_id"] == saved_spelling - assert run.output == output + assert run.status == "pending" + assert run.output is None diff --git a/tests/unit/routes/test_simulation_spm.py b/tests/unit/routes/test_simulation_spm.py index 59b591a5a..22a151696 100644 --- a/tests/unit/routes/test_simulation_spm.py +++ b/tests/unit/routes/test_simulation_spm.py @@ -46,7 +46,7 @@ def test_simulation_records_reject_spm_overrides_before_read_or_write( @pytest.mark.parametrize("population_id", ["00001", "1", "0000001", 1]) -def test_repost_historical_numeric_household_alias_replays_completed_output( +def test_repost_historical_numeric_household_alias_recalculates_old_spm_output( orm_session_factory, population_id ): output = {"household": {"people": {}}, "spm_config": {"geography_kind": "national"}} @@ -81,8 +81,8 @@ def test_repost_historical_numeric_household_alias_replays_completed_output( assert response.status_code == 200 result = response.get_json()["result"] assert result["id"] == historical_id - assert result["status"] == "complete" - assert json.loads(result["output"]) == output + assert result["status"] == "pending" + assert result["output"] is None assert result["population_id"] == "00001" assert ( json.loads(result["simulation_spec_json"])["population_id"] == "00001" @@ -91,6 +91,6 @@ def test_repost_historical_numeric_household_alias_replays_completed_output( with orm_session_factory() as session: assert session.scalar(select(func.count()).select_from(Simulation)) == 1 run = session.scalar(select(SimulationRun)) - assert run.status == "complete" - assert run.output == output + assert run.status == "pending" + assert run.output is None assert run.simulation_spec_snapshot_json["population_id"] == "00001" diff --git a/tests/unit/runtime_cache/test_household_calculations.py b/tests/unit/runtime_cache/test_household_calculations.py index d14f645c2..570a9e3f6 100644 --- a/tests/unit/runtime_cache/test_household_calculations.py +++ b/tests/unit/runtime_cache/test_household_calculations.py @@ -218,8 +218,8 @@ def test_real_country_provider_receipt_round_trips_household_cache(calculate_spm } -def test_household_cache_hits_when_receipt_omits_null_settings(): - """A canonical receipt may omit its null values; that is still the same selection.""" +def test_household_cache_rejects_receipt_omitting_null_settings(): + """A compact receipt's adjacent config must contain all six fields.""" cache = HouseholdCalculationCache(InMemoryCacheBackend(), _namespace()) identity = _identity(spm=SPM_CONFIG) value = CachedHouseholdCalculation( @@ -227,11 +227,8 @@ def test_household_cache_hits_when_receipt_omits_null_settings(): spm_config=OMITTED_NULL_CONFIG, spm_provenance=SPM_RECEIPT, ) - assert cache.set(identity, value) is True - cached = cache.get(identity) - assert cached is not None - assert cached.spm_config == SPM_CONFIG - assert cached.spm_provenance == SPM_RECEIPT + assert cache.set(identity, value) is False + assert cache.get(identity) is None @pytest.mark.parametrize("omitted", sorted(OMITTED_NULL_CONFIG)) diff --git a/tests/unit/services/test_spm_household_identity.py b/tests/unit/services/test_spm_household_identity.py index 2ed2572f1..661d3ec03 100644 --- a/tests/unit/services/test_spm_household_identity.py +++ b/tests/unit/services/test_spm_household_identity.py @@ -158,7 +158,8 @@ def test_replacement_household_preserves_historical_zero_padded_reference( ) assert replay.created is False assert replay.simulation.population_id == str(household.id).zfill(5) - assert replay.simulation.output == {"spm_config": NATIONAL} + assert replay.simulation.status == "pending" + assert replay.simulation.output is None saved = service.get_household("us", household.id) assert saved.household_json == original diff --git a/tests/unit/services/test_worker_spm.py b/tests/unit/services/test_worker_spm.py index 9c406cb62..4cb17b7a6 100644 --- a/tests/unit/services/test_worker_spm.py +++ b/tests/unit/services/test_worker_spm.py @@ -448,15 +448,6 @@ def test_worker_cache_receipts_are_required_and_json_roundtrip(): "spm_provenance": worker_comparison(), } validate_worker_result(json.loads(json.dumps(output)), SELECTION) - validate_worker_result( - { - **output, - "spm_config": { - key: value for key, value in SELECTION.items() if value is not None - }, - }, - SELECTION, - ) validate_worker_result( { "kind": "budgetWindow", @@ -468,6 +459,12 @@ def test_worker_cache_receipts_are_required_and_json_roundtrip(): for incomplete in ( {}, {"spm_config": SELECTION}, + { + **output, + "spm_config": { + key: value for key, value in SELECTION.items() if value is not None + }, + }, {**output, "spm_config": {**SELECTION, "scenario": "zero_real"}}, ): with pytest.raises(SPMValidationError): diff --git a/tests/unit/test_openapi_simulation.py b/tests/unit/test_openapi_simulation.py index ff4fd6c38..d80320eb5 100644 --- a/tests/unit/test_openapi_simulation.py +++ b/tests/unit/test_openapi_simulation.py @@ -147,17 +147,30 @@ def test_simulation_persistence_responses_preserve_spm_output_and_run_fields( "status": "ok", "message": None, "result": {"people": {}}, - "spm_config": {"geography_kind": "national"}, + "spm_config": { + "forecast_content_sha256": "a" * 64, + "scenario": "baseline", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + }, "spm_provenance": { + "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-forecast", "forecast_sha256": "a" * 64, "scenario": "baseline", "geography_kind": "national", - "runtime_versions": {"policyengine-us": "test"}, - "years": {}, - "geographies": [], - "composition_method": "test", - "storage_method": "test", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "runtime_versions": { + "policyengine": "test", + "policyengine-core": "test", + "policyengine-us": "test", + "spm-calculator": "test", + }, + "years": ["2026"], }, } update_payload = { diff --git a/tests/unit/test_openapi_spm.py b/tests/unit/test_openapi_spm.py index 75a6e6d46..38da38748 100644 --- a/tests/unit/test_openapi_spm.py +++ b/tests/unit/test_openapi_spm.py @@ -5,7 +5,13 @@ from flask import Flask from policyengine_api.routes.system_routes import system_bp -from policyengine_api.spm import SPMProvenance, SPMSelection +from policyengine_api.spm import ( + SPMComparisonProvenance, + SPMExecutionProvenance, + SPMProvenance, + SPMRuntimeVersions, + SPMSelection, +) from policyengine_api.query_parameters import ( AnnualEconomyQuery, BudgetWindowEconomyQuery, @@ -50,15 +56,23 @@ def test_economy_query_and_receipt_schemas_match_public_http_contract(): "$ref": "#/components/schemas/SPMSelection" } assert result["properties"]["spm_provenance"] == { - "$ref": "#/components/schemas/SPMWorkerProvenance" + "$ref": "#/components/schemas/SPMComparisonProvenance" } assert operation["responses"]["400"]["$ref"].endswith("SPMValidationError") - receipt = spec["components"]["schemas"]["SPMWorkerProvenance"] - assert receipt["required"] == ["baseline", "reform"] - for side in receipt["required"]: - assert receipt["properties"][side]["items"] == { - "$ref": "#/components/schemas/SPMProvenance" + comparison = spec["components"]["schemas"]["SPMComparisonProvenance"] + assert comparison["required"] == ["schema_version", "baseline", "reform"] + assert comparison["properties"]["schema_version"]["enum"] == [ + "canonical-spm-comparison-v2" + ] + for side in ("baseline", "reform"): + assert comparison["properties"][side] == { + "$ref": "#/components/schemas/SPMExecutionProvenance" } + execution = spec["components"]["schemas"]["SPMExecutionProvenance"] + assert execution["required"] == ["receipt", "execution_count"] + assert execution["properties"]["receipt"] == { + "$ref": "#/components/schemas/SPMProvenance" + } def test_metadata_discovery_and_selection_constraints_are_published(): @@ -106,10 +120,17 @@ def test_spm_documentation_uses_public_models_and_actual_route_envelopes(): for name, model in { "SPMSelection": SPMSelection, + "SPMRuntimeVersions": SPMRuntimeVersions, "SPMProvenance": SPMProvenance, + "SPMExecutionProvenance": SPMExecutionProvenance, + "SPMComparisonProvenance": SPMComparisonProvenance, }.items(): schema = schemas[name] - assert set(schema["properties"]) == set(model.model_fields) + expected_fields = { + field.alias or field_name + for field_name, field in model.model_fields.items() + } + assert set(schema["properties"]) == expected_fields assert schema["additionalProperties"] is False assert '"type": "null"' not in json.dumps(schema) @@ -119,12 +140,38 @@ def test_spm_documentation_uses_public_models_and_actual_route_envelopes(): for field in ("forecast_content_sha256", "scenario", "geography_id", "as_of"): assert selection[field]["type"] == "string" assert selection[field]["nullable"] is True - assert schemas["SPMProvenance"]["properties"]["runtime_versions"][ - "additionalProperties" - ] == { - "type": "string", - "nullable": True, + provenance = schemas["SPMProvenance"] + assert provenance["required"] == [ + "schema_version", + "forecast_id", + "forecast_sha256", + "scenario", + "geography_kind", + "geography_id", + "county_vintage", + "as_of", + "years", + "runtime_versions", + ] + assert provenance["properties"]["runtime_versions"] == { + "$ref": "#/components/schemas/SPMRuntimeVersions" + } + assert set(schemas["SPMRuntimeVersions"]["properties"]) == { + "policyengine", + "policyengine-core", + "policyengine-us", + "spm-calculator", + } + removed_rich_fields = { + "geographies", + "requested", + "effective", + "execution_count", + "sources", + "diagnostics", + "mappings", } + assert not removed_rich_fields.intersection(provenance["properties"]) for route in ("calculate", "calculate-full"): operation = paths[f"/{{country_id}}/{route}"]["post"] diff --git a/tests/unit/test_spm_settings.py b/tests/unit/test_spm_settings.py index f22b3b75e..33e18936b 100644 --- a/tests/unit/test_spm_settings.py +++ b/tests/unit/test_spm_settings.py @@ -403,13 +403,13 @@ def test_canonical_country_cannot_inherit_the_legacy_exception(monkeypatch, vers } -def test_resolved_settings_recover_receipt_omitted_null_values(certified_bundle): - """A receipt that omits its null values still names the same selection.""" +def test_resolved_settings_require_explicit_null_values(certified_bundle): + """A transported resolved config includes all six selection fields.""" assert spm.resolved_spm_settings(RESOLVED_SELECTION) == RESOLVED_SELECTION omitted = { key: value for key, value in RESOLVED_SELECTION.items() if value is not None } - assert spm.resolved_spm_settings(omitted) == RESOLVED_SELECTION + assert spm.resolved_spm_settings(omitted) is None assert spm.resolved_spm_settings(spm.normalize_spm_selection("us", None)) == ( RESOLVED_SELECTION ) @@ -434,11 +434,9 @@ def test_resolved_settings_reject_unreadable_receipt_settings(settings): def test_receipt_expands_config_and_reduces_country_diagnostics(certified_bundle): """The public receipt contains one resolved config and compact provenance.""" - omitted = { - key: value for key, value in RESOLVED_SELECTION.items() if value is not None - } simulation = SimpleNamespace( - spm_config=omitted, spm_provenance=lambda: deepcopy(COUNTRY_RECEIPT) + spm_config=RESOLVED_SELECTION, + spm_provenance=lambda: deepcopy(COUNTRY_RECEIPT), ) receipt = spm.calculation_spm_receipt(simulation) assert receipt["spm_config"] == RESOLVED_SELECTION From b02e3a115582a152a2db111b2f23b0fff91ec8bb Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:42:23 +0400 Subject: [PATCH 07/22] Close compact SPM persistence review gaps --- changelog.d/3862.breaking.md | 2 +- docs/canonical-spm.md | 13 +- docs/migration/stage-12-worker-contract.md | 4 + policyengine_api/constants.py | 2 +- policyengine_api/openapi_spec.yaml | 17 +- .../services/report_output_service.py | 97 +++++++++++ .../services/v2/simulations/types.py | 3 +- policyengine_api/specification.py | 2 + policyengine_api/spm.py | 63 ++++--- policyengine_api/worker_spm.py | 10 -- .../services/test_report_output_service.py | 159 +++++++++++++++++- tests/unit/test_openapi_spm.py | 19 ++- .../unit/test_simulation_and_report_routes.py | 85 +++++++++- tests/unit/test_spm_provenance.py | 32 +++- .../unit/v2/test_stage12_worker_contracts.py | 87 +++++++++- 15 files changed, 538 insertions(+), 57 deletions(-) diff --git a/changelog.d/3862.breaking.md b/changelog.d/3862.breaking.md index 9c9b795a8..b878b5aab 100644 --- a/changelog.d/3862.breaking.md +++ b/changelog.d/3862.breaking.md @@ -1 +1 @@ -Replace large SPM diagnostic provenance payloads with strict compact v2 receipts for household and society-wide results. Older cached or stored SPM output is recalculated instead of translated. +Replace large SPM diagnostic provenance payloads with strict compact v2 receipts for household, society-wide, report, and Stage 12 artifact results. Older cached or stored SPM output is recalculated instead of translated, and runtime version values are required. diff --git a/docs/canonical-spm.md b/docs/canonical-spm.md index 01b2ce313..c7b5fe0a3 100644 --- a/docs/canonical-spm.md +++ b/docs/canonical-spm.md @@ -183,9 +183,9 @@ required compact `canonical-spm-provenance-v2` schema: } ``` -All fields are required. `years` is sorted and unique, and it is empty only -when no SPM dependency ran. Certified deployments populate every runtime -version. Large diagnostic, mapping, request, effective-setting and source +All fields are required. The four runtime-version values are non-null strings. +`years` is sorted and unique, and it is empty only when no SPM dependency ran. +Large diagnostic, mapping, request, effective-setting and source objects from the country package are intentionally excluded from HTTP and cache payloads. The compact receipt remains in JSON form through stored replay and cache hits. Clients saving simulation outputs must retain this response @@ -302,6 +302,13 @@ their code and message for the existing cache lifetime. Later reads replay that failure after API service restarts without polling or resubmitting the failed job. Canonical cache identity and runtime-bundle refresh rules still apply. +Completed US economy report outputs pass the same compact comparison validation +before `report_outputs` or `report_output_runs` persistence. Their report cache +identity uses schema version 2. Reads of schema-version-1 report records resolve +to a new pending record with no copied output, preventing another client from +receiving a previously stored rich receipt. Household and non-US report output +shapes retain their existing behavior. + See the [canonical SPM worker PR](https://github.com/PolicyEngine/policyengine-sim-api/pull/677) for the implemented worker paths and remaining coordinated release gates. diff --git a/docs/migration/stage-12-worker-contract.md b/docs/migration/stage-12-worker-contract.md index dbb7a8aa2..f26c315fa 100644 --- a/docs/migration/stage-12-worker-contract.md +++ b/docs/migration/stage-12-worker-contract.md @@ -171,6 +171,10 @@ SHA-256 content digests, schema versions, row-identity metadata, and bounded provenance. When an option such as SPM produces a detached calculation receipt, the simulation descriptor and Parquet metadata retain it so the coordinator can validate the receipt before reproducing the existing aggregate response. +`SimulationArtifactDescriptor.calculation_provenance` is either null or the +exact typed compact SPM calculation contract: a complete six-field resolved +configuration plus `canonical-spm-provenance-v2`. It rejects partial, rich, and +legacy receipt shapes rather than preserving or normalizing them. Only the Simulation Entrypoint, the versioned worker application, the bucket's object-lifecycle process, and explicitly authorized operators may access the diff --git a/policyengine_api/constants.py b/policyengine_api/constants.py index 683fb4aa9..dffe76ca6 100644 --- a/policyengine_api/constants.py +++ b/policyengine_api/constants.py @@ -177,7 +177,7 @@ def get_bundle_default_dataset_option(country_id: str) -> dict: RUNTIME_CACHE_SCHEMA_VERSIONS = { "economy_impact": 2, - "report_output": 1, + "report_output": 2, } diff --git a/policyengine_api/openapi_spec.yaml b/policyengine_api/openapi_spec.yaml index b03698ab0..21f43d240 100644 --- a/policyengine_api/openapi_spec.yaml +++ b/policyengine_api/openapi_spec.yaml @@ -440,7 +440,7 @@ paths: result: type: object spm_config: - $ref: '#/components/schemas/SPMSelection' + $ref: '#/components/schemas/SPMResolvedConfiguration' spm_provenance: $ref: '#/components/schemas/SPMProvenance' 404: @@ -532,7 +532,7 @@ paths: result: type: object spm_config: - $ref: '#/components/schemas/SPMSelection' + $ref: '#/components/schemas/SPMResolvedConfiguration' spm_provenance: $ref: '#/components/schemas/SPMProvenance' 500: @@ -760,7 +760,10 @@ paths: patch: summary: Update a report output operationId: update_report_output - description: Update a report output status, result, or error message. + description: >- + Update a report output status, result, or error message. Completed US + economy outputs carrying SPM fields require compact v2 comparison + provenance and a matching complete resolved configuration. parameters: - name: country_id in: path @@ -897,7 +900,7 @@ paths: nullable: true properties: spm_config: - $ref: '#/components/schemas/SPMSelection' + $ref: '#/components/schemas/SPMResolvedConfiguration' spm_provenance: $ref: '#/components/schemas/SPMComparisonProvenance' 400: @@ -970,7 +973,7 @@ paths: year: type: string spm_config: - $ref: '#/components/schemas/SPMSelection' + $ref: '#/components/schemas/SPMResolvedConfiguration' spm_provenance: $ref: '#/components/schemas/SPMComparisonProvenance' progress: @@ -1229,7 +1232,7 @@ components: type: object nullable: true spm_config: - $ref: '#/components/schemas/SPMSelection' + $ref: '#/components/schemas/SPMResolvedConfiguration' spm_provenance: $ref: '#/components/schemas/SPMProvenance' SimulationError: @@ -1272,6 +1275,8 @@ components: $ref: '#/components/schemas/SPMSelection' SPMSelection: description: Generated from policyengine_api.spm.SPMSelection. + SPMResolvedConfiguration: + description: Generated from policyengine_api.spm.SPMResolvedConfiguration. SPMRuntimeVersions: description: Generated from policyengine_api.spm.SPMRuntimeVersions. SPMProvenance: diff --git a/policyengine_api/services/report_output_service.py b/policyengine_api/services/report_output_service.py index 28bdd3e9e..abc184f73 100644 --- a/policyengine_api/services/report_output_service.py +++ b/policyengine_api/services/report_output_service.py @@ -1,6 +1,7 @@ import json from dataclasses import dataclass from datetime import datetime, timezone +from typing import cast from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker @@ -8,6 +9,7 @@ from policyengine_api.constants import get_report_output_cache_version from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import ReportOutput, ReportOutputRun, Simulation +from policyengine_api.json_types import JSONObject, JSONValue from policyengine_api.services.report_run_service import ReportRunService from policyengine_api.services.report_spec_service import ( ECONOMY_REPORT_KINDS, @@ -15,6 +17,84 @@ ReportSpecService, ) from policyengine_api.services.simulation_service import SimulationService +from policyengine_api.spm import ( + SPMComparisonProvenance, + error_message, + validate_spm_calculation_provenance, +) + + +def _canonicalize_annual_spm_output( + output: JSONObject, + *, + expected_year: str | None, +) -> JSONObject: + """Validate and serialize the compact SPM fields on one economy result.""" + + has_config = "spm_config" in output + has_provenance = "spm_provenance" in output + if not has_config and not has_provenance: + return output + if not has_config or not has_provenance: + raise ValueError("SPM configuration and provenance must be supplied together") + + comparison = SPMComparisonProvenance.model_validate(output["spm_provenance"]) + calculations = tuple( + validate_spm_calculation_provenance( + output["spm_config"], + execution.receipt, + ) + for execution in (comparison.baseline, comparison.reform) + ) + if expected_year is not None and any( + expected_year not in calculation.spm_provenance.years + for calculation in calculations + ): + raise ValueError("SPM receipt does not cover the report year") + return { + **output, + "spm_config": cast( + JSONValue, + calculations[0].spm_config.model_dump(mode="json"), + ), + "spm_provenance": cast( + JSONValue, + comparison.model_dump(mode="json", by_alias=True), + ), + } + + +def _canonicalize_society_wide_spm_output( + output: object, + *, + expected_year: str | None, +) -> object: + """Validate compact annual or budget-window society-wide SPM output.""" + + if not isinstance(output, dict): + return output + typed_output = cast(JSONObject, output) + if output.get("kind") != "budgetWindow": + return _canonicalize_annual_spm_output( + typed_output, + expected_year=expected_year, + ) + + annual_impacts = output.get("annualImpacts") + if not isinstance(annual_impacts, list): + raise ValueError("SPM budget-window output requires annual impact objects") + canonical_rows: list[JSONValue] = [] + for row in annual_impacts: + if not isinstance(row, dict): + raise ValueError("SPM budget-window output requires annual impact objects") + row_year = row.get("year") + canonical_rows.append( + _canonicalize_annual_spm_output( + cast(JSONObject, row), + expected_year=str(row_year) if row_year is not None else None, + ) + ) + return {**typed_output, "annualImpacts": canonical_rows} @dataclass(frozen=True) @@ -648,6 +728,23 @@ def _update_report_output( "Cannot mark report output running without an active pending " "or running report run" ) + effective_status = values.get("status", report_output.status) + effective_output = values.get("output", report_output.output) + if ( + effective_status == "complete" + and report_output.country_id == "us" + and report_output.report_kind in ECONOMY_REPORT_KINDS + ): + try: + values["output"] = _canonicalize_society_wide_spm_output( + effective_output, + expected_year=report_output.year, + ) + except (ValueError, TypeError, KeyError) as error: + raise ValueError( + "Completed US economy report has invalid SPM provenance: " + f"{error_message(error)}" + ) from error for field, value in values.items(): setattr(report_output, field, value) self._ensure_report_output_dual_write_state( diff --git a/policyengine_api/services/v2/simulations/types.py b/policyengine_api/services/v2/simulations/types.py index cc9b0a723..6e9605102 100644 --- a/policyengine_api/services/v2/simulations/types.py +++ b/policyengine_api/services/v2/simulations/types.py @@ -16,6 +16,7 @@ ) from policyengine_api.query_parameters import CountryId +from policyengine_api.spm import SPMCalculationProvenance ContractText = Annotated[str, Field(min_length=1, max_length=255)] ContractVersion = Literal[1] @@ -205,4 +206,4 @@ class SimulationArtifactDescriptor(StrictContractModel): output_schema_version: ContractVersion = 1 row_identity: RowIdentity bundle: BundleProvenance - calculation_provenance: dict[str, JsonValue] | None = None + calculation_provenance: SPMCalculationProvenance | None = None diff --git a/policyengine_api/specification.py b/policyengine_api/specification.py index e2c9296b6..2830a359c 100644 --- a/policyengine_api/specification.py +++ b/policyengine_api/specification.py @@ -17,6 +17,7 @@ SPMComparisonProvenance, SPMExecutionProvenance, SPMProvenance, + SPMResolvedConfiguration, SPMRuntimeVersions, SPMSelection, ) @@ -89,6 +90,7 @@ def load_specification( schemas = document.get("components", {}).get("schemas", {}) for name, model in { "SPMSelection": SPMSelection, + "SPMResolvedConfiguration": SPMResolvedConfiguration, "SPMRuntimeVersions": SPMRuntimeVersions, "SPMProvenance": SPMProvenance, "SPMExecutionProvenance": SPMExecutionProvenance, diff --git a/policyengine_api/spm.py b/policyengine_api/spm.py index 9cc181f08..7ea5ba638 100644 --- a/policyengine_api/spm.py +++ b/policyengine_api/spm.py @@ -97,6 +97,34 @@ def validate_location(self): return self +class SPMResolvedConfiguration(BaseModel): + """The complete six-field SPM selection recorded beside a receipt.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + forecast_content_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + scenario: str = Field(min_length=1, pattern=r"^\S+$") + geography_kind: Literal["county", "national", "metro"] + geography_id: str | None = Field(min_length=1) + county_vintage: str = Field(pattern=r"^[0-9]{4}$") + as_of: str | None + + @field_validator("as_of") + @classmethod + def validate_as_of(cls, value: str | None) -> str | None: + if value is not None and date.fromisoformat(value).isoformat() != value: + raise ValueError("as_of must be an ISO calendar date (YYYY-MM-DD)") + return value + + @model_validator(mode="after") + def validate_location(self) -> "SPMResolvedConfiguration": + if self.geography_kind == "metro": + if self.geography_id is None or not self.geography_id.strip(): + raise ValueError("An SPM area selection requires geography_id") + elif self.geography_id is not None: + raise ValueError("Only an SPM area selection accepts geography_id") + return self + + class SPMProvenance(BaseModel): """Compact public receipt for one canonical SPM execution.""" @@ -137,10 +165,10 @@ class SPMRuntimeVersions(BaseModel): """Package versions required to reproduce one certified SPM execution.""" model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True) - policyengine: str | None - policyengine_core: str | None = Field(alias="policyengine-core") - policyengine_us: str | None = Field(alias="policyengine-us") - spm_calculator: str | None = Field(alias="spm-calculator") + policyengine: str = Field(min_length=1) + policyengine_core: str = Field(min_length=1, alias="policyengine-core") + policyengine_us: str = Field(min_length=1, alias="policyengine-us") + spm_calculator: str = Field(min_length=1, alias="spm-calculator") SPMProvenance.model_rebuild() @@ -150,7 +178,7 @@ class SPMCalculationProvenance(BaseModel): """A resolved SPM configuration and its matching compact receipt.""" model_config = ConfigDict(frozen=True, extra="forbid") - spm_config: SPMSelection + spm_config: SPMResolvedConfiguration spm_provenance: SPMProvenance @model_validator(mode="after") @@ -257,17 +285,6 @@ def build_spm_calculation_provenance( if not isinstance(source_versions, Mapping): raise ValueError("Country SPM receipt requires runtime_versions") runtime_versions = SPMRuntimeVersions.model_validate(source_versions) - if any( - version is None - for version in ( - runtime_versions.policyengine, - runtime_versions.policyengine_core, - runtime_versions.policyengine_us, - runtime_versions.spm_calculator, - ) - ): - raise ValueError("Certified SPM receipts require every runtime version") - compact = build_spm_provenance( forecast_id=_required_string(receipt, "forecast_id"), forecast_sha256=forecast_sha256, @@ -276,7 +293,14 @@ def build_spm_calculation_provenance( runtime_versions=runtime_versions, ) return SPMCalculationProvenance( - spm_config=config, + spm_config=SPMResolvedConfiguration( + forecast_content_sha256=forecast_sha256, + scenario=scenario, + geography_kind=config.geography_kind, + geography_id=config.geography_id, + county_vintage=config.county_vintage, + as_of=config.as_of, + ), spm_provenance=compact, ) @@ -413,11 +437,8 @@ def validate_spm_calculation_provenance( ) -> SPMCalculationProvenance: """Validate a transported compact receipt against its resolved selection.""" - resolved = resolved_spm_settings(config) - if resolved is None: - raise ValueError("SPM configuration is incomplete") return SPMCalculationProvenance( - spm_config=SPMSelection.model_validate(resolved), + spm_config=SPMResolvedConfiguration.model_validate(config), spm_provenance=SPMProvenance.model_validate(provenance), ) diff --git a/policyengine_api/worker_spm.py b/policyengine_api/worker_spm.py index bcf059c01..e4941a237 100644 --- a/policyengine_api/worker_spm.py +++ b/policyengine_api/worker_spm.py @@ -112,16 +112,6 @@ def validate_worker_result( ).spm_provenance if expected_year is not None and str(expected_year) not in receipt.years: raise ValueError("Worker SPM receipt does not cover requested year") - if any( - version is None - for version in ( - receipt.runtime_versions.policyengine, - receipt.runtime_versions.policyengine_core, - receipt.runtime_versions.policyengine_us, - receipt.runtime_versions.spm_calculator, - ) - ): - raise ValueError("Worker SPM receipt has incomplete runtime versions") except (ValueError, TypeError, KeyError) as exc: raise SPMValidationError( "SPM_CONFIGURATION_UNAVAILABLE", error_message(exc) diff --git a/tests/unit/services/test_report_output_service.py b/tests/unit/services/test_report_output_service.py index f4e6d67ea..93520634d 100644 --- a/tests/unit/services/test_report_output_service.py +++ b/tests/unit/services/test_report_output_service.py @@ -1,6 +1,7 @@ import pytest from sqlalchemy import func, select +from policyengine_api import constants from policyengine_api.constants import get_report_output_cache_version from policyengine_api.data.v1_models import ReportOutput, ReportOutputRun from policyengine_api.services.report_output_service import ReportOutputService @@ -9,6 +10,43 @@ run_service = ReportRunService() +SPM_CONFIG = { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} +SPM_RECEIPT = { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "canonical-forecast", + "forecast_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": ["2025"], + "runtime_versions": { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0", + }, +} + + +def compact_economy_output() -> dict: + return { + "spm_config": SPM_CONFIG, + "spm_provenance": { + "schema_version": "canonical-spm-comparison-v2", + "baseline": {"receipt": SPM_RECEIPT, "execution_count": 2}, + "reform": {"receipt": SPM_RECEIPT, "execution_count": 2}, + }, + "budgetary_impact": 42, + } @pytest.fixture @@ -21,13 +59,14 @@ def create_simulation( *, policy_id=1, population_id="household-1", + population_type="household", ): return ( SimulationService(orm_session_factory) .get_or_create_simulation( country_id="us", population_id=population_id, - population_type="household", + population_type=population_type, policy_id=policy_id, ) .simulation @@ -182,6 +221,124 @@ def test_update_accepts_existing_v1_json_string_boundary( assert view.report_output.output == {"ok": True} +def test_update_complete_persists_only_compact_society_wide_spm_output( + service, + orm_session_factory, +): + baseline = create_simulation( + orm_session_factory, + policy_id=1, + population_id="us", + population_type="geography", + ) + reform = create_simulation( + orm_session_factory, + policy_id=2, + population_id="us", + population_type="geography", + ) + report = service.create_or_reuse_report_output( + "us", baseline.id, reform.id, "2025" + ).view.report_output + + view = service.update_report_output( + "us", + report.id, + status="complete", + output=compact_economy_output(), + ) + + assert view.report_output.output == compact_economy_output() + assert view.display_run.output == compact_economy_output() + + +def test_update_complete_rejects_legacy_society_wide_spm_output( + service, + orm_session_factory, +): + baseline = create_simulation( + orm_session_factory, + policy_id=1, + population_id="us", + population_type="geography", + ) + reform = create_simulation( + orm_session_factory, + policy_id=2, + population_id="us", + population_type="geography", + ) + report = service.create_or_reuse_report_output( + "us", baseline.id, reform.id, "2025" + ).view.report_output + legacy = compact_economy_output() + legacy["spm_provenance"] = { + "baseline": [SPM_RECEIPT], + "reform": [SPM_RECEIPT], + } + + with pytest.raises(ValueError, match="SPM"): + service.update_report_output("us", report.id, status="complete", output=legacy) + + with orm_session_factory() as session: + stored = session.get(ReportOutput, report.id) + assert stored.status == "pending" + assert stored.output is None + run = session.get(ReportOutputRun, stored.active_run_id) + assert run.status == "pending" + assert run.output is None + + +def test_resolve_legacy_spm_report_returns_new_empty_cache_record( + service, + orm_session_factory, + monkeypatch, +): + baseline = create_simulation( + orm_session_factory, + policy_id=1, + population_id="us", + population_type="geography", + ) + reform = create_simulation( + orm_session_factory, + policy_id=2, + population_id="us", + population_type="geography", + ) + monkeypatch.setitem(constants.RUNTIME_CACHE_SCHEMA_VERSIONS, "report_output", 1) + legacy_cache_version = get_report_output_cache_version("us") + monkeypatch.setitem(constants.RUNTIME_CACHE_SCHEMA_VERSIONS, "report_output", 2) + assert legacy_cache_version != get_report_output_cache_version("us") + with orm_session_factory.begin() as session: + legacy = ReportOutput( + country_id="us", + simulation_1_id=baseline.id, + simulation_2_id=reform.id, + api_version=legacy_cache_version, + status="complete", + output={ + "spm_config": SPM_CONFIG, + "spm_provenance": { + "baseline": [SPM_RECEIPT], + "reform": [SPM_RECEIPT], + }, + }, + year="2025", + ) + session.add(legacy) + session.flush() + legacy_id = legacy.id + + view = service.resolve_report_output("us", legacy_id) + + assert view.response_id == legacy_id + assert view.report_output.id != legacy_id + assert view.report_output.api_version == get_report_output_cache_version("us") + assert view.report_output.status == "pending" + assert view.report_output.output is None + + def test_update_running_requires_mutable_run(service, orm_session_factory): simulation = create_simulation(orm_session_factory) report = create_report(service, simulation.id) diff --git a/tests/unit/test_openapi_spm.py b/tests/unit/test_openapi_spm.py index 38da38748..222b056e7 100644 --- a/tests/unit/test_openapi_spm.py +++ b/tests/unit/test_openapi_spm.py @@ -9,6 +9,7 @@ SPMComparisonProvenance, SPMExecutionProvenance, SPMProvenance, + SPMResolvedConfiguration, SPMRuntimeVersions, SPMSelection, ) @@ -53,7 +54,7 @@ def test_economy_query_and_receipt_schemas_match_public_http_contract(): if suffix: result = result["properties"]["annualImpacts"]["items"] assert result["properties"]["spm_config"] == { - "$ref": "#/components/schemas/SPMSelection" + "$ref": "#/components/schemas/SPMResolvedConfiguration" } assert result["properties"]["spm_provenance"] == { "$ref": "#/components/schemas/SPMComparisonProvenance" @@ -116,10 +117,12 @@ def test_spm_documentation_uses_public_models_and_actual_route_envelopes(): paths = spec["paths"] schemas = spec["components"]["schemas"] selection_ref = {"$ref": "#/components/schemas/SPMSelection"} + resolved_ref = {"$ref": "#/components/schemas/SPMResolvedConfiguration"} provenance_ref = {"$ref": "#/components/schemas/SPMProvenance"} for name, model in { "SPMSelection": SPMSelection, + "SPMResolvedConfiguration": SPMResolvedConfiguration, "SPMRuntimeVersions": SPMRuntimeVersions, "SPMProvenance": SPMProvenance, "SPMExecutionProvenance": SPMExecutionProvenance, @@ -162,6 +165,18 @@ def test_spm_documentation_uses_public_models_and_actual_route_envelopes(): "policyengine-us", "spm-calculator", } + assert schemas["SPMRuntimeVersions"]["required"] == [ + "policyengine", + "policyengine-core", + "policyengine-us", + "spm-calculator", + ] + assert all( + "nullable" not in field + for field in schemas["SPMRuntimeVersions"]["properties"].values() + ) + resolved = schemas["SPMResolvedConfiguration"] + assert set(resolved["required"]) == set(resolved["properties"]) removed_rich_fields = { "geographies", "requested", @@ -193,7 +208,7 @@ def test_spm_documentation_uses_public_models_and_actual_route_envelopes(): "properties" ] assert {"status", "message", "result"} <= set(result) - assert result["spm_config"] == selection_ref + assert result["spm_config"] == resolved_ref assert result["spm_provenance"] == provenance_ref assert ( operation["responses"]["400"]["$ref"] diff --git a/tests/unit/test_simulation_and_report_routes.py b/tests/unit/test_simulation_and_report_routes.py index 2d2502f7c..3c0b2c892 100644 --- a/tests/unit/test_simulation_and_report_routes.py +++ b/tests/unit/test_simulation_and_report_routes.py @@ -27,15 +27,57 @@ def create_test_client() -> Flask: return app.test_client() -def create_simulation(factory, *, population_id="household-1", policy_id=1): +def create_simulation( + factory, + *, + population_id="household-1", + policy_id=1, + population_type="household", +): simulation = ( SimulationService(factory) - .get_or_create_simulation("us", population_id, "household", policy_id) + .get_or_create_simulation("us", population_id, population_type, policy_id) .simulation ) return simulation.id +def compact_spm_report_output() -> dict: + config = { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + } + receipt = { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "canonical-forecast", + "forecast_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": ["2025"], + "runtime_versions": { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0", + }, + } + return { + "spm_config": config, + "spm_provenance": { + "schema_version": "canonical-spm-comparison-v2", + "baseline": {"receipt": receipt, "execution_count": 1}, + "reform": {"receipt": receipt, "execution_count": 1}, + }, + } + + def create_report(factory, simulation_id): report = ( ReportOutputService(factory) @@ -261,6 +303,45 @@ def test_report_patch_complete_promotes_active_rerun(orm_session_factory): assert report.latest_successful_run_id == rerun_id +def test_report_patch_rejects_legacy_spm_output_before_persistence( + orm_session_factory, +): + baseline_id = create_simulation( + orm_session_factory, + population_id="us", + policy_id=50, + population_type="geography", + ) + reform_id = create_simulation( + orm_session_factory, + population_id="us", + policy_id=51, + population_type="geography", + ) + report_id = ( + ReportOutputService(orm_session_factory) + .create_or_reuse_report_output("us", baseline_id, reform_id, "2025") + .view.report_output.id + ) + legacy = compact_spm_report_output() + receipt = legacy["spm_provenance"]["baseline"]["receipt"] + legacy["spm_provenance"] = { + "baseline": [receipt], + "reform": [receipt], + } + + response = create_test_client().patch( + "/us/report", + json={"id": report_id, "status": "complete", "output": legacy}, + ) + + assert response.status_code == 400 + with orm_session_factory() as session: + report = session.get(ReportOutput, report_id) + assert report.status == "pending" + assert report.output is None + + def test_simulation_v1_routes_keep_json_fields_as_strings(orm_session_factory): simulation_id = create_simulation(orm_session_factory, policy_id=49) output = {"result": "ok", "values": [1, 2, 3]} diff --git a/tests/unit/test_spm_provenance.py b/tests/unit/test_spm_provenance.py index 2a7147df2..e4f02fa58 100644 --- a/tests/unit/test_spm_provenance.py +++ b/tests/unit/test_spm_provenance.py @@ -6,6 +6,7 @@ from pydantic import ValidationError from policyengine_api.spm import ( + SPMCalculationProvenance, SPMComparisonProvenance, SPMProvenance, SPMRuntimeVersions, @@ -88,6 +89,33 @@ def test_compact_receipt_rejects_noncanonical_values( _receipt(**changes) +@pytest.mark.parametrize( + "invalid_version", + ["policyengine", "policyengine-core", "policyengine-us", "spm-calculator"], +) +@pytest.mark.parametrize("value", [None, ""]) +def test_runtime_versions_require_four_nonempty_strings( + invalid_version: str, + value: object, +) -> None: + versions = VERSIONS.model_dump(mode="json", by_alias=True) + versions[invalid_version] = value + + with pytest.raises(ValidationError): + SPMRuntimeVersions.model_validate(versions) + + +def test_calculation_contract_requires_every_resolved_configuration_field() -> None: + receipt = _receipt().model_dump(mode="json", by_alias=True) + partial = SELECTION.model_dump(mode="json") + partial.pop("geography_id") + + with pytest.raises(ValidationError): + SPMCalculationProvenance.model_validate( + {"spm_config": partial, "spm_provenance": receipt} + ) + + def test_compact_receipt_rejects_the_previous_diagnostic_shape() -> None: previous = { "forecast_id": "spm-rolling-2026-09-09", @@ -120,7 +148,9 @@ def test_country_receipt_adapter_extracts_only_compact_scalars() -> None: calculation = build_spm_calculation_provenance(SELECTION, country_receipt) - assert calculation.spm_config == SELECTION + assert calculation.spm_config.model_dump(mode="json") == SELECTION.model_dump( + mode="json" + ) assert calculation.spm_provenance.years == ("2026", "2027") encoded = calculation.model_dump_json(by_alias=True) assert "diagnostic" not in encoded diff --git a/tests/unit/v2/test_stage12_worker_contracts.py b/tests/unit/v2/test_stage12_worker_contracts.py index df177f792..9e537996d 100644 --- a/tests/unit/v2/test_stage12_worker_contracts.py +++ b/tests/unit/v2/test_stage12_worker_contracts.py @@ -27,6 +27,36 @@ DIGEST_C = "c" * 64 +def _calculation_provenance() -> dict[str, object]: + return { + "spm_config": { + "forecast_content_sha256": DIGEST_A, + "scenario": "official", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + }, + "spm_provenance": { + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "canonical-forecast", + "forecast_sha256": DIGEST_A, + "scenario": "official", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": ["2026"], + "runtime_versions": { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0", + }, + }, + } + + def _bundle() -> dict[str, object]: return { "policyengine_version": "5.2.0", @@ -173,10 +203,7 @@ def test_artifact_descriptors_retain_identity_and_bundle_provenance() -> None: "identity_sha256": DIGEST_B, }, "bundle": _bundle(), - "calculation_provenance": { - "spm_config": {"scenario": "official"}, - "spm_provenance": {"forecast_sha256": DIGEST_A}, - }, + "calculation_provenance": _calculation_provenance(), } ) report_artifact = AggregateReportArtifactDescriptor.model_validate( @@ -197,16 +224,60 @@ def test_artifact_descriptors_retain_identity_and_bundle_provenance() -> None: ) assert simulation_artifact.row_identity.identifier_columns == ("household_id",) - assert simulation_artifact.calculation_provenance == { - "spm_config": {"scenario": "official"}, - "spm_provenance": {"forecast_sha256": DIGEST_A}, - } + assert simulation_artifact.calculation_provenance is not None + assert ( + simulation_artifact.calculation_provenance.model_dump( + mode="json", by_alias=True + ) + == _calculation_provenance() + ) assert ( report_artifact.baseline_artifact_sha256 == simulation_artifact.artifact.content_sha256 ) +@pytest.mark.parametrize( + "provenance", + [ + { + "spm_config": {"scenario": "official"}, + "spm_provenance": {"forecast_sha256": DIGEST_A}, + }, + { + **_calculation_provenance(), + "spm_provenance": { + **_calculation_provenance()["spm_provenance"], + "geographies": [], + }, + }, + ], + ids=["partial", "legacy-rich"], +) +def test_simulation_artifact_rejects_noncompact_calculation_provenance( + provenance: dict[str, object], +) -> None: + descriptor = { + "contract_version": 1, + "evaluation_id": str(EVALUATION_ID), + "simulation_execution_id": str(BASELINE_ID), + "role": "baseline", + "artifact": {**_population_artifact(), "content_sha256": DIGEST_C}, + "output_schema_version": 1, + "row_identity": { + "schema_version": 1, + "identifier_columns": ["household_id"], + "row_count": 100, + "identity_sha256": DIGEST_B, + }, + "bundle": _bundle(), + "calculation_provenance": provenance, + } + + with pytest.raises(ValidationError): + SimulationArtifactDescriptor.model_validate(descriptor) + + def test_artifact_payload_contracts_cover_parquet_and_aggregate_json() -> None: assert SIMULATION_PARQUET_PAYLOAD_CONTRACT.model_dump(mode="json") == { "payload_schema_version": 1, From 777065340974e758b4888499789bed3a1c46233c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:48:23 +0400 Subject: [PATCH 08/22] Require SPM fields on completed US economy reports --- .../services/report_output_service.py | 6 +- .../services/test_report_output_service.py | 76 ++++++++++++++++++- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/policyengine_api/services/report_output_service.py b/policyengine_api/services/report_output_service.py index abc184f73..49cb50051 100644 --- a/policyengine_api/services/report_output_service.py +++ b/policyengine_api/services/report_output_service.py @@ -33,10 +33,10 @@ def _canonicalize_annual_spm_output( has_config = "spm_config" in output has_provenance = "spm_provenance" in output - if not has_config and not has_provenance: - return output if not has_config or not has_provenance: - raise ValueError("SPM configuration and provenance must be supplied together") + raise ValueError( + "Completed US economy output requires both SPM configuration and provenance" + ) comparison = SPMComparisonProvenance.model_validate(output["spm_provenance"]) calculations = tuple( diff --git a/tests/unit/services/test_report_output_service.py b/tests/unit/services/test_report_output_service.py index 93520634d..b264a8129 100644 --- a/tests/unit/services/test_report_output_service.py +++ b/tests/unit/services/test_report_output_service.py @@ -57,6 +57,7 @@ def service(orm_session_factory): def create_simulation( orm_session_factory, *, + country_id="us", policy_id=1, population_id="household-1", population_type="household", @@ -64,7 +65,7 @@ def create_simulation( return ( SimulationService(orm_session_factory) .get_or_create_simulation( - country_id="us", + country_id=country_id, population_id=population_id, population_type=population_type, policy_id=policy_id, @@ -252,6 +253,79 @@ def test_update_complete_persists_only_compact_society_wide_spm_output( assert view.display_run.output == compact_economy_output() +@pytest.mark.parametrize( + "output", + [ + {"budgetary_impact": 42}, + {"budgetary_impact": 42, "spm_config": SPM_CONFIG}, + { + "budgetary_impact": 42, + "spm_provenance": compact_economy_output()["spm_provenance"], + }, + ], + ids=["absent", "config-only", "provenance-only"], +) +def test_update_complete_requires_both_society_wide_spm_fields( + service, + orm_session_factory, + output, +): + baseline = create_simulation( + orm_session_factory, + policy_id=1, + population_id="us", + population_type="geography", + ) + reform = create_simulation( + orm_session_factory, + policy_id=2, + population_id="us", + population_type="geography", + ) + report = service.create_or_reuse_report_output( + "us", baseline.id, reform.id, "2025" + ).view.report_output + + with pytest.raises(ValueError, match="requires both SPM"): + service.update_report_output("us", report.id, status="complete", output=output) + + with orm_session_factory() as session: + stored = session.get(ReportOutput, report.id) + assert stored.status == "pending" + assert stored.output is None + + +@pytest.mark.parametrize( + ("country_id", "population_type"), + [("us", "household"), ("uk", "geography")], + ids=["us-household", "non-us-economy"], +) +def test_update_complete_preserves_optional_spm_for_other_report_kinds( + service, + orm_session_factory, + country_id, + population_type, +): + simulation = create_simulation( + orm_session_factory, + country_id=country_id, + population_id="household-1" if population_type == "household" else country_id, + population_type=population_type, + ) + report = service.create_or_reuse_report_output( + country_id, simulation.id, year="2025" + ).view.report_output + + view = service.update_report_output( + country_id, + report.id, + status="complete", + output={"ok": True}, + ) + + assert view.report_output.output == {"ok": True} + + def test_update_complete_rejects_legacy_society_wide_spm_output( service, orm_session_factory, From 822cbf5bdaebcf5cab1ca6b019be7753b4cc3af7 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 1 Oct 2026 21:06:48 +0400 Subject: [PATCH 09/22] Use SPM provenance as completed-result configuration --- changelog.d/3862.breaking.md | 2 +- docs/canonical-spm.md | 30 ++++---- docs/migration/stage-12-worker-contract.md | 10 ++- policyengine_api/constants.py | 4 +- policyengine_api/openapi_spec.yaml | 30 +++----- policyengine_api/routes/household_routes.py | 6 +- .../runtime_cache/household_calculations.py | 44 +++++------ .../runtime_cache/reform_impacts.py | 2 +- .../services/budget_window_cache.py | 2 +- .../services/household_calculation_service.py | 36 +++------ .../services/report_output_service.py | 26 ++----- .../services/simulation_service.py | 28 +++---- policyengine_api/services/v2/reports/types.py | 11 +++ .../services/v2/simulations/types.py | 4 +- policyengine_api/specification.py | 2 - policyengine_api/spm.py | 64 +++++----------- policyengine_api/worker_spm.py | 17 ++--- tests/integration/test_runtime_cache_redis.py | 2 - tests/unit/routes/test_canonical_spm.py | 19 +++-- .../routes/test_spm_year_worker_polling.py | 1 - .../test_household_calculations.py | 71 ++++++++++-------- .../unit/runtime_cache/test_reform_impacts.py | 8 +- .../unit/services/test_budget_window_cache.py | 8 +- .../services/test_report_output_service.py | 29 +++---- .../unit/services/test_simulation_service.py | 29 ++++--- tests/unit/services/test_worker_spm.py | 33 ++++---- tests/unit/test_country_spm.py | 14 ++-- tests/unit/test_openapi_simulation.py | 8 -- tests/unit/test_openapi_spm.py | 12 +-- .../unit/test_simulation_and_report_routes.py | 9 --- tests/unit/test_spm_provenance.py | 20 ++--- tests/unit/test_spm_settings.py | 8 +- .../unit/v2/test_stage12_worker_contracts.py | 75 +++++++++++-------- 33 files changed, 311 insertions(+), 353 deletions(-) diff --git a/changelog.d/3862.breaking.md b/changelog.d/3862.breaking.md index b878b5aab..4a719cf50 100644 --- a/changelog.d/3862.breaking.md +++ b/changelog.d/3862.breaking.md @@ -1 +1 @@ -Replace large SPM diagnostic provenance payloads with strict compact v2 receipts for household, society-wide, report, and Stage 12 artifact results. Older cached or stored SPM output is recalculated instead of translated, and runtime version values are required. +Replace large SPM diagnostic provenance payloads and duplicate completed-result SPM configuration with strict compact v2 receipts for household, society-wide, report, and Stage 12 artifact results. Older cached or stored SPM output is recalculated instead of translated, and runtime version values are required. diff --git a/docs/canonical-spm.md b/docs/canonical-spm.md index c7b5fe0a3..590f9fe07 100644 --- a/docs/canonical-spm.md +++ b/docs/canonical-spm.md @@ -155,9 +155,12 @@ storage; no additional schema migration is required. selected policy. No independent simulation-level override is supported. Top-level `spm` on simulation POST/PATCH is rejected instead of silently ignored. -Successful canonical calculations add `spm_config` and `spm_provenance` beside -`result`. A receipt describes the measurement the simulation was constructed -with, not a guarantee that every SPM-dependent variable produced a value: a +Successful canonical calculations add only `spm_provenance` beside `result`. +Its forecast, scenario, geography, county-vintage and as-of fields are the +complete resolved calculation configuration; completed results do not repeat +that configuration in a sibling `spm_config` field. A receipt describes the +measurement the simulation was constructed with, not a guarantee that every +SPM-dependent variable produced a value: a calculation that never chose a measurement still carries the inherited one's receipt beside its null cells. Read the values to learn which of them a measurement produced. Provenance comes from the actual simulation and uses the @@ -206,11 +209,11 @@ country, certification, or whether the household has saved `spm`. `PATCH /us/simulation` identifies the record with body `id` and accepts `status` (`pending`, `complete` or `error`), `output` and `error_message`. At least one update field must be non-null, and `complete` requires non-null `output`. Store -the full calculation envelope inside `output`, including its `spm_config` and +the full calculation envelope inside `output`, including its `spm_provenance`. A JSON-encoded output string is also accepted. This endpoint -validates compact SPM receipt integrity and agreement with the adjacent, -fully-resolved six-field `spm_config` before storage. Old or rich receipts are -not translated. Null update fields are ignored. The legacy `api_version` input +validates the one compact SPM receipt strictly before storage. Completed output +that includes the former sibling `spm_config`, an old receipt or a rich receipt +is rejected rather than translated. Null update fields are ignored. The legacy `api_version` input is ignored; writes record the installed country model version. Top-level `spm`, even null, returns HTTP 400 `SPM_SETTINGS_UNSUPPORTED` on POST and PATCH. @@ -227,8 +230,8 @@ Flask and the native specification route. HTTP response cache identity includes the normalized selection and model/bundle versions, and validates certification before reading the cache. The stored -calculated-household cache uses schema version 3, includes the selection in -identity, and stores settings/provenance atomically with the result and its +calculated-household cache uses schema version 4, includes the selection in +identity, and stores provenance atomically with the result and its warnings. Missing, old, rich or mismatched canonical receipts are cache misses and are recalculated rather than translated. @@ -288,12 +291,13 @@ from `/versions`, including agreement with the bundle's independently pinned artifact hash. An old or uncertified worker returns `SPM_CONFIGURATION_UNAVAILABLE`. Normalized settings are sent to both annual and budget-window jobs and participate -in economy cache identity. Results carry `spm_config` and `spm_provenance` using +in economy cache identity. Results carry only `spm_provenance` using `canonical-spm-comparison-v2`. `baseline` and `reform` each contain one compact `receipt` plus an `execution_count`; the receipt collapses identical child execution receipts without transporting a repeated list. Baseline and reform -receipts must be identical. Each budget-window annual row carries these fields; -annual results expose them inside `result`, and budget-window results inside +receipts must be identical, and their receipt fields are the sole resolved SPM +configuration. Each budget-window annual row carries this field; +annual results expose it inside `result`, and budget-window results inside `result.annualImpacts`. Stored results require matching settings and valid receipts. Economy cache identity includes the comparison schema version, so old entries are not selected and are recomputed rather than translated. Typed SPM input errors remain structured 400s @@ -304,7 +308,7 @@ job. Canonical cache identity and runtime-bundle refresh rules still apply. Completed US economy report outputs pass the same compact comparison validation before `report_outputs` or `report_output_runs` persistence. Their report cache -identity uses schema version 2. Reads of schema-version-1 report records resolve +identity uses schema version 3. Reads of earlier report records resolve to a new pending record with no copied output, preventing another client from receiving a previously stored rich receipt. Household and non-US report output shapes retain their existing behavior. diff --git a/docs/migration/stage-12-worker-contract.md b/docs/migration/stage-12-worker-contract.md index f26c315fa..1e830a360 100644 --- a/docs/migration/stage-12-worker-contract.md +++ b/docs/migration/stage-12-worker-contract.md @@ -172,9 +172,13 @@ provenance. When an option such as SPM produces a detached calculation receipt, the simulation descriptor and Parquet metadata retain it so the coordinator can validate the receipt before reproducing the existing aggregate response. `SimulationArtifactDescriptor.calculation_provenance` is either null or the -exact typed compact SPM calculation contract: a complete six-field resolved -configuration plus `canonical-spm-provenance-v2`. It rejects partial, rich, and -legacy receipt shapes rather than preserving or normalizing them. +exact typed `canonical-spm-provenance-v2` receipt. The receipt's forecast, +scenario, geography, county-vintage and as-of fields are the complete resolved +configuration; the descriptor does not transport a duplicate `spm_config`. +It rejects partial, rich, legacy and dual-field receipt shapes rather than +preserving or normalizing them. +`AggregateReportArtifactPayload.result` applies the same rule to comparison +provenance and rejects a sibling `spm_config` before artifact storage. Only the Simulation Entrypoint, the versioned worker application, the bucket's object-lifecycle process, and explicitly authorized operators may access the diff --git a/policyengine_api/constants.py b/policyengine_api/constants.py index dffe76ca6..7056574cb 100644 --- a/policyengine_api/constants.py +++ b/policyengine_api/constants.py @@ -176,8 +176,8 @@ def get_bundle_default_dataset_option(country_id: str) -> dict: ) or _resolve_distribution_version(_dist_versions, "policyengine-core", "policyengine") RUNTIME_CACHE_SCHEMA_VERSIONS = { - "economy_impact": 2, - "report_output": 2, + "economy_impact": 3, + "report_output": 3, } diff --git a/policyengine_api/openapi_spec.yaml b/policyengine_api/openapi_spec.yaml index 21f43d240..3155fe341 100644 --- a/policyengine_api/openapi_spec.yaml +++ b/policyengine_api/openapi_spec.yaml @@ -439,8 +439,6 @@ paths: nullable: true result: type: object - spm_config: - $ref: '#/components/schemas/SPMResolvedConfiguration' spm_provenance: $ref: '#/components/schemas/SPMProvenance' 404: @@ -531,8 +529,6 @@ paths: nullable: true result: type: object - spm_config: - $ref: '#/components/schemas/SPMResolvedConfiguration' spm_provenance: $ref: '#/components/schemas/SPMProvenance' 500: @@ -623,9 +619,9 @@ paths: non-null output. Null update fields are ignored, and omitted fields retain their values. The installed country model version and current run state are updated with the record. Preserve the full household calculation - envelope, including spm_config and spm_provenance, inside output. Its SPM - selection belongs to the linked household; this operation does not select - a measurement or validate stored receipt integrity. Top-level spm is + envelope, including spm_provenance, inside output. Its SPM selection is + derived from that receipt and belongs to the linked household; this + operation does not select a measurement. Top-level spm is rejected, including null, with HTTP 400 SPM_SETTINGS_UNSUPPORTED. parameters: - name: country_id @@ -658,7 +654,7 @@ paths: Object and array outputs and simulation_spec_json use the legacy JSON-encoded string representation, or null when absent. Stored scalar strings are returned unchanged. Decode a saved household output envelope - to read its spm_config and spm_provenance. Run identifiers and + to read its spm_provenance. Run identifiers and specification metadata may be null on historical records. parameters: - name: country_id @@ -762,8 +758,8 @@ paths: operationId: update_report_output description: >- Update a report output status, result, or error message. Completed US - economy outputs carrying SPM fields require compact v2 comparison - provenance and a matching complete resolved configuration. + economy outputs require compact v2 comparison provenance. Each receipt + contains the complete resolved configuration; a sibling spm_config is rejected. parameters: - name: country_id in: path @@ -883,7 +879,7 @@ paths: # Query parameters are generated from policyengine_api.query_parameters. responses: 200: - description: Annual economic impact, progress, or persisted error state. Canonical completed results include the selected SPM configuration and actual baseline/reform receipts. + description: Annual economic impact, progress, or persisted error state. Canonical completed results include actual baseline/reform receipts containing the selected SPM configuration. content: application/json: schema: @@ -899,8 +895,6 @@ paths: type: object nullable: true properties: - spm_config: - $ref: '#/components/schemas/SPMResolvedConfiguration' spm_provenance: $ref: '#/components/schemas/SPMComparisonProvenance' 400: @@ -972,8 +966,6 @@ paths: properties: year: type: string - spm_config: - $ref: '#/components/schemas/SPMResolvedConfiguration' spm_provenance: $ref: '#/components/schemas/SPMComparisonProvenance' progress: @@ -1191,7 +1183,7 @@ components: description: >- Non-string JSON output is JSON-encoded, or null when absent. Stored scalar strings are returned unchanged. Decode a saved household - calculation envelope once to recover spm_config and spm_provenance + calculation envelope once to recover spm_provenance beside its result, not beside this record. x-household-envelope-schema: $ref: '#/components/schemas/StoredHouseholdOutputEnvelope' @@ -1220,7 +1212,7 @@ components: type: object description: >- Household calculation envelope saved as simulation output. Keep result - and both SPM fields together. Legacy calculations omit SPM fields; this + and SPM provenance together. Legacy calculations omit SPM provenance; this descriptive schema does not add validation to the persistence endpoint. properties: status: @@ -1231,8 +1223,6 @@ components: result: type: object nullable: true - spm_config: - $ref: '#/components/schemas/SPMResolvedConfiguration' spm_provenance: $ref: '#/components/schemas/SPMProvenance' SimulationError: @@ -1275,8 +1265,6 @@ components: $ref: '#/components/schemas/SPMSelection' SPMSelection: description: Generated from policyengine_api.spm.SPMSelection. - SPMResolvedConfiguration: - description: Generated from policyengine_api.spm.SPMResolvedConfiguration. SPMRuntimeVersions: description: Generated from policyengine_api.spm.SPMRuntimeVersions. SPMProvenance: diff --git a/policyengine_api/routes/household_routes.py b/policyengine_api/routes/household_routes.py index 071102b4c..300f2d9d3 100644 --- a/policyengine_api/routes/household_routes.py +++ b/policyengine_api/routes/household_routes.py @@ -137,10 +137,8 @@ def _calculation_response(calculation) -> dict: result = dict(status="ok", message=None, result=calculation.household) if calculation.warnings: result["warnings"] = list(calculation.warnings) - for field in ("spm_config", "spm_provenance"): - value = getattr(calculation, field, None) - if value is not None: - result[field] = value + if calculation.spm_provenance is not None: + result["spm_provenance"] = calculation.spm_provenance return result diff --git a/policyengine_api/runtime_cache/household_calculations.py b/policyengine_api/runtime_cache/household_calculations.py index 966afa329..4ab7ceaf7 100644 --- a/policyengine_api/runtime_cache/household_calculations.py +++ b/policyengine_api/runtime_cache/household_calculations.py @@ -10,10 +10,13 @@ CacheNamespace, RecoverableJSONCache, ) -from policyengine_api.spm import validate_spm_calculation_provenance +from policyengine_api.spm import ( + resolved_spm_configuration, + validate_spm_calculation_provenance, +) -HOUSEHOLD_CALCULATION_SCHEMA_VERSION = 3 +HOUSEHOLD_CALCULATION_SCHEMA_VERSION = 4 HOUSEHOLD_CALCULATION_TTL_SECONDS = 86_400 @@ -33,7 +36,6 @@ class HouseholdCalculationIdentity: class CachedHouseholdCalculation: household: dict[str, Any] warnings: tuple[str, ...] = () - spm_config: dict[str, Any] | None = None spm_provenance: dict[str, Any] | None = None @@ -65,30 +67,28 @@ def get( return None if not all(isinstance(warning, str) for warning in warnings): return None - spm_config = payload.get("spm_config") + if "spm_config" in payload: + return None spm_provenance = payload.get("spm_provenance") - if any( - value is not None and not isinstance(value, dict) - for value in (spm_config, spm_provenance) - ): + if spm_provenance is not None and not isinstance(spm_provenance, dict): return None if identity.spm is None: - if spm_config is not None or spm_provenance is not None: + if spm_provenance is not None: return None else: try: - calculation = validate_spm_calculation_provenance( - spm_config, - spm_provenance, - ) + receipt = validate_spm_calculation_provenance(spm_provenance) except (ValidationError, ValueError): return None - if calculation.spm_config.model_dump(mode="json") != identity.spm: + if ( + resolved_spm_configuration(receipt).model_dump(mode="json") + != identity.spm + ): return None + spm_provenance = receipt.model_dump(mode="json", by_alias=True) return CachedHouseholdCalculation( household=household, warnings=tuple(warnings), - spm_config=spm_config, spm_provenance=spm_provenance, ) @@ -98,21 +98,15 @@ def set( value: CachedHouseholdCalculation, ) -> bool: if identity.spm is None: - if value.spm_config is not None or value.spm_provenance is not None: + if value.spm_provenance is not None: return False return self._cache.set(asdict(identity), asdict(value)) try: - calculation = validate_spm_calculation_provenance( - value.spm_config, - value.spm_provenance, - ) + receipt = validate_spm_calculation_provenance(value.spm_provenance) except (ValidationError, ValueError): return False - if calculation.spm_config.model_dump(mode="json") != identity.spm: + if resolved_spm_configuration(receipt).model_dump(mode="json") != identity.spm: return False payload = asdict(value) - payload["spm_config"] = calculation.spm_config.model_dump(mode="json") - payload["spm_provenance"] = calculation.spm_provenance.model_dump( - mode="json", by_alias=True - ) + payload["spm_provenance"] = receipt.model_dump(mode="json", by_alias=True) return self._cache.set(asdict(identity), payload) diff --git a/policyengine_api/runtime_cache/reform_impacts.py b/policyengine_api/runtime_cache/reform_impacts.py index 09c96c0f0..8a7ed5dc8 100644 --- a/policyengine_api/runtime_cache/reform_impacts.py +++ b/policyengine_api/runtime_cache/reform_impacts.py @@ -19,7 +19,7 @@ record_cache_event, ) -REFORM_IMPACT_SCHEMA_VERSION = 2 +REFORM_IMPACT_SCHEMA_VERSION = 3 REFORM_IMPACT_TTL_SECONDS = 2_592_000 REFORM_IMPACT_INDEX_LIMIT = 1_000 REFORM_IMPACT_START_CLAIM_TTL_SECONDS = 300 diff --git a/policyengine_api/services/budget_window_cache.py b/policyengine_api/services/budget_window_cache.py index 9da6475ac..b6190341a 100644 --- a/policyengine_api/services/budget_window_cache.py +++ b/policyengine_api/services/budget_window_cache.py @@ -20,7 +20,7 @@ from policyengine_api.runtime_cache.dependencies import get_runtime_cache_context BUDGET_WINDOW_CACHE_FAMILY = "budget-window" -BUDGET_WINDOW_CACHE_SCHEMA_VERSION = 3 +BUDGET_WINDOW_CACHE_SCHEMA_VERSION = 4 BUDGET_WINDOW_STARTING_TTL_SECONDS = 300 BUDGET_WINDOW_BATCH_TTL_SECONDS = 86_400 BUDGET_WINDOW_RESULT_TTL_SECONDS = 2_592_000 diff --git a/policyengine_api/services/household_calculation_service.py b/policyengine_api/services/household_calculation_service.py index ff8f90845..bce5743c7 100644 --- a/policyengine_api/services/household_calculation_service.py +++ b/policyengine_api/services/household_calculation_service.py @@ -28,6 +28,7 @@ from policyengine_api.utils.input_validation import find_unrecognized_inputs from policyengine_api.spm import ( normalize_spm_selection, + resolved_spm_configuration, validate_spm_calculation_provenance, ) @@ -36,7 +37,6 @@ class CalculationResult: household: dict warnings: tuple[str, ...] = () - spm_config: dict | None = None spm_provenance: dict | None = None @@ -45,7 +45,6 @@ class HouseholdCalculationResult: household: dict warnings: tuple[str, ...] = () cached: bool = False - spm_config: dict | None = None spm_provenance: dict | None = None @@ -76,26 +75,19 @@ def __init__(self, invalid_inputs: list[Any]) -> None: super().__init__("Household or policy contains unrecognized inputs") -def _validated_spm_fields( - spm_config: object, +def _validated_spm_provenance( spm_provenance: object, expected_selection: dict | None, -) -> tuple[dict | None, dict | None]: +) -> dict | None: if expected_selection is None: - if spm_config is None and spm_provenance is None: - return None, None + if spm_provenance is None: + return None raise ValueError("Unexpected SPM provenance for a calculation without SPM") - calculation = validate_spm_calculation_provenance( - spm_config, - spm_provenance, - ) - canonical_config = calculation.spm_config.model_dump(mode="json") + receipt = validate_spm_calculation_provenance(spm_provenance) + canonical_config = resolved_spm_configuration(receipt).model_dump(mode="json") if canonical_config != expected_selection: raise ValueError("SPM calculation configuration differs from the request") - return ( - canonical_config, - calculation.spm_provenance.model_dump(mode="json", by_alias=True), - ) + return receipt.model_dump(mode="json", by_alias=True) def get_household_year(household: dict) -> int | str: @@ -217,7 +209,6 @@ def _store_result( CachedHouseholdCalculation( household=calculation.household, warnings=warnings, - spm_config=calculation.spm_config, spm_provenance=calculation.spm_provenance, ), ) @@ -269,7 +260,6 @@ def calculate_stored_household( household=cached.household, warnings=cached.warnings, cached=True, - spm_config=cached.spm_config, spm_provenance=cached.spm_provenance, ) @@ -321,7 +311,6 @@ def calculate_stored_household( calculation = CalculationResult( household=raw_calculation.household, warnings=tuple(getattr(raw_calculation, "warnings", ())), - spm_config=getattr(raw_calculation, "spm_config", None), spm_provenance=getattr(raw_calculation, "spm_provenance", None), ) else: @@ -330,14 +319,12 @@ def calculate_stored_household( calculation = CalculationResult( household=raw_calculation, ) - spm_config, spm_provenance = _validated_spm_fields( - calculation.spm_config, + spm_provenance = _validated_spm_provenance( calculation.spm_provenance, spm, ) calculation = replace( calculation, - spm_config=spm_config, spm_provenance=spm_provenance, ) record_cache_event( @@ -357,7 +344,6 @@ def calculate_stored_household( return HouseholdCalculationResult( household=calculation.household, warnings=response_warnings, - spm_config=calculation.spm_config, spm_provenance=calculation.spm_provenance, ) @@ -433,15 +419,13 @@ def calculate_prepared_household( else: household = raw_calculation.household calculation_warnings = tuple(getattr(raw_calculation, "warnings", ())) - spm_config, spm_provenance = _validated_spm_fields( - getattr(raw_calculation, "spm_config", None), + spm_provenance = _validated_spm_provenance( getattr(raw_calculation, "spm_provenance", None), prepared.spm, ) return HouseholdCalculationResult( household=household, warnings=(prepared.warnings + calculation_warnings), - spm_config=spm_config, spm_provenance=spm_provenance, ) diff --git a/policyengine_api/services/report_output_service.py b/policyengine_api/services/report_output_service.py index 49cb50051..69246e827 100644 --- a/policyengine_api/services/report_output_service.py +++ b/policyengine_api/services/report_output_service.py @@ -20,7 +20,6 @@ from policyengine_api.spm import ( SPMComparisonProvenance, error_message, - validate_spm_calculation_provenance, ) @@ -29,34 +28,23 @@ def _canonicalize_annual_spm_output( *, expected_year: str | None, ) -> JSONObject: - """Validate and serialize the compact SPM fields on one economy result.""" + """Validate and serialize compact SPM provenance on one economy result.""" - has_config = "spm_config" in output - has_provenance = "spm_provenance" in output - if not has_config or not has_provenance: + if "spm_config" in output: raise ValueError( - "Completed US economy output requires both SPM configuration and provenance" + "Completed US economy output must not include legacy spm_config" ) + if "spm_provenance" not in output: + raise ValueError("Completed US economy output requires SPM provenance") comparison = SPMComparisonProvenance.model_validate(output["spm_provenance"]) - calculations = tuple( - validate_spm_calculation_provenance( - output["spm_config"], - execution.receipt, - ) - for execution in (comparison.baseline, comparison.reform) - ) if expected_year is not None and any( - expected_year not in calculation.spm_provenance.years - for calculation in calculations + expected_year not in execution.receipt.years + for execution in (comparison.baseline, comparison.reform) ): raise ValueError("SPM receipt does not cover the report year") return { **output, - "spm_config": cast( - JSONValue, - calculations[0].spm_config.model_dump(mode="json"), - ), "spm_provenance": cast( JSONValue, comparison.model_dump(mode="json", by_alias=True), diff --git a/policyengine_api/services/simulation_service.py b/policyengine_api/services/simulation_service.py index 64ac114eb..943b0fb00 100644 --- a/policyengine_api/services/simulation_service.py +++ b/policyengine_api/services/simulation_service.py @@ -55,15 +55,14 @@ def _latest_successful_run_id(runs: list[SimulationRun]) -> str | None: @staticmethod def _has_stale_spm_output(output: object) -> bool: - if not isinstance(output, dict) or not ( - "spm_config" in output or "spm_provenance" in output - ): + if not isinstance(output, dict): + return False + if "spm_config" in output: + return True + if "spm_provenance" not in output: return False try: - validate_spm_calculation_provenance( - output.get("spm_config"), - output.get("spm_provenance"), - ) + validate_spm_calculation_provenance(output["spm_provenance"]) except ValueError: return True return False @@ -274,16 +273,17 @@ def update_simulation( if isinstance(values.get("output"), str): values["output"] = json.loads(values["output"]) output_value = values.get("output") - if isinstance(output_value, dict) and ( - "spm_config" in output_value or "spm_provenance" in output_value - ): - calculation = validate_spm_calculation_provenance( - output_value.get("spm_config"), - output_value.get("spm_provenance"), + if isinstance(output_value, dict) and "spm_config" in output_value: + raise ValueError( + "Completed simulation output must not include legacy spm_config" + ) + if isinstance(output_value, dict) and "spm_provenance" in output_value: + receipt = validate_spm_calculation_provenance( + output_value["spm_provenance"] ) values["output"] = { **output_value, - **calculation.model_dump(mode="json", by_alias=True), + "spm_provenance": receipt.model_dump(mode="json", by_alias=True), } with self._sessions.begin() as session: simulation = self._select_simulation( diff --git a/policyengine_api/services/v2/reports/types.py b/policyengine_api/services/v2/reports/types.py index 9d0298f43..ef4d997f4 100644 --- a/policyengine_api/services/v2/reports/types.py +++ b/policyengine_api/services/v2/reports/types.py @@ -16,6 +16,7 @@ SimulationRole, StrictContractModel, ) +from policyengine_api.spm import SPMComparisonProvenance ContractVersion = Literal[1] @@ -102,3 +103,13 @@ def require_unique_aggregates( if len(value) != len(set(value)): raise ValueError("requested aggregates must be unique") return value + + @model_validator(mode="after") + def validate_spm_provenance(self) -> AggregateReportArtifactPayload: + """Reject duplicate or noncanonical SPM data in aggregate artifacts.""" + + if "spm_config" in self.result: + raise ValueError("aggregate result must not include legacy spm_config") + if "spm_provenance" in self.result: + SPMComparisonProvenance.model_validate(self.result["spm_provenance"]) + return self diff --git a/policyengine_api/services/v2/simulations/types.py b/policyengine_api/services/v2/simulations/types.py index 6e9605102..d1e466b13 100644 --- a/policyengine_api/services/v2/simulations/types.py +++ b/policyengine_api/services/v2/simulations/types.py @@ -16,7 +16,7 @@ ) from policyengine_api.query_parameters import CountryId -from policyengine_api.spm import SPMCalculationProvenance +from policyengine_api.spm import SPMProvenance ContractText = Annotated[str, Field(min_length=1, max_length=255)] ContractVersion = Literal[1] @@ -206,4 +206,4 @@ class SimulationArtifactDescriptor(StrictContractModel): output_schema_version: ContractVersion = 1 row_identity: RowIdentity bundle: BundleProvenance - calculation_provenance: SPMCalculationProvenance | None = None + calculation_provenance: SPMProvenance | None = None diff --git a/policyengine_api/specification.py b/policyengine_api/specification.py index 2830a359c..e2c9296b6 100644 --- a/policyengine_api/specification.py +++ b/policyengine_api/specification.py @@ -17,7 +17,6 @@ SPMComparisonProvenance, SPMExecutionProvenance, SPMProvenance, - SPMResolvedConfiguration, SPMRuntimeVersions, SPMSelection, ) @@ -90,7 +89,6 @@ def load_specification( schemas = document.get("components", {}).get("schemas", {}) for name, model in { "SPMSelection": SPMSelection, - "SPMResolvedConfiguration": SPMResolvedConfiguration, "SPMRuntimeVersions": SPMRuntimeVersions, "SPMProvenance": SPMProvenance, "SPMExecutionProvenance": SPMExecutionProvenance, diff --git a/policyengine_api/spm.py b/policyengine_api/spm.py index 7ea5ba638..e48bde68d 100644 --- a/policyengine_api/spm.py +++ b/policyengine_api/spm.py @@ -174,30 +174,6 @@ class SPMRuntimeVersions(BaseModel): SPMProvenance.model_rebuild() -class SPMCalculationProvenance(BaseModel): - """A resolved SPM configuration and its matching compact receipt.""" - - model_config = ConfigDict(frozen=True, extra="forbid") - spm_config: SPMResolvedConfiguration - spm_provenance: SPMProvenance - - @model_validator(mode="after") - def validate_matching_receipt(self) -> "SPMCalculationProvenance": - receipt = self.spm_provenance - config = self.spm_config - receipt_as_of = receipt.as_of.isoformat() if receipt.as_of is not None else None - if ( - receipt.forecast_sha256 != config.forecast_content_sha256 - or receipt.scenario != config.scenario - or receipt.geography_kind != config.geography_kind - or receipt.geography_id != config.geography_id - or receipt.county_vintage != config.county_vintage - or receipt_as_of != config.as_of - ): - raise ValueError("SPM receipt differs from the SPM configuration") - return self - - class SPMExecutionProvenance(BaseModel): """One shared compact receipt and the executions that produced it.""" @@ -259,7 +235,7 @@ def _required_string(source: Mapping[str, object], field: str) -> str: def build_spm_calculation_provenance( config: SPMSelection, receipt: Mapping[str, object], -) -> SPMCalculationProvenance: +) -> SPMProvenance: """Reduce the country-owned rich receipt to the public compact contract.""" forecast_sha256 = _required_string(receipt, "forecast_sha256") @@ -285,23 +261,27 @@ def build_spm_calculation_provenance( if not isinstance(source_versions, Mapping): raise ValueError("Country SPM receipt requires runtime_versions") runtime_versions = SPMRuntimeVersions.model_validate(source_versions) - compact = build_spm_provenance( + return build_spm_provenance( forecast_id=_required_string(receipt, "forecast_id"), forecast_sha256=forecast_sha256, selection=config, years=tuple(sorted(years)), runtime_versions=runtime_versions, ) - return SPMCalculationProvenance( - spm_config=SPMResolvedConfiguration( - forecast_content_sha256=forecast_sha256, - scenario=scenario, - geography_kind=config.geography_kind, - geography_id=config.geography_id, - county_vintage=config.county_vintage, - as_of=config.as_of, - ), - spm_provenance=compact, + + +def resolved_spm_configuration( + provenance: SPMProvenance, +) -> SPMResolvedConfiguration: + """Derive the complete resolved selection from its compact receipt.""" + + return SPMResolvedConfiguration( + forecast_content_sha256=provenance.forecast_sha256, + scenario=provenance.scenario, + geography_kind=provenance.geography_kind, + geography_id=provenance.geography_id, + county_vintage=provenance.county_vintage, + as_of=provenance.as_of.isoformat() if provenance.as_of is not None else None, ) @@ -432,15 +412,11 @@ def resolved_spm_settings(settings: object) -> dict | None: def validate_spm_calculation_provenance( - config: object, provenance: object, -) -> SPMCalculationProvenance: - """Validate a transported compact receipt against its resolved selection.""" +) -> SPMProvenance: + """Validate the sole transported compact receipt for a completed result.""" - return SPMCalculationProvenance( - spm_config=SPMResolvedConfiguration.model_validate(config), - spm_provenance=SPMProvenance.model_validate(provenance), - ) + return SPMProvenance.model_validate(provenance) def _current_bundle() -> dict: @@ -656,7 +632,7 @@ def calculation_spm_receipt(simulation) -> dict: SPMSelection.model_validate(resolved), source, ) - return calculation.model_dump(mode="json", by_alias=True) + return {"spm_provenance": calculation.model_dump(mode="json", by_alias=True)} except (ValidationError, ValueError) as error: raise SPMValidationError( "SPM_CONFIGURATION_UNAVAILABLE", diff --git a/policyengine_api/worker_spm.py b/policyengine_api/worker_spm.py index e4941a237..2d9977d90 100644 --- a/policyengine_api/worker_spm.py +++ b/policyengine_api/worker_spm.py @@ -6,7 +6,7 @@ SPMValidationError, error_message, normalize_spm_selection, - resolved_spm_settings, + resolved_spm_configuration, validate_spm_calculation_provenance, ) @@ -96,20 +96,17 @@ def validate_worker_result( return if expected_years is not None: raise ValueError("Expected a budget window SPM result") - resolved = resolved_spm_settings(result.get("spm_config")) - if resolved is None: - raise ValueError("Worker result has incomplete resolved SPM settings") - if resolved != selection: - raise ValueError("Worker result SPM settings differ from the request") + if "spm_config" in result: + raise ValueError("Worker result must not include legacy spm_config") comparison = SPMComparisonProvenance.model_validate( result.get("spm_provenance") ) for side in ("baseline", "reform"): execution = getattr(comparison, side) - receipt = validate_spm_calculation_provenance( - resolved, - execution.receipt, - ).spm_provenance + receipt = validate_spm_calculation_provenance(execution.receipt) + resolved = resolved_spm_configuration(receipt).model_dump(mode="json") + if resolved != selection: + raise ValueError("Worker result SPM settings differ from the request") if expected_year is not None and str(expected_year) not in receipt.years: raise ValueError("Worker SPM receipt does not cover requested year") except (ValueError, TypeError, KeyError) as exc: diff --git a/tests/integration/test_runtime_cache_redis.py b/tests/integration/test_runtime_cache_redis.py index 2d7c571cc..e12b04575 100644 --- a/tests/integration/test_runtime_cache_redis.py +++ b/tests/integration/test_runtime_cache_redis.py @@ -120,7 +120,6 @@ def test_cached_calculation_receipts_survive_a_real_round_trip(redis_pair) -> No value = CachedHouseholdCalculation( household={"people": {"you": {}}}, warnings=(), - spm_config=selection, spm_provenance={ "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-artifact", @@ -143,7 +142,6 @@ def test_cached_calculation_receipts_survive_a_real_round_trip(redis_pair) -> No assert HouseholdCalculationCache(first, namespace).set(identity, value) cached = HouseholdCalculationCache(second, namespace).get(identity) assert cached is not None - assert cached.spm_config == selection assert cached.spm_provenance == value.spm_provenance other = HouseholdCalculationIdentity( **{**vars(identity), "spm": {**selection, "geography_kind": "county"}} diff --git a/tests/unit/routes/test_canonical_spm.py b/tests/unit/routes/test_canonical_spm.py index 95f51cb83..767de092a 100644 --- a/tests/unit/routes/test_canonical_spm.py +++ b/tests/unit/routes/test_canonical_spm.py @@ -86,7 +86,7 @@ def calculate(self, household, policy, **kwargs): "years": ["2026"], } ) - return CalculationResult(household, (), config, receipt) + return CalculationResult(household, (), receipt) @pytest.fixture @@ -221,7 +221,7 @@ def test_settings_and_provenance_survive_storage_reform_replay_and_replacement( cached = client.get(url + "/policy/2") assert first.status_code == cached.status_code == 200 assert first.json == cached.json - assert first.json["spm_config"] == stored["spm"] + assert "spm_config" not in first.json assert first.json["spm_provenance"]["forecast_sha256"] == FORECAST_HASH assert len(country.calls) == 1 assert "spm" not in country.calls[0][0] @@ -257,7 +257,7 @@ def test_settings_and_provenance_survive_storage_reform_replay_and_replacement( latest = client.get(latest_url).json["result"] assert latest["household_hash"] != edited["household_hash"] assert ( - client.get(latest_url + "/policy/2").json["spm_config"]["geography_kind"] + client.get(latest_url + "/policy/2").json["spm_provenance"]["geography_kind"] == "county" ) assert len(country.calls) == 4 @@ -286,7 +286,7 @@ def test_http_cache_varies_with_measurement_settings(certified, harness, selecti second = client.post("/us/calculate", json=payload) assert second.status_code == 200 assert len(country.calls) == 2 - assert first.json["spm_config"] != second.json["spm_config"] + assert first.json["spm_provenance"] != second.json["spm_provenance"] def test_successful_http_cache_hit_still_starts_observability(certified, harness): @@ -336,7 +336,7 @@ def test_http_cache_default_artifact_hash_is_part_of_identity(certified, harness second = client.post("/us/calculate", json=payload) assert first.status_code == second.status_code == 200 assert len(country.calls) == 2 - assert first.json["spm_config"] != second.json["spm_config"] + assert first.json["spm_provenance"] != second.json["spm_provenance"] def test_linked_household_measurement_is_immutable_for_saved_reports( @@ -544,8 +544,8 @@ def test_valid_spm_database_timeout_keeps_stage11_safe_persistence_response( copy_event.assert_not_called() -def test_stored_replay_rejects_a_receipt_config_that_omits_nulls(certified, harness): - """Country output must state every field in its resolved configuration.""" +def test_stored_replay_rejects_a_receipt_that_omits_null_settings(certified, harness): + """The sole receipt must state every field in its resolved configuration.""" client, country = harness calculate = country.calculate @@ -554,12 +554,11 @@ def omitting_null_settings(household, policy, **kwargs): return CalculationResult( household=result.household, warnings=result.warnings, - spm_config={ + spm_provenance={ key: value - for key, value in result.spm_config.items() + for key, value in result.spm_provenance.items() if value is not None }, - spm_provenance=result.spm_provenance, ) country.calculate = omitting_null_settings diff --git a/tests/unit/routes/test_spm_year_worker_polling.py b/tests/unit/routes/test_spm_year_worker_polling.py index d68b32d38..3faa0ab02 100644 --- a/tests/unit/routes/test_spm_year_worker_polling.py +++ b/tests/unit/routes/test_spm_year_worker_polling.py @@ -49,7 +49,6 @@ def segmented_result(year, invalid_side=None): } result = { "year": year, - "spm_config": SELECTION, "spm_provenance": { "schema_version": "canonical-spm-comparison-v2", "baseline": {"receipt": dict(receipt), "execution_count": 2}, diff --git a/tests/unit/runtime_cache/test_household_calculations.py b/tests/unit/runtime_cache/test_household_calculations.py index 570a9e3f6..cfe422efe 100644 --- a/tests/unit/runtime_cache/test_household_calculations.py +++ b/tests/unit/runtime_cache/test_household_calculations.py @@ -129,7 +129,6 @@ def test_household_cache_rejects_invalid_or_mismatched_spm_receipts(receipt): identity = _identity(spm=SPM_CONFIG) value = CachedHouseholdCalculation( household={"people": {}}, - spm_config=SPM_CONFIG, spm_provenance=receipt, ) assert cache.set(identity, value) is False @@ -157,7 +156,6 @@ def test_household_cache_rejects_previous_diagnostic_receipt_before_write(): identity, CachedHouseholdCalculation( household={"people": {}}, - spm_config=SPM_CONFIG, spm_provenance=previous_receipt, ), ) @@ -172,13 +170,11 @@ def test_household_cache_keeps_valid_receipts_including_lazy_tax_only(years): identity = _identity(spm=SPM_CONFIG) value = CachedHouseholdCalculation( household={"people": {}}, - spm_config=SPM_CONFIG, spm_provenance={**SPM_RECEIPT, "years": years}, ) assert cache.set(identity, value) is True cached = cache.get(identity) assert cached is not None - assert cached.spm_config == SPM_CONFIG assert cached.spm_provenance == {**SPM_RECEIPT, "years": years} @@ -196,52 +192,41 @@ def test_real_country_provider_receipt_round_trips_household_cache(calculate_spm config = country_spm.spm_config(provider) receipt = build_spm_calculation_provenance( SPMSelection.model_validate(config), provider.provenance() - ).spm_provenance.model_dump(mode="json", by_alias=True) + ).model_dump(mode="json", by_alias=True) assert bool(receipt["years"]) is calculate_spm cache = HouseholdCalculationCache(InMemoryCacheBackend(), _namespace()) identity = _identity(spm=config) value = CachedHouseholdCalculation( household={"people": {}}, - spm_config=config, spm_provenance=receipt, ) assert cache.set(identity, value) is True cached = cache.get(identity) assert cached is not None - assert cached.spm_config == config assert cached.spm_provenance == receipt -OMITTED_NULL_CONFIG = { - key: value for key, value in SPM_CONFIG.items() if value is not None -} - - -def test_household_cache_rejects_receipt_omitting_null_settings(): - """A compact receipt's adjacent config must contain all six fields.""" - cache = HouseholdCalculationCache(InMemoryCacheBackend(), _namespace()) - identity = _identity(spm=SPM_CONFIG) - value = CachedHouseholdCalculation( - household={"people": {}}, - spm_config=OMITTED_NULL_CONFIG, - spm_provenance=SPM_RECEIPT, - ) - assert cache.set(identity, value) is False - assert cache.get(identity) is None - - -@pytest.mark.parametrize("omitted", sorted(OMITTED_NULL_CONFIG)) -def test_household_cache_rejects_receipt_omitting_a_nonnull_setting(omitted): - """An omitted non-null setting must never inherit today's resolved default.""" +@pytest.mark.parametrize( + "omitted", + [ + "forecast_sha256", + "scenario", + "geography_kind", + "geography_id", + "county_vintage", + "as_of", + ], +) +def test_household_cache_rejects_receipt_omitting_resolved_setting(omitted): + """Every resolved setting is required on the sole completed-result receipt.""" cache = HouseholdCalculationCache(InMemoryCacheBackend(), _namespace()) identity = _identity(spm=SPM_CONFIG) value = CachedHouseholdCalculation( household={"people": {}}, - spm_config={ - key: item for key, item in OMITTED_NULL_CONFIG.items() if key != omitted + spm_provenance={ + key: item for key, item in SPM_RECEIPT.items() if key != omitted }, - spm_provenance=SPM_RECEIPT, ) assert cache.set(identity, value) is False assert cache.get(identity) is None @@ -263,8 +248,28 @@ def test_household_cache_rejects_receipt_with_different_settings(changed): identity = _identity(spm=SPM_CONFIG) value = CachedHouseholdCalculation( household={"people": {}}, - spm_config={**OMITTED_NULL_CONFIG, **changed}, - spm_provenance=SPM_RECEIPT, + spm_provenance={**SPM_RECEIPT, **changed}, ) assert cache.set(identity, value) is False assert cache.get(identity) is None + + +def test_household_cache_rejects_legacy_dual_field_payload() -> None: + backend = InMemoryCacheBackend() + cache = HouseholdCalculationCache(backend, _namespace()) + identity = _identity(spm=SPM_CONFIG) + backend.set( + cache.cache_key(identity), + encode_envelope( + "household-calculation", + HOUSEHOLD_CALCULATION_SCHEMA_VERSION, + { + "household": {"people": {}}, + "warnings": [], + "spm_config": SPM_CONFIG, + "spm_provenance": SPM_RECEIPT, + }, + ), + ) + + assert cache.get(identity) is None diff --git a/tests/unit/runtime_cache/test_reform_impacts.py b/tests/unit/runtime_cache/test_reform_impacts.py index c7ef0877d..26a02cca7 100644 --- a/tests/unit/runtime_cache/test_reform_impacts.py +++ b/tests/unit/runtime_cache/test_reform_impacts.py @@ -11,6 +11,7 @@ ) from policyengine_api.runtime_cache.fake import InMemoryCacheBackend from policyengine_api.runtime_cache.reform_impacts import ( + REFORM_IMPACT_SCHEMA_VERSION, REFORM_IMPACT_START_CLAIM_TTL_SECONDS, CachedReformImpact, ReformImpactCache, @@ -198,12 +199,15 @@ def test_previous_schema_reform_impact_is_not_read() -> None: namespace = _namespace() cache = ReformImpactCache(backend, namespace) legacy = _impact("legacy", "[spm=canonical]", 1) - legacy_key = namespace.key("reform-impact", 1, {"execution_id": "legacy"}) + previous_version = REFORM_IMPACT_SCHEMA_VERSION - 1 + legacy_key = namespace.key( + "reform-impact", previous_version, {"execution_id": "legacy"} + ) backend.set( legacy_key, encode_envelope( "reform-impact", - 1, + previous_version, { **legacy.__dict__, "start_time": legacy.start_time.isoformat(), diff --git a/tests/unit/services/test_budget_window_cache.py b/tests/unit/services/test_budget_window_cache.py index 24e3896c8..ff344e924 100644 --- a/tests/unit/services/test_budget_window_cache.py +++ b/tests/unit/services/test_budget_window_cache.py @@ -7,6 +7,7 @@ from policyengine_api.runtime_cache.fake import InMemoryCacheBackend from policyengine_api.services.budget_window_cache import ( BUDGET_WINDOW_BATCH_TTL_SECONDS, + BUDGET_WINDOW_CACHE_SCHEMA_VERSION, BUDGET_WINDOW_STARTING_TTL_SECONDS, BudgetWindowCache, BudgetWindowCacheState, @@ -81,7 +82,7 @@ def test_build_key_is_stable_for_request_identity(): ) assert first == second - assert first.startswith("policyengine:test:api:budget-window:v3:") + assert first.startswith("policyengine:test:api:budget-window:v4:") def test_previous_schema_cache_identity_is_not_read(): @@ -98,12 +99,13 @@ def test_previous_schema_cache_identity_is_not_read(): "region": "us", "time_period": "budget_window:2026:2", } - old_key = namespace.key("budget-window", 2, identity) + previous_version = BUDGET_WINDOW_CACHE_SCHEMA_VERSION - 1 + old_key = namespace.key("budget-window", previous_version, identity) backend.set( f"{old_key}:state", encode_envelope( "budget-window", - 2, + previous_version, {"status": "completed", "result": {"legacy": True}}, ), ) diff --git a/tests/unit/services/test_report_output_service.py b/tests/unit/services/test_report_output_service.py index b264a8129..92344b68e 100644 --- a/tests/unit/services/test_report_output_service.py +++ b/tests/unit/services/test_report_output_service.py @@ -39,7 +39,6 @@ def compact_economy_output() -> dict: return { - "spm_config": SPM_CONFIG, "spm_provenance": { "schema_version": "canonical-spm-comparison-v2", "baseline": {"receipt": SPM_RECEIPT, "execution_count": 2}, @@ -254,21 +253,25 @@ def test_update_complete_persists_only_compact_society_wide_spm_output( @pytest.mark.parametrize( - "output", + ("output", "error"), [ - {"budgetary_impact": 42}, - {"budgetary_impact": 42, "spm_config": SPM_CONFIG}, - { - "budgetary_impact": 42, - "spm_provenance": compact_economy_output()["spm_provenance"], - }, + ({"budgetary_impact": 42}, "requires SPM provenance"), + ( + {"budgetary_impact": 42, "spm_config": SPM_CONFIG}, + "legacy spm_config", + ), + ( + {**compact_economy_output(), "spm_config": SPM_CONFIG}, + "legacy spm_config", + ), ], - ids=["absent", "config-only", "provenance-only"], + ids=["absent", "config-only", "dual-field"], ) -def test_update_complete_requires_both_society_wide_spm_fields( +def test_update_complete_requires_only_compact_society_wide_spm_provenance( service, orm_session_factory, output, + error, ): baseline = create_simulation( orm_session_factory, @@ -286,7 +289,7 @@ def test_update_complete_requires_both_society_wide_spm_fields( "us", baseline.id, reform.id, "2025" ).view.report_output - with pytest.raises(ValueError, match="requires both SPM"): + with pytest.raises(ValueError, match=error): service.update_report_output("us", report.id, status="complete", output=output) with orm_session_factory() as session: @@ -380,9 +383,9 @@ def test_resolve_legacy_spm_report_returns_new_empty_cache_record( population_id="us", population_type="geography", ) - monkeypatch.setitem(constants.RUNTIME_CACHE_SCHEMA_VERSIONS, "report_output", 1) - legacy_cache_version = get_report_output_cache_version("us") monkeypatch.setitem(constants.RUNTIME_CACHE_SCHEMA_VERSIONS, "report_output", 2) + legacy_cache_version = get_report_output_cache_version("us") + monkeypatch.setitem(constants.RUNTIME_CACHE_SCHEMA_VERSIONS, "report_output", 3) assert legacy_cache_version != get_report_output_cache_version("us") with orm_session_factory.begin() as session: legacy = ReportOutput( diff --git a/tests/unit/services/test_simulation_service.py b/tests/unit/services/test_simulation_service.py index 5e7b224bb..464ae50b9 100644 --- a/tests/unit/services/test_simulation_service.py +++ b/tests/unit/services/test_simulation_service.py @@ -162,12 +162,12 @@ def test_update_simulation_accepts_legacy_json_text_at_wire_boundary(service): assert updated.output == {"result": 42} -def test_update_simulation_rejects_incomplete_spm_provenance(service): +def test_update_simulation_rejects_legacy_spm_config(service): simulation = service.get_or_create_simulation( "us", "household-1", "household", 1 ).simulation - with pytest.raises(ValueError, match="SPM"): + with pytest.raises(ValueError, match="legacy spm_config"): service.update_simulation( "us", simulation.id, @@ -176,18 +176,24 @@ def test_update_simulation_rejects_incomplete_spm_provenance(service): ) +def test_update_simulation_rejects_incomplete_spm_provenance(service): + simulation = service.get_or_create_simulation( + "us", "household-1", "household", 1 + ).simulation + + with pytest.raises(ValueError, match="Field required"): + service.update_simulation( + "us", + simulation.id, + status="complete", + output={"spm_provenance": {"geography_kind": "national"}}, + ) + + def test_update_simulation_persists_only_compact_spm_provenance(service): simulation = service.get_or_create_simulation( "us", "household-1", "household", 1 ).simulation - config = { - "forecast_content_sha256": "a" * 64, - "scenario": "ce_trend", - "geography_kind": "national", - "geography_id": None, - "county_vintage": "2020", - "as_of": None, - } provenance = { "schema_version": "canonical-spm-provenance-v2", "forecast_id": "spm-rolling-2026-09-09", @@ -210,12 +216,11 @@ def test_update_simulation_persists_only_compact_spm_provenance(service): "us", simulation.id, status="complete", - output={"result": 42, "spm_config": config, "spm_provenance": provenance}, + output={"result": 42, "spm_provenance": provenance}, ) assert updated.output == { "result": 42, - "spm_config": config, "spm_provenance": provenance, } diff --git a/tests/unit/services/test_worker_spm.py b/tests/unit/services/test_worker_spm.py index 4cb17b7a6..e3b35f8d3 100644 --- a/tests/unit/services/test_worker_spm.py +++ b/tests/unit/services/test_worker_spm.py @@ -444,7 +444,6 @@ def test_worker_cache_receipts_are_required_and_json_roundtrip(): from policyengine_api.worker_spm import validate_worker_result output = { - "spm_config": SELECTION, "spm_provenance": worker_comparison(), } validate_worker_result(json.loads(json.dumps(output)), SELECTION) @@ -461,11 +460,17 @@ def test_worker_cache_receipts_are_required_and_json_roundtrip(): {"spm_config": SELECTION}, { **output, - "spm_config": { - key: value for key, value in SELECTION.items() if value is not None - }, + "spm_config": SELECTION, + }, + { + "spm_provenance": worker_comparison( + baseline={ + key: value + for key, value in worker_receipt().items() + if key != "scenario" + } + ) }, - {**output, "spm_config": {**SELECTION, "scenario": "zero_real"}}, ): with pytest.raises(SPMValidationError): validate_worker_result(incomplete, SELECTION) @@ -478,16 +483,22 @@ def test_worker_cache_receipts_are_required_and_json_roundtrip(): def test_worker_receipt_cannot_inherit_nonnull_settings(missing): from policyengine_api.worker_spm import validate_worker_result - config = {key: value for key, value in SELECTION.items() if key != missing} - with pytest.raises(SPMValidationError, match="incomplete resolved"): - validate_worker_result({"spm_config": config}, SELECTION) + receipt_field = ( + "forecast_sha256" if missing == "forecast_content_sha256" else missing + ) + receipt = { + key: value for key, value in worker_receipt().items() if key != receipt_field + } + with pytest.raises(SPMValidationError): + validate_worker_result( + {"spm_provenance": worker_comparison(baseline=receipt)}, SELECTION + ) def test_worker_receipt_must_cover_requested_year(): from policyengine_api.worker_spm import validate_worker_result result = { - "spm_config": SELECTION, "spm_provenance": worker_comparison(), } validate_worker_result(result, SELECTION, expected_year="2026") @@ -501,7 +512,6 @@ def test_worker_receipt_must_cover_requested_year(): def test_worker_receipt_rejects_baseline_reform_mismatch(): result = { - "spm_config": SELECTION, "spm_provenance": worker_comparison( reform=worker_receipt(years=("2025",)), ), @@ -513,7 +523,6 @@ def test_worker_receipt_rejects_baseline_reform_mismatch(): def test_worker_receipt_rejects_removed_segment_list_shape(): result = { - "spm_config": SELECTION, "spm_provenance": { "baseline": [worker_receipt()], "reform": [worker_receipt()], @@ -607,7 +616,6 @@ def test_budget_result_cannot_nest_another_budget_window(): def annual_shape_result(year="2026"): return { "year": year, - "spm_config": SELECTION, "spm_provenance": worker_comparison(), } @@ -686,7 +694,6 @@ def test_a_worker_receipt_failure_quotes_no_validator_internals(receipt): with pytest.raises(SPMValidationError) as caught: validate_worker_result( { - "spm_config": SELECTION, "spm_provenance": worker_comparison(reform=receipt), }, SELECTION, diff --git a/tests/unit/test_country_spm.py b/tests/unit/test_country_spm.py index f392df43e..4b525bb35 100644 --- a/tests/unit/test_country_spm.py +++ b/tests/unit/test_country_spm.py @@ -240,7 +240,6 @@ def calculate(variable, period): assert ( result.household["spm_units"]["spm_unit"]["spm_unit_federal_tax"]["2024"] == 123 ) - assert json.loads(json.dumps(result.spm_config)) == selection assert result.spm_provenance == { "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-artifact", @@ -330,7 +329,7 @@ def test_real_country_state_only_dependency_is_null_when_nothing_was_chosen( """ result = real_canonical_country.calculate(requested_household(variable), None) assert result.household["spm_units"]["spm_unit"][variable]["2024"] is None - assert result.spm_config["geography_kind"] == "county" + assert result.spm_provenance["geography_kind"] == "county" @pytest.mark.parametrize("county", ["99999", "malformed"]) @@ -375,6 +374,7 @@ def test_real_country_explicit_geography_yields_json_receipt( resolved = spm.normalize_spm_selection("us", None) forecast = spm._selected_forecast(resolved["forecast_content_sha256"]) selection["geography_id"] = forecast.resolve_county(2024, "06037")["area_id"] + resolved_selection = spm.normalize_spm_selection("us", selection) original = deepcopy(household) result = real_canonical_country.calculate( household, None, spm=selection, spm_requested=True @@ -383,10 +383,10 @@ def test_real_country_explicit_geography_yields_json_receipt( assert ( result.household["spm_units"]["spm_unit"]["spm_unit_spm_threshold"]["2024"] > 0 ) - assert result.spm_config["geography_kind"] == geography + assert result.spm_provenance["geography_kind"] == geography assert ( result.spm_provenance["forecast_sha256"] - == result.spm_config["forecast_content_sha256"] + == resolved_selection["forecast_content_sha256"] ) assert "2024" in result.spm_provenance["years"] json.dumps(result.spm_provenance) @@ -495,7 +495,7 @@ def test_real_http_missing_primitives_are_null_when_nothing_was_chosen( response = real_http_client.post("/us/calculate", json={"household": household}) assert response.status_code == 200, response.json assert response.json["status"] == "ok" - assert response.json["spm_config"]["geography_kind"] == "county" + assert response.json["spm_provenance"]["geography_kind"] == "county" if kind != "axes": threshold = response.json["result"]["spm_units"]["spm_unit"][ "spm_unit_spm_threshold" @@ -513,7 +513,7 @@ def test_real_http_national_result_and_receipt_survive_response_cache(real_http_ assert response.status_code == 200, response.json assert cached.status_code == 200 assert cached.json == response.json - assert response.json["spm_config"]["geography_kind"] == "national" + assert response.json["spm_provenance"]["geography_kind"] == "national" assert "2024" in response.json["spm_provenance"]["years"] @@ -534,7 +534,7 @@ def test_real_http_full_national_calculation_keeps_threshold_and_provenance( ] > 0 ) - assert response.json["spm_config"]["geography_kind"] == "national" + assert response.json["spm_provenance"]["geography_kind"] == "national" assert "2024" in response.json["spm_provenance"]["years"] diff --git a/tests/unit/test_openapi_simulation.py b/tests/unit/test_openapi_simulation.py index d80320eb5..3d9ca1f76 100644 --- a/tests/unit/test_openapi_simulation.py +++ b/tests/unit/test_openapi_simulation.py @@ -147,14 +147,6 @@ def test_simulation_persistence_responses_preserve_spm_output_and_run_fields( "status": "ok", "message": None, "result": {"people": {}}, - "spm_config": { - "forecast_content_sha256": "a" * 64, - "scenario": "baseline", - "geography_kind": "national", - "geography_id": None, - "county_vintage": "2020", - "as_of": None, - }, "spm_provenance": { "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-forecast", diff --git a/tests/unit/test_openapi_spm.py b/tests/unit/test_openapi_spm.py index 222b056e7..a7d709a60 100644 --- a/tests/unit/test_openapi_spm.py +++ b/tests/unit/test_openapi_spm.py @@ -9,7 +9,6 @@ SPMComparisonProvenance, SPMExecutionProvenance, SPMProvenance, - SPMResolvedConfiguration, SPMRuntimeVersions, SPMSelection, ) @@ -53,9 +52,7 @@ def test_economy_query_and_receipt_schemas_match_public_http_contract(): ]["result"] if suffix: result = result["properties"]["annualImpacts"]["items"] - assert result["properties"]["spm_config"] == { - "$ref": "#/components/schemas/SPMResolvedConfiguration" - } + assert "spm_config" not in result["properties"] assert result["properties"]["spm_provenance"] == { "$ref": "#/components/schemas/SPMComparisonProvenance" } @@ -117,12 +114,10 @@ def test_spm_documentation_uses_public_models_and_actual_route_envelopes(): paths = spec["paths"] schemas = spec["components"]["schemas"] selection_ref = {"$ref": "#/components/schemas/SPMSelection"} - resolved_ref = {"$ref": "#/components/schemas/SPMResolvedConfiguration"} provenance_ref = {"$ref": "#/components/schemas/SPMProvenance"} for name, model in { "SPMSelection": SPMSelection, - "SPMResolvedConfiguration": SPMResolvedConfiguration, "SPMRuntimeVersions": SPMRuntimeVersions, "SPMProvenance": SPMProvenance, "SPMExecutionProvenance": SPMExecutionProvenance, @@ -175,8 +170,7 @@ def test_spm_documentation_uses_public_models_and_actual_route_envelopes(): "nullable" not in field for field in schemas["SPMRuntimeVersions"]["properties"].values() ) - resolved = schemas["SPMResolvedConfiguration"] - assert set(resolved["required"]) == set(resolved["properties"]) + assert "SPMResolvedConfiguration" not in schemas removed_rich_fields = { "geographies", "requested", @@ -208,7 +202,7 @@ def test_spm_documentation_uses_public_models_and_actual_route_envelopes(): "properties" ] assert {"status", "message", "result"} <= set(result) - assert result["spm_config"] == resolved_ref + assert "spm_config" not in result assert result["spm_provenance"] == provenance_ref assert ( operation["responses"]["400"]["$ref"] diff --git a/tests/unit/test_simulation_and_report_routes.py b/tests/unit/test_simulation_and_report_routes.py index 3c0b2c892..23aa8c76f 100644 --- a/tests/unit/test_simulation_and_report_routes.py +++ b/tests/unit/test_simulation_and_report_routes.py @@ -43,14 +43,6 @@ def create_simulation( def compact_spm_report_output() -> dict: - config = { - "forecast_content_sha256": "a" * 64, - "scenario": "ce_trend", - "geography_kind": "national", - "geography_id": None, - "county_vintage": "2020", - "as_of": None, - } receipt = { "schema_version": "canonical-spm-provenance-v2", "forecast_id": "canonical-forecast", @@ -69,7 +61,6 @@ def compact_spm_report_output() -> dict: }, } return { - "spm_config": config, "spm_provenance": { "schema_version": "canonical-spm-comparison-v2", "baseline": {"receipt": receipt, "execution_count": 1}, diff --git a/tests/unit/test_spm_provenance.py b/tests/unit/test_spm_provenance.py index e4f02fa58..28745e900 100644 --- a/tests/unit/test_spm_provenance.py +++ b/tests/unit/test_spm_provenance.py @@ -6,7 +6,6 @@ from pydantic import ValidationError from policyengine_api.spm import ( - SPMCalculationProvenance, SPMComparisonProvenance, SPMProvenance, SPMRuntimeVersions, @@ -14,6 +13,8 @@ build_spm_calculation_provenance, build_spm_comparison_provenance, build_spm_provenance, + resolved_spm_configuration, + validate_spm_calculation_provenance, ) @@ -105,14 +106,15 @@ def test_runtime_versions_require_four_nonempty_strings( SPMRuntimeVersions.model_validate(versions) -def test_calculation_contract_requires_every_resolved_configuration_field() -> None: +def test_completed_calculation_contract_rejects_legacy_dual_field_shape() -> None: receipt = _receipt().model_dump(mode="json", by_alias=True) - partial = SELECTION.model_dump(mode="json") - partial.pop("geography_id") with pytest.raises(ValidationError): - SPMCalculationProvenance.model_validate( - {"spm_config": partial, "spm_provenance": receipt} + validate_spm_calculation_provenance( + { + "spm_config": SELECTION.model_dump(mode="json"), + "spm_provenance": receipt, + } ) @@ -148,10 +150,10 @@ def test_country_receipt_adapter_extracts_only_compact_scalars() -> None: calculation = build_spm_calculation_provenance(SELECTION, country_receipt) - assert calculation.spm_config.model_dump(mode="json") == SELECTION.model_dump( + assert resolved_spm_configuration(calculation).model_dump( mode="json" - ) - assert calculation.spm_provenance.years == ("2026", "2027") + ) == SELECTION.model_dump(mode="json") + assert calculation.years == ("2026", "2027") encoded = calculation.model_dump_json(by_alias=True) assert "diagnostic" not in encoded assert "mapping" not in encoded diff --git a/tests/unit/test_spm_settings.py b/tests/unit/test_spm_settings.py index 33e18936b..c4aa91a04 100644 --- a/tests/unit/test_spm_settings.py +++ b/tests/unit/test_spm_settings.py @@ -432,14 +432,16 @@ def test_resolved_settings_reject_unreadable_receipt_settings(settings): assert spm.resolved_spm_settings(settings) is None -def test_receipt_expands_config_and_reduces_country_diagnostics(certified_bundle): - """The public receipt contains one resolved config and compact provenance.""" +def test_receipt_reduces_country_diagnostics_to_one_compact_provenance( + certified_bundle, +): + """The public receipt itself contains every resolved configuration field.""" simulation = SimpleNamespace( spm_config=RESOLVED_SELECTION, spm_provenance=lambda: deepcopy(COUNTRY_RECEIPT), ) receipt = spm.calculation_spm_receipt(simulation) - assert receipt["spm_config"] == RESOLVED_SELECTION + assert set(receipt) == {"spm_provenance"} assert receipt["spm_provenance"] == { "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-artifact", diff --git a/tests/unit/v2/test_stage12_worker_contracts.py b/tests/unit/v2/test_stage12_worker_contracts.py index 9e537996d..330af0483 100644 --- a/tests/unit/v2/test_stage12_worker_contracts.py +++ b/tests/unit/v2/test_stage12_worker_contracts.py @@ -29,34 +29,33 @@ def _calculation_provenance() -> dict[str, object]: return { - "spm_config": { - "forecast_content_sha256": DIGEST_A, - "scenario": "official", - "geography_kind": "national", - "geography_id": None, - "county_vintage": "2020", - "as_of": None, - }, - "spm_provenance": { - "schema_version": "canonical-spm-provenance-v2", - "forecast_id": "canonical-forecast", - "forecast_sha256": DIGEST_A, - "scenario": "official", - "geography_kind": "national", - "geography_id": None, - "county_vintage": "2020", - "as_of": None, - "years": ["2026"], - "runtime_versions": { - "policyengine": "6.2.1", - "policyengine-core": "3.32.10", - "policyengine-us": "2.2.1", - "spm-calculator": "1.0.0", - }, + "schema_version": "canonical-spm-provenance-v2", + "forecast_id": "canonical-forecast", + "forecast_sha256": DIGEST_A, + "scenario": "official", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + "years": ["2026"], + "runtime_versions": { + "policyengine": "6.2.1", + "policyengine-core": "3.32.10", + "policyengine-us": "2.2.1", + "spm-calculator": "1.0.0", }, } +def _comparison_provenance() -> dict[str, object]: + receipt = _calculation_provenance() + return { + "schema_version": "canonical-spm-comparison-v2", + "baseline": {"receipt": receipt, "execution_count": 1}, + "reform": {"receipt": receipt, "execution_count": 1}, + } + + def _bundle() -> dict[str, object]: return { "policyengine_version": "5.2.0", @@ -240,19 +239,17 @@ def test_artifact_descriptors_retain_identity_and_bundle_provenance() -> None: @pytest.mark.parametrize( "provenance", [ + {"scenario": "official"}, { "spm_config": {"scenario": "official"}, - "spm_provenance": {"forecast_sha256": DIGEST_A}, + "spm_provenance": _calculation_provenance(), }, { **_calculation_provenance(), - "spm_provenance": { - **_calculation_provenance()["spm_provenance"], - "geographies": [], - }, + "geographies": [], }, ], - ids=["partial", "legacy-rich"], + ids=["partial", "legacy-dual-field", "legacy-rich"], ) def test_simulation_artifact_rejects_noncompact_calculation_provenance( provenance: dict[str, object], @@ -301,7 +298,10 @@ def test_artifact_payload_contracts_cover_parquet_and_aggregate_json() -> None: "evaluation_id": str(EVALUATION_ID), "requested_aggregates": ["budget"], "bundle": _bundle(), - "result": {"budget": {"change": 100.0}}, + "result": { + "budget": {"change": 100.0}, + "spm_provenance": _comparison_provenance(), + }, } ) @@ -314,6 +314,19 @@ def test_artifact_payload_contracts_cover_parquet_and_aggregate_json() -> None: } ) + with pytest.raises(ValidationError, match="legacy spm_config"): + AggregateReportArtifactPayload.model_validate( + { + **payload.model_dump(mode="json"), + "result": { + **payload.result, + "spm_config": { + "forecast_content_sha256": DIGEST_A, + }, + }, + } + ) + def test_contract_models_are_immutable_and_reject_duplicate_lists() -> None: report = ReportExecutionInput.model_validate(_report()) From 0e2e536b52ea1e3c13af21a005e47686c31c2b6a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 1 Oct 2026 21:24:05 +0400 Subject: [PATCH 10/22] Clarify internal SPM configuration model --- policyengine_api/spm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/policyengine_api/spm.py b/policyengine_api/spm.py index e48bde68d..4f92a3b96 100644 --- a/policyengine_api/spm.py +++ b/policyengine_api/spm.py @@ -9,8 +9,8 @@ import importlib import inspect from collections.abc import Mapping, Sequence -from functools import lru_cache from datetime import date +from functools import lru_cache from typing import Literal from pydantic import ( @@ -98,7 +98,7 @@ def validate_location(self): class SPMResolvedConfiguration(BaseModel): - """The complete six-field SPM selection recorded beside a receipt.""" + """An internal complete SPM selection derived from a compact receipt.""" model_config = ConfigDict(frozen=True, extra="forbid") forecast_content_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") From 70e5c09871295f9dfcc44541704130f83a6d13b6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 1 Oct 2026 21:50:10 +0400 Subject: [PATCH 11/22] Clarify synthetic SPM version fixture --- tests/unit/test_spm_provenance.py | 32 ++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/unit/test_spm_provenance.py b/tests/unit/test_spm_provenance.py index 28745e900..08521a1cf 100644 --- a/tests/unit/test_spm_provenance.py +++ b/tests/unit/test_spm_provenance.py @@ -29,12 +29,14 @@ "as_of": None, } ) -VERSIONS = SPMRuntimeVersions.model_validate( +# Synthetic fixture values used only to exercise the public receipt contract. +# Deployed workers obtain real package versions from the PolicyEngine.py bundle. +TEST_RUNTIME_VERSIONS = SPMRuntimeVersions.model_validate( { - "policyengine": "6.2.1", - "policyengine-core": "3.32.10", - "policyengine-us": "2.2.1", - "spm-calculator": "1.0.0", + "policyengine": "0.0.0-test-policyengine", + "policyengine-core": "0.0.0-test-policyengine-core", + "policyengine-us": "0.0.0-test-policyengine-us", + "spm-calculator": "0.0.0-test-spm-calculator", } ) @@ -45,7 +47,7 @@ def _receipt(**changes: object) -> SPMProvenance: "forecast_sha256": SHA256, "selection": SELECTION, "years": ("2026",), - "runtime_versions": VERSIONS, + "runtime_versions": TEST_RUNTIME_VERSIONS, } values.update(changes) return build_spm_provenance(**values) @@ -65,10 +67,10 @@ def test_compact_receipt_has_only_the_public_v2_fields() -> None: "as_of": None, "years": ["2026"], "runtime_versions": { - "policyengine": "6.2.1", - "policyengine-core": "3.32.10", - "policyengine-us": "2.2.1", - "spm-calculator": "1.0.0", + "policyengine": "0.0.0-test-policyengine", + "policyengine-core": "0.0.0-test-policyengine-core", + "policyengine-us": "0.0.0-test-policyengine-us", + "spm-calculator": "0.0.0-test-spm-calculator", }, } assert len(receipt.model_dump_json(by_alias=True).encode()) < 1_024 @@ -99,7 +101,7 @@ def test_runtime_versions_require_four_nonempty_strings( invalid_version: str, value: object, ) -> None: - versions = VERSIONS.model_dump(mode="json", by_alias=True) + versions = TEST_RUNTIME_VERSIONS.model_dump(mode="json", by_alias=True) versions[invalid_version] = value with pytest.raises(ValidationError): @@ -141,7 +143,9 @@ def test_country_receipt_adapter_extracts_only_compact_scalars() -> None: "forecast_sha256": SHA256, "scenario": "ce_trend", "geography_kind": "national", - "runtime_versions": VERSIONS.model_dump(mode="json", by_alias=True), + "runtime_versions": TEST_RUNTIME_VERSIONS.model_dump( + mode="json", by_alias=True + ), "years": {"2027": {"large": "diagnostic"}, "2026": {}}, "geographies": [{"large": "mapping"}], "composition_method": "classified inputs", @@ -166,7 +170,9 @@ def test_country_receipt_must_agree_with_adjacent_configuration() -> None: "forecast_sha256": SHA256, "scenario": "different", "geography_kind": "national", - "runtime_versions": VERSIONS.model_dump(mode="json", by_alias=True), + "runtime_versions": TEST_RUNTIME_VERSIONS.model_dump( + mode="json", by_alias=True + ), "years": {}, } From 8ce0246f26b9b86be336d22c0c9a367331751d63 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:02:49 +0400 Subject: [PATCH 12/22] test: prepare fixtures for PolicyEngine 6 --- tests/fixtures/libs/simulation_entrypoint.py | 18 +++- tests/fixtures/services/economy_service.py | 15 ++- tests/fixtures/spm.py | 102 ++++++++++++++++++ .../test_calculate_deprecated_inputs.py | 2 +- .../routes/test_calculate_error_statuses.py | 2 +- .../test_household_calculation_service.py | 4 +- 6 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 tests/fixtures/spm.py diff --git a/tests/fixtures/libs/simulation_entrypoint.py b/tests/fixtures/libs/simulation_entrypoint.py index 47263c121..ef8f5b6d0 100644 --- a/tests/fixtures/libs/simulation_entrypoint.py +++ b/tests/fixtures/libs/simulation_entrypoint.py @@ -15,6 +15,10 @@ MODAL_EXECUTION_STATUS_COMPLETE, MODAL_EXECUTION_STATUS_FAILED, ) +from tests.fixtures.spm import ( + worker_result_fields, + worker_versions_document, +) # Mock data constants MOCK_MODAL_JOB_ID = "fc-abc123xyz" @@ -44,6 +48,7 @@ "poverty_impact": {"baseline": 0.12, "reform": 0.10}, "budget_impact": {"baseline": 1000, "reform": 1200}, "inequality_impact": {"baseline": 0.45, "reform": 0.42}, + **worker_result_fields(years=["2025"]), } MOCK_POLICYENGINE_BUNDLE = { "model_version": "1.459.0", @@ -55,6 +60,10 @@ ), } MOCK_RESOLVED_APP_NAME = "policyengine-us-1-459-0" +MOCK_WORKER_VERSIONS = worker_versions_document( + app_name=MOCK_RESOLVED_APP_NAME, + country_version="1.459.0", +) MOCK_SUBMIT_RESPONSE_SUCCESS = { "job_id": MOCK_MODAL_JOB_ID, @@ -119,7 +128,14 @@ "startYear": "2026", "endYear": "2028", "windowSize": 3, - "annualImpacts": [], + "annualImpacts": [ + { + "year": year, + **MOCK_SIMULATION_RESULT, + **worker_result_fields(years=[year]), + } + for year in ("2026", "2027", "2028") + ], "totals": {}, }, "error": None, diff --git a/tests/fixtures/services/economy_service.py b/tests/fixtures/services/economy_service.py index 666f93880..adc477ddb 100644 --- a/tests/fixtures/services/economy_service.py +++ b/tests/fixtures/services/economy_service.py @@ -9,6 +9,11 @@ MODAL_EXECUTION_STATUS_SUBMITTED, ) from policyengine_api.data.v1_models import ReformImpact +from tests.fixtures.spm import ( + options_hash_segment, + worker_result_fields, + worker_spm_capability, +) # Mock data constants MOCK_COUNTRY_ID = "us" @@ -26,9 +31,10 @@ MOCK_DATA_VERSION = "faux-populace-us-2099-test-release" MOCK_LOOKUP_OPTIONS_HASH = ( "[option1=value1&option2=value2" - "&dataset=default" - "&model_version=1.2.3" - "&policyengine_version=3.4.0]" + + options_hash_segment() + + "&dataset=default" + + "&model_version=1.2.3" + + "&policyengine_version=3.4.0]" ) MOCK_OPTIONS_HASH = ( MOCK_LOOKUP_OPTIONS_HASH[:-1] @@ -56,6 +62,7 @@ "poverty_impact": {"baseline": 0.12, "reform": 0.10}, "budget_impact": {"baseline": 1000, "reform": 1200}, "inequality_impact": {"baseline": 0.45, "reform": 0.42}, + **worker_result_fields(years=[MOCK_TIME_PERIOD]), } MOCK_SIM_CONFIG = { @@ -136,6 +143,7 @@ def mock_simulation_entrypoint(): mock_batch_execution = create_mock_budget_window_batch_execution() mock_api._setup_sim_options.return_value = MOCK_SIM_CONFIG + mock_api.get_spm_capability.return_value = worker_spm_capability() mock_api.run.return_value = mock_execution mock_api.resolve_app_name.side_effect = ( lambda country_id, version=None, policyengine_version=None: ( @@ -316,6 +324,7 @@ def mock_simulation_entrypoint_legacy(): mock_execution = create_mock_modal_execution() mock_api.run.return_value = mock_execution + mock_api.get_spm_capability.return_value = None mock_api.resolve_app_name.side_effect = ( lambda country_id, version=None, policyengine_version=None: ( MOCK_RESOLVED_APP_NAME, diff --git a/tests/fixtures/spm.py b/tests/fixtures/spm.py new file mode 100644 index 000000000..8406251d1 --- /dev/null +++ b/tests/fixtures/spm.py @@ -0,0 +1,102 @@ +"""Shared result shapes for the SPM behavior selected by the installed bundle.""" + +from __future__ import annotations + +from policyengine_api import spm +from policyengine_api.constants import POLICYENGINE_VERSION + + +SPM_CONTRACT_VERSION = "canonical-spm-v1" +INSTALLED_SPM_SELECTION = spm.normalize_spm_selection("us", None) + + +def worker_spm_capability(selection: dict | None = None) -> dict | None: + """Return the capability advertised by a worker using this bundle.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + if resolved is None: + return None + return { + "contract_version": SPM_CONTRACT_VERSION, + "defaults": dict(resolved), + } + + +def worker_versions_document( + *, + bundle_version: str = POLICYENGINE_VERSION, + app_name: str = "test-worker-app", + country: str = "us", + country_version: str | None = None, + selection: dict | None = None, +) -> dict: + """Return the gateway registry document for a matching test worker.""" + document: dict = { + "policyengine": {bundle_version: app_name, "latest": bundle_version}, + } + if country_version is not None: + document[country] = { + country_version: app_name, + "latest": country_version, + } + capability = worker_spm_capability(selection) + if capability is not None: + document["spm_capabilities"] = {bundle_version: capability} + return document + + +def spm_receipt(*, years, selection: dict | None = None) -> dict: + """Return one calculation receipt covering the supplied years.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + if resolved is None: + raise ValueError("An SPM receipt requires a resolved selection") + return { + "forecast_id": "test-only", + "forecast_sha256": resolved["forecast_content_sha256"], + "scenario": resolved["scenario"], + "geography_kind": resolved["geography_kind"], + "runtime_versions": {"policyengine-us": "test-only"}, + "years": {str(year): {"status": "forecast"} for year in years}, + "geographies": [], + "composition_method": "classified-inputs", + "storage_method": "formula", + } + + +def worker_result_fields(*, years, selection: dict | None = None) -> dict: + """Return the SPM fields required on an economy worker result.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + if resolved is None: + return {} + receipt = spm_receipt(years=years, selection=resolved) + return { + "spm_config": dict(resolved), + "spm_provenance": { + "baseline": [dict(receipt)], + "reform": [dict(receipt)], + }, + } + + +def household_result_fields(*, years, selection: dict | None = None) -> dict: + """Return the SPM fields required on a household calculation result.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + if resolved is None: + return {} + return { + "spm_config": dict(resolved), + "spm_provenance": spm_receipt(years=years, selection=resolved), + } + + +def resolved_options(options: dict, selection: dict | None = None) -> dict: + """Return request options after applying the installed bundle default.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + if resolved is None: + return dict(options) + return {**options, "spm": dict(resolved)} + + +def options_hash_segment(selection: dict | None = None) -> str: + """Return the part of a sorted options hash contributed by SPM.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + return "" if resolved is None else f"&spm={resolved}" diff --git a/tests/unit/routes/test_calculate_deprecated_inputs.py b/tests/unit/routes/test_calculate_deprecated_inputs.py index 67ebe09f5..6a2da2a17 100644 --- a/tests/unit/routes/test_calculate_deprecated_inputs.py +++ b/tests/unit/routes/test_calculate_deprecated_inputs.py @@ -38,7 +38,7 @@ def __init__(self): }, } - def calculate(self, household, policy): + def calculate(self, household, policy, **_kwargs): self.household = household self.policy = policy return {"household": household, "policy": policy} diff --git a/tests/unit/routes/test_calculate_error_statuses.py b/tests/unit/routes/test_calculate_error_statuses.py index 206d4dddd..208ac512a 100644 --- a/tests/unit/routes/test_calculate_error_statuses.py +++ b/tests/unit/routes/test_calculate_error_statuses.py @@ -29,7 +29,7 @@ def prepare_calculation(self, household, policy): class CrashingCountry(DummyCountry): - def calculate(self, household, policy): + def calculate(self, household, policy, **_kwargs): raise RuntimeError("engine exploded") diff --git a/tests/unit/services/test_household_calculation_service.py b/tests/unit/services/test_household_calculation_service.py index 5259abad7..325104734 100644 --- a/tests/unit/services/test_household_calculation_service.py +++ b/tests/unit/services/test_household_calculation_service.py @@ -115,7 +115,7 @@ class Country: "parameters": {}, } - def calculate(self, household, policy): + def calculate(self, household, policy, **_kwargs): return CalculationResult( household=household, warnings=("employment_income could not be calculated",), @@ -245,7 +245,7 @@ class Country: "entities": {"person": {"plural": "people", "roles": {}}}, } - def calculate(self, household, policy): + def calculate(self, household, policy, **_kwargs): assert primary.active_scopes == 0 return CalculationResult( household={"people": {"you": {"net_income": {"2026": 42}}}}, From bdd0442281413309430b5745bab4d03b17812bfb Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:05:50 +0400 Subject: [PATCH 13/22] build: update API v1 to PolicyEngine 6.1.2 --- docker/Dockerfile | 4 ++-- pyproject.toml | 9 +++++---- tests/unit/test_cloud_run_deploy_scripts.py | 18 +++++++++--------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7b08cd132..9f5a41635 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ FROM python:3.12 # Match the API bundle in pyproject.toml; the bundle updater changes both pins. -# Exact bundled model requirements prevent pip from silently backtracking. -RUN pip install "policyengine[models]==5.2.0" spm-calculator==0.3.1 ipython +# The models extra pins the complete certified runtime package set. +RUN pip install "policyengine[models]==6.1.2" ipython diff --git a/pyproject.toml b/pyproject.toml index 779e7d7c6..cf184ed9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,10 +43,11 @@ dependencies = [ "policyengine_canada==0.96.3", "policyengine-ng==0.5.1", "policyengine-il==0.1.0", - "policyengine[models]==5.2.0", - # Keep the calculator compatible with the country model in this bundle. - # The Cloud Run image installs this exact selection from uv.lock. - "spm-calculator==0.3.1", + # The models extra pins Core, US, UK, and the SPM calculator to the exact + # versions certified by this PolicyEngine bundle. Cloud Run installs the + # complete frozen environment from uv.lock; the generic image installs + # this requirement directly. + "policyengine[models]==6.1.2", "pydantic", "pymysql", "python-dotenv", diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index ec1b334f4..c81832728 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -652,24 +652,24 @@ def test_cloud_run_dockerfile_installs_the_frozen_uv_environment(): assert "!uv.lock" in dockerignore -def test_active_images_pin_spm_calculator_for_the_legacy_country_bundle(): - """Both deployed images must protect the current bundle's SPM behavior.""" +def test_active_images_install_the_same_policyengine_models_bundle(): + """Both deployed images use the wrapper's exact certified package set.""" import tomllib project = tomllib.loads((REPO / "pyproject.toml").read_text()) - assert "spm-calculator==0.3.1" in project["project"]["dependencies"] - # The independently published GHCR image does not install this project. + dependencies = project["project"]["dependencies"] generic_image = (REPO / "docker/Dockerfile").read_text() - assert "spm-calculator==0.3.1" in generic_image bundle_pin = next( requirement - for requirement in project["project"]["dependencies"] + for requirement in dependencies if requirement.startswith("policyengine[models]==") ) assert bundle_pin in generic_image - assert "pip install policyengine-core policyengine-uk policyengine-us" not in ( - generic_image - ) + assert not any(item.startswith("spm-calculator") for item in dependencies) + assert "spm-calculator==" not in generic_image + for package in ("policyengine-core", "policyengine-us", "policyengine-uk"): + assert not any(item.startswith(package) for item in dependencies) + assert f"{package}==" not in generic_image def test_deployed_startup_execs_only_the_api_server(): From 5fb2c2b9efe6461698bd22d96934cd3e56e96afa Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:07:05 +0400 Subject: [PATCH 14/22] ci: verify simulation worker compatibility --- .../check-policyengine-bundle-supported.sh | 12 +- .github/request-simulation-model-versions.sh | 13 +- .github/workflows/pr.yml | 2 + .github/workflows/push.yml | 4 + policyengine_api/worker_spm_release.py | 122 +++++++++++++++++ tests/unit/test_cloud_run_deploy_scripts.py | 1 + tests/unit/test_worker_spm_release.py | 124 ++++++++++++++++++ 7 files changed, 274 insertions(+), 4 deletions(-) create mode 100644 policyengine_api/worker_spm_release.py create mode 100644 tests/unit/test_worker_spm_release.py diff --git a/.github/check-policyengine-bundle-supported.sh b/.github/check-policyengine-bundle-supported.sh index f48d1f076..20e78734d 100755 --- a/.github/check-policyengine-bundle-supported.sh +++ b/.github/check-policyengine-bundle-supported.sh @@ -51,10 +51,16 @@ if [ "$CHECK_ONLY_IF_CHANGED" = "1" ]; then || true )" - if [ "$current_version" = "$base_version" ]; then - echo "PolicyEngine .py bundle pin is unchanged; skipping simulation API support check." + if [ "$current_version" = "$base_version" ] \ + && git diff --quiet "origin/${BASE_REF}" -- \ + policyengine_api/spm.py policyengine_api/worker_spm.py \ + policyengine_api/worker_spm_release.py policyengine_api/constants.py \ + policyengine_api/country.py \ + .github/check-policyengine-bundle-supported.sh \ + .github/request-simulation-model-versions.sh; then + echo "Bundle pin and SPM integration are unchanged; skipping simulation API support check." exit 0 fi fi -bash "$VERSION_GUARD_SCRIPT" -py "$current_version" +bash "$VERSION_GUARD_SCRIPT" -py "$current_version" --check-installed-spm diff --git a/.github/request-simulation-model-versions.sh b/.github/request-simulation-model-versions.sh index 27a68062c..b2b38240d 100755 --- a/.github/request-simulation-model-versions.sh +++ b/.github/request-simulation-model-versions.sh @@ -23,12 +23,14 @@ usage() { echo "Optional compatibility checks:" echo " -us us_version Expected bundled policyengine-us version" echo " -uk uk_version Expected bundled policyengine-uk version" + echo " --check-installed-spm Compare the installed bundle with worker capability" exit 1 } POLICYENGINE_VERSION="" US_VERSION="" UK_VERSION="" +CHECK_INSTALLED_SPM=0 while [ $# -gt 0 ]; do case "$1" in @@ -44,6 +46,10 @@ while [ $# -gt 0 ]; do UK_VERSION="$2" shift 2 ;; + --check-installed-spm) + CHECK_INSTALLED_SPM=1 + shift + ;; -h|--help) usage ;; @@ -70,7 +76,7 @@ if [ -n "$UK_VERSION" ]; then fi echo "" -VERSIONS_RESPONSE=$(curl -s "${GATEWAY_URL}/versions") +VERSIONS_RESPONSE=$(curl --fail --silent --show-error --connect-timeout 10 --max-time 60 "${GATEWAY_URL}/versions") if [ -z "$VERSIONS_RESPONSE" ]; then echo "ERROR: Failed to fetch versions from gateway" @@ -114,6 +120,11 @@ check_country_route() { check_country_route "us" "$US_VERSION" check_country_route "uk" "$UK_VERSION" +if [ "$CHECK_INSTALLED_SPM" = "1" ]; then + printf '%s' "$VERSIONS_RESPONSE" \ + | uv run --frozen python -m policyengine_api.worker_spm_release "$POLICYENGINE_VERSION" +fi + echo "" echo "SUCCESS: PolicyEngine bundle route is deployed and ready" exit 0 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 9171eba74..b2bfb28ed 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -27,6 +27,8 @@ jobs: fetch-depth: 0 - name: Install jq run: sudo apt-get install -y jq + - name: Set up uv for installed bundle validation + uses: astral-sh/setup-uv@v6 - name: Check simulation API supports updated PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh --if-changed-from-base "${{ github.base_ref }}" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index a602c0138..856b4f570 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -60,6 +60,8 @@ jobs: uses: actions/checkout@v4 - name: Install jq run: sudo apt-get install -y jq + - name: Set up uv for installed bundle validation + uses: astral-sh/setup-uv@v6 - name: Check simulation API supports PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh @@ -411,6 +413,8 @@ jobs: uses: actions/checkout@v4 - name: Install jq run: sudo apt-get install -y jq + - name: Set up uv for installed bundle validation + uses: astral-sh/setup-uv@v6 - name: Check simulation API supports PolicyEngine bundle run: bash .github/check-policyengine-bundle-supported.sh diff --git a/policyengine_api/worker_spm_release.py b/policyengine_api/worker_spm_release.py new file mode 100644 index 000000000..8f876e0d2 --- /dev/null +++ b/policyengine_api/worker_spm_release.py @@ -0,0 +1,122 @@ +"""Validate the installed API bundle against a simulation worker registry.""" + +from __future__ import annotations + +from importlib.metadata import version as distribution_version +import json +import sys + +from policyengine_api import constants +from policyengine_api.worker_spm import validate_worker_spm + + +REQUIRED_BUNDLE_PACKAGES = ( + "policyengine", + "policyengine-core", + "policyengine-us", + "policyengine-uk", + "spm-calculator", +) + + +class RegistryCapability: + """Expose one registry document through the request validator interface.""" + + def __init__(self, registry: dict): + self.registry = registry + + def get_spm_capability( + self, + country, + model_version, + *, + policyengine_version, + ): + capabilities = self.registry.get("spm_capabilities") + if not isinstance(capabilities, dict): + return None + return capabilities.get(policyengine_version) + + +def _bundle_package_versions() -> dict[str, str]: + bundle = constants._policyengine_bundle + packages = bundle.get("packages") if isinstance(bundle, dict) else None + if not isinstance(packages, dict): + raise ValueError("The installed PolicyEngine bundle manifest has no packages") + + versions: dict[str, str] = {} + for package_name in REQUIRED_BUNDLE_PACKAGES: + package = packages.get(package_name) + package_version = package.get("version") if isinstance(package, dict) else None + if not isinstance(package_version, str) or not package_version: + raise ValueError(f"The installed bundle does not identify {package_name}") + versions[package_name] = package_version + return versions + + +def _registered_application( + registry: dict, + route_kind: str, + route_version: str, +) -> str: + routes = registry.get(route_kind) if isinstance(registry, dict) else None + app = routes.get(route_version) if isinstance(routes, dict) else None + if not isinstance(app, str) or not app: + raise ValueError( + f"The worker registry has no {route_kind} {route_version} application" + ) + return app + + +def validate_installed_worker(registry: dict, expected_bundle: str) -> dict | None: + """Validate local package versions, worker routes, and SPM capability.""" + versions = _bundle_package_versions() + if versions["policyengine"] != expected_bundle: + raise ValueError("The bundle manifest differs from the release requirement") + if constants.POLICYENGINE_VERSION != expected_bundle: + raise ValueError("The API runtime differs from the release requirement") + + for package_name, expected_version in versions.items(): + if distribution_version(package_name) != expected_version: + raise ValueError( + f"Installed {package_name} does not match the bundle manifest" + ) + + bundle_app = _registered_application(registry, "policyengine", expected_bundle) + for country_id, package_name in ( + ("us", "policyengine-us"), + ("uk", "policyengine-uk"), + ): + country_app = _registered_application( + registry, + country_id, + versions[package_name], + ) + if country_app != bundle_app: + raise ValueError( + f"The {country_id} route resolves to a different worker application" + ) + + return validate_worker_spm( + "us", + gateway=RegistryCapability(registry), + policyengine_version=expected_bundle, + model_version=versions["policyengine-us"], + ) + + +def main() -> None: + """Read a registry document from stdin and validate one bundle version.""" + if len(sys.argv) != 2: + raise SystemExit( + "Usage: python -m policyengine_api.worker_spm_release BUNDLE_VERSION" + ) + selection = validate_installed_worker(json.load(sys.stdin), sys.argv[1]) + if selection is None: + print("The installed legacy bundle and registered worker agree") + else: + print("The installed bundle and registered worker capability agree") + + +if __name__ == "__main__": + main() diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index c81832728..9860d1d2d 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -604,6 +604,7 @@ def test_policyengine_bundle_support_check_passes_pyproject_pin_to_guard(tmp_pat assert capture_path.read_text(encoding="utf-8").splitlines() == [ "-py", current_version, + "--check-installed-spm", ] diff --git a/tests/unit/test_worker_spm_release.py b/tests/unit/test_worker_spm_release.py new file mode 100644 index 000000000..416af4113 --- /dev/null +++ b/tests/unit/test_worker_spm_release.py @@ -0,0 +1,124 @@ +"""Release checks compare installed packages with the selected worker.""" + +from copy import deepcopy + +import pytest + +from policyengine_api import constants, worker_spm, worker_spm_release +from policyengine_api.spm import SPMValidationError + + +BUNDLE_VERSION = "6.1.2" +PACKAGE_VERSIONS = { + "policyengine": BUNDLE_VERSION, + "policyengine-core": "3.32.5", + "policyengine-us": "2.2.1", + "policyengine-uk": "2.90.2", + "spm-calculator": "1.0.0", +} +SELECTION = { + "forecast_content_sha256": "a" * 64, + "scenario": "fixture", + "geography_kind": "county", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} +WORKER_APP = "fixture-worker" + + +@pytest.fixture +def registry(monkeypatch): + monkeypatch.setattr(constants, "POLICYENGINE_VERSION", BUNDLE_VERSION) + monkeypatch.setattr( + constants, + "_policyengine_bundle", + { + "packages": { + name: {"name": name, "version": package_version} + for name, package_version in PACKAGE_VERSIONS.items() + } + }, + ) + monkeypatch.setattr( + worker_spm_release, + "distribution_version", + PACKAGE_VERSIONS.__getitem__, + ) + monkeypatch.setattr(worker_spm, "normalize_spm_selection", lambda *_: SELECTION) + return { + "policyengine": {BUNDLE_VERSION: WORKER_APP}, + "us": {PACKAGE_VERSIONS["policyengine-us"]: WORKER_APP}, + "uk": {PACKAGE_VERSIONS["policyengine-uk"]: WORKER_APP}, + "spm_capabilities": { + BUNDLE_VERSION: { + "contract_version": "canonical-spm-v1", + "defaults": dict(SELECTION), + } + }, + } + + +def test_release_accepts_matching_installed_bundle_and_worker(registry): + assert ( + worker_spm_release.validate_installed_worker(registry, BUNDLE_VERSION) + == SELECTION + ) + + +@pytest.mark.parametrize( + "mutation", + ["missing-capability", "wrong-contract", "wrong-forecast"], +) +def test_release_rejects_worker_without_matching_spm_capability(registry, mutation): + registry = deepcopy(registry) + capability = registry["spm_capabilities"][BUNDLE_VERSION] + if mutation == "missing-capability": + registry["spm_capabilities"] = {} + elif mutation == "wrong-contract": + capability["contract_version"] = "different-contract" + else: + capability["defaults"]["forecast_content_sha256"] = "b" * 64 + + with pytest.raises(SPMValidationError): + worker_spm_release.validate_installed_worker(registry, BUNDLE_VERSION) + + +@pytest.mark.parametrize("route", ["policyengine", "us", "uk"]) +def test_release_rejects_missing_worker_route(registry, route): + registry[route] = {} + with pytest.raises(ValueError, match="no .* application"): + worker_spm_release.validate_installed_worker(registry, BUNDLE_VERSION) + + +@pytest.mark.parametrize("route", ["us", "uk"]) +def test_release_rejects_country_route_to_different_worker(registry, route): + version = next(iter(registry[route])) + registry[route][version] = "different-worker" + with pytest.raises(ValueError, match="different worker"): + worker_spm_release.validate_installed_worker(registry, BUNDLE_VERSION) + + +def test_release_rejects_installed_distribution_outside_manifest( + registry, + monkeypatch, +): + installed = {**PACKAGE_VERSIONS, "policyengine-core": "0.0.1"} + monkeypatch.setattr( + worker_spm_release, + "distribution_version", + installed.__getitem__, + ) + with pytest.raises(ValueError, match="policyengine-core"): + worker_spm_release.validate_installed_worker(registry, BUNDLE_VERSION) + + +def test_release_rejects_requirement_outside_manifest(registry): + with pytest.raises(ValueError, match="manifest"): + worker_spm_release.validate_installed_worker(registry, "7.0.0") + + +def test_release_rejects_manifest_without_complete_package_set(registry): + constants._policyengine_bundle["packages"].pop("spm-calculator") + with pytest.raises(ValueError, match="spm-calculator"): + worker_spm_release.validate_installed_worker(registry, BUNDLE_VERSION) From a526c298baacd84ef4dabe03cc711c665f2e35bf Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:09:43 +0400 Subject: [PATCH 15/22] ci: update PolicyEngine bundle atomically --- .github/find-api-model-versions.py | 87 ++++++++++++--- .../scripts/update-policyengine-package.sh | 28 +++-- tests/unit/test_bundle_update_pins.py | 105 +++++++++++++++--- 3 files changed, 180 insertions(+), 40 deletions(-) diff --git a/.github/find-api-model-versions.py b/.github/find-api-model-versions.py index 7a9a9f09b..1e5a262a4 100644 --- a/.github/find-api-model-versions.py +++ b/.github/find-api-model-versions.py @@ -1,38 +1,89 @@ import argparse import os +import shlex import sys +from importlib.metadata import version as distribution_version +from policyengine.bundle import get_current_bundle from policyengine_api.constants import ( COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_CORE_VERSION, POLICYENGINE_VERSION, ) +REQUIRED_PACKAGES = ( + "policyengine", + "policyengine-core", + "policyengine-us", + "policyengine-uk", + "spm-calculator", +) + + +def _manifest_versions(bundle: dict) -> dict[str, str]: + packages = bundle.get("packages") + if not isinstance(packages, dict): + raise RuntimeError("PolicyEngine bundle manifest has no package mapping.") + + versions = {} + for package_name in REQUIRED_PACKAGES: + package = packages.get(package_name) + if not isinstance(package, dict) or not package.get("version"): + raise RuntimeError( + f"PolicyEngine bundle manifest has no version for {package_name}." + ) + versions[package_name] = str(package["version"]) + return versions + + +def _data_release_version(bundle: dict, country_id: str) -> str: + data_releases = bundle.get("data_releases") + release = data_releases.get(country_id) if isinstance(data_releases, dict) else None + if not isinstance(release, dict) or not release.get("version"): + raise RuntimeError( + f"PolicyEngine bundle manifest has no {country_id.upper()} data release." + ) + return str(release["version"]) + def find_api_model_versions() -> dict[str, str]: """ - Find the API model versions from the installed PolicyEngine bundle. + Find and validate the package and data versions in the installed bundle. """ - us_version = COUNTRY_PACKAGE_VERSIONS.get("us") - uk_version = COUNTRY_PACKAGE_VERSIONS.get("uk") + bundle = get_current_bundle() + if not isinstance(bundle, dict): + raise RuntimeError("Installed PolicyEngine bundle manifest is not an object.") - if not us_version: - print("Error: US package version not found.", file=sys.stderr) - sys.exit(1) + manifest_versions = _manifest_versions(bundle) + for package_name, manifest_version in manifest_versions.items(): + installed_version = distribution_version(package_name) + if installed_version != manifest_version: + raise RuntimeError( + f"Installed {package_name} version {installed_version} does not match " + f"bundle manifest version {manifest_version}." + ) - if not uk_version: - print("Error: UK package version not found.", file=sys.stderr) - sys.exit(1) - - if not POLICYENGINE_VERSION: - print("Error: PolicyEngine package version not found.", file=sys.stderr) - sys.exit(1) + expected_constants = { + "policyengine": POLICYENGINE_VERSION, + "policyengine-core": POLICYENGINE_CORE_VERSION, + "policyengine-us": COUNTRY_PACKAGE_VERSIONS.get("us"), + "policyengine-uk": COUNTRY_PACKAGE_VERSIONS.get("uk"), + } + for package_name, constant_version in expected_constants.items(): + if constant_version != manifest_versions[package_name]: + raise RuntimeError( + f"API version for {package_name} is {constant_version}, but the bundle " + f"manifest specifies {manifest_versions[package_name]}." + ) return { - "POLICYENGINE_VERSION": POLICYENGINE_VERSION, - "POLICYENGINE_CORE_VERSION": POLICYENGINE_CORE_VERSION, - "US_VERSION": us_version, - "UK_VERSION": uk_version, + "POLICYENGINE_VERSION": manifest_versions["policyengine"], + "POLICYENGINE_CORE_VERSION": manifest_versions["policyengine-core"], + "US_VERSION": manifest_versions["policyengine-us"], + "UK_VERSION": manifest_versions["policyengine-uk"], + "SPM_CALCULATOR_VERSION": manifest_versions["spm-calculator"], + "US_DATA_VERSION": _data_release_version(bundle, "us"), + "UK_DATA_VERSION": _data_release_version(bundle, "uk"), } @@ -57,7 +108,7 @@ def find_api_model_versions_and_output_to_github(): if args.shell: for key, value in find_api_model_versions().items(): - print(f"{key}={value}") + print(f"{key}={shlex.quote(value)}") else: find_api_model_versions_and_output_to_github() print("API model versions found and written to GitHub environment.") diff --git a/.github/scripts/update-policyengine-package.sh b/.github/scripts/update-policyengine-package.sh index ac8345fec..12a47403e 100755 --- a/.github/scripts/update-policyengine-package.sh +++ b/.github/scripts/update-policyengine-package.sh @@ -11,6 +11,7 @@ # repository_dispatch trigger passes the just-released version here); # otherwise the latest version on PyPI is used. # FORCE=1 allows targeting a version that is not newer than the current pin. +# LOCK_RETRY_SECONDS controls the pause between lock retries (default: 30). set -euo pipefail DRY_RUN=0 @@ -30,6 +31,12 @@ Update PolicyEngine .py bundle from ${CURRENT} to ${LATEST}. - policyengine-core: ${POLICYENGINE_CORE_VERSION:-resolved during update} - policyengine-us: ${US_VERSION:-resolved during update} - policyengine-uk: ${UK_VERSION:-resolved during update} +- spm-calculator: ${SPM_CALCULATOR_VERSION:-resolved during update} + +## Certified data releases + +- US: ${US_DATA_VERSION:-resolved during update} +- UK: ${UK_DATA_VERSION:-resolved during update} --- Generated automatically by GitHub Actions @@ -84,11 +91,9 @@ if [[ "$DRY_RUN" == "1" ]]; then exit 0 fi -EXISTING_PR=$(gh pr list \ - --head "$BRANCH" \ - --state open \ - --json number \ - --jq '.[0].number' 2>/dev/null || true) +EXISTING_PR=$(gh pr view "$BRANCH" \ + --json number,state \ + --jq 'select(.state == "OPEN") | .number' 2>/dev/null || true) if [[ -n "$EXISTING_PR" ]]; then echo "PR #${EXISTING_PR} already exists for ${BRANCH}. Skipping." exit 0 @@ -138,13 +143,20 @@ for attempt in 1 2 3; do echo "ERROR: uv lock failed after ${attempt} attempts." >&2 exit 1 fi - echo "uv lock attempt ${attempt} failed; retrying in 30s..." - sleep 30 + echo "uv lock attempt ${attempt} failed; retrying in ${LOCK_RETRY_SECONDS:-30}s..." + sleep "${LOCK_RETRY_SECONDS:-30}" done -VERSIONS_OUTPUT=$(uv run python .github/find-api-model-versions.py --shell) +uv lock --check + +VERSIONS_OUTPUT=$(uv run --frozen python .github/find-api-model-versions.py --shell) eval "$VERSIONS_OUTPUT" +if [[ "$POLICYENGINE_VERSION" != "$LATEST" ]]; then + echo "Installed PolicyEngine version ${POLICYENGINE_VERSION} does not match requested version ${LATEST}." >&2 + exit 1 +fi + FRAGMENT="changelog.d/update-policyengine-bundle-${LATEST}.changed.md" echo "Update the PolicyEngine bundle to ${LATEST}." > "$FRAGMENT" diff --git a/tests/unit/test_bundle_update_pins.py b/tests/unit/test_bundle_update_pins.py index c13abb99b..75ec4a74f 100644 --- a/tests/unit/test_bundle_update_pins.py +++ b/tests/unit/test_bundle_update_pins.py @@ -9,6 +9,8 @@ REPO = Path(__file__).resolve().parents[2] SCRIPT = REPO / ".github/scripts/update-policyengine-package.sh" +SOURCE_VERSION = "4.0.0" +TARGET_VERSION = "4.1.0" @pytest.fixture @@ -16,12 +18,11 @@ def update_checkout(tmp_path): (tmp_path / "docker").mkdir() (tmp_path / "changelog.d").mkdir() (tmp_path / "pyproject.toml").write_text( - '[project]\ndependencies = ["policyengine[models]==5.2.0", ' - '"spm-calculator==0.3.1"]\n' + f'[project]\ndependencies = ["policyengine[models]=={SOURCE_VERSION}"]\n' ) (tmp_path / "docker/Dockerfile").write_text( - 'FROM python:3.12\nRUN pip install "policyengine[models]==5.2.0" ' - "spm-calculator==0.3.1 ipython\n" + "FROM python:3.12\n" + f'RUN pip install "policyengine[models]=={SOURCE_VERSION}" ipython\n' ) (tmp_path / "uv.lock").write_text("# registry resolution is stubbed\n") binaries = tmp_path / "bin" @@ -38,8 +39,23 @@ def update_checkout(tmp_path): stream.write(json.dumps([name, *args]) + "\\n") if name == "git" and args[:1] == ["ls-remote"]: sys.exit(2) +if name == "gh" and args[:2] == ["pr", "view"]: + if os.environ.get("BUNDLE_TEST_OPEN_PR"): + print(os.environ["BUNDLE_TEST_OPEN_PR"]) + sys.exit(0) + sys.exit(1) +if name == "uv" and args[:2] == ["lock", "--upgrade-package"]: + if os.environ.get("BUNDLE_TEST_LOCK_FAIL"): + sys.exit(1) if name == "uv" and args[:1] == ["run"]: - print("POLICYENGINE_VERSION=5.3.0") + reported = os.environ.get("BUNDLE_TEST_REPORTED_VERSION", "4.1.0") + print(f"POLICYENGINE_VERSION={reported}") + print("POLICYENGINE_CORE_VERSION=3.0.0") + print("US_VERSION=2.0.0") + print("UK_VERSION=2.1.0") + print("SPM_CALCULATOR_VERSION=1.0.0") + print("US_DATA_VERSION=us-data-release") + print("UK_DATA_VERSION=uk-data-release") """ for name in ("git", "gh", "uv"): path = binaries / name @@ -48,16 +64,18 @@ def update_checkout(tmp_path): return tmp_path -def run_update(root): +def run_update(root, *arguments, **environment): log = root / "calls.jsonl" result = subprocess.run( - ["bash", str(SCRIPT)], + ["bash", str(SCRIPT), *arguments], cwd=root, env={ **os.environ, "PATH": f"{root / 'bin'}{os.pathsep}{os.environ['PATH']}", - "LATEST_OVERRIDE": "5.3.0", + "LATEST_OVERRIDE": TARGET_VERSION, "BUNDLE_TEST_CALLS": str(log), + "LOCK_RETRY_SECONDS": "0", + **environment, }, capture_output=True, text=True, @@ -72,24 +90,33 @@ def test_automatic_update_changes_and_stages_both_image_pins(update_checkout): assert result.returncode == 0, result.stderr for name in ("pyproject.toml", "docker/Dockerfile"): text = (update_checkout / name).read_text() - assert "policyengine[models]==5.3.0" in text - assert "policyengine[models]==5.2.0" not in text - assert "spm-calculator==0.3.1" in text + assert f"policyengine[models]=={TARGET_VERSION}" in text + assert f"policyengine[models]=={SOURCE_VERSION}" not in text + assert "spm-calculator==" not in text assert [ "git", "add", "pyproject.toml", "docker/Dockerfile", "uv.lock", - "changelog.d/update-policyengine-bundle-5.3.0.changed.md", + f"changelog.d/update-policyengine-bundle-{TARGET_VERSION}.changed.md", ] in calls assert ["uv", "lock", "--upgrade-package", "policyengine"] in calls + assert ["uv", "lock", "--check"] in calls + assert [ + "uv", + "run", + "--frozen", + "python", + ".github/find-api-model-versions.py", + "--shell", + ] in calls -@pytest.mark.parametrize("pin", ["5.1.0", "5.2.01", "5.2.0rc1"]) +@pytest.mark.parametrize("pin", ["3.9.0", "4.0.01", "4.0.0rc1"]) def test_mismatched_image_pin_fails_before_file_changes(update_checkout, pin): image = update_checkout / "docker/Dockerfile" - image.write_text(image.read_text().replace("5.2.0", pin)) + image.write_text(image.read_text().replace(SOURCE_VERSION, pin)) before = { name: (update_checkout / name).read_bytes() for name in ("pyproject.toml", "docker/Dockerfile") @@ -101,3 +128,53 @@ def test_mismatched_image_pin_fails_before_file_changes(update_checkout, pin): assert (update_checkout / name).read_bytes() == data assert not any(call[:1] == ["uv"] for call in calls) assert not any(call[:2] == ["git", "commit"] for call in calls) + + +def test_dry_run_does_not_change_files(update_checkout): + before = { + name: (update_checkout / name).read_bytes() + for name in ("pyproject.toml", "docker/Dockerfile", "uv.lock") + } + result, calls = run_update(update_checkout, "--dry-run") + assert result.returncode == 0, result.stderr + assert "Dry run complete" in result.stdout + for name, data in before.items(): + assert (update_checkout / name).read_bytes() == data + assert not any(call[:1] == ["uv"] for call in calls) + + +def test_lock_failure_stops_before_commit(update_checkout): + result, calls = run_update(update_checkout, BUNDLE_TEST_LOCK_FAIL="1") + assert result.returncode != 0 + assert "uv lock failed after 3 attempts" in result.stderr + assert sum( + call == ["uv", "lock", "--upgrade-package", "policyengine"] + for call in calls + ) == 3 + assert not any(call[:2] == ["git", "commit"] for call in calls) + + +def test_manifest_version_mismatch_stops_before_commit(update_checkout): + result, calls = run_update( + update_checkout, BUNDLE_TEST_REPORTED_VERSION="4.0.9" + ) + assert result.returncode != 0 + assert "does not match requested version" in result.stderr + assert not any(call[:2] == ["git", "commit"] for call in calls) + + +def test_existing_pull_request_is_found_by_exact_branch(update_checkout): + result, calls = run_update(update_checkout, BUNDLE_TEST_OPEN_PR="321") + assert result.returncode == 0, result.stderr + assert "PR #321 already exists" in result.stdout + assert [ + "gh", + "pr", + "view", + f"auto/update-policyengine-bundle-{TARGET_VERSION}", + "--json", + "number,state", + "--jq", + 'select(.state == "OPEN") | .number', + ] in calls + assert not any(call[:1] == ["uv"] for call in calls) From cf6fc70f4777c3b9752864213db6ab358b5267b1 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:27:13 +0400 Subject: [PATCH 16/22] test: update fixtures for PolicyEngine 6 --- .../test_simulation_gateway_contract.py | 9 +++ tests/fixtures/libs/simulation_entrypoint.py | 8 +++ .../test_budget_window_in_flight_dedupe.py | 2 + .../integration/test_v2_catalog_installed.py | 8 +-- tests/unit/libs/test_simulation_entrypoint.py | 14 +++- tests/unit/routes/test_canonical_spm.py | 4 +- ...st_household_and_user_policy_orm_routes.py | 17 ++++- tests/unit/services/test_economy_service.py | 70 +++++++++++-------- .../test_household_calculation_service.py | 7 +- tests/unit/services/test_worker_spm.py | 9 ++- tests/unit/test_country_spm.py | 22 +++--- tests/unit/test_spm_settings.py | 1 + 12 files changed, 122 insertions(+), 49 deletions(-) diff --git a/tests/contract/test_simulation_gateway_contract.py b/tests/contract/test_simulation_gateway_contract.py index 12ccc0e8e..2452e3e38 100644 --- a/tests/contract/test_simulation_gateway_contract.py +++ b/tests/contract/test_simulation_gateway_contract.py @@ -15,6 +15,7 @@ MOCK_RESOLVED_APP_NAME, MOCK_SIMULATION_PAYLOAD_WITH_TELEMETRY, MOCK_SUBMIT_RESPONSE_SUCCESS, + MOCK_WORKER_VERSIONS, ) @@ -56,6 +57,10 @@ def test_gateway_comparison_submit_and_poll_contract(monkeypatch): monkeypatch.setenv("OLD_SIMULATION_GATEWAY_URL", "https://simulation.test") client = _client_for( { + ("GET", "/versions"): _response( + status_code=200, + json_data=MOCK_WORKER_VERSIONS, + ), ("POST", "/simulate/economy/comparison"): _response( status_code=202, json_data=MOCK_SUBMIT_RESPONSE_SUCCESS, @@ -87,6 +92,10 @@ def test_gateway_budget_window_submit_and_poll_contract(monkeypatch): monkeypatch.setenv("OLD_SIMULATION_GATEWAY_URL", "https://simulation.test") client = _client_for( { + ("GET", "/versions"): _response( + status_code=200, + json_data=MOCK_WORKER_VERSIONS, + ), ( "POST", "/simulate/economy/budget-window", diff --git a/tests/fixtures/libs/simulation_entrypoint.py b/tests/fixtures/libs/simulation_entrypoint.py index ef8f5b6d0..02a444be8 100644 --- a/tests/fixtures/libs/simulation_entrypoint.py +++ b/tests/fixtures/libs/simulation_entrypoint.py @@ -16,6 +16,7 @@ MODAL_EXECUTION_STATUS_FAILED, ) from tests.fixtures.spm import ( + worker_spm_capability, worker_result_fields, worker_versions_document, ) @@ -64,6 +65,9 @@ app_name=MOCK_RESOLVED_APP_NAME, country_version="1.459.0", ) +for bundle_version in ("3.4.0", "4.18.3"): + MOCK_WORKER_VERSIONS["policyengine"][bundle_version] = MOCK_RESOLVED_APP_NAME + MOCK_WORKER_VERSIONS["spm_capabilities"][bundle_version] = worker_spm_capability() MOCK_SUBMIT_RESPONSE_SUCCESS = { "job_id": MOCK_MODAL_JOB_ID, @@ -216,6 +220,10 @@ def mock_httpx_client(): ) as mock_client_class: mock_client = MagicMock() mock_client_class.return_value = mock_client + mock_client.get.return_value = create_mock_httpx_response( + status_code=200, + json_data=MOCK_WORKER_VERSIONS, + ) yield mock_client diff --git a/tests/integration/test_budget_window_in_flight_dedupe.py b/tests/integration/test_budget_window_in_flight_dedupe.py index 4c3c45d8e..826154ca9 100644 --- a/tests/integration/test_budget_window_in_flight_dedupe.py +++ b/tests/integration/test_budget_window_in_flight_dedupe.py @@ -2,6 +2,7 @@ from flask import Flask from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from tests.fixtures.spm import worker_spm_capability class FakeRedis(InMemoryCacheBackend): @@ -31,6 +32,7 @@ def test_budget_window_in_flight_dedupe_uses_existing_batch_without_live_db( fake_cache = BudgetWindowCache(client=FakeRedis()) simulation_entrypoint = MagicMock() + simulation_entrypoint.get_spm_capability.return_value = worker_spm_capability() reform_impacts_service = MagicMock() simulation_entrypoint.run_budget_window_batch.return_value = ( diff --git a/tests/integration/test_v2_catalog_installed.py b/tests/integration/test_v2_catalog_installed.py index b09ce746a..f897bce6b 100644 --- a/tests/integration/test_v2_catalog_installed.py +++ b/tests/integration/test_v2_catalog_installed.py @@ -39,10 +39,10 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: assert catalog.entity_counts() == { "models": 2, "model_versions": 2, - "variables": 6_649, - "parameter_nodes": 27_813, - "parameters": 99_006, - "parameter_values": 1_172_130, + "variables": 7_046, + "parameter_nodes": 29_118, + "parameters": 103_705, + "parameter_values": 1_192_826, "datasets": 2, "regions": 826, } diff --git a/tests/unit/libs/test_simulation_entrypoint.py b/tests/unit/libs/test_simulation_entrypoint.py index 8e124ec4f..833c85f68 100644 --- a/tests/unit/libs/test_simulation_entrypoint.py +++ b/tests/unit/libs/test_simulation_entrypoint.py @@ -15,6 +15,7 @@ os.environ.setdefault("FLASK_DEBUG", "1") from policyengine_api.constants import ( # noqa: E402 + POLICYENGINE_VERSION, MODAL_EXECUTION_STATUS_COMPLETE, MODAL_EXECUTION_STATUS_FAILED, MODAL_EXECUTION_STATUS_RUNNING, @@ -57,8 +58,10 @@ MOCK_SIMULATION_PAYLOAD_WITH_TELEMETRY, MOCK_SIMULATION_RESULT, MOCK_SUBMIT_RESPONSE_SUCCESS, + MOCK_WORKER_VERSIONS, create_mock_httpx_response, ) +from tests.fixtures.spm import INSTALLED_SPM_SELECTION # noqa: E402 pytest_plugins = ("tests.fixtures.libs.simulation_entrypoint",) @@ -107,6 +110,9 @@ def _response(self, method, url, json=None): elif "/jobs/" in path: payload = MOCK_POLL_RESPONSE_RUNNING status_code = 202 + elif path == "/versions": + payload = MOCK_WORKER_VERSIONS + status_code = 200 elif "/versions/" in path: payload = { "latest": "1.459.0", @@ -500,6 +506,7 @@ def test__given_request_context__then_all_calls_forward_request_id( "/simulate/economy/budget-window", f"/jobs/{MOCK_MODAL_JOB_ID}", f"/budget-window-jobs/{MOCK_BATCH_JOB_ID}", + "/versions", "/versions/us", "/health", } @@ -601,7 +608,11 @@ def test__given_valid_payload__then_posts_to_correct_endpoint( # Then call_args = mock_httpx_client.post.call_args assert "/simulate/economy/comparison" in call_args[0][0] - assert call_args[1]["json"] == MOCK_SIMULATION_PAYLOAD + assert call_args[1]["json"] == { + **MOCK_SIMULATION_PAYLOAD, + "spm": INSTALLED_SPM_SELECTION, + "policyengine_version": POLICYENGINE_VERSION, + } def test__given_telemetry_payload__then_preserves_non_identity_fields( self, @@ -688,6 +699,7 @@ def test__given_api_v1_default_bundle_payload__then_posts_gateway_contract_body( "include_cliffs", "version", "policyengine_version", + "spm", "_metadata", "_telemetry", } diff --git a/tests/unit/routes/test_canonical_spm.py b/tests/unit/routes/test_canonical_spm.py index 767de092a..7e8c678e8 100644 --- a/tests/unit/routes/test_canonical_spm.py +++ b/tests/unit/routes/test_canonical_spm.py @@ -319,7 +319,9 @@ def test_certification_checked_before_cached_response(certified, harness): @pytest.mark.parametrize("country_id", ["us", "uk"]) -def test_legacy_country_requests_do_not_receive_spm(harness, country_id): +def test_legacy_country_requests_do_not_receive_spm(harness, country_id, monkeypatch): + monkeypatch.setattr(spm, "_current_bundle", lambda: {}) + monkeypatch.setattr(spm, "simulation_supports_spm", lambda _: False) client, country = harness response = client.post(f"/{country_id}/calculate", json={"household": HOUSEHOLD}) assert response.status_code == 200 diff --git a/tests/unit/routes/test_household_and_user_policy_orm_routes.py b/tests/unit/routes/test_household_and_user_policy_orm_routes.py index ee38bea40..43c782c52 100644 --- a/tests/unit/routes/test_household_and_user_policy_orm_routes.py +++ b/tests/unit/routes/test_household_and_user_policy_orm_routes.py @@ -25,8 +25,10 @@ update_user_policy, ) from policyengine_api.services.household_calculation_service import ( + CalculationResult, HouseholdCalculationService, ) +from tests.fixtures.spm import INSTALLED_SPM_SELECTION, household_result_fields def test_household_under_policy_returns_cached_json_object(orm_session_factory): @@ -65,8 +67,12 @@ def test_household_under_policy_returns_cached_json_object(orm_session_factory): policy_hash="policy-hash", country_package_version=COUNTRY_PACKAGE_VERSIONS["us"], policyengine_version=POLICYENGINE_VERSION, + spm=INSTALLED_SPM_SELECTION, + ), + CachedHouseholdCalculation( + household=stored_result, + **household_result_fields(years=["2026"]), ), - CachedHouseholdCalculation(household=stored_result), ) service = HouseholdCalculationService( primary_session_factory=orm_session_factory, @@ -108,7 +114,12 @@ def test_household_under_policy_calculates_and_caches_json_as_an_object( ) calculated = {"people": {"you": {"net_income": {"2026": 42}}}} country = SimpleNamespace( - calculate=Mock(return_value=calculated), + calculate=Mock( + return_value=CalculationResult( + household=calculated, + **household_result_fields(years=["2026"]), + ) + ), metadata={ "variables": {}, "entities": {"person": {"plural": "people", "roles": {}}}, @@ -134,6 +145,8 @@ def test_household_under_policy_calculates_and_caches_json_as_an_object( country.calculate.assert_called_once_with( {"people": {"you": {}}}, {"gov.example.parameter": 1}, + spm=INSTALLED_SPM_SELECTION, + spm_requested=False, ) diff --git a/tests/unit/services/test_economy_service.py b/tests/unit/services/test_economy_service.py index 60f1a42e1..dd5b1bb88 100644 --- a/tests/unit/services/test_economy_service.py +++ b/tests/unit/services/test_economy_service.py @@ -45,6 +45,11 @@ create_mock_budget_window_batch_execution, create_mock_reform_impact, ) +from tests.fixtures.spm import ( + resolved_options, + worker_result_fields, + worker_spm_capability, +) pytest_plugins = ("tests.fixtures.services.economy_service",) @@ -106,6 +111,28 @@ def make_http_status_error( ) +def complete_budget_window_result( + *, annual_values: dict[str, dict] | None = None, totals: dict | None = None +) -> dict: + years = ("2026", "2027", "2028") + annual_values = annual_values or {} + return { + "kind": "budgetWindow", + "startYear": years[0], + "endYear": years[-1], + "windowSize": len(years), + "annualImpacts": [ + { + "year": year, + **annual_values.get(year, {}), + **worker_result_fields(years=[year]), + } + for year in years + ], + "totals": totals or {}, + } + + class TestEconomyService: class TestGetEconomicImpact: @pytest.fixture @@ -394,7 +421,7 @@ def test__given_no_previous_impact__creates_new_simulation( write_values = ( mock_reform_impacts_service.set_reform_impact.call_args.kwargs ) - assert write_values["options"] == MOCK_OPTIONS + assert write_values["options"] == resolved_options(MOCK_OPTIONS) assert write_values["reform_impact_json"] == {} assert write_values["observability_id"] == MOCK_OBSERVABILITY_ID @@ -685,6 +712,9 @@ def test__given_policies_created_through_orm__submits_decoded_json( "policyengine-simulation-test", MOCK_MODEL_VERSION, ) + simulation_gateway.get_spm_capability.return_value = ( + worker_spm_capability() + ) simulation_gateway.get_execution_id.return_value = "execution-1" monkeypatch.setattr( "policyengine_api.services.economy_service.logger", @@ -1207,22 +1237,18 @@ def test__given_completed_cached_result__returns_completed_batch_result( mock_simulation_entrypoint, mock_budget_window_cache, ): - completed_result = { - "kind": "budgetWindow", - "startYear": "2026", - "endYear": "2028", - "windowSize": 3, - "annualImpacts": [ - { + completed_result = complete_budget_window_result( + annual_values={ + "2026": { "year": "2026", "taxRevenueImpact": 100, "federalTaxRevenueImpact": 80, "stateTaxRevenueImpact": 20, "benefitSpendingImpact": -10, "budgetaryImpact": 90, - } - ], - "totals": { + }, + }, + totals={ "year": "Total", "taxRevenueImpact": 100, "federalTaxRevenueImpact": 80, @@ -1230,7 +1256,7 @@ def test__given_completed_cached_result__returns_completed_batch_result( "benefitSpendingImpact": -10, "budgetaryImpact": 90, }, - } + ) mock_budget_window_cache.get_state.return_value = BudgetWindowCacheState( status="completed", result=completed_result, @@ -1289,14 +1315,7 @@ def test__given_completed_batch_poll__caches_result_and_returns_completed( mock_simulation_entrypoint, mock_budget_window_cache, ): - completed_result = { - "kind": "budgetWindow", - "startYear": "2026", - "endYear": "2028", - "windowSize": 3, - "annualImpacts": [], - "totals": {}, - } + completed_result = complete_budget_window_result() mock_budget_window_cache.get_state.return_value = BudgetWindowCacheState( status="submitted", batch_job_id="fc-budget-123", @@ -1367,14 +1386,7 @@ def test__given_completed_batch_cache_write_fails__does_not_clear_batch_id( mock_simulation_entrypoint, mock_budget_window_cache, ): - completed_result = { - "kind": "budgetWindow", - "startYear": "2026", - "endYear": "2028", - "windowSize": 3, - "annualImpacts": [], - "totals": {}, - } + completed_result = complete_budget_window_result() mock_budget_window_cache.get_state.return_value = BudgetWindowCacheState( status="submitted", batch_job_id="fc-budget-123", @@ -1814,7 +1826,7 @@ def test__given_reordered_options__uses_same_budget_window_cache_identity( ): mock_budget_window_cache.get_state.return_value = BudgetWindowCacheState( status="completed", - result={"kind": "budgetWindow"}, + result=complete_budget_window_result(), observability_id=MOCK_OBSERVABILITY_ID, ) diff --git a/tests/unit/services/test_household_calculation_service.py b/tests/unit/services/test_household_calculation_service.py index 325104734..4dd75757b 100644 --- a/tests/unit/services/test_household_calculation_service.py +++ b/tests/unit/services/test_household_calculation_service.py @@ -22,6 +22,7 @@ HouseholdNotFoundError, PolicyNotFoundError, ) +from tests.fixtures.spm import INSTALLED_SPM_SELECTION, household_result_fields PACKAGE_ROOT = Path(__file__).parents[3] / "policyengine_api" @@ -91,6 +92,7 @@ def _identity() -> HouseholdCalculationIdentity: policy_hash="policy-hash", country_package_version=COUNTRY_PACKAGE_VERSIONS["us"], policyengine_version=POLICYENGINE_VERSION, + spm=INSTALLED_SPM_SELECTION, ) @@ -250,6 +252,7 @@ def calculate(self, household, policy, **_kwargs): return CalculationResult( household={"people": {"you": {"net_income": {"2026": 42}}}}, warnings=("net_income could not be calculated",), + **household_result_fields(years=["2026"]), ) service = HouseholdCalculationService( @@ -326,6 +329,7 @@ def test_calculation_uses_local_cache_without_recomputing(orm_session_factory): CachedHouseholdCalculation( household=calculated, warnings=("net_income could not be calculated",), + **household_result_fields(years=["2026"]), ), ) country = SimpleNamespace( @@ -363,8 +367,9 @@ def test_failed_cache_write_does_not_invalidate_successful_calculation( "variables": {}, "entities": {"person": {"plural": "people", "roles": {}}}, }, - calculate=lambda *_: SimpleNamespace( + calculate=lambda *_, **__: SimpleNamespace( household={"people": {"you": {}}}, + **household_result_fields(years=["2026"]), ), ) diff --git a/tests/unit/services/test_worker_spm.py b/tests/unit/services/test_worker_spm.py index e3b35f8d3..bb86be731 100644 --- a/tests/unit/services/test_worker_spm.py +++ b/tests/unit/services/test_worker_spm.py @@ -43,9 +43,12 @@ def test_actual_settings_validator_rejects_explicit_settings_on_legacy_worker( "packages": {"policyengine-us": {"version": "1.764.6"}}, }, ): - assert validate_worker_spm(country_id) is None - with pytest.raises(SPMValidationError) as error: - validate_worker_spm(country_id, {"geography_kind": "national"}) + with patch( + "policyengine_api.spm.simulation_supports_spm", return_value=False + ): + assert validate_worker_spm(country_id) is None + with pytest.raises(SPMValidationError) as error: + validate_worker_spm(country_id, {"geography_kind": "national"}) assert error.value.code == "SPM_SETTINGS_UNSUPPORTED" diff --git a/tests/unit/test_country_spm.py b/tests/unit/test_country_spm.py index 4b525bb35..1cb13234a 100644 --- a/tests/unit/test_country_spm.py +++ b/tests/unit/test_country_spm.py @@ -293,8 +293,6 @@ def test_real_country_state_only_tax_succeeds_and_receipt_stays_empty( "variable", [ "spm_unit_spm_threshold", - "spm_unit_capped_housing_subsidy", - "spm_unit_net_income", "spm_unit_is_in_spm_poverty", ], ) @@ -313,8 +311,6 @@ def test_real_country_state_only_spm_dependency_requires_geography( "variable", [ "spm_unit_spm_threshold", - "spm_unit_capped_housing_subsidy", - "spm_unit_net_income", "spm_unit_is_in_spm_poverty", ], ) @@ -332,13 +328,18 @@ def test_real_country_state_only_dependency_is_null_when_nothing_was_chosen( assert result.spm_provenance["geography_kind"] == "county" -@pytest.mark.parametrize("county", ["99999", "malformed"]) -def test_real_country_unknown_county_is_structured(real_canonical_country, county): +@pytest.mark.parametrize( + ("county", "expected_code"), + [("99999", "SPM_GEOGRAPHY_UNAVAILABLE"), ("malformed", "SPM_GEOGRAPHY_REQUIRED")], +) +def test_real_country_unknown_county_is_structured( + real_canonical_country, county, expected_code +): household = requested_household() household["households"]["household"]["county_fips"] = {"2024": county} with pytest.raises(ValueError) as caught: real_canonical_country.calculate(household, None, spm_requested=True) - assert spm.spm_error_detail(caught.value)["code"] == "SPM_GEOGRAPHY_UNAVAILABLE" + assert spm.spm_error_detail(caught.value)["code"] == expected_code def test_real_country_unknown_area_is_structured(real_canonical_country): @@ -624,10 +625,15 @@ def test_real_http_tax_only_outside_artifact_years_is_lazy( def test_real_http_unsupported_spm_year_is_structured_only_when_calculated( real_http_client, geography, variable, axes ): + household = household_in_year(variable, 2036, axes=axes) + if variable == "spm_unit_net_income": + household["spm_units"]["spm_unit"]["housing_assistance"] = { + "2036": 1_000 + } response = real_http_client.post( "/us/calculate", json={ - "household": household_in_year(variable, 2036, axes=axes), + "household": household, "spm": selection_for_geography(geography), }, ) diff --git a/tests/unit/test_spm_settings.py b/tests/unit/test_spm_settings.py index c4aa91a04..6ed564f1f 100644 --- a/tests/unit/test_spm_settings.py +++ b/tests/unit/test_spm_settings.py @@ -568,6 +568,7 @@ def test_a_model_this_build_cannot_load_reads_as_no_canonical_model( def refuse(name): raise failure + monkeypatch.setattr(spm, "_current_bundle", lambda: deepcopy(LEGACY_BUNDLE)) monkeypatch.setattr(spm.importlib, "import_module", refuse) assert spm._installed_country_implements_spm("us") is False From 3d90999aa70be79ab95c42da99313ce038d0fc0c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:28:53 +0400 Subject: [PATCH 17/22] docs: explain PolicyEngine 6 compatibility --- changelog.d/policyengine-6-1-2.changed.md | 1 + docs/canonical-spm.md | 58 ++++++++++++++++++++--- 2 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 changelog.d/policyengine-6-1-2.changed.md diff --git a/changelog.d/policyengine-6-1-2.changed.md b/changelog.d/policyengine-6-1-2.changed.md new file mode 100644 index 000000000..5e9670f94 --- /dev/null +++ b/changelog.d/policyengine-6-1-2.changed.md @@ -0,0 +1 @@ +Update the API to PolicyEngine.py 6.1.2, including PolicyEngine Core 3.32.5, PolicyEngine US 2.2.1, PolicyEngine UK 2.90.2, and spm-calculator 1.0.0. Validate that simulation workers report the same bundle and SPM measurement settings before submitting US economy calculations. diff --git a/docs/canonical-spm.md b/docs/canonical-spm.md index 590f9fe07..704a9346a 100644 --- a/docs/canonical-spm.md +++ b/docs/canonical-spm.md @@ -1,8 +1,8 @@ # Canonical SPM household API contract -This contract is enabled only by an installed, certified US bundle that pins its -SPM forecast hash/scenario and supports the country `spm` constructor. This change -does not select a new released model or promote a deployment. +This contract is enabled only by an installed, certified US bundle that identifies +one SPM forecast file and scenario and supports the country `spm` constructor. +Installing a package version does not deploy the API or its simulation worker. Whether a bundle predates this contract is decided by the installed country model's capability, never by a bundle version string; the automated bundle update @@ -13,10 +13,50 @@ its existing behavior when settings are omitted and rejects explicit SPM setting with `SPM_SETTINGS_UNSUPPORTED`, whatever its version. A bundle whose US model does implement the constructor but ships no certified `measurements.spm` configuration fails closed with `SPM_CONFIGURATION_UNAVAILABLE`: such a -deployment also fails `/readiness-check`, so the condition is reported where the -release is gated rather than only on each request. Other countries retain their +deployment also fails `/readiness-check`, so the release checks report the +condition rather than leaving it to individual requests. Other countries retain their behavior and reject US-only SPM settings. +## PolicyEngine 6.1.2 compatibility + +This API installs `policyengine[models]==6.1.2`. That requirement installs the +five package versions recorded in the PolicyEngine.py bundle manifest: + +| Package | Version | +| --- | --- | +| `policyengine` | 6.1.2 | +| `policyengine-core` | 3.32.5 | +| `policyengine-us` | 2.2.1 | +| `policyengine-uk` | 2.90.2 | +| `spm-calculator` | 1.0.0 | + +The same manifest identifies `populace-us-2024-spm-20260915` as the US data +release and `policyengine-uk-data-1.56.16` as the UK data release. The API does +not declare separate country-model or SPM calculator requirements. This prevents +an independently selected package version from disagreeing with the tested +combination in the manifest. + +PolicyEngine.py 6 also records the exact SPM forecast file and default scenario +used for US calculations. The API resolves those values before it submits an +economy calculation. It then checks that the worker reports the same values and +that every completed result includes a receipt describing the forecast, scenario, +years, and geography it used. This is the concrete behavior referred to elsewhere +as SPM selection and provenance. + +The API and simulation worker can be reviewed independently. Economy calculations +using this API version require a worker whose `/versions` response maps the +PolicyEngine.py, US, and UK versions above to the same worker application and +reports the matching `canonical-spm-v1` settings. Until such a worker is deployed, +the API returns `SPM_CONFIGURATION_UNAVAILABLE` instead of submitting a calculation +to an incompatible worker. [Simulation API PR 703](https://github.com/PolicyEngine/policyengine-sim-api/pull/703) +implements the corresponding worker package update. + +The automated dependency updater changes the single `policyengine[models]` +requirement in `pyproject.toml` and the generic Docker image, refreshes `uv.lock`, +and checks all five installed distributions against the new manifest. Its pull +request description lists the five package versions and both certified data +release identifiers. + ## Selecting a measurement `GET /us/metadata` exposes `result.spm.available`. When true, `settings_schema` @@ -249,7 +289,8 @@ SPM input failures return HTTP 400 in the existing validation envelope: ``` `SPM_GEOGRAPHY_REQUIRED` indicates missing explicit geography for an SPM -dependency; `SPM_GEOGRAPHY_UNAVAILABLE` indicates malformed/unknown county or area; +dependency, including a county value that is not a five-digit FIPS code; +`SPM_GEOGRAPHY_UNAVAILABLE` indicates a syntactically valid but unknown county or area; `SPM_COMPOSITION_REQUIRED` indicates no classified SPM adult. Country error text is retained. `SPM_YEAR_UNAVAILABLE` indicates an unsupported measurement year. Settings errors use `SPM_SETTINGS_INVALID`, @@ -314,7 +355,10 @@ receiving a previously stored rich receipt. Household and non-US report output shapes retain their existing behavior. See the [canonical SPM worker PR](https://github.com/PolicyEngine/policyengine-sim-api/pull/677) -for the implemented worker paths and remaining coordinated release gates. +for the implemented worker paths and remaining coordinated release validation +checks. See [Simulation API PR 703](https://github.com/PolicyEngine/policyengine-sim-api/pull/703) +for the corresponding PolicyEngine.py 6.2.1 worker package update and compatibility +checks. Partial selections preserve omitted fields in JSON; omissions inherit the certified From 76402aa2561fcaaf01b21c28eee08bcf5cf885c9 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 29 Sep 2026 20:58:47 +0400 Subject: [PATCH 18/22] ci: run API tests before simulation rollout --- .github/workflows/pr.yml | 7 +------ tests/unit/services/test_economy_service.py | 4 +--- tests/unit/services/test_worker_spm.py | 4 +--- tests/unit/test_bundle_update_pins.py | 15 ++++++++------- tests/unit/test_country_spm.py | 4 +--- 5 files changed, 12 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b2bfb28ed..e7b5292e7 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -94,7 +94,6 @@ jobs: test_container_builds: name: Docker runs-on: ubuntu-latest - needs: ensure-policyengine-bundle-supported-by-simulation-api permissions: contents: read packages: write @@ -112,7 +111,6 @@ jobs: test_cloud_run_container_builds: name: Cloud Run container runs-on: ubuntu-latest - needs: ensure-policyengine-bundle-supported-by-simulation-api permissions: contents: read steps: @@ -123,7 +121,6 @@ jobs: test_env_vars: name: Test environment variables runs-on: ubuntu-latest - needs: ensure-policyengine-bundle-supported-by-simulation-api permissions: contents: read id-token: write @@ -154,9 +151,7 @@ jobs: test: name: Test runs-on: ubuntu-latest - needs: - - ensure-policyengine-bundle-supported-by-simulation-api - - test_env_vars + needs: test_env_vars permissions: contents: read id-token: write diff --git a/tests/unit/services/test_economy_service.py b/tests/unit/services/test_economy_service.py index dd5b1bb88..e9c30264f 100644 --- a/tests/unit/services/test_economy_service.py +++ b/tests/unit/services/test_economy_service.py @@ -712,9 +712,7 @@ def test__given_policies_created_through_orm__submits_decoded_json( "policyengine-simulation-test", MOCK_MODEL_VERSION, ) - simulation_gateway.get_spm_capability.return_value = ( - worker_spm_capability() - ) + simulation_gateway.get_spm_capability.return_value = worker_spm_capability() simulation_gateway.get_execution_id.return_value = "execution-1" monkeypatch.setattr( "policyengine_api.services.economy_service.logger", diff --git a/tests/unit/services/test_worker_spm.py b/tests/unit/services/test_worker_spm.py index bb86be731..b09a614d2 100644 --- a/tests/unit/services/test_worker_spm.py +++ b/tests/unit/services/test_worker_spm.py @@ -43,9 +43,7 @@ def test_actual_settings_validator_rejects_explicit_settings_on_legacy_worker( "packages": {"policyengine-us": {"version": "1.764.6"}}, }, ): - with patch( - "policyengine_api.spm.simulation_supports_spm", return_value=False - ): + with patch("policyengine_api.spm.simulation_supports_spm", return_value=False): assert validate_worker_spm(country_id) is None with pytest.raises(SPMValidationError) as error: validate_worker_spm(country_id, {"geography_kind": "national"}) diff --git a/tests/unit/test_bundle_update_pins.py b/tests/unit/test_bundle_update_pins.py index 75ec4a74f..2ccfb19e6 100644 --- a/tests/unit/test_bundle_update_pins.py +++ b/tests/unit/test_bundle_update_pins.py @@ -147,17 +147,18 @@ def test_lock_failure_stops_before_commit(update_checkout): result, calls = run_update(update_checkout, BUNDLE_TEST_LOCK_FAIL="1") assert result.returncode != 0 assert "uv lock failed after 3 attempts" in result.stderr - assert sum( - call == ["uv", "lock", "--upgrade-package", "policyengine"] - for call in calls - ) == 3 + assert ( + sum( + call == ["uv", "lock", "--upgrade-package", "policyengine"] + for call in calls + ) + == 3 + ) assert not any(call[:2] == ["git", "commit"] for call in calls) def test_manifest_version_mismatch_stops_before_commit(update_checkout): - result, calls = run_update( - update_checkout, BUNDLE_TEST_REPORTED_VERSION="4.0.9" - ) + result, calls = run_update(update_checkout, BUNDLE_TEST_REPORTED_VERSION="4.0.9") assert result.returncode != 0 assert "does not match requested version" in result.stderr assert not any(call[:2] == ["git", "commit"] for call in calls) diff --git a/tests/unit/test_country_spm.py b/tests/unit/test_country_spm.py index 1cb13234a..937c0e537 100644 --- a/tests/unit/test_country_spm.py +++ b/tests/unit/test_country_spm.py @@ -627,9 +627,7 @@ def test_real_http_unsupported_spm_year_is_structured_only_when_calculated( ): household = household_in_year(variable, 2036, axes=axes) if variable == "spm_unit_net_income": - household["spm_units"]["spm_unit"]["housing_assistance"] = { - "2036": 1_000 - } + household["spm_units"]["spm_unit"]["housing_assistance"] = {"2036": 1_000} response = real_http_client.post( "/us/calculate", json={ From 07283a0f89da97c52ccee05c344144639c43ec8d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:41:12 +0400 Subject: [PATCH 19/22] Update PolicyEngine bundle to 6.2.1 --- changelog.d/policyengine-6-1-2.changed.md | 1 - changelog.d/policyengine-6-2-1.changed.md | 1 + docker/Dockerfile | 2 +- docs/canonical-spm.md | 32 ++++++------ pyproject.toml | 2 +- .../integration/test_v2_catalog_installed.py | 8 +-- tests/unit/test_worker_spm_release.py | 49 ++++++++++--------- uv.lock | 37 +++++++------- 8 files changed, 66 insertions(+), 66 deletions(-) delete mode 100644 changelog.d/policyengine-6-1-2.changed.md create mode 100644 changelog.d/policyengine-6-2-1.changed.md diff --git a/changelog.d/policyengine-6-1-2.changed.md b/changelog.d/policyengine-6-1-2.changed.md deleted file mode 100644 index 5e9670f94..000000000 --- a/changelog.d/policyengine-6-1-2.changed.md +++ /dev/null @@ -1 +0,0 @@ -Update the API to PolicyEngine.py 6.1.2, including PolicyEngine Core 3.32.5, PolicyEngine US 2.2.1, PolicyEngine UK 2.90.2, and spm-calculator 1.0.0. Validate that simulation workers report the same bundle and SPM measurement settings before submitting US economy calculations. diff --git a/changelog.d/policyengine-6-2-1.changed.md b/changelog.d/policyengine-6-2-1.changed.md new file mode 100644 index 000000000..1f716b484 --- /dev/null +++ b/changelog.d/policyengine-6-2-1.changed.md @@ -0,0 +1 @@ +Update the API to PolicyEngine.py 6.2.1, including PolicyEngine Core 3.32.10, PolicyEngine US 2.2.1, PolicyEngine UK 2.102.3, and spm-calculator 1.0.0. Validate that simulation workers report the same bundle and SPM measurement settings before submitting US economy calculations. diff --git a/docker/Dockerfile b/docker/Dockerfile index 9f5a41635..83b3cabe4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ FROM python:3.12 # Match the API bundle in pyproject.toml; the bundle updater changes both pins. # The models extra pins the complete certified runtime package set. -RUN pip install "policyengine[models]==6.1.2" ipython +RUN pip install "policyengine[models]==6.2.1" ipython diff --git a/docs/canonical-spm.md b/docs/canonical-spm.md index 704a9346a..9794be4bb 100644 --- a/docs/canonical-spm.md +++ b/docs/canonical-spm.md @@ -17,24 +17,22 @@ deployment also fails `/readiness-check`, so the release checks report the condition rather than leaving it to individual requests. Other countries retain their behavior and reject US-only SPM settings. -## PolicyEngine 6.1.2 compatibility +## PolicyEngine 6 compatibility -This API installs `policyengine[models]==6.1.2`. That requirement installs the -five package versions recorded in the PolicyEngine.py bundle manifest: +This API pins one exact `policyengine[models]` release in `pyproject.toml` and +the generic Docker image. That requirement installs the five package versions +recorded in that PolicyEngine.py release's bundle manifest: -| Package | Version | -| --- | --- | -| `policyengine` | 6.1.2 | -| `policyengine-core` | 3.32.5 | -| `policyengine-us` | 2.2.1 | -| `policyengine-uk` | 2.90.2 | -| `spm-calculator` | 1.0.0 | - -The same manifest identifies `populace-us-2024-spm-20260915` as the US data -release and `policyengine-uk-data-1.56.16` as the UK data release. The API does -not declare separate country-model or SPM calculator requirements. This prevents -an independently selected package version from disagreeing with the tested -combination in the manifest. +- `policyengine` +- `policyengine-core` +- `policyengine-us` +- `policyengine-uk` +- `spm-calculator` + +The same manifest identifies the certified US and UK data releases. The API +does not declare separate country-model or SPM calculator requirements. This +prevents an independently selected package version from disagreeing with the +tested combination in the manifest. PolicyEngine.py 6 also records the exact SPM forecast file and default scenario used for US calculations. The API resolves those values before it submits an @@ -45,7 +43,7 @@ as SPM selection and provenance. The API and simulation worker can be reviewed independently. Economy calculations using this API version require a worker whose `/versions` response maps the -PolicyEngine.py, US, and UK versions above to the same worker application and +installed PolicyEngine.py, US, and UK versions to the same worker application and reports the matching `canonical-spm-v1` settings. Until such a worker is deployed, the API returns `SPM_CONFIGURATION_UNAVAILABLE` instead of submitting a calculation to an incompatible worker. [Simulation API PR 703](https://github.com/PolicyEngine/policyengine-sim-api/pull/703) diff --git a/pyproject.toml b/pyproject.toml index cf184ed9f..d4bf88535 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ dependencies = [ # versions certified by this PolicyEngine bundle. Cloud Run installs the # complete frozen environment from uv.lock; the generic image installs # this requirement directly. - "policyengine[models]==6.1.2", + "policyengine[models]==6.2.1", "pydantic", "pymysql", "python-dotenv", diff --git a/tests/integration/test_v2_catalog_installed.py b/tests/integration/test_v2_catalog_installed.py index f897bce6b..88718b122 100644 --- a/tests/integration/test_v2_catalog_installed.py +++ b/tests/integration/test_v2_catalog_installed.py @@ -39,10 +39,10 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: assert catalog.entity_counts() == { "models": 2, "model_versions": 2, - "variables": 7_046, - "parameter_nodes": 29_118, - "parameters": 103_705, - "parameter_values": 1_192_826, + "variables": 7_104, + "parameter_nodes": 29_170, + "parameters": 103_853, + "parameter_values": 1_193_400, "datasets": 2, "regions": 826, } diff --git a/tests/unit/test_worker_spm_release.py b/tests/unit/test_worker_spm_release.py index 416af4113..9df37be59 100644 --- a/tests/unit/test_worker_spm_release.py +++ b/tests/unit/test_worker_spm_release.py @@ -8,13 +8,13 @@ from policyengine_api.spm import SPMValidationError -BUNDLE_VERSION = "6.1.2" -PACKAGE_VERSIONS = { - "policyengine": BUNDLE_VERSION, - "policyengine-core": "3.32.5", - "policyengine-us": "2.2.1", - "policyengine-uk": "2.90.2", - "spm-calculator": "1.0.0", +FIXTURE_BUNDLE_VERSION = "fixture-policyengine" +FIXTURE_PACKAGE_VERSIONS = { + "policyengine": FIXTURE_BUNDLE_VERSION, + "policyengine-core": "fixture-core", + "policyengine-us": "fixture-us", + "policyengine-uk": "fixture-uk", + "spm-calculator": "fixture-spm", } SELECTION = { "forecast_content_sha256": "a" * 64, @@ -29,29 +29,29 @@ @pytest.fixture def registry(monkeypatch): - monkeypatch.setattr(constants, "POLICYENGINE_VERSION", BUNDLE_VERSION) + monkeypatch.setattr(constants, "POLICYENGINE_VERSION", FIXTURE_BUNDLE_VERSION) monkeypatch.setattr( constants, "_policyengine_bundle", { "packages": { name: {"name": name, "version": package_version} - for name, package_version in PACKAGE_VERSIONS.items() + for name, package_version in FIXTURE_PACKAGE_VERSIONS.items() } }, ) monkeypatch.setattr( worker_spm_release, "distribution_version", - PACKAGE_VERSIONS.__getitem__, + FIXTURE_PACKAGE_VERSIONS.__getitem__, ) monkeypatch.setattr(worker_spm, "normalize_spm_selection", lambda *_: SELECTION) return { - "policyengine": {BUNDLE_VERSION: WORKER_APP}, - "us": {PACKAGE_VERSIONS["policyengine-us"]: WORKER_APP}, - "uk": {PACKAGE_VERSIONS["policyengine-uk"]: WORKER_APP}, + "policyengine": {FIXTURE_BUNDLE_VERSION: WORKER_APP}, + "us": {FIXTURE_PACKAGE_VERSIONS["policyengine-us"]: WORKER_APP}, + "uk": {FIXTURE_PACKAGE_VERSIONS["policyengine-uk"]: WORKER_APP}, "spm_capabilities": { - BUNDLE_VERSION: { + FIXTURE_BUNDLE_VERSION: { "contract_version": "canonical-spm-v1", "defaults": dict(SELECTION), } @@ -61,7 +61,10 @@ def registry(monkeypatch): def test_release_accepts_matching_installed_bundle_and_worker(registry): assert ( - worker_spm_release.validate_installed_worker(registry, BUNDLE_VERSION) + worker_spm_release.validate_installed_worker( + registry, + FIXTURE_BUNDLE_VERSION, + ) == SELECTION ) @@ -72,7 +75,7 @@ def test_release_accepts_matching_installed_bundle_and_worker(registry): ) def test_release_rejects_worker_without_matching_spm_capability(registry, mutation): registry = deepcopy(registry) - capability = registry["spm_capabilities"][BUNDLE_VERSION] + capability = registry["spm_capabilities"][FIXTURE_BUNDLE_VERSION] if mutation == "missing-capability": registry["spm_capabilities"] = {} elif mutation == "wrong-contract": @@ -81,14 +84,14 @@ def test_release_rejects_worker_without_matching_spm_capability(registry, mutati capability["defaults"]["forecast_content_sha256"] = "b" * 64 with pytest.raises(SPMValidationError): - worker_spm_release.validate_installed_worker(registry, BUNDLE_VERSION) + worker_spm_release.validate_installed_worker(registry, FIXTURE_BUNDLE_VERSION) @pytest.mark.parametrize("route", ["policyengine", "us", "uk"]) def test_release_rejects_missing_worker_route(registry, route): registry[route] = {} with pytest.raises(ValueError, match="no .* application"): - worker_spm_release.validate_installed_worker(registry, BUNDLE_VERSION) + worker_spm_release.validate_installed_worker(registry, FIXTURE_BUNDLE_VERSION) @pytest.mark.parametrize("route", ["us", "uk"]) @@ -96,29 +99,29 @@ def test_release_rejects_country_route_to_different_worker(registry, route): version = next(iter(registry[route])) registry[route][version] = "different-worker" with pytest.raises(ValueError, match="different worker"): - worker_spm_release.validate_installed_worker(registry, BUNDLE_VERSION) + worker_spm_release.validate_installed_worker(registry, FIXTURE_BUNDLE_VERSION) def test_release_rejects_installed_distribution_outside_manifest( registry, monkeypatch, ): - installed = {**PACKAGE_VERSIONS, "policyengine-core": "0.0.1"} + installed = {**FIXTURE_PACKAGE_VERSIONS, "policyengine-core": "different-core"} monkeypatch.setattr( worker_spm_release, "distribution_version", installed.__getitem__, ) with pytest.raises(ValueError, match="policyengine-core"): - worker_spm_release.validate_installed_worker(registry, BUNDLE_VERSION) + worker_spm_release.validate_installed_worker(registry, FIXTURE_BUNDLE_VERSION) def test_release_rejects_requirement_outside_manifest(registry): with pytest.raises(ValueError, match="manifest"): - worker_spm_release.validate_installed_worker(registry, "7.0.0") + worker_spm_release.validate_installed_worker(registry, "different-bundle") def test_release_rejects_manifest_without_complete_package_set(registry): constants._policyengine_bundle["packages"].pop("spm-calculator") with pytest.raises(ValueError, match="spm-calculator"): - worker_spm_release.validate_installed_worker(registry, BUNDLE_VERSION) + worker_spm_release.validate_installed_worker(registry, FIXTURE_BUNDLE_VERSION) diff --git a/uv.lock b/uv.lock index 9cdc2e332..57a663cc6 100644 --- a/uv.lock +++ b/uv.lock @@ -2743,7 +2743,7 @@ wheels = [ [[package]] name = "policyengine" -version = "5.2.0" +version = "6.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "h5py" }, @@ -2755,9 +2755,9 @@ dependencies = [ { name = "pydantic" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/45/2d/5b09f414e26573748f9ae2e448d6f9f94196b7733e8400d727db9ed17868/policyengine-5.2.0.tar.gz", hash = "sha256:c1e7a3a8d7fb23401aedbe8a9fa2d6ed69e24ae9983b6a4a8a0d5112394b0bc4", size = 732330, upload-time = "2026-08-29T17:24:52.814Z" } +sdist = { url = "https://files.pythonhosted.org/packages/67/f2/3a75eb571fa3eda2e9f043c145cadae9eab4df6959fc447e767f92885d88/policyengine-6.2.1.tar.gz", hash = "sha256:0f2e51be23229ab5d8a8f80fb7b716fe4d3780806e2e16dfcda0362983beac56", size = 772927, upload-time = "2026-09-30T22:19:57.846Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/c5/677144df3800ae41dd587b292bd633c09edd5c998695ca15987c05730bc6/policyengine-5.2.0-py3-none-any.whl", hash = "sha256:e73307f787cf366dc16c080b708c39c28a58b0adf34abe51de5ecd97ecfab154", size = 226102, upload-time = "2026-08-29T17:24:51.377Z" }, + { url = "https://files.pythonhosted.org/packages/5a/8b/4b216e42d150852e51c01ea654455496e1e1197c008e72e70f29392dec8c/policyengine-6.2.1-py3-none-any.whl", hash = "sha256:d07ff400da096be4b43843bd333ea0b7f42b06f5eea584e72c78f75a16fe3cdb", size = 252406, upload-time = "2026-09-30T22:19:56.61Z" }, ] [package.optional-dependencies] @@ -2765,11 +2765,12 @@ models = [ { name = "policyengine-core" }, { name = "policyengine-uk" }, { name = "policyengine-us" }, + { name = "spm-calculator" }, ] [[package]] name = "policyengine-api" -version = "4.2.5" +version = "4.2.2" source = { editable = "." } dependencies = [ { name = "a2wsgi" }, @@ -2801,7 +2802,6 @@ dependencies = [ { name = "python-dotenv" }, { name = "redis" }, { name = "rq" }, - { name = "spm-calculator" }, { name = "sqlalchemy" }, { name = "sqlmodel" }, { name = "streamlit" }, @@ -2844,7 +2844,7 @@ requires-dist = [ { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.15,<2" }, { name = "openai" }, { name = "packaging", specifier = ">=24,<27" }, - { name = "policyengine", extras = ["models"], specifier = "==5.2.0" }, + { name = "policyengine", extras = ["models"], specifier = "==6.2.1" }, { name = "policyengine-canada", specifier = "==0.96.3" }, { name = "policyengine-il", specifier = "==0.1.0" }, { name = "policyengine-ng", specifier = "==0.5.1" }, @@ -2859,7 +2859,6 @@ requires-dist = [ { name = "redis" }, { name = "rq" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9.0" }, - { name = "spm-calculator", specifier = "==0.3.1" }, { name = "sqlalchemy", specifier = ">=2,<3" }, { name = "sqlmodel", specifier = ">=0.0.39,<0.1" }, { name = "streamlit" }, @@ -2899,7 +2898,7 @@ wheels = [ [[package]] name = "policyengine-core" -version = "3.30.1" +version = "3.32.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dpath" }, @@ -2919,9 +2918,9 @@ dependencies = [ { name = "standard-imghdr" }, { name = "wheel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a1/9d/f71da3e348ddc52e058c76b72bbc4b2f20abcf0e7389ecd5aca5d550fd14/policyengine_core-3.30.1.tar.gz", hash = "sha256:a16f29fe51ec01a7be2171386f8bd26b9ba55ffc338adb70adcadafe33d9c55f", size = 500419, upload-time = "2026-07-19T15:50:41.288Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/cd/9fc967c5b787baa5f9d83461773ba0982a8e22ae83c40155a7ee55edd11a/policyengine_core-3.32.10.tar.gz", hash = "sha256:0f51e8abfa8f172c69d03ba8f4304632a348fc6dca780e41021b2fd0cec04fde", size = 427479, upload-time = "2026-09-29T17:56:43.582Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/7e/07624d8add8889af1b4af796d36c38b2e2af494b1d78a064fab29e9bea00/policyengine_core-3.30.1-py3-none-any.whl", hash = "sha256:2dbcf5f590a0199a7b7c77fcbbda2ff6bc289f6c6169ca0af3beace35d8b3e63", size = 244846, upload-time = "2026-07-19T15:50:39.651Z" }, + { url = "https://files.pythonhosted.org/packages/58/50/291d73da16f39a3c4f05dffb6d40b1142fc400ec39b5a2ca7481e6fe9d90/policyengine_core-3.32.10-py3-none-any.whl", hash = "sha256:124afc555fe7c848ae8237b820fed0b6ea0ea1280e8850ad8bf356937ee10e5b", size = 247410, upload-time = "2026-09-29T17:56:42.218Z" }, ] [[package]] @@ -2978,7 +2977,7 @@ otlp-grpc = [ [[package]] name = "policyengine-uk" -version = "2.90.2" +version = "2.102.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -2986,14 +2985,14 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/e1/6652fd4ff8806d81b7c846ce2baeef4b9a8457ed380bb0853419d3cf36d9/policyengine_uk-2.90.2.tar.gz", hash = "sha256:c9baeaded1bb6d6f5ca5fcd962b80a198509513737c194aa867b0ef6d13d6a09", size = 1237478, upload-time = "2026-08-13T05:30:05.961Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/49/1db19a439463f1304e93521debf40794f7b3483d37356c6d99c346112d0f/policyengine_uk-2.102.3.tar.gz", hash = "sha256:cdb0900e78818b46fdb79b2a8299bce6c7d6c14692e26cf38367781beea607be", size = 1314347, upload-time = "2026-09-28T19:50:44.714Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/da/658bff2df6094bd7bf2b0f561ff60fd41508adbd0a3af29b4553c83bc678/policyengine_uk-2.90.2-py3-none-any.whl", hash = "sha256:f100ef97e20b29b84138078f1dbc5b83f8857e6bc74b745fc6abef7ea1eeadf2", size = 2034933, upload-time = "2026-08-13T05:30:03.884Z" }, + { url = "https://files.pythonhosted.org/packages/37/e6/e8d9069d498f1510a1380f24a78e957902d01f00b10248c146d6710d5b37/policyengine_uk-2.102.3-py3-none-any.whl", hash = "sha256:f02a76712a3edf9983c713c1db42e9653a6e974e4ce335dd4b5676bbc2bcd636", size = 2189605, upload-time = "2026-09-28T19:50:42.472Z" }, ] [[package]] name = "policyengine-us" -version = "1.764.6" +version = "2.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -3003,9 +3002,9 @@ dependencies = [ { name = "tables" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6e/cc/9e65a9586069beab1f02102a23f7a0e6332f1a9c2d590cb9f0e68f9ac987/policyengine_us-1.764.6.tar.gz", hash = "sha256:9817d48abc5b7d690b8abe33047005e5cf40b71654a85a2534bb5e3bcf921f79", size = 11009555, upload-time = "2026-07-06T13:13:07.254Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/09/763b21371a62635e15e62b53fb696e299f01460907c27313d40f63031eb2/policyengine_us-2.2.1.tar.gz", hash = "sha256:d5300c91f3eae2e47a61f485fdfde91a892b80e95db3e46ff2c4bbd417d76501", size = 11946510, upload-time = "2026-09-15T04:45:50.33Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/5a/0566666f30ad06e416f0cf25b24bdee73acd71897fb2de3c0ac43b59e3e3/policyengine_us-1.764.6-py3-none-any.whl", hash = "sha256:2b2c5c02c26ab212270948db1f30129728b625fc6109fc9ca0c604b714546909", size = 13117921, upload-time = "2026-07-06T13:13:03.476Z" }, + { url = "https://files.pythonhosted.org/packages/5e/09/86ac3968d2a18e28fce3ba1186fc21728e9f299b96a806d6f204f1e0aaed/policyengine_us-2.2.1-py3-none-any.whl", hash = "sha256:0993a6c73fcdfbe171a796aca9d302741bd8e00ab83388ad81c15a08740318c6", size = 14885537, upload-time = "2026-09-15T04:45:46.649Z" }, ] [[package]] @@ -3924,7 +3923,7 @@ wheels = [ [[package]] name = "spm-calculator" -version = "0.3.1" +version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "census" }, @@ -3934,9 +3933,9 @@ dependencies = [ { name = "requests" }, { name = "us" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/54/3b/b805c7e3e18c5b5c00f61b60112f9690d084c910e2481bc020f35390d8fd/spm_calculator-0.3.1.tar.gz", hash = "sha256:41f2f4d00d8c03422a7d57b800052e7760b88e463a5884802f83ed58d35c18c1", size = 75945, upload-time = "2026-04-17T19:52:39.707Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c2/5e/3f28b212c401795990250bb3daec741f699a8c7dc2a23a3db7005df6feb6/spm_calculator-1.0.0.tar.gz", hash = "sha256:a99aac8c2c0bf81a9455105bbf872366f1ea9066a99cd793714881cd3db846dc", size = 7150825, upload-time = "2026-09-11T15:45:12.732Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/1b/29f705f8a96fc7f55f2c07dfcddbbae78efdc6f174d25d4a0560fc3f5cf9/spm_calculator-0.3.1-py3-none-any.whl", hash = "sha256:52c57ecc5a240ec941b0f2b0d93bc4fa437ef6250e233baed8e11916fa9c1150", size = 57826, upload-time = "2026-04-17T19:52:38.444Z" }, + { url = "https://files.pythonhosted.org/packages/19/a0/c484f69a0ebf88a9b9fd0ac28176f8df66550f46714f1b0c5ec0b360824e/spm_calculator-1.0.0-py3-none-any.whl", hash = "sha256:e354937a5e1a4045d4966ed594a528d8b02866fabaac9bb5672017004b627305", size = 7395384, upload-time = "2026-09-11T15:45:10.492Z" }, ] [[package]] From 92320490953250ab7b774910d80e668adbd2e856 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:36:02 +0400 Subject: [PATCH 20/22] Integrate compact SPM receipts with PolicyEngine 6.2.1 --- tests/fixtures/services/economy_service.py | 5 ++- tests/fixtures/spm.py | 37 ++++++++++++++----- .../test_calculate_deprecated_inputs.py | 7 +++- .../routes/test_calculate_error_statuses.py | 2 +- .../test_household_calculation_service.py | 14 +++++-- uv.lock | 8 ++-- 6 files changed, 53 insertions(+), 20 deletions(-) diff --git a/tests/fixtures/services/economy_service.py b/tests/fixtures/services/economy_service.py index adc477ddb..e1ae2d190 100644 --- a/tests/fixtures/services/economy_service.py +++ b/tests/fixtures/services/economy_service.py @@ -11,6 +11,7 @@ from policyengine_api.data.v1_models import ReformImpact from tests.fixtures.spm import ( options_hash_segment, + provenance_schema_hash_segment, worker_result_fields, worker_spm_capability, ) @@ -34,7 +35,9 @@ + options_hash_segment() + "&dataset=default" + "&model_version=1.2.3" - + "&policyengine_version=3.4.0]" + + "&policyengine_version=3.4.0" + + provenance_schema_hash_segment() + + "]" ) MOCK_OPTIONS_HASH = ( MOCK_LOOKUP_OPTIONS_HASH[:-1] diff --git a/tests/fixtures/spm.py b/tests/fixtures/spm.py index 8406251d1..c295633ec 100644 --- a/tests/fixtures/spm.py +++ b/tests/fixtures/spm.py @@ -50,15 +50,21 @@ def spm_receipt(*, years, selection: dict | None = None) -> dict: if resolved is None: raise ValueError("An SPM receipt requires a resolved selection") return { + "schema_version": "canonical-spm-provenance-v2", "forecast_id": "test-only", "forecast_sha256": resolved["forecast_content_sha256"], "scenario": resolved["scenario"], "geography_kind": resolved["geography_kind"], - "runtime_versions": {"policyengine-us": "test-only"}, - "years": {str(year): {"status": "forecast"} for year in years}, - "geographies": [], - "composition_method": "classified-inputs", - "storage_method": "formula", + "geography_id": resolved["geography_id"], + "county_vintage": resolved["county_vintage"], + "as_of": resolved["as_of"], + "years": sorted({str(year) for year in years}), + "runtime_versions": { + "policyengine": "test-only", + "policyengine-core": "test-only", + "policyengine-us": "test-only", + "spm-calculator": "test-only", + }, } @@ -69,10 +75,16 @@ def worker_result_fields(*, years, selection: dict | None = None) -> dict: return {} receipt = spm_receipt(years=years, selection=resolved) return { - "spm_config": dict(resolved), "spm_provenance": { - "baseline": [dict(receipt)], - "reform": [dict(receipt)], + "schema_version": "canonical-spm-comparison-v2", + "baseline": { + "receipt": dict(receipt), + "execution_count": 1, + }, + "reform": { + "receipt": dict(receipt), + "execution_count": 1, + }, }, } @@ -83,7 +95,6 @@ def household_result_fields(*, years, selection: dict | None = None) -> dict: if resolved is None: return {} return { - "spm_config": dict(resolved), "spm_provenance": spm_receipt(years=years, selection=resolved), } @@ -100,3 +111,11 @@ def options_hash_segment(selection: dict | None = None) -> str: """Return the part of a sorted options hash contributed by SPM.""" resolved = INSTALLED_SPM_SELECTION if selection is None else selection return "" if resolved is None else f"&spm={resolved}" + + +def provenance_schema_hash_segment(selection: dict | None = None) -> str: + """Return the cache-identity segment for compact society-wide receipts.""" + resolved = INSTALLED_SPM_SELECTION if selection is None else selection + if resolved is None: + return "" + return "&spm_provenance_schema=canonical-spm-comparison-v2" diff --git a/tests/unit/routes/test_calculate_deprecated_inputs.py b/tests/unit/routes/test_calculate_deprecated_inputs.py index 6a2da2a17..3c52df5b4 100644 --- a/tests/unit/routes/test_calculate_deprecated_inputs.py +++ b/tests/unit/routes/test_calculate_deprecated_inputs.py @@ -5,8 +5,10 @@ from policyengine_api.routes import household_routes from policyengine_api.extensions import cache from policyengine_api.services.household_calculation_service import ( + CalculationResult, HouseholdCalculationService, ) +from tests.fixtures.spm import household_result_fields class DummyCountry: @@ -41,7 +43,10 @@ def __init__(self): def calculate(self, household, policy, **_kwargs): self.household = household self.policy = policy - return {"household": household, "policy": policy} + return CalculationResult( + household={"household": household, "policy": policy}, + **household_result_fields(years=["2025"]), + ) @pytest.fixture diff --git a/tests/unit/routes/test_calculate_error_statuses.py b/tests/unit/routes/test_calculate_error_statuses.py index 208ac512a..4af690247 100644 --- a/tests/unit/routes/test_calculate_error_statuses.py +++ b/tests/unit/routes/test_calculate_error_statuses.py @@ -21,7 +21,7 @@ class ParsingErrorCountry(DummyCountry): - def prepare_calculation(self, household, policy): + def prepare_calculation(self, household, policy, **_kwargs): raise SituationParsingError( ["people", "you", "employment_income", "2026"], "Can't deal with value: expected type number, received '{}'.", diff --git a/tests/unit/services/test_household_calculation_service.py b/tests/unit/services/test_household_calculation_service.py index 4dd75757b..5199da998 100644 --- a/tests/unit/services/test_household_calculation_service.py +++ b/tests/unit/services/test_household_calculation_service.py @@ -121,6 +121,7 @@ def calculate(self, household, policy, **_kwargs): return CalculationResult( household=household, warnings=("employment_income could not be calculated",), + **household_result_fields(years=["2026"]), ) service = HouseholdCalculationService( @@ -146,10 +147,13 @@ class Country: def __init__(self): self.prepare = Mock(return_value=parsed_calculation) self.calculate = Mock( - return_value=CalculationResult(household={"people": {}}) + return_value=CalculationResult( + household={"people": {}}, + **household_result_fields(years=[]), + ) ) - def prepare_calculation(self, household, policy): + def prepare_calculation(self, household, policy, **_kwargs): return self.prepare(household, policy) country = Country() @@ -170,6 +174,8 @@ def prepare_calculation(self, household, policy): country.calculate.assert_called_once_with( {"people": {}}, {}, + spm=INSTALLED_SPM_SELECTION, + spm_requested=False, prepared=parsed_calculation, ) assert result.household == {"people": {}} @@ -185,7 +191,7 @@ class Country: "parameters": {}, } - def prepare_calculation(self, household, policy): + def prepare_calculation(self, household, policy, **_kwargs): return object() service = HouseholdCalculationService( @@ -213,7 +219,7 @@ class Country: "parameters": {}, } - def prepare_calculation(self, household, policy): + def prepare_calculation(self, household, policy, **_kwargs): raise RuntimeError("invalid situation") service = HouseholdCalculationService( diff --git a/uv.lock b/uv.lock index 57a663cc6..c5ecf2412 100644 --- a/uv.lock +++ b/uv.lock @@ -1,12 +1,12 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", - "python_full_version == '3.14.*' and sys_platform == 'win32'", "python_full_version >= '3.15' and sys_platform == 'emscripten'", - "python_full_version == '3.14.*' and sys_platform == 'emscripten'", "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", @@ -2770,7 +2770,7 @@ models = [ [[package]] name = "policyengine-api" -version = "4.2.2" +version = "4.2.5" source = { editable = "." } dependencies = [ { name = "a2wsgi" }, From e419a93b4ae8ec6ea67c0d8bd6d91419409a2690 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:43:37 +0400 Subject: [PATCH 21/22] Restore simulation bundle prerequisite --- .github/workflows/pr.yml | 7 ++++++- tests/unit/test_cloud_run_deploy_scripts.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index e7b5292e7..b2bfb28ed 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -94,6 +94,7 @@ jobs: test_container_builds: name: Docker runs-on: ubuntu-latest + needs: ensure-policyengine-bundle-supported-by-simulation-api permissions: contents: read packages: write @@ -111,6 +112,7 @@ jobs: test_cloud_run_container_builds: name: Cloud Run container runs-on: ubuntu-latest + needs: ensure-policyengine-bundle-supported-by-simulation-api permissions: contents: read steps: @@ -121,6 +123,7 @@ jobs: test_env_vars: name: Test environment variables runs-on: ubuntu-latest + needs: ensure-policyengine-bundle-supported-by-simulation-api permissions: contents: read id-token: write @@ -151,7 +154,9 @@ jobs: test: name: Test runs-on: ubuntu-latest - needs: test_env_vars + needs: + - ensure-policyengine-bundle-supported-by-simulation-api + - test_env_vars permissions: contents: read id-token: write diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index 9860d1d2d..dd6842606 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -2010,6 +2010,20 @@ def test_workflows_scope_simulation_routing_config_to_github_environments(): assert secret_env in job +def test_pr_build_and_test_jobs_require_simulation_bundle_compatibility(): + workflow = _pr_workflow() + compatibility_job = "ensure-policyengine-bundle-supported-by-simulation-api" + + for job_name in ( + "test_container_builds", + "test_cloud_run_container_builds", + "test_env_vars", + "test", + ): + job = _workflow_job_block(workflow, job_name) + assert compatibility_job in job + + def test_cloud_run_candidate_jobs_use_environment_scoped_migration_selectors(): workflow = _push_workflow() selectors = ( From c04d6a58f293953d306551cebc1d8a5de32ec4a4 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:06:39 +0400 Subject: [PATCH 22/22] Explain legacy SPM result recalculation --- policyengine_api/services/simulation_service.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/policyengine_api/services/simulation_service.py b/policyengine_api/services/simulation_service.py index 943b0fb00..3ef805673 100644 --- a/policyengine_api/services/simulation_service.py +++ b/policyengine_api/services/simulation_service.py @@ -55,6 +55,14 @@ def _latest_successful_run_id(runs: list[SimulationRun]) -> str | None: @staticmethod def _has_stale_spm_output(output: object) -> bool: + """Identify stored results that predate the compact SPM receipt contract. + + Legacy SPM output cannot be translated safely: it may omit the exact + forecast or runtime identity now required to verify a result. Treating + it as stale makes the existing simulation pending and clears its output + so the calculation is run again with the installed certified bundle. + Output without any SPM fields remains valid non-SPM output. + """ if not isinstance(output, dict): return False if "spm_config" in output: