Upgrade PolicyEngine.py, compact SPM provenance, and deploy UK credentials

.env.example

@@ -17,7 +17,9 @@ POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN=policyengine_github_token
 OPENAI_API_KEY=policyengine_openai_api_key
 
 # Token for Hugging Face models 
-HUGGING_FACE_TOKEN=policyengine_huggingface_token
+# PolicyEngine Core compatibility variable. Managed deployments bind the
+# PE_UK_PRIVATE_HF_READ_TOKEN credential to this process-level name.
+HUGGING_FACE_TOKEN=pe_uk_private_hf_read_token_value
 
 # Redis is required for budget-window economy requests and other API cache
 # paths. Configure local development explicitly as described in README.md;

.github/check-policyengine-bundle-supported.sh

@@ -51,10 +51,16 @@ if [ "$CHECK_ONLY_IF_CHANGED" = "1" ]; then
             || true
     )"
 
-    if [ "$current_version" = "$base_version" ]; then
-        echo "PolicyEngine .py bundle pin is unchanged; skipping simulation API support check."
+    if [ "$current_version" = "$base_version" ] \
+        && git diff --quiet "origin/${BASE_REF}" -- \
+            policyengine_api/spm.py policyengine_api/worker_spm.py \
+            policyengine_api/worker_spm_release.py policyengine_api/constants.py \
+            policyengine_api/country.py \
+            .github/check-policyengine-bundle-supported.sh \
+            .github/request-simulation-model-versions.sh; then
+        echo "Bundle pin and SPM integration are unchanged; skipping simulation API support check."
         exit 0
     fi
 fi
 
-bash "$VERSION_GUARD_SCRIPT" -py "$current_version"
+bash "$VERSION_GUARD_SCRIPT" -py "$current_version" --check-installed-spm

.github/find-api-model-versions.py

@@ -1,38 +1,89 @@
 import argparse
 import os
+import shlex
 import sys
+from importlib.metadata import version as distribution_version
 
+from policyengine.bundle import get_current_bundle
 from policyengine_api.constants import (
     COUNTRY_PACKAGE_VERSIONS,
     POLICYENGINE_CORE_VERSION,
     POLICYENGINE_VERSION,
 )
 
+REQUIRED_PACKAGES = (
+    "policyengine",
+    "policyengine-core",
+    "policyengine-us",
+    "policyengine-uk",
+    "spm-calculator",
+)
+
+
+def _manifest_versions(bundle: dict) -> dict[str, str]:
+    packages = bundle.get("packages")
+    if not isinstance(packages, dict):
+        raise RuntimeError("PolicyEngine bundle manifest has no package mapping.")
+
+    versions = {}
+    for package_name in REQUIRED_PACKAGES:
+        package = packages.get(package_name)
+        if not isinstance(package, dict) or not package.get("version"):
+            raise RuntimeError(
+                f"PolicyEngine bundle manifest has no version for {package_name}."
+            )
+        versions[package_name] = str(package["version"])
+    return versions
+
+
+def _data_release_version(bundle: dict, country_id: str) -> str:
+    data_releases = bundle.get("data_releases")
+    release = data_releases.get(country_id) if isinstance(data_releases, dict) else None
+    if not isinstance(release, dict) or not release.get("version"):
+        raise RuntimeError(
+            f"PolicyEngine bundle manifest has no {country_id.upper()} data release."
+        )
+    return str(release["version"])
+
 
 def find_api_model_versions() -> dict[str, str]:
     """
-    Find the API model versions from the installed PolicyEngine bundle.
+    Find and validate the package and data versions in the installed bundle.
     """
-    us_version = COUNTRY_PACKAGE_VERSIONS.get("us")
-    uk_version = COUNTRY_PACKAGE_VERSIONS.get("uk")
+    bundle = get_current_bundle()
+    if not isinstance(bundle, dict):
+        raise RuntimeError("Installed PolicyEngine bundle manifest is not an object.")
 
-    if not us_version:
-        print("Error: US package version not found.", file=sys.stderr)
-        sys.exit(1)
+    manifest_versions = _manifest_versions(bundle)
+    for package_name, manifest_version in manifest_versions.items():
+        installed_version = distribution_version(package_name)
+        if installed_version != manifest_version:
+            raise RuntimeError(
+                f"Installed {package_name} version {installed_version} does not match "
+                f"bundle manifest version {manifest_version}."
+            )
 
-    if not uk_version:
-        print("Error: UK package version not found.", file=sys.stderr)
-        sys.exit(1)
-
-    if not POLICYENGINE_VERSION:
-        print("Error: PolicyEngine package version not found.", file=sys.stderr)
-        sys.exit(1)
+    expected_constants = {
+        "policyengine": POLICYENGINE_VERSION,
+        "policyengine-core": POLICYENGINE_CORE_VERSION,
+        "policyengine-us": COUNTRY_PACKAGE_VERSIONS.get("us"),
+        "policyengine-uk": COUNTRY_PACKAGE_VERSIONS.get("uk"),
+    }
+    for package_name, constant_version in expected_constants.items():
+        if constant_version != manifest_versions[package_name]:
+            raise RuntimeError(
+                f"API version for {package_name} is {constant_version}, but the bundle "
+                f"manifest specifies {manifest_versions[package_name]}."
+            )
 
     return {
-        "POLICYENGINE_VERSION": POLICYENGINE_VERSION,
-        "POLICYENGINE_CORE_VERSION": POLICYENGINE_CORE_VERSION,
-        "US_VERSION": us_version,
-        "UK_VERSION": uk_version,
+        "POLICYENGINE_VERSION": manifest_versions["policyengine"],
+        "POLICYENGINE_CORE_VERSION": manifest_versions["policyengine-core"],
+        "US_VERSION": manifest_versions["policyengine-us"],
+        "UK_VERSION": manifest_versions["policyengine-uk"],
+        "SPM_CALCULATOR_VERSION": manifest_versions["spm-calculator"],
+        "US_DATA_VERSION": _data_release_version(bundle, "us"),
+        "UK_DATA_VERSION": _data_release_version(bundle, "uk"),
     }
 
 
@@ -57,7 +108,7 @@ def find_api_model_versions_and_output_to_github():
 
     if args.shell:
         for key, value in find_api_model_versions().items():
-            print(f"{key}={value}")
+            print(f"{key}={shlex.quote(value)}")
     else:
         find_api_model_versions_and_output_to_github()
         print("API model versions found and written to GitHub environment.")

.github/request-simulation-model-versions.sh

@@ -23,12 +23,14 @@ usage() {
     echo "Optional compatibility checks:"
     echo "  -us  us_version  Expected bundled policyengine-us version"
     echo "  -uk  uk_version  Expected bundled policyengine-uk version"
+    echo "  --check-installed-spm  Compare the installed bundle with worker capability"
     exit 1
 }
 
 POLICYENGINE_VERSION=""
 US_VERSION=""
 UK_VERSION=""
+CHECK_INSTALLED_SPM=0
 
 while [ $# -gt 0 ]; do
     case "$1" in
@@ -44,6 +46,10 @@ while [ $# -gt 0 ]; do
             UK_VERSION="$2"
             shift 2
             ;;
+        --check-installed-spm)
+            CHECK_INSTALLED_SPM=1
+            shift
+            ;;
         -h|--help)
             usage
             ;;
@@ -70,7 +76,7 @@ if [ -n "$UK_VERSION" ]; then
 fi
 echo ""
 
-VERSIONS_RESPONSE=$(curl -s "${GATEWAY_URL}/versions")
+VERSIONS_RESPONSE=$(curl --fail --silent --show-error --connect-timeout 10 --max-time 60 "${GATEWAY_URL}/versions")
 
 if [ -z "$VERSIONS_RESPONSE" ]; then
     echo "ERROR: Failed to fetch versions from gateway"
@@ -114,6 +120,11 @@ check_country_route() {
 check_country_route "us" "$US_VERSION"
 check_country_route "uk" "$UK_VERSION"
 
+if [ "$CHECK_INSTALLED_SPM" = "1" ]; then
+    printf '%s' "$VERSIONS_RESPONSE" \
+        | uv run --frozen python -m policyengine_api.worker_spm_release "$POLICYENGINE_VERSION"
+fi
+
 echo ""
 echo "SUCCESS: PolicyEngine bundle route is deployed and ready"
 exit 0

.github/scripts/cloud_run_env.sh

@@ -39,7 +39,7 @@ cloud_run_set_defaults() {
   CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-}"
   CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}"
   CLOUD_RUN_OPENAI_API_KEY_SECRET="${CLOUD_RUN_OPENAI_API_KEY_SECRET:-policyengine-api-prod-openai-api-key:latest}"
-  CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET="${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET:-policyengine-api-prod-hugging-face-token:latest}"
+  CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET:-pe-uk-private-hf-read-token:latest}"
   CLOUD_RUN_RUNTIME_CACHE_URL_SECRET="${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET:-policyengine-api-prod-runtime-cache-url:latest}"
   CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET="${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET:-policyengine-api-prod-runtime-cache-ca:latest}"
   CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT="${CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT:-production}"
@@ -76,7 +76,7 @@ cloud_run_set_defaults() {
   export CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET
   export CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET
   export CLOUD_RUN_OPENAI_API_KEY_SECRET
-  export CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET
+  export CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET
   export CLOUD_RUN_RUNTIME_CACHE_URL_SECRET
   export CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET
   export CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT

.github/scripts/deploy_cloud_run_candidate.sh

@@ -62,7 +62,9 @@ secret_vars=(
   "POLICYENGINE_DB_PASSWORD=${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}"
   "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN=${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET}"
   "OPENAI_API_KEY=${CLOUD_RUN_OPENAI_API_KEY_SECRET}"
-  "HUGGING_FACE_TOKEN=${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET}"
+  # PolicyEngine Core reads HUGGING_FACE_TOKEN. The stored credential and every
+  # deployment input use the purpose-specific PE_UK_PRIVATE_HF_READ_TOKEN name.
+  "HUGGING_FACE_TOKEN=${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET}"
   "RUNTIME_CACHE_URL=${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET}"
   "RUNTIME_CACHE_CA_CERT=${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET}"
 )

.github/scripts/resolve_cloud_run_candidate_state.sh

@@ -78,6 +78,32 @@ image="$(jq -er '
   | select(type == "string" and contains("@sha256:"))
 ' <<<"${revision_json}")"
 
+expected_hf_secret_resource="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET%:*}"
+expected_hf_secret_name="${expected_hf_secret_resource##*/}"
+expected_hf_secret_version="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET##*:}"
+actual_hf_secret_binding="$(jq -cer '
+  [
+    .spec.containers[0].env[]?
+    | select(.name == "HUGGING_FACE_TOKEN")
+    | {
+        name: .valueFrom.secretKeyRef.name,
+        version: .valueFrom.secretKeyRef.key
+      }
+  ]
+  | if length == 1 then .[0]
+    else error("HUGGING_FACE_TOKEN must have exactly one secret binding")
+    end
+' <<<"${revision_json}")"
+actual_hf_secret_name="$(jq -er '.name' <<<"${actual_hf_secret_binding}")"
+actual_hf_secret_version="$(jq -er '.version' <<<"${actual_hf_secret_binding}")"
+if [[ "${actual_hf_secret_name}" != "${expected_hf_secret_name}" \
+  || "${actual_hf_secret_version}" != "${expected_hf_secret_version}" ]]; then
+  printf 'Revision %s binds HUGGING_FACE_TOKEN to %s:%s; expected %s:%s\n' \
+    "${revision}" "${actual_hf_secret_name}" "${actual_hf_secret_version}" \
+    "${expected_hf_secret_name}" "${expected_hf_secret_version}" >&2
+  exit 2
+fi
+
 deployment_selector_count=0
 for selector in \
   ROUTE_IMPL_HEALTH \