Enable ruff security, logging and debugging rules (S, T20, LOG, G, DTZ, ERA, TD, FIX, EXE)

.github/scripts/update_policyengine_package.py

@@ -32,6 +32,11 @@ def _json_from_stdin() -> object:
     return json.load(sys.stdin)
 
 
+def _write_result(result: object) -> None:
+    """Print one result line for update-policyengine-package.sh to capture."""
+    print(result)  # noqa: T201  # command-line script: the shell script reads stdout
+
+
 def _requirement_name(requirement: str) -> str:
     return re.split(r"[\s\[<>=!~;@]", requirement, maxsplit=1)[0]
 
@@ -155,7 +160,7 @@ def find_issue(title: str) -> None:
         and isinstance(item.get("number"), int)
     )
     if matches:
-        print(matches[0])
+        _write_result(matches[0])
 
 
 def verify_issue(expected_number: int, expected_title: str) -> None:
@@ -191,7 +196,7 @@ def latest_version(package: str) -> None:
     if not isinstance(version, str) or not version:
         msg = f"PyPI response has no latest version for {package}"
         raise SystemExit(msg)
-    print(version)
+    _write_result(version)
 
 
 def bundle_versions() -> None:
@@ -264,7 +269,7 @@ def certificate_sha256(country: str) -> str:
         "uk_certificate_sha256": certificate_sha256("uk"),
     }
     for name, version in outputs.items():
-        print(f"{name}={version}")
+        _write_result(f"{name}={version}")
 
 
 #: Release-dispatch fields compared with the installed bundle: (payload key
@@ -353,9 +358,9 @@ def verify_dispatch() -> None:
     ):
         msg = f"The release dispatch says the UK pair was not promotable ({verdict})"
         raise SystemExit(msg)
-    print(f"dispatch_verified={'true' if compared else 'false'}")
-    print(f"uk_qualification_verdict={verdict}")
-    print(f"uk_qualification_receipt_sha256={receipt_sha256}")
+    _write_result(f"dispatch_verified={'true' if compared else 'false'}")
+    _write_result(f"uk_qualification_verdict={verdict}")
+    _write_result(f"uk_qualification_receipt_sha256={receipt_sha256}")
 
 
 def _parser() -> argparse.ArgumentParser:
@@ -398,7 +403,7 @@ def main() -> None:
     """
     args = _parser().parse_args()
     if args.command == "current-version":
-        print(current_version(args.pyproject, args.package))
+        _write_result(current_version(args.pyproject, args.package))
     elif args.command == "update-requirements":
         update_requirements(args.pyproject, args.package, args.current, args.latest)
     elif args.command == "find-issue":

AGENTS.md

@@ -135,10 +135,11 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`.
 
 ### Code quality and tests
 
+- `sim.quality.deployed-tests-ruff-exemptions`: The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (S101, D1, ANN, PLR2004); every other ruff rule applies to it. (blocking: ruff per-file ignore of ANN, D1, PLR2004, S101 for `**/authenticated_tests/**`)
 - `sim.quality.every-project-tested`: Every project and library with unit tests runs them in the PR test matrix. (doc-only, not checked: follow it yourself)
 - `sim.quality.ruff-format-src`: Source code under projects/*/src and libs/*/src is formatted with ruff format. (blocking: ruff format)
 - `sim.quality.ruff-format-tests`: Test code is formatted with ruff format too; CI currently checks only src directories. (doc-only, not checked: follow it yourself)
-- `sim.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (ANN, BLE, D, E4, E7, E9, EM, F, FLY, FURB, I, PIE, PYI, RET, RSE, TC, TRY, UP, W))
+- `sim.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W))
 - `sim.quality.tests-pass`: Each project's unit tests pass in its own locked environment. (blocking: CI job `test`)
 - `sim.quality.tests-ruff-exemptions`: Test files may use pytest's bare assert and need no docstrings, annotations or named constants for magic numbers (S101, D1, ANN, PLR2004); every other ruff rule applies to tests. (blocking: ruff per-file ignore of ANN, D1, PLR2004, S101 for `**/tests/**`)
 

changelog_entry.yaml

@@ -4,6 +4,7 @@
       - Made CLAUDE.md and the Copilot instructions thin pointers to AGENTS.md, required workflow logic longer than one line to live in tested and shellchecked scripts, and registered the tracking-issue rule
       - Added a rules registry (rules/registry.toml) with a standard-library tool that validates it, checks every rule is enforced where it claims, and renders docs/generated/rules.md and the Rules block in AGENTS.md
     changed:
+      - Enabled ruff's security, logging and debugging rules (S, T20, LOG, G, DTZ, ERA, TD, FIX, EXE); asserts outside tests became explicit checks, command-line scripts keep their printed output, test subprocess calls go through helpers, and every deliberate exception carries a reasoned suppression; the gateway's auth misconfiguration log no longer repeats its message, and the deployed-service tests share the unit-test ruff exemptions
       - Enabled ruff's exception rules (TRY, EM, BLE); exception messages are built before the raise, raises inside try blocks moved into helpers, and every deliberate catch-all or traceback-free error log carries a reasoned suppression; message text is unchanged, and error types change only in an operator script's two type checks
       - Enabled ruff's docstring rules (D) and documented every module, class and function outside tests; published contract models and routes now carry these descriptions in their JSON schema and OpenAPI, with no structural schema change
       - Enabled ruff's annotation rules (ANN) and annotated every function outside tests; test files are exempt from ANN, missing docstrings, magic numbers and bare-assert rules

docs/generated/rules.md

@@ -8,11 +8,11 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf
 
 | Tier | enforced | ratcheted | advisory | doc-only | Total |
 | --- | ---: | ---: | ---: | ---: | ---: |
-| mechanical | 33 | 1 | 0 | 14 | 48 |
+| mechanical | 34 | 1 | 0 | 14 | 49 |
 | metric | 0 | 0 | 0 | 0 | 0 |
 | judgment | 0 | 0 | 0 | 0 | 0 |
 
-**Blocking:** 34 of 48 rules.
+**Blocking:** 35 of 49 rules.
 
 ## Gaps
 
@@ -242,34 +242,39 @@ flowchart LR
 
 | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof |
 | --- | --- | --- | --- | --- | --- | --- | --- |
+| `sim.quality.deployed-tests-ruff-exemptions` | The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (S101, D1, ANN, PLR2004); every other ruff rule applies to it. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, PLR2004, S101 for `**/authenticated_tests/**` | `pr.yml:lint` | — |
 | `sim.quality.every-project-tested` | Every project and library with unit tests runs them in the PR test matrix. | mechanical | doc-only | no | nothing | — | — |
 | `sim.quality.ruff-format-src` | Source code under projects/*/src and libs/*/src is formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — |
 | `sim.quality.ruff-format-tests` | Test code is formatted with ruff format too; CI currently checks only src directories. | mechanical | doc-only | no | nothing | — | — |
-| `sim.quality.ruff-lint` | All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. | mechanical | enforced | yes | ruff check (ANN, BLE, D, E4, E7, E9, EM, F, FLY, FURB, I, PIE, PYI, RET, RSE, TC, TRY, UP, W) | `pr.yml:lint` | — |
+| `sim.quality.ruff-lint` | All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. | mechanical | enforced | yes | ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W) | `pr.yml:lint` | — |
 | `sim.quality.tests-pass` | Each project's unit tests pass in its own locked environment. | mechanical | enforced | yes | CI job `test` | `pr.yml:test` | — |
 | `sim.quality.tests-ruff-exemptions` | Test files may use pytest's bare assert and need no docstrings, annotations or named constants for magic numbers (S101, D1, ANN, PLR2004); every other ruff rule applies to tests. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, PLR2004, S101 for `**/tests/**` | `pr.yml:lint` | — |
 
 ```mermaid
 flowchart LR
+  r_sim_quality_deployed_tests_ruff_exemptions("sim.quality.deployed-tests-ruff-exemptions (enforced)")
+  e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____authenticated_tests___["ruff per-file ignore of ANN, D1, PLR2004, S101 for **/authenticated_tests/**"]
+  j_CI_pr_yml_lint["CI pr.yml:lint"]
   r_sim_quality_every_project_tested("sim.quality.every-project-tested (doc-only)")
   gap_unenforced>"not enforced"]
   r_sim_quality_ruff_format_src("sim.quality.ruff-format-src (enforced)")
   e_ruff_format["ruff format"]
-  j_CI_pr_yml_lint["CI pr.yml:lint"]
   r_sim_quality_ruff_format_tests("sim.quality.ruff-format-tests (doc-only)")
   r_sim_quality_ruff_lint("sim.quality.ruff-lint (enforced)")
-  e_ruff_check__ANN__BLE__D__E4__E7__E9__EM__F__FLY__FURB__I__PIE__PYI__RET__RSE__TC__TRY__UP__W_["ruff check (ANN, BLE, D, E4, E7, E9, EM, F, FLY, FURB, I, PIE, PYI, RET, RSE, TC, TRY, UP, W)"]
+  e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_["ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W)"]
   r_sim_quality_tests_pass("sim.quality.tests-pass (enforced)")
   e_CI_job_test["CI job test"]
   j_CI_pr_yml_test["CI pr.yml:test"]
   r_sim_quality_tests_ruff_exemptions("sim.quality.tests-ruff-exemptions (enforced)")
   e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____tests___["ruff per-file ignore of ANN, D1, PLR2004, S101 for **/tests/**"]
+  r_sim_quality_deployed_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____authenticated_tests___
+  e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____authenticated_tests___ --> j_CI_pr_yml_lint
   r_sim_quality_every_project_tested -.-> gap_unenforced
   r_sim_quality_ruff_format_src --> e_ruff_format
   e_ruff_format --> j_CI_pr_yml_lint
   r_sim_quality_ruff_format_tests -.-> gap_unenforced
-  r_sim_quality_ruff_lint --> e_ruff_check__ANN__BLE__D__E4__E7__E9__EM__F__FLY__FURB__I__PIE__PYI__RET__RSE__TC__TRY__UP__W_
-  e_ruff_check__ANN__BLE__D__E4__E7__E9__EM__F__FLY__FURB__I__PIE__PYI__RET__RSE__TC__TRY__UP__W_ --> j_CI_pr_yml_lint
+  r_sim_quality_ruff_lint --> e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_
+  e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_ --> j_CI_pr_yml_lint
   r_sim_quality_tests_pass --> e_CI_job_test
   e_CI_job_test --> j_CI_pr_yml_test
   r_sim_quality_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____tests___

libs/policyengine-fastapi/src/policyengine_fastapi/auth/jwt_decoder.py

@@ -63,10 +63,10 @@ def _validate(self, token: HTTPAuthorizationCredentials | None) -> dict[str, str
                 token.credentials
             ).key
         except jwt.exceptions.PyJWKClientError as error:
-            LOG.error(f"Unable to get public key for JWT validation: {error}")  # noqa: TRY400  # callers control the token, so a traceback per bad token would let them flood the logs
+            LOG.error("Unable to get public key for JWT validation: %s", error)  # noqa: TRY400  # callers control the token, so a traceback per bad token would let them flood the logs
             raise HTTPException(status_code=status.HTTP_403_FORBIDDEN)
         except jwt.exceptions.DecodeError as error:
-            LOG.error(f"Unable to get public key for JWT validation: {error}")  # noqa: TRY400  # callers control the token, so a traceback per bad token would let them flood the logs
+            LOG.error("Unable to get public key for JWT validation: %s", error)  # noqa: TRY400  # callers control the token, so a traceback per bad token would let them flood the logs
             raise HTTPException(status_code=status.HTTP_403_FORBIDDEN)
 
         try:
@@ -78,6 +78,6 @@ def _validate(self, token: HTTPAuthorizationCredentials | None) -> dict[str, str
                 issuer=self.issuer,
             )
         except Exception as error:  # noqa: BLE001  # any decode or claim-validation failure must become a 403, never a 500
-            LOG.info(f"Invalid bearer token: {error}")
+            LOG.info("Invalid bearer token: %s", error)
             raise HTTPException(status_code=status.HTTP_403_FORBIDDEN)
         return payload

libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py

@@ -51,14 +51,21 @@ def _get_project_id() -> str:
 def _get_project_id_from_metadata() -> str | None:
     from urllib import request
 
-    url = "http://metadata.google.internal/computeMetadata/v1/project/project-id"
-    req = request.Request(url)
-    req.add_header("Metadata-Flavor", "Google")
     try:
-        return request.urlopen(req).read().decode()
+        return (
+            request.urlopen(
+                request.Request(
+                    "http://metadata.google.internal/computeMetadata/v1/project/project-id",
+                    headers={"Metadata-Flavor": "Google"},
+                )
+            )
+            .read()
+            .decode()
+        )
     except Exception as err:  # noqa: BLE001  # best-effort probe: any failure falls back to GOOGLE_CLOUD_PROJECT
         log.info(
-            f"Unable to get the google project id from a local metadata service: {err}"
+            "Unable to get the google project id from a local metadata service: %s",
+            err,
         )
     return None
 

libs/policyengine-simulation-contract/src/policyengine_simulation_contract/hf_dataset.py

@@ -83,9 +83,9 @@ def _fetch_hf_dataset_revision(
     if token is not None:
         headers["Authorization"] = f"Bearer {token}"
 
-    request = Request(url, headers=headers)
+    request = Request(url, headers=headers)  # noqa: S310  # Hub API URL: the HF_ENDPOINT base (https by default) and quoted path parts
     try:
-        with urlopen(request, timeout=HF_REQUEST_TIMEOUT_SECONDS) as response:
+        with urlopen(request, timeout=HF_REQUEST_TIMEOUT_SECONDS) as response:  # noqa: S310  # the Hub API request built above
             return json.loads(response.read().decode("utf-8"))
     except HTTPError as exc:
         detail = exc.reason or f"HTTP {exc.code}"

libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py

@@ -229,7 +229,8 @@ class SPMRuntimeVersions(BaseModel):
     spm_calculator: str = Field(min_length=1, alias="spm-calculator")
 
 
-def _require_complete_selection(selection: SPMSelection) -> None:
+def _require_complete_selection(selection: SPMSelection) -> tuple[str, str]:
+    """Return a complete selection's artifact hash and scenario, else raise ValueError."""
     required = set(SPMSelection.model_fields)
     missing = required.difference(selection.model_fields_set)
     if missing:
@@ -242,18 +243,17 @@ def _require_complete_selection(selection: SPMSelection) -> None:
     if selection.scenario is None:
         msg = "Resolved SPM selection has no scenario"
         raise ValueError(msg)
+    return selection.forecast_content_sha256, selection.scenario
 
 
 def _resolved_spm_configuration(
     selection: SPMSelection,
 ) -> SPMResolvedConfiguration:
     """Convert a resolved request selection to the completed-result contract."""
-    _require_complete_selection(selection)
-    assert selection.forecast_content_sha256 is not None
-    assert selection.scenario is not None
+    forecast_content_sha256, scenario = _require_complete_selection(selection)
     return SPMResolvedConfiguration(
-        forecast_content_sha256=selection.forecast_content_sha256,
-        scenario=selection.scenario,
+        forecast_content_sha256=forecast_content_sha256,
+        scenario=scenario,
         geography_kind=selection.geography_kind,
         geography_id=selection.geography_id,
         county_vintage=selection.county_vintage,
@@ -284,17 +284,15 @@ def build_spm_provenance(
     runtime_versions: SPMRuntimeVersions,
 ) -> SPMProvenance:
     """Build one compact receipt from resolved calculation inputs."""
-    _require_complete_selection(selection)
-    assert selection.forecast_content_sha256 is not None
-    assert selection.scenario is not None
-    if forecast_sha256 != selection.forecast_content_sha256:
+    forecast_content_sha256, scenario = _require_complete_selection(selection)
+    if forecast_sha256 != forecast_content_sha256:
         msg = "SPM receipt artifact hash differs from the selection"
         raise ValueError(msg)
     return SPMProvenance(
         schema_version="canonical-spm-provenance-v2",
         forecast_id=forecast_id,
         forecast_sha256=forecast_sha256,
-        scenario=selection.scenario,
+        scenario=scenario,
         geography_kind=selection.geography_kind,
         geography_id=selection.geography_id,
         county_vintage=selection.county_vintage,

libs/policyengine-simulation-contract/tests/test_hf_dataset.py

@@ -13,6 +13,8 @@
     with_hf_revision,
 )
 
+RUNTIME_TOKEN = "runtime-token"  # noqa: S105  # fake token; the Hub fetch is stubbed
+
 
 class _FakeResponse:
     def __init__(self, payload: dict):
@@ -75,12 +77,12 @@ def fake_fetch(repo_id, revision, token):
         return {"siblings": [{"rfilename": "dataset.h5"}]}
 
     monkeypatch.setattr(hf_dataset, "_fetch_hf_dataset_revision", fake_fetch)
-    monkeypatch.setenv("HUGGING_FACE_TOKEN", "runtime-token")
+    monkeypatch.setenv("HUGGING_FACE_TOKEN", RUNTIME_TOKEN)
     monkeypatch.setenv("PE_UK_PRIVATE_HF_READ_TOKEN", "deployment-token")
     monkeypatch.setenv("HF_TOKEN", "legacy-token")
 
     assert validate_hf_dataset_uri(dataset_uri) == dataset_uri
-    assert seen["token"] == "runtime-token"
+    assert seen["token"] == RUNTIME_TOKEN
 
 
 def test_validate_hf_dataset_uri_ignores_non_runtime_credential_names(monkeypatch):

libs/policyengine-simulation-observability/src/policyengine_simulation_observability/errors.py

@@ -91,7 +91,7 @@ def log_and_redact_exception(
             "gateway_error_redacted",
             attributes=payload,
         )
-    except Exception:  # noqa: BLE001  # pragma: no cover - defensive, never raise from logger
+    except Exception:  # noqa: BLE001, S110  # pragma: no cover - defensive, never raise from logger
         pass
 
     return f"{GENERIC_JOB_FAILURE_MESSAGE} (correlation_id={correlation_id})"

projects/policyengine-simulation-entry/src/policyengine_simulation_entry/generate_openapi.py

@@ -20,7 +20,7 @@ def main() -> None:
         json.dumps(create_app().openapi(), indent=2) + "\n",
         encoding="utf-8",
     )
-    print(f"OpenAPI spec written to {output}")
+    print(f"OpenAPI spec written to {output}")  # noqa: T201  # command-line script: reports where it wrote the spec
 
 
 if __name__ == "__main__":

projects/policyengine-simulation-entry/src/policyengine_simulation_entry/stage12_infrastructure.py

@@ -36,7 +36,7 @@ def main() -> None:
         )
     except (Stage12RuntimeAccessError, ValueError) as error:
         raise SystemExit(str(error)) from None
-    print(
+    print(  # noqa: T201  # command-line check: stdout reports the verified access
         "Shared API v2 runtime database access verified for Stage 12: "
         f"environment={args.environment}, role={args.expected_role}"
     )