From 794ff98e8fbe3eb73f614f8dde2591f16cae05b6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:58:10 +0400 Subject: [PATCH 01/23] Exempt tests from SLF001, INP001 and PLC0415 Tests deliberately reach into private members, pytest test directories aren't packages, and tests often import inside a test after patching the environment. Application code still gets every rule. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 ++-- docs/generated/rules.md | 16 ++++++++-------- ruff.toml | 12 +++++++----- rules/registry.toml | 6 +++--- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 15ba224be..a5bce2ab8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -142,7 +142,7 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. - `sim.quality.ci-cd-scripts-ruff-exemptions`: CI/CD scripts under .github may print their output (T201); every other ruff rule applies to them. (blocking: ruff per-file ignore of T201 for `.github/**`) - `sim.quality.command-line-scripts-ruff-exemptions`: Command-line scripts under a scripts directory may print their output (T201), run other programs (S603, S607) and use today's date (DTZ011); every other ruff rule applies to them. (blocking: ruff per-file ignore of DTZ011, S603, S607, T201 for `**/scripts/**`) -- `sim.quality.deployed-tests-ruff-exemptions`: The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (sim.quality.tests-ruff-exemptions); every other ruff rule applies to it. (blocking: ruff per-file ignore of ANN, D1, DTZ001, DTZ007, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607 for `**/authenticated_tests/**`) +- `sim.quality.deployed-tests-ruff-exemptions`: The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (sim.quality.tests-ruff-exemptions); every other ruff rule applies to it. (blocking: ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001 for `**/authenticated_tests/**`) - `sim.quality.every-project-tested`: Every project and library with unit tests runs them in the PR test matrix. (doc-only, not checked: follow it yourself) - `sim.quality.ruff-format-src`: Source code under projects/*/src and libs/*/src is formatted with ruff format. (blocking: ruff format) - `sim.quality.ruff-format-tests`: Test code is formatted with ruff format too; CI currently checks only src directories. (doc-only, not checked: follow it yourself) @@ -150,7 +150,7 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. - `sim.quality.runtime-evaluated-models-resolve`: Every Pydantic model is runtime-evaluated for ruff: one of its bases, read through the imports of the model's own module, is listed in ruff.toml's runtime-evaluated-base-classes or is a runtime-evaluated class in that same module; and every Pydantic model resolves its annotations once its package is imported. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_runtime_evaluated_annotations.py`, `projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py`) - `sim.quality.src-programs-ruff-exemptions`: Programs under src (the Modal deployment utilities, and the OpenAPI and Stage 12 infrastructure commands) may print their output (T201) and open URLs from their arguments (S310); every other ruff rule applies to them. (blocking: ruff per-file ignore of S310, T201 for `projects/*/src/{modal/utils/*,policyengine_simulation_entry/generate_openapi,policyengine_simulation_entry/stage12_infrastructure,policyengine_simulation_gateway/generate_openapi}.py`) - `sim.quality.tests-pass`: Each project's unit tests pass in its own locked environment. (blocking: CI job `test`) -- `sim.quality.tests-ruff-exemptions`: Test files may use pytest's bare assert and need no docstrings, annotations or named constants for magic numbers (S101, D1, ANN, PLR2004). They may also run repository scripts (S603, S607) and use fake credentials (S105, S106), temporary paths (S108), naive datetimes (DTZ001, DTZ007), pickle (S301) and exec (S102). Every other ruff rule applies to tests. (blocking: ruff per-file ignore of ANN, D1, DTZ001, DTZ007, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607 for `**/tests/**`) +- `sim.quality.tests-ruff-exemptions`: Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415). They may also run repository scripts (S603, S607) and use fake credentials (S105, S106), temporary paths (S108), naive datetimes (DTZ001, DTZ007), pickle (S301) and exec (S102). Every other ruff rule applies to tests. (blocking: ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001 for `**/tests/**`) ### CI workflows and scripts diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 3be6b78e9..309c8b4df 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -248,7 +248,7 @@ flowchart LR | --- | --- | --- | --- | --- | --- | --- | --- | | `sim.quality.ci-cd-scripts-ruff-exemptions` | CI/CD scripts under .github may print their output (T201); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of T201 for `.github/**` | `pr.yml:lint` | — | | `sim.quality.command-line-scripts-ruff-exemptions` | Command-line scripts under a scripts directory may print their output (T201), run other programs (S603, S607) and use today's date (DTZ011); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of DTZ011, S603, S607, T201 for `**/scripts/**` | `pr.yml:lint` | — | -| `sim.quality.deployed-tests-ruff-exemptions` | The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (sim.quality.tests-ruff-exemptions); every other ruff rule applies to it. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, DTZ001, DTZ007, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607 for `**/authenticated_tests/**` | `pr.yml:lint` | — | +| `sim.quality.deployed-tests-ruff-exemptions` | The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (sim.quality.tests-ruff-exemptions); every other ruff rule applies to it. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001 for `**/authenticated_tests/**` | `pr.yml:lint` | — | | `sim.quality.every-project-tested` | Every project and library with unit tests runs them in the PR test matrix. | mechanical | doc-only | no | nothing | — | — | | `sim.quality.ruff-format-src` | Source code under projects/*/src and libs/*/src is formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — | | `sim.quality.ruff-format-tests` | Test code is formatted with ruff format too; CI currently checks only src directories. | mechanical | doc-only | no | nothing | — | — | @@ -256,7 +256,7 @@ flowchart LR | `sim.quality.runtime-evaluated-models-resolve` | Every Pydantic model is runtime-evaluated for ruff: one of its bases, read through the imports of the model's own module, is listed in ruff.toml's runtime-evaluated-base-classes or is a runtime-evaluated class in that same module; and every Pydantic model resolves its annotations once its package is imported. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_runtime_evaluated_annotations.py`, `projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py` | `pr.yml:test` | yes | | `sim.quality.src-programs-ruff-exemptions` | Programs under src (the Modal deployment utilities, and the OpenAPI and Stage 12 infrastructure commands) may print their output (T201) and open URLs from their arguments (S310); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of S310, T201 for `projects/*/src/{modal/utils/*,policyengine_simulation_entry/generate_openapi,policyengine_simulation_entry/stage12_infrastructure,policyengine_simulation_gateway/generate_openapi}.py` | `pr.yml:lint` | — | | `sim.quality.tests-pass` | Each project's unit tests pass in its own locked environment. | mechanical | enforced | yes | CI job `test` | `pr.yml:test` | — | -| `sim.quality.tests-ruff-exemptions` | Test files may use pytest's bare assert and need no docstrings, annotations or named constants for magic numbers (S101, D1, ANN, PLR2004). They may also run repository scripts (S603, S607) and use fake credentials (S105, S106), temporary paths (S108), naive datetimes (DTZ001, DTZ007), pickle (S301) and exec (S102). Every other ruff rule applies to tests. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, DTZ001, DTZ007, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607 for `**/tests/**` | `pr.yml:lint` | — | +| `sim.quality.tests-ruff-exemptions` | Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415). They may also run repository scripts (S603, S607) and use fake credentials (S105, S106), temporary paths (S108), naive datetimes (DTZ001, DTZ007), pickle (S301) and exec (S102). Every other ruff rule applies to tests. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001 for `**/tests/**` | `pr.yml:lint` | — | ```mermaid flowchart LR @@ -266,7 +266,7 @@ flowchart LR r_sim_quality_command_line_scripts_ruff_exemptions("sim.quality.command-line-scripts-ruff-exemptions (enforced)") e_ruff_per_file_ignore_of_DTZ011__S603__S607__T201_for____scripts___["ruff per-file ignore of DTZ011, S603, S607, T201 for **/scripts/**"] r_sim_quality_deployed_tests_ruff_exemptions("sim.quality.deployed-tests-ruff-exemptions (enforced)") - e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607_for____authenticated_tests___["ruff per-file ignore of ANN, D1, DTZ001, DTZ007, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607 for **/authenticated_tests/**"] + e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001_for____authenticated_tests___["ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001 for **/authenticated_tests/**"] r_sim_quality_every_project_tested("sim.quality.every-project-tested (doc-only)") gap_unenforced>"not enforced"] r_sim_quality_ruff_format_src("sim.quality.ruff-format-src (enforced)") @@ -282,13 +282,13 @@ flowchart LR r_sim_quality_tests_pass("sim.quality.tests-pass (enforced)") e_CI_job_test["CI job test"] r_sim_quality_tests_ruff_exemptions("sim.quality.tests-ruff-exemptions (enforced)") - e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607_for____tests___["ruff per-file ignore of ANN, D1, DTZ001, DTZ007, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607 for **/tests/**"] + e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001_for____tests___["ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001 for **/tests/**"] r_sim_quality_ci_cd_scripts_ruff_exemptions --> e_ruff_per_file_ignore_of_T201_for__github___ e_ruff_per_file_ignore_of_T201_for__github___ --> j_CI_pr_yml_lint r_sim_quality_command_line_scripts_ruff_exemptions --> e_ruff_per_file_ignore_of_DTZ011__S603__S607__T201_for____scripts___ e_ruff_per_file_ignore_of_DTZ011__S603__S607__T201_for____scripts___ --> j_CI_pr_yml_lint - r_sim_quality_deployed_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607_for____authenticated_tests___ - e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607_for____authenticated_tests___ --> j_CI_pr_yml_lint + r_sim_quality_deployed_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001_for____authenticated_tests___ + e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001_for____authenticated_tests___ --> j_CI_pr_yml_lint r_sim_quality_every_project_tested -.-> gap_unenforced r_sim_quality_ruff_format_src --> e_ruff_format e_ruff_format --> j_CI_pr_yml_lint @@ -301,8 +301,8 @@ flowchart LR e_ruff_per_file_ignore_of_S310__T201_for_projects___src__modal_utils___policyengine_simulation_entry_generate_openapi_policyengine_simulation_entry_stage12_infrastructure_policyengine_simulation_gateway_generate_openapi__py --> j_CI_pr_yml_lint r_sim_quality_tests_pass --> e_CI_job_test e_CI_job_test --> j_CI_pr_yml_test - r_sim_quality_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607_for____tests___ - e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607_for____tests___ --> j_CI_pr_yml_lint + r_sim_quality_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001_for____tests___ + e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001_for____tests___ --> j_CI_pr_yml_lint ``` ## CI workflows and scripts diff --git a/ruff.toml b/ruff.toml index 5afddf284..ec93e8e82 100644 --- a/ruff.toml +++ b/ruff.toml @@ -32,18 +32,20 @@ ignore = ["D203", "D213"] # docs/engineering/skills/rules-registry.md#file-level-exceptions. # # Tests may use pytest's bare assert, and need no docstrings, annotations or -# named constants for magic numbers. They also run repository scripts and use +# named constants for magic numbers. They may also reach into private members, +# live in directories that aren't packages, and import inside a test (often +# after patching the environment). They also run repository scripts and use # fake credentials, temporary paths, naive datetimes, pickle and exec on # purpose. Every other rule applies to tests. "**/tests/**" = [ - "ANN", "D1", "DTZ001", "DTZ007", "PLR2004", - "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607", + "ANN", "D1", "DTZ001", "DTZ007", "INP001", "PLC0415", "PLR2004", + "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607", "SLF001", ] # The deployed-service test suite (pytest, run against a live candidate) gets # the same test exemptions. "**/authenticated_tests/**" = [ - "ANN", "D1", "DTZ001", "DTZ007", "PLR2004", - "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607", + "ANN", "D1", "DTZ001", "DTZ007", "INP001", "PLC0415", "PLR2004", + "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607", "SLF001", ] # CI/CD scripts report on stdout. ".github/**" = ["T201"] diff --git a/rules/registry.toml b/rules/registry.toml index 99fa344c5..2d88223d7 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -742,7 +742,7 @@ invocation = "ruff check ." [[rule]] id = "sim.quality.tests-ruff-exemptions" category = "quality" -statement = "Test files may use pytest's bare assert and need no docstrings, annotations or named constants for magic numbers (S101, D1, ANN, PLR2004). They may also run repository scripts (S603, S607) and use fake credentials (S105, S106), temporary paths (S108), naive datetimes (DTZ001, DTZ007), pickle (S301) and exec (S102). Every other ruff rule applies to tests." +statement = "Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415). They may also run repository scripts (S603, S607) and use fake credentials (S105, S106), temporary paths (S108), naive datetimes (DTZ001, DTZ007), pickle (S301) and exec (S102). Every other ruff rule applies to tests." rationale = "docs/engineering/skills/rules-registry.md#file-level-exceptions" scope = ["projects/*/tests/**/*.py", "libs/*/tests/**/*.py", "scripts/rules/tests/**/*.py"] tier = "mechanical" @@ -752,7 +752,7 @@ autofix = false kind = "ruff" mode = "per-file-ignore" pattern = "**/tests/**" -codes = ["ANN", "D1", "DTZ001", "DTZ007", "PLR2004", "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607"] +codes = ["ANN", "D1", "DTZ001", "DTZ007", "INP001", "PLC0415", "PLR2004", "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607", "SLF001"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "lint" @@ -771,7 +771,7 @@ autofix = false kind = "ruff" mode = "per-file-ignore" pattern = "**/authenticated_tests/**" -codes = ["ANN", "D1", "DTZ001", "DTZ007", "PLR2004", "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607"] +codes = ["ANN", "D1", "DTZ001", "DTZ007", "INP001", "PLC0415", "PLR2004", "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607", "SLF001"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "lint" From 96d04f4f0e48d71120fdcd8f9778cedc24be7447 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:01:49 +0400 Subject: [PATCH 02/23] Drop response_model arguments that repeat the return type (ruff FAST001) FastAPI takes the response model from the return annotation when response_model is omitted, so the generated OpenAPI documents are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../src/policyengine_fastapi/ping/__init__.py | 2 +- .../src/policyengine_simulation_gateway/endpoints.py | 8 ++++---- .../generate_openapi.py | 12 ++++-------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/libs/policyengine-fastapi/src/policyengine_fastapi/ping/__init__.py b/libs/policyengine-fastapi/src/policyengine_fastapi/ping/__init__.py index 867bf0c3c..3534d2ef8 100644 --- a/libs/policyengine-fastapi/src/policyengine_fastapi/ping/__init__.py +++ b/libs/policyengine-fastapi/src/policyengine_fastapi/ping/__init__.py @@ -28,7 +28,7 @@ async def ping(request: PingRequest) -> PingResponse: """Verify the API can receive and process unauthenticated requests.""" return PingResponse(incremented=request.value + 1) - @router.get("/ping/started", response_model=str) + @router.get("/ping/started") async def started() -> str: """Verify the api is running.""" return "alive" diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py index 7919c0dda..eafb5ee61 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py @@ -1270,7 +1270,7 @@ async def get_budget_window_job_status( return batch_status_response(status_response, headers=response_headers) -@router.get("/versions", response_model=VersionsResponse) +@router.get("/versions") async def list_versions(request: Request) -> VersionsResponse: """List all available routing versions.""" runtime = _runtime(request) @@ -1316,7 +1316,7 @@ def _version_map_from_state(state: VersionRoutingState, kind: str) -> VersionMap return VersionMap(versions) -@router.get("/versions/{kind}", response_model=VersionMap) +@router.get("/versions/{kind}") async def get_country_versions(kind: str, request: Request) -> VersionMap: """Get available versions for policyengine, US, or UK routing.""" runtime = _runtime(request) @@ -1342,13 +1342,13 @@ async def get_country_versions(kind: str, request: Request) -> VersionMap: return VersionMap(dict(version_dict)) -@router.get("/health", response_model=HealthResponse) +@router.get("/health") async def health() -> HealthResponse: """Health check endpoint.""" return HealthResponse() -@router.post("/ping", response_model=PingResponse) +@router.post("/ping") async def ping(request: PingRequest) -> PingResponse: """Verify the API can receive and process requests. diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/generate_openapi.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/generate_openapi.py index fea8f2a3b..7590fbf9d 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/generate_openapi.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/generate_openapi.py @@ -37,7 +37,6 @@ def create_openapi_app() -> FastAPI: @app.post( "/simulate/economy/comparison", - response_model=JobSubmitResponse, responses={ 200: {"description": "Job submitted successfully"}, 400: {"description": "Invalid request (unknown country/version)"}, @@ -54,7 +53,6 @@ async def submit_simulation(request: SimulationRequest) -> JobSubmitResponse: @app.post( "/simulate/economy/budget-window", - response_model=BudgetWindowBatchSubmitResponse, responses={ 200: {"description": "Budget-window batch submitted successfully"}, 400: {"description": "Invalid request (unknown country/version/year)"}, @@ -72,7 +70,6 @@ async def submit_budget_window_batch( @app.get( "/jobs/{job_id}", - response_model=JobStatusResponse, responses={ 200: {"description": "Job complete", "model": JobStatusResponse}, 202: {"description": "Job still running"}, @@ -95,7 +92,6 @@ async def get_job_status(job_id: str) -> JobStatusResponse: @app.get( "/budget-window-jobs/{batch_job_id}", - response_model=BudgetWindowBatchStatusResponse, responses={ 200: { "description": "Batch complete", @@ -113,24 +109,24 @@ async def get_budget_window_job_status( msg = "Stub for OpenAPI generation" raise NotImplementedError(msg) - @app.get("/versions", response_model=VersionsResponse) + @app.get("/versions") async def list_versions() -> VersionsResponse: """List all available routing versions.""" msg = "Stub for OpenAPI generation" raise NotImplementedError(msg) - @app.get("/versions/{kind}", response_model=VersionMap) + @app.get("/versions/{kind}") async def get_country_versions(kind: str) -> VersionMap: """Get available versions for policyengine, US, or UK routing.""" msg = "Stub for OpenAPI generation" raise NotImplementedError(msg) - @app.get("/health", response_model=HealthResponse) + @app.get("/health") async def health() -> HealthResponse: """Health check endpoint.""" return HealthResponse() - @app.post("/ping", response_model=PingResponse) + @app.post("/ping") async def ping(request: PingRequest) -> PingResponse: """Verify the API is able to receive and process requests.""" msg = "Stub for OpenAPI generation" From a2bec1f1ac790ce901424a88de8058a99a3e778c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:06:10 +0400 Subject: [PATCH 03/23] Fix ruff naming and builtin-shadowing findings (N, A, PLC0414, SLF001) - Alias the shared exit registry as app_exit where it is imported, and keep the policyengine_fastapi.exit.exit name that projects import. - Use snake_case for the OpenTelemetry provider and log-handler locals. - Add the Error suffix to exceptions that never leave their project: BackendUnavailableError, BackendTimeoutError, TemporaryStage12UnsupportedRequestError and ParityMismatchError. TemporaryStage12DispatchFailed keeps its name, which the entry app logs as error_type. - Keep the camelCase budget-window fields, which are published JSON, the explicit re-exports in stage12_execution, and the two deliberate private accesses, each with a reason. Co-Authored-By: Claude Opus 5.5 --- .../src/policyengine_fastapi/exit.py | 2 +- .../opentelemetry/console.py | 18 ++++----- .../policyengine_fastapi/opentelemetry/gcp.py | 26 ++++++------- libs/policyengine-fastapi/tests/test_exit.py | 6 +-- .../gateway_models.py | 28 +++++++------- .../stage12_execution.py | 4 +- .../src/policyengine_simulation_entry/app.py | 12 +++--- .../policyengine_simulation_entry/backend.py | 22 +++++------ .../stage12_backend.py | 8 ++-- .../tests/test_app.py | 16 +++++--- .../tests/test_backend.py | 4 +- .../tests/test_stage12_backend.py | 4 +- .../baseline_artifacts.py | 2 +- .../policyengine_simulation_executor/main.py | 4 +- .../simulation_runtime.py | 2 +- .../stage12_parity.py | 38 +++++++++---------- .../stage12_qualification.py | 24 ++++++------ .../tests/test_stage12_parity.py | 10 ++--- .../tests/test_stage12_qualification.py | 8 ++-- 19 files changed, 122 insertions(+), 116 deletions(-) diff --git a/libs/policyengine-fastapi/src/policyengine_fastapi/exit.py b/libs/policyengine-fastapi/src/policyengine_fastapi/exit.py index 88e484d29..aaf978d94 100644 --- a/libs/policyengine-fastapi/src/policyengine_fastapi/exit.py +++ b/libs/policyengine-fastapi/src/policyengine_fastapi/exit.py @@ -47,4 +47,4 @@ async def lifespan(self) -> AsyncIterator[None]: self._exit() -exit = AppExit() +exit = AppExit() # noqa: A001 # public name: projects import policyengine_fastapi.exit.exit diff --git a/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/console.py b/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/console.py index 86a26712e..e8adc1cb3 100644 --- a/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/console.py +++ b/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/console.py @@ -15,7 +15,7 @@ ConsoleSpanExporter, ) -from policyengine_fastapi.exit import exit +from policyengine_fastapi.exit import exit as app_exit # Configure opentelemetry @@ -27,16 +27,16 @@ def export_ot_to_console(resource: Resource) -> None: Installs global tracer and meter providers for ``resource`` and registers an ``exit`` callback that flushes both at shutdown. """ - traceProvider = TracerProvider(resource=resource) + trace_provider = TracerProvider(resource=resource) processor = BatchSpanProcessor(ConsoleSpanExporter()) - traceProvider.add_span_processor(processor) - trace.set_tracer_provider(traceProvider) + trace_provider.add_span_processor(processor) + trace.set_tracer_provider(trace_provider) reader = PeriodicExportingMetricReader(ConsoleMetricExporter()) - meterProvider = MeterProvider(resource=resource, metric_readers=[reader]) - metrics.set_meter_provider(meterProvider) + meter_provider = MeterProvider(resource=resource, metric_readers=[reader]) + metrics.set_meter_provider(meter_provider) - @exit() + @app_exit() def flush_ot() -> None: - traceProvider.force_flush() - meterProvider.force_flush() + trace_provider.force_flush() + meter_provider.force_flush() diff --git a/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py b/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py index fc2b13c21..51e1a0631 100644 --- a/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py +++ b/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py @@ -10,7 +10,7 @@ from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor -from policyengine_fastapi.exit import exit +from policyengine_fastapi.exit import exit as app_exit log = logging.getLogger(__name__) @@ -94,7 +94,7 @@ def instrument(self) -> None: from opentelemetry.instrumentation.logging import LoggingInstrumentor from pythonjsonlogger.json import JsonFormatter - logHandler = logging.StreamHandler() + log_handler = logging.StreamHandler() formatter = JsonFormatter( "%(asctime)s %(levelname)s %(message)s %(otelTraceID)s %(otelSpanID)s %(otelTraceSampled)s", rename_fields={ @@ -107,12 +107,12 @@ def instrument(self) -> None: datefmt="%Y-%m-%dT%H:%M:%SZ", ) - logHandler.setFormatter(formatter) - logHandler.addFilter(AddGcpProjectToTraceFilter(self.project_id)) + log_handler.setFormatter(formatter) + log_handler.addFilter(AddGcpProjectToTraceFilter(self.project_id)) logging.basicConfig( level=logging.INFO, - handlers=[logHandler], + handlers=[log_handler], ) LoggingInstrumentor().instrument() @@ -129,18 +129,18 @@ def export_ot_to_gcp(resource: Resource) -> None: ) from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter - traceProvider = TracerProvider(resource=resource) + trace_provider = TracerProvider(resource=resource) processor = BatchSpanProcessor(CloudTraceSpanExporter()) - traceProvider.add_span_processor(processor) - trace.set_tracer_provider(traceProvider) + trace_provider.add_span_processor(processor) + trace.set_tracer_provider(trace_provider) reader = PeriodicExportingMetricReader(CloudMonitoringMetricsExporter()) - meterProvider = MeterProvider(metric_readers=[reader], resource=resource) - metrics.set_meter_provider(meterProvider) + meter_provider = MeterProvider(metric_readers=[reader], resource=resource) + metrics.set_meter_provider(meter_provider) - @exit() + @app_exit() def flush_ot() -> None: log.info("Flushing traces and metrics before shutdown") - traceProvider.force_flush() - meterProvider.force_flush() + trace_provider.force_flush() + meter_provider.force_flush() log.info("All metics and traces flushed.") diff --git a/libs/policyengine-fastapi/tests/test_exit.py b/libs/policyengine-fastapi/tests/test_exit.py index dd580fe03..c1cfc733a 100644 --- a/libs/policyengine-fastapi/tests/test_exit.py +++ b/libs/policyengine-fastapi/tests/test_exit.py @@ -1,7 +1,7 @@ from unittest.mock import Mock, call import pytest -from policyengine_fastapi.exit import exit +from policyengine_fastapi.exit import exit as app_exit @pytest.mark.asyncio @@ -13,9 +13,9 @@ async def test_lifecycle_executes_callbacks__when_app_ends(): parent.app = app parent.callback = callback - exit()(callback) + app_exit()(callback) - async with exit.lifespan(): + async with app_exit.lifespan(): app() parent.assert_has_calls([call.app(), call.callback()]) diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py index b57e3b057..a56d899a9 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py @@ -325,32 +325,32 @@ class BudgetWindowAnnualImpact(BaseModel): spm_provenance: SPMComparisonProvenance | None = None year: str - taxRevenueImpact: float - federalTaxRevenueImpact: float - stateTaxRevenueImpact: float - benefitSpendingImpact: float - budgetaryImpact: float + taxRevenueImpact: float # noqa: N815 # published budget-window JSON field taxRevenueImpact + federalTaxRevenueImpact: float # noqa: N815 # published budget-window JSON field federalTaxRevenueImpact + stateTaxRevenueImpact: float # noqa: N815 # published budget-window JSON field stateTaxRevenueImpact + benefitSpendingImpact: float # noqa: N815 # published budget-window JSON field benefitSpendingImpact + budgetaryImpact: float # noqa: N815 # published budget-window JSON field budgetaryImpact class BudgetWindowTotals(BaseModel): """Aggregate totals for a completed budget-window response.""" year: Literal["Total"] = "Total" - taxRevenueImpact: float - federalTaxRevenueImpact: float - stateTaxRevenueImpact: float - benefitSpendingImpact: float - budgetaryImpact: float + taxRevenueImpact: float # noqa: N815 # published budget-window JSON field taxRevenueImpact + federalTaxRevenueImpact: float # noqa: N815 # published budget-window JSON field federalTaxRevenueImpact + stateTaxRevenueImpact: float # noqa: N815 # published budget-window JSON field stateTaxRevenueImpact + benefitSpendingImpact: float # noqa: N815 # published budget-window JSON field benefitSpendingImpact + budgetaryImpact: float # noqa: N815 # published budget-window JSON field budgetaryImpact class BudgetWindowResult(BaseModel): """Completed budget-window output.""" kind: Literal["budgetWindow"] = "budgetWindow" - startYear: str - endYear: str - windowSize: int - annualImpacts: list[BudgetWindowAnnualImpact] = Field(default_factory=list) + startYear: str # noqa: N815 # published budget-window JSON field startYear + endYear: str # noqa: N815 # published budget-window JSON field endYear + windowSize: int # noqa: N815 # published budget-window JSON field windowSize + annualImpacts: list[BudgetWindowAnnualImpact] = Field(default_factory=list) # noqa: N815 # published budget-window JSON field annualImpacts totals: BudgetWindowTotals diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution.py index dd4e6ca42..7ef082277 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution.py @@ -24,10 +24,10 @@ SPMProvenance, ) from policyengine_simulation_contract.uk_geography import ( - UKLocalAuthorityBoundaryVersion as UKLocalAuthorityBoundaryVersion, + UKLocalAuthorityBoundaryVersion as UKLocalAuthorityBoundaryVersion, # noqa: PLC0414 # explicit re-export: callers import it from stage12_execution ) from policyengine_simulation_contract.uk_geography import ( - UKLocalAuthorityMetadata as UKLocalAuthorityMetadata, + UKLocalAuthorityMetadata as UKLocalAuthorityMetadata, # noqa: PLC0414 # explicit re-export: callers import it from stage12_execution ) ContractText = Annotated[str, Field(min_length=1, max_length=255)] diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index 4651c0044..b7656433b 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -52,8 +52,8 @@ ) from policyengine_simulation_entry.backend import ( BackendResponse, - BackendTimeout, - BackendUnavailable, + BackendTimeoutError, + BackendUnavailableError, OldGatewayBackend, SimulationBackend, ) @@ -68,7 +68,7 @@ ) from policyengine_simulation_entry.stage12_backend import ( TemporaryStage12DispatchFailed, - TemporaryStage12UnsupportedRequest, + TemporaryStage12UnsupportedRequestError, ) if TYPE_CHECKING: @@ -379,7 +379,7 @@ async def forward( response.headers[OBSERVABILITY_ID_HEADER] = observability_id elif OBSERVABILITY_ID_HEADER in response.headers: del response.headers[OBSERVABILITY_ID_HEADER] - except BackendTimeout: + except BackendTimeoutError: runtime.event( "simulation_entry_backend_timeout", attributes={ @@ -396,7 +396,7 @@ async def forward( **BACKEND_RESPONSE_HEADER, }, ) - except BackendUnavailable: + except BackendUnavailableError: runtime.event( "simulation_entry_backend_unavailable", attributes={ @@ -466,7 +466,7 @@ async def submit_temporary_stage12_report( content={"detail": "Stage 12 submission acknowledgement timed out."}, headers={"Retry-After": "10"}, ) - except TemporaryStage12UnsupportedRequest as error: + except TemporaryStage12UnsupportedRequestError as error: return JSONResponse( status_code=422, content={ diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/backend.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/backend.py index 81842927f..955dbc29b 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/backend.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/backend.py @@ -35,15 +35,15 @@ ) -class BackendUnavailable(RuntimeError): +class BackendUnavailableError(RuntimeError): """The backend could not be reached.""" -class BackendTimeout(RuntimeError): +class BackendTimeoutError(RuntimeError): """The backend did not answer within the configured timeout.""" -class BackendAuthenticationError(BackendUnavailable): +class BackendAuthenticationError(BackendUnavailableError): """The service could not authenticate to the backend.""" @@ -85,7 +85,7 @@ async def request( ) -> BackendResponse: """Send one request and return the backend's response. - Raise BackendUnavailable or BackendTimeout when no response arrives. + Raise BackendUnavailableError or BackendTimeoutError when no response arrives. """ ... @@ -196,14 +196,14 @@ async def close(self) -> None: def _runtime(self) -> tuple[httpx.AsyncClient, ClientCredentialsTokenProvider]: if self.client is None or self.token_provider is None: msg = "Old-gateway backend is not started." - raise BackendUnavailable(msg) + raise BackendUnavailableError(msg) return self.client, self.token_provider async def ready(self) -> bool: """Return whether the old gateway answers GET /health with 200.""" try: response = await self.request("GET", "/health") - except (BackendUnavailable, BackendTimeout): + except (BackendUnavailableError, BackendTimeoutError): return False return response.status_code == 200 @@ -220,8 +220,8 @@ async def request( A connection failure is retried once for GET only. A 401 or 403 refreshes the token and replays once for any method, then raises - BackendAuthenticationError. Raise BackendUnavailable if the client is - not started or the gateway is unreachable, and BackendTimeout on timeout. + BackendAuthenticationError. Raise BackendUnavailableError if the client is + not started or the gateway is unreachable, and BackendTimeoutError on timeout. """ client, token_provider = self._runtime() method = method.upper() @@ -248,13 +248,13 @@ async def request( connection_retries_remaining -= 1 continue msg = "Old gateway is unavailable." - raise BackendUnavailable(msg) from exc + raise BackendUnavailableError(msg) from exc except httpx.TimeoutException as exc: msg = "Old gateway timed out." - raise BackendTimeout(msg) from exc + raise BackendTimeoutError(msg) from exc except httpx.RequestError as exc: msg = "Old gateway is unavailable." - raise BackendUnavailable(msg) from exc + raise BackendUnavailableError(msg) from exc if response.status_code in {401, 403}: if auth_refreshes_remaining: diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/stage12_backend.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/stage12_backend.py index 33af3d078..91bfd9991 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/stage12_backend.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/stage12_backend.py @@ -88,7 +88,7 @@ async def spawn( ] -class TemporaryStage12UnsupportedRequest(ValueError): +class TemporaryStage12UnsupportedRequestError(ValueError): """A temporary direct request cannot be normalized by the reviewed adapter.""" def __init__(self, reason: str) -> None: @@ -97,7 +97,7 @@ def __init__(self, reason: str) -> None: self.reason = reason -class TemporaryStage12DispatchFailed(RuntimeError): +class TemporaryStage12DispatchFailed(RuntimeError): # noqa: N818 # app logs its name as error_type in stage12_comparison_dispatch_failed """Modal did not acknowledge a Stage 12 coordinator submission.""" def __init__(self, report: ComparisonReportRecord) -> None: @@ -371,7 +371,9 @@ def _prepare_temporary_report( ) ) if report is None: - raise TemporaryStage12UnsupportedRequest(skip_reason or "unsupported_input") + raise TemporaryStage12UnsupportedRequestError( + skip_reason or "unsupported_input" + ) parent = self._parent_record( evaluation_id=evaluation_id, request_id=request_id, diff --git a/projects/policyengine-simulation-entry/tests/test_app.py b/projects/policyengine-simulation-entry/tests/test_app.py index 9baef55dc..061d07a7b 100644 --- a/projects/policyengine-simulation-entry/tests/test_app.py +++ b/projects/policyengine-simulation-entry/tests/test_app.py @@ -19,12 +19,12 @@ from policyengine_simulation_entry.app import create_app from policyengine_simulation_entry.backend import ( BackendResponse, - BackendTimeout, - BackendUnavailable, + BackendTimeoutError, + BackendUnavailableError, ) from policyengine_simulation_entry.stage12_backend import ( TemporaryStage12DispatchFailed, - TemporaryStage12UnsupportedRequest, + TemporaryStage12UnsupportedRequestError, ) from policyengine_simulation_observability.identifiers import OBSERVABILITY_ID_HEADER from stage12_fixtures import ( @@ -584,7 +584,7 @@ def test_temporary_stage12_poll_returns_404_for_unknown_report(backend): ("error", "status_code", "expected"), [ ( - TemporaryStage12UnsupportedRequest("unsupported_cliff_calculation"), + TemporaryStage12UnsupportedRequestError("unsupported_cliff_calculation"), 422, {"reason": "unsupported_cliff_calculation"}, ), @@ -866,8 +866,12 @@ def test_request_validation_matches_shared_contract(client): @pytest.mark.parametrize( ("error", "expected_status", "expected_detail"), [ - (BackendUnavailable("unavailable"), 503, "Simulation backend is unavailable."), - (BackendTimeout("timed out"), 504, "Simulation backend timed out."), + ( + BackendUnavailableError("unavailable"), + 503, + "Simulation backend is unavailable.", + ), + (BackendTimeoutError("timed out"), 504, "Simulation backend timed out."), ], ) def test_backend_failures_are_sanitized(error, expected_status, expected_detail): diff --git a/projects/policyengine-simulation-entry/tests/test_backend.py b/projects/policyengine-simulation-entry/tests/test_backend.py index e28cb2a1d..502542868 100644 --- a/projects/policyengine-simulation-entry/tests/test_backend.py +++ b/projects/policyengine-simulation-entry/tests/test_backend.py @@ -8,7 +8,7 @@ from policyengine_observability import REQUEST_ID_HEADER from policyengine_simulation_entry.backend import ( BackendAuthenticationError, - BackendUnavailable, + BackendUnavailableError, OldGatewayBackend, ) @@ -208,7 +208,7 @@ def handler(request: httpx.Request) -> httpx.Response: ) await backend.start() try: - with pytest.raises(BackendUnavailable): + with pytest.raises(BackendUnavailableError): await backend.request( "POST", "/simulate/economy/comparison", diff --git a/projects/policyengine-simulation-entry/tests/test_stage12_backend.py b/projects/policyengine-simulation-entry/tests/test_stage12_backend.py index b18cc02a8..0145e54dc 100644 --- a/projects/policyengine-simulation-entry/tests/test_stage12_backend.py +++ b/projects/policyengine-simulation-entry/tests/test_stage12_backend.py @@ -18,7 +18,7 @@ ModalReportInvoker, Stage12ComparisonBackend, TemporaryStage12DispatchFailed, - TemporaryStage12UnsupportedRequest, + TemporaryStage12UnsupportedRequestError, ) from stage12_fixtures import eligible_payload, worker @@ -330,7 +330,7 @@ def test_temporary_submission_rejects_unsupported_input_without_side_effects() - store = FakeStore() invoker = FakeInvoker() - with pytest.raises(TemporaryStage12UnsupportedRequest) as error: + with pytest.raises(TemporaryStage12UnsupportedRequestError) as error: asyncio.run( _backend(store, invoker).submit_temporary_report( request_payload={**eligible_payload(), "include_cliffs": "true"}, diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py index 3725a522e..3ca1dff08 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py @@ -232,7 +232,7 @@ def ensure(self) -> None: # existing key and exposes no remove, so evict directly first. from policyengine.core.simulation import _cache - _cache._cache.pop(getattr(self, "storage_id", self.id), None) + _cache._cache.pop(getattr(self, "storage_id", self.id), None) # noqa: SLF001 # policyengine's LRUCache has no public remove _cache.add( getattr(self, "storage_id", self.id), self.model_copy(deep=False) if selection is not None else self, diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/main.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/main.py index 2e674a885..327f8ab7e 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/main.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/main.py @@ -6,7 +6,7 @@ from fastapi import FastAPI from policyengine_fastapi import ping -from policyengine_fastapi.exit import exit +from policyengine_fastapi.exit import exit as app_exit from policyengine_fastapi.health import ( HealthRegistry, HealthSystemReporter, @@ -30,7 +30,7 @@ @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: """Run the app inside the exit handler and flush observability on shutdown.""" - async with exit.lifespan(): + async with app_exit.lifespan(): try: yield finally: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py index 5e3d9d079..aac927130 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py @@ -456,7 +456,7 @@ def release(reference: ArtifactPathReference) -> tuple[str, str]: weight_matrix_bucket=scoping_strategy.weight_matrix_bucket, lookup_csv_bucket=scoping_strategy.lookup_csv_bucket, ) - simulation_output_geographic._required_uk_geography_lookup_csv_path(spec) + simulation_output_geographic._required_uk_geography_lookup_csv_path(spec) # noqa: SLF001 # package-internal helper, called through its module so tests can patch it paths = resolve_uk_geography_asset_paths(spec, download_missing_assets=False) if Path(paths.weight_matrix_path).resolve() != Path(certified.path).resolve(): msg = ( diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_parity.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_parity.py index 57d7b7d15..483360ab1 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_parity.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_parity.py @@ -24,7 +24,7 @@ def __post_init__(self) -> None: raise ValueError(msg) -class ParityMismatch(ValueError): +class ParityMismatchError(ValueError): """A bounded diagnostic containing a code and schema path, never values.""" def __init__(self, code: str, path: str) -> None: @@ -39,7 +39,7 @@ def canonicalize_simulation_frames( ) -> dict[str, pd.DataFrame]: """Sort each entity's rows by ``_id`` and its columns by name. - Raises ``ParityMismatch`` if an entity lacks that id column or repeats an id. + Raises ``ParityMismatchError`` if an entity lacks that id column or repeats an id. """ canonical = {} for entity in sorted(frames): @@ -47,10 +47,10 @@ def canonicalize_simulation_frames( frame = pd.DataFrame(frames[entity]).copy() if identifier not in frame.columns: error_code = "missing_row_identity" - raise ParityMismatch(error_code, entity) + raise ParityMismatchError(error_code, entity) if frame[identifier].duplicated().any(): error_code = "duplicate_row_identity" - raise ParityMismatch(error_code, entity) + raise ParityMismatchError(error_code, entity) canonical[entity] = ( frame.sort_values(identifier, kind="stable") .reset_index(drop=True) @@ -65,7 +65,7 @@ def compare_simulation_frames( *, tolerances: Mapping[str, NumericalTolerance] | None = None, ) -> None: - """Raise ``ParityMismatch`` at the first difference between two simulation outputs. + """Raise ``ParityMismatchError`` at the first difference of two simulation outputs. Entities, columns, row ids and dtypes must match exactly. Numeric columns compare within the tolerance keyed by ``entity.column`` (exact by default, @@ -75,27 +75,27 @@ def compare_simulation_frames( actual = canonicalize_simulation_frames(candidate) if expected.keys() != actual.keys(): error_code = "entity_membership_mismatch" - raise ParityMismatch(error_code, "simulation") + raise ParityMismatchError(error_code, "simulation") allowed = tolerances or {} for entity, expected_frame in expected.items(): actual_frame = actual[entity] if tuple(expected_frame.columns) != tuple(actual_frame.columns): error_code = "column_membership_mismatch" - raise ParityMismatch(error_code, entity) + raise ParityMismatchError(error_code, entity) if len(expected_frame) != len(actual_frame): error_code = "row_membership_mismatch" - raise ParityMismatch(error_code, entity) + raise ParityMismatchError(error_code, entity) identifier = f"{entity}_id" if not expected_frame[identifier].equals(actual_frame[identifier]): error_code = "row_identity_mismatch" - raise ParityMismatch(error_code, entity) + raise ParityMismatchError(error_code, entity) for column in expected_frame.columns: path = f"{entity}.{column}" expected_values = expected_frame[column] actual_values = actual_frame[column] if str(expected_values.dtype) != str(actual_values.dtype): error_code = "dtype_mismatch" - raise ParityMismatch(error_code, path) + raise ParityMismatchError(error_code, path) if pd.api.types.is_numeric_dtype(expected_values.dtype): tolerance = allowed.get(path, NumericalTolerance()) if not np.allclose( @@ -106,10 +106,10 @@ def compare_simulation_frames( equal_nan=True, ): error_code = "numeric_value_mismatch" - raise ParityMismatch(error_code, path) + raise ParityMismatchError(error_code, path) elif not expected_values.equals(actual_values): error_code = "value_mismatch" - raise ParityMismatch(error_code, path) + raise ParityMismatchError(error_code, path) def compare_aggregate_reports( @@ -118,7 +118,7 @@ def compare_aggregate_reports( *, tolerances: Mapping[str, NumericalTolerance] | None = None, ) -> None: - """Raise ``ParityMismatch`` at the first difference between two aggregate reports. + """Raise ``ParityMismatchError`` at the first difference of two aggregate reports. Mappings and lists must match in shape. Numbers (not booleans) compare within the tolerance keyed by their path, such as ``budget.total`` or @@ -130,7 +130,7 @@ def compare(expected: object, actual: object, path: str) -> None: if isinstance(expected, Mapping): if not isinstance(actual, Mapping) or expected.keys() != actual.keys(): error_code = "field_membership_mismatch" - raise ParityMismatch(error_code, path or "report") + raise ParityMismatchError(error_code, path or "report") for key in sorted(expected): compare(expected[key], actual[key], f"{path}.{key}".lstrip(".")) return @@ -141,10 +141,10 @@ def compare(expected: object, actual: object, path: str) -> None: actual, (str, bytes, bytearray) ): error_code = "value_type_mismatch" - raise ParityMismatch(error_code, path) + raise ParityMismatchError(error_code, path) if len(expected) != len(actual): error_code = "list_length_mismatch" - raise ParityMismatch(error_code, path) + raise ParityMismatchError(error_code, path) for index, value in enumerate(expected): compare(value, actual[index], f"{path}[{index}]") return @@ -162,13 +162,13 @@ def compare(expected: object, actual: object, path: str) -> None: abs_tol=tolerance.absolute, ): error_code = "numeric_value_mismatch" - raise ParityMismatch(error_code, path) + raise ParityMismatchError(error_code, path) return if type(expected) is not type(actual): error_code = "value_type_mismatch" - raise ParityMismatch(error_code, path) + raise ParityMismatchError(error_code, path) if expected != actual: error_code = "value_mismatch" - raise ParityMismatch(error_code, path) + raise ParityMismatchError(error_code, path) compare(incumbent, candidate, "") diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_qualification.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_qualification.py index 34473ae93..00c43929b 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_qualification.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_qualification.py @@ -35,7 +35,7 @@ ) from policyengine_simulation_executor.stage12_parity import ( NumericalTolerance, - ParityMismatch, + ParityMismatchError, compare_aggregate_reports, compare_simulation_frames, ) @@ -132,17 +132,17 @@ def _validate_matching_input( continue if actual_value != expected_value: error_code = "normalized_input_mismatch" - raise ParityMismatch(error_code, field) + raise ParityMismatchError(error_code, field) if existing_request.get("data") not in accepted_dataset_values: error_code = "dataset_provenance_mismatch" - raise ParityMismatch(error_code, "data") + raise ParityMismatchError(error_code, "data") population = baseline.population if population.kind != "dataset": error_code = "population_kind_mismatch" - raise ParityMismatch(error_code, "population.kind") + raise ParityMismatchError(error_code, "population.kind") if population.artifact.uri != bundle.dataset.uri: error_code = "dataset_provenance_mismatch" - raise ParityMismatch( + raise ParityMismatchError( error_code, "population.artifact.uri", ) @@ -155,25 +155,25 @@ def _existing_frames( ) -> dict[str, pd.DataFrame]: if not isinstance(microdata, Mapping): error_code = "missing_existing_microdata" - raise ParityMismatch(error_code, "report._microdata") + raise ParityMismatchError(error_code, "report._microdata") raw_side = microdata.get(side) raw_dtypes = microdata.get("dtypes") side_dtypes = raw_dtypes.get(side) if isinstance(raw_dtypes, Mapping) else None if not isinstance(raw_side, Mapping) or not isinstance(side_dtypes, Mapping): error_code = "invalid_existing_microdata" - raise ParityMismatch(error_code, f"report._microdata.{side}") + raise ParityMismatchError(error_code, f"report._microdata.{side}") frames: dict[str, pd.DataFrame] = {} for entity, columns in raw_side.items(): entity_dtypes = side_dtypes.get(entity) if not isinstance(entity, str) or not isinstance(columns, dict): error_code = "invalid_existing_microdata" - raise ParityMismatch( + raise ParityMismatchError( error_code, f"report._microdata.{side}", ) if not isinstance(entity_dtypes, dict): error_code = "invalid_existing_dtypes" - raise ParityMismatch( + raise ParityMismatchError( error_code, f"report._microdata.dtypes.{side}.{entity}", ) @@ -275,13 +275,13 @@ def run_existing(request: dict[str, Any]) -> Mapping[str, Any]: bundle = report.baseline.bundle if existing_result.get("model_version") != bundle.country_package_version: error_code = "country_package_provenance_mismatch" - raise ParityMismatch( + raise ParityMismatchError( error_code, "report.model_version", ) if existing_result.get("data_version") != bundle.dataset.artifact_revision: error_code = "dataset_provenance_mismatch" - raise ParityMismatch( + raise ParityMismatchError( error_code, "report.data_version", ) @@ -321,7 +321,7 @@ def run_existing(request: dict[str, Any]) -> Mapping[str, Any]: v2_result = v2_report.get("result") if not isinstance(v2_result, Mapping): error_code = "invalid_v2_aggregate" - raise ParityMismatch(error_code, "report.result") + raise ParityMismatchError(error_code, "report.result") compare_aggregate_reports( existing_result, v2_result, diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_parity.py b/projects/policyengine-simulation-executor/tests/test_stage12_parity.py index f5eb260e5..3a006766a 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_parity.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_parity.py @@ -5,7 +5,7 @@ import pytest from policyengine_simulation_executor.stage12_parity import ( NumericalTolerance, - ParityMismatch, + ParityMismatchError, compare_aggregate_reports, compare_simulation_frames, ) @@ -42,7 +42,7 @@ def test_simulation_parity_requires_reviewed_field_specific_tolerance() -> None: incumbent = _frames() candidate = _frames(100.005) - with pytest.raises(ParityMismatch) as error: + with pytest.raises(ParityMismatchError) as error: compare_simulation_frames(incumbent, candidate) assert error.value.code == "numeric_value_mismatch" assert "100" not in str(error.value) @@ -61,12 +61,12 @@ def test_simulation_parity_compares_schema_dtypes_and_row_membership() -> None: wrong_dtype["person"]["person_household_id"] = wrong_dtype["person"][ "person_household_id" ].astype("int32") - with pytest.raises(ParityMismatch, match="dtype_mismatch"): + with pytest.raises(ParityMismatchError, match="dtype_mismatch"): compare_simulation_frames(_frames(), wrong_dtype) missing_row = _frames() missing_row["person"] = missing_row["person"].iloc[:1] - with pytest.raises(ParityMismatch, match="row_membership_mismatch"): + with pytest.raises(ParityMismatchError, match="row_membership_mismatch"): compare_simulation_frames(_frames(), missing_row) @@ -80,7 +80,7 @@ def test_aggregate_parity_compares_every_field_and_bounds_diagnostics() -> None: "poverty": [{"name": "all", "rate": 0.12}], } - with pytest.raises(ParityMismatch) as error: + with pytest.raises(ParityMismatchError) as error: compare_aggregate_reports(incumbent, candidate) assert error.value.path == "budget.total" assert "100" not in str(error.value) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_qualification.py b/projects/policyengine-simulation-executor/tests/test_stage12_qualification.py index eae07ae75..e2927ab51 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_qualification.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_qualification.py @@ -11,7 +11,7 @@ from policyengine_simulation_executor.simulation_microdata import ( extract_output_microdata, ) -from policyengine_simulation_executor.stage12_parity import ParityMismatch +from policyengine_simulation_executor.stage12_parity import ParityMismatchError from policyengine_simulation_executor.stage12_qualification import ( qualify_report_parity, ) @@ -145,7 +145,7 @@ def single_runner(simulation): ) return SimulationCalculation(frames=frames) - with pytest.raises(ParityMismatch) as error: + with pytest.raises(ParityMismatchError) as error: qualify_report_parity( _existing_request(), report, @@ -169,7 +169,7 @@ def single_runner(simulation): def test_qualification_rejects_different_input_provenance(field, value, code) -> None: request = {**_existing_request(), field: value} - with pytest.raises(ParityMismatch) as error: + with pytest.raises(ParityMismatchError) as error: qualify_report_parity(request, _report()) assert error.value.code == code @@ -191,7 +191,7 @@ def existing_runner(_request): ), } - with pytest.raises(ParityMismatch) as error: + with pytest.raises(ParityMismatchError) as error: qualify_report_parity( _existing_request(), report, From 9f5611bcbce86d5f75a58e5089f405c486794611 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:09:01 +0400 Subject: [PATCH 04/23] Resolve ruff INP001 per directory Declare the script directories (.github/scripts, scripts and the executor's scripts) and the rules proof fixtures as namespace packages in ruff.toml. Ruff's full output, with every rule selected, is otherwise unchanged by the setting. The executor's src/modal/utils scripts keep an inline suppression instead: declaring that directory, or adding an __init__.py, makes ruff treat it as part of a first-party "modal" package and regroup its Modal SDK imports. An __init__.py would also change the src.modal source baked into the Modal images. Co-Authored-By: Claude Opus 5.5 --- .../src/modal/utils/extract_bundle_versions.py | 2 +- .../src/modal/utils/extract_stage12_bundle.py | 2 +- .../src/modal/utils/qualify_stage12_parity.py | 2 +- .../src/modal/utils/record_deployment.py | 2 +- .../src/modal/utils/update_v2_version_manifest.py | 2 +- .../src/modal/utils/update_version_registry.py | 2 +- .../src/modal/utils/validate_hf_access.py | 2 +- .../src/modal/utils/validate_v2_worker_release.py | 2 +- ruff.toml | 14 ++++++++++++++ 9 files changed, 22 insertions(+), 8 deletions(-) diff --git a/projects/policyengine-simulation-executor/src/modal/utils/extract_bundle_versions.py b/projects/policyengine-simulation-executor/src/modal/utils/extract_bundle_versions.py index 4e41068fa..b70ef031b 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/extract_bundle_versions.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/extract_bundle_versions.py @@ -1,4 +1,4 @@ -"""Print policyengine.py bundle versions for deployment scripts.""" +"""Print policyengine.py bundle versions for deployment scripts.""" # noqa: INP001 # a script run as python -m src.modal.utils.extract_bundle_versions; utils is not a regular package from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/modal/utils/extract_stage12_bundle.py b/projects/policyengine-simulation-executor/src/modal/utils/extract_stage12_bundle.py index 3850ed768..19ebeb54b 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/extract_stage12_bundle.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/extract_stage12_bundle.py @@ -1,4 +1,4 @@ -"""Export the strict Stage 12 worker bundle selected by PolicyEngine.py.""" +"""Export the strict Stage 12 worker bundle selected by PolicyEngine.py.""" # noqa: INP001 # a script run as python -m src.modal.utils.extract_stage12_bundle; utils is not a regular package from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/modal/utils/qualify_stage12_parity.py b/projects/policyengine-simulation-executor/src/modal/utils/qualify_stage12_parity.py index 994750336..8ffea2d40 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/qualify_stage12_parity.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/qualify_stage12_parity.py @@ -1,4 +1,4 @@ -"""Run one controlled Stage 12 parity qualification from canonical JSON files.""" +"""Run one controlled Stage 12 parity qualification from canonical JSON files.""" # noqa: INP001 # a script run as python -m src.modal.utils.qualify_stage12_parity; utils is not a regular package from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/modal/utils/record_deployment.py b/projects/policyengine-simulation-executor/src/modal/utils/record_deployment.py index 3b0426453..1e0a0fe12 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/record_deployment.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/record_deployment.py @@ -4,7 +4,7 @@ ``deployed/.json`` marker (last-writer-wins). The marker is the artifact GC's liveness signal, so a deploy that cannot record itself must fail the job even though Modal already serves it. -""" +""" # noqa: INP001 # a script run as python -m src.modal.utils.record_deployment; utils is not a regular package from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/modal/utils/update_v2_version_manifest.py b/projects/policyengine-simulation-executor/src/modal/utils/update_v2_version_manifest.py index c695073e7..e3cb6bf5d 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/update_v2_version_manifest.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/update_v2_version_manifest.py @@ -1,4 +1,4 @@ -"""Publish a validated worker to the separate Stage 12 v2 manifest.""" +"""Publish a validated worker to the separate Stage 12 v2 manifest.""" # noqa: INP001 # a script run as python -m src.modal.utils.update_v2_version_manifest; utils is not a regular package from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py b/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py index 9ab9f69f0..6f7df7327 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py @@ -4,7 +4,7 @@ group ``simulation-deploy-main`` (with ``cancel-in-progress: false``). Library writers and the CLI permit writes only from its deployment and rollback workflows; read-only and plan modes remain available locally. -""" +""" # noqa: INP001 # a script run as python -m src.modal.utils.update_version_registry; utils is not a regular package import argparse import hashlib diff --git a/projects/policyengine-simulation-executor/src/modal/utils/validate_hf_access.py b/projects/policyengine-simulation-executor/src/modal/utils/validate_hf_access.py index 895fc9549..e2f2be5c5 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/validate_hf_access.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/validate_hf_access.py @@ -1,4 +1,4 @@ -"""Validate UK private-data access directly in CI.""" +"""Validate UK private-data access directly in CI.""" # noqa: INP001 # a script run as python -m src.modal.utils.validate_hf_access; utils is not a regular package from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/modal/utils/validate_v2_worker_release.py b/projects/policyengine-simulation-executor/src/modal/utils/validate_v2_worker_release.py index 0192c50f4..e029a09b6 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/validate_v2_worker_release.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/validate_v2_worker_release.py @@ -1,4 +1,4 @@ -"""Invoke and verify every deployed Stage 12 country validation function.""" +"""Invoke and verify every deployed Stage 12 country validation function.""" # noqa: INP001 # a script run as python -m src.modal.utils.validate_v2_worker_release; utils is not a regular package from __future__ import annotations diff --git a/ruff.toml b/ruff.toml index ec93e8e82..4aa63ec9a 100644 --- a/ruff.toml +++ b/ruff.toml @@ -5,6 +5,20 @@ # be treated as Python 3.9. scripts/rules supports Python 3.11+ (and is shared # byte-for-byte with policyengine-api, which targets 3.11). per-file-target-version = { "scripts/**" = "py311" } +# Directories that are not regular packages on purpose, so they have no +# __init__.py: standalone scripts, the implicit namespace package that holds +# scripts.rules, and proof fixtures. Ruff resolves modules as if each listed +# directory, and every directory under it, had an __init__.py. This is a +# top-level setting: ruff rejects it in [lint]. The executor's src/modal/utils +# scripts are not listed, because ruff would then treat them as part of a +# first-party "modal" package and regroup their Modal SDK imports; each keeps +# an inline INP001 suppression instead. +namespace-packages = [ + ".github/scripts", + "projects/policyengine-simulation-executor/scripts", + "rules/fixtures", + "scripts", +] [lint] select = [ From c7bbd4592e49da6def4851c0c2241423e3263fa9 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:27:35 +0400 Subject: [PATCH 05/23] Keep deliberate deferred imports in the shared library and CI script (ruff PLC0415) Each deferred import keeps an inline reason: the GCP exporters, logging instrumentation and metadata probe load only when used, and only the bundle-versions command runs where policyengine is installed. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/update_policyengine_package.py | 4 +++- .../policyengine_fastapi/opentelemetry/gcp.py | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/scripts/update_policyengine_package.py b/.github/scripts/update_policyengine_package.py index af723b329..a51e5ffe1 100644 --- a/.github/scripts/update_policyengine_package.py +++ b/.github/scripts/update_policyengine_package.py @@ -201,7 +201,9 @@ def latest_version(package: str) -> None: def bundle_versions() -> None: """Print the package and data versions selected by PolicyEngine.py.""" - from policyengine.bundle import get_current_bundle + from policyengine.bundle import ( # noqa: PLC0415 # only bundle-versions runs under uv with policyengine installed; the other commands use the runner's python3 + get_current_bundle, + ) bundle = _mapping(get_current_bundle()) packages = _mapping(bundle.get("packages")) diff --git a/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py b/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py index 51e1a0631..c7d83da44 100644 --- a/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py +++ b/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py @@ -49,7 +49,9 @@ def _get_project_id() -> str: def _get_project_id_from_metadata() -> str | None: - from urllib import request + from urllib import ( # noqa: PLC0415 # urllib.request loads only when the metadata probe runs + request, + ) try: return ( @@ -91,8 +93,12 @@ def instrument(self) -> None: Uses ``logging.basicConfig``, so the handler is not installed if the root logger already has handlers. """ - from opentelemetry.instrumentation.logging import LoggingInstrumentor - from pythonjsonlogger.json import JsonFormatter + from opentelemetry.instrumentation.logging import ( # noqa: PLC0415 # the logging instrumentation loads only when GCP logging is set up + LoggingInstrumentor, + ) + from pythonjsonlogger.json import ( # noqa: PLC0415 # the JSON formatter loads only when GCP logging is set up + JsonFormatter, + ) log_handler = logging.StreamHandler() formatter = JsonFormatter( @@ -124,10 +130,12 @@ def export_ot_to_gcp(resource: Resource) -> None: providers for ``resource`` and registers an ``exit`` callback that flushes both at shutdown. """ - from opentelemetry.exporter.cloud_monitoring import ( + from opentelemetry.exporter.cloud_monitoring import ( # noqa: PLC0415 # the Google Cloud exporters load only when GCP export is set up CloudMonitoringMetricsExporter, ) - from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter + from opentelemetry.exporter.cloud_trace import ( # noqa: PLC0415 # the Google Cloud exporters load only when GCP export is set up + CloudTraceSpanExporter, + ) trace_provider = TracerProvider(resource=resource) processor = BatchSpanProcessor(CloudTraceSpanExporter()) From 60b8dbaf8e76a6b1d4d0476303ef38a6ed2d2df0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:27:35 +0400 Subject: [PATCH 06/23] Import the Stage 12 backend at the top of the entry app (ruff PLC0415) app.py already imports stage12_backend at module level for its exception types, so binding Stage12ComparisonBackend there too loads nothing new. Co-Authored-By: Claude Opus 5.5 --- .../src/policyengine_simulation_entry/app.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index b7656433b..140932660 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -67,6 +67,7 @@ TemporaryStage12SubmissionResponse, ) from policyengine_simulation_entry.stage12_backend import ( + Stage12ComparisonBackend, TemporaryStage12DispatchFailed, TemporaryStage12UnsupportedRequestError, ) @@ -260,10 +261,6 @@ def adopt_backend_observability_id( stage12_operator = authorizer.require_permission("stage12:operate") runtime_comparison = comparison_backend if runtime_comparison is None and runtime_settings.stage12_resources_configured: - from policyengine_simulation_entry.stage12_backend import ( - Stage12ComparisonBackend, - ) - runtime_comparison = Stage12ComparisonBackend.from_settings( runtime_settings, runtime=runtime, From 2d0def700355c13216c55fb671900ce7796b64c6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:27:35 +0400 Subject: [PATCH 07/23] Resolve ruff PLC0415 in the gateway Bind SPMSelection in endpoints.py's existing policyengine_simulation_contract.spm import. Keep the Modal function body's imports, which build the web app in its container, and the smoke app's in-image imports, each with a reason. Co-Authored-By: Claude Opus 5.5 --- .../fixtures/gateway_endpoints.py | 6 ++++-- .../src/policyengine_simulation_gateway/app.py | 14 +++++++++----- .../policyengine_simulation_gateway/endpoints.py | 3 +-- .../policyengine_simulation_gateway/smoke_app.py | 16 +++++++++------- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/projects/policyengine-simulation-gateway/fixtures/gateway_endpoints.py b/projects/policyengine-simulation-gateway/fixtures/gateway_endpoints.py index 1e01a6454..6741f8862 100644 --- a/projects/policyengine-simulation-gateway/fixtures/gateway_endpoints.py +++ b/projects/policyengine-simulation-gateway/fixtures/gateway_endpoints.py @@ -355,12 +355,14 @@ class OutputExpiredError(Exception): @pytest.fixture def mock_modal(monkeypatch: pytest.MonkeyPatch) -> dict[str, object]: """Patch Modal calls in the gateway endpoints module.""" - from policyengine_simulation_contract import ( + from policyengine_simulation_contract import ( # noqa: PLC0415 # patch targets, imported when the fixture runs budget_window_state, dataset_uri, job_progress_state, ) - from policyengine_simulation_gateway import endpoints + from policyengine_simulation_gateway import ( # noqa: PLC0415 # patch targets, imported when the fixture runs + endpoints, + ) mock_func = MockFunction() mock_dicts = { diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/app.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/app.py index 23984108c..25b5b618f 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/app.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/app.py @@ -80,19 +80,23 @@ def web_app() -> FastAPI: GET /versions - List available versions GET /health - Health check """ - from fastapi import FastAPI - from policyengine_simulation_observability.observability import ( + from fastapi import ( # noqa: PLC0415 # Modal function body: the gateway app is built in its container, not at deploy time + FastAPI, + ) + from policyengine_simulation_observability.observability import ( # noqa: PLC0415 # Modal function body: the gateway app is built in its container, not at deploy time init_simulation_observability, ) - from policyengine_simulation_gateway.auth import ( + from policyengine_simulation_gateway.auth import ( # noqa: PLC0415 # Modal function body: the gateway app is built in its container, not at deploy time enforce_auth_configured_guard, enforce_production_auth_guard, ) - from policyengine_simulation_gateway.correlation import ( + from policyengine_simulation_gateway.correlation import ( # noqa: PLC0415 # Modal function body: the gateway app is built in its container, not at deploy time install_observability_id_middleware, ) - from policyengine_simulation_gateway.endpoints import router + from policyengine_simulation_gateway.endpoints import ( # noqa: PLC0415 # Modal function body: the gateway app is built in its container, not at deploy time + router, + ) api = FastAPI( title="PolicyEngine Simulation Gateway", diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py index eafb5ee61..367d89e93 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py @@ -48,6 +48,7 @@ from policyengine_simulation_contract.spm import ( SPMCapability, SPMInputError, + SPMSelection, resolve_spm_selection, spm_error_detail, ) @@ -673,8 +674,6 @@ def _resolve_request_spm( route_provenance=route.route_provenance, ) if selection is not None: - from policyengine_simulation_contract.spm import SPMSelection - request.spm = SPMSelection.model_validate(selection) return selection diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/smoke_app.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/smoke_app.py index 647afeb65..8eedc377f 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/smoke_app.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/smoke_app.py @@ -26,14 +26,14 @@ def smoke_import_gateway() -> dict: Return the number of modules imported and the app's ``METHOD path`` routes. """ - import importlib - import os - import pkgutil + import importlib # noqa: PLC0415 # smoke app: standard-library import kept with the in-image check's other imports + import os # noqa: PLC0415 # smoke app: standard-library import kept with the in-image check's other imports + import pkgutil # noqa: PLC0415 # smoke app: standard-library import kept with the in-image check's other imports - import policyengine_simulation_contract - import policyengine_simulation_observability + import policyengine_simulation_contract # noqa: PLC0415 # smoke check: the import itself, inside the Modal image, is what this verifies + import policyengine_simulation_observability # noqa: PLC0415 # smoke check: the import itself, inside the Modal image, is what this verifies - import policyengine_simulation_gateway + import policyengine_simulation_gateway # noqa: PLC0415 # smoke check: the import itself, inside the Modal image, is what this verifies imported = [] for package in ( @@ -52,7 +52,9 @@ def smoke_import_gateway() -> dict: # import surface is exercised without real JWT material. os.environ.setdefault("GATEWAY_AUTH_ISSUER", "https://smoke.invalid/") os.environ.setdefault("GATEWAY_AUTH_AUDIENCE", "smoke") - from policyengine_simulation_gateway.testing import create_gateway_app + from policyengine_simulation_gateway.testing import ( # noqa: PLC0415 # smoke check: imported after the auth environment is staged + create_gateway_app, + ) asgi_app = create_gateway_app() return { From a2c274af11ea7687859aee49ff0a714b7b2f2080 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:28:49 +0400 Subject: [PATCH 08/23] Resolve ruff PLC0415 in the executor Move only imports that load nothing new and that no test patches: pathlib and importlib, the policyengine_simulation_contract helpers, names from a module the file already imports at the top, and RegionGroupStrategy, which policyengine.core already loads. Every other deferred import keeps an inline reason: Modal function bodies and the image-build and smoke functions, lazily loaded policyengine models and heavy libraries, call-time resolution that the tests' patch seams rely on, and import cycles in the runtime. Co-Authored-By: Claude Opus 5.5 --- .../fixtures/identity_stubs.py | 14 ++-- .../fixtures/spm_doubles.py | 4 +- .../src/modal/_image_setup.py | 28 +++++--- .../src/modal/app.py | 18 +++-- .../src/modal/budget_window_batch.py | 4 +- .../src/modal/precompute_app.py | 44 +++++++++--- .../src/modal/segmented_national.py | 10 +-- .../src/modal/smoke_app.py | 26 ++++--- .../modal/utils/update_version_registry.py | 6 +- .../src/modal/v2_app.py | 8 +-- .../artifact_keys.py | 20 ++++-- .../artifact_store.py | 6 +- .../baseline_artifacts.py | 16 ++--- .../hf_access_validation.py | 4 +- .../precompute.py | 56 ++++++++------- .../release_bundle.py | 16 +++-- .../segmented_national_reduce.py | 2 +- .../simulation_output_common.py | 14 ++-- .../simulation_output_distribution.py | 2 +- .../simulation_output_geographic.py | 15 ++-- .../simulation_runtime.py | 68 ++++++++++++------- .../policyengine_simulation_executor/spm.py | 26 +++++-- .../stage12_bundle/manifest.py | 4 +- .../stage12_qualification.py | 12 +--- .../stage12_runtime/aggregation.py | 20 +++--- .../stage12_runtime/dependencies.py | 5 +- .../stage12_runtime/output_planning.py | 10 ++- .../stage12_runtime/simulation.py | 12 ++-- .../stage12_worker_validation.py | 4 +- .../uk_local_authority_metadata.py | 4 +- 30 files changed, 299 insertions(+), 179 deletions(-) diff --git a/projects/policyengine-simulation-executor/fixtures/identity_stubs.py b/projects/policyengine-simulation-executor/fixtures/identity_stubs.py index 21204db4c..dc4255760 100644 --- a/projects/policyengine-simulation-executor/fixtures/identity_stubs.py +++ b/projects/policyengine-simulation-executor/fixtures/identity_stubs.py @@ -34,9 +34,13 @@ def install_identity_stubs(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace: # This module imports ``get_release_manifest`` by value while constructing # country models. Import it before replacing the manifest module's function # so full-suite ordering cannot bind this test's stub into production code. - import policyengine.tax_benefit_models.common.model_version # noqa: F401 - from policyengine.provenance import manifest as manifest_module - from policyengine_simulation_executor import release_bundle + import policyengine.tax_benefit_models.common.model_version # noqa: F401, PLC0415 # must load before the manifest stubs are installed (see above) + from policyengine.provenance import ( # noqa: PLC0415 # patch target, imported after model_version above + manifest as manifest_module, + ) + from policyengine_simulation_executor import ( # noqa: PLC0415 # patch target, imported when the stubs are installed + release_bundle, + ) bundle = SimpleNamespace( country="us", @@ -84,7 +88,9 @@ def install_identity_stubs(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace: ), ) - from policyengine_simulation_executor import spm as executor_spm + from policyengine_simulation_executor import ( # noqa: PLC0415 # patch target, imported when the stubs are installed + spm as executor_spm, + ) state.installed_spm_capability = executor_spm.runtime_spm_capability monkeypatch.setattr( diff --git a/projects/policyengine-simulation-executor/fixtures/spm_doubles.py b/projects/policyengine-simulation-executor/fixtures/spm_doubles.py index cd8c1afda..b4cf34424 100644 --- a/projects/policyengine-simulation-executor/fixtures/spm_doubles.py +++ b/projects/policyengine-simulation-executor/fixtures/spm_doubles.py @@ -11,7 +11,9 @@ def installed_spm_selection() -> dict[str, object] | None: """Return the installed US bundle's default SPM selection, if any.""" - from policyengine_simulation_executor.spm import runtime_spm_capability + from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # resolved per call so tests can patch spm.runtime_spm_capability + runtime_spm_capability, + ) capability = runtime_spm_capability() return None if capability is None else capability.defaults.model_dump() diff --git a/projects/policyengine-simulation-executor/src/modal/_image_setup.py b/projects/policyengine-simulation-executor/src/modal/_image_setup.py index 82290a293..6019805c2 100644 --- a/projects/policyengine-simulation-executor/src/modal/_image_setup.py +++ b/projects/policyengine-simulation-executor/src/modal/_image_setup.py @@ -39,10 +39,12 @@ def fetch_artifacts( ``client`` is a test seam; real builds construct a storage client from the layer's GCP secret (or ambient ADC for local drill runs). """ - import json - import logging - import os - from pathlib import Path + import json # noqa: PLC0415 # image-build function: this module has no module-level imports by design + import logging # noqa: PLC0415 # image-build function: this module has no module-level imports by design + import os # noqa: PLC0415 # image-build function: this module has no module-level imports by design + from pathlib import ( # noqa: PLC0415 # image-build function: this module has no module-level imports by design + Path, + ) logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -118,7 +120,9 @@ def fetch_artifacts( raise info = json.loads(parsed) if isinstance(parsed, str) else parsed - from google.oauth2 import service_account + from google.oauth2 import ( # noqa: PLC0415 # image-build function: this module has no module-level imports by design + service_account, + ) # Credentials built in memory, never written to disk: this # layer's container filesystem is committed into the image, @@ -130,7 +134,9 @@ def fetch_artifacts( "project": info.get("project_id"), } # No blob: ambient ADC (the local fault-drill path). - from google.cloud import storage + from google.cloud import ( # noqa: PLC0415 # image-build function: this module has no module-level imports by design + storage, + ) client = storage.Client(**creds_kwargs) @@ -175,18 +181,22 @@ def fetch_artifacts( def snapshot_models() -> None: """Pre-load models at image build time for fast cold starts.""" - import logging + import logging # noqa: PLC0415 # image-build function: this module has no module-level imports by design logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) logger.info("Pre-loading US tax-benefit system...") - from policyengine_us import CountryTaxBenefitSystem as USSystem + from policyengine_us import ( # noqa: PLC0415 # image-build function: this module has no module-level imports by design + CountryTaxBenefitSystem as USSystem, + ) USSystem() logger.info("Pre-loading UK tax-benefit system...") - from policyengine_uk import CountryTaxBenefitSystem as UKSystem + from policyengine_uk import ( # noqa: PLC0415 # image-build function: this module has no module-level imports by design + CountryTaxBenefitSystem as UKSystem, + ) UKSystem() diff --git a/projects/policyengine-simulation-executor/src/modal/app.py b/projects/policyengine-simulation-executor/src/modal/app.py index 97734e86b..2f1c9bad7 100644 --- a/projects/policyengine-simulation-executor/src/modal/app.py +++ b/projects/policyengine-simulation-executor/src/modal/app.py @@ -152,8 +152,12 @@ def _deploy_time_artifact_inputs() -> tuple[str, dict | None]: bucket = os.environ.get("POLICYENGINE_ARTIFACT_BUCKET", "") if not digest: return bucket, None - from policyengine_simulation_executor.artifact_store import ArtifactStore - from policyengine_simulation_executor.precompute_models import ArtifactManifest + from policyengine_simulation_executor.artifact_store import ( # noqa: PLC0415 # deploy-time manifest check only; the app import stays light for the precompute and smoke apps + ArtifactStore, + ) + from policyengine_simulation_executor.precompute_models import ( # noqa: PLC0415 # deploy-time manifest check only; the app import stays light for the precompute and smoke apps + ArtifactManifest, + ) store = ArtifactStore(bucket or None) payload = store.read_manifest(digest) @@ -361,7 +365,9 @@ def run_simulation( remote_context=propagated, ): _set_modal_call_attributes(runtime) - from src.modal.segmented_national import dispatch_run_simulation + from src.modal.segmented_national import ( # noqa: PLC0415 # Modal function body: loaded in the worker container, not at deploy time + dispatch_run_simulation, + ) try: # Plain national macro requests fan out across region groups @@ -431,7 +437,7 @@ def run_simulation_segment( remote_context=propagated, ): _set_modal_call_attributes(runtime) - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # Modal function body: loaded in the worker container, not at deploy time run_simulation_impl, ) @@ -471,7 +477,9 @@ def run_budget_window_batch( remote_context=propagated, ): _set_modal_call_attributes(runtime) - from src.modal.budget_window_batch import run_budget_window_batch_impl + from src.modal.budget_window_batch import ( # noqa: PLC0415 # Modal function body: loaded in the worker container, not at deploy time + run_budget_window_batch_impl, + ) return run_budget_window_batch_impl(params, runtime=runtime) finally: diff --git a/projects/policyengine-simulation-executor/src/modal/budget_window_batch.py b/projects/policyengine-simulation-executor/src/modal/budget_window_batch.py index 285b9ad1b..92635502e 100644 --- a/projects/policyengine-simulation-executor/src/modal/budget_window_batch.py +++ b/projects/policyengine-simulation-executor/src/modal/budget_window_batch.py @@ -24,7 +24,9 @@ def run_budget_window_batch_impl( Must run inside a Modal call: the current function call id becomes the batch job id that child jobs and stored batch state are keyed by. """ - from policyengine_simulation_executor.spm import normalize_runtime_spm + from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # Modal worker code: executor spm is loaded per call, in the container + normalize_runtime_spm, + ) selection = normalize_runtime_spm(params) if selection is not None: diff --git a/projects/policyengine-simulation-executor/src/modal/precompute_app.py b/projects/policyengine-simulation-executor/src/modal/precompute_app.py index a83b801c5..4dacc38ea 100644 --- a/projects/policyengine-simulation-executor/src/modal/precompute_app.py +++ b/projects/policyengine-simulation-executor/src/modal/precompute_app.py @@ -54,7 +54,9 @@ ) def plan_artifacts(bucket: str) -> dict: """Return the dumped ``PrecomputePlan``: expected artifacts and store presence.""" - from policyengine_simulation_executor.precompute import plan_artifacts_impl + from policyengine_simulation_executor.precompute import ( # noqa: PLC0415 # Modal function body: loaded in the precompute worker, not at deploy time + plan_artifacts_impl, + ) return plan_artifacts_impl(bucket).model_dump() @@ -72,8 +74,12 @@ def build_dataset(bucket: str, expected: dict) -> dict: Raises if the dataset identity resolved in this container differs from the planned one, so planner/worker drift cannot upload under the wrong key. """ - from policyengine_simulation_executor.precompute import build_dataset_impl - from policyengine_simulation_executor.precompute_models import DatasetPlanEntry + from policyengine_simulation_executor.precompute import ( # noqa: PLC0415 # Modal function body: loaded in the precompute worker, not at deploy time + build_dataset_impl, + ) + from policyengine_simulation_executor.precompute_models import ( # noqa: PLC0415 # Modal function body: loaded in the precompute worker, not at deploy time + DatasetPlanEntry, + ) return build_dataset_impl( bucket, DatasetPlanEntry.model_validate(expected) @@ -89,8 +95,12 @@ def build_dataset(bucket: str, expected: dict) -> dict: ) def compute_baseline(bucket: str, expected: dict) -> dict: """Compute and upload one cohort's national baseline output (wave 2).""" - from policyengine_simulation_executor.precompute import compute_baseline_impl - from policyengine_simulation_executor.precompute_models import BaselinePlanEntry + from policyengine_simulation_executor.precompute import ( # noqa: PLC0415 # Modal function body: loaded in the precompute worker, not at deploy time + compute_baseline_impl, + ) + from policyengine_simulation_executor.precompute_models import ( # noqa: PLC0415 # Modal function body: loaded in the precompute worker, not at deploy time + BaselinePlanEntry, + ) return compute_baseline_impl( bucket, BaselinePlanEntry.model_validate(expected) @@ -106,8 +116,12 @@ def compute_baseline(bucket: str, expected: dict) -> dict: ) def verify_determinism(bucket: str, expected: dict) -> dict: """Recompute one stored cohort baseline afresh and report columns that differ.""" - from policyengine_simulation_executor.precompute import verify_determinism_impl - from policyengine_simulation_executor.precompute_models import BaselinePlanEntry + from policyengine_simulation_executor.precompute import ( # noqa: PLC0415 # Modal function body: loaded in the precompute worker, not at deploy time + verify_determinism_impl, + ) + from policyengine_simulation_executor.precompute_models import ( # noqa: PLC0415 # Modal function body: loaded in the precompute worker, not at deploy time + BaselinePlanEntry, + ) return verify_determinism_impl( bucket, BaselinePlanEntry.model_validate(expected) @@ -123,8 +137,12 @@ def verify_determinism(bucket: str, expected: dict) -> dict: ) def publish_manifest(bucket: str, plan: dict) -> str: """Write the deploy manifest for ``plan`` to the store and return its digest.""" - from policyengine_simulation_executor.precompute import publish_manifest_impl - from policyengine_simulation_executor.precompute_models import PrecomputePlan + from policyengine_simulation_executor.precompute import ( # noqa: PLC0415 # Modal function body: loaded in the precompute worker, not at deploy time + publish_manifest_impl, + ) + from policyengine_simulation_executor.precompute_models import ( # noqa: PLC0415 # Modal function body: loaded in the precompute worker, not at deploy time + PrecomputePlan, + ) return publish_manifest_impl(bucket, PrecomputePlan.model_validate(plan)) @@ -132,8 +150,12 @@ def publish_manifest(bucket: str, plan: dict) -> str: @app.local_entrypoint() def main(force: bool = False) -> None: """Fill the store for the installed bundle; ``--force`` recomputes stored ones.""" - from policyengine_simulation_executor.artifact_store import resolve_bucket_name - from policyengine_simulation_executor.precompute import run_precompute + from policyengine_simulation_executor.artifact_store import ( # noqa: PLC0415 # local entrypoint: loaded only when the precompute runs + resolve_bucket_name, + ) + from policyengine_simulation_executor.precompute import ( # noqa: PLC0415 # local entrypoint: loaded only when the precompute runs + run_precompute, + ) run_precompute( resolve_bucket_name(), diff --git a/projects/policyengine-simulation-executor/src/modal/segmented_national.py b/projects/policyengine-simulation-executor/src/modal/segmented_national.py index d66b0775a..01efffc28 100644 --- a/projects/policyengine-simulation-executor/src/modal/segmented_national.py +++ b/projects/policyengine-simulation-executor/src/modal/segmented_national.py @@ -251,7 +251,7 @@ def run(self) -> dict[str, Any]: return self._reduce(child_results, country_module=country_module) def _country_module(self) -> ModuleType: - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # resolved per call so tests can patch simulation_runtime._country_module _country_module, ) @@ -540,7 +540,7 @@ def _cancel_all(self, children: list[ChildExecution]) -> None: def _reduce( self, child_results: list[dict], *, country_module: ModuleType ) -> dict[str, Any]: - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # Modal worker code: bound per call, like this module's other simulation_runtime imports _parse_year, ) @@ -600,7 +600,9 @@ def dispatch_run_simulation( Segmented national fan-out serves eligible requests (see ``should_run_segmented_national``); everything else runs monolithically. """ - from policyengine_simulation_executor.spm import normalize_runtime_spm + from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # resolved per call so tests can patch spm.normalize_runtime_spm + normalize_runtime_spm, + ) selection = normalize_runtime_spm(params) if selection is not None: @@ -614,7 +616,7 @@ def dispatch_run_simulation( progress=progress, ) - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # resolved per call so tests can patch simulation_runtime.run_simulation_impl run_simulation_impl, ) diff --git a/projects/policyengine-simulation-executor/src/modal/smoke_app.py b/projects/policyengine-simulation-executor/src/modal/smoke_app.py index 40320881c..36c022349 100644 --- a/projects/policyengine-simulation-executor/src/modal/smoke_app.py +++ b/projects/policyengine-simulation-executor/src/modal/smoke_app.py @@ -60,7 +60,7 @@ def _summarize_bundle_status(status: dict) -> dict: """Reject an incomplete or mismatched runtime bundle and summarize it.""" - import json + import json # noqa: PLC0415 # smoke app: standard-library import kept with the in-image check that uses it package_names = {check.get("package") for check in status.get("packages", [])} dataset_countries = {check.get("country") for check in status.get("datasets", [])} @@ -112,12 +112,16 @@ def smoke_import_executor() -> dict: Raises on any bundle mismatch or import failure; otherwise returns the bundle summary and how many shared-library modules imported. """ - import importlib - import os - import pkgutil - from pathlib import Path + import importlib # noqa: PLC0415 # smoke app: standard-library import kept with the in-image check's other imports + import os # noqa: PLC0415 # smoke app: standard-library import kept with the in-image check's other imports + import pkgutil # noqa: PLC0415 # smoke app: standard-library import kept with the in-image check's other imports + from pathlib import ( # noqa: PLC0415 # smoke app: standard-library import kept with the in-image check's other imports + Path, + ) - from policyengine.bundle import inspect_bundle_status + from policyengine.bundle import ( # noqa: PLC0415 # smoke check: inspects the bundle installed in the Modal image + inspect_bundle_status, + ) bundle = _summarize_bundle_status( inspect_bundle_status( @@ -139,15 +143,15 @@ def smoke_import_executor() -> dict: # policyengine.core chain) on EVERY request, so a break there crashes # all requests at import time — exactly the #602 failure class this # smoke exists to catch. - import policyengine_simulation_contract - import policyengine_simulation_observability - from policyengine_simulation_executor.simulation_runtime import ( # noqa: F401 + import policyengine_simulation_contract # noqa: PLC0415 # smoke check: the import itself, inside the Modal image, is what this verifies + import policyengine_simulation_observability # noqa: PLC0415 # smoke check: the import itself, inside the Modal image, is what this verifies + from policyengine_simulation_executor.simulation_runtime import ( # noqa: F401, PLC0415 # smoke check: the import itself, inside the Modal image, is what this verifies run_simulation_impl, ) - from src.modal.budget_window_batch import ( # noqa: F401 + from src.modal.budget_window_batch import ( # noqa: F401, PLC0415 # smoke check: the import itself, inside the Modal image, is what this verifies run_budget_window_batch_impl, ) - from src.modal.segmented_national import ( # noqa: F401 + from src.modal.segmented_national import ( # noqa: F401, PLC0415 # smoke check: the import itself, inside the Modal image, is what this verifies dispatch_run_simulation, ) diff --git a/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py b/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py index 6f7df7327..9c5ff4641 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py @@ -108,7 +108,7 @@ def _is_newer_version(candidate: str, current: str | None) -> bool: def _country_bundle_metadata(country: str) -> CountryBundleMetadata: - from policyengine_simulation_executor.release_bundle import ( + from policyengine_simulation_executor.release_bundle import ( # noqa: PLC0415 # the executor package loads only when bundle metadata is built get_country_release_bundle, ) @@ -138,7 +138,9 @@ def build_bundle_manifest_metadata( from policyengine_simulation_contract.job_progress import ( JOB_PROGRESS_CAPABILITY_VERSION, ) - from policyengine_simulation_executor.spm import runtime_spm_capability + from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # the executor package loads only when bundle metadata is built + runtime_spm_capability, + ) capability = runtime_spm_capability() metadata: BundleManifestMetadata = { diff --git a/projects/policyengine-simulation-executor/src/modal/v2_app.py b/projects/policyengine-simulation-executor/src/modal/v2_app.py index f7c04b6af..5ff581fb8 100644 --- a/projects/policyengine-simulation-executor/src/modal/v2_app.py +++ b/projects/policyengine-simulation-executor/src/modal/v2_app.py @@ -154,7 +154,7 @@ def build_v2_image(countries: tuple[CountryId, ...]) -> modal.Image: def _validate(country: CountryId) -> dict: - from policyengine_simulation_executor.stage12_worker_validation import ( + from policyengine_simulation_executor.stage12_worker_validation import ( # noqa: PLC0415 # Modal function body: loaded in the worker container, not at deploy time validate_country_worker, ) @@ -206,7 +206,7 @@ def run_single_simulation_us( observability_context: dict | None = None, ) -> dict: """Calculate one planned US simulation and return its artifact descriptor.""" - from policyengine_simulation_executor.stage12_runtime import ( + from policyengine_simulation_executor.stage12_runtime import ( # noqa: PLC0415 # Modal function body: loaded in the worker container, not at deploy time run_single_simulation, ) @@ -252,7 +252,7 @@ def run_single_simulation_uk( observability_context: dict | None = None, ) -> dict: """Calculate one planned UK simulation and return its artifact descriptor.""" - from policyengine_simulation_executor.stage12_runtime import ( + from policyengine_simulation_executor.stage12_runtime import ( # noqa: PLC0415 # Modal function body: loaded in the worker container, not at deploy time run_single_simulation, ) @@ -306,7 +306,7 @@ def coordinate_report( result is compared with that production job's. A report another invocation already claimed returns a ``deduplicated`` marker instead. """ - from policyengine_simulation_executor.stage12_runtime import ( + from policyengine_simulation_executor.stage12_runtime import ( # noqa: PLC0415 # Modal function body: loaded in the worker container, not at deploy time coordinate_report as run_report_coordinator, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py index 645f19026..06365584a 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py @@ -297,7 +297,9 @@ def _receipt_source_sha256(country: str, data_version: str) -> str | None: version (mirroring ``resolve_local_bundle_dataset_path``) — a receipt left over from a different install must not leak into the key. """ - from policyengine_simulation_executor.release_bundle import _receipt_dataset + from policyengine_simulation_executor.release_bundle import ( # noqa: PLC0415 # identity seam: resolved per call so tests can patch release_bundle._receipt_dataset + _receipt_dataset, + ) entry = _receipt_dataset(country) if not isinstance(entry, Mapping): @@ -312,7 +314,9 @@ def _receipt_source_sha256(country: str, data_version: str) -> str | None: def _certification_fingerprint(country: str) -> str | None: - from policyengine.provenance.manifest import get_release_manifest + from policyengine.provenance.manifest import ( # noqa: PLC0415 # identity seam: resolved per call so tests can patch the release manifest + get_release_manifest, + ) certification = getattr(get_release_manifest(country), "certification", None) fingerprint = getattr(certification, "data_build_fingerprint", None) @@ -328,12 +332,12 @@ def collect_dataset_identity(country: str, year: int) -> DatasetArtifactIdentity runtime lookup uses, so filename coupling is inherited, not re-derived. """ - from policyengine.provenance.manifest import ( + from policyengine.provenance.manifest import ( # noqa: PLC0415 # identity seam: resolved per call so tests can patch the release manifest dataset_logical_name, resolve_dataset_reference, ) - from policyengine_simulation_executor.release_bundle import ( + from policyengine_simulation_executor.release_bundle import ( # noqa: PLC0415 # identity seam: resolved per call so tests can patch release_bundle get_country_release_bundle, ) @@ -341,7 +345,9 @@ def collect_dataset_identity(country: str, year: int) -> DatasetArtifactIdentity stem = dataset_logical_name( resolve_dataset_reference(bundle.country, bundle.default_dataset) ) - from policyengine_simulation_executor.spm import normalize_runtime_spm + from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # executor spm is not loaded at import; loaded per call + normalize_runtime_spm, + ) selection = normalize_runtime_spm({"country": country, "time_period": year}) return DatasetArtifactIdentity( @@ -372,7 +378,9 @@ def collect_baseline_identity( ``spm`` is normalized against the installed image's capability first, so a request that omits it still keys to the image's default SPM selection. """ - from policyengine_simulation_executor.spm import normalize_runtime_spm + from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # executor spm is not loaded at import; loaded per call + normalize_runtime_spm, + ) selection = normalize_runtime_spm( {"country": country, "time_period": year, "spm": spm} diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_store.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_store.py index 98476ba85..88cb572e5 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_store.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_store.py @@ -49,9 +49,11 @@ def resolve_bucket_name(explicit: str | None = None) -> str: def _default_client() -> storage.Client: - from google.cloud import storage + from google.cloud import ( # noqa: PLC0415 # google-cloud-storage loads only when a default client is built + storage, + ) - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # resolved per call so tests can patch simulation_runtime.setup_gcp_credentials setup_gcp_credentials, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py index 3ca1dff08..2b9f4fbf3 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py @@ -23,6 +23,8 @@ import logging from typing import TYPE_CHECKING, Any +from policyengine.core.scoping_strategy import RegionGroupStrategy +from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail from pydantic import PrivateAttr from policyengine_simulation_executor import artifact_keys @@ -73,8 +75,6 @@ def qualifying_baseline_identity( reader cannot disagree by construction. Identity-collection errors propagate: the writer must fail loudly on them. """ - from policyengine.core.scoping_strategy import RegionGroupStrategy - if policy is not None: return None if str(params.get("scope") or "").lower() != "macro": @@ -132,8 +132,6 @@ def deterministic_baseline_id( year=year, ) except Exception as exc: - from policyengine_simulation_contract.spm import spm_error_detail - if spm_error_detail(exc): raise logger.warning( @@ -194,9 +192,9 @@ def ensure(self) -> None: self._record_outcome(OUTCOME_MISS) return - from policyengine_simulation_contract.spm import SPMInputError - - from policyengine_simulation_executor.spm import simulation_spm_result + from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # executor spm is not loaded at import; loaded per call + simulation_spm_result, + ) # A tax-only artifact can legitimately have no receipt years. Check its # columns first, then treat unusable cached receipts as another gap. @@ -230,7 +228,9 @@ def ensure(self) -> None: # completed output instead of revalidating (and re-running) against # the incomplete one. LRUCache.add keeps the OLD value for an # existing key and exposes no remove, so evict directly first. - from policyengine.core.simulation import _cache + from policyengine.core.simulation import ( # noqa: PLC0415 # resolved per call so tests can patch policyengine's simulation cache + _cache, + ) _cache._cache.pop(getattr(self, "storage_id", self.id), None) # noqa: SLF001 # policyengine's LRUCache has no public remove _cache.add( diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/hf_access_validation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/hf_access_validation.py index d08f89188..5ccd6d5d9 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/hf_access_validation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/hf_access_validation.py @@ -138,7 +138,9 @@ def build_uk_bundle_access_plan() -> tuple[HFAccessTarget, ...]: def _load_whoami(token: str) -> Mapping[str, object]: - from huggingface_hub import HfApi + from huggingface_hub import ( # noqa: PLC0415 # huggingface_hub stays out of this module's import-time imports + HfApi, + ) return HfApi().whoami(token=token) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py index 9b147d908..95d9ec6fa 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py @@ -34,6 +34,7 @@ import logging import time +from pathlib import Path from typing import TYPE_CHECKING, Any from policyengine_simulation_executor.artifact_keys import ( @@ -56,7 +57,6 @@ if TYPE_CHECKING: from collections.abc import Callable - from pathlib import Path from types import ModuleType from policyengine.core import Simulation @@ -103,7 +103,7 @@ def _cohort_resolution( resolution or the in-container id checks abort the run; sharing this prologue removes the drift surface. """ - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # resolved per call so tests can patch simulation_runtime helpers _country_module, _resolve_region, ) @@ -120,10 +120,10 @@ def _cohort_resolution( def cohort_identity(year: int, group: list[str]) -> BaselineArtifactIdentity: """Identity via the executor's own resolution path (writer==reader).""" - from policyengine_simulation_executor.baseline_artifacts import ( + from policyengine_simulation_executor.baseline_artifacts import ( # noqa: PLC0415 # resolved per call so tests can patch baseline_artifacts qualifying_baseline_identity, ) - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # resolved per call so tests can patch simulation_runtime helpers _resolve_dataset_selection, ) @@ -187,14 +187,16 @@ def build_manifest(plan: PrecomputePlan) -> ArtifactManifest: def plan_artifacts_impl(bucket: str) -> PrecomputePlan: """Compute every expected artifact identity and its store presence.""" - from policyengine_simulation_executor.artifact_keys import ( + from policyengine_simulation_executor.artifact_keys import ( # noqa: PLC0415 # resolved per call so tests can patch artifact_keys.collect_dataset_identity collect_dataset_identity, ) - from policyengine_simulation_executor.artifact_store import ArtifactStore - from policyengine_simulation_executor.national_partition import ( + from policyengine_simulation_executor.artifact_store import ( # noqa: PLC0415 # resolved per call so tests can patch artifact_store.ArtifactStore + ArtifactStore, + ) + from policyengine_simulation_executor.national_partition import ( # noqa: PLC0415 # resolved per call so tests can patch national_partition.national_region_groups national_region_groups, ) - from policyengine_simulation_executor.release_bundle import ( + from policyengine_simulation_executor.release_bundle import ( # noqa: PLC0415 # resolved per call so tests can patch release_bundle.get_country_release_bundle get_country_release_bundle, ) @@ -244,16 +246,16 @@ def plan_artifacts_impl(bucket: str) -> PrecomputePlan: def build_dataset_impl(bucket: str, expected: DatasetPlanEntry) -> DatasetBuildResult: """Wave 1: build one single-year dataset and upload it.""" - from pathlib import Path - - from policyengine_simulation_executor.artifact_keys import ( + from policyengine_simulation_executor.artifact_keys import ( # noqa: PLC0415 # resolved per call so tests can patch artifact_keys.collect_dataset_identity collect_dataset_identity, ) - from policyengine_simulation_executor.artifact_store import ArtifactStore - from policyengine_simulation_executor.release_bundle import ( + from policyengine_simulation_executor.artifact_store import ( # noqa: PLC0415 # resolved per call so tests can patch artifact_store.ArtifactStore + ArtifactStore, + ) + from policyengine_simulation_executor.release_bundle import ( # noqa: PLC0415 # resolved per call so tests can patch release_bundle.get_country_release_bundle get_country_release_bundle, ) - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # resolved per call so tests can patch simulation_runtime helpers _country_module, resolve_data_folder, ) @@ -295,13 +297,13 @@ def _prepare_cohort_baseline( bucket: str, expected: BaselinePlanEntry ) -> tuple[ArtifactStore, Path, Simulation]: """Shared by compute and verify: dataset download + simulation build.""" - from pathlib import Path - - from policyengine_simulation_executor.artifact_keys import ( + from policyengine_simulation_executor.artifact_keys import ( # noqa: PLC0415 # resolved per call so tests can patch artifact_keys.collect_dataset_identity collect_dataset_identity, ) - from policyengine_simulation_executor.artifact_store import ArtifactStore - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.artifact_store import ( # noqa: PLC0415 # resolved per call so tests can patch artifact_store.ArtifactStore + ArtifactStore, + ) + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # resolved per call so tests can patch simulation_runtime helpers _build_simulation, _load_dataset, _resolve_dataset_selection, @@ -351,7 +353,7 @@ def _prepare_cohort_baseline( # The extras economic_impact_analysis applies unconditionally before # ensure(); the artifact must carry them or every request would fail # the column guard and recompute. - from policyengine.tax_benefit_models.us.analysis import ( + from policyengine.tax_benefit_models.us.analysis import ( # noqa: PLC0415 # policyengine's US model loads only for a US cohort baseline configure_budgetary_impact_variables, ) @@ -363,7 +365,7 @@ def compute_baseline_impl( bucket: str, expected: BaselinePlanEntry ) -> BaselineComputeResult: """Wave 2: compute one cohort baseline and upload its output.""" - from policyengine_simulation_executor.baseline_artifacts import ( + from policyengine_simulation_executor.baseline_artifacts import ( # noqa: PLC0415 # resolved per call so tests can patch baseline_artifacts.ArtifactBaselineSimulation ArtifactBaselineSimulation, ) @@ -409,9 +411,13 @@ def verify_determinism_impl( deliberately not used (HDF5 container metadata is not guaranteed stable). """ - from policyengine.core import Simulation + from policyengine.core import ( # noqa: PLC0415 # resolved per call so tests can patch policyengine.core.Simulation + Simulation, + ) - from policyengine_simulation_executor.baseline_artifacts import OUTCOME_HIT + from policyengine_simulation_executor.baseline_artifacts import ( # noqa: PLC0415 # baseline_artifacts is not loaded at import; loaded per call + OUTCOME_HIT, + ) store, data_folder, baseline = _prepare_cohort_baseline(bucket, expected) artifact_file = data_folder / baseline_artifact_filename( @@ -459,7 +465,9 @@ def verify_determinism_impl( def publish_manifest_impl(bucket: str, plan: PrecomputePlan) -> str: """Store the plan's manifest write-once and return its content digest.""" - from policyengine_simulation_executor.artifact_store import ArtifactStore + from policyengine_simulation_executor.artifact_store import ( # noqa: PLC0415 # resolved per call so tests can patch artifact_store.ArtifactStore + ArtifactStore, + ) manifest = build_manifest(plan) return ArtifactStore(bucket).write_manifest(manifest.canonical_payload()) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/release_bundle.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/release_bundle.py index 2c2aadf7c..ca5f794a4 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/release_bundle.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/release_bundle.py @@ -145,7 +145,9 @@ def _artifact_hf_uri( data_package: DataPackageVersion, reference: object, ) -> tuple[str | None, str | None]: - from policyengine.provenance.manifest import build_hf_uri + from policyengine.provenance.manifest import ( # noqa: PLC0415 # policyengine stays out of this module's import-time imports + build_hf_uri, + ) path = _reference_value(reference, "path") if not isinstance(path, str) or not path: @@ -188,7 +190,9 @@ def _dataset_uris_from_release( data_release: Mapping, fallback_data_package: DataPackageVersion, ) -> tuple[dict[str, str], dict[str, str]]: - from policyengine.provenance.manifest import build_hf_uri + from policyengine.provenance.manifest import ( # noqa: PLC0415 # policyengine stays out of this module's import-time imports + build_hf_uri, + ) data_package = _mapping(data_release.get("data_package")) datasets = _mapping(data_release.get("datasets")) @@ -227,7 +231,9 @@ def _dataset_uris_from_release( def _current_policyengine_bundle() -> Mapping | None: try: - from policyengine.bundle import get_current_bundle + from policyengine.bundle import ( # noqa: PLC0415 # inside the fallback guard: a failing policyengine.bundle import returns None + get_current_bundle, + ) return get_current_bundle() except Exception: # noqa: BLE001 # preserve the fallback for invalid legacy bundles @@ -300,7 +306,9 @@ def _derive_data_package_version( def get_country_release_bundle(country: str) -> CountryReleaseBundle: """Return package and dataset versions from policyengine.py metadata.""" country = _normalise_country(country) - from policyengine.provenance.manifest import get_release_manifest + from policyengine.provenance.manifest import ( # noqa: PLC0415 # resolved per call so tests can patch the release manifest + get_release_manifest, + ) manifest = get_release_manifest(country) bundle_metadata = _bundle_country_metadata(country) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/segmented_national_reduce.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/segmented_national_reduce.py index 669897178..8a75b99ef 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/segmented_national_reduce.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/segmented_national_reduce.py @@ -99,7 +99,7 @@ def _us_dataset_from_frames( Weights need no special handling: they ride as ``_weight`` columns and ``MicroDataFrame(df, weights=...)`` re-derives them. """ - from policyengine.tax_benefit_models.us.datasets import ( + from policyengine.tax_benefit_models.us.datasets import ( # noqa: PLC0415 # policyengine's US model loads only when frames are rebuilt PolicyEngineUSDataset, USYearData, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py index f14b24a6e..4531b9542 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py @@ -8,6 +8,8 @@ from importlib import import_module from typing import TYPE_CHECKING, Any +from policyengine_simulation_contract.spm import spm_error_detail + if TYPE_CHECKING: import pandas as pd from _typeshed import ConvertibleToFloat @@ -76,7 +78,10 @@ def _sum_output_variable(simulation: Simulation, variable: str, entity: str) -> if variable in data.columns: return float(data[variable].sum()) - from policyengine.outputs import Aggregate, AggregateType + from policyengine.outputs import ( # noqa: PLC0415 # policyengine stays out of this module's import-time imports + Aggregate, + AggregateType, + ) output = Aggregate( simulation=simulation, @@ -96,7 +101,10 @@ def _change_output_variable( if variable in baseline_data.columns and variable in reform_data.columns: return float((reform_data[variable] - baseline_data[variable]).sum()) - from policyengine.outputs import ChangeAggregate, ChangeAggregateType + from policyengine.outputs import ( # noqa: PLC0415 # policyengine stays out of this module's import-time imports + ChangeAggregate, + ChangeAggregateType, + ) output = ChangeAggregate( baseline_simulation=baseline, @@ -124,8 +132,6 @@ def _try_compute_output[**P, T]( try: return fn(*args, **kwargs) except Exception as exc: - from policyengine_simulation_contract.spm import spm_error_detail - if spm_error_detail(exc) is not None: raise logger.warning("Unable to calculate %s", label, exc_info=True) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_distribution.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_distribution.py index 50bd794eb..83af46d83 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_distribution.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_distribution.py @@ -61,7 +61,7 @@ def build_intra_decile_output( A calculation failure is logged and yields empty deciles with zero overall shares rather than failing the request; SPM selection errors propagate. """ - from policyengine.outputs.intra_decile_impact import ( + from policyengine.outputs.intra_decile_impact import ( # noqa: PLC0415 # policyengine stays out of this module's import-time imports compute_intra_decile_impacts, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py index f7c97449c..cff991ccf 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py @@ -31,11 +31,11 @@ def _required_uk_geography_lookup_csv_path(spec: UKGeographyAssetSpec) -> str: """Resolve one required lookup under the runtime GCP identity.""" - from policyengine.outputs.uk_geography_impact import ( + from policyengine.outputs.uk_geography_impact import ( # noqa: PLC0415 # resolved per call so tests can patch policyengine's lookup resolver resolve_uk_geography_lookup_csv_path, ) - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # import cycle: simulation_runtime imports this module through its output builder setup_gcp_credentials, ) @@ -82,7 +82,10 @@ def _complete_uk_geography_output( @lru_cache(maxsize=1) def _policyengine_us_district_metadata() -> tuple[dict[int, str], frozenset[str]]: - from policyengine.countries.us.data import AT_LARGE_STATES, US_STATE_FIPS + from policyengine.countries.us.data import ( # noqa: PLC0415 # policyengine's US data loads only for US district output + AT_LARGE_STATES, + US_STATE_FIPS, + ) return ( { @@ -219,7 +222,7 @@ def build_congressional_district_impact( if not _should_build_us_congressional_district_impact(region_code): return None - from policyengine.outputs.congressional_district_impact import ( + from policyengine.outputs.congressional_district_impact import ( # noqa: PLC0415 # bound per call, like this module's patched policyengine.outputs import above compute_us_congressional_district_impacts, ) @@ -305,11 +308,11 @@ def build_uk_local_authority_impact( msg = "UK local-authority boundary metadata is required" raise ValueError(msg) - from policyengine.outputs.uk_geography_impact import ( + from policyengine.outputs.uk_geography_impact import ( # noqa: PLC0415 # bound per call, like this module's patched policyengine.outputs import above compute_longwise_uk_geography_impacts, ) - from policyengine_simulation_executor.uk_local_authority_metadata import ( + from policyengine_simulation_executor.uk_local_authority_metadata import ( # noqa: PLC0415 # the UK metadata module loads only for UK local-authority output load_uk_local_authority_resources, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py index aac927130..4b35a3052 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py @@ -17,10 +17,12 @@ from dataclasses import dataclass from datetime import datetime from importlib import import_module +from pathlib import Path from typing import TYPE_CHECKING, Any from policyengine_simulation_contract.dataset_uri import runtime_dataset_uri from policyengine_simulation_contract.job_progress import JobStage +from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail from policyengine_simulation_observability.stages import ( ANNUAL_IMPACT_STAGES, Stage, @@ -189,11 +191,6 @@ def run_simulation_impl( except ValueError as exc: # Gateway images intentionally do not install country packages. # Transport the public error through the shared contract instead. - from policyengine_simulation_contract.spm import ( - SPMInputError, - spm_error_detail, - ) - detail = spm_error_detail(exc) if detail is not None: raise SPMInputError(detail.code, detail.message) from None @@ -263,9 +260,16 @@ def _compile_interval_policy( ): return policy - from policyengine.core import ParameterValue, Policy - from policyengine.core.parameter import Parameter - from policyengine.tax_benefit_models.common.reform import compile_reform + from policyengine.core import ( # noqa: PLC0415 # this module keeps policyengine out of its module-level imports (see its docstring) + ParameterValue, + Policy, + ) + from policyengine.core.parameter import ( # noqa: PLC0415 # this module keeps policyengine out of its module-level imports (see its docstring) + Parameter, + ) + from policyengine.tax_benefit_models.common.reform import ( # noqa: PLC0415 # this module keeps policyengine out of its module-level imports (see its docstring) + compile_reform, + ) compiled = compile_reform(policy, year=year, model_version=country_module.model) parameter_values = [] @@ -346,9 +350,13 @@ def _build_uk_weight_replacement_region(region_code: str) -> Region | None: if prefix not in {"constituency", "local_authority"}: return None - from policyengine.core.region import Region - from policyengine.core.scoping_strategy import WeightReplacementStrategy - from policyengine.data.uk_geography_assets import ( + from policyengine.core.region import ( # noqa: PLC0415 # this module keeps policyengine out of its module-level imports (see its docstring) + Region, + ) + from policyengine.core.scoping_strategy import ( # noqa: PLC0415 # this module keeps policyengine out of its module-level imports (see its docstring) + WeightReplacementStrategy, + ) + from policyengine.data.uk_geography_assets import ( # noqa: PLC0415 # this module keeps policyengine out of its module-level imports (see its docstring) CONSTITUENCY_ASSET_SPEC, LOCAL_AUTHORITY_ASSET_SPEC, ) @@ -395,24 +403,30 @@ def _require_certified_uk_weight_matrix( release replaces, is never downloaded (see ``_build_uk_weight_replacement_region``). """ - from policyengine.core.scoping_strategy import WeightReplacementStrategy + from policyengine.core.scoping_strategy import ( # noqa: PLC0415 # this module keeps policyengine out of its module-level imports (see its docstring) + WeightReplacementStrategy, + ) if not isinstance(scoping_strategy, WeightReplacementStrategy): return - from pathlib import Path - - import h5py - import pandas as pd - from policyengine.data.uk_geography_assets import ( + import h5py # noqa: PLC0415 # h5py loads only for the UK weight-matrix check + import pandas as pd # noqa: PLC0415 # UK weight-matrix check: kept with its lazily loaded data dependencies + from policyengine.data.uk_geography_assets import ( # noqa: PLC0415 # this module keeps policyengine out of its module-level imports (see its docstring) UKGeographyAssetSpec, default_download_dir, resolve_uk_geography_asset_paths, ) - from policyengine.provenance.dataset_materialization import materialize_dataset - from policyengine.provenance.manifest import get_release_manifest + from policyengine.provenance.dataset_materialization import ( # noqa: PLC0415 # resolved per call so tests can patch materialize_dataset + materialize_dataset, + ) + from policyengine.provenance.manifest import ( # noqa: PLC0415 # resolved per call so tests can patch the release manifest + get_release_manifest, + ) - from policyengine_simulation_executor import simulation_output_geographic + from policyengine_simulation_executor import ( # noqa: PLC0415 # simulation_output_geographic imports this module back lazily; kept lazy on both sides + simulation_output_geographic, + ) region = scoping_strategy.region_code matrix_name = scoping_strategy.weight_matrix_key @@ -595,7 +609,7 @@ def _resolve_region_group( Members must be row-filter regions (states/CDs) so national weights are preserved (INV-5); the group filters the national dataset. """ - from policyengine.core.scoping_strategy import ( + from policyengine.core.scoping_strategy import ( # noqa: PLC0415 # this module keeps policyengine out of its module-level imports (see its docstring) RegionGroupStrategy, RowFilterStrategy, ) @@ -730,11 +744,13 @@ def _build_simulation( scoping_strategy: ScopingStrategy | None = None, region_code: str | None = None, ) -> Simulation: - from policyengine_simulation_executor.baseline_artifacts import ( + from policyengine_simulation_executor.baseline_artifacts import ( # noqa: PLC0415 # resolved per call so tests can patch baseline_artifacts ArtifactBaselineSimulation, deterministic_baseline_id, ) - from policyengine_simulation_executor.spm import normalize_runtime_spm + from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # resolved per call so tests can patch spm.normalize_runtime_spm + normalize_runtime_spm, + ) from policyengine_simulation_executor.stage_reporting_simulation import ( StageReportingSimulation, ) @@ -792,7 +808,7 @@ def _run_simulation_impl_core( with runtime.span(ANNUAL_IMPACT_STAGES.name(Stage.REQUEST_PARSE)): simulation_params, telemetry, metadata = split_internal_payload(params) metadata = metadata or {} - from policyengine_simulation_executor.spm import ( + from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # resolved per call so tests can patch spm.normalize_runtime_spm normalize_runtime_spm, simulation_spm_result, ) @@ -837,7 +853,7 @@ def _run_simulation_impl_core( selection=dataset_selection, country_module=country_module, ) - from policyengine_simulation_executor.uk_local_authority_metadata import ( + from policyengine_simulation_executor.uk_local_authority_metadata import ( # noqa: PLC0415 # the UK metadata module is not loaded at import; loaded per call detect_uk_local_authority_metadata, ) @@ -908,7 +924,7 @@ def _run_simulation_impl_core( logger.info("Baseline artifact outcome: %s", artifact_outcome) logger.info("Comparison complete") if params.get("_emit_microdata"): - from policyengine_simulation_executor.simulation_microdata import ( + from policyengine_simulation_executor.simulation_microdata import ( # noqa: PLC0415 # the microdata export loads only when a run asks for it extract_output_microdata, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py index 62ab5532a..881369879 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py @@ -26,7 +26,9 @@ @lru_cache(maxsize=4) def _forecast(expected_sha256: str | None) -> SPMForecast: - from spm_calculator import load_forecast + from spm_calculator import ( # noqa: PLC0415 # spm_calculator loads only when a forecast is needed + load_forecast, + ) return load_forecast(expected_sha256=expected_sha256) @@ -51,8 +53,12 @@ def runtime_spm_capability() -> SPMCapability | None: already succeeded. Tests that stub the installed environment must call :func:`reset_spm_runtime_caches` (the executor suite does, automatically). """ - from policyengine.bundle import get_current_bundle - from policyengine.core import Simulation + from policyengine.bundle import ( # noqa: PLC0415 # resolved per call so tests can patch policyengine.bundle.get_current_bundle + get_current_bundle, + ) + from policyengine.core import ( # noqa: PLC0415 # resolved per call so tests can patch policyengine.core.Simulation + Simulation, + ) try: bundle = get_current_bundle() @@ -86,7 +92,9 @@ def _certified_us_spm_capability(configured: object) -> SPMCapability: Raises ``ImportError``, ``ValueError``, ``TypeError`` or ``OSError`` when the US model, the configuration or the pinned forecast cannot support them. """ - from policyengine_us import Microsimulation + from policyengine_us import ( # noqa: PLC0415 # the US model loads only to certify the SPM capability + Microsimulation, + ) if not ( hasattr(Microsimulation, "spm_config") @@ -122,7 +130,9 @@ def _prevalidate_selection( an exception -- so a rejected selection is rejected again, with the same typed error, every time it is asked for. """ - from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider + from spm_calculator.policyengine_adapter import ( # noqa: PLC0415 # spm_calculator loads only when a selection is prevalidated + PolicyEngineSPMProvider, + ) selection = json.loads(selection_json) forecast = _forecast(selection["forecast_content_sha256"]) @@ -182,7 +192,7 @@ def reset_spm_runtime_caches() -> None: def normalize_runtime_spm(params: dict[str, Any]) -> dict[str, Any] | None: """Resolve before dataset loading, artifact lookup, or child submission.""" - from policyengine_simulation_executor.release_bundle import ( + from policyengine_simulation_executor.release_bundle import ( # noqa: PLC0415 # resolved per call so tests can patch release_bundle.get_country_release_bundle get_country_release_bundle, ) @@ -198,7 +208,9 @@ def normalize_runtime_spm(params: dict[str, Any]) -> dict[str, Any] | None: if selection is not None: canonical = json.dumps(selection, sort_keys=True, separators=(",", ":")) try: - from policyengine_simulation_executor.simulation_runtime import _parse_year + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # simulation_runtime imports this module lazily; kept lazy on both sides + _parse_year, + ) try: start = int(params.get("start_year") or _parse_year(params)) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_bundle/manifest.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_bundle/manifest.py index 15c0d4356..20c06e6c5 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_bundle/manifest.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_bundle/manifest.py @@ -122,6 +122,8 @@ def resolve_stage12_bundle(raw_bundle: object) -> ResolvedStage12Bundle: def load_stage12_bundle() -> ResolvedStage12Bundle: """Read the installed PolicyEngine.py release's packaged bundle.""" - from policyengine.bundle import get_current_bundle + from policyengine.bundle import ( # noqa: PLC0415 # resolved per call so tests can patch policyengine.bundle.get_current_bundle + get_current_bundle, + ) return resolve_stage12_bundle(get_current_bundle()) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_qualification.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_qualification.py index 00c43929b..ca02627f3 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_qualification.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_qualification.py @@ -41,6 +41,8 @@ ) from policyengine_simulation_executor.stage12_runtime import ( SimulationCalculation, + build_aggregate_report, + calculate_simulation_frames, simulation_input_sha256, ) from policyengine_simulation_executor.stage12_runtime.aggregation import ( @@ -237,7 +239,7 @@ def qualify_report_parity( local_runtime: ObservabilityRuntime | None = None if existing_runner is None: - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # default runner, bound only when none is injected run_simulation_impl, ) @@ -249,16 +251,8 @@ def run_existing(request: dict[str, Any]) -> Mapping[str, Any]: existing_runner = run_existing if single_simulation_runner is None: - from policyengine_simulation_executor.stage12_runtime import ( - calculate_simulation_frames, - ) - single_simulation_runner = calculate_simulation_frames if aggregate_builder is None: - from policyengine_simulation_executor.stage12_runtime import ( - build_aggregate_report, - ) - aggregate_builder = build_aggregate_report request_with_microdata = dict(existing_request) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py index 90932edb1..e6e1b7281 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any +from policyengine_simulation_contract.spm import build_spm_comparison_provenance from policyengine_simulation_contract.stage12_execution import ( ReportExecutionInput, SimulationArtifactDescriptor, @@ -38,7 +39,7 @@ def validate_uk_local_authority_metadata( """ if country == "uk": if baseline is None and reform is None: - from policyengine_simulation_executor.uk_local_authority_metadata import ( + from policyengine_simulation_executor.uk_local_authority_metadata import ( # noqa: PLC0415 # UK-only branch, reached only when a UK report has no metadata uk_area_codes_required, ) @@ -103,7 +104,9 @@ def _dataset_from_frames( country: CountryId, year: int, ) -> PolicyEngineUSDataset | PolicyEngineUKDataset: - from microdf import MicroDataFrame + from microdf import ( # noqa: PLC0415 # dataset rebuild: kept with the country dataset classes below, which load lazily + MicroDataFrame, + ) fields = {} for entity, frame in frames.items(): @@ -114,7 +117,7 @@ def _dataset_from_frames( else MicroDataFrame(frame) ) if country == "us": - from policyengine.tax_benefit_models.us.datasets import ( + from policyengine.tax_benefit_models.us.datasets import ( # noqa: PLC0415 # policyengine's US model loads only for a US report PolicyEngineUSDataset, USYearData, ) @@ -126,7 +129,7 @@ def _dataset_from_frames( name="stage12-comparison", description="Retained Stage 12 output", ) - from policyengine.tax_benefit_models.uk.datasets import ( + from policyengine.tax_benefit_models.uk.datasets import ( # noqa: PLC0415 # policyengine's UK model loads only for a UK report PolicyEngineUKDataset, UKYearData, ) @@ -154,13 +157,13 @@ def build_aggregate_report( Runs the standard macro output builder over stand-in simulations that load the stored frames, so no model is recomputed. """ - from policyengine_simulation_executor.segmented_national_reduce import ( + from policyengine_simulation_executor.segmented_national_reduce import ( # noqa: PLC0415 # the segmented reduce module loads only when a report is aggregated PrecomputedSimulation, ) - from policyengine_simulation_executor.simulation_output_builder import ( + from policyengine_simulation_executor.simulation_output_builder import ( # noqa: PLC0415 # resolved per call so tests can patch simulation_output_builder.SimulationOutputBuilder SimulationOutputBuilder, ) - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # resolved per call so tests can patch simulation_runtime._country_module _country_module, _runtime_policy, ) @@ -247,9 +250,6 @@ def build_spm_result( requested_spm = report.baseline.options.get("spm") if requested_spm is None: return {} - from policyengine_simulation_contract.spm import ( - build_spm_comparison_provenance, - ) baseline_provenance = baseline_descriptor.calculation_provenance reform_provenance = reform_descriptor.calculation_provenance diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/dependencies.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/dependencies.py index bbac965d2..553ba4c76 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/dependencies.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/dependencies.py @@ -4,6 +4,7 @@ import os from functools import partial +from importlib import import_module from typing import TYPE_CHECKING, Any, Protocol from policyengine_stage12_persistence import Stage12PersistenceStore @@ -130,8 +131,6 @@ def spawn( observability_context: dict[str, Any] | None, ) -> ChildCall: """Look up ``application_name.function_name`` in ``environment``; spawn it.""" - from importlib import import_module - modal: Any = import_module("modal") function = modal.Function.from_name( application_name, @@ -146,8 +145,6 @@ def spawn( def restore(self, invocation_id: str) -> ChildCall: """Return the ``modal.FunctionCall`` with this id.""" - from importlib import import_module - modal: Any = import_module("modal") return modal.FunctionCall.from_id(invocation_id) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py index 8c067732f..a17efc9c0 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py @@ -58,10 +58,14 @@ def _unmonitored_stage_scope( def _country_model(country: CountryId) -> OutputVariableModel: if country == "us": - from policyengine.tax_benefit_models import us + from policyengine.tax_benefit_models import ( # noqa: PLC0415 # policyengine's US model loads only for a US simulation + us, + ) return us.model - from policyengine.tax_benefit_models import uk + from policyengine.tax_benefit_models import ( # noqa: PLC0415 # policyengine's UK model loads only for a UK simulation + uk, + ) return uk.model @@ -129,7 +133,7 @@ def _configure_country_outputs( if include_cliff_impacts: configure_cliff_impact_variables(baseline, reform) if country == "us": - from policyengine.tax_benefit_models.us.analysis import ( + from policyengine.tax_benefit_models.us.analysis import ( # noqa: PLC0415 # policyengine's US model loads only for a US simulation configure_budgetary_impact_variables, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py index 20f2874c1..89c3fb4d1 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py @@ -11,9 +11,12 @@ import pandas as pd from policyengine_simulation_contract.stage12_execution import ( + ArtifactMediaType, + ArtifactReference, ComparisonRunLifecycleStatus, ComparisonSimulationRecord, PlannedSimulationExecutionInput, + RowIdentity, SimulationArtifactDescriptor, SimulationExecutionInput, Stage12InvocationContext, @@ -153,7 +156,7 @@ def calculate_simulation_frames( "region": simulation.geography.region, **simulation.options, } - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # resolved per call so tests can patch simulation_runtime helpers _build_simulation, _country_module, _load_dataset, @@ -219,7 +222,7 @@ def calculate_simulation_frames( selection = getattr(model, "spm_config", None) calculation_provenance = None if selection is not None: - from policyengine_simulation_executor.spm import ( + from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # executor spm loads only for an SPM selection simulation_spm_calculation_provenance, ) @@ -259,11 +262,6 @@ def descriptor_from_record( ): msg = "successful child record has incomplete output metadata" raise ValueError(msg) - from policyengine_simulation_contract.stage12_execution import ( - ArtifactMediaType, - ArtifactReference, - RowIdentity, - ) return SimulationArtifactDescriptor( evaluation_id=simulation.evaluation_id, diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py index 37bb6a310..8bd9d0926 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_worker_validation.py @@ -72,7 +72,7 @@ def _run_non_serving_calculation(country: CountryId) -> None: def _check_uk_local_authority_resources() -> None: - from policyengine_simulation_executor.uk_local_authority_metadata import ( + from policyengine_simulation_executor.uk_local_authority_metadata import ( # noqa: PLC0415 # the UK metadata module is not loaded at import; loaded per call load_uk_local_authority_resources, ) @@ -82,7 +82,7 @@ def _check_uk_local_authority_resources() -> None: def _check_uk_local_authority_dataset(dataset_path: str) -> None: - from policyengine_simulation_executor.uk_local_authority_metadata import ( + from policyengine_simulation_executor.uk_local_authority_metadata import ( # noqa: PLC0415 # resolved per call so tests can patch uk_local_authority_metadata detect_uk_local_authority_metadata_from_hdf, uk_area_regions_routed, uk_hdf_household_has_local_authority_codes, diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py index 90e49a4f4..a01b45c65 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py @@ -293,7 +293,9 @@ def uk_area_regions_routed() -> bool: region types have one may the UK default, a national file, carry no area codes: their runs and breakdowns then belong to that local-area dataset. """ - from policyengine.provenance.manifest import get_release_manifest + from policyengine.provenance.manifest import ( # noqa: PLC0415 # resolved per call so tests can patch the release manifest + get_release_manifest, + ) region_datasets = get_release_manifest("uk").region_datasets return all(region_type in region_datasets for region_type in UK_AREA_REGION_TYPES) From 3c78d7a1e9840bd70dff2bea93f60e29dcf295aa Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:33:29 +0400 Subject: [PATCH 09/23] Wrap long lines in the shared libraries (ruff E501) Split long messages and the GCP log format with implicit string concatenation, so every string is unchanged, and rewrap two docstrings that no published schema uses. Co-Authored-By: Claude Opus 5.5 --- .../src/policyengine_fastapi/observability/__init__.py | 2 +- .../src/policyengine_fastapi/opentelemetry/gcp.py | 3 ++- .../src/policyengine_simulation_contract/spm.py | 5 ++++- .../tests/test_dataset_uri.py | 5 ++++- .../src/policyengine_stage12_persistence/store.py | 5 ++++- .../src/policyengine_stage12_persistence/validators.py | 5 ++++- 6 files changed, 19 insertions(+), 6 deletions(-) diff --git a/libs/policyengine-fastapi/src/policyengine_fastapi/observability/__init__.py b/libs/policyengine-fastapi/src/policyengine_fastapi/observability/__init__.py index 15f67934a..ca463b260 100644 --- a/libs/policyengine-fastapi/src/policyengine_fastapi/observability/__init__.py +++ b/libs/policyengine-fastapi/src/policyengine_fastapi/observability/__init__.py @@ -1,4 +1,4 @@ -"""Shared observability contracts and utilities for the PolicyEngine simulation API services. +"""Observability contracts and utilities shared by the PolicyEngine simulation APIs. This stays in `policyengine-fastapi` for now because only one active service needs it and the main value today is shared contract stability, not a separate diff --git a/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py b/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py index c7d83da44..109ce9583 100644 --- a/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py +++ b/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py @@ -102,7 +102,8 @@ def instrument(self) -> None: log_handler = logging.StreamHandler() formatter = JsonFormatter( - "%(asctime)s %(levelname)s %(message)s %(otelTraceID)s %(otelSpanID)s %(otelTraceSampled)s", + "%(asctime)s %(levelname)s %(message)s " + "%(otelTraceID)s %(otelSpanID)s %(otelTraceSampled)s", rename_fields={ "levelname": "severity", "asctime": "timestamp", diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py index bf47aadc3..e7f04efa4 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py @@ -230,7 +230,10 @@ class SPMRuntimeVersions(BaseModel): def _require_complete_selection(selection: SPMSelection) -> tuple[str, str]: - """Return a complete selection's artifact hash and scenario, else raise ValueError.""" + """Return a complete selection's artifact hash and scenario. + + Raise ValueError if the selection is incomplete. + """ required = set(SPMSelection.model_fields) missing = required.difference(selection.model_fields_set) if missing: diff --git a/libs/policyengine-simulation-contract/tests/test_dataset_uri.py b/libs/policyengine-simulation-contract/tests/test_dataset_uri.py index e06019764..7e4f2e4e9 100644 --- a/libs/policyengine-simulation-contract/tests/test_dataset_uri.py +++ b/libs/policyengine-simulation-contract/tests/test_dataset_uri.py @@ -12,7 +12,10 @@ def test_runtime_dataset_uri_preserves_populace_build_revision_when_package_diff ) def reject_hf_validation(dataset_uri: str, revision: str) -> str: - msg = f"HF validation should not run for trusted bundle data: {dataset_uri}@{revision}" + msg = ( + "HF validation should not run for trusted bundle data: " + f"{dataset_uri}@{revision}" + ) raise AssertionError(msg) monkeypatch.setattr( diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py index df8658561..ab7a9db8b 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/store.py @@ -454,7 +454,10 @@ def create_or_resolve_simulation( .one_or_none() ) if existing is None: - msg = "comparison simulation conflict did not resolve to an existing identity" + msg = ( + "comparison simulation conflict did not resolve to an " + "existing identity" + ) raise ValueError(msg) resolved = _simulation(existing) require_simulation_conflict_matches(resolved, record) diff --git a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/validators.py b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/validators.py index 1d60019d7..206f075da 100644 --- a/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/validators.py +++ b/libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/validators.py @@ -109,7 +109,10 @@ def _require_equal_fields( ) -> None: for field_name in fields: if getattr(existing, field_name) != getattr(candidate, field_name): - msg = f"comparison-run field {field_name} is immutable for an existing identity" + msg = ( + f"comparison-run field {field_name} is immutable for an " + "existing identity" + ) raise ValueError(msg) From 3460877e06ebcd23b7da049b50e8b4d49b527b09 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:33:29 +0400 Subject: [PATCH 10/23] Wrap long generated-client calls in the integration tests (ruff E501) Bind the long generated endpoint functions to shorter local names before calling them. Co-Authored-By: Claude Opus 5.5 --- .../policyengine-apis-integ/tests/simulation/conftest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/policyengine-apis-integ/tests/simulation/conftest.py b/projects/policyengine-apis-integ/tests/simulation/conftest.py index e1c2c808e..220713638 100644 --- a/projects/policyengine-apis-integ/tests/simulation/conftest.py +++ b/projects/policyengine-apis-integ/tests/simulation/conftest.py @@ -96,8 +96,9 @@ def _poll_budget_window_batch( last_status_code: HTTPStatus | None = None last_content = b"" + status_endpoint = get_budget_window_job_status_budget_window_jobs_batch_job_id_get while time.monotonic() < deadline: - response = get_budget_window_job_status_budget_window_jobs_batch_job_id_get.sync_detailed( + response = status_endpoint.sync_detailed( batch_job_id=batch_job_id, client=client ) last_status_code = response.status_code @@ -177,9 +178,10 @@ def decode_response_content(): @pytest.fixture() def submit_budget_window_batch(client: Client | AuthenticatedClient): """Submit a budget-window batch through the generated client.""" + submit_endpoint = submit_budget_window_batch_simulate_economy_budget_window_post def submit(request: BudgetWindowBatchRequest): - return submit_budget_window_batch_simulate_economy_budget_window_post.sync_detailed( + return submit_endpoint.sync_detailed( client=client, body=request, ) From 02d55b764508d70c26bffac0d9e01cb073f0d2c4 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:33:29 +0400 Subject: [PATCH 11/23] Wrap long lines in the entry project (ruff E501) Split long strings with implicit concatenation, so every value is unchanged. The fake curl and gcloud scripts keep their single-line printf output with a suppression on the string. Co-Authored-By: Claude Opus 5.5 --- .../policyengine_simulation_entry/config.py | 5 ++- .../tests/test_deploy_workflow_scripts.py | 7 +-- .../tests/test_deployment_assets.py | 43 ++++++++++++------- .../tests/test_openapi.py | 4 +- .../tests/workflow_script_support.py | 5 ++- 5 files changed, 43 insertions(+), 21 deletions(-) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py index 62bbed40c..5757485d9 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py @@ -170,7 +170,10 @@ def validate(self) -> None: raise ConfigurationError(msg) if bool(self.auth_issuer) != bool(self.auth_audience): - msg = "Set both SIMULATION_ENTRYPOINT_AUTH_ISSUER and SIMULATION_ENTRYPOINT_AUTH_AUDIENCE." + msg = ( + "Set both SIMULATION_ENTRYPOINT_AUTH_ISSUER and " + "SIMULATION_ENTRYPOINT_AUTH_AUDIENCE." + ) raise ConfigurationError(msg) if self.auth_required and not self.auth_issuer: msg = "Caller authentication is required but issuer/audience are missing." diff --git a/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py index c7c715b59..3ad8ab6ee 100644 --- a/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py +++ b/projects/policyengine-simulation-entry/tests/test_deploy_workflow_scripts.py @@ -32,7 +32,8 @@ def fakes(tmp_path: Path) -> FakeCommands: def _step(workflow: str, name: str) -> str: """Return one workflow step's text, from its ``- name:`` to the next step.""" match = re.search( - rf"^(?P +)- name: {re.escape(name)}\n(?P.*?)(?=^(?P=indent)- |^ \S|\Z)", + rf"^(?P +)- name: {re.escape(name)}\n" + r"(?P.*?)(?=^(?P=indent)- |^ \S|\Z)", workflow, flags=re.MULTILINE | re.DOTALL, ) @@ -323,8 +324,8 @@ def test_deploy_workflow_runs_the_stage12_worker_scripts(): validation ) assert ( - f'run: ../../.github/scripts/{VALIDATE_V2_WORKERS} "${{{{ inputs.modal_environment }}}}"' - in validation + f"run: ../../.github/scripts/{VALIDATE_V2_WORKERS} " + '"${{ inputs.modal_environment }}"' in validation ) assert "application_name: ${{ steps.application.outputs.name }}" in workflow diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index 58a0d8f50..8af6655ce 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -153,7 +153,7 @@ def _write_fake_smoke_curl(tmp_path: Path, revisions: list[str]) -> tuple[Path, exit 2 ;; esac - """), + """), # noqa: E501 # the fake curl writes its HTTP header block with one printf encoding="utf-8", ) fake_curl.chmod(0o755) @@ -705,16 +705,17 @@ def test_full_stack_promotion_order_is_explicit(): "update_routing", } <= promote_needs assert ( - 'cloud-run-simulation-entry-smoke.sh "${STABLE_URL}" "${TARGET_REVISION}" "${PREVIOUS_REVISION}"' - in reusable_workflow + 'cloud-run-simulation-entry-smoke.sh "${STABLE_URL}" "${TARGET_REVISION}" ' + '"${PREVIOUS_REVISION}"' in reusable_workflow ) assert ( - 'cloud-run-simulation-entry-smoke.sh "${CUSTOM_URL}" "${TARGET_REVISION}" "${PREVIOUS_REVISION}"' - in reusable_workflow + 'cloud-run-simulation-entry-smoke.sh "${CUSTOM_URL}" "${TARGET_REVISION}" ' + '"${PREVIOUS_REVISION}"' in reusable_workflow ) assert ( - 'cloud-run-simulation-entry-smoke.sh "${{ needs.deploy_entrypoint.outputs.candidate_url }}" "${{ needs.deploy_entrypoint.outputs.revision }}"' - in reusable_workflow + "cloud-run-simulation-entry-smoke.sh " + '"${{ needs.deploy_entrypoint.outputs.candidate_url }}" ' + '"${{ needs.deploy_entrypoint.outputs.revision }}"' in reusable_workflow ) assert "failure() && steps.promote.outcome == 'success'" in reusable_workflow assert "needs: beta" in deploy_workflow @@ -1686,7 +1687,7 @@ def test_traffic_changes_are_exact_revision_validated_and_reversible(tmp_path): exit 2 ;; esac - """), + """), # noqa: E501 # the fake gcloud prints each JSON response on one line, as gcloud does encoding="utf-8", ) fake_gcloud.chmod(0o755) @@ -1705,8 +1706,12 @@ def test_traffic_changes_are_exact_revision_validated_and_reversible(tmp_path): capture_output=True, env={ **base_env, - "SIMULATION_ENTRYPOINT_TARGET_REVISION": "policyengine-simulation-entry-00002-new", - "SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION": "policyengine-simulation-entry-00001-old", + "SIMULATION_ENTRYPOINT_TARGET_REVISION": ( + "policyengine-simulation-entry-00002-new" + ), + "SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION": ( + "policyengine-simulation-entry-00001-old" + ), }, ) assert ( @@ -1720,8 +1725,12 @@ def test_traffic_changes_are_exact_revision_validated_and_reversible(tmp_path): capture_output=True, env={ **base_env, - "SIMULATION_ENTRYPOINT_TARGET_REVISION": "policyengine-simulation-entry-00001-old", - "SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION": "policyengine-simulation-entry-00002-new", + "SIMULATION_ENTRYPOINT_TARGET_REVISION": ( + "policyengine-simulation-entry-00001-old" + ), + "SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION": ( + "policyengine-simulation-entry-00002-new" + ), }, ) assert ( @@ -1743,7 +1752,7 @@ def test_traffic_change_refuses_an_intervening_promotion(tmp_path): set -euo pipefail active="$(cat "${FAKE_GCLOUD_STATE}")" printf '{"status":{"traffic":[{"revisionName":"%s","percent":100}]}}\\n' "${active}" - """), + """), # noqa: E501 # the fake gcloud prints its JSON response on one line, as gcloud does encoding="utf-8", ) fake_gcloud.chmod(0o755) @@ -1758,8 +1767,12 @@ def test_traffic_change_refuses_an_intervening_promotion(tmp_path): "FAKE_GCLOUD_STATE": str(state_file), "SIMULATION_ENTRYPOINT_GCP_PROJECT_ID": "simulation-entry-test", "SIMULATION_ENTRYPOINT_SERVICE": "policyengine-simulation-entry", - "SIMULATION_ENTRYPOINT_TARGET_REVISION": "policyengine-simulation-entry-00002-new", - "SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION": "policyengine-simulation-entry-00001-old", + "SIMULATION_ENTRYPOINT_TARGET_REVISION": ( + "policyengine-simulation-entry-00002-new" + ), + "SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION": ( + "policyengine-simulation-entry-00001-old" + ), }, ) diff --git a/projects/policyengine-simulation-entry/tests/test_openapi.py b/projects/policyengine-simulation-entry/tests/test_openapi.py index 03146eb4e..e293b7e0c 100644 --- a/projects/policyengine-simulation-entry/tests/test_openapi.py +++ b/projects/policyengine-simulation-entry/tests/test_openapi.py @@ -68,7 +68,9 @@ def normalized_compatibility_paths(spec: OpenAPIDocument) -> OpenAPIPathMap: operation.pop("security", None) for path, description in { "/simulate/economy/comparison": "Invalid request (unknown country/version)", - "/simulate/economy/budget-window": "Invalid request (unknown country/version/year)", + "/simulate/economy/budget-window": ( + "Invalid request (unknown country/version/year)" + ), }.items(): paths[path]["post"]["responses"]["400"] = {"description": description} for path in ("/jobs/{job_id}", "/budget-window-jobs/{batch_job_id}"): diff --git a/projects/policyengine-simulation-entry/tests/workflow_script_support.py b/projects/policyengine-simulation-entry/tests/workflow_script_support.py index f9111816e..2ee55c08d 100644 --- a/projects/policyengine-simulation-entry/tests/workflow_script_support.py +++ b/projects/policyengine-simulation-entry/tests/workflow_script_support.py @@ -47,7 +47,10 @@ def add(self, name: str, body: str = "") -> None: path.chmod(0o755) def calls(self, name: str | None = None) -> list[list[str]]: - """Every recorded call (``[command, *arguments]``), optionally of one command.""" + """Return each recorded call as ``[command, *arguments]``. + + Given ``name``, return only that command's calls. + """ if not self.log.exists(): return [] recorded = [ From 24ba096aeaa35059a0b9150e035bfd7d0fd8015f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:33:29 +0400 Subject: [PATCH 12/23] Wrap long lines in the executor (ruff E501) Split long messages and test strings with implicit concatenation, so every value is unchanged, and shorten docstrings that no published schema uses. The fake gh script keeps its single-line JSON output with a suppression on the string. Co-Authored-By: Claude Opus 5.5 --- ...est_policyengine_package_update_scripts.py | 2 +- .../src/modal/app.py | 3 ++- .../simulation_runtime.py | 2 +- .../stage12_bundle/parsing.py | 20 +++++++++++++++---- .../uk_local_authority_metadata.py | 5 ++++- .../tests/test_budget_window_results.py | 5 ++++- .../tests/test_canonical_spm_native.py | 5 ++++- .../tests/test_ci_complete_gate.py | 3 ++- .../tests/test_gcp_credentials.py | 2 +- .../tests/test_modal_scripts.py | 4 ++-- .../test_policyengine_dependency_source.py | 2 +- .../tests/test_simulation_output_budget.py | 2 +- .../tests/test_stage12_worker_validation.py | 2 +- .../tests/test_update_v2_version_manifest.py | 20 +++++++++++++++---- 14 files changed, 56 insertions(+), 21 deletions(-) diff --git a/projects/policyengine-simulation-executor/fixtures/test_policyengine_package_update_scripts.py b/projects/policyengine-simulation-executor/fixtures/test_policyengine_package_update_scripts.py index edc93e3fc..3aa97ee0b 100644 --- a/projects/policyengine-simulation-executor/fixtures/test_policyengine_package_update_scripts.py +++ b/projects/policyengine-simulation-executor/fixtures/test_policyengine_package_update_scripts.py @@ -187,7 +187,7 @@ def install_fake_gh( fi exit 0 -""", +""", # noqa: E501 # the fake gh prints each JSON response on one line, as gh does ) diff --git a/projects/policyengine-simulation-executor/src/modal/app.py b/projects/policyengine-simulation-executor/src/modal/app.py index 2f1c9bad7..3e02f47df 100644 --- a/projects/policyengine-simulation-executor/src/modal/app.py +++ b/projects/policyengine-simulation-executor/src/modal/app.py @@ -3,7 +3,8 @@ This app contains the heavy simulation workload with snapshotted models. Each deployment creates a versioned app (e.g., policyengine-simulation-py4-10-0). -The gateway app (policyengine-simulation-gateway) routes requests to these versioned apps. +The gateway app (policyengine-simulation-gateway) routes requests to these +versioned apps. """ import os diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py index 4b35a3052..e16e9220f 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py @@ -213,7 +213,7 @@ def _parse_policy_period(period_key: object) -> tuple[datetime, datetime | None] def _parse_policy_period_dates(text: str) -> tuple[datetime, datetime | None]: - """Parse a date or a ``start.stop`` range; raise a bare ``ValueError`` if invalid.""" + """Parse a date or ``start.stop`` range; raise a bare ``ValueError`` if invalid.""" parts = text.split(".") if len(parts) not in {1, 2}: raise ValueError diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_bundle/parsing.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_bundle/parsing.py index c5581c299..b6975584c 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_bundle/parsing.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_bundle/parsing.py @@ -52,7 +52,10 @@ def require_exact_package( f"packages.{package_name}.install_requirement", ) if name != package_name: - msg = f"PolicyEngine.py bundle package key {package_name!r} does not match its name" + msg = ( + f"PolicyEngine.py bundle package key {package_name!r} " + "does not match its name" + ) raise Stage12BundleError(msg) expected_requirement = f"{name}=={version}" if requirement != expected_requirement: @@ -100,7 +103,10 @@ def normalize_country_bundle( ) != policyengine_version ): - msg = f"PolicyEngine.py bundle data release for {country!r} names another PolicyEngine.py version" + msg = ( + f"PolicyEngine.py bundle data release for {country!r} names another " + "PolicyEngine.py version" + ) raise Stage12BundleError(msg) release_model = require_mapping( @@ -119,7 +125,10 @@ def normalize_country_bundle( ) != country_package_version ): - msg = f"PolicyEngine.py bundle country package metadata is inconsistent for {country!r}" + msg = ( + "PolicyEngine.py bundle country package metadata is inconsistent " + f"for {country!r}" + ) raise Stage12BundleError(msg) data_package = require_mapping( @@ -201,7 +210,10 @@ def normalize_country_bundle( msg = f"PolicyEngine.py bundle default dataset is absent for {country!r}" raise Stage12BundleError(msg) if selected_dataset.uri != default_dataset_uri: - msg = f"PolicyEngine.py bundle default dataset URI is inconsistent for {country!r}" + msg = ( + "PolicyEngine.py bundle default dataset URI is inconsistent " + f"for {country!r}" + ) raise Stage12BundleError(msg) certified = require_mapping( diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py index a01b45c65..bd32b8f7c 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py @@ -194,7 +194,10 @@ def load_uk_local_authority_resources() -> UKLocalAuthorityResources: lad23 = _load_coordinates("coordinates_lad23.csv") supported_codes = lad22.keys() | lad23.keys() if names.keys() != supported_codes: - msg = "UK local-authority names must exactly cover the supported boundary versions" + msg = ( + "UK local-authority names must exactly cover the supported " + "boundary versions" + ) raise ValueError(msg) if lad22.keys() - lad23.keys() != _LAD22_ONLY_CODES: msg = "LAD22 distinguishing codes do not match its coordinate file" diff --git a/projects/policyengine-simulation-executor/tests/test_budget_window_results.py b/projects/policyengine-simulation-executor/tests/test_budget_window_results.py index d17c416de..8e05b79ae 100644 --- a/projects/policyengine-simulation-executor/tests/test_budget_window_results.py +++ b/projects/policyengine-simulation-executor/tests/test_budget_window_results.py @@ -66,7 +66,10 @@ def test_extract_annual_impact_defaults_state_tax_for_uk_child_result(): def test_extract_annual_impact_rejects_malformed_child_result(): with pytest.raises( ValueError, - match="Malformed budget-window child result: missing numeric budget.tax_revenue_impact", + match=( + "Malformed budget-window child result: " + "missing numeric budget.tax_revenue_impact" + ), ): extract_annual_impact( simulation_year="2026", diff --git a/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py b/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py index 799197748..39e95805c 100644 --- a/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py +++ b/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py @@ -13,7 +13,10 @@ SOURCE = os.environ.get("SPM_NATIVE_SMOKE_SOURCE") pytestmark = pytest.mark.skipif( not SOURCE, - reason="Set SPM_NATIVE_SMOKE_SOURCE to the local native H5 for the bounded source integration smoke", + reason=( + "Set SPM_NATIVE_SMOKE_SOURCE to the local native H5 for the bounded " + "source integration smoke" + ), ) diff --git a/projects/policyengine-simulation-executor/tests/test_ci_complete_gate.py b/projects/policyengine-simulation-executor/tests/test_ci_complete_gate.py index 94fec7bb4..0dda3201a 100644 --- a/projects/policyengine-simulation-executor/tests/test_ci_complete_gate.py +++ b/projects/policyengine-simulation-executor/tests/test_ci_complete_gate.py @@ -69,7 +69,8 @@ def test_image_smoke_runs_only_at_the_bottom_of_a_stack() -> None: ) stack_condition = ( "(!github.event.pull_request.stack || " - "github.event.pull_request.stack.base.ref == github.event.pull_request.base.ref)" + "github.event.pull_request.stack.base.ref == " + "github.event.pull_request.base.ref)" ) for job_id, job in smoke["jobs"].items(): diff --git a/projects/policyengine-simulation-executor/tests/test_gcp_credentials.py b/projects/policyengine-simulation-executor/tests/test_gcp_credentials.py index e4eb76cb2..4994d3d3b 100644 --- a/projects/policyengine-simulation-executor/tests/test_gcp_credentials.py +++ b/projects/policyengine-simulation-executor/tests/test_gcp_credentials.py @@ -1,4 +1,4 @@ -"""Tests for GCP credentials setup in ``policyengine_simulation_executor.simulation_runtime``.""" +"""Tests for GCP credentials setup in the executor's ``simulation_runtime``.""" from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/tests/test_modal_scripts.py b/projects/policyengine-simulation-executor/tests/test_modal_scripts.py index 150748110..6dd07f6a9 100644 --- a/projects/policyengine-simulation-executor/tests/test_modal_scripts.py +++ b/projects/policyengine-simulation-executor/tests/test_modal_scripts.py @@ -78,8 +78,8 @@ def test_deploy_workflow_passes_core_version_to_modal(self): assert "policyengine_core_version" in workflow assert ( - "POLICYENGINE_CORE_VERSION: ${{ needs.prepare.outputs.policyengine_core_version }}" - in workflow + "POLICYENGINE_CORE_VERSION: " + "${{ needs.prepare.outputs.policyengine_core_version }}" in workflow ) def test_deploy_workflow_threads_force_latest_to_routing_publisher(self): diff --git a/projects/policyengine-simulation-executor/tests/test_policyengine_dependency_source.py b/projects/policyengine-simulation-executor/tests/test_policyengine_dependency_source.py index ab3fc4828..7bd4656e3 100644 --- a/projects/policyengine-simulation-executor/tests/test_policyengine_dependency_source.py +++ b/projects/policyengine-simulation-executor/tests/test_policyengine_dependency_source.py @@ -99,7 +99,7 @@ def _modal_import_env() -> dict[str, str]: def _run_python_snippet( code: str, env: dict[str, str] ) -> subprocess.CompletedProcess[str]: - """Run ``code`` in a fresh interpreter from the project root, capturing text output.""" + """Run ``code`` in a fresh interpreter from the project root; capture its text.""" return subprocess.run( [sys.executable, "-c", code], cwd=REPO_ROOT, diff --git a/projects/policyengine-simulation-executor/tests/test_simulation_output_budget.py b/projects/policyengine-simulation-executor/tests/test_simulation_output_budget.py index 42dca109d..87fe8d2af 100644 --- a/projects/policyengine-simulation-executor/tests/test_simulation_output_budget.py +++ b/projects/policyengine-simulation-executor/tests/test_simulation_output_budget.py @@ -12,7 +12,7 @@ def _simulation(columns: dict[str, list[float]]): - """Build a simulation whose household output is a MicroDataFrame, as policyengine does.""" + """Build a simulation whose household output is a policyengine MicroDataFrame.""" household = MicroDataFrame(pd.DataFrame(columns), weights="household_weight") return SimpleNamespace( output_dataset=SimpleNamespace(data=SimpleNamespace(household=household)) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_worker_validation.py b/projects/policyengine-simulation-executor/tests/test_stage12_worker_validation.py index 057790992..ee5471a76 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_worker_validation.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_worker_validation.py @@ -51,7 +51,7 @@ def test_validation_checks_dataset_and_non_serving_calculation() -> None: def test_non_serving_us_calculation_supports_national_spm() -> None: - """The deployed-worker check must not invent a county for its synthetic household.""" + """The deployed-worker check invents no county for its synthetic household.""" _run_non_serving_calculation("us") diff --git a/projects/policyengine-simulation-executor/tests/test_update_v2_version_manifest.py b/projects/policyengine-simulation-executor/tests/test_update_v2_version_manifest.py index 67eb7b0a4..7d074a446 100644 --- a/projects/policyengine-simulation-executor/tests/test_update_v2_version_manifest.py +++ b/projects/policyengine-simulation-executor/tests/test_update_v2_version_manifest.py @@ -69,11 +69,23 @@ def test_deployment_workflow_keeps_v2_optional_and_separate() -> None: for declaration in ( "OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}", "OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}", - "OBSERVABILITY_LOGGING_PROJECT_ID: ${{ vars.OBSERVABILITY_LOGGING_PROJECT_ID }}", + ( + "OBSERVABILITY_LOGGING_PROJECT_ID: " + "${{ vars.OBSERVABILITY_LOGGING_PROJECT_ID }}" + ), "OBSERVABILITY_LOG_NAME: ${{ vars.OBSERVABILITY_LOG_NAME }}", - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}", - "OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }}", + ( + "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER: " + "${{ vars.OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}" + ), + ( + "OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: " + "${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }}" + ), "OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}", - "POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}", + ( + "POLICYENGINE_OTEL_GOOGLE_AUDIENCE: " + "${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}" + ), ): assert declaration in deploy_section From 061b0b12cb4514c601903828c24c123b1f011bf3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:33:29 +0400 Subject: [PATCH 13/23] Wrap long lines in the gateway (ruff E501) Split the Modal app's FastAPI description with implicit concatenation, so the published text is unchanged, and shorten four test names. Co-Authored-By: Claude Opus 5.5 --- .../src/policyengine_simulation_gateway/app.py | 4 +++- .../tests/test_endpoints.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/app.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/app.py index 25b5b618f..59b30f633 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/app.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/app.py @@ -100,7 +100,9 @@ def web_app() -> FastAPI: api = FastAPI( title="PolicyEngine Simulation Gateway", - description="Submit and poll simulation jobs. Routes to versioned simulation apps.", + description=( + "Submit and poll simulation jobs. Routes to versioned simulation apps." + ), version="1.0.0", ) runtime = init_simulation_observability( diff --git a/projects/policyengine-simulation-gateway/tests/test_endpoints.py b/projects/policyengine-simulation-gateway/tests/test_endpoints.py index be8a30f73..17672641c 100644 --- a/projects/policyengine-simulation-gateway/tests/test_endpoints.py +++ b/projects/policyengine-simulation-gateway/tests/test_endpoints.py @@ -136,7 +136,7 @@ def test__given_active_state_absent__then_falls_back_to_legacy_country_dict( assert resolved_version == "1.500.0" assert app_name == "legacy-app" - def test__given_active_state_with_only_legacy_country_routes__then_no_version_uses_country_latest( + def test__given_only_legacy_country_routes__then_no_version_uses_country_latest( self, mock_modal ): from policyengine_simulation_gateway.endpoints import get_app_name @@ -542,7 +542,7 @@ def test__given_legacy_alias_in_bundle_snapshot__then_gateway_rejects_it( assert response.json()["detail"][0]["type"] == "extra_forbidden" assert mock_modal["func"].last_payload is None - def test__given_us_state_region_without_data__then_keeps_contract_and_uses_default_dataset( + def test__given_us_state_without_data__then_keeps_contract_and_uses_default_dataset( self, mock_modal, client: TestClient ): mock_modal["dicts"]["simulation-api-us-versions"] = { @@ -734,7 +734,7 @@ def test__given_policyengine_data_default__then_uses_package_storage_version( ) assert response_bundle["data_version"] == ("policyengine-uk-data-1.56.16") - def test__given_uk_submission_without_data_and_manifest_commit__then_uses_populace_default( + def test__given_uk_without_data_and_manifest_commit__then_uses_populace_default( self, mock_modal, client: TestClient, @@ -760,7 +760,10 @@ def test__given_uk_submission_without_data_and_manifest_commit__then_uses_popula mock_modal["dicts"]["simulation-api-routing-state"] = {"active": state} def reject_revision(dataset_uri, revision): - msg = f"HF validation should not run for private GCS data: {dataset_uri}@{revision}" + msg = ( + "HF validation should not run for private GCS data: " + f"{dataset_uri}@{revision}" + ) raise HuggingFaceDatasetReferenceError(msg) monkeypatch.setattr( @@ -1673,7 +1676,7 @@ def test__given_non_general_target__then_budget_window_submit_returns_422( assert response.status_code == 422 assert response.json()["detail"][0]["loc"] == ["body", "target"] - def test__given_max_parallel_above_active_limit__then_budget_window_submit_returns_422( + def test__given_max_parallel_above_active_limit__then_submit_returns_422( self, mock_modal, client: TestClient ): response = client.post( From 05d8e2df6d6f7d47cad1ea02f733dfa371f84de0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:33:29 +0400 Subject: [PATCH 14/23] Wrap the Stage 12 DDL proof fixtures' docstrings (ruff E501) Co-Authored-By: Claude Opus 5.5 --- .../sim.stage12.no-schema-ddl/bad_stage12_runtime.py | 5 ++++- .../sim.stage12.no-schema-ddl/good_stage12_runtime.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py b/rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py index 1db8aeaff..c243fa60e 100644 --- a/rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py +++ b/rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py @@ -1,4 +1,7 @@ -"""Proof fixture: a Stage 12 runtime that creates its own table. The rule must reject it.""" +"""Proof fixture: a Stage 12 runtime that creates its own table. + +The rule must reject it. +""" from __future__ import annotations diff --git a/rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py b/rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py index bb03ec6fd..020be3f60 100644 --- a/rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py +++ b/rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py @@ -1,4 +1,7 @@ -"""Proof fixture: a Stage 12 runtime that writes rows with typed DML. The rule must accept it.""" +"""Proof fixture: a Stage 12 runtime that writes rows with typed DML. + +The rule must accept it. +""" from __future__ import annotations From a77f4d2f4e64fda8d9d55951369b8b83672dfa72 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:33:43 +0400 Subject: [PATCH 15/23] Enable ruff group 7 (structure and imports) Select A, E, FAST, INP, N, PLC and SLF. Every finding outside the shared scripts/rules tool is fixed or carries an inline reason; the rules tool's findings are fixed upstream and synced separately. Co-Authored-By: Claude Opus 5.5 --- ruff.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ruff.toml b/ruff.toml index 4aa63ec9a..07490c093 100644 --- a/ruff.toml +++ b/ruff.toml @@ -35,6 +35,8 @@ select = [ "BLE", "EM", "TRY", # Group 6: security, logging and debugging leftovers. "DTZ", "ERA", "EXE", "FIX", "G", "LOG", "S", "T20", "TD", + # Group 7: structure and imports. + "A", "E", "FAST", "INP", "N", "PLC", "SLF", ] # Mutually exclusive pydocstyle pairs: keep D211 (no blank line before a class # docstring) and D212 (summary on the first line). From b9c73275ba96da1a1c1ed99c77df8d84051f286a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:36:23 +0400 Subject: [PATCH 16/23] Register group 7 in the ruff-lint rule sim.quality.ruff-lint now lists A, E, FAST, INP, N, PLC and SLF. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- changelog_entry.yaml | 1 + docs/generated/rules.md | 8 ++++---- rules/registry.toml | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a5bce2ab8..546f3a860 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -146,7 +146,7 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. - `sim.quality.every-project-tested`: Every project and library with unit tests runs them in the PR test matrix. (doc-only, not checked: follow it yourself) - `sim.quality.ruff-format-src`: Source code under projects/*/src and libs/*/src is formatted with ruff format. (blocking: ruff format) - `sim.quality.ruff-format-tests`: Test code is formatted with ruff format too; CI currently checks only src directories. (doc-only, not checked: follow it yourself) -- `sim.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W)) +- `sim.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TRY, UP, W)) - `sim.quality.runtime-evaluated-models-resolve`: Every Pydantic model is runtime-evaluated for ruff: one of its bases, read through the imports of the model's own module, is listed in ruff.toml's runtime-evaluated-base-classes or is a runtime-evaluated class in that same module; and every Pydantic model resolves its annotations once its package is imported. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_runtime_evaluated_annotations.py`, `projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py`) - `sim.quality.src-programs-ruff-exemptions`: Programs under src (the Modal deployment utilities, and the OpenAPI and Stage 12 infrastructure commands) may print their output (T201) and open URLs from their arguments (S310); every other ruff rule applies to them. (blocking: ruff per-file ignore of S310, T201 for `projects/*/src/{modal/utils/*,policyengine_simulation_entry/generate_openapi,policyengine_simulation_entry/stage12_infrastructure,policyengine_simulation_gateway/generate_openapi}.py`) - `sim.quality.tests-pass`: Each project's unit tests pass in its own locked environment. (blocking: CI job `test`) diff --git a/changelog_entry.yaml b/changelog_entry.yaml index 99adea86d..cc6b5269e 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -4,6 +4,7 @@ - Made CLAUDE.md and the Copilot instructions thin pointers to AGENTS.md, required workflow logic longer than one line to live in tested and shellchecked scripts, and registered the tracking-issue rule - Added a rules registry (rules/registry.toml) with a standard-library tool that validates it, checks every rule is enforced where it claims, and renders docs/generated/rules.md and the Rules block in AGENTS.md changed: + - Enabled ruff's structure and import rules (INP, PLC, SLF, N, A, FAST, E); long lines are wrapped, deferred imports that Modal images, test seams, import cycles or heavy dependencies need carry a reason, script directories are declared to ruff as namespace packages, four internal exceptions gained an Error suffix, and published field names, OpenAPI documents and logged error types are unchanged; tests are also exempt from SLF001, INP001 and PLC0415 - Enabled ruff's security, logging and debugging rules (S, T20, LOG, G, DTZ, ERA, TD, FIX, EXE); asserts outside tests became explicit checks, command-line scripts keep their printed output, test subprocess calls go through helpers, and every deliberate exception carries a reasoned suppression; the gateway's auth misconfiguration log no longer repeats its message, and the deployed-service tests share the unit-test ruff exemptions; tests, CI/CD scripts and command-line scripts take their deliberate exceptions as registered file-level ruff ignores instead of inline suppressions, and the rules guard rejects file-level ignores anywhere else - Enabled ruff's exception rules (TRY, EM, BLE); exception messages are built before the raise, raises inside try blocks moved into helpers, and every deliberate catch-all or traceback-free error log carries a reasoned suppression; message text is unchanged, and error types change only in an operator script's two type checks - Enabled ruff's docstring rules (D) and documented every module, class and function outside tests; published contract models and routes now carry these descriptions in their JSON schema and OpenAPI, with no structural schema change diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 309c8b4df..25f07198c 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -252,7 +252,7 @@ flowchart LR | `sim.quality.every-project-tested` | Every project and library with unit tests runs them in the PR test matrix. | mechanical | doc-only | no | nothing | — | — | | `sim.quality.ruff-format-src` | Source code under projects/*/src and libs/*/src is formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — | | `sim.quality.ruff-format-tests` | Test code is formatted with ruff format too; CI currently checks only src directories. | mechanical | doc-only | no | nothing | — | — | -| `sim.quality.ruff-lint` | All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. | mechanical | enforced | yes | ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W) | `pr.yml:lint` | — | +| `sim.quality.ruff-lint` | All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. | mechanical | enforced | yes | ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TRY, UP, W) | `pr.yml:lint` | — | | `sim.quality.runtime-evaluated-models-resolve` | Every Pydantic model is runtime-evaluated for ruff: one of its bases, read through the imports of the model's own module, is listed in ruff.toml's runtime-evaluated-base-classes or is a runtime-evaluated class in that same module; and every Pydantic model resolves its annotations once its package is imported. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_runtime_evaluated_annotations.py`, `projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py` | `pr.yml:test` | yes | | `sim.quality.src-programs-ruff-exemptions` | Programs under src (the Modal deployment utilities, and the OpenAPI and Stage 12 infrastructure commands) may print their output (T201) and open URLs from their arguments (S310); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of S310, T201 for `projects/*/src/{modal/utils/*,policyengine_simulation_entry/generate_openapi,policyengine_simulation_entry/stage12_infrastructure,policyengine_simulation_gateway/generate_openapi}.py` | `pr.yml:lint` | — | | `sim.quality.tests-pass` | Each project's unit tests pass in its own locked environment. | mechanical | enforced | yes | CI job `test` | `pr.yml:test` | — | @@ -273,7 +273,7 @@ flowchart LR e_ruff_format["ruff format"] r_sim_quality_ruff_format_tests("sim.quality.ruff-format-tests (doc-only)") r_sim_quality_ruff_lint("sim.quality.ruff-lint (enforced)") - e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_["ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W)"] + e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TRY__UP__W_["ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TRY, UP, W)"] r_sim_quality_runtime_evaluated_models_resolve("sim.quality.runtime-evaluated-models-resolve (enforced)") e_pytest_projects_policyengine_simulation_entry_tests_test_runtime_evaluated_annotations_py__projects_policyengine_simulation_executor_tests_test_runtime_evaluated_annotations_py["pytest projects/policyengine-simulation-entry/tests/test_runtime_evaluated_annotations.py, projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py"] j_CI_pr_yml_test["CI pr.yml:test"] @@ -293,8 +293,8 @@ flowchart LR r_sim_quality_ruff_format_src --> e_ruff_format e_ruff_format --> j_CI_pr_yml_lint r_sim_quality_ruff_format_tests -.-> gap_unenforced - r_sim_quality_ruff_lint --> e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_ - e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_ --> j_CI_pr_yml_lint + r_sim_quality_ruff_lint --> e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TRY__UP__W_ + e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TRY__UP__W_ --> j_CI_pr_yml_lint r_sim_quality_runtime_evaluated_models_resolve --> e_pytest_projects_policyengine_simulation_entry_tests_test_runtime_evaluated_annotations_py__projects_policyengine_simulation_executor_tests_test_runtime_evaluated_annotations_py e_pytest_projects_policyengine_simulation_entry_tests_test_runtime_evaluated_annotations_py__projects_policyengine_simulation_executor_tests_test_runtime_evaluated_annotations_py --> j_CI_pr_yml_test r_sim_quality_src_programs_ruff_exemptions --> e_ruff_per_file_ignore_of_S310__T201_for_projects___src__modal_utils___policyengine_simulation_entry_generate_openapi_policyengine_simulation_entry_stage12_infrastructure_policyengine_simulation_gateway_generate_openapi__py diff --git a/rules/registry.toml b/rules/registry.toml index 2d88223d7..89dfee8db 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -733,7 +733,7 @@ autofix = true [rule.enforcer] kind = "ruff" mode = "lint" -codes = ["ANN", "BLE", "D", "DTZ", "E4", "E7", "E9", "EM", "ERA", "EXE", "F", "FIX", "FLY", "FURB", "G", "I", "LOG", "PIE", "PYI", "RET", "RSE", "S", "T20", "TC", "TD", "TRY", "UP", "W"] +codes = ["A", "ANN", "BLE", "D", "DTZ", "E", "E4", "E7", "E9", "EM", "ERA", "EXE", "F", "FAST", "FIX", "FLY", "FURB", "G", "I", "INP", "LOG", "N", "PIE", "PLC", "PYI", "RET", "RSE", "S", "SLF", "T20", "TC", "TD", "TRY", "UP", "W"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "lint" From 69a54f67d18f0b6691678f5d65814caccbb662b0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:03:03 +0400 Subject: [PATCH 17/23] Sync the rules tool from the Public API's group 7 The shared scripts/rules tool takes the Public API's group 7 fixes: wrapped long lines, BaseUnavailableError, and assert_violation in its tests. The rendered rules page picks up the tool's new hash. Co-Authored-By: Claude Opus 5.5 --- scripts/rules/adapters.py | 3 +- scripts/rules/baseline.py | 9 +- scripts/rules/ci_scripts.py | 6 +- scripts/rules/cli.py | 5 +- scripts/rules/discover.py | 6 +- scripts/rules/guard.py | 7 +- scripts/rules/render.py | 16 ++-- scripts/rules/resolve.py | 6 +- scripts/rules/tests/test_ci_scripts.py | 10 +- scripts/rules/tests/test_guard.py | 123 +++++++++++++++---------- scripts/rules/tests/test_workflows.py | 13 ++- scripts/rules/validate.py | 6 +- 12 files changed, 130 insertions(+), 80 deletions(-) diff --git a/scripts/rules/adapters.py b/scripts/rules/adapters.py index 19a7ee979..a16c5b40c 100644 --- a/scripts/rules/adapters.py +++ b/scripts/rules/adapters.py @@ -48,7 +48,8 @@ def v(message: str, fix: str) -> None: if others: v( f"references other docs ({', '.join(others)})", - f"list skills and rules in {instructions} only; keep {adapter} a pointer", + f"list skills and rules in {instructions} only; " + f"keep {adapter} a pointer", ) if BEGIN in text: v( diff --git a/scripts/rules/baseline.py b/scripts/rules/baseline.py index 5bc3feb90..2af9b1ac5 100644 --- a/scripts/rules/baseline.py +++ b/scripts/rules/baseline.py @@ -13,7 +13,7 @@ from pathlib import Path -class BaseUnavailable(Exception): +class BaseUnavailableError(Exception): """The base ref could not be read.""" @@ -26,7 +26,7 @@ def default_base_ref() -> str | None: def load_base_registry(root: Path, ref: str) -> Registry | None: """Return the registry as it is on ``ref``, or None if the base has none yet. - Raise BaseUnavailable when ``ref`` does not name a commit. + Raise BaseUnavailableError when ``ref`` does not name a commit. """ verify = subprocess.run( ["git", "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"], @@ -35,7 +35,7 @@ def load_base_registry(root: Path, ref: str) -> Registry | None: text=True, ) if verify.returncode != 0: - raise BaseUnavailable(ref) + raise BaseUnavailableError(ref) shown = subprocess.run( ["git", "show", f"{ref}:{REGISTRY_PATH.as_posix()}"], cwd=root, @@ -81,7 +81,8 @@ def check_baselines( new.id, "baseline", f"status weakened from {old.status} to {new.status}", - 'add [rule.downgrade] reason = "..." so the downgrade is explicit and reviewable', + 'add [rule.downgrade] reason = "..." so the downgrade is ' + "explicit and reviewable", ) ) for rule_id in sorted(current_ids - {rule.id for rule in base.rules}): diff --git a/scripts/rules/ci_scripts.py b/scripts/rules/ci_scripts.py index 571b63ac2..77fe38324 100644 --- a/scripts/rules/ci_scripts.py +++ b/scripts/rules/ci_scripts.py @@ -166,7 +166,8 @@ def compare(registry: Registry, findings: Findings, root: Path) -> list[Violatio "untested workflow script", findings.untested, config.untested_baseline, - "add a test that runs the script (see docs/engineering/skills/ci-workflows.md)", + "add a test that runs the script " + "(see docs/engineering/skills/ci-workflows.md)", ), ( "shellcheck", @@ -190,7 +191,8 @@ def compare(registry: Registry, findings: Findings, root: Path) -> list[Violatio rule_id, "ci-scripts", f"baseline entry no longer applies: {entry}", - f"remove it from {baseline_path} and lower the rule's baseline count", + f"remove it from {baseline_path} and lower the rule's " + "baseline count", ) ) return violations diff --git a/scripts/rules/cli.py b/scripts/rules/cli.py index 78ebf6149..eb9edf9fb 100644 --- a/scripts/rules/cli.py +++ b/scripts/rules/cli.py @@ -47,7 +47,10 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--write-baselines", action="store_true", - help="ci-scripts only: record today's findings as the baselines (when adopting the rule)", + help=( + "ci-scripts only: record today's findings as the baselines " + "(when adopting the rule)" + ), ) args = parser.parse_args(argv) root = args.root.resolve() diff --git a/scripts/rules/discover.py b/scripts/rules/discover.py index a352c05ef..152ec9469 100644 --- a/scripts/rules/discover.py +++ b/scripts/rules/discover.py @@ -149,7 +149,8 @@ def discover_hidden(registry: Registry, root: Path) -> list[Violation]: violations.append( _hidden( f"guard test {node} is not referenced by any rule", - "add it to a pytest rule's nodes, or to discovery.ignore_tests with a reason", + "add it to a pytest rule's nodes, or to discovery.ignore_tests " + "with a reason", ) ) @@ -162,7 +163,8 @@ def discover_hidden(registry: Registry, root: Path) -> list[Violation]: violations.append( _hidden( f"CI job '{job_id}' in {workflow} enforces no registered rule", - "reference it from a rule's [[rule.ci]], or add it to discovery.ignore_jobs with a reason", + "reference it from a rule's [[rule.ci]], or add it to " + "discovery.ignore_jobs with a reason", ) ) return violations diff --git a/scripts/rules/guard.py b/scripts/rules/guard.py index 3efa3d4f5..99ba5e357 100644 --- a/scripts/rules/guard.py +++ b/scripts/rules/guard.py @@ -9,7 +9,7 @@ from scripts.rules.adapters import check_adapters from scripts.rules.baseline import ( - BaseUnavailable, + BaseUnavailableError, check_baselines, default_base_ref, load_base_registry, @@ -104,12 +104,13 @@ def run_guard( return result if ref is None: result.notes.append( - "no base ref (set --base-ref or GITHUB_BASE_REF); baseline comparison skipped" + "no base ref (set --base-ref or GITHUB_BASE_REF); " + "baseline comparison skipped" ) else: try: base = load_base_registry(root, ref) - except BaseUnavailable: + except BaseUnavailableError: if os.environ.get("GITHUB_ACTIONS"): result.violations.append( Violation( diff --git a/scripts/rules/render.py b/scripts/rules/render.py index 2b70361e4..68002382e 100644 --- a/scripts/rules/render.py +++ b/scripts/rules/render.py @@ -129,9 +129,10 @@ def render_rules_md(registry: Registry) -> str: "Generated by `python -m scripts.rules render` from `rules/registry.toml`. " "Do not edit by hand.", "", - f"Every rule that governs `{registry.repo}` is listed here with how it is enforced. " - "Blocking rules fail CI. The Gaps section lists what is written down but not yet " - f"enforced or proven. To add or change a rule, see `{SKILL_DOC}`.", + f"Every rule that governs `{registry.repo}` is listed here with how it " + "is enforced. Blocking rules fail CI. The Gaps section lists what is " + "written down but not yet enforced or proven. To add or change a rule, " + f"see `{SKILL_DOC}`.", "", "## Summary", "", @@ -184,7 +185,8 @@ def section(title: str, items: list[str]) -> None: out += [ f"## {title}", "", - "| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof |", + "| Rule | Statement | Tier | Status | Blocking " + "| Enforced by | CI | Proof |", "| --- | --- | --- | --- | --- | --- | --- | --- |", ] for r in group: @@ -220,9 +222,11 @@ def render_agents_block(registry: Registry) -> str: out = [ BEGIN, "", - "Every rule for this repository lives in `rules/registry.toml` and is rendered,", + "Every rule for this repository lives in " + "`rules/registry.toml` and is rendered,", "with its enforcement and gaps, in `docs/generated/rules.md`. Blocking rules", - "fail CI. Doc-only rules are not checked automatically, so follow them yourself.", + "fail CI. Doc-only rules are not checked automatically, " + "so follow them yourself.", f"To add or change a rule, read `{SKILL_DOC}`.", "", ] diff --git a/scripts/rules/resolve.py b/scripts/rules/resolve.py index d00d15e91..78a41a2d8 100644 --- a/scripts/rules/resolve.py +++ b/scripts/rules/resolve.py @@ -467,7 +467,8 @@ def resolve_proof_ci(rule: Rule, root: Path) -> list[Violation]: rule.id, "proof", "the proof test is not tied to a CI job", - "add proof.workflow, proof.job and proof.invocation naming the job that runs it", + "add proof.workflow, proof.job and proof.invocation naming the " + "job that runs it", ) ] violations = [] @@ -516,7 +517,8 @@ def _test_coverage( Violation( rule_id, check, - f"{path} is not collected by {described} (collects {collected})", + f"{path} is not collected by {described} " + f"(collects {collected})", "add it to what the job runs, or move the test", ) ) diff --git a/scripts/rules/tests/test_ci_scripts.py b/scripts/rules/tests/test_ci_scripts.py index c05a4bc3f..354ad5adc 100644 --- a/scripts/rules/tests/test_ci_scripts.py +++ b/scripts/rules/tests/test_ci_scripts.py @@ -38,7 +38,7 @@ def require_shellcheck(test: unittest.TestCase) -> None: class TestCiScriptsProof(unittest.TestCase): - """Proofs that each ci-scripts check rejects a bad example and accepts a good one.""" + """Proofs: each ci-scripts check rejects a bad example and accepts a good one.""" def test_rule_proof_multiline_run(self) -> None: self.assertEqual( @@ -80,7 +80,10 @@ def test_blank_lines_do_not_count(self) -> None: self.assertEqual(code_lines("\nmake build\n\n"), 1) def test_folded_block_counts_each_line(self) -> None: - text = "jobs:\n a:\n steps:\n - run: >-\n pytest a\n b\n" + text = ( + "jobs:\n a:\n steps:\n - run: >-\n pytest a\n" + " b\n" + ) jobs, _ = parse_jobs(text) block = jobs["a"].run_blocks[0] self.assertEqual(block.code_lines, 2) @@ -194,7 +197,8 @@ def test_enforced_rule_without_baseline_keeps_its_id(self) -> None: REGISTRY.replace("ex.ci.multiline", "ex.ci.workflow-scripts-tested") .replace('status = "ratcheted"', 'status = "enforced"') .replace( - '[rule.baseline]\nfile = "rules/baselines/ci-multiline-run.txt"\ncount = 1\n', + '[rule.baseline]\nfile = "rules/baselines/ci-multiline-run.txt"\n' + "count = 1\n", "", ) ) diff --git a/scripts/rules/tests/test_guard.py b/scripts/rules/tests/test_guard.py index 00151ed23..0c5773ed9 100644 --- a/scripts/rules/tests/test_guard.py +++ b/scripts/rules/tests/test_guard.py @@ -91,7 +91,7 @@ def test_rules_md_lists_gaps(self) -> None: class SchemaTest(unittest.TestCase): - def assertViolation(self, repo: Repo, fragment: str) -> None: + def assert_violation(self, repo: Repo, fragment: str) -> None: messages = repo.messages() self.assertTrue(any(fragment in m for m in messages), messages) @@ -100,19 +100,19 @@ def test_missing_required_field(self) -> None: repo.edit( "rules/registry.toml", 'statement = "Routes never issue SQL."\n', "" ) - self.assertViolation(repo, "statement must be a non-empty string") + self.assert_violation(repo, "statement must be a non-empty string") def test_unknown_enforcer_kind(self) -> None: with Repo(render=False) as repo: repo.edit("rules/registry.toml", 'kind = "mypy"', 'kind = "flake8"') - self.assertViolation(repo, "enforcer.kind is 'flake8'") + self.assert_violation(repo, "enforcer.kind is 'flake8'") def test_bad_status(self) -> None: with Repo(render=False) as repo: repo.edit( "rules/registry.toml", 'status = "doc-only"', 'status = "optional"' ) - self.assertViolation(repo, "status is 'optional'") + self.assert_violation(repo, "status is 'optional'") def test_unknown_key(self) -> None: with Repo(render=False) as repo: @@ -121,7 +121,7 @@ def test_unknown_key(self) -> None: "autofix = true", 'autofix = true\nowner = "someone"', ) - self.assertViolation(repo, "unknown key 'owner'") + self.assert_violation(repo, "unknown key 'owner'") def test_duplicate_rule_id(self) -> None: with Repo(render=False) as repo: @@ -130,7 +130,7 @@ def test_duplicate_rule_id(self) -> None: 'id = "ex.quality.types"', 'id = "ex.quality.format"', ) - self.assertViolation(repo, "duplicate rule id") + self.assert_violation(repo, "duplicate rule id") def test_scope_glob_must_match(self) -> None: with Repo(render=False) as repo: @@ -139,7 +139,7 @@ def test_scope_glob_must_match(self) -> None: 'scope = ["rules/registry.toml"]', 'scope = ["nowhere/*.py"]', ) - self.assertViolation(repo, "scope 'nowhere/*.py' matches no files") + self.assert_violation(repo, "scope 'nowhere/*.py' matches no files") def test_doc_only_rule_cannot_claim_enforcer(self) -> None: with Repo(render=False) as repo: @@ -148,7 +148,7 @@ def test_doc_only_rule_cannot_claim_enforcer(self) -> None: 'status = "enforced"\nautofix = true', 'status = "doc-only"\nautofix = true', ) - self.assertViolation(repo, "doc-only rules have no enforcer") + self.assert_violation(repo, "doc-only rules have no enforcer") def test_enforced_rule_needs_ci(self) -> None: extra = """ @@ -166,7 +166,7 @@ def test_enforced_rule_needs_ci(self) -> None: nodes = ["tests/unit/test_boundaries.py::test_routes_have_no_sql"] """ with Repo(with_rule(extra), render=False) as repo: - self.assertViolation(repo, "enforced rules must run in CI") + self.assert_violation(repo, "enforced rules must run in CI") def test_ruff_is_never_ratcheted(self) -> None: with Repo(render=False) as repo: @@ -174,9 +174,10 @@ def test_ruff_is_never_ratcheted(self) -> None: repo.edit( "rules/registry.toml", 'kind = "ruff"\nmode = "format"', - 'kind = "ruff"\nmode = "format"\n[rule.baseline]\nfile = "rules/baselines/ruff.txt"\ncount = 1', + 'kind = "ruff"\nmode = "format"\n[rule.baseline]\n' + 'file = "rules/baselines/ruff.txt"\ncount = 1', ) - self.assertViolation(repo, "ruff is never ratcheted") + self.assert_violation(repo, "ruff is never ratcheted") def test_ratcheted_without_baseline(self) -> None: with Repo(render=False) as repo: @@ -185,7 +186,7 @@ def test_ratcheted_without_baseline(self) -> None: 'status = "enforced"\nautofix = false\n[rule.enforcer]\nkind = "mypy"', 'status = "ratcheted"\nautofix = false\n[rule.enforcer]\nkind = "mypy"', ) - self.assertViolation(repo, "ratcheted rules need a baseline") + self.assert_violation(repo, "ratcheted rules need a baseline") def test_baseline_count_must_match_file(self) -> None: with Repo(with_rule(RATCHETED_RULE.format(count=3)), render=False) as repo: @@ -193,7 +194,7 @@ def test_baseline_count_must_match_file(self) -> None: "rules/baselines/ratcheted.txt", "# tolerated\npkg/routes.py\npkg/other.py\n", ) - self.assertViolation( + self.assert_violation( repo, "baseline count is 3 but rules/baselines/ratcheted.txt lists 2" ) @@ -204,11 +205,11 @@ def test_judgment_rules_need_review_enforcer(self) -> None: 'tier = "mechanical"\nstatus = "enforced"\nautofix = true', 'tier = "judgment"\nstatus = "enforced"\nautofix = true', ) - self.assertViolation(repo, "judgment rules are enforced by review") + self.assert_violation(repo, "judgment rules are enforced by review") class ResolutionTest(unittest.TestCase): - def assertViolation(self, repo: Repo, fragment: str) -> None: + def assert_violation(self, repo: Repo, fragment: str) -> None: messages = repo.messages() self.assertTrue(any(fragment in m for m in messages), messages) @@ -219,7 +220,7 @@ def test_unresolved_pytest_node(self) -> None: "def test_routes_have_no_sql", "def test_renamed", ) - self.assertViolation(repo, "has no test 'test_routes_have_no_sql'") + self.assert_violation(repo, "has no test 'test_routes_have_no_sql'") def test_parametrized_node_resolves(self) -> None: with Repo() as repo: @@ -234,7 +235,7 @@ def test_parametrized_node_resolves(self) -> None: def test_test_outside_make_target_is_not_collected(self) -> None: with Repo() as repo: repo.edit("Makefile", "tests/unit", "tests/contract") - self.assertViolation(repo, "is not collected by 'make test'") + self.assert_violation(repo, "is not collected by 'make test'") def test_project_must_be_in_matrix(self) -> None: workflow = """ @@ -275,7 +276,7 @@ def test_project_must_be_in_matrix(self) -> None: **with_rule(extra), } with Repo(files) as repo: - self.assertViolation( + self.assert_violation( repo, "project projects/beta is not in the matrix of job 'test'" ) @@ -283,12 +284,12 @@ def test_missing_job(self) -> None: with Repo() as repo: repo.edit("rules/registry.toml", 'job = "typecheck"', 'job = "types"') repo.render() - self.assertViolation(repo, "no job 'types'") + self.assert_violation(repo, "no job 'types'") def test_invocation_only_in_comment_does_not_count(self) -> None: with Repo() as repo: repo.edit(".github/workflows/pr.yml", "run: make test", "run: echo skipped") - self.assertViolation(repo, "does not run 'make test'") + self.assert_violation(repo, "does not run 'make test'") def test_unknown_quality_guard(self) -> None: with Repo() as repo: @@ -296,7 +297,7 @@ def test_unknown_quality_guard(self) -> None: "rules/registry.toml", 'name = "rules-registry"', 'name = "renamed"' ) repo.render() - self.assertViolation(repo, "no guard named 'renamed'") + self.assert_violation(repo, "no guard named 'renamed'") def test_mypy_path_must_be_configured(self) -> None: with Repo() as repo: @@ -304,7 +305,7 @@ def test_mypy_path_must_be_configured(self) -> None: "rules/registry.toml", 'paths = ["pkg"]', 'paths = ["pkg", "other"]' ) repo.render() - self.assertViolation(repo, "['other'] not in [tool.mypy].files") + self.assert_violation(repo, "['other'] not in [tool.mypy].files") def test_ruff_lint_codes_must_be_selected(self) -> None: extra = """ @@ -323,17 +324,19 @@ def test_ruff_lint_codes_must_be_selected(self) -> None: codes = ["B"] """ with Repo(with_rule(extra)) as repo: - self.assertViolation(repo, "ruff does not select ['B']") + self.assert_violation(repo, "ruff does not select ['B']") def test_proof_files_must_exist(self) -> None: with Repo(render=False) as repo: repo.edit( "rules/registry.toml", 'kind = "mypy"\npaths = ["pkg"]', - 'kind = "mypy"\npaths = ["pkg"]\n[rule.proof]\nbad_fixture = "rules/fixtures/bad.py"\n' - 'good_fixture = "rules/fixtures/good.py"\ntest = "tests/unit/test_boundaries.py::test_routes_have_no_sql"', + 'kind = "mypy"\npaths = ["pkg"]\n[rule.proof]\n' + 'bad_fixture = "rules/fixtures/bad.py"\n' + 'good_fixture = "rules/fixtures/good.py"\n' + 'test = "tests/unit/test_boundaries.py::test_routes_have_no_sql"', ) - self.assertViolation( + self.assert_violation( repo, "proof bad_fixture rules/fixtures/bad.py does not exist" ) @@ -359,7 +362,8 @@ def test_pytest_rule_proof_runs_in_its_ci_job(self) -> None: self.with_proof( repo, 'nodes = ["tests/unit/test_boundaries.py::test_routes_have_no_sql"]', - 'bad_fixture = "rules/fixtures/bad.py"\ngood_fixture = "rules/fixtures/good.py"\n' + 'bad_fixture = "rules/fixtures/bad.py"\n' + 'good_fixture = "rules/fixtures/good.py"\n' 'test = "tests/unit/test_proof.py::test_rule_proof_routes"', ) self.assertEqual(repo.messages(), []) @@ -373,7 +377,8 @@ def test_proof_outside_the_ci_command_is_rejected(self) -> None: self.with_proof( repo, 'nodes = ["tests/unit/test_boundaries.py::test_routes_have_no_sql"]', - 'bad_fixture = "rules/fixtures/bad.py"\ngood_fixture = "rules/fixtures/good.py"\n' + 'bad_fixture = "rules/fixtures/bad.py"\n' + 'good_fixture = "rules/fixtures/good.py"\n' 'test = "tests/contract/test_proof.py::test_rule_proof_routes"', ) messages = "\n".join(repo.messages()) @@ -391,7 +396,8 @@ def test_non_pytest_rule_proof_must_name_its_job(self) -> None: self.with_proof( repo, 'paths = ["pkg"]', - 'bad_fixture = "rules/fixtures/bad.py"\ngood_fixture = "rules/fixtures/good.py"\n' + 'bad_fixture = "rules/fixtures/bad.py"\n' + 'good_fixture = "rules/fixtures/good.py"\n' 'test = "tests/unit/test_proof.py::test_rule_proof_types"', ) self.assertIn( @@ -405,7 +411,9 @@ def test_proof_with_named_unittest_job(self) -> None: ) files = { **self.PROOF_FILES, - "tools/tests/test_proof.py": "class TestProof:\n def test_rule_proof_types(self):\n pass\n", + "tools/tests/test_proof.py": ( + "class TestProof:\n def test_rule_proof_types(self):\n pass\n" + ), } with Repo(files, render=False) as repo: repo.write( @@ -415,10 +423,12 @@ def test_proof_with_named_unittest_job(self) -> None: repo.edit( "rules/registry.toml", "[discovery]\n", - '[discovery]\nignore_jobs = [{ job = "tool-tests", reason = "runs tool tests" }]\n', + "[discovery]\n" + 'ignore_jobs = [{ job = "tool-tests", reason = "runs tool tests" }]\n', ) proof = ( - 'bad_fixture = "rules/fixtures/bad.py"\ngood_fixture = "rules/fixtures/good.py"\n' + 'bad_fixture = "rules/fixtures/bad.py"\n' + 'good_fixture = "rules/fixtures/good.py"\n' 'test = "tools/tests/test_proof.py::TestProof::test_rule_proof_types"\n' 'workflow = ".github/workflows/pr.yml"\njob = "tool-tests"\n' 'invocation = "unittest discover -s tools/tests"' @@ -434,7 +444,7 @@ def test_proof_with_named_unittest_job(self) -> None: class HiddenRuleTest(unittest.TestCase): - def assertViolation(self, repo: Repo, fragment: str) -> None: + def assert_violation(self, repo: Repo, fragment: str) -> None: messages = repo.messages() self.assertTrue(any(fragment in m for m in messages), messages) @@ -442,9 +452,10 @@ def test_unregistered_quality_guard(self) -> None: with Repo() as repo: repo.write( "scripts/run_quality_guards.py", - 'GUARDS = (("rules-registry", lambda: []), ("secret-guard", lambda: []))\n', + 'GUARDS = (("rules-registry", lambda: []), ' + '("secret-guard", lambda: []))\n', ) - self.assertViolation(repo, "quality guard 'secret-guard'") + self.assert_violation(repo, "quality guard 'secret-guard'") def test_unregistered_guard_test(self) -> None: with Repo() as repo: @@ -453,22 +464,23 @@ def test_unregistered_guard_test(self) -> None: "tests/unit/test_boundaries.py", text + "\n\ndef test_new_boundary():\n pass\n", ) - self.assertViolation( + self.assert_violation( repo, "test_boundaries.py::test_new_boundary is not referenced" ) def test_unregistered_mypy_file(self) -> None: with Repo() as repo: repo.write("pyproject.toml", '[tool.mypy]\nfiles = ["pkg", "scripts"]\n') - self.assertViolation(repo, "mypy checks 'scripts'") + self.assert_violation(repo, "mypy checks 'scripts'") def test_unregistered_ruff_ignore(self) -> None: with Repo() as repo: repo.write( "pyproject.toml", - '[tool.mypy]\nfiles = ["pkg"]\n\n[tool.ruff.lint.per-file-ignores]\n"pkg/*.py" = ["E501"]\n', + '[tool.mypy]\nfiles = ["pkg"]\n\n' + '[tool.ruff.lint.per-file-ignores]\n"pkg/*.py" = ["E501"]\n', ) - self.assertViolation(repo, "ruff per-file-ignores 'pkg/*.py' has no rule") + self.assert_violation(repo, "ruff per-file-ignores 'pkg/*.py' has no rule") def test_file_level_exceptions_only_for_tests_ci_and_scripts(self) -> None: program = 'def main():\n pass\n\n\nif __name__ == "__main__":\n main()\n' @@ -495,7 +507,7 @@ def test_file_level_exceptions_only_for_tests_ci_and_scripts(self) -> None: def test_file_level_exception_outside_allowed_files_is_rejected(self) -> None: with Repo(with_rule(per_file_ignore_rule("routes", "pkg/*.py"))) as repo: repo.write("pyproject.toml", per_file_ignore_config("pkg/*.py")) - self.assertViolation( + self.assert_violation( repo, "covers files that may not take file-level exceptions: pkg/routes.py", ) @@ -510,25 +522,27 @@ def test_file_level_exception_needs_a_program_outside_scripts(self) -> None: } ) as repo: repo.write("pyproject.toml", per_file_ignore_config("pkg/cli.py")) - self.assertViolation(repo, "may not take file-level exceptions: pkg/cli.py") + self.assert_violation( + repo, "may not take file-level exceptions: pkg/cli.py" + ) def test_file_level_exception_star_crosses_directories(self) -> None: # Ruff's * also matches "/", so "*/routes.py" reaches pkg/routes.py. with Repo(with_rule(per_file_ignore_rule("star", "*/routes.py"))) as repo: repo.write("pyproject.toml", per_file_ignore_config("*/routes.py")) - self.assertViolation( + self.assert_violation( repo, "may not take file-level exceptions: pkg/routes.py" ) def test_negated_file_level_exception_is_rejected(self) -> None: with Repo(with_rule(per_file_ignore_rule("negated", "!tests/**"))) as repo: repo.write("pyproject.toml", per_file_ignore_config("!tests/**")) - self.assertViolation(repo, "pattern '!tests/**' is negated") + self.assert_violation(repo, "pattern '!tests/**' is negated") def test_file_level_exception_must_match_a_file(self) -> None: with Repo(with_rule(per_file_ignore_rule("stale", "nowhere/**"))) as repo: repo.write("pyproject.toml", per_file_ignore_config("nowhere/**")) - self.assertViolation(repo, "pattern 'nowhere/**' matches no files") + self.assert_violation(repo, "pattern 'nowhere/**' matches no files") def test_unregistered_ci_job(self) -> None: with Repo() as repo: @@ -537,7 +551,7 @@ def test_unregistered_ci_job(self) -> None: ".github/workflows/pr.yml", text + " deploy:\n steps:\n - run: ./deploy.sh\n", ) - self.assertViolation(repo, "CI job 'deploy'") + self.assert_violation(repo, "CI job 'deploy'") def test_ignored_ci_job_with_reason_passes(self) -> None: with Repo() as repo: @@ -549,7 +563,9 @@ def test_ignored_ci_job_with_reason_passes(self) -> None: repo.edit( "rules/registry.toml", "[discovery]\n", - '[discovery]\nignore_jobs = [{ job = "deploy", reason = "deploys, enforces nothing" }]\n', + "[discovery]\n" + 'ignore_jobs = [{ job = "deploy", ' + 'reason = "deploys, enforces nothing" }]\n', ) repo.render() self.assertEqual(repo.messages(), []) @@ -608,7 +624,8 @@ def test_expired_exception(self) -> None: repo.edit( "rules/registry.toml", 'kind = "mypy"\npaths = ["pkg"]', - 'kind = "mypy"\npaths = ["pkg"]\n[[rule.exception]]\npath = "pkg/routes.py"\n' + 'kind = "mypy"\npaths = ["pkg"]\n[[rule.exception]]\n' + 'path = "pkg/routes.py"\n' 'reason = "migrating"\nexpires = 2026-05-01', ) repo.render() @@ -623,7 +640,8 @@ def test_exception_is_listed_in_rules_md(self) -> None: repo.edit( "rules/registry.toml", 'kind = "mypy"\npaths = ["pkg"]', - 'kind = "mypy"\npaths = ["pkg"]\n[[rule.exception]]\npath = "pkg/routes.py"\n' + 'kind = "mypy"\npaths = ["pkg"]\n[[rule.exception]]\n' + 'path = "pkg/routes.py"\n' 'reason = "migrating"\nexpires = 2026-12-01', ) repo.render() @@ -653,9 +671,12 @@ def test_baseline_may_not_grow(self) -> None: def test_downgrade_needs_reason(self) -> None: base = self.registry("") weakened = REGISTRY.replace( - 'status = "enforced"\nautofix = true\n[rule.enforcer]\nkind = "ruff"\nmode = "format"\n[[rule.ci]]\n' - 'workflow = ".github/workflows/pr.yml"\njob = "lint"\ninvocation = "ruff format --check"', - 'status = "advisory"\nautofix = true\n[rule.enforcer]\nkind = "ruff"\nmode = "format"', + 'status = "enforced"\nautofix = true\n[rule.enforcer]\nkind = "ruff"\n' + 'mode = "format"\n[[rule.ci]]\n' + 'workflow = ".github/workflows/pr.yml"\njob = "lint"\n' + 'invocation = "ruff format --check"', + 'status = "advisory"\nautofix = true\n[rule.enforcer]\nkind = "ruff"\n' + 'mode = "format"', ) current, _ = load_text(weakened) assert current is not None diff --git a/scripts/rules/tests/test_workflows.py b/scripts/rules/tests/test_workflows.py index 137337c52..bf03f566e 100644 --- a/scripts/rules/tests/test_workflows.py +++ b/scripts/rules/tests/test_workflows.py @@ -1,4 +1,7 @@ -"""Tests for the GitHub Actions subset reader, pinned against both repos' real PR workflows.""" +"""Tests for the GitHub Actions subset reader. + +They are pinned against both repos' real PR workflows. +""" from __future__ import annotations @@ -55,7 +58,10 @@ def test_block_scalar_stops_at_sibling_key(self) -> None: self.assertTrue(jobs["build"].invokes("make build")) def test_comments_are_ignored(self) -> None: - text = "jobs:\n a:\n steps:\n # run: make test\n - run: echo hi # make test\n" + text = ( + "jobs:\n a:\n steps:\n # run: make test\n" + " - run: echo hi # make test\n" + ) jobs, _ = parse_jobs(text) self.assertFalse(jobs["a"].invokes("make test")) @@ -67,7 +73,8 @@ def test_quoted_hash_is_not_a_comment(self) -> None: def test_inline_matrix_list(self) -> None: jobs, _ = parse_jobs( - 'jobs:\n a:\n strategy:\n matrix:\n python: ["3.12", "3.13"]\n' + "jobs:\n a:\n strategy:\n matrix:\n" + ' python: ["3.12", "3.13"]\n' ) self.assertEqual(jobs["a"].matrix["python"], ("3.12", "3.13")) diff --git a/scripts/rules/validate.py b/scripts/rules/validate.py index 406b40ef6..94f6824c9 100644 --- a/scripts/rules/validate.py +++ b/scripts/rules/validate.py @@ -178,7 +178,8 @@ def v(message: str, fix: str) -> None: if kind is Kind.RUFF and rule.baseline is not None: v( "ruff is never ratcheted", - "remove the baseline and fix every violation in a standalone PR that also changes the ruff select list", + "remove the baseline and fix every violation in a standalone PR that " + "also changes the ruff select list", ) if tier is Tier.JUDGMENT and kind not in (Kind.LLM_REVIEW, Kind.NONE): v( @@ -330,7 +331,8 @@ def v(rule_id: str | None, message: str, fix: str, check: str = "schema") -> Non elif baseline_entries(path) != rule.baseline.count: v( rule.id, - f"baseline count is {rule.baseline.count} but {rule.baseline.file} lists {baseline_entries(path)}", + f"baseline count is {rule.baseline.count} but " + f"{rule.baseline.file} lists {baseline_entries(path)}", "make the count match the file", "baseline", ) From 229aff27cd91a4748ccf63a3d142953cb7cd86ef Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 14 Oct 2026 20:50:22 +0400 Subject: [PATCH 18/23] Fix the ruff group 7 findings in the code #743 and #744 added #743 and #744 landed on main after this stack branched. Group 7, which this PR enables, flags their new code; each fix follows this PR's conventions: - E501: wrap the long strings in their routing, sentinel, gateway and test code; every message and expected value reads the same. - N818: RoutingConflict never leaves the executor, so it becomes RoutingConflictError, as this PR renames the other in-project errors. - INP001: the sentinel script in src/modal/utils keeps the inline reason the other scripts there carry. - PLC0415: the sentinel's deferred contract and runtime imports and the package updater's optional receipt import keep inline reasons. Co-Authored-By: Claude Opus 5.5 --- .../scripts/update_policyengine_package.py | 2 +- .../tests/test_deployment_assets.py | 11 +++-- .../tests/test_routing_workflow_invariants.py | 24 +++++++---- .../tests/test_simulation_rollback.py | 5 ++- .../src/modal/utils/release_sentinel.py | 15 ++++--- .../modal/utils/update_version_registry.py | 38 +++++++++-------- .../tests/test_update_version_registry.py | 42 ++++++++++--------- .../endpoints.py | 3 +- 8 files changed, 84 insertions(+), 56 deletions(-) diff --git a/.github/scripts/update_policyengine_package.py b/.github/scripts/update_policyengine_package.py index a51e5ffe1..50e5708cb 100644 --- a/.github/scripts/update_policyengine_package.py +++ b/.github/scripts/update_policyengine_package.py @@ -249,7 +249,7 @@ def dataset_sha256(country: str) -> str: def certificate_sha256(country: str) -> str: # Receipt-v1 certificate digest; older wrappers lack the helper. try: - from policyengine.provenance.receipt import ( + from policyengine.provenance.receipt import ( # noqa: PLC0415 # older policyengine wrappers lack the receipt helper, so the ImportError is handled here certificate_sha256 as digest, ) except ImportError: diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index 8af6655ce..2818540a1 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -1426,8 +1426,8 @@ def test_each_publication_is_named_by_its_run_attempt_and_environment(): if step.get("name") == "Plan the routing change" ) assert ( - '--publication-id "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RELEASE_ENVIRONMENT}"' - in plan["run"] + '--publication-id "${GITHUB_RUN_ID}-' + '${GITHUB_RUN_ATTEMPT}-${RELEASE_ENVIRONMENT}"' in plan["run"] ) assert plan["env"]["RELEASE_ENVIRONMENT"] == "${{ inputs.release_environment }}" @@ -2083,7 +2083,9 @@ def test_deploy_gates_pass_the_release_identity_to_the_smoke(): if "cloud-run-simulation-entry-smoke.sh" in step.get("run", "") ) expected = { - "EXPECTED_IDENTITY": "${{ inputs.deploy_existing_stack && 'required' || 'none' }}", + "EXPECTED_IDENTITY": ( + "${{ inputs.deploy_existing_stack && 'required' || 'none' }}" + ), "EXPECTED_APP_NAME": ( "${{ inputs.deploy_existing_stack && " "needs.deploy_executor.outputs.app_name || '' }}" @@ -2093,7 +2095,8 @@ def test_deploy_gates_pass_the_release_identity_to_the_smoke(): "needs.prepare.outputs.policyengine_version || '' }}" ), "EXPECTED_UK_MODEL_VERSION": ( - "${{ inputs.deploy_existing_stack && needs.prepare.outputs.uk_version || '' }}" + "${{ inputs.deploy_existing_stack && " + "needs.prepare.outputs.uk_version || '' }}" ), "EXPECTED_UK_DATASET_SHA256": ( "${{ inputs.deploy_existing_stack && " diff --git a/projects/policyengine-simulation-entry/tests/test_routing_workflow_invariants.py b/projects/policyengine-simulation-entry/tests/test_routing_workflow_invariants.py index 72fdfc29c..17c299eda 100644 --- a/projects/policyengine-simulation-entry/tests/test_routing_workflow_invariants.py +++ b/projects/policyengine-simulation-entry/tests/test_routing_workflow_invariants.py @@ -15,7 +15,8 @@ WORKFLOW_DIRECTORY = REPOSITORY_ROOT / ".github" / "workflows" REGISTRY_MODULE = ( REPOSITORY_ROOT - / "projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py" + / "projects/policyengine-simulation-executor/src/modal/utils" + / "update_version_registry.py" ) REGISTRY_COMMAND = "src.modal.utils.update_version_registry" REGISTRY_INVOCATION = re.compile( @@ -359,7 +360,8 @@ def _writer_environment(workflow: dict, job: dict, step: dict, inputs: dict): and "|" in _shell_tokens(line) ): assert len(REGISTRY_INVOCATION.findall(line)) <= 1, ( - "Routing pipeline commands can execute in parallel; use foreground sequential commands" + "Routing pipeline commands can execute in parallel; " + "use foreground sequential commands" ) for arguments in commands: values = [] @@ -370,7 +372,8 @@ def _writer_environment(workflow: dict, job: dict, step: dict, inputs: dict): elif argument.startswith("--environment="): values.append(argument.partition("=")[2]) assert values == ["${MODAL_ENVIRONMENT}"], ( - "Routing writes must explicitly use the proven MODAL_ENVIRONMENT exactly once" + "Routing writes must explicitly use the proven MODAL_ENVIRONMENT " + "exactly once" ) return _resolve_environment(environment["MODAL_ENVIRONMENT"], inputs) @@ -453,8 +456,9 @@ def test_stage12_callers_propagate_distinct_modal_routing_environments(): ("--complete-plan plan.json", True), ( ( - "--finalize-publication --publication-run 744 --release-environment prod " - "--gates-passed true --github-output github-output.txt" + "--finalize-publication --publication-run 744 " + "--release-environment prod --gates-passed true " + "--github-output github-output.txt" ), True, ), @@ -658,8 +662,14 @@ def test_routing_pipeline_cannot_make_writers_parallel(): f"{{ uv run python -m {REGISTRY_COMMAND} --apply-plan plan.json; }}", f'uv run python -c "import {REGISTRY_COMMAND}"', f"uv run python -m {REGISTRY_COMMAND} --dry-run $flags", - f"uv run python -m {REGISTRY_COMMAND} --environment $MODAL_ENVIRONMENT --apply-plan plan.json", - f"uv run python -m {REGISTRY_COMMAND} --environment '${{MODAL_ENVIRONMENT}}' --apply-plan plan.json", + ( + f"uv run python -m {REGISTRY_COMMAND} --environment $MODAL_ENVIRONMENT " + "--apply-plan plan.json" + ), + ( + f"uv run python -m {REGISTRY_COMMAND} --environment " + "'${MODAL_ENVIRONMENT}' --apply-plan plan.json" + ), ], ) def test_routing_discovery_rejects_unsupported_shell_forms(script): diff --git a/projects/policyengine-simulation-entry/tests/test_simulation_rollback.py b/projects/policyengine-simulation-entry/tests/test_simulation_rollback.py index 1288e79f3..1be1b381a 100644 --- a/projects/policyengine-simulation-entry/tests/test_simulation_rollback.py +++ b/projects/policyengine-simulation-entry/tests/test_simulation_rollback.py @@ -275,7 +275,10 @@ def _fake_gh( jobs.extend( [ _inventory_job( - job="Complete beta deployment / Publish routing after all services pass" + job=( + "Complete beta deployment / " + "Publish routing after all services pass" + ) ), _inventory_job(step="Record the plan artifact"), _inventory_job(conclusion="failure"), diff --git a/projects/policyengine-simulation-executor/src/modal/utils/release_sentinel.py b/projects/policyengine-simulation-executor/src/modal/utils/release_sentinel.py index c930145f9..7b66468fe 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/release_sentinel.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/release_sentinel.py @@ -19,7 +19,7 @@ python -m src.modal.utils.release_sentinel expected --output sentinel.json python -m src.modal.utils.release_sentinel check --base-url URL \ --expected sentinel.json -""" +""" # noqa: INP001 # a script run as python -m src.modal.utils.release_sentinel; utils is not a regular package from __future__ import annotations @@ -80,7 +80,9 @@ class SentinelError(RuntimeError): def worker_payload(request: Mapping[str, Any]) -> dict[str, Any]: """Build the payload the gateway forwards to the worker for ``request``.""" - from policyengine_simulation_contract.gateway_models import SimulationRequest + from policyengine_simulation_contract.gateway_models import ( # noqa: PLC0415 # the contract models load only when a payload is built + SimulationRequest, + ) return SimulationRequest.model_validate(dict(request)).model_dump( exclude={"version", "policyengine_version", "telemetry"}, @@ -114,7 +116,7 @@ def agree(expected: float, actual: float) -> bool: def _bundle_identity() -> dict[str, Any]: - from policyengine_simulation_executor.release_bundle import ( + from policyengine_simulation_executor.release_bundle import ( # noqa: PLC0415 # resolved at call time, as the tests' module patches rely on get_country_release_bundle, ) @@ -130,10 +132,10 @@ def _bundle_identity() -> dict[str, Any]: def _local_runner() -> Callable[[dict[str, Any]], Mapping[str, Any]]: - from policyengine_simulation_executor.simulation_runtime import ( + from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # loads the policyengine runtime only for the expected command; the tests patch it run_simulation_impl, ) - from policyengine_simulation_executor.stage12_qualification import ( + from policyengine_simulation_executor.stage12_qualification import ( # noqa: PLC0415 # loads the policyengine runtime only for the expected command; the tests patch it _local_qualification_runtime, ) @@ -278,7 +280,8 @@ def check_deployment( ): if served.get(field) != wanted: problems.append( - f"{label}: served {field} {served.get(field)!r}, expected {wanted!r}" + f"{label}: served {field} {served.get(field)!r}, " + f"expected {wanted!r}" ) deadline = clock() + timeout_seconds result = None diff --git a/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py b/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py index 9c5ff4641..f1255d76e 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py @@ -407,7 +407,7 @@ def build_restored_routing_state( ROUTING_PLAN_SCHEMA = "policyengine-simulation.routing-plan/v1" -class RoutingConflict(RuntimeError): +class RoutingConflictError(RuntimeError): """The live routing state is not the one a plan expects.""" @@ -536,8 +536,8 @@ def _require_serialized_routing_workflow() -> None: "Routing writes require GitHub Actions in " f"{CANONICAL_GITHUB_REPOSITORY} through {workflows}. " "Use those workflows, whose simulation-deploy-main concurrency group " - "serializes routing writes. Read-only, --dry-run and --plan-out modes are available " - "locally." + "serializes routing writes. Read-only, --dry-run and --plan-out modes " + "are available locally." ) raise RuntimeError(msg) @@ -552,7 +552,7 @@ def _unfinished_publication(store: "modal.Dict | None") -> dict | None: "restored", }: msg = "invalid routing publication record; refusing routing writes" - raise RoutingConflict(msg) + raise RoutingConflictError(msg) validate_routing_plan(record.get("plan")) return record["plan"] if record["status"] == "pending" else None @@ -568,7 +568,7 @@ def _require_finished_publication( "the recorded plan through simulation-rollback with target and " "resolve_pending_publication=true before deploying again." ) - raise RoutingConflict(msg) + raise RoutingConflictError(msg) def _write_publication_record(store: "modal.Dict", *, plan: dict, status: str) -> None: @@ -576,7 +576,7 @@ def _write_publication_record(store: "modal.Dict", *, plan: dict, status: str) - store[ROUTING_PUBLICATION_KEY] = record if store.get(ROUTING_PUBLICATION_KEY) != record: msg = "the routing publication record reads back differently" - raise RoutingConflict(msg) + raise RoutingConflictError(msg) def record_routing_plan(*, environment: str, plan: dict) -> None: @@ -595,7 +595,7 @@ def record_routing_plan(*, environment: str, plan: dict) -> None: != plan["before_sha256"] ): msg = "the routing state changed before recording the plan" - raise RoutingConflict(msg) + raise RoutingConflictError(msg) _write_publication_record(store, plan=plan, status="pending") @@ -606,13 +606,13 @@ def complete_routing_plan(*, environment: str, plan: dict) -> None: store = _routing_store(environment, create_if_missing=False) if _unfinished_publication(store) != plan: msg = "no matching unfinished routing publication to complete" - raise RoutingConflict(msg) + raise RoutingConflictError(msg) if ( routing_state_digest(store.get(ROUTING_STATE_ACTIVE_KEY)) != plan["after_sha256"] ): msg = "the completed release's routing state is not live" - raise RoutingConflict(msg) + raise RoutingConflictError(msg) _write_publication_record(store, plan=plan, status="completed") @@ -628,12 +628,12 @@ def apply_routing_plan(*, environment: str, plan: dict) -> None: "the routing state changed after this release was planned; " "nothing was published" ) - raise RoutingConflict(msg) + raise RoutingConflictError(msg) store[ROUTING_STATE_ACTIVE_KEY] = plan["after"] written = store.get(ROUTING_STATE_ACTIVE_KEY) if routing_state_digest(written) != plan["after_sha256"]: msg = "the published routing state reads back differently" - raise RoutingConflict(msg) + raise RoutingConflictError(msg) def restore_routing_plans(*, environment: str, plans: list[dict]) -> str: @@ -658,7 +658,7 @@ def restore_routing_plans(*, environment: str, plans: list[dict]) -> str: if all(plan["before"] is None for plan in plans): return "unchanged" msg = "there is no routing state to restore into" - raise RoutingConflict(msg) + raise RoutingConflictError(msg) remaining = [ plan for plan in plans if plan["before_sha256"] != plan["after_sha256"] ] @@ -679,7 +679,7 @@ def restore_routing_plans(*, environment: str, plans: list[dict]) -> str: != plan["before_sha256"] ): msg = "the restored routing state reads back differently" - raise RoutingConflict(msg) + raise RoutingConflictError(msg) outcome = "restored" if outcome == "unchanged" and not any( plan["before_sha256"] == digest for plan in plans @@ -688,7 +688,7 @@ def restore_routing_plans(*, environment: str, plans: list[dict]) -> str: "the routing state is neither what this release replaced nor what " "it published; another change was made, so nothing was restored" ) - raise RoutingConflict(msg) + raise RoutingConflictError(msg) if pending is not None: _write_publication_record(store, plan=pending, status="restored") return outcome @@ -1163,7 +1163,9 @@ def main() -> None: parser.add_argument( "--dry-run", action="store_true", - help="With publishing arguments: print a routing plan's digests; change nothing.", + help=( + "With publishing arguments: print a routing plan's digests; change nothing." + ), ) parser.add_argument( "--plan-out", @@ -1279,7 +1281,8 @@ def active_state() -> object: if args.finalize_publication: if args.release_environment is None or args.gates_passed is None: parser.error( - "--finalize-publication requires --release-environment and --gates-passed" + "--finalize-publication requires --release-environment and " + "--gates-passed" ) if args.publication_run is None and args.gates_passed != "false": parser.error("operator recovery requires --gates-passed false") @@ -1300,7 +1303,8 @@ def active_state() -> object: ) if outcome == "foreign-publication": _report( - "Pending routing publication belongs to another run or environment; routing is unchanged." + "Pending routing publication belongs to another run or " + "environment; routing is unchanged." ) _report(f"Routing publication finalization: {outcome}") if args.github_output: diff --git a/projects/policyengine-simulation-executor/tests/test_update_version_registry.py b/projects/policyengine-simulation-executor/tests/test_update_version_registry.py index 578e32784..efc1c1a63 100644 --- a/projects/policyengine-simulation-executor/tests/test_update_version_registry.py +++ b/projects/policyengine-simulation-executor/tests/test_update_version_registry.py @@ -723,7 +723,7 @@ def test_a_plan_is_applied_only_onto_its_own_starting_state(monkeypatch, patched plan = _plan(monkeypatch, state, "6.3.0", "1.1.0", "2.2.0") moved = _publish(monkeypatch, state, "6.2.7", "1.1.0", "2.1.0") store = _store(patched_modal, moved) - with pytest.raises(registry.RoutingConflict, match="nothing was published"): + with pytest.raises(registry.RoutingConflictError, match="nothing was published"): registry.apply_routing_plan(environment="staging", plan=plan) assert store.get("active") == json.loads(json.dumps(moved)) @@ -735,7 +735,7 @@ def test_a_restore_never_overwrites_a_later_change(monkeypatch, patched_modal): registry.apply_routing_plan(environment="staging", plan=plan) later = _publish(monkeypatch, store.get("active"), "6.4.0", "1.2.0", "2.2.0") store["active"] = later - with pytest.raises(registry.RoutingConflict, match="nothing was restored"): + with pytest.raises(registry.RoutingConflictError, match="nothing was restored"): registry.restore_routing_plan(environment="staging", plan=plan) assert store.get("active") == later @@ -914,7 +914,7 @@ def test_an_old_plan_cannot_withdraw_a_healthy_republication( ) healthy = _plan(monkeypatch, a, "6.1.0", "1.1.0", "2.2.0") registry.apply_routing_plan(environment="staging", plan=healthy) - with pytest.raises(registry.RoutingConflict, match="nothing was restored"): + with pytest.raises(registry.RoutingConflictError, match="nothing was restored"): registry.restore_routing_plan(environment="staging", plan=failed) assert store.get("active") == healthy["after"] @@ -1273,7 +1273,8 @@ def __setitem__(self, key, value): store = DroppedWrite({"active": state}) patched_modal["staging/simulation-api-routing-state"] = store with pytest.raises( - registry.RoutingConflict, match="published routing state reads back differently" + registry.RoutingConflictError, + match="published routing state reads back differently", ): registry.apply_routing_plan(environment="staging", plan=plan) assert store.get("active") == state @@ -1300,7 +1301,8 @@ def pop(self, key): store = DroppedRestore({"active": plan["after"]}) patched_modal["staging/simulation-api-routing-state"] = store with pytest.raises( - registry.RoutingConflict, match="restored routing state reads back differently" + registry.RoutingConflictError, + match="restored routing state reads back differently", ): registry.restore_routing_plan(environment="staging", plan=plan) assert store.get("active") == plan["after"] @@ -1317,7 +1319,7 @@ def test_runner_loss_cannot_be_adopted_by_the_next_release(monkeypatch, patched_ attempted_b = _plan(monkeypatch, store.get("active"), "6.2.0", "1.2.0", "2.3.0") for operation in (registry.record_routing_plan, registry.apply_routing_plan): with pytest.raises( - registry.RoutingConflict, match="Unfinished routing publication" + registry.RoutingConflictError, match="Unfinished routing publication" ): operation(environment="staging", plan=attempted_b) assert store.get("active") == failed_a["after"] @@ -1335,7 +1337,7 @@ def test_runner_loss_cannot_be_adopted_by_the_next_release(monkeypatch, patched_ registry.apply_routing_plan(environment="staging", plan=healthy_b) registry.complete_routing_plan(environment="staging", plan=healthy_b) assert "6.1.0" not in store.get("active")["routes"]["policyengine"] - with pytest.raises(registry.RoutingConflict, match="nothing was restored"): + with pytest.raises(registry.RoutingConflictError, match="nothing was restored"): registry.restore_routing_plan(environment="staging", plan=failed_a) # A subsequent failed release restores healthy B and cannot resurrect A. failed_c = _plan(monkeypatch, healthy_b["after"], "6.3.0", "1.3.0", "2.4.0") @@ -1372,7 +1374,7 @@ def test_a_failed_same_version_redeploy_is_resolved_by_its_restore( # A release gate fails here. Until the restore runs, nothing else may write. later = _plan(monkeypatch, live, "6.1.0", "1.1.0", "2.2.0") with pytest.raises( - registry.RoutingConflict, match="Unfinished routing publication" + registry.RoutingConflictError, match="Unfinished routing publication" ): registry.record_routing_plan(environment="staging", plan=later) assert ( @@ -1398,7 +1400,7 @@ def test_recorded_but_unpublished_plan_blocks_writes_until_restored( registry.record_routing_plan(environment="staging", plan=pending) # Covers loss after recording, before upload or before apply. with pytest.raises( - registry.RoutingConflict, match="Unfinished routing publication" + registry.RoutingConflictError, match="Unfinished routing publication" ): registry.record_routing_plan(environment="staging", plan=pending) assert ( @@ -1422,7 +1424,7 @@ def test_every_other_routing_write_refuses_an_unfinished_release( saved = json.loads(json.dumps(store.snapshot())) other = _plan(monkeypatch, pending["after"], "6.2.0", "1.2.0", "2.3.0") with pytest.raises( - registry.RoutingConflict, match="Unfinished routing publication" + registry.RoutingConflictError, match="Unfinished routing publication" ): if writer == "publish": registry.publish_routing_state( @@ -1447,14 +1449,16 @@ def test_only_matching_live_publication_can_be_completed(monkeypatch, patched_mo before = _publish(monkeypatch, None, "6.0.0", "1.1.0", "2.1.0") store = _store(patched_modal, before) pending = _plan(monkeypatch, before, "6.1.0", "1.1.0", "2.2.0") - with pytest.raises(registry.RoutingConflict, match="no matching unfinished"): + with pytest.raises(registry.RoutingConflictError, match="no matching unfinished"): registry.complete_routing_plan(environment="staging", plan=pending) registry.record_routing_plan(environment="staging", plan=pending) - with pytest.raises(registry.RoutingConflict, match="routing state is not live"): + with pytest.raises( + registry.RoutingConflictError, match="routing state is not live" + ): registry.complete_routing_plan(environment="staging", plan=pending) registry.apply_routing_plan(environment="staging", plan=pending) other = _plan(monkeypatch, before, "6.1.0", "1.1.0", "2.2.0") - with pytest.raises(registry.RoutingConflict, match="no matching unfinished"): + with pytest.raises(registry.RoutingConflictError, match="no matching unfinished"): registry.complete_routing_plan(environment="staging", plan=other) assert store.get("publication")["status"] == "pending" registry.complete_routing_plan(environment="staging", plan=pending) @@ -1482,7 +1486,7 @@ def test_publication_ownership_compares_the_entire_plan( "restore": registry.restore_routing_plan, "complete": registry.complete_routing_plan, }[operation] - with pytest.raises(registry.RoutingConflict): + with pytest.raises(registry.RoutingConflictError): writer(environment="staging", plan=changed) assert store.snapshot() == saved @@ -1574,7 +1578,7 @@ def test_lost_finalizer_is_recovered_by_operator_without_an_artifact( registry.apply_routing_plan(environment="staging", plan=pending) later = _plan(monkeypatch, pending["after"], "6.2.0", "1.2.0", "2.3.0") with pytest.raises( - registry.RoutingConflict, match="Unfinished routing publication" + registry.RoutingConflictError, match="Unfinished routing publication" ): registry.record_routing_plan(environment="staging", plan=later) assert store.get("publication") == {"status": "pending", "plan": pending} @@ -1652,7 +1656,7 @@ def test_finalizer_conflict_is_loud_and_keeps_the_journal_pending( registry.apply_routing_plan(environment="staging", plan=pending) store["active"] = _publish(monkeypatch, pending["after"], "6.2.0", "1.2.0", "2.3.0") saved = json.loads(json.dumps(store.snapshot())) - with pytest.raises(registry.RoutingConflict, match="nothing was restored"): + with pytest.raises(registry.RoutingConflictError, match="nothing was restored"): registry.finalize_routing_publication( environment="staging", release_environment="beta", @@ -1863,7 +1867,7 @@ def __setitem__(self, key, value): "restored": registry.restore_routing_plan, }[status] with pytest.raises( - registry.RoutingConflict, match="publication record reads back differently" + registry.RoutingConflictError, match="publication record reads back differently" ): operation(environment="staging", plan=plan) assert store.get("active") == (plan["after"] if status == "completed" else before) @@ -1875,7 +1879,7 @@ def test_recording_checks_the_live_starting_state(monkeypatch, patched_modal): plan = _plan(monkeypatch, before, "6.1.0", "1.1.0", "2.2.0") later = _publish(monkeypatch, before, "6.2.0", "1.2.0", "2.3.0") store = _store(patched_modal, later) - with pytest.raises(registry.RoutingConflict, match="changed before recording"): + with pytest.raises(registry.RoutingConflictError, match="changed before recording"): registry.record_routing_plan(environment="staging", plan=plan) assert store.snapshot() == {"active": later} @@ -1889,7 +1893,7 @@ def test_malformed_publication_record_refuses_writes( store = _store(patched_modal, None) store["publication"] = record plan = _plan(monkeypatch, None, "6.1.0", "1.1.0", "2.2.0") - with pytest.raises((registry.RoutingConflict, ValueError)): + with pytest.raises((registry.RoutingConflictError, ValueError)): registry.apply_routing_plan(environment="staging", plan=plan) assert store.snapshot() == {"publication": record} diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py index 367d89e93..cce2b275c 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py @@ -615,7 +615,8 @@ def _bundle_identities(state: dict) -> dict[str, dict[str, PolicyEngineBundle]]: # One historical entry with an unresolvable dataset must not # take /versions down for every caller; report it without one. logger.warning( - "Cannot resolve the %s dataset of policyengine %s for /versions: %s", + "Cannot resolve the %s dataset of policyengine %s " + "for /versions: %s", country, policyengine_version, exc, From e1a3825b137bc0b1bd29bcf6f8b2adecea849f41 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:23:15 +0400 Subject: [PATCH 19/23] Keep the long Stage 12 test name #789 added under E501 #789 landed on main after this stack branched. E501, which this PR enables, flags the signature line of its lost-commit coordinator test, whose name cannot be wrapped; the line keeps a reasoned suppression rather than a shorter, vaguer name. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_stage12_runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_runtime.py b/projects/policyengine-simulation-executor/tests/test_stage12_runtime.py index f76dbeda3..4ef65c5df 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_runtime.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_runtime.py @@ -1938,7 +1938,7 @@ def record_store_runtime(runtime): assert store.parent.status is ComparisonRunLifecycleStatus.SUCCEEDED -def test_coordinator_does_not_deduplicate_against_its_own_claim_after_a_lost_commit() -> ( +def test_coordinator_does_not_deduplicate_against_its_own_claim_after_a_lost_commit() -> ( # noqa: E501 # the test name spells out the lost-commit scenario None ): # SQLAlchemy raises a failed COMMIT without a statement, like a refused From 482834b231766e8ba72a2e0a68f8369047fc8189 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 15 Oct 2026 03:07:44 +0400 Subject: [PATCH 20/23] Fix the ruff group 7 findings in the code #791 added #791 landed on main after this stack branched. E501, which this PR enables, flags three long string literals in its Stage 12 failure-cause tests; each is wrapped by implicit concatenation, with its value unchanged. N818 flags Stage12FailureCause, whose name telemetry records as the error type of every recorded cause, so it keeps its name under a reasoned suppression, like TemporaryStage12DispatchFailed. Co-Authored-By: Claude Opus 5.5 --- .../stage12_runtime/failure_causes.py | 2 +- .../tests/test_stage12_failure_causes.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/failure_causes.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/failure_causes.py index 1fa12f85c..4147c439e 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/failure_causes.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/failure_causes.py @@ -180,7 +180,7 @@ def tracked_stage( yield -class Stage12FailureCause(Exception): +class Stage12FailureCause(Exception): # noqa: N818 # telemetry records its name as the error type of every recorded Stage 12 failure cause """A redacted stand-in for a failure, safe to export to telemetry.""" diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_failure_causes.py b/projects/policyengine-simulation-executor/tests/test_stage12_failure_causes.py index b6636d9a5..7ba827738 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_failure_causes.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_failure_causes.py @@ -107,7 +107,8 @@ def test_database_url_credentials_are_redacted_from_the_summary() -> None: def test_connection_string_password_assignment_is_redacted() -> None: error = RuntimeError( - f"invalid dsn: host=pooler.example.com user=postgres password={DATABASE_PASSWORD}" + "invalid dsn: host=pooler.example.com user=postgres " + f"password={DATABASE_PASSWORD}" ) summary = failure_summary(error) @@ -138,7 +139,10 @@ def test_secret_environment_values_are_redacted(monkeypatch) -> None: [ ("Authorization: Bearer abc.def-ghi", "abc.def-ghi"), ( - "download https://storage.example.com/data.h5?X-Goog-Signature=signed123 failed", + ( + "download https://storage.example.com/data.h5" + "?X-Goog-Signature=signed123 failed" + ), "signed123", ), ("token rejected: hf_abcdefghijklmnopqrstuvwxyz0123", "hf_abcdefghij"), @@ -335,7 +339,8 @@ def test_database_errors_report_the_driver_cause_without_parameters() -> None: ( ( 'null value in column "status" violates not-null constraint\n' - "DETAIL: Failing row contains (stored-identifier, stored\nvalue, null)." + "DETAIL: Failing row contains " + "(stored-identifier, stored\nvalue, null)." ), ( 'null value in column "status" violates not-null constraint ' From 7803713d8b47749a221bdd98f55e30d569d03b9b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:04:30 +0400 Subject: [PATCH 21/23] Fix the ruff group 7 findings in the code #758 added #758 landed on main after this stack branched. FAST002, which this PR enables, flags the six entry routes that take the caller's identity as a "caller: CallerIdentity | None = Depends(...)" parameter. FastAPI's Annotated form can't be used: this module postpones annotations, and FastAPI resolves them against the module's globals, where the per-app permission dependency doesn't exist. No handler reads the parameter. Each route now declares its permission dependency in the decorator's dependencies, as the gateway's routes declare require_auth and as these routes declared authentication before #758. FastAPI still runs the same dependency before the body is read, so each route checks the same permission, logs or rejects a missing one the same way, and the OpenAPI document is unchanged. The CallerIdentity import is no longer used. Co-Authored-By: Claude Opus 5.5 --- .../src/policyengine_simulation_entry/app.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index 140932660..86fbe3efe 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -61,7 +61,6 @@ from policyengine_simulation_entry.schemas import ( BackendTelemetryAttributes, BackendTelemetryPayload, - CallerIdentity, RequestIdentifiers, TemporaryStage12ReportResponse, TemporaryStage12SubmissionResponse, @@ -428,11 +427,11 @@ def temporary_submission_payload( @app.post( "/internal/stage12/reports", include_in_schema=False, + dependencies=[Depends(stage12_operator)], ) async def submit_temporary_stage12_report( body: SimulationRequest, request: Request, - caller: CallerIdentity | None = Depends(stage12_operator), ) -> Response: if not runtime_settings.stage12_enabled: return JSONResponse( @@ -507,11 +506,11 @@ async def submit_temporary_stage12_report( @app.get( "/internal/stage12/reports/{evaluation_id}", include_in_schema=False, + dependencies=[Depends(stage12_operator)], ) async def get_temporary_stage12_report( evaluation_id: UUID, request: Request, - caller: CallerIdentity | None = Depends(stage12_operator), ) -> Response: comparison = runtime_comparison if comparison is None: @@ -586,11 +585,11 @@ async def get_temporary_stage12_report( "model": SimulationErrorResponse, }, }, + dependencies=[Depends(simulation_submitter)], ) async def submit_comparison( body: SimulationRequest, request: Request, - caller: CallerIdentity | None = Depends(simulation_submitter), ) -> Response: observability_id = start_workflow_observability(request) request_payload = _model_json(body) @@ -655,11 +654,11 @@ async def submit_comparison( "model": SimulationErrorResponse, }, }, + dependencies=[Depends(simulation_submitter)], ) async def submit_budget_window( body: BudgetWindowBatchRequest, request: Request, - caller: CallerIdentity | None = Depends(simulation_submitter), ) -> Response: start_workflow_observability(request) return await forward( @@ -694,12 +693,9 @@ async def submit_budget_window( 404: {"description": "Job not found"}, 500: {"description": "Job failed"}, }, + dependencies=[Depends(simulation_reader)], ) - async def get_job( - job_id: str, - request: Request, - caller: CallerIdentity | None = Depends(simulation_reader), - ) -> Response: + async def get_job(job_id: str, request: Request) -> Response: return await forward( request, "GET", @@ -729,11 +725,11 @@ async def get_job( 404: {"description": "Batch job not found"}, 500: {"description": "Batch failed"}, }, + dependencies=[Depends(simulation_reader)], ) async def get_budget_window_job( batch_job_id: str, request: Request, - caller: CallerIdentity | None = Depends(simulation_reader), ) -> Response: return await forward( request, From 19b613a28387972a901b3c0b8070e205c181480c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:08:24 +0400 Subject: [PATCH 22/23] Fix the ruff group 7 findings in the code #781 added #781 landed on main after this stack branched. PLC0415, which this PR enables, flags four of its deferred imports. As for the executor's other deferred imports, only an import that loads nothing new and that no test patches moves to the top: - the Modal app's NULL_JOB_PROGRESS and StoredJobProgressReporter, whose module the app's top-level imports already load, move to the top; - the job_progress_state store functions stay deferred, resolved per call so the tests can patch job_progress_state; - StageReportingSimulation stays deferred in _build_simulation, resolved per call like its neighbours so the tests can patch it; - the bundle-metadata builder's JOB_PROGRESS_CAPABILITY_VERSION stays deferred: the contract package, and pydantic with it, loads only when bundle metadata is built. Each kept import carries its reason inline. Co-Authored-By: Claude Opus 5.5 --- .../policyengine-simulation-executor/src/modal/app.py | 10 +++++----- .../src/modal/utils/update_version_registry.py | 2 +- .../simulation_runtime.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/policyengine-simulation-executor/src/modal/app.py b/projects/policyengine-simulation-executor/src/modal/app.py index 3e02f47df..699afe762 100644 --- a/projects/policyengine-simulation-executor/src/modal/app.py +++ b/projects/policyengine-simulation-executor/src/modal/app.py @@ -15,6 +15,10 @@ HF_MODAL_SECRET_NAME, HF_RUNTIME_ENV_NAME, ) +from policyengine_simulation_executor.job_progress import ( + NULL_JOB_PROGRESS, + StoredJobProgressReporter, +) from policyengine_simulation_executor.release_bundle import ( get_bundled_country_model_version, get_bundled_package_version, @@ -279,14 +283,10 @@ def _job_progress_reporter() -> "JobProgressReporter": writes a record under its own call id. ``/budget-window-jobs`` does not read those records yet; the writes are a few small Dict puts per year. """ - from policyengine_simulation_contract.job_progress_state import ( + from policyengine_simulation_contract.job_progress_state import ( # noqa: PLC0415 # resolved per call so tests can patch job_progress_state get_job_progress, put_job_progress, ) - from policyengine_simulation_executor.job_progress import ( - NULL_JOB_PROGRESS, - StoredJobProgressReporter, - ) try: job_id = modal.current_function_call_id() diff --git a/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py b/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py index f1255d76e..4099f9763 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py @@ -135,7 +135,7 @@ def build_bundle_manifest_metadata( policyengine_version: str, ) -> BundleManifestMetadata: """Describe the installed release bundle as served by ``app_name``.""" - from policyengine_simulation_contract.job_progress import ( + from policyengine_simulation_contract.job_progress import ( # noqa: PLC0415 # the contract package loads only when bundle metadata is built JOB_PROGRESS_CAPABILITY_VERSION, ) from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # the executor package loads only when bundle metadata is built diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py index e16e9220f..c33d1e954 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py @@ -751,7 +751,7 @@ def _build_simulation( from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # resolved per call so tests can patch spm.normalize_runtime_spm normalize_runtime_spm, ) - from policyengine_simulation_executor.stage_reporting_simulation import ( + from policyengine_simulation_executor.stage_reporting_simulation import ( # noqa: PLC0415 # resolved per call so tests can patch stage_reporting_simulation StageReportingSimulation, ) From e6e185004e683afc156c5058d5487221e6d75eed Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:19:45 +0400 Subject: [PATCH 23/23] List the lazily loaded packages for ruff instead of suppressing each import Fourteen heavy or optional packages and modules are only ever imported inside the function that needs them: the policyengine bundle, countries, provenance and tax-benefit-model modules, policyengine_us, spm_calculator, h5py, google.cloud, three OpenTelemetry exporters and instrumentations, pythonjsonlogger, and the executor's spm and baseline_artifacts modules. No application module imports any of them at module level. ruff.toml now lists them in banned-module-level-imports and selects TID253, so importing one at module level fails (TYPE_CHECKING imports are still fine), and PLC0415 accepts their in-function imports without a suppression. That removes 51 PLC0415 suppressions. Tests may still import them at module level (TID253 joins the test exemptions). Packages that some application module does import at module level (pandas, microdf, huggingface_hub, policyengine.core and others) aren't listed, so their deferred imports keep their reasons. The nine Modal utility programs, run with python -m from a directory that isn't a package, take INP001 in their existing file-level entry instead of one suppression each. Comments and configuration only: no code changes. Co-Authored-By: Claude Opus 5.5 --- .../scripts/update_policyengine_package.py | 4 +-- AGENTS.md | 8 ++--- changelog_entry.yaml | 2 +- docs/generated/rules.md | 32 ++++++++--------- .../policyengine_fastapi/opentelemetry/gcp.py | 8 ++--- .../fixtures/identity_stubs.py | 6 ++-- .../fixtures/spm_doubles.py | 2 +- .../src/modal/_image_setup.py | 4 +-- .../src/modal/budget_window_batch.py | 2 +- .../src/modal/segmented_national.py | 2 +- .../src/modal/smoke_app.py | 2 +- .../modal/utils/extract_bundle_versions.py | 2 +- .../src/modal/utils/extract_stage12_bundle.py | 2 +- .../src/modal/utils/qualify_stage12_parity.py | 2 +- .../src/modal/utils/record_deployment.py | 2 +- .../src/modal/utils/release_sentinel.py | 2 +- .../modal/utils/update_v2_version_manifest.py | 2 +- .../modal/utils/update_version_registry.py | 4 +-- .../src/modal/utils/validate_hf_access.py | 2 +- .../modal/utils/validate_v2_worker_release.py | 2 +- .../artifact_keys.py | 8 ++--- .../artifact_store.py | 2 +- .../baseline_artifacts.py | 2 +- .../precompute.py | 8 ++--- .../release_bundle.py | 8 ++--- .../segmented_national_reduce.py | 2 +- .../simulation_output_geographic.py | 2 +- .../simulation_runtime.py | 14 ++++---- .../policyengine_simulation_executor/spm.py | 8 ++--- .../stage12_bundle/manifest.py | 2 +- .../stage12_runtime/aggregation.py | 4 +-- .../stage12_runtime/output_planning.py | 6 ++-- .../stage12_runtime/simulation.py | 2 +- .../uk_local_authority_metadata.py | 2 +- ruff.toml | 35 ++++++++++++++++--- rules/registry.toml | 12 +++---- 36 files changed, 118 insertions(+), 91 deletions(-) diff --git a/.github/scripts/update_policyengine_package.py b/.github/scripts/update_policyengine_package.py index 50e5708cb..9cd78e1a7 100644 --- a/.github/scripts/update_policyengine_package.py +++ b/.github/scripts/update_policyengine_package.py @@ -201,7 +201,7 @@ def latest_version(package: str) -> None: def bundle_versions() -> None: """Print the package and data versions selected by PolicyEngine.py.""" - from policyengine.bundle import ( # noqa: PLC0415 # only bundle-versions runs under uv with policyengine installed; the other commands use the runner's python3 + from policyengine.bundle import ( get_current_bundle, ) @@ -249,7 +249,7 @@ def dataset_sha256(country: str) -> str: def certificate_sha256(country: str) -> str: # Receipt-v1 certificate digest; older wrappers lack the helper. try: - from policyengine.provenance.receipt import ( # noqa: PLC0415 # older policyengine wrappers lack the receipt helper, so the ImportError is handled here + from policyengine.provenance.receipt import ( certificate_sha256 as digest, ) except ImportError: diff --git a/AGENTS.md b/AGENTS.md index 546f3a860..2dddc0b51 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -142,15 +142,15 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. - `sim.quality.ci-cd-scripts-ruff-exemptions`: CI/CD scripts under .github may print their output (T201); every other ruff rule applies to them. (blocking: ruff per-file ignore of T201 for `.github/**`) - `sim.quality.command-line-scripts-ruff-exemptions`: Command-line scripts under a scripts directory may print their output (T201), run other programs (S603, S607) and use today's date (DTZ011); every other ruff rule applies to them. (blocking: ruff per-file ignore of DTZ011, S603, S607, T201 for `**/scripts/**`) -- `sim.quality.deployed-tests-ruff-exemptions`: The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (sim.quality.tests-ruff-exemptions); every other ruff rule applies to it. (blocking: ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001 for `**/authenticated_tests/**`) +- `sim.quality.deployed-tests-ruff-exemptions`: The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (sim.quality.tests-ruff-exemptions); every other ruff rule applies to it. (blocking: ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001, TID253 for `**/authenticated_tests/**`) - `sim.quality.every-project-tested`: Every project and library with unit tests runs them in the PR test matrix. (doc-only, not checked: follow it yourself) - `sim.quality.ruff-format-src`: Source code under projects/*/src and libs/*/src is formatted with ruff format. (blocking: ruff format) - `sim.quality.ruff-format-tests`: Test code is formatted with ruff format too; CI currently checks only src directories. (doc-only, not checked: follow it yourself) -- `sim.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TRY, UP, W)) +- `sim.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TID253, TRY, UP, W)) - `sim.quality.runtime-evaluated-models-resolve`: Every Pydantic model is runtime-evaluated for ruff: one of its bases, read through the imports of the model's own module, is listed in ruff.toml's runtime-evaluated-base-classes or is a runtime-evaluated class in that same module; and every Pydantic model resolves its annotations once its package is imported. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_runtime_evaluated_annotations.py`, `projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py`) -- `sim.quality.src-programs-ruff-exemptions`: Programs under src (the Modal deployment utilities, and the OpenAPI and Stage 12 infrastructure commands) may print their output (T201) and open URLs from their arguments (S310); every other ruff rule applies to them. (blocking: ruff per-file ignore of S310, T201 for `projects/*/src/{modal/utils/*,policyengine_simulation_entry/generate_openapi,policyengine_simulation_entry/stage12_infrastructure,policyengine_simulation_gateway/generate_openapi}.py`) +- `sim.quality.src-programs-ruff-exemptions`: Programs under src (the Modal deployment utilities, and the OpenAPI and Stage 12 infrastructure commands) may print their output (T201), open URLs from their arguments (S310) and, for the Modal utilities run with python -m, live in a directory that isn't a package (INP001); every other ruff rule applies to them. (blocking: ruff per-file ignore of INP001, S310, T201 for `projects/*/src/{modal/utils/*,policyengine_simulation_entry/generate_openapi,policyengine_simulation_entry/stage12_infrastructure,policyengine_simulation_gateway/generate_openapi}.py`) - `sim.quality.tests-pass`: Each project's unit tests pass in its own locked environment. (blocking: CI job `test`) -- `sim.quality.tests-ruff-exemptions`: Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415). They may also run repository scripts (S603, S607) and use fake credentials (S105, S106), temporary paths (S108), naive datetimes (DTZ001, DTZ007), pickle (S301) and exec (S102). Every other ruff rule applies to tests. (blocking: ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001 for `**/tests/**`) +- `sim.quality.tests-ruff-exemptions`: Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415). They may also run repository scripts (S603, S607) and use fake credentials (S105, S106), temporary paths (S108), naive datetimes (DTZ001, DTZ007), pickle (S301) and exec (S102), and may import the lazily loaded packages at module level (TID253). Every other ruff rule applies to tests. (blocking: ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001, TID253 for `**/tests/**`) ### CI workflows and scripts diff --git a/changelog_entry.yaml b/changelog_entry.yaml index cc6b5269e..cefcea73d 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -4,7 +4,7 @@ - Made CLAUDE.md and the Copilot instructions thin pointers to AGENTS.md, required workflow logic longer than one line to live in tested and shellchecked scripts, and registered the tracking-issue rule - Added a rules registry (rules/registry.toml) with a standard-library tool that validates it, checks every rule is enforced where it claims, and renders docs/generated/rules.md and the Rules block in AGENTS.md changed: - - Enabled ruff's structure and import rules (INP, PLC, SLF, N, A, FAST, E); long lines are wrapped, deferred imports that Modal images, test seams, import cycles or heavy dependencies need carry a reason, script directories are declared to ruff as namespace packages, four internal exceptions gained an Error suffix, and published field names, OpenAPI documents and logged error types are unchanged; tests are also exempt from SLF001, INP001 and PLC0415 + - Enabled ruff's structure and import rules (INP, PLC, SLF, N, A, FAST, E); long lines are wrapped, deferred imports that Modal images, test seams, import cycles or heavy dependencies need carry a reason, script directories are declared to ruff as namespace packages, four internal exceptions gained an Error suffix, and published field names, OpenAPI documents and logged error types are unchanged; tests are also exempt from SLF001, INP001 and PLC0415; heavy or optional packages that load only inside functions are listed in ruff's banned-module-level-imports, which keeps them out of module-level imports (TID253) and lets their deferred imports go without a suppression, and the Modal utility programs take INP001 file-wide - Enabled ruff's security, logging and debugging rules (S, T20, LOG, G, DTZ, ERA, TD, FIX, EXE); asserts outside tests became explicit checks, command-line scripts keep their printed output, test subprocess calls go through helpers, and every deliberate exception carries a reasoned suppression; the gateway's auth misconfiguration log no longer repeats its message, and the deployed-service tests share the unit-test ruff exemptions; tests, CI/CD scripts and command-line scripts take their deliberate exceptions as registered file-level ruff ignores instead of inline suppressions, and the rules guard rejects file-level ignores anywhere else - Enabled ruff's exception rules (TRY, EM, BLE); exception messages are built before the raise, raises inside try blocks moved into helpers, and every deliberate catch-all or traceback-free error log carries a reasoned suppression; message text is unchanged, and error types change only in an operator script's two type checks - Enabled ruff's docstring rules (D) and documented every module, class and function outside tests; published contract models and routes now carry these descriptions in their JSON schema and OpenAPI, with no structural schema change diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 25f07198c..0afff1f79 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -248,15 +248,15 @@ flowchart LR | --- | --- | --- | --- | --- | --- | --- | --- | | `sim.quality.ci-cd-scripts-ruff-exemptions` | CI/CD scripts under .github may print their output (T201); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of T201 for `.github/**` | `pr.yml:lint` | — | | `sim.quality.command-line-scripts-ruff-exemptions` | Command-line scripts under a scripts directory may print their output (T201), run other programs (S603, S607) and use today's date (DTZ011); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of DTZ011, S603, S607, T201 for `**/scripts/**` | `pr.yml:lint` | — | -| `sim.quality.deployed-tests-ruff-exemptions` | The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (sim.quality.tests-ruff-exemptions); every other ruff rule applies to it. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001 for `**/authenticated_tests/**` | `pr.yml:lint` | — | +| `sim.quality.deployed-tests-ruff-exemptions` | The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (sim.quality.tests-ruff-exemptions); every other ruff rule applies to it. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001, TID253 for `**/authenticated_tests/**` | `pr.yml:lint` | — | | `sim.quality.every-project-tested` | Every project and library with unit tests runs them in the PR test matrix. | mechanical | doc-only | no | nothing | — | — | | `sim.quality.ruff-format-src` | Source code under projects/*/src and libs/*/src is formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — | | `sim.quality.ruff-format-tests` | Test code is formatted with ruff format too; CI currently checks only src directories. | mechanical | doc-only | no | nothing | — | — | -| `sim.quality.ruff-lint` | All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. | mechanical | enforced | yes | ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TRY, UP, W) | `pr.yml:lint` | — | +| `sim.quality.ruff-lint` | All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. | mechanical | enforced | yes | ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TID253, TRY, UP, W) | `pr.yml:lint` | — | | `sim.quality.runtime-evaluated-models-resolve` | Every Pydantic model is runtime-evaluated for ruff: one of its bases, read through the imports of the model's own module, is listed in ruff.toml's runtime-evaluated-base-classes or is a runtime-evaluated class in that same module; and every Pydantic model resolves its annotations once its package is imported. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_runtime_evaluated_annotations.py`, `projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py` | `pr.yml:test` | yes | -| `sim.quality.src-programs-ruff-exemptions` | Programs under src (the Modal deployment utilities, and the OpenAPI and Stage 12 infrastructure commands) may print their output (T201) and open URLs from their arguments (S310); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of S310, T201 for `projects/*/src/{modal/utils/*,policyengine_simulation_entry/generate_openapi,policyengine_simulation_entry/stage12_infrastructure,policyengine_simulation_gateway/generate_openapi}.py` | `pr.yml:lint` | — | +| `sim.quality.src-programs-ruff-exemptions` | Programs under src (the Modal deployment utilities, and the OpenAPI and Stage 12 infrastructure commands) may print their output (T201), open URLs from their arguments (S310) and, for the Modal utilities run with python -m, live in a directory that isn't a package (INP001); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of INP001, S310, T201 for `projects/*/src/{modal/utils/*,policyengine_simulation_entry/generate_openapi,policyengine_simulation_entry/stage12_infrastructure,policyengine_simulation_gateway/generate_openapi}.py` | `pr.yml:lint` | — | | `sim.quality.tests-pass` | Each project's unit tests pass in its own locked environment. | mechanical | enforced | yes | CI job `test` | `pr.yml:test` | — | -| `sim.quality.tests-ruff-exemptions` | Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415). They may also run repository scripts (S603, S607) and use fake credentials (S105, S106), temporary paths (S108), naive datetimes (DTZ001, DTZ007), pickle (S301) and exec (S102). Every other ruff rule applies to tests. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001 for `**/tests/**` | `pr.yml:lint` | — | +| `sim.quality.tests-ruff-exemptions` | Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415). They may also run repository scripts (S603, S607) and use fake credentials (S105, S106), temporary paths (S108), naive datetimes (DTZ001, DTZ007), pickle (S301) and exec (S102), and may import the lazily loaded packages at module level (TID253). Every other ruff rule applies to tests. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001, TID253 for `**/tests/**` | `pr.yml:lint` | — | ```mermaid flowchart LR @@ -266,43 +266,43 @@ flowchart LR r_sim_quality_command_line_scripts_ruff_exemptions("sim.quality.command-line-scripts-ruff-exemptions (enforced)") e_ruff_per_file_ignore_of_DTZ011__S603__S607__T201_for____scripts___["ruff per-file ignore of DTZ011, S603, S607, T201 for **/scripts/**"] r_sim_quality_deployed_tests_ruff_exemptions("sim.quality.deployed-tests-ruff-exemptions (enforced)") - e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001_for____authenticated_tests___["ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001 for **/authenticated_tests/**"] + e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001__TID253_for____authenticated_tests___["ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001, TID253 for **/authenticated_tests/**"] r_sim_quality_every_project_tested("sim.quality.every-project-tested (doc-only)") gap_unenforced>"not enforced"] r_sim_quality_ruff_format_src("sim.quality.ruff-format-src (enforced)") e_ruff_format["ruff format"] r_sim_quality_ruff_format_tests("sim.quality.ruff-format-tests (doc-only)") r_sim_quality_ruff_lint("sim.quality.ruff-lint (enforced)") - e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TRY__UP__W_["ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TRY, UP, W)"] + e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TID253__TRY__UP__W_["ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TID253, TRY, UP, W)"] r_sim_quality_runtime_evaluated_models_resolve("sim.quality.runtime-evaluated-models-resolve (enforced)") e_pytest_projects_policyengine_simulation_entry_tests_test_runtime_evaluated_annotations_py__projects_policyengine_simulation_executor_tests_test_runtime_evaluated_annotations_py["pytest projects/policyengine-simulation-entry/tests/test_runtime_evaluated_annotations.py, projects/policyengine-simulation-executor/tests/test_runtime_evaluated_annotations.py"] j_CI_pr_yml_test["CI pr.yml:test"] r_sim_quality_src_programs_ruff_exemptions("sim.quality.src-programs-ruff-exemptions (enforced)") - e_ruff_per_file_ignore_of_S310__T201_for_projects___src__modal_utils___policyengine_simulation_entry_generate_openapi_policyengine_simulation_entry_stage12_infrastructure_policyengine_simulation_gateway_generate_openapi__py["ruff per-file ignore of S310, T201 for projects/*/src/{modal/utils/*,policyengine_simulation_entry/generate_openapi,policyengine_simulation_entry/stage12_infrastructure,policyengine_simulation_gateway/generate_openapi}.py"] + e_ruff_per_file_ignore_of_INP001__S310__T201_for_projects___src__modal_utils___policyengine_simulation_entry_generate_openapi_policyengine_simulation_entry_stage12_infrastructure_policyengine_simulation_gateway_generate_openapi__py["ruff per-file ignore of INP001, S310, T201 for projects/*/src/{modal/utils/*,policyengine_simulation_entry/generate_openapi,policyengine_simulation_entry/stage12_infrastructure,policyengine_simulation_gateway/generate_openapi}.py"] r_sim_quality_tests_pass("sim.quality.tests-pass (enforced)") e_CI_job_test["CI job test"] r_sim_quality_tests_ruff_exemptions("sim.quality.tests-ruff-exemptions (enforced)") - e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001_for____tests___["ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001 for **/tests/**"] + e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001__TID253_for____tests___["ruff per-file ignore of ANN, D1, DTZ001, DTZ007, INP001, PLC0415, PLR2004, S101, S102, S105, S106, S108, S301, S603, S607, SLF001, TID253 for **/tests/**"] r_sim_quality_ci_cd_scripts_ruff_exemptions --> e_ruff_per_file_ignore_of_T201_for__github___ e_ruff_per_file_ignore_of_T201_for__github___ --> j_CI_pr_yml_lint r_sim_quality_command_line_scripts_ruff_exemptions --> e_ruff_per_file_ignore_of_DTZ011__S603__S607__T201_for____scripts___ e_ruff_per_file_ignore_of_DTZ011__S603__S607__T201_for____scripts___ --> j_CI_pr_yml_lint - r_sim_quality_deployed_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001_for____authenticated_tests___ - e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001_for____authenticated_tests___ --> j_CI_pr_yml_lint + r_sim_quality_deployed_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001__TID253_for____authenticated_tests___ + e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001__TID253_for____authenticated_tests___ --> j_CI_pr_yml_lint r_sim_quality_every_project_tested -.-> gap_unenforced r_sim_quality_ruff_format_src --> e_ruff_format e_ruff_format --> j_CI_pr_yml_lint r_sim_quality_ruff_format_tests -.-> gap_unenforced - r_sim_quality_ruff_lint --> e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TRY__UP__W_ - e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TRY__UP__W_ --> j_CI_pr_yml_lint + r_sim_quality_ruff_lint --> e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TID253__TRY__UP__W_ + e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TID253__TRY__UP__W_ --> j_CI_pr_yml_lint r_sim_quality_runtime_evaluated_models_resolve --> e_pytest_projects_policyengine_simulation_entry_tests_test_runtime_evaluated_annotations_py__projects_policyengine_simulation_executor_tests_test_runtime_evaluated_annotations_py e_pytest_projects_policyengine_simulation_entry_tests_test_runtime_evaluated_annotations_py__projects_policyengine_simulation_executor_tests_test_runtime_evaluated_annotations_py --> j_CI_pr_yml_test - r_sim_quality_src_programs_ruff_exemptions --> e_ruff_per_file_ignore_of_S310__T201_for_projects___src__modal_utils___policyengine_simulation_entry_generate_openapi_policyengine_simulation_entry_stage12_infrastructure_policyengine_simulation_gateway_generate_openapi__py - e_ruff_per_file_ignore_of_S310__T201_for_projects___src__modal_utils___policyengine_simulation_entry_generate_openapi_policyengine_simulation_entry_stage12_infrastructure_policyengine_simulation_gateway_generate_openapi__py --> j_CI_pr_yml_lint + r_sim_quality_src_programs_ruff_exemptions --> e_ruff_per_file_ignore_of_INP001__S310__T201_for_projects___src__modal_utils___policyengine_simulation_entry_generate_openapi_policyengine_simulation_entry_stage12_infrastructure_policyengine_simulation_gateway_generate_openapi__py + e_ruff_per_file_ignore_of_INP001__S310__T201_for_projects___src__modal_utils___policyengine_simulation_entry_generate_openapi_policyengine_simulation_entry_stage12_infrastructure_policyengine_simulation_gateway_generate_openapi__py --> j_CI_pr_yml_lint r_sim_quality_tests_pass --> e_CI_job_test e_CI_job_test --> j_CI_pr_yml_test - r_sim_quality_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001_for____tests___ - e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001_for____tests___ --> j_CI_pr_yml_lint + r_sim_quality_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001__TID253_for____tests___ + e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ007__INP001__PLC0415__PLR2004__S101__S102__S105__S106__S108__S301__S603__S607__SLF001__TID253_for____tests___ --> j_CI_pr_yml_lint ``` ## CI workflows and scripts diff --git a/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py b/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py index 109ce9583..e54b8e02b 100644 --- a/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py +++ b/libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py @@ -93,10 +93,10 @@ def instrument(self) -> None: Uses ``logging.basicConfig``, so the handler is not installed if the root logger already has handlers. """ - from opentelemetry.instrumentation.logging import ( # noqa: PLC0415 # the logging instrumentation loads only when GCP logging is set up + from opentelemetry.instrumentation.logging import ( LoggingInstrumentor, ) - from pythonjsonlogger.json import ( # noqa: PLC0415 # the JSON formatter loads only when GCP logging is set up + from pythonjsonlogger.json import ( JsonFormatter, ) @@ -131,10 +131,10 @@ def export_ot_to_gcp(resource: Resource) -> None: providers for ``resource`` and registers an ``exit`` callback that flushes both at shutdown. """ - from opentelemetry.exporter.cloud_monitoring import ( # noqa: PLC0415 # the Google Cloud exporters load only when GCP export is set up + from opentelemetry.exporter.cloud_monitoring import ( CloudMonitoringMetricsExporter, ) - from opentelemetry.exporter.cloud_trace import ( # noqa: PLC0415 # the Google Cloud exporters load only when GCP export is set up + from opentelemetry.exporter.cloud_trace import ( CloudTraceSpanExporter, ) diff --git a/projects/policyengine-simulation-executor/fixtures/identity_stubs.py b/projects/policyengine-simulation-executor/fixtures/identity_stubs.py index dc4255760..6965e8581 100644 --- a/projects/policyengine-simulation-executor/fixtures/identity_stubs.py +++ b/projects/policyengine-simulation-executor/fixtures/identity_stubs.py @@ -34,8 +34,8 @@ def install_identity_stubs(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace: # This module imports ``get_release_manifest`` by value while constructing # country models. Import it before replacing the manifest module's function # so full-suite ordering cannot bind this test's stub into production code. - import policyengine.tax_benefit_models.common.model_version # noqa: F401, PLC0415 # must load before the manifest stubs are installed (see above) - from policyengine.provenance import ( # noqa: PLC0415 # patch target, imported after model_version above + import policyengine.tax_benefit_models.common.model_version # noqa: F401 # must load before the manifest stubs are installed (see above) + from policyengine.provenance import ( manifest as manifest_module, ) from policyengine_simulation_executor import ( # noqa: PLC0415 # patch target, imported when the stubs are installed @@ -88,7 +88,7 @@ def install_identity_stubs(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace: ), ) - from policyengine_simulation_executor import ( # noqa: PLC0415 # patch target, imported when the stubs are installed + from policyengine_simulation_executor import ( spm as executor_spm, ) diff --git a/projects/policyengine-simulation-executor/fixtures/spm_doubles.py b/projects/policyengine-simulation-executor/fixtures/spm_doubles.py index b4cf34424..677403837 100644 --- a/projects/policyengine-simulation-executor/fixtures/spm_doubles.py +++ b/projects/policyengine-simulation-executor/fixtures/spm_doubles.py @@ -11,7 +11,7 @@ def installed_spm_selection() -> dict[str, object] | None: """Return the installed US bundle's default SPM selection, if any.""" - from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # resolved per call so tests can patch spm.runtime_spm_capability + from policyengine_simulation_executor.spm import ( runtime_spm_capability, ) diff --git a/projects/policyengine-simulation-executor/src/modal/_image_setup.py b/projects/policyengine-simulation-executor/src/modal/_image_setup.py index 6019805c2..fec0df487 100644 --- a/projects/policyengine-simulation-executor/src/modal/_image_setup.py +++ b/projects/policyengine-simulation-executor/src/modal/_image_setup.py @@ -134,7 +134,7 @@ def fetch_artifacts( "project": info.get("project_id"), } # No blob: ambient ADC (the local fault-drill path). - from google.cloud import ( # noqa: PLC0415 # image-build function: this module has no module-level imports by design + from google.cloud import ( storage, ) @@ -187,7 +187,7 @@ def snapshot_models() -> None: logger = logging.getLogger(__name__) logger.info("Pre-loading US tax-benefit system...") - from policyengine_us import ( # noqa: PLC0415 # image-build function: this module has no module-level imports by design + from policyengine_us import ( CountryTaxBenefitSystem as USSystem, ) diff --git a/projects/policyengine-simulation-executor/src/modal/budget_window_batch.py b/projects/policyengine-simulation-executor/src/modal/budget_window_batch.py index 92635502e..efe7f8e91 100644 --- a/projects/policyengine-simulation-executor/src/modal/budget_window_batch.py +++ b/projects/policyengine-simulation-executor/src/modal/budget_window_batch.py @@ -24,7 +24,7 @@ def run_budget_window_batch_impl( Must run inside a Modal call: the current function call id becomes the batch job id that child jobs and stored batch state are keyed by. """ - from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # Modal worker code: executor spm is loaded per call, in the container + from policyengine_simulation_executor.spm import ( normalize_runtime_spm, ) diff --git a/projects/policyengine-simulation-executor/src/modal/segmented_national.py b/projects/policyengine-simulation-executor/src/modal/segmented_national.py index 01efffc28..8d40d85bc 100644 --- a/projects/policyengine-simulation-executor/src/modal/segmented_national.py +++ b/projects/policyengine-simulation-executor/src/modal/segmented_national.py @@ -600,7 +600,7 @@ def dispatch_run_simulation( Segmented national fan-out serves eligible requests (see ``should_run_segmented_national``); everything else runs monolithically. """ - from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # resolved per call so tests can patch spm.normalize_runtime_spm + from policyengine_simulation_executor.spm import ( normalize_runtime_spm, ) diff --git a/projects/policyengine-simulation-executor/src/modal/smoke_app.py b/projects/policyengine-simulation-executor/src/modal/smoke_app.py index 36c022349..1b655bdd5 100644 --- a/projects/policyengine-simulation-executor/src/modal/smoke_app.py +++ b/projects/policyengine-simulation-executor/src/modal/smoke_app.py @@ -119,7 +119,7 @@ def smoke_import_executor() -> dict: Path, ) - from policyengine.bundle import ( # noqa: PLC0415 # smoke check: inspects the bundle installed in the Modal image + from policyengine.bundle import ( inspect_bundle_status, ) diff --git a/projects/policyengine-simulation-executor/src/modal/utils/extract_bundle_versions.py b/projects/policyengine-simulation-executor/src/modal/utils/extract_bundle_versions.py index b70ef031b..4e41068fa 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/extract_bundle_versions.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/extract_bundle_versions.py @@ -1,4 +1,4 @@ -"""Print policyengine.py bundle versions for deployment scripts.""" # noqa: INP001 # a script run as python -m src.modal.utils.extract_bundle_versions; utils is not a regular package +"""Print policyengine.py bundle versions for deployment scripts.""" from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/modal/utils/extract_stage12_bundle.py b/projects/policyengine-simulation-executor/src/modal/utils/extract_stage12_bundle.py index 19ebeb54b..3850ed768 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/extract_stage12_bundle.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/extract_stage12_bundle.py @@ -1,4 +1,4 @@ -"""Export the strict Stage 12 worker bundle selected by PolicyEngine.py.""" # noqa: INP001 # a script run as python -m src.modal.utils.extract_stage12_bundle; utils is not a regular package +"""Export the strict Stage 12 worker bundle selected by PolicyEngine.py.""" from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/modal/utils/qualify_stage12_parity.py b/projects/policyengine-simulation-executor/src/modal/utils/qualify_stage12_parity.py index 8ffea2d40..994750336 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/qualify_stage12_parity.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/qualify_stage12_parity.py @@ -1,4 +1,4 @@ -"""Run one controlled Stage 12 parity qualification from canonical JSON files.""" # noqa: INP001 # a script run as python -m src.modal.utils.qualify_stage12_parity; utils is not a regular package +"""Run one controlled Stage 12 parity qualification from canonical JSON files.""" from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/modal/utils/record_deployment.py b/projects/policyengine-simulation-executor/src/modal/utils/record_deployment.py index 1e0a0fe12..3b0426453 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/record_deployment.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/record_deployment.py @@ -4,7 +4,7 @@ ``deployed/.json`` marker (last-writer-wins). The marker is the artifact GC's liveness signal, so a deploy that cannot record itself must fail the job even though Modal already serves it. -""" # noqa: INP001 # a script run as python -m src.modal.utils.record_deployment; utils is not a regular package +""" from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/modal/utils/release_sentinel.py b/projects/policyengine-simulation-executor/src/modal/utils/release_sentinel.py index 7b66468fe..c4d4f6867 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/release_sentinel.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/release_sentinel.py @@ -19,7 +19,7 @@ python -m src.modal.utils.release_sentinel expected --output sentinel.json python -m src.modal.utils.release_sentinel check --base-url URL \ --expected sentinel.json -""" # noqa: INP001 # a script run as python -m src.modal.utils.release_sentinel; utils is not a regular package +""" from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/modal/utils/update_v2_version_manifest.py b/projects/policyengine-simulation-executor/src/modal/utils/update_v2_version_manifest.py index e3cb6bf5d..c695073e7 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/update_v2_version_manifest.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/update_v2_version_manifest.py @@ -1,4 +1,4 @@ -"""Publish a validated worker to the separate Stage 12 v2 manifest.""" # noqa: INP001 # a script run as python -m src.modal.utils.update_v2_version_manifest; utils is not a regular package +"""Publish a validated worker to the separate Stage 12 v2 manifest.""" from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py b/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py index 4099f9763..5f3be563a 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py @@ -4,7 +4,7 @@ group ``simulation-deploy-main`` (with ``cancel-in-progress: false``). Library writers and the CLI permit writes only from its deployment and rollback workflows; read-only and plan modes remain available locally. -""" # noqa: INP001 # a script run as python -m src.modal.utils.update_version_registry; utils is not a regular package +""" import argparse import hashlib @@ -138,7 +138,7 @@ def build_bundle_manifest_metadata( from policyengine_simulation_contract.job_progress import ( # noqa: PLC0415 # the contract package loads only when bundle metadata is built JOB_PROGRESS_CAPABILITY_VERSION, ) - from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # the executor package loads only when bundle metadata is built + from policyengine_simulation_executor.spm import ( runtime_spm_capability, ) diff --git a/projects/policyengine-simulation-executor/src/modal/utils/validate_hf_access.py b/projects/policyengine-simulation-executor/src/modal/utils/validate_hf_access.py index e2f2be5c5..895fc9549 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/validate_hf_access.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/validate_hf_access.py @@ -1,4 +1,4 @@ -"""Validate UK private-data access directly in CI.""" # noqa: INP001 # a script run as python -m src.modal.utils.validate_hf_access; utils is not a regular package +"""Validate UK private-data access directly in CI.""" from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/modal/utils/validate_v2_worker_release.py b/projects/policyengine-simulation-executor/src/modal/utils/validate_v2_worker_release.py index e029a09b6..0192c50f4 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/validate_v2_worker_release.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/validate_v2_worker_release.py @@ -1,4 +1,4 @@ -"""Invoke and verify every deployed Stage 12 country validation function.""" # noqa: INP001 # a script run as python -m src.modal.utils.validate_v2_worker_release; utils is not a regular package +"""Invoke and verify every deployed Stage 12 country validation function.""" from __future__ import annotations diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py index 06365584a..30cc9a781 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py @@ -314,7 +314,7 @@ def _receipt_source_sha256(country: str, data_version: str) -> str | None: def _certification_fingerprint(country: str) -> str | None: - from policyengine.provenance.manifest import ( # noqa: PLC0415 # identity seam: resolved per call so tests can patch the release manifest + from policyengine.provenance.manifest import ( get_release_manifest, ) @@ -332,7 +332,7 @@ def collect_dataset_identity(country: str, year: int) -> DatasetArtifactIdentity runtime lookup uses, so filename coupling is inherited, not re-derived. """ - from policyengine.provenance.manifest import ( # noqa: PLC0415 # identity seam: resolved per call so tests can patch the release manifest + from policyengine.provenance.manifest import ( dataset_logical_name, resolve_dataset_reference, ) @@ -345,7 +345,7 @@ def collect_dataset_identity(country: str, year: int) -> DatasetArtifactIdentity stem = dataset_logical_name( resolve_dataset_reference(bundle.country, bundle.default_dataset) ) - from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # executor spm is not loaded at import; loaded per call + from policyengine_simulation_executor.spm import ( normalize_runtime_spm, ) @@ -378,7 +378,7 @@ def collect_baseline_identity( ``spm`` is normalized against the installed image's capability first, so a request that omits it still keys to the image's default SPM selection. """ - from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # executor spm is not loaded at import; loaded per call + from policyengine_simulation_executor.spm import ( normalize_runtime_spm, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_store.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_store.py index 88cb572e5..bcf95f524 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_store.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_store.py @@ -49,7 +49,7 @@ def resolve_bucket_name(explicit: str | None = None) -> str: def _default_client() -> storage.Client: - from google.cloud import ( # noqa: PLC0415 # google-cloud-storage loads only when a default client is built + from google.cloud import ( storage, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py index 2b9f4fbf3..0baa69d69 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py @@ -192,7 +192,7 @@ def ensure(self) -> None: self._record_outcome(OUTCOME_MISS) return - from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # executor spm is not loaded at import; loaded per call + from policyengine_simulation_executor.spm import ( simulation_spm_result, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py index 95d9ec6fa..cab2de6e3 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py @@ -120,7 +120,7 @@ def _cohort_resolution( def cohort_identity(year: int, group: list[str]) -> BaselineArtifactIdentity: """Identity via the executor's own resolution path (writer==reader).""" - from policyengine_simulation_executor.baseline_artifacts import ( # noqa: PLC0415 # resolved per call so tests can patch baseline_artifacts + from policyengine_simulation_executor.baseline_artifacts import ( qualifying_baseline_identity, ) from policyengine_simulation_executor.simulation_runtime import ( # noqa: PLC0415 # resolved per call so tests can patch simulation_runtime helpers @@ -353,7 +353,7 @@ def _prepare_cohort_baseline( # The extras economic_impact_analysis applies unconditionally before # ensure(); the artifact must carry them or every request would fail # the column guard and recompute. - from policyengine.tax_benefit_models.us.analysis import ( # noqa: PLC0415 # policyengine's US model loads only for a US cohort baseline + from policyengine.tax_benefit_models.us.analysis import ( configure_budgetary_impact_variables, ) @@ -365,7 +365,7 @@ def compute_baseline_impl( bucket: str, expected: BaselinePlanEntry ) -> BaselineComputeResult: """Wave 2: compute one cohort baseline and upload its output.""" - from policyengine_simulation_executor.baseline_artifacts import ( # noqa: PLC0415 # resolved per call so tests can patch baseline_artifacts.ArtifactBaselineSimulation + from policyengine_simulation_executor.baseline_artifacts import ( ArtifactBaselineSimulation, ) @@ -415,7 +415,7 @@ def verify_determinism_impl( Simulation, ) - from policyengine_simulation_executor.baseline_artifacts import ( # noqa: PLC0415 # baseline_artifacts is not loaded at import; loaded per call + from policyengine_simulation_executor.baseline_artifacts import ( OUTCOME_HIT, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/release_bundle.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/release_bundle.py index ca5f794a4..716654cd5 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/release_bundle.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/release_bundle.py @@ -145,7 +145,7 @@ def _artifact_hf_uri( data_package: DataPackageVersion, reference: object, ) -> tuple[str | None, str | None]: - from policyengine.provenance.manifest import ( # noqa: PLC0415 # policyengine stays out of this module's import-time imports + from policyengine.provenance.manifest import ( build_hf_uri, ) @@ -190,7 +190,7 @@ def _dataset_uris_from_release( data_release: Mapping, fallback_data_package: DataPackageVersion, ) -> tuple[dict[str, str], dict[str, str]]: - from policyengine.provenance.manifest import ( # noqa: PLC0415 # policyengine stays out of this module's import-time imports + from policyengine.provenance.manifest import ( build_hf_uri, ) @@ -231,7 +231,7 @@ def _dataset_uris_from_release( def _current_policyengine_bundle() -> Mapping | None: try: - from policyengine.bundle import ( # noqa: PLC0415 # inside the fallback guard: a failing policyengine.bundle import returns None + from policyengine.bundle import ( get_current_bundle, ) @@ -306,7 +306,7 @@ def _derive_data_package_version( def get_country_release_bundle(country: str) -> CountryReleaseBundle: """Return package and dataset versions from policyengine.py metadata.""" country = _normalise_country(country) - from policyengine.provenance.manifest import ( # noqa: PLC0415 # resolved per call so tests can patch the release manifest + from policyengine.provenance.manifest import ( get_release_manifest, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/segmented_national_reduce.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/segmented_national_reduce.py index 8a75b99ef..669897178 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/segmented_national_reduce.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/segmented_national_reduce.py @@ -99,7 +99,7 @@ def _us_dataset_from_frames( Weights need no special handling: they ride as ``_weight`` columns and ``MicroDataFrame(df, weights=...)`` re-derives them. """ - from policyengine.tax_benefit_models.us.datasets import ( # noqa: PLC0415 # policyengine's US model loads only when frames are rebuilt + from policyengine.tax_benefit_models.us.datasets import ( PolicyEngineUSDataset, USYearData, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py index cff991ccf..bc1588462 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py @@ -82,7 +82,7 @@ def _complete_uk_geography_output( @lru_cache(maxsize=1) def _policyengine_us_district_metadata() -> tuple[dict[int, str], frozenset[str]]: - from policyengine.countries.us.data import ( # noqa: PLC0415 # policyengine's US data loads only for US district output + from policyengine.countries.us.data import ( AT_LARGE_STATES, US_STATE_FIPS, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py index c33d1e954..3d4987b7c 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py @@ -267,7 +267,7 @@ def _compile_interval_policy( from policyengine.core.parameter import ( # noqa: PLC0415 # this module keeps policyengine out of its module-level imports (see its docstring) Parameter, ) - from policyengine.tax_benefit_models.common.reform import ( # noqa: PLC0415 # this module keeps policyengine out of its module-level imports (see its docstring) + from policyengine.tax_benefit_models.common.reform import ( compile_reform, ) @@ -410,17 +410,17 @@ def _require_certified_uk_weight_matrix( if not isinstance(scoping_strategy, WeightReplacementStrategy): return - import h5py # noqa: PLC0415 # h5py loads only for the UK weight-matrix check + import h5py import pandas as pd # noqa: PLC0415 # UK weight-matrix check: kept with its lazily loaded data dependencies from policyengine.data.uk_geography_assets import ( # noqa: PLC0415 # this module keeps policyengine out of its module-level imports (see its docstring) UKGeographyAssetSpec, default_download_dir, resolve_uk_geography_asset_paths, ) - from policyengine.provenance.dataset_materialization import ( # noqa: PLC0415 # resolved per call so tests can patch materialize_dataset + from policyengine.provenance.dataset_materialization import ( materialize_dataset, ) - from policyengine.provenance.manifest import ( # noqa: PLC0415 # resolved per call so tests can patch the release manifest + from policyengine.provenance.manifest import ( get_release_manifest, ) @@ -744,11 +744,11 @@ def _build_simulation( scoping_strategy: ScopingStrategy | None = None, region_code: str | None = None, ) -> Simulation: - from policyengine_simulation_executor.baseline_artifacts import ( # noqa: PLC0415 # resolved per call so tests can patch baseline_artifacts + from policyengine_simulation_executor.baseline_artifacts import ( ArtifactBaselineSimulation, deterministic_baseline_id, ) - from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # resolved per call so tests can patch spm.normalize_runtime_spm + from policyengine_simulation_executor.spm import ( normalize_runtime_spm, ) from policyengine_simulation_executor.stage_reporting_simulation import ( # noqa: PLC0415 # resolved per call so tests can patch stage_reporting_simulation @@ -808,7 +808,7 @@ def _run_simulation_impl_core( with runtime.span(ANNUAL_IMPACT_STAGES.name(Stage.REQUEST_PARSE)): simulation_params, telemetry, metadata = split_internal_payload(params) metadata = metadata or {} - from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # resolved per call so tests can patch spm.normalize_runtime_spm + from policyengine_simulation_executor.spm import ( normalize_runtime_spm, simulation_spm_result, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py index 881369879..0f84c3c77 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py @@ -26,7 +26,7 @@ @lru_cache(maxsize=4) def _forecast(expected_sha256: str | None) -> SPMForecast: - from spm_calculator import ( # noqa: PLC0415 # spm_calculator loads only when a forecast is needed + from spm_calculator import ( load_forecast, ) @@ -53,7 +53,7 @@ def runtime_spm_capability() -> SPMCapability | None: already succeeded. Tests that stub the installed environment must call :func:`reset_spm_runtime_caches` (the executor suite does, automatically). """ - from policyengine.bundle import ( # noqa: PLC0415 # resolved per call so tests can patch policyengine.bundle.get_current_bundle + from policyengine.bundle import ( get_current_bundle, ) from policyengine.core import ( # noqa: PLC0415 # resolved per call so tests can patch policyengine.core.Simulation @@ -92,7 +92,7 @@ def _certified_us_spm_capability(configured: object) -> SPMCapability: Raises ``ImportError``, ``ValueError``, ``TypeError`` or ``OSError`` when the US model, the configuration or the pinned forecast cannot support them. """ - from policyengine_us import ( # noqa: PLC0415 # the US model loads only to certify the SPM capability + from policyengine_us import ( Microsimulation, ) @@ -130,7 +130,7 @@ def _prevalidate_selection( an exception -- so a rejected selection is rejected again, with the same typed error, every time it is asked for. """ - from spm_calculator.policyengine_adapter import ( # noqa: PLC0415 # spm_calculator loads only when a selection is prevalidated + from spm_calculator.policyengine_adapter import ( PolicyEngineSPMProvider, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_bundle/manifest.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_bundle/manifest.py index 20c06e6c5..26d5487f4 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_bundle/manifest.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_bundle/manifest.py @@ -122,7 +122,7 @@ def resolve_stage12_bundle(raw_bundle: object) -> ResolvedStage12Bundle: def load_stage12_bundle() -> ResolvedStage12Bundle: """Read the installed PolicyEngine.py release's packaged bundle.""" - from policyengine.bundle import ( # noqa: PLC0415 # resolved per call so tests can patch policyengine.bundle.get_current_bundle + from policyengine.bundle import ( get_current_bundle, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py index e6e1b7281..6bc07a39c 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/aggregation.py @@ -117,7 +117,7 @@ def _dataset_from_frames( else MicroDataFrame(frame) ) if country == "us": - from policyengine.tax_benefit_models.us.datasets import ( # noqa: PLC0415 # policyengine's US model loads only for a US report + from policyengine.tax_benefit_models.us.datasets import ( PolicyEngineUSDataset, USYearData, ) @@ -129,7 +129,7 @@ def _dataset_from_frames( name="stage12-comparison", description="Retained Stage 12 output", ) - from policyengine.tax_benefit_models.uk.datasets import ( # noqa: PLC0415 # policyengine's UK model loads only for a UK report + from policyengine.tax_benefit_models.uk.datasets import ( PolicyEngineUKDataset, UKYearData, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py index a17efc9c0..cd0cbee0b 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py @@ -58,12 +58,12 @@ def _unmonitored_stage_scope( def _country_model(country: CountryId) -> OutputVariableModel: if country == "us": - from policyengine.tax_benefit_models import ( # noqa: PLC0415 # policyengine's US model loads only for a US simulation + from policyengine.tax_benefit_models import ( us, ) return us.model - from policyengine.tax_benefit_models import ( # noqa: PLC0415 # policyengine's UK model loads only for a UK simulation + from policyengine.tax_benefit_models import ( uk, ) @@ -133,7 +133,7 @@ def _configure_country_outputs( if include_cliff_impacts: configure_cliff_impact_variables(baseline, reform) if country == "us": - from policyengine.tax_benefit_models.us.analysis import ( # noqa: PLC0415 # policyengine's US model loads only for a US simulation + from policyengine.tax_benefit_models.us.analysis import ( configure_budgetary_impact_variables, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py index 89c3fb4d1..c1c0eed39 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/simulation.py @@ -222,7 +222,7 @@ def calculate_simulation_frames( selection = getattr(model, "spm_config", None) calculation_provenance = None if selection is not None: - from policyengine_simulation_executor.spm import ( # noqa: PLC0415 # executor spm loads only for an SPM selection + from policyengine_simulation_executor.spm import ( simulation_spm_calculation_provenance, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py index bd32b8f7c..5f036ecb5 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/uk_local_authority_metadata.py @@ -296,7 +296,7 @@ def uk_area_regions_routed() -> bool: region types have one may the UK default, a national file, carry no area codes: their runs and breakdowns then belong to that local-area dataset. """ - from policyengine.provenance.manifest import ( # noqa: PLC0415 # resolved per call so tests can patch the release manifest + from policyengine.provenance.manifest import ( get_release_manifest, ) diff --git a/ruff.toml b/ruff.toml index 07490c093..1b9aa1535 100644 --- a/ruff.toml +++ b/ruff.toml @@ -36,7 +36,7 @@ select = [ # Group 6: security, logging and debugging leftovers. "DTZ", "ERA", "EXE", "FIX", "G", "LOG", "S", "T20", "TD", # Group 7: structure and imports. - "A", "E", "FAST", "INP", "N", "PLC", "SLF", + "A", "E", "FAST", "INP", "N", "PLC", "SLF", "TID253", ] # Mutually exclusive pydocstyle pairs: keep D211 (no blank line before a class # docstring) and D212 (summary on the first line). @@ -52,16 +52,19 @@ ignore = ["D203", "D213"] # live in directories that aren't packages, and import inside a test (often # after patching the environment). They also run repository scripts and use # fake credentials, temporary paths, naive datetimes, pickle and exec on -# purpose. Every other rule applies to tests. +# purpose, and may import the lazily loaded packages at module level. Every +# other rule applies to tests. "**/tests/**" = [ "ANN", "D1", "DTZ001", "DTZ007", "INP001", "PLC0415", "PLR2004", "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607", "SLF001", + "TID253", ] # The deployed-service test suite (pytest, run against a live candidate) gets # the same test exemptions. "**/authenticated_tests/**" = [ "ANN", "D1", "DTZ001", "DTZ007", "INP001", "PLC0415", "PLR2004", "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607", "SLF001", + "TID253", ] # CI/CD scripts report on stdout. ".github/**" = ["T201"] @@ -70,8 +73,32 @@ ignore = ["D203", "D213"] "**/scripts/**" = ["DTZ011", "S603", "S607", "T201"] # Programs under src: the Modal deployment utilities, and the OpenAPI and # Stage 12 infrastructure commands. They report on stdout, and the release -# sentinel opens URLs from its arguments. -"projects/*/src/{modal/utils/*,policyengine_simulation_entry/generate_openapi,policyengine_simulation_entry/stage12_infrastructure,policyengine_simulation_gateway/generate_openapi}.py" = ["S310", "T201"] +# sentinel opens URLs from its arguments. The Modal utilities run with +# python -m from a directory that isn't a package. +"projects/*/src/{modal/utils/*,policyengine_simulation_entry/generate_openapi,policyengine_simulation_entry/stage12_infrastructure,policyengine_simulation_gateway/generate_openapi}.py" = ["INP001", "S310", "T201"] + +[lint.flake8-tidy-imports] +# Heavy or optional packages that load only inside the function that needs +# them, never at module level (TID253 enforces it; TYPE_CHECKING imports are +# fine). Ruff's PLC0415 accepts in-function imports of these without a +# suppression. A package belongs here only if no application module imports it +# at module level. +banned-module-level-imports = [ + "google.cloud", + "h5py", + "opentelemetry.exporter.cloud_monitoring", + "opentelemetry.exporter.cloud_trace", + "opentelemetry.instrumentation.logging", + "policyengine.bundle", + "policyengine.countries", + "policyengine.provenance", + "policyengine.tax_benefit_models", + "policyengine_simulation_executor.baseline_artifacts", + "policyengine_simulation_executor.spm", + "policyengine_us", + "pythonjsonlogger", + "spm_calculator", +] [lint.flake8-type-checking] # Classes whose annotations Pydantic evaluates at runtime, so their imports must diff --git a/rules/registry.toml b/rules/registry.toml index 89dfee8db..69510e010 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -733,7 +733,7 @@ autofix = true [rule.enforcer] kind = "ruff" mode = "lint" -codes = ["A", "ANN", "BLE", "D", "DTZ", "E", "E4", "E7", "E9", "EM", "ERA", "EXE", "F", "FAST", "FIX", "FLY", "FURB", "G", "I", "INP", "LOG", "N", "PIE", "PLC", "PYI", "RET", "RSE", "S", "SLF", "T20", "TC", "TD", "TRY", "UP", "W"] +codes = ["A", "ANN", "BLE", "D", "DTZ", "E", "E4", "E7", "E9", "EM", "ERA", "EXE", "F", "FAST", "FIX", "FLY", "FURB", "G", "I", "INP", "LOG", "N", "PIE", "PLC", "PYI", "RET", "RSE", "S", "SLF", "T20", "TC", "TD", "TID253", "TRY", "UP", "W"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "lint" @@ -742,7 +742,7 @@ invocation = "ruff check ." [[rule]] id = "sim.quality.tests-ruff-exemptions" category = "quality" -statement = "Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415). They may also run repository scripts (S603, S607) and use fake credentials (S105, S106), temporary paths (S108), naive datetimes (DTZ001, DTZ007), pickle (S301) and exec (S102). Every other ruff rule applies to tests." +statement = "Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415). They may also run repository scripts (S603, S607) and use fake credentials (S105, S106), temporary paths (S108), naive datetimes (DTZ001, DTZ007), pickle (S301) and exec (S102), and may import the lazily loaded packages at module level (TID253). Every other ruff rule applies to tests." rationale = "docs/engineering/skills/rules-registry.md#file-level-exceptions" scope = ["projects/*/tests/**/*.py", "libs/*/tests/**/*.py", "scripts/rules/tests/**/*.py"] tier = "mechanical" @@ -752,7 +752,7 @@ autofix = false kind = "ruff" mode = "per-file-ignore" pattern = "**/tests/**" -codes = ["ANN", "D1", "DTZ001", "DTZ007", "INP001", "PLC0415", "PLR2004", "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607", "SLF001"] +codes = ["ANN", "D1", "DTZ001", "DTZ007", "INP001", "PLC0415", "PLR2004", "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607", "SLF001", "TID253"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "lint" @@ -771,7 +771,7 @@ autofix = false kind = "ruff" mode = "per-file-ignore" pattern = "**/authenticated_tests/**" -codes = ["ANN", "D1", "DTZ001", "DTZ007", "INP001", "PLC0415", "PLR2004", "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607", "SLF001"] +codes = ["ANN", "D1", "DTZ001", "DTZ007", "INP001", "PLC0415", "PLR2004", "S101", "S102", "S105", "S106", "S108", "S301", "S603", "S607", "SLF001", "TID253"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "lint" @@ -818,7 +818,7 @@ invocation = "ruff check ." [[rule]] id = "sim.quality.src-programs-ruff-exemptions" category = "quality" -statement = "Programs under src (the Modal deployment utilities, and the OpenAPI and Stage 12 infrastructure commands) may print their output (T201) and open URLs from their arguments (S310); every other ruff rule applies to them." +statement = "Programs under src (the Modal deployment utilities, and the OpenAPI and Stage 12 infrastructure commands) may print their output (T201), open URLs from their arguments (S310) and, for the Modal utilities run with python -m, live in a directory that isn't a package (INP001); every other ruff rule applies to them." rationale = "docs/engineering/skills/rules-registry.md#file-level-exceptions" scope = ["projects/policyengine-simulation-executor/src/modal/utils/*.py", "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/generate_openapi.py", "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/stage12_infrastructure.py", "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/generate_openapi.py"] tier = "mechanical" @@ -828,7 +828,7 @@ autofix = false kind = "ruff" mode = "per-file-ignore" pattern = "projects/*/src/{modal/utils/*,policyengine_simulation_entry/generate_openapi,policyengine_simulation_entry/stage12_infrastructure,policyengine_simulation_gateway/generate_openapi}.py" -codes = ["S310", "T201"] +codes = ["INP001", "S310", "T201"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "lint"