From 4a80d801d9db7e00c4df5c97ff47435c7ec35a3d Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 07:59:27 -0600 Subject: [PATCH 01/29] docs: tests give the same result every time (D40); check_junit --same-as D40 records the maintainer's rule: the same code gives every test the same outcome on any machine, date, and run order; tests read no wall clock, reach no network, use no unseeded randomness, and bound every wait; they assert on structured results, never on a document's wording; hard inputs come from software the project builds; and acceptance names a test, a check, or a command, never a text search or a review. - The plan, CLAUDE.md, docs/testing.md, and CONTRIBUTING.md state D40. - Steps 3b-2a-2d, 3b-2a-3, 3b-2b-1, and 3b-2b-2 accept on software: the step's tests failing on the parent's code and passing on the head, or two suite runs agreeing. - scripts/check_junit.py --same-as fails unless a second JUnit report holds the same tests with the same outcome each. --- CLAUDE.md | 18 ++++++++++++++++++ CONTRIBUTING.md | 14 ++++++++++++++ docs/redesign-plan.md | 40 +++++++++++++++++++++------------------- docs/testing.md | 15 +++++++++++++++ scripts/check_junit.py | 42 ++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 108 insertions(+), 21 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a531734..a2a8614 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -58,6 +58,24 @@ a caller can fail is tested on both sides. A fix comes with a regression test th parent commit's code. Tests merged before D39 that change a file or a copy of one, or write a file, a revision, a call, a connection, or a decoy holding a flaw, to see whether a check catches it are replaced in Step 3b-2a-2d. See [Testing](docs/testing.md). +## Deterministic Tests + +Tests are software that gives the same result every time (redesign plan D40). The same code gives +every test the same outcome on any machine, on any date, and in any run order: a test, and every +code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no +network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state +outside its own temporary directory, the repository, and the session's database template, and bounds +every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or +exits with: where the code gives a structured value (an exit code, rows, parsed fields, an +exception's class and fields), the test asserts on it rather than on printed wording, and no test +checks the wording of a document. Where a test needs an input the code cannot easily be given, the +project builds software that produces it (a fake clock, the scenario model, the replay, a recorder +mode), never a planted file (D39). Evidence is what software reproduces: an acceptance criterion +names a test, a check, or a command whose output shows it, and a search of text, a reading, or a +review finds work to do but is never acceptance. `tests/test_suite_rules.py` enforces the rules it +can model, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test (Step +3b-2a-2d). + ## Current Scope **Implemented:** diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 54cb981..34e8942 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -265,6 +265,20 @@ file or a copy of one, or write a file, a revision, a call, a connection, or a d to see whether a check catches it are replaced in Step 3b-2a-2d. See the [Testing Guide](docs/testing.md#principles). +**4. Write tests that give the same result every time** (redesign plan D40). The same code gives +every test the same outcome on any machine, on any date, and in any run order: a test, and every +code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no +network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state +outside its own temporary directory, the repository, and the session's database template, and bounds +every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or +exits with: where the code gives a structured value (an exit code, rows, parsed fields, an +exception's class and fields), the test asserts on it rather than on printed wording, and no test +checks the wording of a document. Where a test needs an input the code cannot easily be given, the +project builds software that produces it (a fake clock, the scenario model, the replay, a recorder +mode), never a planted file (D39). Evidence is what software reproduces: an acceptance criterion +names a test, a check, or a command whose output shows it, and a search of text, a reading, or a +review finds work to do but is never acceptance. + ### Running Tests ```bash diff --git a/docs/redesign-plan.md b/docs/redesign-plan.md index 78bf06e..2b043cf 100644 --- a/docs/redesign-plan.md +++ b/docs/redesign-plan.md @@ -94,6 +94,7 @@ Within scope the database collects and distinguishes data and never interprets i | D37 | Decision of 2026-10-21 (the maintainer): an error is better than a silent failure or silently wrong data. Where the code cannot check something, or cannot accept what GitHub returned, it reports a failure that stops publication or turns the run red, never a pass. For verify: an unchecked item and an unverified repository fail verify in both modes, not only `--exact`, and a lookup whose answer verify cannot accept is an error (5.4, 5.8, gap 97), so D25 does not publish a database verify could not check. | | D38 | Decisions of 2026-10-21 applying D37 to the sync and to the steps not yet built (gaps 98-110). The sync exits 2 when a 24-hour account refresh fails, when the run reaches its deadline, and when a hydration contradicts the run's own index after one retry (Step 3b-2a-3); the failures a batch made final before a deadline are recorded, not left pending. An `accepted_failures` entry accepts one error class and one cause, the first line of the failure's message, of an item, an account's refresh, or from Step 4b a compare, so a permanent failure can be accepted without turning every run red while a new cause is not. A PR with more closing links than the index carries is hydrated every run. Every node id a sighting, a lookup, or a commit link reports for a stored account is stored, and `sync_runs` records the `--repo` filter, each configured entry's `github_id`, and its first commit-link lookups and those GitHub answered `NOT_FOUND`. The source refuses what it cannot read (gap 104), and the run records refuse what they cannot write and mark `-unknown` what they cannot determine (gap 105). Step 4b marks a compare file list at GitHub's 300-file cap and counts it in verify's report and the manifest, completes an activity read only at the floor or at the log's first event, as a read in ascending order finds it, fails a history read whose count still differs after a rescan, and records a compare that still fails, which an entry can accept. Step 5 measures which edits leave `updatedAt` unchanged. Step 6 compares the App's reads with the token's before any App run publishes to `db`, pins the database file the job checked, tests that every exit-code `if:` compares a non-empty string, and ties a snapshot to the job's own sync. A schedule that stops is the one silent case this repository cannot see: no run starts, so nothing turns red; 5.6 states the staleness rule a consumer applies, and building it belongs to the dashboard's integration (section 9). Left as they are, because the stored data already says what happened: a commit-linked account a lookup answers `NOT_FOUND` (D28; now counted per run and in the manifest, and from Step 6a a run whose `links_not_found` exceeds the share of its `link_lookups` Step 5 sets, with at least Step 5's minimum number of lookups, exits 2 (an account an earlier lookup answered `NOT_FOUND` counted in neither), since a lookup form GitHub stopped resolving would look the same per link), a compare at GitHub's 300-file cap and one GitHub refuses (marked by `files_at_cap` and `http_status`, counted in verify's report and the manifest), a REST git author without a name or email (stored null, as GitHub reports none), deletions recorded as a count (`items_deleted`, and from Step 4b `events_deleted` and `history_commits_deleted`; the snapshots retained for 7 days hold the identities; older ones are not kept), which accounts failed a refresh (printed; the count is stored, the run is red unless an account entry accepts the failure, and the accounts are retried), an `author_association` difference (verify reports it as drift with its own count: GitHub computes association when read and changes it without moving `updatedAt`, section 7, so no read tells a change from a defect), and the merge state as of `hydrated_at` (D32). New tables and columns of GitHub-derived data on a populated file follow gap 110's rule (5.1). | | D39 | Decision of 2026-10-21 (the maintainer). Nothing is planted or tampered, no flaw is seeded, and there is no mutation testing, under any name. No test, review, or acceptance changes a file or a copy of one (other than a stub or configuration the inputs below allow), or writes a file, a row, a revision, a call site, or a decoy holding a flaw, to see whether something catches it, and none measures how many such flaws are found. A test's or an acceptance's inputs are GitHub's recorded responses; stubs of a response GitHub or the environment can send (a status, a body, a timeout, an interrupt, a failing `git`, a clock), which a test may build in memory from a recorded unit changed into that shape; calls the code under test must refuse, and configuration it must refuse or that names a state GitHub can produce (a repository that does not resolve, one no longer listed); valid rows a test writes (through the models with `tests/db/seed.py`, or in SQL); and databases and files this project's own code wrote, at any revision. Each test asserts the exact result. A check over stored data, files, or source that only a defect of this code could fail is tested on its passing side, with the exact count of the rows, files, or values it compared, so it cannot pass on nothing; a check that GitHub, the environment, or a caller can fail is tested on both sides. A fix comes with a regression test that fails against the parent commit's code. Tests merged before D39 that change a file or a copy of one, or write a file, a revision, a call, a connection, or a decoy holding a flaw, to see whether a check catches it are replaced in Step 3b-2a-2d. | +| D40 | Decision of 2026-10-22 (the maintainer). Tests are software that gives the same result every time. The same code gives every test the same outcome on any machine, on any date, and in any run order: a test, and every code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state outside its own temporary directory, the repository, and the session's database template, and bounds every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on printed wording, and no test checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode), never a planted file (D39). Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. `tests/test_suite_rules.py` enforces the rules it can model, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test (Step 3b-2a-2d). | ## 3. Coverage rule @@ -1581,7 +1582,7 @@ Every Step 3 test of a GitHub interaction runs through the real `GitHubSource` o schema comparison. Evidence is direct, with inputs at and on each side of every boundary the design states (the coverage floor, the 10-minute unsettled window, 10 minutes past `deadline_at` for the lock, the 24-hour refresh, the delete guard's max(25, 2%)) and regression tests that fail against -the parent's code. D39 governs every test. Nothing is planted or tampered, no flaw is seeded, and +the parent's code. D39 and D40 govern every test. Nothing is planted or tampered, no flaw is seeded, and there is no mutation testing, under any name. No test, review, or acceptance changes a file or a copy of one (other than a stub or configuration the inputs below allow), or writes a file, a row, a revision, a call site, or a decoy holding a flaw, to see whether something catches it, and none measures how many @@ -1938,12 +1939,15 @@ syncs, the pull request records that PR's `updatedAt` and `mergeStateStatus` fro 8). An adversarial review of the change, by reviewers who did not write it, finds no invariant check without a test that targets it; CI passes on the exact PR head. -#### Step 3b-2a-2d: Tests without planted inputs (D39) +#### Step 3b-2a-2d: Tests without planted inputs, deterministic (D39, D40) Prerequisite: Step 3b-2a-2c has merged, and D39 is on `main`. -Scope: tests and `docs/testing.md`; nothing under `src/`, `scripts/`, or `alembic/` changes, and no -rule loses its test. Every test merged before D39 that changes a file or a copy of one, or writes a +Scope: tests, `docs/testing.md`, and `scripts/check_junit.py`'s `--same-as`; nothing under `src/` +or `alembic/` changes, and no rule loses its test. Every test meets D40, and +`tests/test_suite_rules.py` gains the determinism rules it can model: no wall-clock read, no real +network transport, and no unseeded randomness in a test file, each rule tested on its passing side +over every file under `tests/`, the files it scanned counted against `git ls-files`. Every test merged before D39 that changes a file or a copy of one, or writes a file, a revision, a call, a connection, or a decoy holding a flaw, to see whether a check catches it, is replaced under D39: a check that GitHub, the environment, or a caller can fail keeps a refusing side built from D39's inputs, and a check that only a defect of this code could fail is tested on @@ -1983,14 +1987,9 @@ environment, or a caller can fail it, or on its passing side with the exact coun pull request lists each test it replaces or removes, with the rule it held and the test that now holds it. -Acceptance: the pull request lists every test under `tests/**/*.py` that writes a source file, -a revision, a row, a database, or a package copy, changes a recorded unit, patches a call or a -constant, or runs a call or a query holding a flaw, with the D39 input it uses or the test that -replaced it; a search of `tests/**/*.py` -(recorded GitHub text under `tests/fixtures/` is outside it) for `plant`, `tamper`, `decoy`, -`corrupt`, and `mutat` finds none; an adversarial review of the change, by reviewers who did not -write it, finds no rule that lost its test and no test that holds a flaw; CI -passes on the exact PR head. +Acceptance: `tests/test_suite_rules.py`'s determinism and subprocess rules pass over every test file +and report the files they scanned; two runs of the full suite give the same outcome for every test, +compared by `scripts/check_junit.py --same-as`; CI passes on the exact PR head. #### Step 3b-2a-3: D37 in the sync @@ -2110,8 +2109,9 @@ gone-item, leaving-coverage, and 11-link tests; in `tests/sync/test_runs.py` the `trigger`, and `code_version` tests; and in `tests/test_tracking_config.py` the defaults without `[automation]` and the accepted-failure entries. -Acceptance: an adversarial review of the change, by reviewers who did not write it, finds no rule of -this step without a test that targets it; CI passes on the exact PR head. +Acceptance: the step's tests, run on the parent commit's code, fail for each rule the step adds or +changes and pass on the head, and the pull request records the command and its output (D40); CI +passes on the exact PR head. #### Step 3b-2b-1: REST sampler @@ -2153,8 +2153,9 @@ and with `--only documents` leaving the mode's directory, REST and GraphQL units sampler reading only the repositories `--repo` names and refusing `prebid-agent-skills` before any request. -Acceptance: an adversarial review of the change, by reviewers who did not write it, finds no rule -of the sampler without a test that targets it; CI passes on the exact PR head. +Acceptance: the step's tests, run on the parent commit's code, fail for each rule the step adds or +changes and pass on the head, and the pull request records the command and its output (D40); CI +passes on the exact PR head. #### Step 3b-2b-2: Verify @@ -2232,9 +2233,10 @@ closing-link error verify reports, with its PR and link (for a difference in the found by reading the PR's full closing list; section 8). `verify --exact` exits 0, or exits 2 with no failing difference but such closing-link errors, each with REST evidence that GitHub changed the link without changing the PR's `updatedAt`. A live item failure that -cannot pass is listed in `[sync].accepted_failures` with REST evidence of why, as in Step 5. An -adversarial review of the change, by reviewers who did not write it, finds no invariant check -without a test that targets it; CI passes on the exact PR head. +cannot pass is listed in `[sync].accepted_failures` with REST evidence of why, as in Step 5. The +step's tests, run on the parent commit's code, fail for each rule the step adds or changes and +pass on the head, and the pull request records the command and its output (D40); CI passes on the +exact PR head. ### Step 4: Issues, closing-issue links, and default-branch activity diff --git a/docs/testing.md b/docs/testing.md index e721b7c..0428569 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -27,6 +27,21 @@ How the test suite is built and what it counts as evidence. The plan of record i both sides. A fix comes with a regression test that fails against the parent commit's code. Tests merged before D39 that change a file or a copy of one, or write a file, a revision, a call, a connection, or a decoy holding a flaw, to see whether a check catches it are replaced in Step 3b-2a-2d. +- **The same result every time** (redesign plan D40). The same code gives every test the same + outcome on any machine, on any date, and in any run order: a test, and every code path it runs, + reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no network (GitHub + answers from recordings and stubs), uses no unseeded randomness, depends on no state outside its + own temporary directory, the repository, and the session's database template, and bounds every + wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits + with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's + class and fields), the test asserts on it rather than on printed wording, and no test checks the + wording of a document. Where a test needs an input the code cannot easily be given, the project + builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode), + never a planted file (D39). Evidence is what software reproduces: an acceptance criterion names a + test, a check, or a command whose output shows it, and a search of text, a reading, or a review + finds work to do but is never acceptance. `tests/test_suite_rules.py` enforces the rules it can + model, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test (Step + 3b-2a-2d). - **No skipped tests** (D22). CI runs `scripts/check_junit.py --max-skipped 0`, which fails a run where any test was skipped (pytest reports `xfail` as skipped) or none ran. - **Time.** Source tests pass a `FakeClock`, so waits and retries are asserted, never slept. diff --git a/scripts/check_junit.py b/scripts/check_junit.py index 9a9438a..aaa7721 100644 --- a/scripts/check_junit.py +++ b/scripts/check_junit.py @@ -2,8 +2,10 @@ pytest exits 0 when every collected test is skipped. This gate reads the report instead: at least one test executed, none failed or errored, and no more skips than allowed. +With ``--same-as``, it also fails unless a second report holds the same tests, each with the +same outcome, so two runs of the suite can be shown to agree (redesign plan D40). -Usage: python scripts/check_junit.py REPORT.xml [--max-skipped N] +Usage: python scripts/check_junit.py REPORT.xml [--max-skipped N] [--same-as OTHER.xml] """ from __future__ import annotations @@ -25,6 +27,36 @@ def read_counts(report: Path) -> dict[str, int]: return {key: sum(int(suite.get(key, "0")) for suite in suites) for key in COUNTED} +def read_outcomes(report: Path) -> dict[str, str]: + """Each test's id (classname::name) and its outcome: passed, failed, error, or skipped.""" + root = ET.parse(report).getroot() + suites = [root] if root.tag == "testsuite" else root.findall("testsuite") + if not suites: + raise ValueError(f"no element in {report}") + outcomes: dict[str, str] = {} + for suite in suites: + for case in suite.iter("testcase"): + test = f"{case.get('classname', '')}::{case.get('name', '')}" + if test in outcomes: + raise ValueError(f"{test} appears twice in {report}") + outcome = "passed" + for kind in ("failure", "error", "skipped"): + if case.find(kind) is not None: + outcome = "failed" if kind == "failure" else kind + break + outcomes[test] = outcome + return outcomes + + +def differences(first: dict[str, str], second: dict[str, str]) -> list[str]: + """Every test the two reports do not agree on, sorted.""" + return [ + f"{test}: {first.get(test, 'absent')} vs {second.get(test, 'absent')}" + for test in sorted(first.keys() | second.keys()) + if first.get(test) != second.get(test) + ] + + def problems(counts: dict[str, int], max_skipped: int) -> list[str]: executed = counts["tests"] - counts["skipped"] found = [] @@ -43,16 +75,22 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("report", type=Path) parser.add_argument("--max-skipped", type=int, default=0) + parser.add_argument("--same-as", type=Path, help="a second report that must agree test by test") args = parser.parse_args(argv) if args.max_skipped < 0: parser.error("--max-skipped must be 0 or more") try: counts = read_counts(args.report) + compared = ( + differences(read_outcomes(args.report), read_outcomes(args.same_as)) + if args.same_as is not None + else [] + ) except (OSError, ET.ParseError, ValueError) as e: print(f"cannot read JUnit report: {e}", file=sys.stderr) return 1 print(counts) - found = problems(counts, args.max_skipped) + found = problems(counts, args.max_skipped) + [f"differs: {line}" for line in compared] for problem in found: print(f"FAIL: {problem}", file=sys.stderr) return 1 if found else 0 From 51459683f55d7d10ef15fe5e87745fcd1cdc7b2a Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 08:00:33 -0600 Subject: [PATCH 02/29] Step 3b-2a-2d: db tests without planted inputs (D39, D40) tests/db/test_schema.py - Remove _tampered and _redefined and every test that rebuilt a copy of the head template (changed definitions, a dropped index, probe indexes and triggers, literal whitespace, table options, changed views, objects outside the comparison, changed plan rules, a dropped foreign-key index) and the create_all checker's written module. - Head against SQLAlchemy's metadata, table by table: columns, foreign keys, created index SQL (CreateIndex), definition items (CreateTable), named CHECKs, flags, no trigger. - Databases the migrations built at 33f8570f6540, 94a52e4f1d57, and 2d2ba76b177c: the readings differ only in the table 2d2ba76b177c lists, in its added column, index, and foreign key; 94a52e4f1d57's changed CHECK text; view_differences and objects_outside_the_comparison against the views frozen through 94a52e4f1d57, expected from DROPPED_VIEWS, REPLACED_VIEWS, and VIEWS; nullable columns and UNIQUE sets across 33f8570f6540 and 94a52e4f1d57. - Scratch databases of valid SQL for the readers: CHECK text, partial and expression indexes, a trigger and a view with whitespace in a literal, STRICT and WITHOUT ROWID flags, a nullable composite primary-key column, and a column with two foreign keys. - Exact counts from the models: sqlite_master rows at head, delete actions, references to github_users, foreign keys examined for an index; create_all_uses counts the files it reads (checked with os.walk) and the tokenizer's names and strings. - CHECK failures assert sqlite_errorname SQLITE_CONSTRAINT_CHECK and the constraint's name. tests/db/schema_support.py - frozen_views takes an optional revision to stop at (default head); foreign_key_columns lists the foreign keys unindexed_foreign_keys examines. tests/db/test_migrations.py - Remove the test-only revisions holding an orphan row, module-level code, a failure, COMMIT, END, ROLLBACK, BEGIN, refused statements, ATTACH, or a batch alter under views; the connection put inside a transaction; the copies changed for the definition reading; the revision checker's written files. - current, check, stamp, and upgrade over a file of seeded rows, and the chain from 94a52e4f1d57 over seeded rows, each with the rows of every table with a foreign key. - An interrupt at a set point after the second revision: the file stays at the first schema. - env.py's own authorizer, captured through a recording connection factory and called at each revision of the real chain and after the commit, with every answer exact. - A failed commit and the write lock on the real chain from 94a52e4f1d57, the timeout from the URL, sqlite_errorname SQLITE_BUSY, and the statements run. - The tag-table refusal over valid tag rows; the step 3a upgrade's views as frozen; table counts for the CHANGED_TABLES tests; legacy_alter_table in the connection probe; the refused tracked-file connection is closed; revision_source counts imports and VIEWS against the tokenizer and the loaded modules. - Settings read no .env from the checkout; the alembic command runs outside it. tests/db/test_tracked_database.py - The frozen-content check fails on databases the migrations wrote (head, first schema); the printed digests equal the files' SHA-256. - git and pre-commit run with an explicit environment (own HOME, no global or system config); the scripts with PATH only. tests/db/test_views.py - Remove the row id projected under a consumer name and the single-column index copies; exact row counts for the views compared. tests/db/test_types.py - The column default reads a fixed clock; rejected values assert the driver error's class. tests/db/test_application_engine.py, tests/test_db_engine.py - Settings read no .env from the checkout; foreign-key failures assert sqlite_errorname; the connect listener's failure is an interrupt at its first statement. tests/test_data_dictionary.py - differences and lock_differences over the dictionary and schema lock of 21a4267 and 77663a7 and databases at 33f8570f6540, 94a52e4f1d57, and 2d2ba76b177c; every frozen view's digest; the scratch database renamed. tests/test_check_query_lock.py - Removed and deleted entries, pairs, and versions come from the lock files of c1f474b, 6fedb05, 21a4267, and 77663a7; the history's own changes pass; edits with no real producer are held on the passing side. - The CI step runs on those files; the base checker is shown by a pull request that deletes its copy. - git, the checker, and the step run with an explicit environment and fixed commit dates. tests/fixtures/history - The lock files and data dictionaries above, copied unchanged from git show, listed in README.md. --- tests/db/schema_support.py | 41 +- tests/db/test_application_engine.py | 9 +- tests/db/test_migrations.py | 789 ++++++++---------- tests/db/test_schema.py | 641 +++++++------- tests/db/test_tracked_database.py | 131 ++- tests/db/test_types.py | 37 +- tests/db/test_views.py | 86 +- .../history/21a4267/docs/data-dictionary.md | 678 +++++++++++++++ .../history/21a4267/tests/query_versions.json | 4 + .../21a4267/tests/schema_versions.json | 496 +++++++++++ .../6fedb05/tests/schema_versions.json | 105 +++ .../history/77663a7/docs/data-dictionary.md | 679 +++++++++++++++ .../77663a7/tests/schema_versions.json | 497 +++++++++++ tests/fixtures/history/README.md | 15 + .../history/c1f474b/tests/query_versions.json | 3 + tests/test_check_query_lock.py | 425 ++++------ tests/test_data_dictionary.py | 139 +-- tests/test_db_engine.py | 32 +- 18 files changed, 3658 insertions(+), 1149 deletions(-) create mode 100644 tests/fixtures/history/21a4267/docs/data-dictionary.md create mode 100644 tests/fixtures/history/21a4267/tests/query_versions.json create mode 100644 tests/fixtures/history/21a4267/tests/schema_versions.json create mode 100644 tests/fixtures/history/6fedb05/tests/schema_versions.json create mode 100644 tests/fixtures/history/77663a7/docs/data-dictionary.md create mode 100644 tests/fixtures/history/77663a7/tests/schema_versions.json create mode 100644 tests/fixtures/history/README.md create mode 100644 tests/fixtures/history/c1f474b/tests/query_versions.json diff --git a/tests/db/schema_support.py b/tests/db/schema_support.py index d65b5dd..37d1818 100644 --- a/tests/db/schema_support.py +++ b/tests/db/schema_support.py @@ -214,10 +214,27 @@ def objects_outside_the_comparison(database: Path, frozen: Iterable[str]) -> lis ] +def foreign_key_columns(database: Path) -> list[tuple[str, tuple[str, ...]]]: + """Each foreign key of each schema table, as its table and its columns in key order.""" + found: list[tuple[str, tuple[str, ...]]] = [] + for table in schema_tables(database): + foreign_keys: dict[int, list[tuple[int, str]]] = {} + for fk_id, seq, _parent, column, *_rest in _rows( + database, f'PRAGMA foreign_key_list("{table}")' + ): + foreign_keys.setdefault(fk_id, []).append((seq, column)) + found += [ + (table, tuple(column for _seq, column in sorted(parts))) + for parts in foreign_keys.values() + ] + return found + + def unindexed_foreign_keys(database: Path) -> set[tuple[str, tuple[str, ...]]]: - """Foreign keys whose columns lead no index; a table's INTEGER PRIMARY KEY is its rowid.""" + """The foreign keys ``foreign_key_columns`` reads whose columns lead no index; a table's + INTEGER PRIMARY KEY is its rowid.""" missing: set[tuple[str, tuple[str, ...]]] = set() - for table in schema_tables(database): + for table, columns in foreign_key_columns(database): leading = [ tuple(row[2] for row in _rows(database, f'PRAGMA index_xinfo("{name}")') if row[5]) for _seq, name, *_rest in _rows(database, f'PRAGMA index_list("{table}")') @@ -229,24 +246,18 @@ def unindexed_foreign_keys(database: Path) -> set[tuple[str, tuple[str, ...]]]: ) if len(primary_key) == 1 and primary_key[0][2] == "INTEGER": leading.append((primary_key[0][1],)) - foreign_keys: dict[int, list[tuple[int, str]]] = {} - for fk_id, seq, _parent, column, *_rest in _rows( - database, f'PRAGMA foreign_key_list("{table}")' - ): - foreign_keys.setdefault(fk_id, []).append((seq, column)) - for parts in foreign_keys.values(): - columns = tuple(column for _seq, column in sorted(parts)) - if not any(keys[: len(columns)] == columns for keys in leading): - missing.add((table, columns)) + if not any(keys[: len(columns)] == columns for keys in leading): + missing.add((table, columns)) return missing -def frozen_views(config: Config) -> dict[str, str]: - """Each view at head with the SQL of the newest revision that creates or replaces it (its - ``VIEWS``); a view a later revision lists in ``DROPPED_VIEWS`` is not at head.""" +def frozen_views(config: Config, through: str = "heads") -> dict[str, str]: + """Each view at head, or at the revision ``through``, with the SQL of the newest revision up + to it that creates or replaces it (its ``VIEWS``); a view a revision up to it lists in + ``DROPPED_VIEWS`` is not there.""" script = ScriptDirectory.from_config(config) frozen: dict[str, str] = {} - for revision in reversed(list(script.walk_revisions())): + for revision in reversed(list(script.walk_revisions(head=through))): for name, sql in getattr(revision.module, "VIEWS", ()): frozen[name] = sql for name in getattr(revision.module, "DROPPED_VIEWS", ()): diff --git a/tests/db/test_application_engine.py b/tests/db/test_application_engine.py index 1b7af31..2723dbd 100644 --- a/tests/db/test_application_engine.py +++ b/tests/db/test_application_engine.py @@ -1,6 +1,7 @@ """The application's own engine and sessions apply the SQLite connection settings.""" import shutil +import sqlite3 from collections.abc import AsyncIterator from pathlib import Path @@ -19,9 +20,10 @@ @pytest.fixture async def application_database( - migrated_database: Path, monkeypatch: pytest.MonkeyPatch + migrated_database: Path, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> AsyncIterator[Path]: """Point the settings at a database migrated to head; reset the cached engine around it.""" + monkeypatch.chdir(tmp_path) # the settings read no .env but this test's monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{migrated_database}") get_settings.cache_clear() await dispose_engine() @@ -46,11 +48,14 @@ async def test_every_application_session_gets_the_connection_settings( async def test_application_session_rejects_an_orphan_row(application_database: Path) -> None: - with pytest.raises(IntegrityError, match="FOREIGN KEY constraint failed"): + with pytest.raises(IntegrityError) as error: async with get_session() as session: await session.execute( text("INSERT INTO pr_user_tags (pr_id, user_tag_id) VALUES (999, 999)") ) + driver_error = error.value.orig + assert isinstance(driver_error, sqlite3.IntegrityError), repr(driver_error) + assert driver_error.sqlite_errorname == "SQLITE_CONSTRAINT_FOREIGNKEY" async def test_dispose_engine_makes_the_next_engine_read_the_settings_again( diff --git a/tests/db/test_migrations.py b/tests/db/test_migrations.py index 6c5a8a6..2d26b93 100644 --- a/tests/db/test_migrations.py +++ b/tests/db/test_migrations.py @@ -14,15 +14,19 @@ import sqlite3 import subprocess import sys -from collections.abc import Iterator -from contextlib import closing +import tokenize +from collections.abc import Callable, Iterator +from contextlib import closing, contextmanager +from dataclasses import dataclass from pathlib import Path from typing import Any import pytest from alembic import command from alembic.script import ScriptDirectory -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event +from sqlalchemy.engine import Engine +from sqlalchemy.exc import OperationalError, PendingRollbackError from github_activity_db.config import get_settings from github_activity_db.db.models import Base @@ -37,15 +41,20 @@ upgrade, write_revision, ) -from tests.db.schema_support import frozen_views, named_constraints, schema_of +from tests.db.schema_support import ( + frozen_views, + named_constraints, + schema_of, + schema_tables, + views, +) from tests.db.seed import T0, Seed, connect from tests.test_import_boundaries import imported_modules TRACKED = REPO_ROOT / "github_activity.db" TEST_REVISION = "f1e57000a001" -# A pr_labels row whose pull request does not exist. -PLANTED_LABEL = "INSERT INTO pr_labels (pr_id, name, label_node_id) VALUES (999, {!r}, 'x')" -ORPHAN = f"op.execute({PLANTED_LABEL.format('orphan')!r})" +# The revisions an upgrade from the first schema's head applies. +AFTER_THE_FIRST_SCHEMA = MIGRATION_REVISIONS[MIGRATION_REVISIONS.index(FIRST_SCHEMA_HEAD) + 1 :] VIEWS_READING_PULL_REQUESTS = ( "v_pr_reviews", "v_pr_comments", @@ -83,6 +92,33 @@ def refusal(error: pytest.ExceptionInfo[BaseException]) -> str: return str(error.value) +def sqlite_error_name(error: pytest.ExceptionInfo[OperationalError]) -> str: + """The SQLite result code of the driver error SQLAlchemy's error wraps.""" + driver_error = error.value.orig + assert isinstance(driver_error, sqlite3.Error), repr(driver_error) + return str(driver_error.sqlite_errorname) + + +def foreign_key_rows(database: Path) -> dict[str, int]: + """The rows of each table that has a foreign key: what ``PRAGMA foreign_key_check`` reads.""" + counted = {} + for table in sorted(table_names(database)): + if query(database, f'PRAGMA foreign_key_list("{table}")'): + ((rows,),) = query(database, f'SELECT COUNT(*) FROM "{table}"') + counted[table] = int(str(rows)) + return counted + + +@contextmanager +def listening(name: str, listener: Callable[..., None]) -> Iterator[None]: + """``listener`` on the ``name`` event of every engine, env.py's own included.""" + event.listen(Engine, name, listener) + try: + yield + finally: + event.remove(Engine, name, listener) + + @pytest.fixture(scope="module") def first_schema_template(tmp_path_factory: pytest.TempPathFactory) -> Path: """A database at the first schema's head holding a repository, a PR, and a sync failure.""" @@ -131,13 +167,13 @@ def test_upgrade_builds_head_with_no_autogenerate_diff( ) -> None: target = tmp_path / "migrations.db" # If env.py ignored the database_url attribute it would migrate DATABASE_URL instead; - # point that at a decoy to make it visible. - decoy = tmp_path / "decoy.db" - monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{decoy}") + # point that at a file of its own to make it visible. + settings_target = tmp_path / "settings-target.db" + monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{settings_target}") config = alembic_config(target) command.upgrade(config, "head") command.check(config) # raises AutogenerateDiffsDetected when models and chain disagree - assert not decoy.exists() + assert not settings_target.exists() assert revision_of(target) == HEAD_REVISION def test_the_packaged_revision_tuple_equals_the_chain(self, tmp_path: Path) -> None: @@ -166,68 +202,20 @@ def test_downgrade_raises_and_changes_nothing(self, migrated_database: Path) -> pytest.param(lambda config: command.upgrade(config, "head"), id="upgrade at head"), ], ) - def test_every_command_refuses_a_file_holding_a_foreign_key_violation( + def test_every_command_commits_a_file_of_seeded_rows( self, migrated_database: Path, run: Any ) -> None: - """No command commits a file that fails the check, whether it ran a revision or not.""" - with closing(sqlite3.connect(migrated_database)) as connection, connection: - connection.execute(PLANTED_LABEL.format("orphan")) - before = snapshot(migrated_database) - with pytest.raises(RuntimeError) as error: - run(alembic_config(migrated_database)) - assert "1 foreign-key violations in the file before the commit" in refusal(error) - assert snapshot(migrated_database) == before - - @pytest.mark.parametrize( - ("run", "statement"), - [ - *( - pytest.param(run, statement, id=f"{change}, {name}") - for change, statement in ( - ("a row", ORPHAN), - # The seeded pull request's author row goes with the table. - ("the schema", 'op.execute("DROP TABLE github_users")'), - ) - for name, run in ( - ("current", command.current), - ("stamp head", lambda config: command.stamp(config, "head")), - ("upgrade at head", lambda config: command.upgrade(config, "head")), - ) - ), - # With the table gone, check's own comparison fails before the commit instead. - pytest.param(command.check, ORPHAN, id="a row, check"), - ], - ) - def test_revision_module_code_that_breaks_a_reference_is_refused( - self, migrated_database: Path, tmp_path: Path, run: Any, statement: str - ) -> None: - """alembic loads each revision module inside the transaction, so code at a module's top - level runs even when no upgrade() does.""" - with connect(migrated_database) as seeded: - seed = Seed(seeded) - seed.pull_request(seed.repository(), 1, seed.user("author")) - versions = tmp_path / "versions" - versions.mkdir() - (versions / f"{TEST_REVISION}_module_level.py").write_text( - "from alembic import op\n\n" - f"revision = {TEST_REVISION!r}\n" - f"down_revision = {HEAD_REVISION!r}\n" - "branch_labels = None\n" - "depends_on = None\n\n" - f"{statement}\n\n\n" - "def upgrade() -> None:\n" - " pass\n\n\n" - "def downgrade() -> None:\n" - " raise RuntimeError('test-only revision')\n" - ) - # At the test revision already, so no command runs a revision or finds the file behind. - with closing(sqlite3.connect(migrated_database)) as connection, connection: - connection.execute("UPDATE alembic_version SET version_num = ?", (TEST_REVISION,)) + """Each command runs the foreign-key check before it commits, whether it ran a revision + or not; a file of valid rows passes it, and each table the check reads holds the rows + the seed wrote.""" + written = _seed_a_pull_request_with_children(migrated_database) before = snapshot(migrated_database) - with pytest.raises(RuntimeError) as error: - run(alembic_config(migrated_database, test_versions=versions)) - assert "1 foreign-key violations" in refusal(error) + run(alembic_config(migrated_database)) assert snapshot(migrated_database) == before + read = foreign_key_rows(migrated_database) + assert {table: rows for table, rows in read.items() if rows} == { + table: rows for table, rows in written.items() if table in read + } def test_an_in_memory_database_is_migrated( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -278,26 +266,38 @@ def test_upgrades_a_scratch_copy_of_the_tracked_file(self, tmp_path: Path) -> No assert digest(TRACKED) == tracked_digest @pytest.mark.parametrize( - ("insert", "held"), + ("inserts", "held"), [ pytest.param( - "INSERT INTO user_tags (name, created_at) VALUES ('keep', '2026-01-01 00:00:00')", + ( + "INSERT INTO user_tags (name, created_at) " + "VALUES ('keep', '2026-01-01 00:00:00')", + ), "user_tags 1, pr_user_tags 0", id="user_tags", ), pytest.param( - # Foreign keys are off on this connection: a link whose tag row is missing. - "INSERT INTO pr_user_tags (pr_id, user_tag_id) VALUES (1, 99)", - "user_tags 0, pr_user_tags 1", + ( + "INSERT INTO user_tags (id, name, created_at) " + "VALUES (1, 'keep', '2026-01-01 00:00:00')", + "INSERT INTO user_tags (id, name, created_at) " + "VALUES (2, 'other', '2026-01-01 00:00:00')", + # The template's pull request 1, tagged. + "INSERT INTO pr_user_tags (pr_id, user_tag_id) VALUES (1, 1)", + ), + "user_tags 2, pr_user_tags 1", id="pr_user_tags", ), ], ) def test_aborts_while_a_tag_table_holds_rows( - self, first_schema: Path, insert: str, held: str + self, first_schema: Path, inserts: tuple[str, ...], held: str ) -> None: with closing(sqlite3.connect(first_schema)) as connection, connection: - connection.execute(insert) + connection.execute("PRAGMA foreign_keys=ON") + for insert in inserts: + connection.execute(insert) + assert query(first_schema, "PRAGMA foreign_key_check") == [] before = snapshot(first_schema) with pytest.raises(RuntimeError, match=rf"tag rows would be lost \({held}\)"): upgrade(first_schema) @@ -381,10 +381,15 @@ def test_refuses_a_database_holding_rows_in_a_table_it_adds_columns_to( def test_upgrades_a_database_without_rows( self, step3a_database: Path, head_template: Path ) -> None: + """Its table rebuilds run with Step 3a's views in place, which it drops and recreates + from its own frozen SQL: every view at head is there, as frozen.""" + assert views(step3a_database) upgrade(step3a_database) assert revision_of(step3a_database) == HEAD_REVISION assert query(step3a_database, "PRAGMA user_version") == [(SCHEMA_VERSION,)] assert schema_of(step3a_database) == schema_of(head_template) + frozen = frozen_views(alembic_config(head_template)) + assert views(step3a_database) == {name: sql.strip() for name, sql in frozen.items()} def test_a_row_in_a_table_it_keeps_is_not_refused(self, step3a_database: Path) -> None: """A user tag is kept as it was; only the tables the revision recreates must be empty.""" @@ -406,6 +411,9 @@ def test_the_refusal_covers_every_table_the_revision_recreates( step3b1 = tmp_path / "at-step3b1.db" upgrade(step3b1, STEP3B1) before, after = table_definitions(step3a_database), table_definitions(step3b1) + # Every table of each file is compared. + assert len(before) == len(schema_tables(step3a_database)) + assert len(after) == len(schema_tables(step3b1)) changed = {table for table in before if before[table] != after.get(table)} assert set(module.CHANGED_TABLES) == changed assert after.keys() - before.keys() == { @@ -415,42 +423,6 @@ def test_the_refusal_covers_every_table_the_revision_recreates( "account_profiles", } - def test_the_definition_reading_sees_a_changed_table( - self, step3a_database: Path, tmp_path: Path - ) -> None: - """Positive control: a column added to a table is a changed definition.""" - changed = tmp_path / "changed.db" - shutil.copyfile(step3a_database, changed) - with closing(sqlite3.connect(changed)) as connection, connection: - connection.execute("ALTER TABLE pr_files ADD COLUMN probe INTEGER") - assert ( - table_definitions(changed)["pr_files"] != table_definitions(step3a_database)["pr_files"] - ) - - def test_the_definition_reading_sees_a_literal_s_whitespace( - self, step3a_database: Path, tmp_path: Path - ) -> None: - """Whitespace inside a literal is data: a CHECK on 'a b' and one on 'a b' differ.""" - - def defined(literal: str, name: str) -> str: - copy = tmp_path / f"{name}.db" - shutil.copyfile(step3a_database, copy) - with closing(sqlite3.connect(copy)) as connection, connection: - ((sql,),) = connection.execute( - "SELECT sql FROM sqlite_master WHERE name = 'pr_files'" - ).fetchall() - assert "path TEXT NOT NULL" in sql - connection.execute("DROP TABLE pr_files") - connection.execute( - sql.replace( - "path TEXT NOT NULL", f"path TEXT NOT NULL CHECK (path != {literal})" - ) - ) - return table_definitions(copy)["pr_files"] - - assert defined("'a b'", "one") == defined("'a b'", "again") - assert defined("'a b'", "two") != defined("'a b'", "one") - class TestSyncFailuresLastRunIdMigration: FAILURE = ( @@ -499,6 +471,9 @@ def test_the_refusal_covers_every_table_the_revision_recreates( migrated = tmp_path / "at-last-run-id.db" upgrade(migrated, LAST_RUN_ID) before, after = table_definitions(step3b1_database), table_definitions(migrated) + # Every table of each file is compared. + assert len(before) == len(schema_tables(step3b1_database)) + assert len(after) == len(schema_tables(migrated)) changed = {table for table in before if before[table] != after.get(table)} assert set(module.CHANGED_TABLES) == changed == {"sync_failures"} assert after.keys() == before.keys() @@ -517,39 +492,46 @@ def table_definitions(database: Path) -> dict[str, str]: class TestOneTransaction: - def test_a_failure_in_the_second_revision_applies_neither( - self, first_schema: Path, tmp_path: Path - ) -> None: - versions = tmp_path / "versions" - write_revision( - versions, TEST_REVISION, HEAD_REVISION, "raise RuntimeError('planted failure')" - ) + def test_an_interrupt_after_the_second_revision_applies_none(self, first_schema: Path) -> None: + """An interrupt at a set point, once the second of the revisions after the first schema + has run: the file stays at the revision the upgrade started from.""" + second = AFTER_THE_FIRST_SCHEMA[1] + + def interrupt(_connection: Any, _cursor: Any, statement: str, *_rest: Any) -> None: + if statement.startswith(f"UPDATE alembic_version SET version_num='{second}'"): + raise KeyboardInterrupt + before = snapshot(first_schema) - with pytest.raises(RuntimeError, match="planted failure"): - upgrade(first_schema, test_versions=versions) + # Both caught, so an interrupt that reached here would fail this test, not end the run. + with ( + listening("after_cursor_execute", interrupt), + pytest.raises((PendingRollbackError, KeyboardInterrupt)) as error, + ): + upgrade(first_schema) + # SQLAlchemy invalidates the connection for the interrupt; env.py's handler then reads + # connection.info, which raises on it with the interrupt as its context. + assert [type(error.value), type(error.value.__context__)] == [ + PendingRollbackError, + KeyboardInterrupt, + ] assert snapshot(first_schema) == before assert revision_of(first_schema) == FIRST_SCHEMA_HEAD - def test_a_foreign_key_violation_fails_the_upgrade_before_commit( - self, migrated_database: Path, tmp_path: Path - ) -> None: - versions = tmp_path / "versions" - write_revision( - versions, - TEST_REVISION, - HEAD_REVISION, - ORPHAN, - ) - before = snapshot(migrated_database) - with pytest.raises(RuntimeError) as error: - upgrade(migrated_database, test_versions=versions) - assert "1 foreign-key violations in the file before the commit" in refusal(error) - assert "('pr_labels', 1, 'pull_requests', 0)" in refusal(error) - assert snapshot(migrated_database) == before + def test_the_chain_commits_a_file_of_seeded_rows(self, step3b1_database: Path) -> None: + """The foreign-key check before the upgrade's commit passes a file of valid rows, and + each table it reads holds the rows the seed wrote.""" + written = _seed_a_pull_request_with_children(step3b1_database) + upgrade(step3b1_database) + assert revision_of(step3b1_database) == HEAD_REVISION + read = foreign_key_rows(step3b1_database) + assert {table: rows for table, rows in read.items() if rows} == { + table: rows for table, rows in written.items() if table in read + } def test_the_migration_connection_has_foreign_keys_off_inside_one_transaction( self, migrated_database: Path, tmp_path: Path ) -> None: + # One row: one connection, configured once, with legacy_alter_table still off. versions = tmp_path / "versions" write_revision( versions, @@ -559,287 +541,196 @@ def test_the_migration_connection_has_foreign_keys_off_inside_one_transaction( bind = op.get_bind() dbapi = bind.connection.dbapi_connection foreign_keys = bind.exec_driver_sql("PRAGMA foreign_keys").scalar_one() + legacy_alter_table = bind.exec_driver_sql("PRAGMA legacy_alter_table").scalar_one() op.execute( - "CREATE TABLE probe (foreign_keys INTEGER, in_transaction INTEGER, " - "autocommit INTEGER)" + "CREATE TABLE probe (foreign_keys INTEGER, legacy_alter_table INTEGER, " + "in_transaction INTEGER, autocommit INTEGER)" ) bind.exec_driver_sql( - "INSERT INTO probe VALUES (?, ?, ?)", - (foreign_keys, dbapi.in_transaction, dbapi.isolation_level is None), + "INSERT INTO probe VALUES (?, ?, ?, ?)", + ( + foreign_keys, + legacy_alter_table, + dbapi.in_transaction, + dbapi.isolation_level is None, + ), ) """, ) upgrade(migrated_database, test_versions=versions) - assert query(migrated_database, "SELECT * FROM probe") == [(0, 1, 1)] - - @pytest.mark.parametrize( - ("foreign_keys", "message"), - [ - pytest.param("ON", "PRAGMA foreign_keys reads 1 after turning it off", id="fk on"), - pytest.param("OFF", "already inside a transaction", id="fk off"), - ], - ) - def test_a_connection_already_inside_a_transaction_is_refused( - self, - first_schema: Path, - monkeypatch: pytest.MonkeyPatch, - foreign_keys: str, - message: str, - ) -> None: - """Planted through env.py's own engine: the DBAPI connect function SQLAlchemy calls.""" - real_connect = sqlite3.dbapi2.connect - planted: list[sqlite3.Connection] = [] - - def connect_inside_a_transaction(*args: Any, **kwargs: Any) -> sqlite3.Connection: - connection = real_connect(*args, **kwargs) - connection.execute(f"PRAGMA foreign_keys={foreign_keys}") - connection.execute("BEGIN") - planted.append(connection) - return connection - - before = snapshot(first_schema) - monkeypatch.setattr(sqlite3.dbapi2, "connect", connect_inside_a_transaction) - with pytest.raises(RuntimeError) as error: - upgrade(first_schema) - monkeypatch.undo() - assert message in refusal(error) - assert len(planted) == 1 - with pytest.raises(sqlite3.ProgrammingError, match="closed database"): - planted[0].execute("SELECT 1") - assert snapshot(first_schema) == before - - -# SQLite rolls back by itself; the revision swallows the error and goes on. -SWALLOWED_ROLLBACK = ( - "try:\n" - ' op.execute("INSERT OR ROLLBACK INTO alembic_version "\n' - f" \"VALUES ('{HEAD_REVISION}')\")\n" - "except Exception:\n" - " pass\n" -) -STOP_TRIGGER = ( - "CREATE TRIGGER stop BEFORE INSERT ON pr_labels WHEN NEW.name = 'stop' " - "BEGIN SELECT RAISE(ROLLBACK, 'stop'); END" -) -STOPPED_INSERT = ( - f"try:\n op.execute({PLANTED_LABEL.format('stop')!r})\nexcept Exception:\n pass\n" -) -BOUND_INSERT = ( - 'op.get_bind().exec_driver_sql("INSERT INTO pr_labels (pr_id, name, label_node_id) ' - "VALUES (?, ?, 'LA_bound')\", (999, {!r}))" -) + assert query(migrated_database, "SELECT * FROM probe") == [(0, 0, 1, 1)] + + +Authorizer = Callable[[int, str | None, str | None, str | None, str | None], int] + + +class RecordingConnection(sqlite3.Connection): + """A driver connection that keeps the authorizer env.py installs on it, so a test can call + that authorizer itself.""" + + authorizer: Authorizer + + def set_authorizer(self, authorizer_callback: Authorizer | None) -> None: + if authorizer_callback is not None: + self.authorizer = authorizer_callback + super().set_authorizer(authorizer_callback) + + +# Calls env.py's authorizer must answer, as SQLite makes them: (action, its two arguments, the +# database, the trigger or view). +REFUSED_WHILE_OPEN = { + "ATTACH": (sqlite3.SQLITE_ATTACH, "other.db", None, None, None), + "DETACH": (sqlite3.SQLITE_DETACH, "other", None, None, None), + "SAVEPOINT": (sqlite3.SQLITE_SAVEPOINT, "BEGIN", "early", None, None), + "ANALYZE": (sqlite3.SQLITE_ANALYZE, "pr_labels", None, "main", None), + "CREATE TEMP TABLE": (sqlite3.SQLITE_CREATE_TEMP_TABLE, "scratch", None, "temp", None), + "CREATE TEMP VIEW": (sqlite3.SQLITE_CREATE_TEMP_VIEW, "scratch", None, "temp", None), + "CREATE TEMP TRIGGER": (sqlite3.SQLITE_CREATE_TEMP_TRIGGER, "t", "pr_labels", "temp", None), + # Creating a temp. object writes its row to the temp schema. + "a temp schema row": (sqlite3.SQLITE_INSERT, "sqlite_temp_master", None, "temp", None), + "CREATE VIRTUAL TABLE": (sqlite3.SQLITE_CREATE_VTABLE, "scratch", "fts5", "main", None), + "BEGIN": (sqlite3.SQLITE_TRANSACTION, "BEGIN", None, None, None), + "COMMIT": (sqlite3.SQLITE_TRANSACTION, "COMMIT", None, None, None), + "ROLLBACK": (sqlite3.SQLITE_TRANSACTION, "ROLLBACK", None, None, None), + "PRAGMA journal_mode=OFF": (sqlite3.SQLITE_PRAGMA, "journal_mode", "OFF", "main", None), + "PRAGMA legacy_alter_table=ON": (sqlite3.SQLITE_PRAGMA, "legacy_alter_table", "ON", None, None), + "PRAGMA ignore_check_constraints=ON": ( + sqlite3.SQLITE_PRAGMA, + "ignore_check_constraints", + "ON", + None, + None, + ), + "PRAGMA writable_schema=ON": (sqlite3.SQLITE_PRAGMA, "writable_schema", "ON", None, None), +} +ALLOWED_WHILE_OPEN = { + "PRAGMA user_version=2": (sqlite3.SQLITE_PRAGMA, "user_version", "2", None, None), + "PRAGMA table_info(pr_labels)": (sqlite3.SQLITE_PRAGMA, "table_info", "pr_labels", None, None), + "PRAGMA foreign_keys": (sqlite3.SQLITE_PRAGMA, "foreign_keys", None, None, None), + "INSERT": (sqlite3.SQLITE_INSERT, "pr_labels", None, "main", None), + "SELECT": (sqlite3.SQLITE_SELECT, None, None, None, None), +} class TestTransactionControl: """Only env.py opens, commits, and rolls back the upgrade's transaction (plan 5.1).""" - @pytest.mark.parametrize( - "body", - [ - pytest.param(f"{ORPHAN}\nop.execute('COMMIT')", id="COMMIT"), - pytest.param(f"{ORPHAN}\nop.execute('END')", id="END"), - pytest.param(f"{ORPHAN}\nop.execute('ROLLBACK')\n{ORPHAN}", id="ROLLBACK"), - pytest.param(f"op.execute('BEGIN')\n{ORPHAN}", id="BEGIN"), - pytest.param( - f"with op.get_context().autocommit_block():\n {ORPHAN}", id="autocommit_block" - ), - pytest.param( - "op.execute('CREATE TABLE early (x INTEGER)')\n" - f"{SWALLOWED_ROLLBACK}" - f"op.execute('CREATE TABLE late (x INTEGER)')\n{ORPHAN}", - id="INSERT OR ROLLBACK", - ), - pytest.param( - "op.execute('CREATE TRIGGER stop BEFORE INSERT ON pr_labels " - "BEGIN SELECT RAISE(ROLLBACK, \\'stop\\'); END')\n" - "try:\n" - f" {ORPHAN}\n" - "except Exception:\n" - " pass\n" - "op.execute('CREATE TABLE late (x INTEGER)')", - id="RAISE(ROLLBACK)", - ), - ], - ) - def test_a_revision_cannot_end_the_transaction( - self, first_schema: Path, tmp_path: Path, body: str + def test_the_authorizer_answers_each_call_as_the_upgrade_allows( + self, first_schema: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - versions = tmp_path / "versions" - write_revision(versions, TEST_REVISION, HEAD_REVISION, body) - before = snapshot(first_schema) - with pytest.raises(Exception, match="not authorized"): - upgrade(first_schema, test_versions=versions) - assert snapshot(first_schema) == before + """env.py's own authorizer, called at a set point in each revision of the real chain, + while the upgrade's transaction is open, and again once it has committed.""" + connections: list[RecordingConnection] = [] + real_connect = sqlite3.dbapi2.connect - @pytest.mark.parametrize( - ("body", "url_query"), - [ - pytest.param(f"{ORPHAN}\n{SWALLOWED_ROLLBACK}{ORPHAN}", "", id="INSERT OR ROLLBACK"), - pytest.param( - f"{ORPHAN}\n{STOPPED_INSERT}{ORPHAN}", - "", - id="RAISE(ROLLBACK) of a trigger in the file", - ), - pytest.param( - f"{BOUND_INSERT.format('a')}\n{SWALLOWED_ROLLBACK}{BOUND_INSERT.format('b')}", - "", - id="bound parameters", - ), - pytest.param( - f"{ORPHAN}\n{SWALLOWED_ROLLBACK}{ORPHAN}", - "?cached_statements=128", - id="statement cache asked for by the URL", - ), - ], - ) - def test_a_statement_run_before_sqlite_ended_the_transaction_is_refused_after( - self, migrated_database: Path, tmp_path: Path, body: str, url_query: str - ) -> None: - """SQLite authorizes a statement when it prepares it. A data-only upgrade changes no - schema, so SQLite's own rollback expires nothing the driver could reuse by its text.""" - with closing(sqlite3.connect(migrated_database)) as connection, connection: - connection.execute(STOP_TRIGGER) - versions = tmp_path / "versions" - write_revision(versions, TEST_REVISION, HEAD_REVISION, body) - config = alembic_config(migrated_database, test_versions=versions) - config.attributes["database_url"] += url_query - before = snapshot(migrated_database) - with pytest.raises(Exception, match="not authorized"): - command.upgrade(config, "head") - assert snapshot(migrated_database) == before + def connect(*args: Any, **kwargs: Any) -> sqlite3.Connection: + connection = real_connect(*args, factory=RecordingConnection, **kwargs) + connections.append(connection) + return connection - @pytest.mark.parametrize( - "statement", - [ - # The rollback of a failed upgrade could not restore pages spilled to the file. - "PRAGMA journal_mode=OFF", - "PRAGMA main.JOURNAL_MODE=OFF", - # A batch alter under a view would pass (gap 71). - "PRAGMA legacy_alter_table=ON", - # Rows the named CHECK constraints refuse would be committed. - "PRAGMA ignore_check_constraints=ON", - "PRAGMA writable_schema=ON", - "SAVEPOINT early", - "CREATE TEMP TABLE scratch (x INTEGER)", - # Schema-qualified, these reach SQLite as the main database's action codes. - "CREATE TABLE temp.scratch (x INTEGER)", - "CREATE VIEW temp.scratch AS SELECT 1 AS x", - "CREATE TRIGGER temp.drop_labels BEFORE INSERT ON pr_labels " - "BEGIN SELECT RAISE(IGNORE); END", - "ANALYZE", - ], - ) - def test_a_revision_runs_only_the_actions_the_authorizer_lists( - self, migrated_database: Path, tmp_path: Path, statement: str - ) -> None: - versions = tmp_path / "versions" - write_revision(versions, TEST_REVISION, HEAD_REVISION, f"op.execute({statement!r})") - before = snapshot(migrated_database) - with pytest.raises(Exception, match="not authorized"): - upgrade(migrated_database, test_versions=versions) - assert snapshot(migrated_database) == before + calls = {**REFUSED_WHILE_OPEN, **ALLOWED_WHILE_OPEN} + answers: list[tuple[bool, dict[str, int]]] = [] - def test_a_revision_cannot_attach_another_database( - self, migrated_database: Path, tmp_path: Path - ) -> None: - """ATTACH would let a revision write a file the refusals never saw, such as another - checkout's github_activity.db.""" - other = tmp_path / "other-checkout" / "github_activity.db" - other.parent.mkdir() - versions = tmp_path / "versions" - write_revision( - versions, - TEST_REVISION, - HEAD_REVISION, - f'op.execute("ATTACH DATABASE {str(other)!r} AS other")\n' - "op.execute('CREATE TABLE other.probe (x INTEGER)')", - ) - before = snapshot(migrated_database) - with pytest.raises(Exception, match="not authorized"): - upgrade(migrated_database, test_versions=versions) - assert snapshot(migrated_database) == before - assert not other.exists() - - def test_a_failed_commit_rolls_back_cleanly( - self, - migrated_database: Path, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, - ) -> None: - """A reader's lock makes the COMMIT fail; the rollback that follows is env.py's own.""" - versions = tmp_path / "versions" - write_revision( - versions, TEST_REVISION, HEAD_REVISION, "op.execute('CREATE TABLE probe (x INTEGER)')" + def ask(_connection: Any, _cursor: Any, statement: str, *_rest: Any) -> None: + if statement.startswith("UPDATE alembic_version"): + (connection,) = connections + answers.append( + ( + connection.in_transaction, + {name: connection.authorizer(*call) for name, call in calls.items()}, + ) + ) + + monkeypatch.setattr(sqlite3.dbapi2, "connect", connect) + with listening("after_cursor_execute", ask): + upgrade(first_schema) + monkeypatch.undo() + assert revision_of(first_schema) == HEAD_REVISION + while_open = dict.fromkeys(REFUSED_WHILE_OPEN, sqlite3.SQLITE_DENY) | dict.fromkeys( + ALLOWED_WHILE_OPEN, sqlite3.SQLITE_OK ) - before = snapshot(migrated_database) - real_connect = sqlite3.dbapi2.connect + assert answers == [(True, while_open)] * len(AFTER_THE_FIRST_SCHEMA) + # Once the check has passed: env.py's COMMIT, and its ROLLBACK should the commit fail. + (connection,) = connections + assert { + name: connection.authorizer(*REFUSED_WHILE_OPEN[name]) + for name in ("BEGIN", "COMMIT", "ROLLBACK") + } == { + "BEGIN": sqlite3.SQLITE_DENY, + "COMMIT": sqlite3.SQLITE_OK, + "ROLLBACK": sqlite3.SQLITE_OK, + } - def connect_with_a_short_timeout(*args: Any, **kwargs: Any) -> sqlite3.Connection: - return real_connect(*args, **{**kwargs, "timeout": 0.1}) + def test_a_failed_commit_rolls_back_cleanly(self, step3b1_database: Path) -> None: + """A reader's lock makes the COMMIT fail once the check has run; the rollback that + follows is env.py's own, so the error is the commit's, not the authorizer's.""" + statements: list[str] = [] - reader = sqlite3.connect(migrated_database, isolation_level=None) + def record(_connection: Any, _cursor: Any, statement: str, *_rest: Any) -> None: + statements.append(statement) + + before = snapshot(step3b1_database) + config = alembic_config(step3b1_database) + config.attributes["database_url"] += "?timeout=0.1" + reader = sqlite3.connect(step3b1_database, isolation_level=None) try: reader.execute("BEGIN") reader.execute("SELECT COUNT(*) FROM pr_labels").fetchall() # holds a shared lock - monkeypatch.setattr(sqlite3.dbapi2, "connect", connect_with_a_short_timeout) - with pytest.raises(Exception, match="database is locked"): - upgrade(migrated_database, test_versions=versions) - monkeypatch.undo() + with ( + listening("before_cursor_execute", record), + pytest.raises(OperationalError) as error, + ): + command.upgrade(config, "head") finally: reader.close() - assert snapshot(migrated_database) == before - assert not [record for record in caplog.records if "not authorized" in str(record.exc_info)] + assert sqlite_error_name(error) == "SQLITE_BUSY" + assert statements[-1] == "PRAGMA foreign_key_check" + assert snapshot(step3b1_database) == before + assert revision_of(step3b1_database) == STEP3B1 def test_the_upgrade_takes_the_write_lock_before_any_revision_runs( - self, migrated_database: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + self, step3b1_database: Path ) -> None: - """BEGIN IMMEDIATE: while another connection holds the write lock, no revision runs.""" - marker = tmp_path / "revision-ran" - versions = tmp_path / "versions" - write_revision( - versions, - TEST_REVISION, - HEAD_REVISION, - f"import pathlib\npathlib.Path({str(marker)!r}).write_text('ran')\n{ORPHAN}", - ) - before = snapshot(migrated_database) - real_connect = sqlite3.dbapi2.connect + """BEGIN IMMEDIATE: while another connection holds the write lock, the upgrade's first + statement fails, so no revision runs.""" + statements: list[str] = [] - def connect_with_a_short_timeout(*args: Any, **kwargs: Any) -> sqlite3.Connection: - return real_connect(*args, **{**kwargs, "timeout": 0.1}) + def record(_connection: Any, _cursor: Any, statement: str, *_rest: Any) -> None: + statements.append(statement) - holder = sqlite3.connect(migrated_database, isolation_level=None) + before = snapshot(step3b1_database) + config = alembic_config(step3b1_database) + config.attributes["database_url"] += "?timeout=0.1" + holder = sqlite3.connect(step3b1_database, isolation_level=None) try: holder.execute("BEGIN IMMEDIATE") - monkeypatch.setattr(sqlite3.dbapi2, "connect", connect_with_a_short_timeout) - with pytest.raises(Exception, match="database is locked"): - upgrade(migrated_database, test_versions=versions) - monkeypatch.undo() + with ( + listening("before_cursor_execute", record), + pytest.raises(OperationalError) as error, + ): + command.upgrade(config, "head") finally: holder.close() - assert not marker.exists() - assert snapshot(migrated_database) == before + assert sqlite_error_name(error) == "SQLITE_BUSY" + assert statements == ["BEGIN IMMEDIATE"] + assert snapshot(step3b1_database) == before + assert revision_of(step3b1_database) == STEP3B1 -def _batch_alter_revision(views_sql: tuple[tuple[str, str], ...] | None) -> str: - """A revision that batch-recreates pull_requests; with ``views_sql`` it drops and recreates - the views reading it, from that frozen copy, as plan 5.1 requires.""" - drop = recreate = "" - if views_sql is not None: - drop = ( - f"for name in {VIEWS_READING_PULL_REQUESTS!r}:\n op.execute(f'DROP VIEW {{name}}')\n" - ) - recreate = f"for _name, sql in {views_sql!r}:\n op.execute(sql)\n" +def _batch_alter_revision(views_sql: tuple[tuple[str, str], ...]) -> str: + """A revision that batch-recreates pull_requests, dropping and recreating the views reading + it from ``views_sql``, a frozen copy, as plan 5.1 requires.""" return ( - drop - + "with op.batch_alter_table('pull_requests', recreate='always') as batch:\n" - + " batch.add_column(sa.Column('probe', sa.Integer(), nullable=True))\n" - + recreate + f"for name in {VIEWS_READING_PULL_REQUESTS!r}:\n op.execute(f'DROP VIEW {{name}}')\n" + "with op.batch_alter_table('pull_requests', recreate='always') as batch:\n" + " batch.add_column(sa.Column('probe', sa.Integer(), nullable=True))\n" + f"for _name, sql in {views_sql!r}:\n op.execute(sql)\n" ) -def _seed_a_pull_request_with_children(database: Path) -> None: +def _seed_a_pull_request_with_children(database: Path) -> dict[str, int]: + """Valid rows: a merged pull request with a row in each of its child tables; returns the + rows written to each table.""" with connect(database) as connection: seed = Seed(connection) repository = seed.repository() @@ -870,6 +761,21 @@ def _seed_a_pull_request_with_children(database: Path) -> None: connection.exec_driver_sql( f"INSERT INTO pr_user_tags (pr_id, user_tag_id) VALUES ({pr}, 1)" ) + return { + "repositories": 1, + "github_users": 3, + "pull_requests": 1, + "pr_reviews": 1, + "pr_comments": 2, + "pr_commits": 1, + "pr_labels": 1, + "pr_files": 1, + "pr_closing_issues": 1, + "pr_review_requests": 1, + "pr_assignees": 1, + "user_tags": 1, + "pr_user_tags": 1, + } CHILD_TABLES = ( @@ -928,17 +834,6 @@ def test_a_batch_alter_of_pull_requests_keeps_child_rows_and_constraints( } <= constraints.keys() assert query(migrated_database, PULL_REQUEST_INDEXES) == indexes - def test_a_batch_alter_under_a_view_fails_and_applies_nothing( - self, migrated_database: Path, tmp_path: Path - ) -> None: - """legacy_alter_table stays off, so a view left in place fails the migration (gap 71).""" - versions = tmp_path / "versions" - write_revision(versions, TEST_REVISION, HEAD_REVISION, _batch_alter_revision(None)) - before = snapshot(migrated_database) - with pytest.raises(Exception, match=r"error in view v_\w+"): - upgrade(migrated_database, test_versions=versions) - assert snapshot(migrated_database) == before - class TestRefusals: @pytest.fixture(autouse=True) @@ -948,9 +843,10 @@ def tracked_file_unchanged(self) -> Iterator[None]: assert (digest(TRACKED), TRACKED.stat().st_mtime_ns) == before def test_refuses_the_tracked_file_named_by_database_url( - self, monkeypatch: pytest.MonkeyPatch, isolated_settings: None + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, isolated_settings: None ) -> None: """The spelling gap 73 found: DATABASE_URL, as the documented .env line sets it.""" + monkeypatch.chdir(tmp_path) # the settings read no .env but this test's monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{TRACKED}") config = alembic_config(TRACKED) del config.attributes["database_url"] @@ -1026,18 +922,19 @@ def test_refuses_the_tracked_file_of_a_copied_checkout( command.upgrade(config, "head") assert "refusing to migrate" in refusal(error) assert snapshot(target) == before - # Positive control: the copied environment migrates a file that is not the tracked one. + # The copied environment migrates a file that is not the tracked one. other = tmp_path / "other.db" shutil.copyfile(first_schema, other) upgrade(other, script_location=checkout / "alembic") assert revision_of(other) == HEAD_REVISION def test_the_file_sqlite_opened_is_refused_by_its_name( - self, tmp_path: Path, first_schema: Path + self, tmp_path: Path, first_schema: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """The opened-file name check alone: the URL names alias.db, Python's realpath (symlinks first, then ..) names a missing file, and the driver, which drops .. as text, opens - alias.db, a symlink to another checkout's github_activity.db.""" + alias.db, a symlink to another checkout's github_activity.db. The connect listener + refuses it there, and closes the connection, which SQLAlchemy would not.""" target = tmp_path / "other-checkout" / "github_activity.db" target.parent.mkdir() shutil.copyfile(first_schema, target) @@ -1049,9 +946,22 @@ def test_the_file_sqlite_opened_is_refused_by_its_name( before = snapshot(target) config = alembic_config(tmp_path / "unused.db") config.attributes["database_url"] = f"sqlite+aiosqlite:///{checkout}/link/../alias.db" + connections: list[sqlite3.Connection] = [] + real_connect = sqlite3.dbapi2.connect + + def connect(*args: Any, **kwargs: Any) -> sqlite3.Connection: + connection = real_connect(*args, **kwargs) + connections.append(connection) + return connection + + monkeypatch.setattr(sqlite3.dbapi2, "connect", connect) with pytest.raises(RuntimeError) as error: command.upgrade(config, "head") + monkeypatch.undo() assert "(the file SQLite opened)" in refusal(error) + (connection,) = connections + with pytest.raises(sqlite3.ProgrammingError): # a closed connection runs nothing + connection.execute("SELECT 1") assert snapshot(target) == before @pytest.mark.parametrize( @@ -1112,8 +1022,8 @@ def test_a_caller_supplied_connection_is_refused( ) -> None: """alembic's documented attributes["connection"] would otherwise be ignored, and the settings' database migrated instead.""" - decoy = tmp_path / "decoy.db" - monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{decoy}") + settings_target = tmp_path / "settings-target.db" + monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{settings_target}") callers = tmp_path / "callers.db" config = alembic_config(callers) del config.attributes["database_url"] @@ -1126,14 +1036,14 @@ def test_a_caller_supplied_connection_is_refused( finally: engine.dispose() assert "a caller-supplied connection is not used" in refusal(error) - assert not decoy.exists() + assert not settings_target.exists() assert table_names(callers) == set() def test_sqlalchemy_url_is_refused( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, isolated_settings: None ) -> None: - decoy = tmp_path / "decoy.db" - monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{decoy}") + settings_target = tmp_path / "settings-target.db" + monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{settings_target}") named = tmp_path / "named.db" config = alembic_config(named) del config.attributes["database_url"] @@ -1141,14 +1051,16 @@ def test_sqlalchemy_url_is_refused( with pytest.raises(RuntimeError) as error: command.upgrade(config, "head") assert "sqlalchemy.url is not used" in refusal(error) - assert not decoy.exists() + assert not settings_target.exists() assert not named.exists() - def test_the_alembic_command_refuses_the_tracked_file(self) -> None: + def test_the_alembic_command_refuses_the_tracked_file(self, tmp_path: Path) -> None: + # Run outside the checkout, so the settings read no .env a developer keeps there. + config = str(REPO_ROOT / "alembic.ini") result = subprocess.run( - [sys.executable, "-m", "alembic", "upgrade", "head"], - cwd=REPO_ROOT, - env={"PATH": "", "DATABASE_URL": "sqlite+aiosqlite:///./github_activity.db"}, + [sys.executable, "-m", "alembic", "-c", config, "upgrade", "head"], + cwd=tmp_path, + env={"PATH": "", "DATABASE_URL": f"sqlite+aiosqlite:///{TRACKED}"}, capture_output=True, text=True, timeout=60, @@ -1177,7 +1089,17 @@ def test_head_tables_are_the_models_tables(head_template: Path) -> None: assert table_names(head_template) == set(Base.metadata.tables) | {"alembic_version"} -def revision_source_problems(path: Path) -> list[str]: +@dataclass(frozen=True) +class RevisionSource: + """What ``revision_source`` read of a revision file: the problems it found, and how many + import statements and ``VIEWS`` assignments it checked.""" + + problems: list[str] + imports: int + views: int + + +def revision_source(path: Path) -> RevisionSource: """Plan 5.1's rules for a revision file: it imports only the standard library, alembic, and SQLAlchemy (no application code, no relative import), and its ``VIEWS`` is a literal.""" tree = ast.parse(path.read_bytes()) @@ -1186,11 +1108,13 @@ def revision_source_problems(path: Path) -> list[str]: for line, module in imported_modules(path) if module.split(".")[0] not in REVISION_IMPORTS | sys.stdlib_module_names ] + imports = [node for node in ast.walk(tree) if isinstance(node, ast.Import | ast.ImportFrom)] problems += [ f"{path.name}:{node.lineno}: relative import" - for node in ast.walk(tree) + for node in imports if isinstance(node, ast.ImportFrom) and node.level > 0 ] + views = 0 for node in tree.body: if isinstance(node, ast.Assign): targets, value = node.targets, node.value @@ -1199,25 +1123,44 @@ def revision_source_problems(path: Path) -> list[str]: else: continue if any(isinstance(t, ast.Name) and t.id == "VIEWS" for t in targets): + views += 1 try: ast.literal_eval(value) except ValueError: problems.append(f"{path.name}:{node.lineno}: VIEWS is not a literal") - return problems + return RevisionSource(problems, len(imports), views) REVISION_IMPORTS = frozenset({"alembic", "sqlalchemy"}) +def import_keywords(path: Path) -> int: + """The ``import`` keywords the tokenizer reads in the file: one per import statement.""" + with path.open("rb") as source: + return sum( + token.type == tokenize.NAME and token.string == "import" + for token in tokenize.tokenize(source.readline) + ) + + class TestRevisionSources: - def test_every_revision_follows_the_rules(self) -> None: + def test_every_revision_follows_the_rules(self, tmp_path: Path) -> None: revisions = sorted((REPO_ROOT / "alembic" / "versions").glob("*.py")) assert len(revisions) == len(MIGRATION_REVISIONS) - assert [problem for path in revisions for problem in revision_source_problems(path)] == [] + read = [revision_source(path) for path in revisions] + assert [problem for source in read for problem in source.problems] == [] + # What the rules checked, counted another way: the tokenizer's import keywords, and the + # VIEWS of the modules alembic loads. + assert sum(source.imports for source in read) == sum(map(import_keywords, revisions)) + script = ScriptDirectory.from_config(alembic_config(tmp_path / "unused.db")) + assert sum(source.views for source in read) == sum( + hasattr(script.get_revision(revision).module, "VIEWS") + for revision in MIGRATION_REVISIONS + ) def test_the_newest_view_revision_has_a_literal_views_constant(self) -> None: - """Positive control: the rule reads the VIEWS the chain actually has, in the newest - revision that defines any.""" + """The rule reads the VIEWS the chain actually has, in the newest revision that defines + any.""" def defines_views(path: Path) -> bool: return any( @@ -1236,28 +1179,6 @@ def defines_views(path: Path) -> bool: ) assert newest.name.startswith(STEP3B1) - @pytest.mark.parametrize( - ("source", "flagged"), - [ - ("import json\nimport sqlalchemy as sa\nfrom alembic import op\n", False), - ("VIEWS = (('v', 'CREATE VIEW v AS SELECT 1'),)\n", False), - ("from typing import Final\nVIEWS: Final = (('v', 'SELECT 1'),)\n", False), - ("from github_activity_db.db import models\n", True), - ("import githubkit\n", True), - ("from . import shared\n", True), - ("import importlib\nimportlib.import_module('github_activity_db')\n", True), - ("VIEWS = SHARED_VIEWS\n", True), - ("VIEWS: tuple = tuple(view for view in ())\n", True), - ("VIEWS = (('v', 'CREATE VIEW v AS SELECT ' + '1'),)\n", True), - ], - ) - def test_flags_each_rule_on_planted_files( - self, tmp_path: Path, source: str, flagged: bool - ) -> None: - path = tmp_path / "f00000000001_planted.py" - path.write_text(source) - assert bool(revision_source_problems(path)) is flagged - def test_a_generated_revision_raises_in_downgrade_and_follows_the_rules(tmp_path: Path) -> None: """``alembic revision`` writes from alembic/script.py.mako.""" @@ -1266,7 +1187,7 @@ def test_a_generated_revision_raises_in_downgrade_and_follows_the_rules(tmp_path config = alembic_config(tmp_path / "unused.db", test_versions=versions) command.revision(config, message="probe", rev_id=TEST_REVISION, version_path=str(versions)) (path,) = versions.glob(f"{TEST_REVISION}_*.py") - assert revision_source_problems(path) == [] + assert revision_source(path) == RevisionSource([], import_keywords(path), 0) spec = importlib.util.spec_from_file_location("generated_revision", path) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) diff --git a/tests/db/test_schema.py b/tests/db/test_schema.py index 47ae539..bf7e9eb 100644 --- a/tests/db/test_schema.py +++ b/tests/db/test_schema.py @@ -2,22 +2,29 @@ ``alembic check`` compares neither views nor CHECK constraints, so these tests compare a database migrated to head with one ``metadata.create_all`` builds, structurally, and each view -with the newest migration's frozen SQL. +with the newest migration's frozen SQL. Where the readings must tell two schemas apart, the two +are databases the migrations built at neighbouring revisions; a shape no revision has is read +from a scratch database of valid SQL. """ from __future__ import annotations import ast -import shutil +import os +import re import sqlite3 +import tokenize from collections.abc import Callable from contextlib import closing from pathlib import Path from typing import Any, get_args import pytest -from sqlalchemy import create_engine, insert, pool +from alembic.script import ScriptDirectory +from sqlalchemy import CheckConstraint, Table, UniqueConstraint, create_engine, insert, pool +from sqlalchemy.dialects import sqlite from sqlalchemy.exc import IntegrityError +from sqlalchemy.schema import CreateIndex, CreateTable from github_activity_db.db.models import ( Base, @@ -26,10 +33,13 @@ SyncPending, ) from github_activity_db.source.records import AccountType, ReviewState -from tests.db.alembic_support import alembic_config, query +from tests.db.alembic_support import alembic_config, query, upgrade from tests.db.schema_support import ( + ALEMBIC_OBJECTS, check_constraints, + create_table_items, delete_actions, + foreign_key_columns, frozen_views, named_constraints, nullable_columns, @@ -39,9 +49,15 @@ unindexed_foreign_keys, unique_sets, view_differences, + views, ) from tests.db.seed import T0, Seed, connect +STEP3A = "33f8570f6540" +STEP3B1 = "94a52e4f1d57" +LAST_RUN_ID = "2d2ba76b177c" +SQLITE = sqlite.dialect() + @pytest.fixture(scope="module") def models_database(tmp_path_factory: pytest.TempPathFactory) -> Path: @@ -54,20 +70,36 @@ def models_database(tmp_path_factory: pytest.TempPathFactory) -> Path: return path -def _tampered(source: Path, target: Path, *statements: str) -> Path: - shutil.copyfile(source, target) - with closing(sqlite3.connect(target)) as connection, connection: +@pytest.fixture(scope="module") +def revisions(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Path]: + """Databases the migrations built at Step 3a, at Step 3b-1, and at the revision after it; + read-only, so no test changes one.""" + folder = tmp_path_factory.mktemp("revisions") + built = {} + for revision in (STEP3A, STEP3B1, LAST_RUN_ID): + built[revision] = folder / f"{revision}.db" + upgrade(built[revision], revision) + built[revision].chmod(0o444) + return built + + +def revision_module(database: Path, revision: str) -> Any: + """The revision file's module, for the constants it lists (CHANGED_TABLES, VIEWS, ...).""" + return ScriptDirectory.from_config(alembic_config(database)).get_revision(revision).module + + +def scratch(path: Path, *statements: str) -> Path: + """A database of valid SQL, which no migration built, for a reader's exact readings.""" + with closing(sqlite3.connect(path)) as connection, connection: for statement in statements: connection.execute(statement) - return target + return path -def _redefined(database: Path, table: str, old: str, new: str) -> tuple[str, str]: - """Statements that recreate ``table`` with ``old`` replaced by ``new`` in its definition.""" - with closing(sqlite3.connect(database)) as connection: - ((sql,),) = connection.execute("SELECT sql FROM sqlite_master WHERE name = ?", (table,)) - assert old in sql, (old, sql) - return f"DROP TABLE {table}", sql.replace(old, new) +def is_rowid(table: Table) -> bool: + """Whether the table's primary key is its rowid: one column declared INTEGER.""" + (*others, first) = table.primary_key.columns + return not others and str(first.type.compile(dialect=SQLITE)) == "INTEGER" @pytest.mark.parametrize( @@ -95,6 +127,66 @@ def test_a_named_check_is_read_whatever_whitespace_separates_its_words( } +# Shapes no table at head has, each with whitespace inside a literal, which is data. +TABLE = ( + "CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT NOT NULL, " + "CONSTRAINT ck_t_name CHECK (name != 'a b'))" +) +PARTIAL_INDEX = "CREATE INDEX ix_t_partial ON t (name) WHERE name != 'a b'" +EXPRESSION_INDEX = "CREATE INDEX ix_t_lower ON t (lower(name))" +TRIGGER = "CREATE TRIGGER tr_t BEFORE INSERT ON t BEGIN SELECT RAISE(ABORT, 'a b'); END" +VIEW = "CREATE VIEW v AS SELECT 'a b' AS value" +STRICT_TABLE = "CREATE TABLE s (k TEXT PRIMARY KEY, v INTEGER) STRICT" +WITHOUT_ROWID_TABLE = "CREATE TABLE w (k TEXT PRIMARY KEY) WITHOUT ROWID" + + +def test_the_readings_hold_what_no_pragma_reports(tmp_path: Path) -> None: + """A CHECK's text, a partial index's WHERE, an index expression, a trigger, and a view, each + exactly as written, and the STRICT and WITHOUT ROWID flags.""" + database = scratch( + tmp_path / "scratch.db", + TABLE, + PARTIAL_INDEX, + EXPRESSION_INDEX, + TRIGGER, + VIEW, + STRICT_TABLE, + WITHOUT_ROWID_TABLE, + ) + read = schema_of(database) + assert sorted(read) == ["s", "t", "w"] + assert read["t"]["definition"] == [ + "CONSTRAINT ck_t_name CHECK (name != 'a b')", + "id INTEGER PRIMARY KEY", + "name TEXT NOT NULL", + ] + # (created name and SQL, unique, origin, partial, index_xinfo rows); -2 is an expression. + assert read["t"]["indexes"] == [ + ( + ("ix_t_lower", EXPRESSION_INDEX), + 0, + "c", + 0, + ((0, -2, None, 0, "BINARY", 1), (1, -1, None, 0, "BINARY", 0)), + ), + ( + ("ix_t_partial", PARTIAL_INDEX), + 0, + "c", + 1, + ((0, 1, "name", 0, "BINARY", 1), (1, -1, None, 0, "BINARY", 0)), + ), + ] + assert read["t"]["triggers"] == [("tr_t", TRIGGER)] + assert views(database) == {"v": VIEW} + # (type, ncol, without rowid, strict) + assert [read[table]["flags"] for table in ("t", "s", "w")] == [ + [("table", 2, 0, 0)], + [("table", 2, 0, 1)], + [("table", 1, 1, 0)], + ] + + class TestModelsComparison: def test_head_matches_the_models(self, head_template: Path, models_database: Path) -> None: migrated, models = schema_of(head_template), schema_of(models_database) @@ -102,8 +194,60 @@ def test_head_matches_the_models(self, head_template: Path, models_database: Pat for table in models: assert migrated[table] == models[table], table + def test_the_readings_at_head_are_the_models_declarations(self, head_template: Path) -> None: + """Each table's reading against SQLAlchemy's metadata, not a pragma: its columns, its + foreign keys, its created indexes as the models compile them, the items of its compiled + definition, its flags, and no trigger.""" + read = schema_of(head_template) + assert sorted(read) == sorted(Base.metadata.tables) + for name, table in Base.metadata.tables.items(): + reading = read[name] + assert [(row[1], row[2], bool(row[3]), bool(row[5])) for row in reading["columns"]] == [ + ( + column.name, + str(column.type.compile(dialect=SQLITE)), + not column.nullable, + column.primary_key, + ) + for column in table.columns + ], name + assert reading["foreign_keys"] == sorted( + ( + tuple( + ( + seq, + element.column.table.name, + element.parent.name, + element.column.name, + key.onupdate or "NO ACTION", + key.ondelete or "NO ACTION", + "NONE", + ) + for seq, element in enumerate(key.elements) + ) + for key in table.foreign_key_constraints + ), + key=repr, + ), name + assert sorted(created for created, *_rest in reading["indexes"] if created) == sorted( + (index.name, str(CreateIndex(index).compile(dialect=SQLITE)).strip()) + for index in table.indexes + ), name + assert reading["definition"] == sorted( + create_table_items(str(CreateTable(table).compile(dialect=SQLITE))) + ), name + assert set(check_constraints(head_template, name)) == { + constraint.name + for constraint in table.constraints + if isinstance(constraint, CheckConstraint) + }, name + options = table.dialect_options["sqlite"] + without_rowid, strict = int(not options["with_rowid"]), int(options["strict"]) + assert reading["flags"] == [("table", len(table.columns), without_rowid, strict)], name + assert reading["triggers"] == [], name + def test_the_comparison_reads_every_table_and_check(self, models_database: Path) -> None: - """Positive control: the readings hold the CHECKs and tables the models declare.""" + """The readings hold the CHECKs and tables the models declare.""" assert set(schema_tables(models_database)) == set(Base.metadata.tables) checks = { name @@ -112,144 +256,47 @@ def test_the_comparison_reads_every_table_and_check(self, models_database: Path) } assert checks == set(CHECKS) | PAIRED_CHECKS - @pytest.mark.parametrize( - ("table", "old", "new"), - [ - pytest.param( - "automation_emails", - "CONSTRAINT ck_automation_emails_email_lowercase CHECK", - "CONSTRAINT ck_automation_emails_lower CHECK", - id="renamed CHECK", - ), - pytest.param( - "automation_emails", - "email = lower(email)", - "email = upper(email)", - id="changed CHECK text", - ), - pytest.param( - "automation_emails", - "CONSTRAINT ck_automation_emails_email_lowercase ", - "", - id="unnamed CHECK", - ), - pytest.param("pr_files", "path TEXT NOT NULL", "path TEXT", id="nullability"), - pytest.param( - "pr_commits", - "authored_at DATETIME NOT NULL", - "authored_at TEXT NOT NULL", - id="column type", - ), - pytest.param("pr_labels", " ON DELETE CASCADE", "", id="foreign key action"), - pytest.param( - "pr_labels", - "CONSTRAINT fk_pr_labels_pr_id_pull_requests", - "CONSTRAINT fk_pr_labels_pr", - id="foreign key name", - ), - pytest.param( - "pr_reviews", - "CONSTRAINT uq_pr_reviews_github_id UNIQUE", - "UNIQUE", - id="unnamed UNIQUE", - ), - ], - ) - def test_a_planted_difference_is_reported( - self, - head_template: Path, - models_database: Path, - tmp_path: Path, - table: str, - old: str, - new: str, - ) -> None: - tampered = _tampered( - head_template, tmp_path / "tampered.db", *_redefined(head_template, table, old, new) - ) - assert schema_of(tampered)[table] != schema_of(models_database)[table] - - def test_a_missing_index_is_reported( - self, head_template: Path, models_database: Path, tmp_path: Path - ) -> None: - tampered = _tampered( - head_template, tmp_path / "tampered.db", "DROP INDEX ix_pr_reviews_reviewer_id" - ) - assert schema_of(tampered)["pr_reviews"] != schema_of(models_database)["pr_reviews"] - - @pytest.mark.parametrize( - ("first", "second"), - [ - pytest.param( - ["CREATE INDEX ix_probe ON pr_labels (name) WHERE name <> 'a'"], - ["CREATE INDEX ix_probe ON pr_labels (name) WHERE name <> 'b'"], - id="partial index WHERE", - ), - pytest.param( - ["CREATE INDEX ix_probe ON pr_labels (lower(name))"], - ["CREATE INDEX ix_probe ON pr_labels (upper(name))"], - id="expression index", - ), - pytest.param( - [], - ["CREATE TRIGGER tr_probe AFTER DELETE ON pr_labels BEGIN SELECT 1; END"], - id="trigger", - ), - ], - ) - def test_index_sql_and_triggers_are_compared( - self, head_template: Path, tmp_path: Path, first: list[str], second: list[str] + def test_a_revision_s_changes_are_read_in_the_tables_it_lists( + self, revisions: dict[str, Path] ) -> None: - """Shapes no pragma reports: a partial index's WHERE, an expression, a trigger.""" - one = _tampered(head_template, tmp_path / "one.db", *first) - two = _tampered(head_template, tmp_path / "two.db", *second) - assert schema_of(one)["pr_labels"] != schema_of(two)["pr_labels"] - - @pytest.mark.parametrize("kind", ["check", "index", "trigger"]) - def test_whitespace_inside_a_literal_is_a_difference( - self, head_template: Path, tmp_path: Path, kind: str - ) -> None: - """Two schemas whose SQL differs only in a literal's whitespace read differently.""" - - def read(literal: str, name: str) -> dict[str, Any]: - statements: tuple[str, ...] - if kind == "check": - statements = _redefined( - head_template, - "pr_files", - "path TEXT NOT NULL", - f"path TEXT NOT NULL CHECK (path != {literal})", - ) - elif kind == "index": - statements = (f"CREATE INDEX ix_probe ON pr_files (path) WHERE path != {literal}",) - else: - statements = ( - "CREATE TRIGGER tr_probe BEFORE INSERT ON pr_files " - f"BEGIN SELECT RAISE(ABORT, {literal}); END", - ) - tampered = _tampered(head_template, tmp_path / f"{name}.db", *statements) - return schema_of(tampered)["pr_files"] - - one, again, two = read("'a b'", "one"), read("'a b'", "again"), read("'a b'", "two") - assert one == again - # The difference is read where the kind's SQL is: a table's items, an index, a trigger. - assert [key for key in one if one[key] != two[key]] == [ - {"check": "definition", "index": "indexes", "trigger": "triggers"}[kind] + """2d2ba76b177c adds sync_failures.last_run_id, its foreign key to sync_runs, and its + index; the readings before and after it differ in that table alone, in those readings.""" + before, after = schema_of(revisions[STEP3B1]), schema_of(revisions[LAST_RUN_ID]) + assert sorted(before) == sorted(after) + changed = {table for table in after if before[table] != after[table]} + assert changed == set(revision_module(revisions[LAST_RUN_ID], LAST_RUN_ID).CHANGED_TABLES) + old, new = before["sync_failures"], after["sync_failures"] + assert [key for key in new if old[key] != new[key]] == [ + "columns", + "indexes", + "foreign_keys", + "definition", + "flags", ] + assert {row[1] for row in new["columns"]} - {row[1] for row in old["columns"]} == { + "last_run_id" + } + assert {created[0] for created, *_ in new["indexes"] if created} - { + created[0] for created, *_ in old["indexes"] if created + } == {"ix_sync_failures_last_run_id"} + assert set(new["foreign_keys"]) - set(old["foreign_keys"]) == { + ((0, "sync_runs", "last_run_id", "id", "NO ACTION", "NO ACTION", "NONE"),) + } + assert set(old["foreign_keys"]) <= set(new["foreign_keys"]) - @pytest.mark.parametrize("option", ["STRICT", "WITHOUT ROWID"]) - def test_a_table_option_is_reported( - self, head_template: Path, models_database: Path, tmp_path: Path, option: str - ) -> None: - tampered = _tampered( - head_template, - tmp_path / "tampered.db", - *_redefined(head_template, "pr_labels", "\n)", f"\n) {option}"), - ) - assert ( - schema_of(tampered)["pr_labels"]["flags"] - != schema_of(models_database)["pr_labels"]["flags"] - ) + def test_a_check_s_changed_text_is_read(self, revisions: dict[str, Path]) -> None: + """94a52e4f1d57 widened two CHECKs, keeping their names.""" + before, after = revisions[STEP3A], revisions[STEP3B1] + changed = { + name + for table in set(schema_tables(before)) & set(schema_tables(after)) + for name, item in check_constraints(before, table).items() + if check_constraints(after, table).get(name, item) != item + } + assert changed == {"ck_github_users_type", "ck_pull_requests_state"} + old, new = schema_of(before), schema_of(after) + assert old["github_users"]["definition"] != new["github_users"]["definition"] + assert old["pull_requests"]["definition"] != new["pull_requests"]["definition"] class TestViews: @@ -264,37 +311,21 @@ def test_views_match_the_newest_frozen_sql(self, head_template: Path) -> None: "v_pr_relationships", } assert view_differences(head_template, frozen) == [] - - def test_a_changed_view_is_reported(self, head_template: Path, tmp_path: Path) -> None: - tampered = _tampered( - head_template, - tmp_path / "tampered.db", - "DROP VIEW v_github_users", - "CREATE VIEW v_github_users AS SELECT github_id, login FROM github_users", - "CREATE VIEW v_extra AS SELECT 1 AS one", - "DROP VIEW v_commit_authors", - ) - assert view_differences(tampered, frozen_views(alembic_config(head_template))) == [ - "missing v_commit_authors", - "not frozen v_extra", - "differs v_github_users", + # Each view compared, by plain equality of its SQL. + assert views(head_template) == {name: sql.strip() for name, sql in frozen.items()} + + def test_an_earlier_revision_s_views_are_reported(self, revisions: dict[str, Path]) -> None: + """Against Step 3b-1's frozen views, Step 3a lacks the views Step 3b-1 adds, has the two + it drops, and has the two it replaces with other SQL.""" + at_step3b1 = frozen_views(alembic_config(revisions[STEP3B1]), through=STEP3B1) + step3a = revision_module(revisions[STEP3B1], STEP3A) + step3b1 = revision_module(revisions[STEP3B1], STEP3B1) + assert view_differences(revisions[STEP3A], at_step3b1) == [ + *(f"missing {name}" for name in sorted(set(at_step3b1) - dict(step3a.VIEWS).keys())), + *(f"not frozen {name}" for name in sorted(step3b1.DROPPED_VIEWS)), + *(f"differs {name}" for name in sorted(step3b1.REPLACED_VIEWS)), ] - - def test_whitespace_inside_a_literal_is_a_difference( - self, head_template: Path, tmp_path: Path - ) -> None: - tampered = _tampered( - head_template, - tmp_path / "tampered.db", - "CREATE VIEW v_literal AS SELECT 'a b' AS value", - "CREATE VIEW v_kept AS SELECT 'a b' AS value", - ) - frozen = { - **frozen_views(alembic_config(head_template)), - "v_literal": "\nCREATE VIEW v_literal AS SELECT 'a b' AS value\n", - "v_kept": "\nCREATE VIEW v_kept AS SELECT 'a b' AS value\n", - } - assert view_differences(tampered, frozen) == ["differs v_literal"] + assert view_differences(revisions[STEP3B1], at_step3b1) == [] class TestObjectsOutsideTheComparison: @@ -304,22 +335,27 @@ class TestObjectsOutsideTheComparison: def test_head_has_none(self, head_template: Path) -> None: frozen = frozen_views(alembic_config(head_template)) assert objects_outside_the_comparison(head_template, frozen) == [] - - def test_each_planted_object_is_reported(self, head_template: Path, tmp_path: Path) -> None: - tampered = _tampered( - head_template, - tmp_path / "tampered.db", - "CREATE TRIGGER tr_view INSTEAD OF DELETE ON v_github_users BEGIN SELECT 1; END", - "CREATE INDEX ix_version ON alembic_version (version_num)", - "CREATE TRIGGER tr_version AFTER UPDATE ON alembic_version BEGIN SELECT 1; END", - "ANALYZE", + # Every row of sqlite_master, counted from the models, the frozen views, and alembic's + # own: each table, its created indexes, the index behind each UNIQUE, and the one behind + # a primary key that is not the rowid. + tables = Base.metadata.tables.values() + indexes = sum( + len(table.indexes) + + sum(isinstance(constraint, UniqueConstraint) for constraint in table.constraints) + + (not is_rowid(table)) + for table in tables ) - frozen = frozen_views(alembic_config(head_template)) - assert objects_outside_the_comparison(tampered, frozen) == [ - ("index", "ix_version"), - ("table", "sqlite_stat1"), - ("trigger", "tr_version"), - ("trigger", "tr_view"), + assert query(head_template, "SELECT COUNT(*) FROM sqlite_master") == [ + (len(tables) + indexes + len(frozen) + len(ALEMBIC_OBJECTS),) + ] + + def test_the_views_a_later_revision_drops_are_reported( + self, revisions: dict[str, Path] + ) -> None: + frozen = frozen_views(alembic_config(revisions[STEP3B1]), through=STEP3B1) + dropped = revision_module(revisions[STEP3B1], STEP3B1).DROPPED_VIEWS + assert objects_outside_the_comparison(revisions[STEP3A], frozen) == [ + ("view", name) for name in sorted(dropped) ] @@ -489,33 +525,29 @@ def test_the_unique_column_sets_are_the_plans(self, head_template: Path) -> None def test_references_cascade_or_set_null_as_the_plan_says(self, head_template: Path) -> None: assert delete_rule_problems(head_template) == (set(), OUTSIDE_THE_DELETE_RULE) - - def test_a_changed_rule_is_reported(self, head_template: Path, tmp_path: Path) -> None: - tampered = _tampered( - head_template, - tmp_path / "tampered.db", - *_redefined(head_template, "pr_files", "path TEXT NOT NULL", "path TEXT"), - *_redefined( - head_template, - "pr_reviews", - "REFERENCES github_users (id) ON DELETE SET NULL, \n\t" - "CONSTRAINT uq_pr_reviews_github_id UNIQUE (github_id)", - "REFERENCES github_users (id) ON DELETE CASCADE", - ), - *_redefined( - head_template, - "sync_failures", - "REFERENCES sync_runs (id), \n\tCONSTRAINT uq_sync_failures", - "REFERENCES sync_runs (id) ON DELETE CASCADE, \n\tCONSTRAINT uq_sync_failures", - ), - ) - assert nullable_columns(tampered) - NULLABLE == {("pr_files", "path")} - assert unique_sets(tampered)["pr_reviews"] == set() - assert delete_rule_problems(tampered) == ( - {("pr_reviews", "editor_id", "CASCADE"), ("sync_failures", "last_run_id", "CASCADE")}, - OUTSIDE_THE_DELETE_RULE, + # Every foreign-key column the models declare was read. + assert len(delete_actions(head_template)) == sum( + len(table.foreign_keys) for table in Base.metadata.tables.values() ) + def test_an_earlier_revision_reads_its_own_columns_and_keys( + self, revisions: dict[str, Path] + ) -> None: + """94a52e4f1d57 dropped two nullable columns, added a UNIQUE to repositories, and added + four tables; its own references keep the rule, as Step 3a's do.""" + before, after = revisions[STEP3A], revisions[STEP3B1] + assert nullable_columns(before) - nullable_columns(after) == { + ("github_users", "deleted_at"), + ("sync_runs", "halt_reason"), + } + old, new = unique_sets(before), unique_sets(after) + assert {table for table in new if old.get(table) != new[table]} == ( + set(schema_tables(after)) - set(schema_tables(before)) + ) | {"repositories"} + assert new["repositories"] - old["repositories"] == {("u", ("node_id",))} + for database in (before, after): + assert delete_rule_problems(database) == (set(), OUTSIDE_THE_DELETE_RULE) + def test_the_cascade_exceptions_are_the_primary_key_references( self, head_template: Path ) -> None: @@ -530,49 +562,45 @@ def test_the_cascade_exceptions_are_the_primary_key_references( ) } assert in_a_primary_key == GITHUB_USERS_IN_A_PRIMARY_KEY - - def test_a_primary_key_reference_set_null_is_reported( - self, head_template: Path, tmp_path: Path - ) -> None: - """The cascade exception is read per column: SET NULL on pr_assignees.account_id breaks - it.""" - tampered = _tampered( - head_template, - tmp_path / "tampered.db", - *_redefined( - head_template, - "pr_assignees", - "REFERENCES github_users (id) ON DELETE CASCADE", - "REFERENCES github_users (id) ON DELETE SET NULL", - ), - ) - assert delete_rule_problems(tampered) == ( - {("pr_assignees", "account_id", "SET NULL")}, - OUTSIDE_THE_DELETE_RULE, + # Each reference to github_users the models declare was read, per column. + assert len( + {action for action in delete_actions(head_template) if action[2] == "github_users"} + ) == sum( + key.column.table.name == "github_users" + for table in Base.metadata.tables.values() + for key in table.foreign_keys ) - def test_a_primary_key_column_and_a_second_reference_are_read( - self, head_template: Path, tmp_path: Path + def test_every_composite_primary_key_column_is_declared_not_null( + self, head_template: Path ) -> None: - """SQLite holds a composite primary key's columns NOT NULL only as declared, and one - column can carry two foreign keys.""" - tampered = _tampered( - head_template, - tmp_path / "tampered.db", - *_redefined(head_template, "pr_labels", "name TEXT NOT NULL", "name TEXT"), - *_redefined( - head_template, - "pr_reviews", - "ON DELETE SET NULL, \n\t", - "ON DELETE SET NULL, \n\t" - "FOREIGN KEY(reviewer_id) REFERENCES github_users (id) ON DELETE CASCADE, \n\t", - ), - ) - assert nullable_columns(tampered) - NULLABLE == {("pr_labels", "name")} - assert delete_rule_problems(tampered) == ( - {("pr_reviews", "reviewer_id", "CASCADE")}, - OUTSIDE_THE_DELETE_RULE, + """SQLite holds a composite primary key's columns NOT NULL only as declared.""" + composite = { + (table.name, column.name) + for table in Base.metadata.tables.values() + if len(table.primary_key.columns) > 1 + for column in table.primary_key.columns + } + assert composite + assert composite.isdisjoint(nullable_columns(head_template)) + + def test_the_plan_readers_read_each_column_and_key(self, tmp_path: Path) -> None: + """A composite primary key's column without NOT NULL is nullable, and each foreign key + of a column that references two tables is read.""" + database = scratch( + tmp_path / "scratch.db", + "CREATE TABLE p (id INTEGER NOT NULL PRIMARY KEY)", + "CREATE TABLE q (id INTEGER NOT NULL PRIMARY KEY)", + "CREATE TABLE c (a INTEGER, b TEXT NOT NULL, PRIMARY KEY (a, b), " + "FOREIGN KEY (a) REFERENCES p (id) ON DELETE CASCADE, " + "FOREIGN KEY (a) REFERENCES q (id) ON DELETE SET NULL)", ) + assert nullable_columns(database) == {("c", "a")} + assert unique_sets(database) == {"c": {("pk", ("a", "b"))}, "p": set(), "q": set()} + assert delete_actions(database) == { + ("c", "a", "p", "CASCADE"), + ("c", "a", "q", "SET NULL"), + } class TestForeignKeyIndexes: @@ -580,15 +608,13 @@ class TestForeignKeyIndexes: EXEMPT = frozenset({("pr_user_tags", ("user_tag_id",))}) def test_every_foreign_key_column_leads_an_index(self, head_template: Path) -> None: + # Every foreign key the models declare is examined. + assert len(foreign_key_columns(head_template)) == sum( + len(table.foreign_key_constraints) for table in Base.metadata.tables.values() + ) # Equality, not a subset: the exemption is still needed, and nothing else is missing. assert unindexed_foreign_keys(head_template) == self.EXEMPT - def test_a_missing_index_is_reported(self, head_template: Path, tmp_path: Path) -> None: - tampered = _tampered( - head_template, tmp_path / "tampered.db", "DROP INDEX ix_pr_comments_author_id" - ) - assert unindexed_foreign_keys(tampered) == self.EXEMPT | {("pr_comments", ("author_id",))} - def test_a_composite_primary_key_and_the_rowid_count(self, head_template: Path) -> None: """pr_labels.pr_id leads its primary key; repo_sync_state.repository_id is the rowid.""" foreign_keys = { @@ -646,6 +672,17 @@ def _review(seed: Seed, state: str) -> None: seed.review(pr, seed.next_id(), None, state) +def failed_check(error: pytest.ExceptionInfo[IntegrityError]) -> str: + """The CHECK SQLite refused a row for: the driver's error code says a CHECK failed, and the + constraint's name is in SQLite's fixed ``CHECK constraint failed: `` form.""" + driver_error = error.value.orig + assert isinstance(driver_error, sqlite3.IntegrityError), repr(driver_error) + assert driver_error.sqlite_errorname == "SQLITE_CONSTRAINT_CHECK" + named = re.fullmatch(r"CHECK constraint failed: (\w+)", str(driver_error)) + assert named is not None, str(driver_error) + return named[1] + + Write = Callable[[Seed, str], object] # CHECKs tying two columns together, each with its own test below. PAIRED_CHECKS = frozenset( @@ -712,9 +749,10 @@ def test_accepts_its_values_and_rejects_another( for value in accepted: with connect(migrated_database) as connection: write(Seed(connection), value) - with pytest.raises(IntegrityError, match=f"CHECK constraint failed: {name}"): + with pytest.raises(IntegrityError) as error: with connect(migrated_database) as connection: write(Seed(connection), rejected) + assert failed_check(error) == name @pytest.mark.parametrize( ("state", "merged_at", "accepted"), @@ -736,11 +774,9 @@ def write() -> None: if accepted: write() else: - with pytest.raises( - IntegrityError, - match="CHECK constraint failed: ck_pull_requests_merged_at_iff_merged", - ): + with pytest.raises(IntegrityError) as error: write() + assert failed_check(error) == "ck_pull_requests_merged_at_iff_merged" class TestEnterpriseAccountId: @@ -764,26 +800,42 @@ def write() -> None: if accepted: write() else: - with pytest.raises( - IntegrityError, - match="CHECK constraint failed: " - "ck_github_users_github_id_null_iff_enterprise_account", - ): + with pytest.raises(IntegrityError) as error: write() + assert failed_check(error) == "ck_github_users_github_id_null_iff_enterprise_account" ROOT = Path(__file__).resolve().parents[2] +ROOTS = ("src", "tests", "scripts", "alembic") -def create_all_uses(*roots: Path) -> list[str]: - """Every ``.create_all`` attribute, called or passed as ``run_sync(metadata.create_all)``.""" - found = [] +def create_all_uses(*roots: Path) -> tuple[list[str], int]: + """Every ``.create_all`` attribute, called or passed as ``run_sync(metadata.create_all)``, + and the number of files read.""" + found, files = [], 0 for root in roots: for path in sorted(root.rglob("*.py")): + files += 1 for node in ast.walk(ast.parse(path.read_bytes())): if isinstance(node, ast.Attribute) and node.attr == "create_all": found.append(f"{path.relative_to(root.parent)}:{node.lineno}") - return found + return found, files + + +def create_all_tokens(*roots: Path) -> tuple[int, set[str]]: + """As the tokenizer reads the same files: the ``create_all`` names, and the lines of the + strings and comments that hold the text.""" + text_tokens = {tokenize.STRING, tokenize.COMMENT, tokenize.FSTRING_MIDDLE} + names, texts = 0, set() + for root in roots: + for path in sorted(root.rglob("*.py")): + with path.open("rb") as source: + for token in tokenize.tokenize(source.readline): + if token.type == tokenize.NAME and token.string == "create_all": + names += 1 + elif token.type in text_tokens and "create_all" in token.string: + texts.add(f"{path.relative_to(root.parent)}:{token.start[0]}") + return names, texts class TestBuiltByMigrations: @@ -791,22 +843,17 @@ class TestBuiltByMigrations: side of the comparison above (plan Step 3).""" def test_create_all_is_used_only_by_the_comparison(self) -> None: - uses = create_all_uses(ROOT / "src", ROOT / "tests", ROOT / "scripts", ROOT / "alembic") + roots = [ROOT / name for name in ROOTS] + uses, files = create_all_uses(*roots) + assert files == sum( + name.endswith(".py") + for root in roots + for _directory, _subdirectories, names in os.walk(root) + for name in names + ) assert [use.split(":")[0] for use in uses] == ["tests/db/test_schema.py"], uses - - @pytest.mark.parametrize( - ("source", "flagged"), - [ - ("Base.metadata.create_all(engine)\n", True), - ("conn.run_sync(Base.metadata.create_all)\n", True), - ("def f():\n metadata.create_all(bind=engine)\n", True), - ('DECOY = "Base.metadata.create_all(engine)"\n', False), - ("# Base.metadata.create_all(engine)\n", False), - ("Base.metadata.drop_all(engine)\n", False), - ], - ) - def test_flags_every_use_and_no_text(self, tmp_path: Path, source: str, flagged: bool) -> None: - package = tmp_path / "pkg" - package.mkdir() - (package / "module.py").write_text(source) - assert bool(create_all_uses(package)) is flagged + # Each use is a name the tokenizer reads too; the text in strings and comments is none. + names, texts = create_all_tokens(*roots) + assert len(uses) == names + assert texts + assert texts.isdisjoint(uses) diff --git a/tests/db/test_tracked_database.py b/tests/db/test_tracked_database.py index 20d0db6..dd3d784 100644 --- a/tests/db/test_tracked_database.py +++ b/tests/db/test_tracked_database.py @@ -4,7 +4,9 @@ github_activity.db. Step 6 moves the database out of git and revisits these tests. """ +import hashlib import os +import re import shutil import sqlite3 import subprocess @@ -13,18 +15,38 @@ import pytest +from tests.db.alembic_support import FIRST_SCHEMA_HEAD, upgrade + REPO_ROOT = Path(__file__).resolve().parents[2] +TRACKED = REPO_ROOT / "github_activity.db" SCRIPT = REPO_ROOT / "scripts" / "check_db_journal.py" +# The scripts' whole environment: they read no variable, and run from this interpreter's path. +SCRIPT_ENVIRONMENT = {"PATH": os.environ["PATH"]} + + +def git_environment(home: Path) -> dict[str, str]: + """A git or pre-commit process's whole environment: a home of the test's own, so no + user-level ignore file applies, and no global or system configuration.""" + return { + "PATH": os.environ["PATH"], + "HOME": str(home), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + } def _check(path: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( - [sys.executable, str(SCRIPT), str(path)], capture_output=True, text=True, timeout=60 + [sys.executable, str(SCRIPT), str(path)], + capture_output=True, + text=True, + env=SCRIPT_ENVIRONMENT, + timeout=60, ) def test_tracked_database_is_in_rollback_journal_mode() -> None: - result = _check(REPO_ROOT / "github_activity.db") + result = _check(TRACKED) assert result.returncode == 0, result.stdout assert "rollback journal" in result.stdout @@ -57,35 +79,54 @@ def test_non_database_file_is_rejected(tmp_path: Path) -> None: def _frozen(*paths: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( - [sys.executable, str(FROZEN), *map(str, paths)], capture_output=True, text=True, timeout=60 + [sys.executable, str(FROZEN), *map(str, paths)], + capture_output=True, + text=True, + env=SCRIPT_ENVIRONMENT, + timeout=60, ) +def printed_digests(stdout: str) -> dict[str, str]: + """Each file the script read, with the SHA-256 it prints for it (``: sha256 ``).""" + found = {} + for line in stdout.splitlines(): + printed = re.fullmatch(r"(?P.+?): sha256 (?P[0-9a-f]{64})(?:\W.*)?", line) + if printed is not None: + found[printed["path"]] = printed["digest"] + return found + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + class TestFrozenContent: """scripts/check_frozen_db.py pins the tracked file's pre-Step-3 content (D26).""" def test_the_tracked_file_has_the_pinned_digest(self) -> None: - result = _frozen(REPO_ROOT / "github_activity.db") + result = _frozen(TRACKED) assert result.returncode == 0, result.stdout - assert "the pinned frozen content" in result.stdout + assert printed_digests(result.stdout) == {str(TRACKED): sha256(TRACKED)} def test_an_identical_copy_passes(self, tmp_path: Path) -> None: copy = tmp_path / "copy.db" - shutil.copyfile(REPO_ROOT / "github_activity.db", copy) + shutil.copyfile(TRACKED, copy) assert _frozen(copy).returncode == 0 - def test_a_changed_copy_fails(self, tmp_path: Path) -> None: - copy = tmp_path / "changed.db" - shutil.copyfile(REPO_ROOT / "github_activity.db", copy) - connection = sqlite3.connect(copy) - try: - connection.execute("PRAGMA user_version = 1") - finally: - connection.close() - result = _frozen(REPO_ROOT / "github_activity.db", copy) + def test_a_database_the_migrations_wrote_fails( + self, tmp_path: Path, head_template: Path + ) -> None: + """Files this project's migrations wrote, at head and at the first schema's head, which + the tracked file is at: neither has the tracked file's bytes.""" + first_schema = tmp_path / "first.db" + upgrade(first_schema, FIRST_SCHEMA_HEAD) + result = _frozen(TRACKED, head_template, first_schema) assert result.returncode == 1 - assert f"{copy}: sha256 " in result.stdout - assert "differs from the pinned" in result.stdout + assert printed_digests(result.stdout) == { + str(path): sha256(path) for path in (TRACKED, head_template, first_schema) + } + assert sha256(TRACKED) not in {sha256(head_template), sha256(first_schema)} def test_a_missing_file_fails(self, tmp_path: Path) -> None: result = _frozen(tmp_path / "absent.db") @@ -95,13 +136,13 @@ def test_a_missing_file_fails(self, tmp_path: Path) -> None: def test_a_symlink_to_an_identical_copy_fails(self, tmp_path: Path) -> None: """git would store the link text, not the database.""" copy = tmp_path / "copy.db" - shutil.copyfile(REPO_ROOT / "github_activity.db", copy) + shutil.copyfile(TRACKED, copy) (tmp_path / "github_activity.db").symlink_to(copy) result = _frozen(tmp_path / "github_activity.db") assert result.returncode == 1 assert "is a symlink" in result.stdout - def test_git_stores_the_tracked_file_as_a_regular_file(self) -> None: + def test_git_stores_the_tracked_file_as_a_regular_file(self, tmp_path: Path) -> None: """What a clone or a raw download gets: deleted, renamed, or committed as a symlink, the file is not this one entry.""" assert GIT is not None @@ -111,6 +152,7 @@ def test_git_stores_the_tracked_file_as_a_regular_file(self) -> None: capture_output=True, text=True, check=True, + env=git_environment(tmp_path), timeout=60, ) (entry,) = result.stdout.splitlines() @@ -122,13 +164,14 @@ class TestGitignore: """Only the repository-root github_activity.db escapes the *.db rule (D26).""" @staticmethod - def _ignored(path: str) -> bool: + def _ignored(path: str, home: Path) -> bool: assert GIT is not None result = subprocess.run( [GIT, "check-ignore", "--no-index", "--quiet", path], cwd=REPO_ROOT, capture_output=True, text=True, + env=git_environment(home), timeout=60, ) assert result.returncode in (0, 1), result.stderr @@ -137,22 +180,29 @@ def _ignored(path: str) -> bool: @pytest.mark.parametrize( "path", ["github_activity.local.db", "data/github_activity.db", "scratch.db"] ) - def test_other_database_files_are_ignored(self, path: str) -> None: - assert self._ignored(path) + def test_other_database_files_are_ignored(self, tmp_path: Path, path: str) -> None: + assert self._ignored(path, tmp_path) @pytest.mark.parametrize( "path", ["x.db-journal", "x.sqlite-journal", "x.sqlite3-wal", "x.sqlite-shm"] ) - def test_transient_files_are_ignored(self, path: str) -> None: - assert self._ignored(path) + def test_transient_files_are_ignored(self, tmp_path: Path, path: str) -> None: + assert self._ignored(path, tmp_path) - def test_the_tracked_file_is_not_ignored(self) -> None: - assert not self._ignored("github_activity.db") + def test_the_tracked_file_is_not_ignored(self, tmp_path: Path) -> None: + assert not self._ignored("github_activity.db", tmp_path) -def _git(repo: Path, *args: str) -> None: +def _git(repo: Path, home: Path, *args: str) -> None: assert GIT is not None - subprocess.run([GIT, *args], cwd=repo, check=True, capture_output=True, timeout=60) + subprocess.run( + [GIT, *args], + cwd=repo, + check=True, + capture_output=True, + env=git_environment(home), + timeout=60, + ) class TestNoSidecarFiles: @@ -161,8 +211,9 @@ class TestNoSidecarFiles: SQLite finds it under any letter case.""" SIDECARS = ("github_activity.db-journal", "github_activity.db-wal", "github_activity.db-shm") + HOOK = "tracked-db-no-sidecar" - def test_none_is_tracked(self) -> None: + def test_none_is_tracked(self, tmp_path: Path) -> None: assert GIT is not None result = subprocess.run( [GIT, "ls-files", "--", *(f":(icase){name}" for name in self.SIDECARS)], @@ -170,37 +221,45 @@ def test_none_is_tracked(self) -> None: capture_output=True, text=True, check=True, + env=git_environment(tmp_path), timeout=60, ) assert result.stdout == "" - @staticmethod - def _run_hook(tmp_path: Path, name: str, *, symlink: bool = False) -> int: + @classmethod + def _run_hook(cls, tmp_path: Path, name: str, *, symlink: bool = False) -> int: """pre-commit itself, on a scratch repository holding ``name``, with the hook's lines copied from the real config (a local hook, so nothing is fetched).""" config = (REPO_ROOT / ".pre-commit-config.yaml").read_text() - start = config.index(" - id: tracked-db-no-sidecar\n") + start = config.index(f" - id: {cls.HOOK}\n") hook = config[start : config.index("\n - id: ", start)] + (hook_name,) = re.findall(r"^ name: (.+)$", hook, re.MULTILINE) hooks = tmp_path / "hooks.yaml" hooks.write_text(f"repos:\n - repo: local\n hooks:\n{hook}\n") + home = tmp_path / "home" + home.mkdir() repo = tmp_path / "repo" repo.mkdir() - _git(repo, "init", "-q") + _git(repo, home, "init", "-q") if symlink: (repo / "target").write_text("journal bytes") (repo / name).symlink_to("target") else: (repo / name).write_text("journal bytes") - _git(repo, "add", "-f", "--", name) + _git(repo, home, "add", "-f", "--", name) result = subprocess.run( [sys.executable, "-m", "pre_commit", "run", "--all-files", "--config", str(hooks)], cwd=repo, - env={**os.environ, "PRE_COMMIT_HOME": str(tmp_path / "pre-commit-home")}, + env={ + **git_environment(home), + "PRE_COMMIT_HOME": str(tmp_path / "pre-commit-home"), + }, capture_output=True, text=True, timeout=120, ) - assert "no journal, WAL, or shared-memory file" in result.stdout, result.stdout + # The config held the hook: pre-commit reports it by name, run or skipped. + assert hook_name in result.stdout, result.stdout return result.returncode @pytest.mark.parametrize( diff --git a/tests/db/test_types.py b/tests/db/test_types.py index 59be93b..aafaabb 100644 --- a/tests/db/test_types.py +++ b/tests/db/test_types.py @@ -9,6 +9,7 @@ from sqlalchemy import DateTime, insert, select, update from sqlalchemy.exc import StatementError +from github_activity_db.db import types from github_activity_db.db.models import Base, GitHubUser, UserTag from github_activity_db.db.types import UtcDateTime, utc_now from tests.db.alembic_support import query @@ -40,6 +41,11 @@ def _stored_refreshed_at(database: Path) -> str: return str(text) +def _stored_created_at(database: Path) -> str: + ((text,),) = query(database, "SELECT created_at FROM user_tags") + return str(text) + + def test_no_column_uses_a_bare_datetime() -> None: types = { f"{table.name}.{column.name}": column.type @@ -60,14 +66,17 @@ def test_no_column_uses_a_bare_datetime() -> None: ], ) def test_a_naive_value_is_rejected(migrated_database: Path, value: datetime) -> None: - with pytest.raises(StatementError, match="naive datetime 2026-01-01T12:00:00"): + with pytest.raises(StatementError) as error: _write_refreshed_at(migrated_database, value) + assert type(error.value.orig) is ValueError assert query(migrated_database, "SELECT COUNT(*) FROM github_users") == [(0,)] def test_a_value_that_is_not_a_datetime_is_rejected(migrated_database: Path) -> None: - with pytest.raises(StatementError, match="expected a datetime, got str"): + with pytest.raises(StatementError) as error: _write_refreshed_at(migrated_database, "2026-01-01 12:00:00") + assert type(error.value.orig) is TypeError + assert query(migrated_database, "SELECT COUNT(*) FROM github_users") == [(0,)] def test_an_aware_value_is_stored_as_utc_and_read_back_aware(migrated_database: Path) -> None: @@ -105,11 +114,27 @@ def test_null_passes_through(migrated_database: Path) -> None: assert connection.execute(select(GitHubUser.not_found_at)).scalar_one() is None -def test_the_default_is_now_in_utc(migrated_database: Path) -> None: - before = utc_now() +# A fixed instant for the clock the column default reads. +NOW = datetime(2026, 1, 2, 3, 4, 5, 678901, tzinfo=UTC) + + +class FixedClock(datetime): + """datetime as types.py sees it while a test runs: ``now`` is NOW in the zone asked for.""" + + @classmethod + def now(cls, tz: tzinfo | None = None) -> FixedClock: + assert tz is UTC + return cls.combine(NOW.date(), NOW.timetz()) + + +def test_the_default_is_now_in_utc( + migrated_database: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(types, "datetime", FixedClock) + assert utc_now() == NOW with connect(migrated_database) as connection: connection.execute(insert(UserTag).values(name="kept")) created_at = connection.execute(select(UserTag.created_at)).scalar_one() - assert before <= created_at <= utc_now() + assert created_at == NOW assert created_at.tzinfo is UTC - assert utc_now().tzinfo is UTC + assert _stored_created_at(migrated_database) == "2026-01-02 03:04:05.678901" diff --git a/tests/db/test_views.py b/tests/db/test_views.py index 109e414..193cbb5 100644 --- a/tests/db/test_views.py +++ b/tests/db/test_views.py @@ -235,13 +235,34 @@ def test_no_view_value_is_a_row_id(self, seed: Seed) -> None: seed.assignee(pr, author) seed.sync_run(id=next(ids)) row_ids = set(range(7_000_000_001, next(ids))) - for view in COLUMNS: - values = {value for row in rows(seed, f"SELECT * FROM {view}") for value in row} - assert values, view + # Every id drawn is a row id the file holds. + assert { + value + for table in ( + "github_users", + "repositories", + "pull_requests", + "pr_reviews", + "pr_comments", + "sync_runs", + ) + for (value,) in rows(seed, f"SELECT id FROM {table}") + } == row_ids + read = {view: rows(seed, f"SELECT * FROM {view}") for view in COLUMNS} + # The rows compared: one per seeded account, profile, review, comment, and commit, and + # one per relationship of the pull request (author, merger, editor, reviewer, review + # editor, commenter, comment editor, commit author, requested reviewer, assignee). + assert {view: len(table) for view, table in read.items()} == { + "v_github_users": 2, + "v_account_profiles": 1, + "v_pr_reviews": 1, + "v_pr_comments": 1, + "v_commit_authors": 1, + "v_pr_relationships": 10, + } + for view, table in read.items(): + values = {value for row in table for value in row} assert values.isdisjoint(row_ids), (view, values & row_ids) - # Positive control: a row id projected under a consumer name is caught. - renamed = {value for (value,) in rows(seed, "SELECT id AS pr_number FROM pull_requests")} - assert renamed & row_ids class Scenario: @@ -1209,23 +1230,24 @@ def count() -> int: return steps -# One index per foreign-key column instead, created in the order the migration once created -# them. With two single-column candidates the planner took the one created last, the account's, -# and read every row of the account for each PR (observed on SQLite 3.45.3 and 3.47.1). -SINGLE_COLUMN_INDEXES = ( - "DROP INDEX ix_pr_reviews_pr_id_reviewer_id_github_id", - "DROP INDEX ix_pr_reviews_reviewer_id", - "CREATE INDEX ix_pr_reviews_pr_id ON pr_reviews (pr_id)", - "CREATE INDEX ix_pr_reviews_reviewer_id ON pr_reviews (reviewer_id)", -) - - class TestConcentratedParticipation: """Four times the PRs costs v_pr_comments about four times the work, not sixteen times: - ``author_also_reviewed`` is an index lookup on (PR, account) (plan 5.1).""" + ``author_also_reviewed`` is an index lookup on (PR, account) (plan 5.1). With one index per + foreign-key column instead, the planner took the account's and read every row of the + account for each PR (observed on SQLite 3.45.3 and 3.47.1).""" SIZES = (100, 400) + @staticmethod + def rows_read(prs: int) -> dict[str, int]: + """The rows each view returns for ``prs`` pull requests of the seed loop: a review and + two comments on each, and in v_pr_relationships an author row for each as well.""" + return { + "v_pr_reviews": prs, + "v_pr_comments": 2 * prs, + "v_pr_relationships": prs + prs + 2 * prs, + } + @pytest.fixture(scope="class") def databases( self, head_template: Path, tmp_path_factory: pytest.TempPathFactory @@ -1236,27 +1258,17 @@ def databases( built[prs] = folder / f"{prs}.db" shutil.copyfile(head_template, built[prs]) _one_account_on_every_pull_request(built[prs], prs) + built[prs].chmod(0o444) return built - def growth(self, databases: dict[int, Path], tmp_path: Path, view: str, *alter: str) -> float: - work = [] - for prs in self.SIZES: - database = tmp_path / f"{prs}.db" - shutil.copyfile(databases[prs], database) - with closing(sqlite3.connect(database)) as connection, connection: - for statement in alter: - connection.execute(statement) - work.append(vm_steps(database, f"SELECT * FROM {view}")) + def growth(self, databases: dict[int, Path], view: str) -> float: + work = [vm_steps(databases[prs], f"SELECT * FROM {view}") for prs in self.SIZES] return work[1] / work[0] @pytest.mark.parametrize("view", ["v_pr_reviews", "v_pr_comments", "v_pr_relationships"]) - def test_work_grows_with_the_pull_requests( - self, databases: dict[int, Path], tmp_path: Path, view: str - ) -> None: - assert self.growth(databases, tmp_path, view) < 6 - - def test_a_lookup_by_the_account_alone_is_reported( - self, databases: dict[int, Path], tmp_path: Path - ) -> None: - """Positive control: with single-column indexes the same measure grows with the square.""" - assert self.growth(databases, tmp_path, "v_pr_comments", *SINGLE_COLUMN_INDEXES) > 10 + def test_work_grows_with_the_pull_requests(self, databases: dict[int, Path], view: str) -> None: + for prs in self.SIZES: + assert query(databases[prs], f"SELECT COUNT(*) FROM {view}") == [ + (self.rows_read(prs)[view],) + ], prs + assert self.growth(databases, view) < 6 diff --git a/tests/fixtures/history/21a4267/docs/data-dictionary.md b/tests/fixtures/history/21a4267/docs/data-dictionary.md new file mode 100644 index 0000000..c4e778a --- /dev/null +++ b/tests/fixtures/history/21a4267/docs/data-dictionary.md @@ -0,0 +1,678 @@ +# Data Dictionary + +Every table, view, and column of the database at the migration head, each marked **consumer** +or **internal**. `tests/test_data_dictionary.py` fails when an object or column at head has no +entry here, or an entry names one that does not exist. + +## Consumer interface + +- The interface is the `v_*` views, every column of which is consumer, and the table columns + marked consumer below. Every column holding what GitHub reports is consumer. +- Internal columns can change in any migration. +- Row ids join rows within one file: a pull request's labels, reviews, and commits by `pr_id`, + its author by `author_id`. The `*_id` columns of consumer tables, and the `id` columns they + reference, are consumer; other row ids are internal. Row ids change on a rebuild, so never + store one or compare it across files. Across files, identify a row by its `github_id` + (`repositories.github_id`, `pull_requests.github_id`, ...), an account with a null + `github_id` by its `node_id`, and a label, commit, file, closing link, review request, or + assignee by its pull request's `github_id` and its own key; (`repositories.full_name`, + `pull_requests.number`) identifies a pull request among tracked repositories within one file, + and changes when the repository is renamed. +- `PRAGMA user_version` holds the schema version (`SCHEMA_VERSION` in + `src/github_activity_db/db/schema.py`). It is bumped when a consumer column is removed, + renamed, or changes meaning, or a view's frozen SQL changes (its rows, its derivation, or its + layout alone); an added table column, table, or view does not bump it. + `tests/schema_versions.json` lists each version's consumer columns and, from version 2, the + SHA-256 of each view's SQL exactly as its migration embeds it (SQLite stores that text without + its leading and trailing line breaks); on pull requests CI fails when an + entry, a column of one, or a view digest of one that exists on the base branch is removed or + changed. +- Timestamps are UTC, stored as `YYYY-MM-DD HH:MM:SS.ffffff` text, so text order is time order. + Booleans are stored as 0 and 1. +- Values GitHub reports keep GitHub's names in snake case and its values. Every GitHub enum + except a pull request's and a review's `state` (and `github_users.type`) is GitHub's text, + stored without a check, such as `review_decision` or `mergeable`, so a value GitHub adds is + stored. No column of GitHub's data holds a value the database chose, except the + classifications named below (`targets_default_branch`, `is_pr_author`, `author_also_reviewed`, + `automation_reason`, `email_automation_reason`), each a rule over stored columns of the same + file (D29). An `ordinal` column is the entry's position in GitHub's list, numbered from 0. +- The references D28 keeps (a pull request's author and merger, a review's reviewer, a + comment's author, a commit's account) are null when GitHub returned no actor and the row had + none stored; GitHub does not say why. A reference the database kept when GitHub later + returned no actor has its `*_reported` column 0; on a commit, `linked_github_id` holds the + account id GitHub linked in the latest hydration (D28). Editors and requested accounts take + GitHub's current value, null included, and `pr_assignees` holds GitHub's current + `assignedActors` list. Each column's entry gives its further null cases. An account a lookup answered `NOT_FOUND` for keeps its + row, its `not_found_at` set. +- An account's `type` and the reason of the `[automation]` entry listing it are separate + columns; nothing merges them into one flag, and the views keep every row (D29). +- The run records (`sync_runs`, `repo_sync_state`, `sync_pending`, `sync_failures`), the + automation lists, `is_tracked`, `repo_group`, and `pull_requests.hydration_version` describe + the database's own configuration and runs, not GitHub's data. A view's `relationship` column names the stored row a view row comes + from. + +The decisions cited (D3, D17, ...) are in [docs/redesign-plan.md](redesign-plan.md). + +## Tables + +### `repositories` + +One row per repository the sync has resolved, with every field the resolve reads (redesign plan +5.8), as the latest complete resolve read them. + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | consumer | Row id | +| `github_id` | consumer | GitHub's numeric repository id | +| `node_id` | consumer | GitHub's GraphQL node id | +| `owner` | consumer | Owner login, in GitHub's spelling | +| `name` | consumer | Repository name, in GitHub's spelling | +| `full_name` | consumer | `owner/name`; changes when the repository is renamed. With a pull request's `number` it identifies the pull request among tracked repositories | +| `repo_group` | consumer | The group `tracking.toml` assigned at the last resolve, a name the maintainer gives (`Client`, `Server`, `Mobile`, `Docs`, `Agentic`, `Tools`) | +| `is_private` | consumer | GitHub's `isPrivate` | +| `is_archived` | consumer | GitHub's `isArchived` | +| `default_branch` | consumer | The default branch's name | +| `labels_total` | consumer | GitHub's `totalCount` of the repository's labels (`repo_labels` rows) | +| `is_tracked` | consumer | 1 when `tracking.toml` listed the repository at the last sync that checked (redesign plan 5.3 step 1); an untracked repository's rows stay until `ghactivity sync --prune-untracked` (D27) | +| `resolved_at` | consumer | GitHub's `Date` header of the first response of the resolve that wrote the row: the as-of time of its GitHub fields and labels | +| `is_security_policy_enabled` | consumer | GitHub's `isSecurityPolicyEnabled`; null when GitHub returns none | +| `visibility` | consumer | GitHub's `visibility` (`PUBLIC`, `PRIVATE`, ...) | +| `description` | consumer | GitHub's `description`; null when there is none | +| `homepage_url` | consumer | GitHub's `homepageUrl`, a link the owner sets; null when there is none | +| `created_at` | consumer | GitHub's `createdAt` | +| `updated_at` | consumer | GitHub's `updatedAt` | +| `pushed_at` | consumer | GitHub's `pushedAt`; null when GitHub returns none | +| `archived_at` | consumer | GitHub's `archivedAt`; null when the repository is not archived | +| `stargazer_count` | consumer | GitHub's `stargazerCount` | +| `fork_count` | consumer | GitHub's `forkCount` | +| `disk_usage` | consumer | GitHub's `diskUsage`, in kilobytes; null when GitHub returns none | +| `forking_allowed` | consumer | GitHub's `forkingAllowed` | +| `allow_update_branch` | consumer | GitHub's `allowUpdateBranch` | +| `web_commit_signoff_required` | consumer | GitHub's `webCommitSignoffRequired` | +| `has_wiki_enabled` | consumer | GitHub's `hasWikiEnabled` | +| `has_discussions_enabled` | consumer | GitHub's `hasDiscussionsEnabled` | +| `has_projects_enabled` | consumer | GitHub's `hasProjectsEnabled` | +| `has_vulnerability_alerts_enabled` | consumer | GitHub's `hasVulnerabilityAlertsEnabled` | +| `uses_custom_open_graph_image` | consumer | GitHub's `usesCustomOpenGraphImage` | +| `merge_commit_allowed` | consumer | GitHub's `mergeCommitAllowed` | +| `squash_merge_allowed` | consumer | GitHub's `squashMergeAllowed` | +| `rebase_merge_allowed` | consumer | GitHub's `rebaseMergeAllowed` | +| `delete_branch_on_merge` | consumer | GitHub's `deleteBranchOnMerge` | +| `merge_commit_title` | consumer | GitHub's `mergeCommitTitle` setting | +| `merge_commit_message` | consumer | GitHub's `mergeCommitMessage` setting | +| `squash_merge_commit_title` | consumer | GitHub's `squashMergeCommitTitle` setting | +| `squash_merge_commit_message` | consumer | GitHub's `squashMergeCommitMessage` setting | +| `primary_language` | consumer | The name of GitHub's `primaryLanguage`; null when there is none | +| `license_spdx_id` | consumer | The `spdxId` of GitHub's `licenseInfo`; null when GitHub returns no license or no `spdxId` | +| `plan_features_codeowners` | consumer | GitHub's `planFeatures.codeowners` | +| `plan_features_team_review_requests` | consumer | GitHub's `planFeatures.teamReviewRequests` | +| `plan_features_maximum_manual_review_requests` | consumer | GitHub's `planFeatures.maximumManualReviewRequests` | + +### `repo_labels` + +Each repository's label set as the latest complete resolve read it (D8). + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_id` | consumer | Row id of the repository | +| `node_id` | consumer | GitHub's label id | +| `name` | consumer | Label name | +| `color` | consumer | GitHub's `color` | +| `description` | consumer | GitHub's `description`; null when there is none | +| `is_default` | consumer | GitHub's `isDefault` | +| `created_at` | consumer | GitHub's `createdAt`; null when GitHub returns none | +| `updated_at` | consumer | GitHub's `updatedAt`; null when GitHub returns none | + +### `github_users` + +Every account the database has stored, kept after a lookup answers `NOT_FOUND` for it (D28). + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | consumer | Row id | +| `github_id` | consumer | GitHub's numeric account id; null only for an `EnterpriseUserAccount`, which has none | +| `node_id` | consumer | GitHub's GraphQL node id; the key of an account with a null `github_id` | +| `login` | consumer | Last known login, not unique; an actor sighting sets it when it inserts the account and only lookups change it; each sync refreshes accounts last refreshed more than 24 hours ago, except those with `not_found_at` set | +| `type` | consumer | GitHub's type: `User`, `Bot`, `Organization`, `Mannequin`, or `EnterpriseUserAccount` | +| `name` | consumer | GitHub's `name`; null until the account's first lookup, for a `Bot`, and when GitHub returns none | +| `created_at` | consumer | GitHub's `createdAt`; null until the account's first lookup | +| `updated_at` | consumer | GitHub's `updatedAt`; null until the account's first lookup | +| `linked_user_github_id` | consumer | An `EnterpriseUserAccount`'s `user.databaseId`; null for another type or when GitHub returns none | +| `claimant_github_id` | consumer | A `Mannequin`'s `claimant.databaseId`; null until the account's first lookup, for another type, and when GitHub returns none | +| `refreshed_at` | consumer | GitHub's `Date` header of the response that inserted the row or of its latest lookup | +| `not_found_at` | consumer | When a lookup answered `NOT_FOUND` for the account; null otherwise. No reference reads it | + +### `account_profiles` + +A `User`'s or an `Organization`'s public profile as the latest lookup returned it (D31); a +`Bot`, `Mannequin`, or `EnterpriseUserAccount` has no row. A lookup that answers `NOT_FOUND` +leaves the row as it was. + +| Column | Use | Meaning | +|--------|-----|---------| +| `account_id` | consumer | Row id of the account | +| `company` | consumer | A user's public profile `company`; null for an organization and when there is none | +| `location` | consumer | The profile's `location`; null when there is none | +| `website_url` | consumer | The profile's `websiteUrl`; null when there is none | +| `description` | consumer | An organization's `description`; null for a user and when there is none | +| `email` | consumer | An organization's public `email`; null for a user and when there is none. A user's profile email is not collected (D31) | +| `twitter_username` | consumer | An organization's `twitterUsername`; null for a user and when there is none | +| `is_verified` | consumer | An organization's `isVerified`; null for a user | +| `archived_at` | consumer | An organization's `archivedAt`; null for a user and when it is not archived | +| `has_sponsors_listing` | consumer | An organization's `hasSponsorsListing`; null for a user | +| `pinned_items_remaining` | consumer | An organization's `pinnedItemsRemaining`; null for a user | +| `refreshed_at` | consumer | GitHub's `Date` header of the lookup that wrote the row | + +### `pull_requests` + +Pull requests in coverage: open ones, ones merged on or after the coverage floor in +`tracking.toml`, and ones closed without merge on or after it (redesign plan section 3, D3). + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | consumer | Row id | +| `repository_id` | consumer | Row id of the repository | +| `number` | consumer | Pull request number | +| `github_id` | consumer | GitHub's numeric pull request id (can exceed 2^31) | +| `node_id` | consumer | GitHub's GraphQL node id | +| `state` | consumer | GitHub's `state`: `OPEN`, `CLOSED`, or `MERGED` | +| `is_draft` | consumer | GitHub's `isDraft` | +| `title` | consumer | Title | +| `body` | consumer | Description, empty when there is none (D19) | +| `author_id` | consumer | Row id of the author's account; null when GitHub returned no author and the row had none stored | +| `author_reported` | consumer | 1 when GitHub returned an author in the row's latest hydration; 0 when it returned none, `author_id` then being a reference kept from earlier or null (D28) | +| `author_association` | consumer | GitHub's current `authorAssociation` (`MEMBER`, `CONTRIBUTOR`, ...), which GitHub computes when read | +| `created_at` | consumer | GitHub's `createdAt` | +| `merged_at` | consumer | GitHub's `mergedAt`; set exactly when `state` is `MERGED` | +| `closed_at` | consumer | GitHub's `closedAt`; set for every `CLOSED` or `MERGED` row (plan 5.3 step 5) | +| `published_at` | consumer | GitHub's `publishedAt`; null when GitHub returns none | +| `last_edited_at` | consumer | GitHub's `lastEditedAt`; null when GitHub returns none | +| `editor_id` | consumer | Row id of the account GitHub lists as the last editor; null when GitHub returns none | +| `includes_created_edit` | consumer | GitHub's `includesCreatedEdit` | +| `maintainer_can_modify` | consumer | GitHub's `maintainerCanModify` | +| `merged_by_id` | consumer | Row id of the merger's account; null on an unmerged pull request, and when GitHub returned no merger and the row had none stored | +| `merged_by_reported` | consumer | 1 when GitHub returned a merger in the row's latest hydration (D28) | +| `base_ref` | consumer | GitHub's `baseRefName` | +| `base_ref_oid` | consumer | GitHub's `baseRefOid` | +| `head_ref_name` | consumer | GitHub's `headRefName` | +| `head_ref_oid` | consumer | GitHub's `headRefOid` | +| `head_repo_github_id` | consumer | Numeric id of GitHub's `headRepository`; null when GitHub returns no head repository or no id for it | +| `head_repo_full_name` | consumer | `nameWithOwner` of GitHub's `headRepository`; null when GitHub returns none | +| `head_repo_owner_node_id` | consumer | Node id of GitHub's `headRepositoryOwner`; null when GitHub returns none | +| `head_repo_owner_login` | consumer | Login of GitHub's `headRepositoryOwner`; null when GitHub returns none | +| `merge_commit_oid` | consumer | Oid of GitHub's `mergeCommit`; null when GitHub returns none | +| `targets_default_branch` | consumer | Classification: whether `base_ref` equals the repository's current `default_branch`, recomputed when the default branch changes | +| `is_cross_repository` | consumer | GitHub's `isCrossRepository` | +| `milestone_number` | consumer | Number of GitHub's `milestone` reference; null when the pull request is in none | +| `milestone_title` | consumer | Title of GitHub's `milestone` reference; null when the pull request is in none | +| `review_decision` | consumer | GitHub's `reviewDecision` when the row was hydrated; null when GitHub returns none | +| `mergeable` | consumer | GitHub's `mergeable` when the row was hydrated; GitHub recomputes it, so it is as of `hydrated_at` | +| `merge_state_status` | consumer | GitHub's `mergeStateStatus` when the row was hydrated, as of `hydrated_at` | +| `can_be_rebased` | consumer | GitHub's `canBeRebased` when the row was hydrated, as of `hydrated_at` | +| `potential_merge_commit_oid` | consumer | Oid of GitHub's `potentialMergeCommit` when the row was hydrated; GitHub returns none "if the pull request is merged, or if the test merge commit is still being generated" | +| `stack_node_id` | consumer | Node id of GitHub's `stack`; null when the pull request is in none | +| `stack_number` | consumer | GitHub's stack `number`, "A number uniquely identifying the stack within its repository"; null when the pull request is in none | +| `stack_size` | consumer | GitHub's stack `size`; null when the pull request is in none | +| `stack_base_ref_name` | consumer | GitHub's stack `baseRefName`; null when the pull request is in none | +| `stack_entry_node_id` | consumer | Node id of GitHub's `stackEntry`; null when GitHub returns none | +| `stack_position` | consumer | GitHub's stack entry `position`, "where 1 is the closest to the base branch"; null when GitHub returns no entry | +| `additions` | consumer | GitHub's `additions` | +| `deletions` | consumer | GitHub's `deletions` | +| `changed_files` | consumer | GitHub's `changedFiles`; a path listed twice counts once, so it can differ from `files_total` | +| `total_comments_count` | consumer | GitHub's `totalCommentsCount`; null when GitHub returns none | +| `commits_total` | consumer | GitHub's `totalCount` of the commits (`pr_commits` rows) | +| `files_total` | consumer | GitHub's `totalCount` of the file list (`pr_files` rows) | +| `reviews_total` | consumer | GitHub's `totalCount` of the submitted reviews (`pr_reviews` rows) | +| `comments_total` | consumer | GitHub's `totalCount` of the conversation comments (`pr_comments` rows) | +| `labels_total` | consumer | GitHub's `totalCount` of the labels applied (`pr_labels` rows) | +| `closing_issues_total` | consumer | GitHub's `totalCount` of `closingIssuesReferences` (`pr_closing_issues` rows) | +| `user_linked_closing_issues_total` | consumer | GitHub's `totalCount` of `closingIssuesReferences` with `userLinkedOnly: true` (rows with `user_linked`) | +| `review_requests_total` | consumer | GitHub's `totalCount` of `reviewRequests` (`pr_review_requests` rows) | +| `assignees_total` | consumer | GitHub's `totalCount` of `assignedActors` (`pr_assignees` rows) | +| `github_updated_at` | consumer | GitHub's `updatedAt` as of the hydration that wrote the row; updates from the index alone leave it | +| `hydrated_at` | consumer | GitHub's `Date` header of the response that read the pull request: the as-of time of its hydrated fields | +| `hydration_version` | consumer | Version of the queries that read the pull request; a column added later is null on rows read at an older version until they are read again | + +### `pr_labels` + +Labels as currently applied, without history (D8). + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | consumer | Row id of the pull request | +| `name` | consumer | Label name | +| `label_node_id` | consumer | GitHub's label id | + +### `pr_reviews` + +Submitted reviews; pending reviews are not stored (D4). Review ids do not follow submission +order (D6), so `submitted_at` gives that order. No review text (D24). + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | internal | Row id | +| `github_id` | consumer | GitHub's numeric review id | +| `node_id` | consumer | GitHub's GraphQL node id | +| `pr_id` | consumer | Row id of the pull request | +| `reviewer_id` | consumer | Row id of the reviewer's account; null when GitHub returned no author and the row had none stored | +| `reviewer_reported` | consumer | 1 when GitHub returned an author in the review's latest hydration (D28) | +| `state` | consumer | GitHub's `state`: `APPROVED`, `CHANGES_REQUESTED`, `COMMENTED`, or `DISMISSED` | +| `created_at` | consumer | GitHub's `createdAt` | +| `submitted_at` | consumer | GitHub's `submittedAt`; null when GitHub returns none | +| `published_at` | consumer | GitHub's `publishedAt`; null when GitHub returns none | +| `updated_at` | consumer | GitHub's `updatedAt` | +| `last_edited_at` | consumer | GitHub's `lastEditedAt`; null when GitHub returns none | +| `editor_id` | consumer | Row id of the account GitHub lists as the last editor; null when GitHub returns none | +| `includes_created_edit` | consumer | GitHub's `includesCreatedEdit` | +| `commit_oid` | consumer | Oid of the review's `commit`; null when GitHub returns none | +| `author_association` | consumer | GitHub's `authorAssociation` when the pull request was hydrated | +| `author_can_push_to_repository` | consumer | GitHub's `authorCanPushToRepository` when the pull request was hydrated | +| `inline_comments_total` | consumer | GitHub's `comments.totalCount` for the review: its inline comments (D5) | + +### `pr_comments` + +Conversation comments, one row each, without their text (D5, D24). + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | internal | Row id | +| `github_id` | consumer | GitHub's numeric comment id | +| `node_id` | consumer | GitHub's GraphQL node id | +| `pr_id` | consumer | Row id of the pull request | +| `author_id` | consumer | Row id of the author's account; null when GitHub returned no author and the row had none stored | +| `author_reported` | consumer | 1 when GitHub returned an author in the comment's latest hydration (D28) | +| `created_at` | consumer | GitHub's `createdAt` | +| `published_at` | consumer | GitHub's `publishedAt`; null when GitHub returns none | +| `updated_at` | consumer | GitHub's `updatedAt` | +| `last_edited_at` | consumer | GitHub's `lastEditedAt`; null when GitHub returns none | +| `editor_id` | consumer | Row id of the account GitHub lists as the last editor; null when GitHub returns none | +| `created_via_email` | consumer | GitHub's `createdViaEmail` | +| `includes_created_edit` | consumer | GitHub's `includesCreatedEdit` | +| `author_association` | consumer | GitHub's `authorAssociation` when the pull request was hydrated | + +### `pr_commits` + +The pull request's commits, each like a comment: its author link and git author, never its +message (D19); co-authors are not stored (D15). + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | consumer | Row id of the pull request | +| `sha` | consumer | Commit SHA | +| `author_id` | consumer | Row id of the account GitHub links the commit to; null when GitHub links none and the row had none stored, and when the linked account was never stored because a lookup answered `NOT_FOUND` for it (`linked_github_id` then holds its id) | +| `linked_github_id` | consumer | The account id GitHub linked the commit to in the latest hydration, whether or not a lookup found the account; null when it linked none (D28) | +| `author_name` | consumer | Git author name | +| `author_email` | consumer | Git author email | +| `authored_at` | consumer | Git author date, set by the committer's client (D6) | + +### `pr_files` + +The file list, one row per entry; a path can appear twice. + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | consumer | Row id of the pull request | +| `ordinal` | consumer | Position in GitHub's file list, from 0 | +| `path` | consumer | File path | +| `change_type` | consumer | GitHub's `changeType` (`ADDED`, `MODIFIED`, ...) | +| `additions` | consumer | Lines added to the file | +| `deletions` | consumer | Lines deleted from the file | + +### `pr_closing_issues` + +The pull request's `closingIssuesReferences`, which GitHub describes as "List of issues that may +be closed by this pull request", possibly in repositories the database does not track (D23). + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | consumer | Row id of the pull request | +| `issue_repo_github_id` | consumer | GitHub's numeric id of the issue's repository | +| `issue_repo_full_name` | consumer | `owner/name` of the issue's repository when the link was last written | +| `issue_number` | consumer | Issue number | +| `issue_github_id` | consumer | The issue's `fullDatabaseId` | +| `user_linked` | consumer | 1 when GitHub also lists the link under `userLinkedOnly: true`, which GitHub describes as "Return only manually linked Issues" | + +### `pr_review_requests` + +GitHub's `reviewRequests` for the pull request, one row per entry as GitHub currently lists it +(D4). + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | consumer | Row id of the pull request | +| `ordinal` | consumer | Position in GitHub's list, from 0 | +| `request_github_id` | consumer | The request's own `databaseId`; null when GitHub returns none | +| `request_node_id` | consumer | The request's own node id | +| `requested_type` | consumer | GitHub's type of the requested reviewer (`User`, `Bot`, `Mannequin`, `Team`, `EnterpriseTeam`); null when GitHub returns no requested reviewer | +| `requested_github_id` | consumer | The requested account's or `Team`'s `databaseId`, or an `EnterpriseTeam`'s `fullDatabaseId`; null when GitHub returns none | +| `requested_node_id` | consumer | The requested account's, team's, or enterprise team's node id; null when GitHub returns no requested reviewer | +| `account_id` | consumer | Row id of the requested account; null for a `Team` or an `EnterpriseTeam`, which is not an account, and when GitHub returns no requested reviewer | +| `team_slug` | consumer | The requested team's or enterprise team's `slug`; null for an account and when GitHub returns no requested reviewer | +| `as_code_owner` | consumer | GitHub's `asCodeOwner` | + +### `pr_assignees` + +The actors GitHub lists in the pull request's `assignedActors` (D14): users, bots, mannequins, +and organizations. + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | consumer | Row id of the pull request | +| `account_id` | consumer | Row id of the assigned account | + +### `automation_accounts` + +The `[automation].accounts` list of `tracking.toml`, copied at the start of every sync. + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | internal | Row id | +| `github_id` | consumer | GitHub's numeric account id | +| `login` | consumer | Login as `tracking.toml` gives it | +| `reason` | consumer | The reason `tracking.toml` gives for the entry | +| `run_id` | consumer | Row id of the run that copied the list | + +### `automation_emails` + +The `[automation].emails` list of `tracking.toml`, copied at the start of every sync. + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | internal | Row id | +| `email` | consumer | Commit author email, lowercase ASCII | +| `reason` | consumer | The reason `tracking.toml` gives for the entry | +| `run_id` | consumer | Row id of the run that copied the list | + +### `sync_runs` + +One row per sync run. + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | consumer | Row id | +| `started_at` | consumer | When the run started | +| `deadline_at` | consumer | `started_at` plus `--max-minutes`; null without it | +| `finished_at` | consumer | When the run closed; null while it runs | +| `status` | consumer | `running`, `complete`, `time_limited`, `failed`, or `aborted` | +| `exit_code` | consumer | The run's exit code; null while it runs | +| `trigger` | consumer | `schedule` or `workflow_dispatch` from the Actions event, else `manual` | +| `code_version` | consumer | Commit or package version of the code that ran | +| `config_source` | consumer | `packaged`, or the `TRACKING_CONFIG_PATH` path the run read | +| `config_sha256` | consumer | SHA-256 of the `tracking.toml` bytes the run read | +| `graphql_points` | internal | GraphQL rate-limit points spent | +| `rest_calls` | internal | REST requests sent | +| `items_planned` | internal | Distinct items planned for hydration | +| `items_hydrated` | internal | Hydrated items written | +| `items_deleted` | internal | Item rows deleted | +| `failures` | internal | Item failures in the run, accepted ones included | +| `deletions_approved` | consumer | JSON text: the deletions an operator approved, per guarded set, and the repositories pruned; null when none was approved | + +### `repo_sync_state` + +Per-repository sync state, written as each run closes. + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_id` | consumer | Row id of the repository | +| `last_index_started_at` | consumer | Start of the last complete index the run did not later fail: the as-of time of everything the index wrote | +| `index_pr_total` | consumer | Pull requests that index counted | +| `index_issue_total` | consumer | Issues that index counted; null until issues are indexed | +| `last_success_at` | consumer | When the last fully successful run finished | +| `last_run_id` | consumer | Row id of the latest run that attempted the repository | +| `last_outcome` | consumer | That run's result for the repository | +| `halt_reason` | consumer | Why deletions halted in the latest run that halted them; cleared by a run that did not halt | + +### `sync_pending` + +Items a time-limited run planned but did not reach: an in-coverage item not yet hydrated. + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_id` | consumer | Row id of the repository | +| `item_kind` | consumer | `pull_request` or `issue` | +| `number` | consumer | Item number | +| `run_id` | consumer | Row id of the run that left the item | + +### `sync_failures` + +Items whose latest attempt failed, in a hydration or in the planner's check of an index entry +(redesign plan 5.3 step 3). The next run plans the item again; its row is deleted when the item is +written, or when the item leaves coverage. + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | internal | Row id | +| `repository_id` | consumer | Row id of the repository | +| `item_kind` | consumer | `pull_request` or `issue` | +| `number` | consumer | Item number | +| `attempts` | internal | Runs in which the item failed | +| `last_error` | consumer | The last error's class and message | +| `first_failed_at` | consumer | When the item first failed | +| `last_failed_at` | consumer | When the item last failed | + +### `user_tags` + +Kept from the first schema; nothing writes tags yet (D17). + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | internal | Row id | +| `name` | internal | Tag name | +| `description` | internal | Tag description | +| `color` | internal | Tag color | +| `created_at` | internal | When the tag was created | + +### `pr_user_tags` + +Tags applied to pull requests; empty until a tag producer exists (D17). + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | internal | Row id of the pull request | +| `user_tag_id` | internal | Row id of the tag | +| `created_at` | internal | When the tag was applied | + +### `alembic_version` + +The migration revision the file is at. + +| Column | Use | Meaning | +|--------|-----|---------| +| `version_num` | internal | Alembic revision id | + +## Views + +Every view row comes from one stored row, and every row is kept: a null reference gives null +account columns. A pull request's keys are its repository's `github_id`, `full_name`, and +`is_tracked`, and its `number` and `github_id`. An account carries its `github_id`, `node_id`, +`login`, `type`, and `automation_reason`, the reason of the `automation_accounts` entry for its +`github_id` (null when none lists it). + +### `v_github_users` + +One row per stored account. + +| Column | Use | Meaning | +|--------|-----|---------| +| `github_id` | consumer | GitHub's numeric account id; null for an `EnterpriseUserAccount` | +| `node_id` | consumer | GitHub's GraphQL node id | +| `login` | consumer | Last known login | +| `type` | consumer | GitHub's type | +| `name` | consumer | GitHub's `name`; null until the first lookup, for a `Bot`, and when there is none | +| `created_at` | consumer | GitHub's `createdAt`; null until the first lookup | +| `updated_at` | consumer | GitHub's `updatedAt`; null until the first lookup | +| `linked_user_github_id` | consumer | An `EnterpriseUserAccount`'s user id | +| `claimant_github_id` | consumer | A `Mannequin`'s claimant id | +| `refreshed_at` | consumer | GitHub's `Date` header of the response that inserted the row or of its latest lookup | +| `not_found_at` | consumer | When a lookup answered `NOT_FOUND` for the account; null otherwise | +| `automation_reason` | consumer | Classification: the reason of the `automation_accounts` entry for the account's `github_id`; null when none lists it | + +### `v_account_profiles` + +One row per `account_profiles` row, with its account's keys. + +| Column | Use | Meaning | +|--------|-----|---------| +| `github_id` | consumer | The account's numeric id | +| `node_id` | consumer | The account's node id | +| `login` | consumer | The account's last known login | +| `type` | consumer | `User` or `Organization` | +| `automation_reason` | consumer | Classification: the reason of the `automation_accounts` entry for the account's `github_id`; null when none lists it | +| `company` | consumer | A user's profile `company` | +| `location` | consumer | The profile's `location` | +| `website_url` | consumer | The profile's `websiteUrl` | +| `description` | consumer | An organization's `description` | +| `email` | consumer | An organization's public `email` | +| `twitter_username` | consumer | An organization's `twitterUsername` | +| `is_verified` | consumer | An organization's `isVerified` | +| `archived_at` | consumer | An organization's `archivedAt` | +| `has_sponsors_listing` | consumer | An organization's `hasSponsorsListing` | +| `pinned_items_remaining` | consumer | An organization's `pinnedItemsRemaining` | +| `refreshed_at` | consumer | GitHub's `Date` header of the lookup that wrote the profile | + +### `v_pr_reviews` + +One row per stored review. + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_github_id` | consumer | The repository's numeric id | +| `repository_full_name` | consumer | `owner/name` of the repository | +| `repository_is_tracked` | consumer | The repository's `is_tracked` | +| `pr_number` | consumer | Pull request number | +| `pr_github_id` | consumer | GitHub's numeric pull request id | +| `review_github_id` | consumer | GitHub's numeric review id | +| `review_node_id` | consumer | The review's node id | +| `state` | consumer | GitHub's review `state` | +| `created_at` | consumer | GitHub's `createdAt` | +| `submitted_at` | consumer | GitHub's `submittedAt` | +| `published_at` | consumer | GitHub's `publishedAt` | +| `updated_at` | consumer | GitHub's `updatedAt` | +| `last_edited_at` | consumer | GitHub's `lastEditedAt` | +| `includes_created_edit` | consumer | GitHub's `includesCreatedEdit` | +| `commit_oid` | consumer | Oid of the review's commit | +| `author_association` | consumer | GitHub's `authorAssociation` when hydrated | +| `author_can_push_to_repository` | consumer | GitHub's `authorCanPushToRepository` when hydrated | +| `inline_comments_total` | consumer | GitHub's count of the review's inline comments | +| `reviewer_reported` | consumer | 1 when GitHub returned the reviewer in the latest hydration | +| `reviewer_github_id` | consumer | The reviewer's numeric id | +| `reviewer_node_id` | consumer | The reviewer's node id | +| `reviewer_login` | consumer | The reviewer's last known login | +| `reviewer_type` | consumer | The reviewer's type | +| `reviewer_automation_reason` | consumer | The reviewer's `[automation]` reason | +| `editor_github_id` | consumer | The last editor's numeric id | +| `editor_node_id` | consumer | The last editor's node id | +| `editor_login` | consumer | The last editor's last known login | +| `editor_type` | consumer | The last editor's type | +| `editor_automation_reason` | consumer | The last editor's `[automation]` reason | +| `is_pr_author` | consumer | Classification: 1 when the reviewer is the pull request's author, 0 when not; null when either reference is null | + +### `v_pr_comments` + +One row per stored conversation comment. + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_github_id` | consumer | The repository's numeric id | +| `repository_full_name` | consumer | `owner/name` of the repository | +| `repository_is_tracked` | consumer | The repository's `is_tracked` | +| `pr_number` | consumer | Pull request number | +| `pr_github_id` | consumer | GitHub's numeric pull request id | +| `comment_github_id` | consumer | GitHub's numeric comment id | +| `comment_node_id` | consumer | The comment's node id | +| `created_at` | consumer | GitHub's `createdAt` | +| `published_at` | consumer | GitHub's `publishedAt` | +| `updated_at` | consumer | GitHub's `updatedAt` | +| `last_edited_at` | consumer | GitHub's `lastEditedAt` | +| `created_via_email` | consumer | GitHub's `createdViaEmail` | +| `includes_created_edit` | consumer | GitHub's `includesCreatedEdit` | +| `author_association` | consumer | GitHub's `authorAssociation` when hydrated | +| `author_reported` | consumer | 1 when GitHub returned the author in the latest hydration | +| `author_github_id` | consumer | The author's numeric id | +| `author_node_id` | consumer | The author's node id | +| `author_login` | consumer | The author's last known login | +| `author_type` | consumer | The author's type | +| `author_automation_reason` | consumer | The author's `[automation]` reason | +| `editor_github_id` | consumer | The last editor's numeric id | +| `editor_node_id` | consumer | The last editor's node id | +| `editor_login` | consumer | The last editor's last known login | +| `editor_type` | consumer | The last editor's type | +| `editor_automation_reason` | consumer | The last editor's `[automation]` reason | +| `is_pr_author` | consumer | Classification: 1 when the comment's author is the pull request's author, 0 when not; null when either reference is null | +| `author_also_reviewed` | consumer | Classification: 1 when the author has a stored review on the same pull request, 0 when not; null when the author is null | + +### `v_commit_authors` + +One row per stored commit. + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_github_id` | consumer | The repository's numeric id | +| `repository_full_name` | consumer | `owner/name` of the repository | +| `repository_is_tracked` | consumer | The repository's `is_tracked` | +| `pr_number` | consumer | Pull request number | +| `pr_github_id` | consumer | GitHub's numeric pull request id | +| `sha` | consumer | Commit SHA | +| `linked_github_id` | consumer | The account id GitHub linked the commit to in the latest hydration | +| `author_name` | consumer | Git author name | +| `author_email` | consumer | Git author email | +| `authored_at` | consumer | Git author date (D6) | +| `account_github_id` | consumer | The linked stored account's numeric id | +| `account_node_id` | consumer | The linked stored account's node id | +| `account_login` | consumer | The linked stored account's last known login | +| `account_type` | consumer | The linked stored account's type | +| `account_automation_reason` | consumer | The linked stored account's `[automation]` reason | +| `email_automation_reason` | consumer | Classification: the reason of the `automation_emails` entry equal to `author_email` with ASCII letters lowercased; null when none | + +### `v_pr_relationships` + +One row per relationship of a pull request to an account, a team, or an actor GitHub did not +return, from the stored rows, with the pull request's `additions`, `deletions`, and +`changed_files`. `relationship` names the source: +`author` (one per pull request), `merged_by` (one per merged pull request), `review` (one per +review), `comment` (one per comment), `review_request` (one per request), `assignee` (one per +assignee), `commit_author` (one per commit), and `pr_editor`, `review_editor`, and +`comment_editor` (one per pull request, review, or comment that GitHub lists an editor or a +`last_edited_at` for). Columns that belong to another source are null. + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_github_id` | consumer | The repository's numeric id | +| `repository_full_name` | consumer | `owner/name` of the repository | +| `repository_is_tracked` | consumer | The repository's `is_tracked` | +| `pr_number` | consumer | Pull request number | +| `pr_github_id` | consumer | GitHub's numeric pull request id | +| `relationship` | consumer | The stored row the view row comes from | +| `review_github_id` | consumer | The review's id, on `review` and `review_editor` rows | +| `comment_github_id` | consumer | The comment's id, on `comment` and `comment_editor` rows | +| `request_node_id` | consumer | The request's node id, on `review_request` rows | +| `request_github_id` | consumer | The request's `databaseId`, on `review_request` rows | +| `request_ordinal` | consumer | The request's position in GitHub's list, from 0, on `review_request` rows | +| `commit_sha` | consumer | The commit's SHA, on `commit_author` rows | +| `reported` | consumer | The source row's `*_reported` value, on `author`, `merged_by`, `review`, and `comment` rows | +| `account_github_id` | consumer | The account's numeric id | +| `account_node_id` | consumer | The account's node id | +| `account_login` | consumer | The account's last known login | +| `account_type` | consumer | The account's type | +| `account_automation_reason` | consumer | The account's `[automation]` reason | +| `account_not_found_at` | consumer | The account's `not_found_at` | +| `review_state` | consumer | The review's `state`, on `review` rows | +| `review_submitted_at` | consumer | The review's `submitted_at`, on `review` rows | +| `requested_type` | consumer | The requested reviewer's type, on `review_request` rows | +| `requested_github_id` | consumer | The requested account's or team's id, on `review_request` rows | +| `team_slug` | consumer | The requested team's slug, on `review_request` rows | +| `as_code_owner` | consumer | GitHub's `asCodeOwner`, on `review_request` rows | +| `linked_github_id` | consumer | The commit's `linked_github_id`, on `commit_author` rows | +| `author_name` | consumer | The commit's git author name, on `commit_author` rows | +| `author_email` | consumer | The commit's git author email, on `commit_author` rows | +| `email_automation_reason` | consumer | The commit email's `[automation]` reason, on `commit_author` rows | +| `pr_state` | consumer | The pull request's `state` | +| `pr_created_at` | consumer | The pull request's `created_at` | +| `pr_merged_at` | consumer | The pull request's `merged_at` | +| `pr_closed_at` | consumer | The pull request's `closed_at` | +| `base_ref` | consumer | The pull request's base branch name | +| `targets_default_branch` | consumer | The pull request's `targets_default_branch` | +| `merge_commit_oid` | consumer | The pull request's merge commit oid | +| `additions` | consumer | The pull request's `additions` | +| `deletions` | consumer | The pull request's `deletions` | +| `changed_files` | consumer | The pull request's `changed_files` | diff --git a/tests/fixtures/history/21a4267/tests/query_versions.json b/tests/fixtures/history/21a4267/tests/query_versions.json new file mode 100644 index 0000000..bd342e4 --- /dev/null +++ b/tests/fixtures/history/21a4267/tests/query_versions.json @@ -0,0 +1,4 @@ +{ + "pull_request:1": "6aff7254be69c54b70e4e633c4b9d89640e5d9b99492b70ba94f7808a5e9cc76", + "pull_request:2": "459c43a60ae47e98f73fc2eed23aa4a59f7e28b7fdebd9f29ed39f762513eb18" +} diff --git a/tests/fixtures/history/21a4267/tests/schema_versions.json b/tests/fixtures/history/21a4267/tests/schema_versions.json new file mode 100644 index 0000000..067e5a8 --- /dev/null +++ b/tests/fixtures/history/21a4267/tests/schema_versions.json @@ -0,0 +1,496 @@ +{ + "1": [ + ["automation_accounts", "github_id"], + ["automation_accounts", "login"], + ["automation_accounts", "reason"], + ["automation_emails", "email"], + ["automation_emails", "reason"], + ["github_users", "deleted_at"], + ["github_users", "github_id"], + ["github_users", "id"], + ["github_users", "login"], + ["github_users", "node_id"], + ["github_users", "type"], + ["pr_closing_issues", "issue_number"], + ["pr_closing_issues", "issue_repo_full_name"], + ["pr_closing_issues", "issue_repo_github_id"], + ["pr_closing_issues", "pr_id"], + ["pr_comments", "author_id"], + ["pr_comments", "github_id"], + ["pr_comments", "pr_id"], + ["pr_commits", "author_email"], + ["pr_commits", "author_id"], + ["pr_commits", "author_name"], + ["pr_commits", "authored_at"], + ["pr_commits", "pr_id"], + ["pr_commits", "sha"], + ["pr_files", "additions"], + ["pr_files", "change_type"], + ["pr_files", "deletions"], + ["pr_files", "ordinal"], + ["pr_files", "path"], + ["pr_files", "pr_id"], + ["pr_labels", "name"], + ["pr_labels", "pr_id"], + ["pr_reviews", "github_id"], + ["pr_reviews", "pr_id"], + ["pr_reviews", "reviewer_id"], + ["pr_reviews", "state"], + ["pull_requests", "additions"], + ["pull_requests", "author_association"], + ["pull_requests", "author_id"], + ["pull_requests", "base_ref"], + ["pull_requests", "body"], + ["pull_requests", "changed_files"], + ["pull_requests", "closing_issues_total"], + ["pull_requests", "comments_total"], + ["pull_requests", "commits_total"], + ["pull_requests", "created_at"], + ["pull_requests", "deletions"], + ["pull_requests", "files_total"], + ["pull_requests", "github_id"], + ["pull_requests", "github_updated_at"], + ["pull_requests", "id"], + ["pull_requests", "is_cross_repository"], + ["pull_requests", "is_draft"], + ["pull_requests", "labels_total"], + ["pull_requests", "merged_at"], + ["pull_requests", "merged_by_id"], + ["pull_requests", "node_id"], + ["pull_requests", "number"], + ["pull_requests", "repository_id"], + ["pull_requests", "reviews_total"], + ["pull_requests", "state"], + ["pull_requests", "targets_default_branch"], + ["pull_requests", "title"], + ["repositories", "default_branch"], + ["repositories", "full_name"], + ["repositories", "github_id"], + ["repositories", "id"], + ["repositories", "is_archived"], + ["repositories", "is_private"], + ["repositories", "is_tracked"], + ["repositories", "name"], + ["repositories", "owner"], + ["repositories", "repo_group"], + ["v_commit_authors", "author_email"], + ["v_commit_authors", "author_github_id"], + ["v_commit_authors", "author_name"], + ["v_commit_authors", "authored_at"], + ["v_commit_authors", "is_automation"], + ["v_commit_authors", "pr_github_id"], + ["v_commit_authors", "pr_number"], + ["v_commit_authors", "repository_full_name"], + ["v_commit_authors", "sha"], + ["v_github_users", "deleted_at"], + ["v_github_users", "github_id"], + ["v_github_users", "is_automation"], + ["v_github_users", "login"], + ["v_github_users", "node_id"], + ["v_github_users", "type"], + ["v_pr_commenters", "comment_count"], + ["v_pr_commenters", "commenter_github_id"], + ["v_pr_commenters", "commenter_login"], + ["v_pr_commenters", "pr_github_id"], + ["v_pr_commenters", "pr_number"], + ["v_pr_commenters", "repository_full_name"], + ["v_pr_reviewers", "comment_count"], + ["v_pr_reviewers", "final_verdict"], + ["v_pr_reviewers", "pr_github_id"], + ["v_pr_reviewers", "pr_number"], + ["v_pr_reviewers", "repository_full_name"], + ["v_pr_reviewers", "reviewer_github_id"], + ["v_pr_reviewers", "reviewer_login"] + ], + "2": { + "columns": [ + ["account_profiles", "account_id"], + ["account_profiles", "archived_at"], + ["account_profiles", "company"], + ["account_profiles", "description"], + ["account_profiles", "email"], + ["account_profiles", "has_sponsors_listing"], + ["account_profiles", "is_verified"], + ["account_profiles", "location"], + ["account_profiles", "pinned_items_remaining"], + ["account_profiles", "refreshed_at"], + ["account_profiles", "twitter_username"], + ["account_profiles", "website_url"], + ["automation_accounts", "github_id"], + ["automation_accounts", "login"], + ["automation_accounts", "reason"], + ["automation_accounts", "run_id"], + ["automation_emails", "email"], + ["automation_emails", "reason"], + ["automation_emails", "run_id"], + ["github_users", "claimant_github_id"], + ["github_users", "created_at"], + ["github_users", "github_id"], + ["github_users", "id"], + ["github_users", "linked_user_github_id"], + ["github_users", "login"], + ["github_users", "name"], + ["github_users", "node_id"], + ["github_users", "not_found_at"], + ["github_users", "refreshed_at"], + ["github_users", "type"], + ["github_users", "updated_at"], + ["pr_assignees", "account_id"], + ["pr_assignees", "pr_id"], + ["pr_closing_issues", "issue_github_id"], + ["pr_closing_issues", "issue_number"], + ["pr_closing_issues", "issue_repo_full_name"], + ["pr_closing_issues", "issue_repo_github_id"], + ["pr_closing_issues", "pr_id"], + ["pr_closing_issues", "user_linked"], + ["pr_comments", "author_association"], + ["pr_comments", "author_id"], + ["pr_comments", "author_reported"], + ["pr_comments", "created_at"], + ["pr_comments", "created_via_email"], + ["pr_comments", "editor_id"], + ["pr_comments", "github_id"], + ["pr_comments", "includes_created_edit"], + ["pr_comments", "last_edited_at"], + ["pr_comments", "node_id"], + ["pr_comments", "pr_id"], + ["pr_comments", "published_at"], + ["pr_comments", "updated_at"], + ["pr_commits", "author_email"], + ["pr_commits", "author_id"], + ["pr_commits", "author_name"], + ["pr_commits", "authored_at"], + ["pr_commits", "linked_github_id"], + ["pr_commits", "pr_id"], + ["pr_commits", "sha"], + ["pr_files", "additions"], + ["pr_files", "change_type"], + ["pr_files", "deletions"], + ["pr_files", "ordinal"], + ["pr_files", "path"], + ["pr_files", "pr_id"], + ["pr_labels", "label_node_id"], + ["pr_labels", "name"], + ["pr_labels", "pr_id"], + ["pr_review_requests", "account_id"], + ["pr_review_requests", "as_code_owner"], + ["pr_review_requests", "ordinal"], + ["pr_review_requests", "pr_id"], + ["pr_review_requests", "request_github_id"], + ["pr_review_requests", "request_node_id"], + ["pr_review_requests", "requested_github_id"], + ["pr_review_requests", "requested_node_id"], + ["pr_review_requests", "requested_type"], + ["pr_review_requests", "team_slug"], + ["pr_reviews", "author_association"], + ["pr_reviews", "author_can_push_to_repository"], + ["pr_reviews", "commit_oid"], + ["pr_reviews", "created_at"], + ["pr_reviews", "editor_id"], + ["pr_reviews", "github_id"], + ["pr_reviews", "includes_created_edit"], + ["pr_reviews", "inline_comments_total"], + ["pr_reviews", "last_edited_at"], + ["pr_reviews", "node_id"], + ["pr_reviews", "pr_id"], + ["pr_reviews", "published_at"], + ["pr_reviews", "reviewer_id"], + ["pr_reviews", "reviewer_reported"], + ["pr_reviews", "state"], + ["pr_reviews", "submitted_at"], + ["pr_reviews", "updated_at"], + ["pull_requests", "additions"], + ["pull_requests", "assignees_total"], + ["pull_requests", "author_association"], + ["pull_requests", "author_id"], + ["pull_requests", "author_reported"], + ["pull_requests", "base_ref"], + ["pull_requests", "base_ref_oid"], + ["pull_requests", "body"], + ["pull_requests", "can_be_rebased"], + ["pull_requests", "changed_files"], + ["pull_requests", "closed_at"], + ["pull_requests", "closing_issues_total"], + ["pull_requests", "comments_total"], + ["pull_requests", "commits_total"], + ["pull_requests", "created_at"], + ["pull_requests", "deletions"], + ["pull_requests", "editor_id"], + ["pull_requests", "files_total"], + ["pull_requests", "github_id"], + ["pull_requests", "github_updated_at"], + ["pull_requests", "head_ref_name"], + ["pull_requests", "head_ref_oid"], + ["pull_requests", "head_repo_full_name"], + ["pull_requests", "head_repo_github_id"], + ["pull_requests", "head_repo_owner_login"], + ["pull_requests", "head_repo_owner_node_id"], + ["pull_requests", "hydrated_at"], + ["pull_requests", "hydration_version"], + ["pull_requests", "id"], + ["pull_requests", "includes_created_edit"], + ["pull_requests", "is_cross_repository"], + ["pull_requests", "is_draft"], + ["pull_requests", "labels_total"], + ["pull_requests", "last_edited_at"], + ["pull_requests", "maintainer_can_modify"], + ["pull_requests", "merge_commit_oid"], + ["pull_requests", "merge_state_status"], + ["pull_requests", "mergeable"], + ["pull_requests", "merged_at"], + ["pull_requests", "merged_by_id"], + ["pull_requests", "merged_by_reported"], + ["pull_requests", "milestone_number"], + ["pull_requests", "milestone_title"], + ["pull_requests", "node_id"], + ["pull_requests", "number"], + ["pull_requests", "potential_merge_commit_oid"], + ["pull_requests", "published_at"], + ["pull_requests", "repository_id"], + ["pull_requests", "review_decision"], + ["pull_requests", "review_requests_total"], + ["pull_requests", "reviews_total"], + ["pull_requests", "stack_base_ref_name"], + ["pull_requests", "stack_entry_node_id"], + ["pull_requests", "stack_node_id"], + ["pull_requests", "stack_number"], + ["pull_requests", "stack_position"], + ["pull_requests", "stack_size"], + ["pull_requests", "state"], + ["pull_requests", "targets_default_branch"], + ["pull_requests", "title"], + ["pull_requests", "total_comments_count"], + ["pull_requests", "user_linked_closing_issues_total"], + ["repo_labels", "color"], + ["repo_labels", "created_at"], + ["repo_labels", "description"], + ["repo_labels", "is_default"], + ["repo_labels", "name"], + ["repo_labels", "node_id"], + ["repo_labels", "repository_id"], + ["repo_labels", "updated_at"], + ["repo_sync_state", "halt_reason"], + ["repo_sync_state", "index_issue_total"], + ["repo_sync_state", "index_pr_total"], + ["repo_sync_state", "last_index_started_at"], + ["repo_sync_state", "last_outcome"], + ["repo_sync_state", "last_run_id"], + ["repo_sync_state", "last_success_at"], + ["repo_sync_state", "repository_id"], + ["repositories", "allow_update_branch"], + ["repositories", "archived_at"], + ["repositories", "created_at"], + ["repositories", "default_branch"], + ["repositories", "delete_branch_on_merge"], + ["repositories", "description"], + ["repositories", "disk_usage"], + ["repositories", "fork_count"], + ["repositories", "forking_allowed"], + ["repositories", "full_name"], + ["repositories", "github_id"], + ["repositories", "has_discussions_enabled"], + ["repositories", "has_projects_enabled"], + ["repositories", "has_vulnerability_alerts_enabled"], + ["repositories", "has_wiki_enabled"], + ["repositories", "homepage_url"], + ["repositories", "id"], + ["repositories", "is_archived"], + ["repositories", "is_private"], + ["repositories", "is_security_policy_enabled"], + ["repositories", "is_tracked"], + ["repositories", "labels_total"], + ["repositories", "license_spdx_id"], + ["repositories", "merge_commit_allowed"], + ["repositories", "merge_commit_message"], + ["repositories", "merge_commit_title"], + ["repositories", "name"], + ["repositories", "node_id"], + ["repositories", "owner"], + ["repositories", "plan_features_codeowners"], + ["repositories", "plan_features_maximum_manual_review_requests"], + ["repositories", "plan_features_team_review_requests"], + ["repositories", "primary_language"], + ["repositories", "pushed_at"], + ["repositories", "rebase_merge_allowed"], + ["repositories", "repo_group"], + ["repositories", "resolved_at"], + ["repositories", "squash_merge_allowed"], + ["repositories", "squash_merge_commit_message"], + ["repositories", "squash_merge_commit_title"], + ["repositories", "stargazer_count"], + ["repositories", "updated_at"], + ["repositories", "uses_custom_open_graph_image"], + ["repositories", "visibility"], + ["repositories", "web_commit_signoff_required"], + ["sync_failures", "first_failed_at"], + ["sync_failures", "item_kind"], + ["sync_failures", "last_error"], + ["sync_failures", "last_failed_at"], + ["sync_failures", "number"], + ["sync_failures", "repository_id"], + ["sync_pending", "item_kind"], + ["sync_pending", "number"], + ["sync_pending", "repository_id"], + ["sync_pending", "run_id"], + ["sync_runs", "code_version"], + ["sync_runs", "config_sha256"], + ["sync_runs", "config_source"], + ["sync_runs", "deadline_at"], + ["sync_runs", "deletions_approved"], + ["sync_runs", "exit_code"], + ["sync_runs", "finished_at"], + ["sync_runs", "id"], + ["sync_runs", "started_at"], + ["sync_runs", "status"], + ["sync_runs", "trigger"], + ["v_account_profiles", "archived_at"], + ["v_account_profiles", "automation_reason"], + ["v_account_profiles", "company"], + ["v_account_profiles", "description"], + ["v_account_profiles", "email"], + ["v_account_profiles", "github_id"], + ["v_account_profiles", "has_sponsors_listing"], + ["v_account_profiles", "is_verified"], + ["v_account_profiles", "location"], + ["v_account_profiles", "login"], + ["v_account_profiles", "node_id"], + ["v_account_profiles", "pinned_items_remaining"], + ["v_account_profiles", "refreshed_at"], + ["v_account_profiles", "twitter_username"], + ["v_account_profiles", "type"], + ["v_account_profiles", "website_url"], + ["v_commit_authors", "account_automation_reason"], + ["v_commit_authors", "account_github_id"], + ["v_commit_authors", "account_login"], + ["v_commit_authors", "account_node_id"], + ["v_commit_authors", "account_type"], + ["v_commit_authors", "author_email"], + ["v_commit_authors", "author_name"], + ["v_commit_authors", "authored_at"], + ["v_commit_authors", "email_automation_reason"], + ["v_commit_authors", "linked_github_id"], + ["v_commit_authors", "pr_github_id"], + ["v_commit_authors", "pr_number"], + ["v_commit_authors", "repository_full_name"], + ["v_commit_authors", "repository_github_id"], + ["v_commit_authors", "repository_is_tracked"], + ["v_commit_authors", "sha"], + ["v_github_users", "automation_reason"], + ["v_github_users", "claimant_github_id"], + ["v_github_users", "created_at"], + ["v_github_users", "github_id"], + ["v_github_users", "linked_user_github_id"], + ["v_github_users", "login"], + ["v_github_users", "name"], + ["v_github_users", "node_id"], + ["v_github_users", "not_found_at"], + ["v_github_users", "refreshed_at"], + ["v_github_users", "type"], + ["v_github_users", "updated_at"], + ["v_pr_comments", "author_also_reviewed"], + ["v_pr_comments", "author_association"], + ["v_pr_comments", "author_automation_reason"], + ["v_pr_comments", "author_github_id"], + ["v_pr_comments", "author_login"], + ["v_pr_comments", "author_node_id"], + ["v_pr_comments", "author_reported"], + ["v_pr_comments", "author_type"], + ["v_pr_comments", "comment_github_id"], + ["v_pr_comments", "comment_node_id"], + ["v_pr_comments", "created_at"], + ["v_pr_comments", "created_via_email"], + ["v_pr_comments", "editor_automation_reason"], + ["v_pr_comments", "editor_github_id"], + ["v_pr_comments", "editor_login"], + ["v_pr_comments", "editor_node_id"], + ["v_pr_comments", "editor_type"], + ["v_pr_comments", "includes_created_edit"], + ["v_pr_comments", "is_pr_author"], + ["v_pr_comments", "last_edited_at"], + ["v_pr_comments", "pr_github_id"], + ["v_pr_comments", "pr_number"], + ["v_pr_comments", "published_at"], + ["v_pr_comments", "repository_full_name"], + ["v_pr_comments", "repository_github_id"], + ["v_pr_comments", "repository_is_tracked"], + ["v_pr_comments", "updated_at"], + ["v_pr_relationships", "account_automation_reason"], + ["v_pr_relationships", "account_github_id"], + ["v_pr_relationships", "account_login"], + ["v_pr_relationships", "account_node_id"], + ["v_pr_relationships", "account_not_found_at"], + ["v_pr_relationships", "account_type"], + ["v_pr_relationships", "additions"], + ["v_pr_relationships", "as_code_owner"], + ["v_pr_relationships", "author_email"], + ["v_pr_relationships", "author_name"], + ["v_pr_relationships", "base_ref"], + ["v_pr_relationships", "changed_files"], + ["v_pr_relationships", "comment_github_id"], + ["v_pr_relationships", "commit_sha"], + ["v_pr_relationships", "deletions"], + ["v_pr_relationships", "email_automation_reason"], + ["v_pr_relationships", "linked_github_id"], + ["v_pr_relationships", "merge_commit_oid"], + ["v_pr_relationships", "pr_closed_at"], + ["v_pr_relationships", "pr_created_at"], + ["v_pr_relationships", "pr_github_id"], + ["v_pr_relationships", "pr_merged_at"], + ["v_pr_relationships", "pr_number"], + ["v_pr_relationships", "pr_state"], + ["v_pr_relationships", "relationship"], + ["v_pr_relationships", "reported"], + ["v_pr_relationships", "repository_full_name"], + ["v_pr_relationships", "repository_github_id"], + ["v_pr_relationships", "repository_is_tracked"], + ["v_pr_relationships", "request_github_id"], + ["v_pr_relationships", "request_node_id"], + ["v_pr_relationships", "request_ordinal"], + ["v_pr_relationships", "requested_github_id"], + ["v_pr_relationships", "requested_type"], + ["v_pr_relationships", "review_github_id"], + ["v_pr_relationships", "review_state"], + ["v_pr_relationships", "review_submitted_at"], + ["v_pr_relationships", "targets_default_branch"], + ["v_pr_relationships", "team_slug"], + ["v_pr_reviews", "author_association"], + ["v_pr_reviews", "author_can_push_to_repository"], + ["v_pr_reviews", "commit_oid"], + ["v_pr_reviews", "created_at"], + ["v_pr_reviews", "editor_automation_reason"], + ["v_pr_reviews", "editor_github_id"], + ["v_pr_reviews", "editor_login"], + ["v_pr_reviews", "editor_node_id"], + ["v_pr_reviews", "editor_type"], + ["v_pr_reviews", "includes_created_edit"], + ["v_pr_reviews", "inline_comments_total"], + ["v_pr_reviews", "is_pr_author"], + ["v_pr_reviews", "last_edited_at"], + ["v_pr_reviews", "pr_github_id"], + ["v_pr_reviews", "pr_number"], + ["v_pr_reviews", "published_at"], + ["v_pr_reviews", "repository_full_name"], + ["v_pr_reviews", "repository_github_id"], + ["v_pr_reviews", "repository_is_tracked"], + ["v_pr_reviews", "review_github_id"], + ["v_pr_reviews", "review_node_id"], + ["v_pr_reviews", "reviewer_automation_reason"], + ["v_pr_reviews", "reviewer_github_id"], + ["v_pr_reviews", "reviewer_login"], + ["v_pr_reviews", "reviewer_node_id"], + ["v_pr_reviews", "reviewer_reported"], + ["v_pr_reviews", "reviewer_type"], + ["v_pr_reviews", "state"], + ["v_pr_reviews", "submitted_at"], + ["v_pr_reviews", "updated_at"] + ], + "views": { + "v_account_profiles": "01c76ece53f4d3009b51f31b33077582d9db93c72f314cf9e95995f7faf2aab0", + "v_commit_authors": "6fdf1eb9572f0e2a24222669a1d56baf317f7c4a6f5ffceab5f0e995030038af", + "v_github_users": "31636d6122550ca013950fbcd35cf91d88ff02f59ee758d712a98f30eeecea1a", + "v_pr_comments": "89505c43b3d3ebc728844401c0c36724359a7ab057ddceb2ac420b2da95a704d", + "v_pr_relationships": "6aeb3202811037377c3fa734af207e0c51c5512307f3a13d46cc4be234041b08", + "v_pr_reviews": "73be434d0eb3ffa46b47dac729c1091e168736df2f54344256e528c678145c0a" + } + } +} diff --git a/tests/fixtures/history/6fedb05/tests/schema_versions.json b/tests/fixtures/history/6fedb05/tests/schema_versions.json new file mode 100644 index 0000000..30f862a --- /dev/null +++ b/tests/fixtures/history/6fedb05/tests/schema_versions.json @@ -0,0 +1,105 @@ +{ + "1": [ + ["automation_accounts", "github_id"], + ["automation_accounts", "login"], + ["automation_accounts", "reason"], + ["automation_emails", "email"], + ["automation_emails", "reason"], + ["github_users", "deleted_at"], + ["github_users", "github_id"], + ["github_users", "id"], + ["github_users", "login"], + ["github_users", "node_id"], + ["github_users", "type"], + ["pr_closing_issues", "issue_number"], + ["pr_closing_issues", "issue_repo_full_name"], + ["pr_closing_issues", "issue_repo_github_id"], + ["pr_closing_issues", "pr_id"], + ["pr_comments", "author_id"], + ["pr_comments", "github_id"], + ["pr_comments", "pr_id"], + ["pr_commits", "author_email"], + ["pr_commits", "author_id"], + ["pr_commits", "author_name"], + ["pr_commits", "authored_at"], + ["pr_commits", "pr_id"], + ["pr_commits", "sha"], + ["pr_files", "additions"], + ["pr_files", "change_type"], + ["pr_files", "deletions"], + ["pr_files", "ordinal"], + ["pr_files", "path"], + ["pr_files", "pr_id"], + ["pr_labels", "name"], + ["pr_labels", "pr_id"], + ["pr_reviews", "github_id"], + ["pr_reviews", "pr_id"], + ["pr_reviews", "reviewer_id"], + ["pr_reviews", "state"], + ["pull_requests", "additions"], + ["pull_requests", "author_association"], + ["pull_requests", "author_id"], + ["pull_requests", "base_ref"], + ["pull_requests", "body"], + ["pull_requests", "changed_files"], + ["pull_requests", "closing_issues_total"], + ["pull_requests", "comments_total"], + ["pull_requests", "commits_total"], + ["pull_requests", "created_at"], + ["pull_requests", "deletions"], + ["pull_requests", "files_total"], + ["pull_requests", "github_id"], + ["pull_requests", "github_updated_at"], + ["pull_requests", "id"], + ["pull_requests", "is_cross_repository"], + ["pull_requests", "is_draft"], + ["pull_requests", "labels_total"], + ["pull_requests", "merged_at"], + ["pull_requests", "merged_by_id"], + ["pull_requests", "node_id"], + ["pull_requests", "number"], + ["pull_requests", "repository_id"], + ["pull_requests", "reviews_total"], + ["pull_requests", "state"], + ["pull_requests", "targets_default_branch"], + ["pull_requests", "title"], + ["repositories", "default_branch"], + ["repositories", "full_name"], + ["repositories", "github_id"], + ["repositories", "id"], + ["repositories", "is_archived"], + ["repositories", "is_private"], + ["repositories", "is_tracked"], + ["repositories", "name"], + ["repositories", "owner"], + ["repositories", "repo_group"], + ["v_commit_authors", "author_email"], + ["v_commit_authors", "author_github_id"], + ["v_commit_authors", "author_name"], + ["v_commit_authors", "authored_at"], + ["v_commit_authors", "is_automation"], + ["v_commit_authors", "pr_github_id"], + ["v_commit_authors", "pr_number"], + ["v_commit_authors", "repository_full_name"], + ["v_commit_authors", "sha"], + ["v_github_users", "deleted_at"], + ["v_github_users", "github_id"], + ["v_github_users", "is_automation"], + ["v_github_users", "login"], + ["v_github_users", "node_id"], + ["v_github_users", "type"], + ["v_pr_commenters", "comment_count"], + ["v_pr_commenters", "commenter_github_id"], + ["v_pr_commenters", "commenter_login"], + ["v_pr_commenters", "pr_github_id"], + ["v_pr_commenters", "pr_number"], + ["v_pr_commenters", "repository_full_name"], + ["v_pr_reviewers", "comment_count"], + ["v_pr_reviewers", "final_verdict"], + ["v_pr_reviewers", "pr_github_id"], + ["v_pr_reviewers", "pr_number"], + ["v_pr_reviewers", "repository_full_name"], + ["v_pr_reviewers", "reviewer_github_id"], + ["v_pr_reviewers", "reviewer_login"] + ] +} diff --git a/tests/fixtures/history/77663a7/docs/data-dictionary.md b/tests/fixtures/history/77663a7/docs/data-dictionary.md new file mode 100644 index 0000000..f639e61 --- /dev/null +++ b/tests/fixtures/history/77663a7/docs/data-dictionary.md @@ -0,0 +1,679 @@ +# Data Dictionary + +Every table, view, and column of the database at the migration head, each marked **consumer** +or **internal**. `tests/test_data_dictionary.py` fails when an object or column at head has no +entry here, or an entry names one that does not exist. + +## Consumer interface + +- The interface is the `v_*` views, every column of which is consumer, and the table columns + marked consumer below. Every column holding what GitHub reports is consumer. +- Internal columns can change in any migration. +- Row ids join rows within one file: a pull request's labels, reviews, and commits by `pr_id`, + its author by `author_id`. The `*_id` columns of consumer tables, and the `id` columns they + reference, are consumer; other row ids are internal. Row ids change on a rebuild, so never + store one or compare it across files. Across files, identify a row by its `github_id` + (`repositories.github_id`, `pull_requests.github_id`, ...), an account with a null + `github_id` by its `node_id`, and a label, commit, file, closing link, review request, or + assignee by its pull request's `github_id` and its own key; (`repositories.full_name`, + `pull_requests.number`) identifies a pull request among tracked repositories within one file, + and changes when the repository is renamed. +- `PRAGMA user_version` holds the schema version (`SCHEMA_VERSION` in + `src/github_activity_db/db/schema.py`). It is bumped when a consumer column is removed, + renamed, or changes meaning, or a view's frozen SQL changes (its rows, its derivation, or its + layout alone); an added table column, table, or view does not bump it. + `tests/schema_versions.json` lists each version's consumer columns and, from version 2, the + SHA-256 of each view's SQL exactly as its migration embeds it (SQLite stores that text without + its leading and trailing line breaks); on pull requests CI fails when an + entry, a column of one, or a view digest of one that exists on the base branch is removed or + changed. +- Timestamps are UTC, stored as `YYYY-MM-DD HH:MM:SS.ffffff` text, so text order is time order. + Booleans are stored as 0 and 1. +- Values GitHub reports keep GitHub's names in snake case and its values. Every GitHub enum + except a pull request's and a review's `state` (and `github_users.type`) is GitHub's text, + stored without a check, such as `review_decision` or `mergeable`, so a value GitHub adds is + stored. No column of GitHub's data holds a value the database chose, except the + classifications named below (`targets_default_branch`, `is_pr_author`, `author_also_reviewed`, + `automation_reason`, `email_automation_reason`), each a rule over stored columns of the same + file (D29). An `ordinal` column is the entry's position in GitHub's list, numbered from 0. +- The references D28 keeps (a pull request's author and merger, a review's reviewer, a + comment's author, a commit's account) are null when GitHub returned no actor and the row had + none stored; GitHub does not say why. A reference the database kept when GitHub later + returned no actor has its `*_reported` column 0; on a commit, `linked_github_id` holds the + account id GitHub linked in the latest hydration (D28). Editors and requested accounts take + GitHub's current value, null included, and `pr_assignees` holds GitHub's current + `assignedActors` list. Each column's entry gives its further null cases. An account a lookup answered `NOT_FOUND` for keeps its + row, its `not_found_at` set. +- An account's `type` and the reason of the `[automation]` entry listing it are separate + columns; nothing merges them into one flag, and the views keep every row (D29). +- The run records (`sync_runs`, `repo_sync_state`, `sync_pending`, `sync_failures`), the + automation lists, `is_tracked`, `repo_group`, and `pull_requests.hydration_version` describe + the database's own configuration and runs, not GitHub's data. A view's `relationship` column names the stored row a view row comes + from. + +The decisions cited (D3, D17, ...) are in [docs/redesign-plan.md](redesign-plan.md). + +## Tables + +### `repositories` + +One row per repository the sync has resolved, with every field the resolve reads (redesign plan +5.8), as the latest complete resolve read them. + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | consumer | Row id | +| `github_id` | consumer | GitHub's numeric repository id | +| `node_id` | consumer | GitHub's GraphQL node id | +| `owner` | consumer | Owner login, in GitHub's spelling | +| `name` | consumer | Repository name, in GitHub's spelling | +| `full_name` | consumer | `owner/name`; changes when the repository is renamed. With a pull request's `number` it identifies the pull request among tracked repositories | +| `repo_group` | consumer | The group `tracking.toml` assigned at the last resolve, a name the maintainer gives (`Client`, `Server`, `Mobile`, `Docs`, `Agentic`, `Tools`) | +| `is_private` | consumer | GitHub's `isPrivate` | +| `is_archived` | consumer | GitHub's `isArchived` | +| `default_branch` | consumer | The default branch's name | +| `labels_total` | consumer | GitHub's `totalCount` of the repository's labels (`repo_labels` rows) | +| `is_tracked` | consumer | 1 when `tracking.toml` listed the repository at the last sync that checked (redesign plan 5.3 step 1); an untracked repository's rows stay until `ghactivity sync --prune-untracked` (D27) | +| `resolved_at` | consumer | GitHub's `Date` header of the first response of the resolve that wrote the row: the as-of time of its GitHub fields and labels | +| `is_security_policy_enabled` | consumer | GitHub's `isSecurityPolicyEnabled`; null when GitHub returns none | +| `visibility` | consumer | GitHub's `visibility` (`PUBLIC`, `PRIVATE`, ...) | +| `description` | consumer | GitHub's `description`; null when there is none | +| `homepage_url` | consumer | GitHub's `homepageUrl`, a link the owner sets; null when there is none | +| `created_at` | consumer | GitHub's `createdAt` | +| `updated_at` | consumer | GitHub's `updatedAt` | +| `pushed_at` | consumer | GitHub's `pushedAt`; null when GitHub returns none | +| `archived_at` | consumer | GitHub's `archivedAt`; null when the repository is not archived | +| `stargazer_count` | consumer | GitHub's `stargazerCount` | +| `fork_count` | consumer | GitHub's `forkCount` | +| `disk_usage` | consumer | GitHub's `diskUsage`, in kilobytes; null when GitHub returns none | +| `forking_allowed` | consumer | GitHub's `forkingAllowed` | +| `allow_update_branch` | consumer | GitHub's `allowUpdateBranch` | +| `web_commit_signoff_required` | consumer | GitHub's `webCommitSignoffRequired` | +| `has_wiki_enabled` | consumer | GitHub's `hasWikiEnabled` | +| `has_discussions_enabled` | consumer | GitHub's `hasDiscussionsEnabled` | +| `has_projects_enabled` | consumer | GitHub's `hasProjectsEnabled` | +| `has_vulnerability_alerts_enabled` | consumer | GitHub's `hasVulnerabilityAlertsEnabled` | +| `uses_custom_open_graph_image` | consumer | GitHub's `usesCustomOpenGraphImage` | +| `merge_commit_allowed` | consumer | GitHub's `mergeCommitAllowed` | +| `squash_merge_allowed` | consumer | GitHub's `squashMergeAllowed` | +| `rebase_merge_allowed` | consumer | GitHub's `rebaseMergeAllowed` | +| `delete_branch_on_merge` | consumer | GitHub's `deleteBranchOnMerge` | +| `merge_commit_title` | consumer | GitHub's `mergeCommitTitle` setting | +| `merge_commit_message` | consumer | GitHub's `mergeCommitMessage` setting | +| `squash_merge_commit_title` | consumer | GitHub's `squashMergeCommitTitle` setting | +| `squash_merge_commit_message` | consumer | GitHub's `squashMergeCommitMessage` setting | +| `primary_language` | consumer | The name of GitHub's `primaryLanguage`; null when there is none | +| `license_spdx_id` | consumer | The `spdxId` of GitHub's `licenseInfo`; null when GitHub returns no license or no `spdxId` | +| `plan_features_codeowners` | consumer | GitHub's `planFeatures.codeowners` | +| `plan_features_team_review_requests` | consumer | GitHub's `planFeatures.teamReviewRequests` | +| `plan_features_maximum_manual_review_requests` | consumer | GitHub's `planFeatures.maximumManualReviewRequests` | + +### `repo_labels` + +Each repository's label set as the latest complete resolve read it (D8). + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_id` | consumer | Row id of the repository | +| `node_id` | consumer | GitHub's label id | +| `name` | consumer | Label name | +| `color` | consumer | GitHub's `color` | +| `description` | consumer | GitHub's `description`; null when there is none | +| `is_default` | consumer | GitHub's `isDefault` | +| `created_at` | consumer | GitHub's `createdAt`; null when GitHub returns none | +| `updated_at` | consumer | GitHub's `updatedAt`; null when GitHub returns none | + +### `github_users` + +Every account the database has stored, kept after a lookup answers `NOT_FOUND` for it (D28). + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | consumer | Row id | +| `github_id` | consumer | GitHub's numeric account id; null only for an `EnterpriseUserAccount`, which has none | +| `node_id` | consumer | GitHub's GraphQL node id; the key of an account with a null `github_id` | +| `login` | consumer | Last known login, not unique; an actor sighting sets it when it inserts the account and only lookups change it; each sync refreshes accounts last refreshed more than 24 hours ago, except those with `not_found_at` set | +| `type` | consumer | GitHub's type: `User`, `Bot`, `Organization`, `Mannequin`, or `EnterpriseUserAccount` | +| `name` | consumer | GitHub's `name`; null until the account's first lookup, for a `Bot`, and when GitHub returns none | +| `created_at` | consumer | GitHub's `createdAt`; null until the account's first lookup | +| `updated_at` | consumer | GitHub's `updatedAt`; null until the account's first lookup | +| `linked_user_github_id` | consumer | An `EnterpriseUserAccount`'s `user.databaseId`; null for another type or when GitHub returns none | +| `claimant_github_id` | consumer | A `Mannequin`'s `claimant.databaseId`; null until the account's first lookup, for another type, and when GitHub returns none | +| `refreshed_at` | consumer | GitHub's `Date` header of the response that inserted the row or of its latest lookup that returned the account; a `NOT_FOUND` lookup leaves it | +| `not_found_at` | consumer | GitHub's `Date` header of the first lookup that answered `NOT_FOUND` since the account was last seen; null otherwise. No reference reads it | + +### `account_profiles` + +A `User`'s or an `Organization`'s public profile as the latest lookup returned it (D31); a +`Bot`, `Mannequin`, or `EnterpriseUserAccount` has no row. A lookup that answers `NOT_FOUND` +leaves the row as it was. + +| Column | Use | Meaning | +|--------|-----|---------| +| `account_id` | consumer | Row id of the account | +| `company` | consumer | A user's public profile `company`; null for an organization and when there is none | +| `location` | consumer | The profile's `location`; null when there is none | +| `website_url` | consumer | The profile's `websiteUrl`; null when there is none | +| `description` | consumer | An organization's `description`; null for a user and when there is none | +| `email` | consumer | An organization's public `email`; null for a user and when there is none. A user's profile email is not collected (D31) | +| `twitter_username` | consumer | An organization's `twitterUsername`; null for a user and when there is none | +| `is_verified` | consumer | An organization's `isVerified`; null for a user | +| `archived_at` | consumer | An organization's `archivedAt`; null for a user and when it is not archived | +| `has_sponsors_listing` | consumer | An organization's `hasSponsorsListing`; null for a user | +| `pinned_items_remaining` | consumer | An organization's `pinnedItemsRemaining`; null for a user | +| `refreshed_at` | consumer | GitHub's `Date` header of the lookup that wrote the row | + +### `pull_requests` + +Pull requests in coverage: open ones, ones merged on or after the coverage floor in +`tracking.toml`, and ones closed without merge on or after it (redesign plan section 3, D3). + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | consumer | Row id | +| `repository_id` | consumer | Row id of the repository | +| `number` | consumer | Pull request number | +| `github_id` | consumer | GitHub's numeric pull request id (can exceed 2^31) | +| `node_id` | consumer | GitHub's GraphQL node id | +| `state` | consumer | GitHub's `state`: `OPEN`, `CLOSED`, or `MERGED` | +| `is_draft` | consumer | GitHub's `isDraft` | +| `title` | consumer | Title | +| `body` | consumer | Description, empty when there is none (D19) | +| `author_id` | consumer | Row id of the author's account; null when GitHub returned no author and the row had none stored | +| `author_reported` | consumer | 1 when GitHub returned an author in the row's latest hydration; 0 when it returned none, `author_id` then being a reference kept from earlier or null (D28) | +| `author_association` | consumer | GitHub's current `authorAssociation` (`MEMBER`, `CONTRIBUTOR`, ...), which GitHub computes when read | +| `created_at` | consumer | GitHub's `createdAt` | +| `merged_at` | consumer | GitHub's `mergedAt`; set exactly when `state` is `MERGED` | +| `closed_at` | consumer | GitHub's `closedAt`; set for every `CLOSED` or `MERGED` row (plan 5.3 step 5) | +| `published_at` | consumer | GitHub's `publishedAt`; null when GitHub returns none | +| `last_edited_at` | consumer | GitHub's `lastEditedAt`; null when GitHub returns none | +| `editor_id` | consumer | Row id of the account GitHub lists as the last editor; null when GitHub returns none | +| `includes_created_edit` | consumer | GitHub's `includesCreatedEdit` | +| `maintainer_can_modify` | consumer | GitHub's `maintainerCanModify` | +| `merged_by_id` | consumer | Row id of the merger's account; null on an unmerged pull request, and when GitHub returned no merger and the row had none stored | +| `merged_by_reported` | consumer | 1 when GitHub returned a merger in the row's latest hydration (D28) | +| `base_ref` | consumer | GitHub's `baseRefName` | +| `base_ref_oid` | consumer | GitHub's `baseRefOid` | +| `head_ref_name` | consumer | GitHub's `headRefName` | +| `head_ref_oid` | consumer | GitHub's `headRefOid` | +| `head_repo_github_id` | consumer | Numeric id of GitHub's `headRepository`; null when GitHub returns no head repository or no id for it | +| `head_repo_full_name` | consumer | `nameWithOwner` of GitHub's `headRepository`; null when GitHub returns none | +| `head_repo_owner_node_id` | consumer | Node id of GitHub's `headRepositoryOwner`; null when GitHub returns none | +| `head_repo_owner_login` | consumer | Login of GitHub's `headRepositoryOwner`; null when GitHub returns none | +| `merge_commit_oid` | consumer | Oid of GitHub's `mergeCommit`; null when GitHub returns none | +| `targets_default_branch` | consumer | Classification: whether `base_ref` equals the repository's current `default_branch`, recomputed when the default branch changes | +| `is_cross_repository` | consumer | GitHub's `isCrossRepository` | +| `milestone_number` | consumer | Number of GitHub's `milestone` reference; null when the pull request is in none | +| `milestone_title` | consumer | Title of GitHub's `milestone` reference; null when the pull request is in none | +| `review_decision` | consumer | GitHub's `reviewDecision` when the row was hydrated; null when GitHub returns none | +| `mergeable` | consumer | GitHub's `mergeable` when the row was hydrated; GitHub recomputes it, so it is as of `hydrated_at` | +| `merge_state_status` | consumer | GitHub's `mergeStateStatus` when the row was hydrated, as of `hydrated_at` | +| `can_be_rebased` | consumer | GitHub's `canBeRebased` when the row was hydrated, as of `hydrated_at` | +| `potential_merge_commit_oid` | consumer | Oid of GitHub's `potentialMergeCommit` when the row was hydrated; GitHub returns none "if the pull request is merged, or if the test merge commit is still being generated" | +| `stack_node_id` | consumer | Node id of GitHub's `stack`; null when the pull request is in none | +| `stack_number` | consumer | GitHub's stack `number`, "A number uniquely identifying the stack within its repository"; null when the pull request is in none | +| `stack_size` | consumer | GitHub's stack `size`; null when the pull request is in none | +| `stack_base_ref_name` | consumer | GitHub's stack `baseRefName`; null when the pull request is in none | +| `stack_entry_node_id` | consumer | Node id of GitHub's `stackEntry`; null when GitHub returns none | +| `stack_position` | consumer | GitHub's stack entry `position`, "where 1 is the closest to the base branch"; null when GitHub returns no entry | +| `additions` | consumer | GitHub's `additions` | +| `deletions` | consumer | GitHub's `deletions` | +| `changed_files` | consumer | GitHub's `changedFiles`; a path listed twice counts once, so it can differ from `files_total` | +| `total_comments_count` | consumer | GitHub's `totalCommentsCount`; null when GitHub returns none | +| `commits_total` | consumer | GitHub's `totalCount` of the commits (`pr_commits` rows) | +| `files_total` | consumer | GitHub's `totalCount` of the file list (`pr_files` rows) | +| `reviews_total` | consumer | GitHub's `totalCount` of the submitted reviews (`pr_reviews` rows) | +| `comments_total` | consumer | GitHub's `totalCount` of the conversation comments (`pr_comments` rows) | +| `labels_total` | consumer | GitHub's `totalCount` of the labels applied (`pr_labels` rows) | +| `closing_issues_total` | consumer | GitHub's `totalCount` of `closingIssuesReferences` (`pr_closing_issues` rows) | +| `user_linked_closing_issues_total` | consumer | GitHub's `totalCount` of `closingIssuesReferences` with `userLinkedOnly: true` (rows with `user_linked`) | +| `review_requests_total` | consumer | GitHub's `totalCount` of `reviewRequests` (`pr_review_requests` rows) | +| `assignees_total` | consumer | GitHub's `totalCount` of `assignedActors` (`pr_assignees` rows) | +| `github_updated_at` | consumer | GitHub's `updatedAt` as of the hydration that wrote the row; updates from the index alone leave it | +| `hydrated_at` | consumer | GitHub's `Date` header of the response that read the pull request: the as-of time of its hydrated fields | +| `hydration_version` | consumer | Version of the queries that read the pull request; a column added later is null on rows read at an older version until they are read again | + +### `pr_labels` + +Labels as currently applied, without history (D8). + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | consumer | Row id of the pull request | +| `name` | consumer | Label name | +| `label_node_id` | consumer | GitHub's label id | + +### `pr_reviews` + +Submitted reviews; pending reviews are not stored (D4). Review ids do not follow submission +order (D6), so `submitted_at` gives that order. No review text (D24). + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | internal | Row id | +| `github_id` | consumer | GitHub's numeric review id | +| `node_id` | consumer | GitHub's GraphQL node id | +| `pr_id` | consumer | Row id of the pull request | +| `reviewer_id` | consumer | Row id of the reviewer's account; null when GitHub returned no author and the row had none stored | +| `reviewer_reported` | consumer | 1 when GitHub returned an author in the review's latest hydration (D28) | +| `state` | consumer | GitHub's `state`: `APPROVED`, `CHANGES_REQUESTED`, `COMMENTED`, or `DISMISSED` | +| `created_at` | consumer | GitHub's `createdAt` | +| `submitted_at` | consumer | GitHub's `submittedAt`; null when GitHub returns none | +| `published_at` | consumer | GitHub's `publishedAt`; null when GitHub returns none | +| `updated_at` | consumer | GitHub's `updatedAt` | +| `last_edited_at` | consumer | GitHub's `lastEditedAt`; null when GitHub returns none | +| `editor_id` | consumer | Row id of the account GitHub lists as the last editor; null when GitHub returns none | +| `includes_created_edit` | consumer | GitHub's `includesCreatedEdit` | +| `commit_oid` | consumer | Oid of the review's `commit`; null when GitHub returns none | +| `author_association` | consumer | GitHub's `authorAssociation` when the pull request was hydrated | +| `author_can_push_to_repository` | consumer | GitHub's `authorCanPushToRepository` when the pull request was hydrated | +| `inline_comments_total` | consumer | GitHub's `comments.totalCount` for the review: its inline comments (D5) | + +### `pr_comments` + +Conversation comments, one row each, without their text (D5, D24). + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | internal | Row id | +| `github_id` | consumer | GitHub's numeric comment id | +| `node_id` | consumer | GitHub's GraphQL node id | +| `pr_id` | consumer | Row id of the pull request | +| `author_id` | consumer | Row id of the author's account; null when GitHub returned no author and the row had none stored | +| `author_reported` | consumer | 1 when GitHub returned an author in the comment's latest hydration (D28) | +| `created_at` | consumer | GitHub's `createdAt` | +| `published_at` | consumer | GitHub's `publishedAt`; null when GitHub returns none | +| `updated_at` | consumer | GitHub's `updatedAt` | +| `last_edited_at` | consumer | GitHub's `lastEditedAt`; null when GitHub returns none | +| `editor_id` | consumer | Row id of the account GitHub lists as the last editor; null when GitHub returns none | +| `created_via_email` | consumer | GitHub's `createdViaEmail` | +| `includes_created_edit` | consumer | GitHub's `includesCreatedEdit` | +| `author_association` | consumer | GitHub's `authorAssociation` when the pull request was hydrated | + +### `pr_commits` + +The pull request's commits, each like a comment: its author link and git author, never its +message (D19); co-authors are not stored (D15). + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | consumer | Row id of the pull request | +| `sha` | consumer | Commit SHA | +| `author_id` | consumer | Row id of the account GitHub links the commit to; null when GitHub links none and the row had none stored, and when the linked account was never stored because a lookup answered `NOT_FOUND` for it (`linked_github_id` then holds its id) | +| `linked_github_id` | consumer | The account id GitHub linked the commit to in the latest hydration, whether or not a lookup found the account; null when it linked none (D28) | +| `author_name` | consumer | Git author name | +| `author_email` | consumer | Git author email | +| `authored_at` | consumer | Git author date, set by the committer's client (D6) | + +### `pr_files` + +The file list, one row per entry; a path can appear twice. + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | consumer | Row id of the pull request | +| `ordinal` | consumer | Position in GitHub's file list, from 0 | +| `path` | consumer | File path | +| `change_type` | consumer | GitHub's `changeType` (`ADDED`, `MODIFIED`, ...) | +| `additions` | consumer | Lines added to the file | +| `deletions` | consumer | Lines deleted from the file | + +### `pr_closing_issues` + +The pull request's `closingIssuesReferences`, which GitHub describes as "List of issues that may +be closed by this pull request", possibly in repositories the database does not track (D23). + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | consumer | Row id of the pull request | +| `issue_repo_github_id` | consumer | GitHub's numeric id of the issue's repository | +| `issue_repo_full_name` | consumer | `owner/name` of the issue's repository when the link was last written | +| `issue_number` | consumer | Issue number | +| `issue_github_id` | consumer | The issue's `fullDatabaseId` | +| `user_linked` | consumer | 1 when GitHub also lists the link under `userLinkedOnly: true`, which GitHub describes as "Return only manually linked Issues" | + +### `pr_review_requests` + +GitHub's `reviewRequests` for the pull request, one row per entry as GitHub currently lists it +(D4). + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | consumer | Row id of the pull request | +| `ordinal` | consumer | Position in GitHub's list, from 0 | +| `request_github_id` | consumer | The request's own `databaseId`; null when GitHub returns none | +| `request_node_id` | consumer | The request's own node id | +| `requested_type` | consumer | GitHub's type of the requested reviewer (`User`, `Bot`, `Mannequin`, `Team`, `EnterpriseTeam`); null when GitHub returns no requested reviewer | +| `requested_github_id` | consumer | The requested account's or `Team`'s `databaseId`, or an `EnterpriseTeam`'s `fullDatabaseId`; null when GitHub returns none | +| `requested_node_id` | consumer | The requested account's, team's, or enterprise team's node id; null when GitHub returns no requested reviewer | +| `account_id` | consumer | Row id of the requested account; null for a `Team` or an `EnterpriseTeam`, which is not an account, and when GitHub returns no requested reviewer | +| `team_slug` | consumer | The requested team's or enterprise team's `slug`; null for an account and when GitHub returns no requested reviewer | +| `as_code_owner` | consumer | GitHub's `asCodeOwner` | + +### `pr_assignees` + +The actors GitHub lists in the pull request's `assignedActors` (D14): users, bots, mannequins, +and organizations. + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | consumer | Row id of the pull request | +| `account_id` | consumer | Row id of the assigned account | + +### `automation_accounts` + +The `[automation].accounts` list of `tracking.toml`, copied at the start of every sync. + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | internal | Row id | +| `github_id` | consumer | GitHub's numeric account id | +| `login` | consumer | Login as `tracking.toml` gives it | +| `reason` | consumer | The reason `tracking.toml` gives for the entry | +| `run_id` | consumer | Row id of the run that copied the list | + +### `automation_emails` + +The `[automation].emails` list of `tracking.toml`, copied at the start of every sync. + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | internal | Row id | +| `email` | consumer | Commit author email, lowercase ASCII | +| `reason` | consumer | The reason `tracking.toml` gives for the entry | +| `run_id` | consumer | Row id of the run that copied the list | + +### `sync_runs` + +One row per sync run. + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | consumer | Row id | +| `started_at` | consumer | When the run started | +| `deadline_at` | consumer | `started_at` plus `--max-minutes`; null without it | +| `finished_at` | consumer | When the run closed; null while it runs, and on a row another process marked `aborted` | +| `status` | consumer | `running`, `complete`, `time_limited`, `failed`, or `aborted` | +| `exit_code` | consumer | The run's exit code; null while it runs, and on a row another process marked `aborted` | +| `trigger` | consumer | `schedule` or `workflow_dispatch` from the Actions event, else `manual` | +| `code_version` | consumer | `GITHUB_SHA`; else the commit of the git checkout the package ran from, with `-dirty` when its tracked files had changes; else the package version | +| `config_source` | consumer | `packaged`, or the `TRACKING_CONFIG_PATH` path the run read, made absolute without following symlinks | +| `config_sha256` | consumer | SHA-256 of the `tracking.toml` bytes the run read | +| `graphql_points` | internal | GraphQL rate-limit points spent | +| `rest_calls` | internal | REST requests sent | +| `items_planned` | internal | Distinct items planned for hydration | +| `items_hydrated` | internal | Hydrated items written | +| `items_deleted` | internal | Item rows deleted | +| `failures` | internal | Item failures in the run, accepted ones included | +| `deletions_approved` | consumer | JSON text, keys sorted: `allow_deletions` (`repository` as configured, its `github_id`, `approved` N, and `sets` with `items`' `candidates` and whether they were `deleted`) and `prune_untracked` (each pruned repository's `github_id`, `full_name`, and `items_deleted`), each written with the deletion it records; null when neither applied | + +### `repo_sync_state` + +Per-repository sync state, written as each run closes. + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_id` | consumer | Row id of the repository | +| `last_index_started_at` | consumer | Start of the last complete index the run did not later fail: the as-of time of everything the index wrote | +| `index_pr_total` | consumer | Pull requests that index counted | +| `index_issue_total` | consumer | Issues that index counted; null until issues are indexed | +| `last_success_at` | consumer | `finished_at` of the latest run in which the repository's outcome was `complete` and none of its items failed outside `[sync].accepted_failures` | +| `last_run_id` | consumer | Row id of the latest run that attempted the repository | +| `last_outcome` | consumer | That run's result for the repository | +| `halt_reason` | consumer | Why deletions halted in the latest run that halted them, with the candidate count; cleared by a run whose step 7 ran for the repository without halting, and left by one whose step 7 did not run for it | + +### `sync_pending` + +Items a time-limited run planned but did not reach: an in-coverage item not yet hydrated. + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_id` | consumer | Row id of the repository | +| `item_kind` | consumer | `pull_request` or `issue` | +| `number` | consumer | Item number | +| `run_id` | consumer | Row id of the run that left the item | + +### `sync_failures` + +Items whose latest attempt failed, in a hydration or in the planner's check of an index entry +(redesign plan 5.3 step 3). The next run plans the item again; its row is deleted when the item is +written, or when the item leaves coverage. + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | internal | Row id | +| `repository_id` | consumer | Row id of the repository | +| `item_kind` | consumer | `pull_request` or `issue` | +| `number` | consumer | Item number | +| `attempts` | internal | Runs in which the item failed | +| `last_error` | consumer | The last error's class and message | +| `first_failed_at` | consumer | When the item first failed | +| `last_failed_at` | consumer | When the item last failed | +| `last_run_id` | consumer | Row id of the latest run in which the item failed; `attempts` adds one only for a failure in another run | + +### `user_tags` + +Kept from the first schema; nothing writes tags yet (D17). + +| Column | Use | Meaning | +|--------|-----|---------| +| `id` | internal | Row id | +| `name` | internal | Tag name | +| `description` | internal | Tag description | +| `color` | internal | Tag color | +| `created_at` | internal | When the tag was created | + +### `pr_user_tags` + +Tags applied to pull requests; empty until a tag producer exists (D17). + +| Column | Use | Meaning | +|--------|-----|---------| +| `pr_id` | internal | Row id of the pull request | +| `user_tag_id` | internal | Row id of the tag | +| `created_at` | internal | When the tag was applied | + +### `alembic_version` + +The migration revision the file is at. + +| Column | Use | Meaning | +|--------|-----|---------| +| `version_num` | internal | Alembic revision id | + +## Views + +Every view row comes from one stored row, and every row is kept: a null reference gives null +account columns. A pull request's keys are its repository's `github_id`, `full_name`, and +`is_tracked`, and its `number` and `github_id`. An account carries its `github_id`, `node_id`, +`login`, `type`, and `automation_reason`, the reason of the `automation_accounts` entry for its +`github_id` (null when none lists it). + +### `v_github_users` + +One row per stored account. + +| Column | Use | Meaning | +|--------|-----|---------| +| `github_id` | consumer | GitHub's numeric account id; null for an `EnterpriseUserAccount` | +| `node_id` | consumer | GitHub's GraphQL node id | +| `login` | consumer | Last known login | +| `type` | consumer | GitHub's type | +| `name` | consumer | GitHub's `name`; null until the first lookup, for a `Bot`, and when there is none | +| `created_at` | consumer | GitHub's `createdAt`; null until the first lookup | +| `updated_at` | consumer | GitHub's `updatedAt`; null until the first lookup | +| `linked_user_github_id` | consumer | An `EnterpriseUserAccount`'s user id | +| `claimant_github_id` | consumer | A `Mannequin`'s claimant id | +| `refreshed_at` | consumer | GitHub's `Date` header of the response that inserted the row or of its latest lookup that returned the account; a `NOT_FOUND` lookup leaves it | +| `not_found_at` | consumer | GitHub's `Date` header of the first lookup that answered `NOT_FOUND` since the account was last seen; null otherwise | +| `automation_reason` | consumer | Classification: the reason of the `automation_accounts` entry for the account's `github_id`; null when none lists it | + +### `v_account_profiles` + +One row per `account_profiles` row, with its account's keys. + +| Column | Use | Meaning | +|--------|-----|---------| +| `github_id` | consumer | The account's numeric id | +| `node_id` | consumer | The account's node id | +| `login` | consumer | The account's last known login | +| `type` | consumer | `User` or `Organization` | +| `automation_reason` | consumer | Classification: the reason of the `automation_accounts` entry for the account's `github_id`; null when none lists it | +| `company` | consumer | A user's profile `company` | +| `location` | consumer | The profile's `location` | +| `website_url` | consumer | The profile's `websiteUrl` | +| `description` | consumer | An organization's `description` | +| `email` | consumer | An organization's public `email` | +| `twitter_username` | consumer | An organization's `twitterUsername` | +| `is_verified` | consumer | An organization's `isVerified` | +| `archived_at` | consumer | An organization's `archivedAt` | +| `has_sponsors_listing` | consumer | An organization's `hasSponsorsListing` | +| `pinned_items_remaining` | consumer | An organization's `pinnedItemsRemaining` | +| `refreshed_at` | consumer | GitHub's `Date` header of the lookup that wrote the profile | + +### `v_pr_reviews` + +One row per stored review. + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_github_id` | consumer | The repository's numeric id | +| `repository_full_name` | consumer | `owner/name` of the repository | +| `repository_is_tracked` | consumer | The repository's `is_tracked` | +| `pr_number` | consumer | Pull request number | +| `pr_github_id` | consumer | GitHub's numeric pull request id | +| `review_github_id` | consumer | GitHub's numeric review id | +| `review_node_id` | consumer | The review's node id | +| `state` | consumer | GitHub's review `state` | +| `created_at` | consumer | GitHub's `createdAt` | +| `submitted_at` | consumer | GitHub's `submittedAt` | +| `published_at` | consumer | GitHub's `publishedAt` | +| `updated_at` | consumer | GitHub's `updatedAt` | +| `last_edited_at` | consumer | GitHub's `lastEditedAt` | +| `includes_created_edit` | consumer | GitHub's `includesCreatedEdit` | +| `commit_oid` | consumer | Oid of the review's commit | +| `author_association` | consumer | GitHub's `authorAssociation` when hydrated | +| `author_can_push_to_repository` | consumer | GitHub's `authorCanPushToRepository` when hydrated | +| `inline_comments_total` | consumer | GitHub's count of the review's inline comments | +| `reviewer_reported` | consumer | 1 when GitHub returned the reviewer in the latest hydration | +| `reviewer_github_id` | consumer | The reviewer's numeric id | +| `reviewer_node_id` | consumer | The reviewer's node id | +| `reviewer_login` | consumer | The reviewer's last known login | +| `reviewer_type` | consumer | The reviewer's type | +| `reviewer_automation_reason` | consumer | The reviewer's `[automation]` reason | +| `editor_github_id` | consumer | The last editor's numeric id | +| `editor_node_id` | consumer | The last editor's node id | +| `editor_login` | consumer | The last editor's last known login | +| `editor_type` | consumer | The last editor's type | +| `editor_automation_reason` | consumer | The last editor's `[automation]` reason | +| `is_pr_author` | consumer | Classification: 1 when the reviewer is the pull request's author, 0 when not; null when either reference is null | + +### `v_pr_comments` + +One row per stored conversation comment. + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_github_id` | consumer | The repository's numeric id | +| `repository_full_name` | consumer | `owner/name` of the repository | +| `repository_is_tracked` | consumer | The repository's `is_tracked` | +| `pr_number` | consumer | Pull request number | +| `pr_github_id` | consumer | GitHub's numeric pull request id | +| `comment_github_id` | consumer | GitHub's numeric comment id | +| `comment_node_id` | consumer | The comment's node id | +| `created_at` | consumer | GitHub's `createdAt` | +| `published_at` | consumer | GitHub's `publishedAt` | +| `updated_at` | consumer | GitHub's `updatedAt` | +| `last_edited_at` | consumer | GitHub's `lastEditedAt` | +| `created_via_email` | consumer | GitHub's `createdViaEmail` | +| `includes_created_edit` | consumer | GitHub's `includesCreatedEdit` | +| `author_association` | consumer | GitHub's `authorAssociation` when hydrated | +| `author_reported` | consumer | 1 when GitHub returned the author in the latest hydration | +| `author_github_id` | consumer | The author's numeric id | +| `author_node_id` | consumer | The author's node id | +| `author_login` | consumer | The author's last known login | +| `author_type` | consumer | The author's type | +| `author_automation_reason` | consumer | The author's `[automation]` reason | +| `editor_github_id` | consumer | The last editor's numeric id | +| `editor_node_id` | consumer | The last editor's node id | +| `editor_login` | consumer | The last editor's last known login | +| `editor_type` | consumer | The last editor's type | +| `editor_automation_reason` | consumer | The last editor's `[automation]` reason | +| `is_pr_author` | consumer | Classification: 1 when the comment's author is the pull request's author, 0 when not; null when either reference is null | +| `author_also_reviewed` | consumer | Classification: 1 when the author has a stored review on the same pull request, 0 when not; null when the author is null | + +### `v_commit_authors` + +One row per stored commit. + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_github_id` | consumer | The repository's numeric id | +| `repository_full_name` | consumer | `owner/name` of the repository | +| `repository_is_tracked` | consumer | The repository's `is_tracked` | +| `pr_number` | consumer | Pull request number | +| `pr_github_id` | consumer | GitHub's numeric pull request id | +| `sha` | consumer | Commit SHA | +| `linked_github_id` | consumer | The account id GitHub linked the commit to in the latest hydration | +| `author_name` | consumer | Git author name | +| `author_email` | consumer | Git author email | +| `authored_at` | consumer | Git author date (D6) | +| `account_github_id` | consumer | The linked stored account's numeric id | +| `account_node_id` | consumer | The linked stored account's node id | +| `account_login` | consumer | The linked stored account's last known login | +| `account_type` | consumer | The linked stored account's type | +| `account_automation_reason` | consumer | The linked stored account's `[automation]` reason | +| `email_automation_reason` | consumer | Classification: the reason of the `automation_emails` entry equal to `author_email` with ASCII letters lowercased; null when none | + +### `v_pr_relationships` + +One row per relationship of a pull request to an account, a team, or an actor GitHub did not +return, from the stored rows, with the pull request's `additions`, `deletions`, and +`changed_files`. `relationship` names the source: +`author` (one per pull request), `merged_by` (one per merged pull request), `review` (one per +review), `comment` (one per comment), `review_request` (one per request), `assignee` (one per +assignee), `commit_author` (one per commit), and `pr_editor`, `review_editor`, and +`comment_editor` (one per pull request, review, or comment that GitHub lists an editor or a +`last_edited_at` for). Columns that belong to another source are null. + +| Column | Use | Meaning | +|--------|-----|---------| +| `repository_github_id` | consumer | The repository's numeric id | +| `repository_full_name` | consumer | `owner/name` of the repository | +| `repository_is_tracked` | consumer | The repository's `is_tracked` | +| `pr_number` | consumer | Pull request number | +| `pr_github_id` | consumer | GitHub's numeric pull request id | +| `relationship` | consumer | The stored row the view row comes from | +| `review_github_id` | consumer | The review's id, on `review` and `review_editor` rows | +| `comment_github_id` | consumer | The comment's id, on `comment` and `comment_editor` rows | +| `request_node_id` | consumer | The request's node id, on `review_request` rows | +| `request_github_id` | consumer | The request's `databaseId`, on `review_request` rows | +| `request_ordinal` | consumer | The request's position in GitHub's list, from 0, on `review_request` rows | +| `commit_sha` | consumer | The commit's SHA, on `commit_author` rows | +| `reported` | consumer | The source row's `*_reported` value, on `author`, `merged_by`, `review`, and `comment` rows | +| `account_github_id` | consumer | The account's numeric id | +| `account_node_id` | consumer | The account's node id | +| `account_login` | consumer | The account's last known login | +| `account_type` | consumer | The account's type | +| `account_automation_reason` | consumer | The account's `[automation]` reason | +| `account_not_found_at` | consumer | The account's `not_found_at` | +| `review_state` | consumer | The review's `state`, on `review` rows | +| `review_submitted_at` | consumer | The review's `submitted_at`, on `review` rows | +| `requested_type` | consumer | The requested reviewer's type, on `review_request` rows | +| `requested_github_id` | consumer | The requested account's or team's id, on `review_request` rows | +| `team_slug` | consumer | The requested team's slug, on `review_request` rows | +| `as_code_owner` | consumer | GitHub's `asCodeOwner`, on `review_request` rows | +| `linked_github_id` | consumer | The commit's `linked_github_id`, on `commit_author` rows | +| `author_name` | consumer | The commit's git author name, on `commit_author` rows | +| `author_email` | consumer | The commit's git author email, on `commit_author` rows | +| `email_automation_reason` | consumer | The commit email's `[automation]` reason, on `commit_author` rows | +| `pr_state` | consumer | The pull request's `state` | +| `pr_created_at` | consumer | The pull request's `created_at` | +| `pr_merged_at` | consumer | The pull request's `merged_at` | +| `pr_closed_at` | consumer | The pull request's `closed_at` | +| `base_ref` | consumer | The pull request's base branch name | +| `targets_default_branch` | consumer | The pull request's `targets_default_branch` | +| `merge_commit_oid` | consumer | The pull request's merge commit oid | +| `additions` | consumer | The pull request's `additions` | +| `deletions` | consumer | The pull request's `deletions` | +| `changed_files` | consumer | The pull request's `changed_files` | diff --git a/tests/fixtures/history/77663a7/tests/schema_versions.json b/tests/fixtures/history/77663a7/tests/schema_versions.json new file mode 100644 index 0000000..bc40c9c --- /dev/null +++ b/tests/fixtures/history/77663a7/tests/schema_versions.json @@ -0,0 +1,497 @@ +{ + "1": [ + ["automation_accounts", "github_id"], + ["automation_accounts", "login"], + ["automation_accounts", "reason"], + ["automation_emails", "email"], + ["automation_emails", "reason"], + ["github_users", "deleted_at"], + ["github_users", "github_id"], + ["github_users", "id"], + ["github_users", "login"], + ["github_users", "node_id"], + ["github_users", "type"], + ["pr_closing_issues", "issue_number"], + ["pr_closing_issues", "issue_repo_full_name"], + ["pr_closing_issues", "issue_repo_github_id"], + ["pr_closing_issues", "pr_id"], + ["pr_comments", "author_id"], + ["pr_comments", "github_id"], + ["pr_comments", "pr_id"], + ["pr_commits", "author_email"], + ["pr_commits", "author_id"], + ["pr_commits", "author_name"], + ["pr_commits", "authored_at"], + ["pr_commits", "pr_id"], + ["pr_commits", "sha"], + ["pr_files", "additions"], + ["pr_files", "change_type"], + ["pr_files", "deletions"], + ["pr_files", "ordinal"], + ["pr_files", "path"], + ["pr_files", "pr_id"], + ["pr_labels", "name"], + ["pr_labels", "pr_id"], + ["pr_reviews", "github_id"], + ["pr_reviews", "pr_id"], + ["pr_reviews", "reviewer_id"], + ["pr_reviews", "state"], + ["pull_requests", "additions"], + ["pull_requests", "author_association"], + ["pull_requests", "author_id"], + ["pull_requests", "base_ref"], + ["pull_requests", "body"], + ["pull_requests", "changed_files"], + ["pull_requests", "closing_issues_total"], + ["pull_requests", "comments_total"], + ["pull_requests", "commits_total"], + ["pull_requests", "created_at"], + ["pull_requests", "deletions"], + ["pull_requests", "files_total"], + ["pull_requests", "github_id"], + ["pull_requests", "github_updated_at"], + ["pull_requests", "id"], + ["pull_requests", "is_cross_repository"], + ["pull_requests", "is_draft"], + ["pull_requests", "labels_total"], + ["pull_requests", "merged_at"], + ["pull_requests", "merged_by_id"], + ["pull_requests", "node_id"], + ["pull_requests", "number"], + ["pull_requests", "repository_id"], + ["pull_requests", "reviews_total"], + ["pull_requests", "state"], + ["pull_requests", "targets_default_branch"], + ["pull_requests", "title"], + ["repositories", "default_branch"], + ["repositories", "full_name"], + ["repositories", "github_id"], + ["repositories", "id"], + ["repositories", "is_archived"], + ["repositories", "is_private"], + ["repositories", "is_tracked"], + ["repositories", "name"], + ["repositories", "owner"], + ["repositories", "repo_group"], + ["v_commit_authors", "author_email"], + ["v_commit_authors", "author_github_id"], + ["v_commit_authors", "author_name"], + ["v_commit_authors", "authored_at"], + ["v_commit_authors", "is_automation"], + ["v_commit_authors", "pr_github_id"], + ["v_commit_authors", "pr_number"], + ["v_commit_authors", "repository_full_name"], + ["v_commit_authors", "sha"], + ["v_github_users", "deleted_at"], + ["v_github_users", "github_id"], + ["v_github_users", "is_automation"], + ["v_github_users", "login"], + ["v_github_users", "node_id"], + ["v_github_users", "type"], + ["v_pr_commenters", "comment_count"], + ["v_pr_commenters", "commenter_github_id"], + ["v_pr_commenters", "commenter_login"], + ["v_pr_commenters", "pr_github_id"], + ["v_pr_commenters", "pr_number"], + ["v_pr_commenters", "repository_full_name"], + ["v_pr_reviewers", "comment_count"], + ["v_pr_reviewers", "final_verdict"], + ["v_pr_reviewers", "pr_github_id"], + ["v_pr_reviewers", "pr_number"], + ["v_pr_reviewers", "repository_full_name"], + ["v_pr_reviewers", "reviewer_github_id"], + ["v_pr_reviewers", "reviewer_login"] + ], + "2": { + "columns": [ + ["account_profiles", "account_id"], + ["account_profiles", "archived_at"], + ["account_profiles", "company"], + ["account_profiles", "description"], + ["account_profiles", "email"], + ["account_profiles", "has_sponsors_listing"], + ["account_profiles", "is_verified"], + ["account_profiles", "location"], + ["account_profiles", "pinned_items_remaining"], + ["account_profiles", "refreshed_at"], + ["account_profiles", "twitter_username"], + ["account_profiles", "website_url"], + ["automation_accounts", "github_id"], + ["automation_accounts", "login"], + ["automation_accounts", "reason"], + ["automation_accounts", "run_id"], + ["automation_emails", "email"], + ["automation_emails", "reason"], + ["automation_emails", "run_id"], + ["github_users", "claimant_github_id"], + ["github_users", "created_at"], + ["github_users", "github_id"], + ["github_users", "id"], + ["github_users", "linked_user_github_id"], + ["github_users", "login"], + ["github_users", "name"], + ["github_users", "node_id"], + ["github_users", "not_found_at"], + ["github_users", "refreshed_at"], + ["github_users", "type"], + ["github_users", "updated_at"], + ["pr_assignees", "account_id"], + ["pr_assignees", "pr_id"], + ["pr_closing_issues", "issue_github_id"], + ["pr_closing_issues", "issue_number"], + ["pr_closing_issues", "issue_repo_full_name"], + ["pr_closing_issues", "issue_repo_github_id"], + ["pr_closing_issues", "pr_id"], + ["pr_closing_issues", "user_linked"], + ["pr_comments", "author_association"], + ["pr_comments", "author_id"], + ["pr_comments", "author_reported"], + ["pr_comments", "created_at"], + ["pr_comments", "created_via_email"], + ["pr_comments", "editor_id"], + ["pr_comments", "github_id"], + ["pr_comments", "includes_created_edit"], + ["pr_comments", "last_edited_at"], + ["pr_comments", "node_id"], + ["pr_comments", "pr_id"], + ["pr_comments", "published_at"], + ["pr_comments", "updated_at"], + ["pr_commits", "author_email"], + ["pr_commits", "author_id"], + ["pr_commits", "author_name"], + ["pr_commits", "authored_at"], + ["pr_commits", "linked_github_id"], + ["pr_commits", "pr_id"], + ["pr_commits", "sha"], + ["pr_files", "additions"], + ["pr_files", "change_type"], + ["pr_files", "deletions"], + ["pr_files", "ordinal"], + ["pr_files", "path"], + ["pr_files", "pr_id"], + ["pr_labels", "label_node_id"], + ["pr_labels", "name"], + ["pr_labels", "pr_id"], + ["pr_review_requests", "account_id"], + ["pr_review_requests", "as_code_owner"], + ["pr_review_requests", "ordinal"], + ["pr_review_requests", "pr_id"], + ["pr_review_requests", "request_github_id"], + ["pr_review_requests", "request_node_id"], + ["pr_review_requests", "requested_github_id"], + ["pr_review_requests", "requested_node_id"], + ["pr_review_requests", "requested_type"], + ["pr_review_requests", "team_slug"], + ["pr_reviews", "author_association"], + ["pr_reviews", "author_can_push_to_repository"], + ["pr_reviews", "commit_oid"], + ["pr_reviews", "created_at"], + ["pr_reviews", "editor_id"], + ["pr_reviews", "github_id"], + ["pr_reviews", "includes_created_edit"], + ["pr_reviews", "inline_comments_total"], + ["pr_reviews", "last_edited_at"], + ["pr_reviews", "node_id"], + ["pr_reviews", "pr_id"], + ["pr_reviews", "published_at"], + ["pr_reviews", "reviewer_id"], + ["pr_reviews", "reviewer_reported"], + ["pr_reviews", "state"], + ["pr_reviews", "submitted_at"], + ["pr_reviews", "updated_at"], + ["pull_requests", "additions"], + ["pull_requests", "assignees_total"], + ["pull_requests", "author_association"], + ["pull_requests", "author_id"], + ["pull_requests", "author_reported"], + ["pull_requests", "base_ref"], + ["pull_requests", "base_ref_oid"], + ["pull_requests", "body"], + ["pull_requests", "can_be_rebased"], + ["pull_requests", "changed_files"], + ["pull_requests", "closed_at"], + ["pull_requests", "closing_issues_total"], + ["pull_requests", "comments_total"], + ["pull_requests", "commits_total"], + ["pull_requests", "created_at"], + ["pull_requests", "deletions"], + ["pull_requests", "editor_id"], + ["pull_requests", "files_total"], + ["pull_requests", "github_id"], + ["pull_requests", "github_updated_at"], + ["pull_requests", "head_ref_name"], + ["pull_requests", "head_ref_oid"], + ["pull_requests", "head_repo_full_name"], + ["pull_requests", "head_repo_github_id"], + ["pull_requests", "head_repo_owner_login"], + ["pull_requests", "head_repo_owner_node_id"], + ["pull_requests", "hydrated_at"], + ["pull_requests", "hydration_version"], + ["pull_requests", "id"], + ["pull_requests", "includes_created_edit"], + ["pull_requests", "is_cross_repository"], + ["pull_requests", "is_draft"], + ["pull_requests", "labels_total"], + ["pull_requests", "last_edited_at"], + ["pull_requests", "maintainer_can_modify"], + ["pull_requests", "merge_commit_oid"], + ["pull_requests", "merge_state_status"], + ["pull_requests", "mergeable"], + ["pull_requests", "merged_at"], + ["pull_requests", "merged_by_id"], + ["pull_requests", "merged_by_reported"], + ["pull_requests", "milestone_number"], + ["pull_requests", "milestone_title"], + ["pull_requests", "node_id"], + ["pull_requests", "number"], + ["pull_requests", "potential_merge_commit_oid"], + ["pull_requests", "published_at"], + ["pull_requests", "repository_id"], + ["pull_requests", "review_decision"], + ["pull_requests", "review_requests_total"], + ["pull_requests", "reviews_total"], + ["pull_requests", "stack_base_ref_name"], + ["pull_requests", "stack_entry_node_id"], + ["pull_requests", "stack_node_id"], + ["pull_requests", "stack_number"], + ["pull_requests", "stack_position"], + ["pull_requests", "stack_size"], + ["pull_requests", "state"], + ["pull_requests", "targets_default_branch"], + ["pull_requests", "title"], + ["pull_requests", "total_comments_count"], + ["pull_requests", "user_linked_closing_issues_total"], + ["repo_labels", "color"], + ["repo_labels", "created_at"], + ["repo_labels", "description"], + ["repo_labels", "is_default"], + ["repo_labels", "name"], + ["repo_labels", "node_id"], + ["repo_labels", "repository_id"], + ["repo_labels", "updated_at"], + ["repo_sync_state", "halt_reason"], + ["repo_sync_state", "index_issue_total"], + ["repo_sync_state", "index_pr_total"], + ["repo_sync_state", "last_index_started_at"], + ["repo_sync_state", "last_outcome"], + ["repo_sync_state", "last_run_id"], + ["repo_sync_state", "last_success_at"], + ["repo_sync_state", "repository_id"], + ["repositories", "allow_update_branch"], + ["repositories", "archived_at"], + ["repositories", "created_at"], + ["repositories", "default_branch"], + ["repositories", "delete_branch_on_merge"], + ["repositories", "description"], + ["repositories", "disk_usage"], + ["repositories", "fork_count"], + ["repositories", "forking_allowed"], + ["repositories", "full_name"], + ["repositories", "github_id"], + ["repositories", "has_discussions_enabled"], + ["repositories", "has_projects_enabled"], + ["repositories", "has_vulnerability_alerts_enabled"], + ["repositories", "has_wiki_enabled"], + ["repositories", "homepage_url"], + ["repositories", "id"], + ["repositories", "is_archived"], + ["repositories", "is_private"], + ["repositories", "is_security_policy_enabled"], + ["repositories", "is_tracked"], + ["repositories", "labels_total"], + ["repositories", "license_spdx_id"], + ["repositories", "merge_commit_allowed"], + ["repositories", "merge_commit_message"], + ["repositories", "merge_commit_title"], + ["repositories", "name"], + ["repositories", "node_id"], + ["repositories", "owner"], + ["repositories", "plan_features_codeowners"], + ["repositories", "plan_features_maximum_manual_review_requests"], + ["repositories", "plan_features_team_review_requests"], + ["repositories", "primary_language"], + ["repositories", "pushed_at"], + ["repositories", "rebase_merge_allowed"], + ["repositories", "repo_group"], + ["repositories", "resolved_at"], + ["repositories", "squash_merge_allowed"], + ["repositories", "squash_merge_commit_message"], + ["repositories", "squash_merge_commit_title"], + ["repositories", "stargazer_count"], + ["repositories", "updated_at"], + ["repositories", "uses_custom_open_graph_image"], + ["repositories", "visibility"], + ["repositories", "web_commit_signoff_required"], + ["sync_failures", "first_failed_at"], + ["sync_failures", "item_kind"], + ["sync_failures", "last_error"], + ["sync_failures", "last_failed_at"], + ["sync_failures", "last_run_id"], + ["sync_failures", "number"], + ["sync_failures", "repository_id"], + ["sync_pending", "item_kind"], + ["sync_pending", "number"], + ["sync_pending", "repository_id"], + ["sync_pending", "run_id"], + ["sync_runs", "code_version"], + ["sync_runs", "config_sha256"], + ["sync_runs", "config_source"], + ["sync_runs", "deadline_at"], + ["sync_runs", "deletions_approved"], + ["sync_runs", "exit_code"], + ["sync_runs", "finished_at"], + ["sync_runs", "id"], + ["sync_runs", "started_at"], + ["sync_runs", "status"], + ["sync_runs", "trigger"], + ["v_account_profiles", "archived_at"], + ["v_account_profiles", "automation_reason"], + ["v_account_profiles", "company"], + ["v_account_profiles", "description"], + ["v_account_profiles", "email"], + ["v_account_profiles", "github_id"], + ["v_account_profiles", "has_sponsors_listing"], + ["v_account_profiles", "is_verified"], + ["v_account_profiles", "location"], + ["v_account_profiles", "login"], + ["v_account_profiles", "node_id"], + ["v_account_profiles", "pinned_items_remaining"], + ["v_account_profiles", "refreshed_at"], + ["v_account_profiles", "twitter_username"], + ["v_account_profiles", "type"], + ["v_account_profiles", "website_url"], + ["v_commit_authors", "account_automation_reason"], + ["v_commit_authors", "account_github_id"], + ["v_commit_authors", "account_login"], + ["v_commit_authors", "account_node_id"], + ["v_commit_authors", "account_type"], + ["v_commit_authors", "author_email"], + ["v_commit_authors", "author_name"], + ["v_commit_authors", "authored_at"], + ["v_commit_authors", "email_automation_reason"], + ["v_commit_authors", "linked_github_id"], + ["v_commit_authors", "pr_github_id"], + ["v_commit_authors", "pr_number"], + ["v_commit_authors", "repository_full_name"], + ["v_commit_authors", "repository_github_id"], + ["v_commit_authors", "repository_is_tracked"], + ["v_commit_authors", "sha"], + ["v_github_users", "automation_reason"], + ["v_github_users", "claimant_github_id"], + ["v_github_users", "created_at"], + ["v_github_users", "github_id"], + ["v_github_users", "linked_user_github_id"], + ["v_github_users", "login"], + ["v_github_users", "name"], + ["v_github_users", "node_id"], + ["v_github_users", "not_found_at"], + ["v_github_users", "refreshed_at"], + ["v_github_users", "type"], + ["v_github_users", "updated_at"], + ["v_pr_comments", "author_also_reviewed"], + ["v_pr_comments", "author_association"], + ["v_pr_comments", "author_automation_reason"], + ["v_pr_comments", "author_github_id"], + ["v_pr_comments", "author_login"], + ["v_pr_comments", "author_node_id"], + ["v_pr_comments", "author_reported"], + ["v_pr_comments", "author_type"], + ["v_pr_comments", "comment_github_id"], + ["v_pr_comments", "comment_node_id"], + ["v_pr_comments", "created_at"], + ["v_pr_comments", "created_via_email"], + ["v_pr_comments", "editor_automation_reason"], + ["v_pr_comments", "editor_github_id"], + ["v_pr_comments", "editor_login"], + ["v_pr_comments", "editor_node_id"], + ["v_pr_comments", "editor_type"], + ["v_pr_comments", "includes_created_edit"], + ["v_pr_comments", "is_pr_author"], + ["v_pr_comments", "last_edited_at"], + ["v_pr_comments", "pr_github_id"], + ["v_pr_comments", "pr_number"], + ["v_pr_comments", "published_at"], + ["v_pr_comments", "repository_full_name"], + ["v_pr_comments", "repository_github_id"], + ["v_pr_comments", "repository_is_tracked"], + ["v_pr_comments", "updated_at"], + ["v_pr_relationships", "account_automation_reason"], + ["v_pr_relationships", "account_github_id"], + ["v_pr_relationships", "account_login"], + ["v_pr_relationships", "account_node_id"], + ["v_pr_relationships", "account_not_found_at"], + ["v_pr_relationships", "account_type"], + ["v_pr_relationships", "additions"], + ["v_pr_relationships", "as_code_owner"], + ["v_pr_relationships", "author_email"], + ["v_pr_relationships", "author_name"], + ["v_pr_relationships", "base_ref"], + ["v_pr_relationships", "changed_files"], + ["v_pr_relationships", "comment_github_id"], + ["v_pr_relationships", "commit_sha"], + ["v_pr_relationships", "deletions"], + ["v_pr_relationships", "email_automation_reason"], + ["v_pr_relationships", "linked_github_id"], + ["v_pr_relationships", "merge_commit_oid"], + ["v_pr_relationships", "pr_closed_at"], + ["v_pr_relationships", "pr_created_at"], + ["v_pr_relationships", "pr_github_id"], + ["v_pr_relationships", "pr_merged_at"], + ["v_pr_relationships", "pr_number"], + ["v_pr_relationships", "pr_state"], + ["v_pr_relationships", "relationship"], + ["v_pr_relationships", "reported"], + ["v_pr_relationships", "repository_full_name"], + ["v_pr_relationships", "repository_github_id"], + ["v_pr_relationships", "repository_is_tracked"], + ["v_pr_relationships", "request_github_id"], + ["v_pr_relationships", "request_node_id"], + ["v_pr_relationships", "request_ordinal"], + ["v_pr_relationships", "requested_github_id"], + ["v_pr_relationships", "requested_type"], + ["v_pr_relationships", "review_github_id"], + ["v_pr_relationships", "review_state"], + ["v_pr_relationships", "review_submitted_at"], + ["v_pr_relationships", "targets_default_branch"], + ["v_pr_relationships", "team_slug"], + ["v_pr_reviews", "author_association"], + ["v_pr_reviews", "author_can_push_to_repository"], + ["v_pr_reviews", "commit_oid"], + ["v_pr_reviews", "created_at"], + ["v_pr_reviews", "editor_automation_reason"], + ["v_pr_reviews", "editor_github_id"], + ["v_pr_reviews", "editor_login"], + ["v_pr_reviews", "editor_node_id"], + ["v_pr_reviews", "editor_type"], + ["v_pr_reviews", "includes_created_edit"], + ["v_pr_reviews", "inline_comments_total"], + ["v_pr_reviews", "is_pr_author"], + ["v_pr_reviews", "last_edited_at"], + ["v_pr_reviews", "pr_github_id"], + ["v_pr_reviews", "pr_number"], + ["v_pr_reviews", "published_at"], + ["v_pr_reviews", "repository_full_name"], + ["v_pr_reviews", "repository_github_id"], + ["v_pr_reviews", "repository_is_tracked"], + ["v_pr_reviews", "review_github_id"], + ["v_pr_reviews", "review_node_id"], + ["v_pr_reviews", "reviewer_automation_reason"], + ["v_pr_reviews", "reviewer_github_id"], + ["v_pr_reviews", "reviewer_login"], + ["v_pr_reviews", "reviewer_node_id"], + ["v_pr_reviews", "reviewer_reported"], + ["v_pr_reviews", "reviewer_type"], + ["v_pr_reviews", "state"], + ["v_pr_reviews", "submitted_at"], + ["v_pr_reviews", "updated_at"] + ], + "views": { + "v_account_profiles": "01c76ece53f4d3009b51f31b33077582d9db93c72f314cf9e95995f7faf2aab0", + "v_commit_authors": "6fdf1eb9572f0e2a24222669a1d56baf317f7c4a6f5ffceab5f0e995030038af", + "v_github_users": "31636d6122550ca013950fbcd35cf91d88ff02f59ee758d712a98f30eeecea1a", + "v_pr_comments": "89505c43b3d3ebc728844401c0c36724359a7ab057ddceb2ac420b2da95a704d", + "v_pr_relationships": "6aeb3202811037377c3fa734af207e0c51c5512307f3a13d46cc4be234041b08", + "v_pr_reviews": "73be434d0eb3ffa46b47dac729c1091e168736df2f54344256e528c678145c0a" + } + } +} diff --git a/tests/fixtures/history/README.md b/tests/fixtures/history/README.md new file mode 100644 index 0000000..df611bd --- /dev/null +++ b/tests/fixtures/history/README.md @@ -0,0 +1,15 @@ +# Earlier versions of repository files + +Each file here is a real earlier version of a repository file, copied unchanged from +`git show :` to `/`. CI checks out with depth 1 and has no +history, so the tests read these copies instead. + +| File | Commit | +|------|--------| +| `21a4267/docs/data-dictionary.md` | 21a4267adc84e69327f2795d6f045ea5514881b1 | +| `77663a7/docs/data-dictionary.md` | 77663a7a1039fbf1892fa7fc42ef3ecefa55ea5f | +| `c1f474b/tests/query_versions.json` | c1f474bf86117a38a5b6c45dd5808cf99e3f0da0 | +| `21a4267/tests/query_versions.json` | 21a4267adc84e69327f2795d6f045ea5514881b1 | +| `6fedb05/tests/schema_versions.json` | 6fedb05fde7bd26cc79b0a7eb2dd91d020655a58 | +| `21a4267/tests/schema_versions.json` | 21a4267adc84e69327f2795d6f045ea5514881b1 | +| `77663a7/tests/schema_versions.json` | 77663a7a1039fbf1892fa7fc42ef3ecefa55ea5f | diff --git a/tests/fixtures/history/c1f474b/tests/query_versions.json b/tests/fixtures/history/c1f474b/tests/query_versions.json new file mode 100644 index 0000000..6269da1 --- /dev/null +++ b/tests/fixtures/history/c1f474b/tests/query_versions.json @@ -0,0 +1,3 @@ +{ + "pull_request:1": "6aff7254be69c54b70e4e633c4b9d89640e5d9b99492b70ba94f7808a5e9cc76" +} diff --git a/tests/test_check_query_lock.py b/tests/test_check_query_lock.py index cd47c0e..f47fa3a 100644 --- a/tests/test_check_query_lock.py +++ b/tests/test_check_query_lock.py @@ -1,4 +1,8 @@ -"""The append-only check on tests/query_versions.json and tests/schema_versions.json.""" +"""The append-only check on tests/query_versions.json and tests/schema_versions.json. + +A lock a pull request changes or removes entries of is a real earlier version of the lock file +(tests/fixtures/history), placed over a base that holds a later one. +""" from __future__ import annotations @@ -7,21 +11,47 @@ import shutil import subprocess import sys +from collections.abc import Callable from pathlib import Path from typing import Any import pytest SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "check_query_lock.py" +HISTORY = Path(__file__).resolve().parent / "fixtures" / "history" GIT = shutil.which("git") BASH = shutil.which("bash") LOCK = "tests/query_versions.json" SCHEMA_LOCK = "tests/schema_versions.json" +# The query lock gained pull_request:2 at 21a4267; the schema lock gained version 2 there, and +# sync_failures.last_run_id in version 2 at 77663a7. +QUERY_LOCKS = ("c1f474b", "21a4267") +SCHEMA_LOCKS = ("6fedb05", "21a4267", "77663a7") + + +def environment(directory: Path) -> dict[str, str]: + """A git, checker, or step process's whole environment: the directory it runs in, inside + the test's own, as its home; no global or system git configuration; fixed commit dates.""" + return { + "PATH": os.environ["PATH"], + "HOME": str(directory), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_DATE": "2026-01-01T00:00:00Z", + "GIT_COMMITTER_DATE": "2026-01-01T00:00:00Z", + } def git(repo: Path, *args: str) -> None: assert GIT is not None - subprocess.run([GIT, *args], cwd=repo, check=True, capture_output=True, timeout=60) + subprocess.run( + [GIT, *args], + cwd=repo, + check=True, + capture_output=True, + env=environment(repo), + timeout=60, + ) @pytest.fixture @@ -39,21 +69,51 @@ def write_lock(repo: Path, entries: dict[str, str]) -> None: path.write_text(json.dumps(entries)) +def place(repo: Path, lock: str, commit: str) -> None: + """The lock file as it was at ``commit``, unchanged.""" + path = repo / lock + path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(HISTORY / commit / lock, path) + + +def entries_at(commit: str, lock: str) -> dict[str, Any]: + entries: dict[str, Any] = json.loads((HISTORY / commit / lock).read_text()) + return entries + + def commit(repo: Path) -> None: git(repo, "add", "-A") git(repo, "commit", "-q", "-m", "lock") -def check(repo: Path, base: str = "main") -> subprocess.CompletedProcess[str]: +def run_checker( + repo: Path, *options: str, cwd: Path | None = None, isolated: bool = False +) -> subprocess.CompletedProcess[str]: + """The checker on ``repo``, as CI runs it with ``isolated`` (``-I -S``).""" + flags = ["-I", "-S"] if isolated else [] + directory = cwd or repo return subprocess.run( - [sys.executable, str(SCRIPT), "--base", base], - cwd=repo, + [sys.executable, *flags, str(SCRIPT), *options], + cwd=directory, capture_output=True, text=True, + env=environment(directory), timeout=60, ) +def check(repo: Path, base: str = "main") -> subprocess.CompletedProcess[str]: + return run_checker(repo, "--base", base) + + +def kept(lock: str, before: str, now: str) -> str: + """The line the checker prints for a lock whose entries at ``before`` ``now`` keeps.""" + return ( + f"{lock}: {len(entries_at(before, lock))} entries at main kept; " + f"{len(entries_at(now, lock))} entries now" + ) + + def test_a_new_entry_passes(repo: Path) -> None: write_lock(repo, {"pull_request:1": "a" * 64}) commit(repo) @@ -62,22 +122,33 @@ def test_a_new_entry_passes(repo: Path) -> None: assert result.returncode == 0, result.stderr -def test_a_changed_entry_fails(repo: Path) -> None: - write_lock(repo, {"pull_request:1": "a" * 64}) +@pytest.mark.parametrize( + ("lock", "before", "now"), + [ + pytest.param(LOCK, *QUERY_LOCKS, id="query lock gained an entry"), + pytest.param(SCHEMA_LOCK, *SCHEMA_LOCKS[:2], id="schema lock gained a version"), + pytest.param(SCHEMA_LOCK, *SCHEMA_LOCKS[1:], id="newest schema version gained a pair"), + ], +) +def test_each_change_the_history_made_passes(repo: Path, lock: str, before: str, now: str) -> None: + """Every entry, pair, and view at the base is kept, unchanged; what was added was added to + a new entry or to the newest one.""" + place(repo, lock, before) commit(repo) - write_lock(repo, {"pull_request:1": "c" * 64}) + place(repo, lock, now) result = check(repo) - assert result.returncode == 1 - assert "pull_request:1 changed" in result.stderr + assert result.returncode == 0, result.stderr + assert kept(lock, before, now) in result.stdout.splitlines() def test_a_removed_entry_fails(repo: Path) -> None: - write_lock(repo, {"pull_request:1": "a" * 64, "pull_request:2": "b" * 64}) + place(repo, LOCK, QUERY_LOCKS[1]) commit(repo) - write_lock(repo, {"pull_request:2": "b" * 64}) + place(repo, LOCK, QUERY_LOCKS[0]) result = check(repo) assert result.returncode == 1 - assert "pull_request:1 was removed" in result.stderr + (removed,) = entries_at(QUERY_LOCKS[1], LOCK).keys() - entries_at(QUERY_LOCKS[0], LOCK).keys() + assert f"{removed} was removed" in result.stderr def test_a_lock_absent_at_the_base_passes(repo: Path) -> None: @@ -94,12 +165,13 @@ def test_an_unknown_base_is_an_error_not_a_pass(repo: Path) -> None: def test_a_deleted_lock_removes_every_entry(repo: Path) -> None: - write_lock(repo, {"pull_request:1": "a" * 64}) + place(repo, LOCK, QUERY_LOCKS[0]) commit(repo) (repo / LOCK).unlink() result = check(repo) assert result.returncode == 1 - assert "pull_request:1 was removed" in result.stderr + for key in entries_at(QUERY_LOCKS[0], LOCK): + assert f"{key} was removed" in result.stderr @pytest.mark.parametrize("text", ["{not json", "[1, 2]", '{"pull_request:1": 1}']) @@ -112,34 +184,31 @@ def test_a_malformed_lock_is_an_error_not_a_pass(repo: Path, text: str) -> None: def test_an_absolute_lock_path_is_an_error_not_a_pass(repo: Path) -> None: - write_lock(repo, {"pull_request:1": "a" * 64}) + place(repo, LOCK, QUERY_LOCKS[1]) commit(repo) - write_lock(repo, {"pull_request:1": "c" * 64}) - result = subprocess.run( - [sys.executable, str(SCRIPT), "--base", "main", "--lock", str(repo / LOCK)], - cwd=repo, - capture_output=True, - text=True, - timeout=60, - ) + result = run_checker(repo, "--base", "main", "--lock", str(repo / LOCK)) assert result.returncode == 2 assert "is not one of tests/query_versions.json" in result.stderr -def test_the_checker_runs_isolated_from_site_packages(repo: Path) -> None: - # CI runs the base copy with -I -S, outside the pull request's environment. - write_lock(repo, {"pull_request:1": "a" * 64}) +@pytest.mark.parametrize( + ("isolated", "directory"), + [ + # CI runs the base copy with -I -S, outside the pull request's environment. + pytest.param(True, ".", id="isolated from site-packages"), + pytest.param(False, "tests", id="from a subdirectory"), + ], +) +def test_the_verdict_does_not_depend_on_how_the_checker_runs( + repo: Path, isolated: bool, directory: str +) -> None: + place(repo, LOCK, QUERY_LOCKS[1]) commit(repo) - write_lock(repo, {"pull_request:1": "c" * 64}) - result = subprocess.run( - [sys.executable, "-I", "-S", str(SCRIPT), "--base", "main"], - cwd=repo, - capture_output=True, - text=True, - timeout=60, - ) - assert result.returncode == 1 - assert "pull_request:1 changed" in result.stderr + place(repo, LOCK, QUERY_LOCKS[0]) + result = run_checker(repo, "--base", "main", cwd=repo / directory, isolated=isolated) + assert result.returncode == 1, result.stdout + result.stderr + (removed,) = entries_at(QUERY_LOCKS[1], LOCK).keys() - entries_at(QUERY_LOCKS[0], LOCK).keys() + assert f"{removed} was removed" in result.stderr @pytest.mark.parametrize( @@ -163,21 +232,6 @@ def test_an_integer_past_pythons_digit_limit_is_an_error_not_a_pass(repo: Path) assert result.returncode == 2, result.stderr -def test_the_verdict_does_not_depend_on_the_working_directory(repo: Path) -> None: - write_lock(repo, {"pull_request:1": "a" * 64}) - commit(repo) - write_lock(repo, {"pull_request:1": "c" * 64}) - result = subprocess.run( - [sys.executable, str(SCRIPT), "--base", "main"], - cwd=repo / "tests", - capture_output=True, - text=True, - timeout=60, - ) - assert result.returncode == 1, result.stdout + result.stderr - assert "pull_request:1 changed" in result.stderr - - def test_a_malformed_lock_at_the_base_is_an_error_not_a_pass(repo: Path) -> None: (repo / "tests").mkdir() (repo / LOCK).write_text("{not json") @@ -191,13 +245,7 @@ def test_a_malformed_lock_at_the_base_is_an_error_not_a_pass(repo: Path) -> None def test_a_lock_path_outside_the_repository_is_an_error_not_a_pass(repo: Path) -> None: write_lock(repo, {"pull_request:1": "a" * 64}) commit(repo) - result = subprocess.run( - [sys.executable, str(SCRIPT), "--base", "main", "--lock", "../query_versions.json"], - cwd=repo, - capture_output=True, - text=True, - timeout=60, - ) + result = run_checker(repo, "--base", "main", "--lock", "../query_versions.json") assert result.returncode == 2 assert "is not one of tests/query_versions.json" in result.stderr @@ -233,16 +281,6 @@ def write_schema_lock(repo: Path, entries: dict[str, Any]) -> None: } -# Two entries with views, so a view can be edited in one that is not the newest at the base. -V3: dict[str, Any] = { - **V2, - "3": { - "columns": V2["2"]["columns"], - "views": {"v_pr_reviews": "c" * 64, "v_github_users": "b" * 64}, - }, -} - - def with_views(views: dict[str, str]) -> dict[str, Any]: """V2 with version 2's views replaced.""" return {**V2, "2": {**V2["2"], "views": views}} @@ -268,60 +306,37 @@ def test_an_added_version_passes(self, repo: Path) -> None: assert result.returncode == 0, result.stderr assert "tests/schema_versions.json: 1 entries at main kept; 2 entries now" in result.stdout - def test_a_pair_added_to_a_superseded_entry_fails(self, repo: Path) -> None: - """Version 1 describes databases that never had a column version 2 added.""" - write_schema_lock(repo, V1) - commit(repo) - write_schema_lock( - repo, - {"1": [*V1["1"], ["pull_requests", "title"]], "2": [["pull_requests", "number"]]}, - ) - result = check(repo) - assert result.returncode == 1 - assert ( - "schema version 1 gained pull_requests.title; only the newest version, 2, gains " - "columns" in result.stderr - ) - - def test_a_pair_removed_from_a_base_entry_fails(self, repo: Path) -> None: - write_schema_lock(repo, V1) - commit(repo) - write_schema_lock(repo, {"1": [["pull_requests", "number"]]}) - result = check(repo) - assert result.returncode == 1 - assert "schema version 1 lost v_pr_reviewers.final_verdict" in result.stderr - - def test_a_renamed_pair_fails(self, repo: Path) -> None: - write_schema_lock(repo, V1) - commit(repo) - write_schema_lock(repo, {"1": [["pull_requests", "pr_number"], V1["1"][1]]}) - result = check(repo) - assert result.returncode == 1 - assert "schema version 1 lost pull_requests.number" in result.stderr - def test_a_removed_entry_fails(self, repo: Path) -> None: - write_schema_lock(repo, {**V1, "2": V1["1"]}) + place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[1]) commit(repo) - write_schema_lock(repo, {"2": V1["1"]}) + place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[0]) result = check(repo) assert result.returncode == 1 - assert "schema version 1 was removed" in result.stderr + (removed,) = ( + entries_at(SCHEMA_LOCKS[1], SCHEMA_LOCK).keys() + - entries_at(SCHEMA_LOCKS[0], SCHEMA_LOCK).keys() + ) + assert f"schema version {removed} was removed" in result.stderr def test_a_deleted_lock_removes_every_entry(self, repo: Path) -> None: - write_schema_lock(repo, V1) + place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[0]) commit(repo) (repo / SCHEMA_LOCK).unlink() result = check(repo) assert result.returncode == 1 - assert "schema version 1 was removed" in result.stderr + for version in entries_at(SCHEMA_LOCKS[0], SCHEMA_LOCK): + assert f"schema version {version} was removed" in result.stderr def test_a_pair_removed_from_an_object_entry_fails(self, repo: Path) -> None: - write_schema_lock(repo, V2) + place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[2]) commit(repo) - write_schema_lock(repo, {**V2, "2": {**V2["2"], "columns": [["pull_requests", "number"]]}}) + place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[1]) result = check(repo) assert result.returncode == 1 - assert "schema version 2 lost v_pr_reviews.state" in result.stderr + before = entries_at(SCHEMA_LOCKS[2], SCHEMA_LOCK)["2"]["columns"] + now = entries_at(SCHEMA_LOCKS[1], SCHEMA_LOCK)["2"]["columns"] + ((table, column),) = {tuple(pair) for pair in before} - {tuple(pair) for pair in now} + assert f"schema version 2 lost {table}.{column}" in result.stderr def test_a_pair_added_to_the_newest_object_entry_passes(self, repo: Path) -> None: write_schema_lock(repo, V2) @@ -331,75 +346,6 @@ def test_a_pair_added_to_the_newest_object_entry_passes(self, repo: Path) -> Non result = check(repo) assert result.returncode == 0, result.stderr - def test_a_pair_added_to_a_superseded_object_entry_fails(self, repo: Path) -> None: - write_schema_lock(repo, V2) - commit(repo) - columns = [*V2["2"]["columns"], ["pull_requests", "title"]] - write_schema_lock(repo, {**V2, "2": {**V2["2"], "columns": columns}, "3": V2["2"]}) - result = check(repo) - assert result.returncode == 1 - assert ( - "schema version 2 gained pull_requests.title; only the newest version, 3, gains " - "columns" in result.stderr - ) - - @pytest.mark.parametrize( - ("current", "message"), - [ - pytest.param( - {**with_views({**V2["2"]["views"], "v_pr_reviews": "c" * 64}), "3": V2["2"]}, - "schema version 2 changed view v_pr_reviews", - id="changed digest", - ), - pytest.param( - {**with_views({"v_pr_reviews": "a" * 64}), "3": V2["2"]}, - "schema version 2 lost view v_github_users", - id="removed view", - ), - pytest.param( - {**V2, "1": [["pull_requests", "number"]], "3": V2["2"]}, - "schema version 1 lost v_pr_reviewers.final_verdict", - id="removed pair", - ), - ], - ) - def test_an_edit_to_a_superseded_entry_fails( - self, repo: Path, current: dict[str, Any], message: str - ) -> None: - """An entry stays fixed once a newer one is added.""" - write_schema_lock(repo, V2) - commit(repo) - write_schema_lock(repo, current) - result = check(repo) - assert result.returncode == 1 - assert message in result.stderr - - @pytest.mark.parametrize( - ("views", "message"), - [ - pytest.param( - {**V2["2"]["views"], "v_pr_reviews": "d" * 64}, - "schema version 2 changed view v_pr_reviews", - id="changed digest", - ), - pytest.param( - {"v_pr_reviews": "a" * 64}, - "schema version 2 lost view v_github_users", - id="removed view", - ), - ], - ) - def test_a_view_edit_below_the_newest_entry_at_the_base_fails( - self, repo: Path, views: dict[str, str], message: str - ) -> None: - """Views are kept in every base entry, not only in the base's newest one.""" - write_schema_lock(repo, V3) - commit(repo) - write_schema_lock(repo, {**V3, "2": {**V3["2"], "views": views}}) - result = check(repo) - assert result.returncode == 1 - assert message in result.stderr - def test_a_version_has_at_most_nine_digits(self, repo: Path) -> None: write_schema_lock(repo, V1) commit(repo) @@ -421,40 +367,6 @@ def test_a_view_added_to_the_newest_entry_passes(self, repo: Path) -> None: result = check(repo) assert result.returncode == 0, result.stderr - def test_a_changed_view_digest_fails(self, repo: Path) -> None: - write_schema_lock(repo, V2) - commit(repo) - write_schema_lock(repo, with_views({**V2["2"]["views"], "v_pr_reviews": "c" * 64})) - result = check(repo) - assert result.returncode == 1 - assert "schema version 2 changed view v_pr_reviews" in result.stderr - - def test_a_removed_view_fails(self, repo: Path) -> None: - write_schema_lock(repo, V2) - commit(repo) - write_schema_lock(repo, with_views({"v_pr_reviews": "a" * 64})) - result = check(repo) - assert result.returncode == 1 - assert "schema version 2 lost view v_github_users" in result.stderr - - def test_an_entry_rewritten_as_a_list_loses_its_views(self, repo: Path) -> None: - write_schema_lock(repo, V2) - commit(repo) - write_schema_lock(repo, {**V2, "2": V2["2"]["columns"]}) - result = check(repo) - assert result.returncode == 1 - assert "schema version 2 lost view v_github_users" in result.stderr - assert "schema version 2 lost view v_pr_reviews" in result.stderr - - def test_a_view_added_to_a_superseded_entry_fails(self, repo: Path) -> None: - write_schema_lock(repo, V2) - commit(repo) - superseded = {"columns": V1["1"], "views": {"v_late": "d" * 64}} - write_schema_lock(repo, {**V2, "1": superseded, "3": V2["2"]}) - result = check(repo) - assert result.returncode == 1 - assert "schema version 1 gained view v_late; only the newest version, 3" in result.stderr - @pytest.mark.parametrize( "entry", [ @@ -550,30 +462,26 @@ def test_a_key_listed_twice_is_an_error_not_a_pass( assert "is not JSON" not in result.stderr def test_both_locks_are_checked_by_default(self, repo: Path) -> None: - write_lock(repo, {"pull_request:1": "a" * 64}) - write_schema_lock(repo, V1) + place(repo, LOCK, QUERY_LOCKS[1]) + place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[2]) commit(repo) - write_lock(repo, {"pull_request:1": "c" * 64}) - write_schema_lock(repo, {"1": []}) + place(repo, LOCK, QUERY_LOCKS[0]) + place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[1]) result = check(repo) assert result.returncode == 1 - assert "pull_request:1 changed" in result.stderr - assert "schema version 1 lost pull_requests.number" in result.stderr + stderr = result.stderr.splitlines() + assert [line.split(":")[0] for line in stderr] == [LOCK, SCHEMA_LOCK] def test_lock_selects_one(self, repo: Path) -> None: - write_lock(repo, {"pull_request:1": "a" * 64}) - write_schema_lock(repo, V1) + place(repo, LOCK, QUERY_LOCKS[1]) + place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[2]) commit(repo) - write_lock(repo, {"pull_request:1": "c" * 64}) - result = subprocess.run( - [sys.executable, str(SCRIPT), "--base", "main", "--lock", SCHEMA_LOCK], - cwd=repo, - capture_output=True, - text=True, - timeout=60, - ) + place(repo, LOCK, QUERY_LOCKS[0]) + assert check(repo).returncode == 1 + result = run_checker(repo, "--base", "main", "--lock", SCHEMA_LOCK) assert result.returncode == 0, result.stderr - assert "query_versions" not in result.stdout + result.stderr + assert result.stdout.splitlines() == [kept(SCHEMA_LOCK, SCHEMA_LOCKS[2], SCHEMA_LOCKS[2])] + assert result.stderr == "" def test_the_repository_locks_are_read_by_their_rules(self, repo: Path) -> None: """The real lock files parse under the rule the checker gives each, in a scratch @@ -619,7 +527,7 @@ def lock_step_script() -> str: @pytest.fixture def origin(tmp_path: Path) -> Path: bare = tmp_path / "origin.git" - git(tmp_path, "init", "-q", "--bare", str(bare)) + git(tmp_path, "init", "-q", "--bare", "-b", "main", str(bare)) return bare @@ -631,12 +539,22 @@ def work_clone(tmp_path: Path, origin: Path, name: str) -> Path: return work -def push_main(work: Path, *, checker: bool, entries: dict[str, str] | None) -> None: +def push_main( + work: Path, + *, + checker: bool, + entries: dict[str, str] | None = None, + history: dict[str, str] | None = None, +) -> None: + """The base: the checker, a query lock of ``entries``, and each lock of ``history`` as it + was at its commit.""" if checker: (work / "scripts").mkdir(exist_ok=True) (work / "scripts" / "check_query_lock.py").write_text(SCRIPT.read_text()) if entries is not None: write_lock(work, entries) + for lock, at in (history or {}).items(): + place(work, lock, at) (work / "README").write_text("base\n") git(work, "add", "-A") git(work, "commit", "-q", "-m", "base") @@ -644,7 +562,7 @@ def push_main(work: Path, *, checker: bool, entries: dict[str, str] | None) -> N def run_step_on_merge( - tmp_path: Path, origin: Path, work: Path, change + tmp_path: Path, origin: Path, work: Path, change: Callable[[Path], None] ) -> subprocess.CompletedProcess[str]: """Commit ``change`` on a PR branch, push its merge into main as a ref, and run the step in a depth-1 checkout of that merge, as a pull_request run does.""" @@ -673,33 +591,39 @@ def run_step(checkout: Path, runner_temp: Path) -> subprocess.CompletedProcess[s cwd=checkout, capture_output=True, text=True, - env={**os.environ, "RUNNER_TEMP": str(runner_temp)}, + env={**environment(checkout), "RUNNER_TEMP": str(runner_temp)}, timeout=60, ) -def edit_entry(work: Path) -> None: - write_lock(work, {"pull_request:1": "c" * 64}) +# The base holds the later lock of each pair; the pull request puts back the earlier one. +LATER_LOCKS = {LOCK: QUERY_LOCKS[1], SCHEMA_LOCK: SCHEMA_LOCKS[2]} -def edit_entry_and_relax_checker(work: Path) -> None: - edit_entry(work) - checker = work / "scripts" / "check_query_lock.py" - checker.write_text("raise SystemExit(0)\n") +def put_back_the_earlier_query_lock(work: Path) -> None: + place(work, LOCK, QUERY_LOCKS[0]) -def test_the_step_fails_an_edited_entry(tmp_path: Path, origin: Path) -> None: +def test_the_step_fails_a_removed_entry(tmp_path: Path, origin: Path) -> None: work = work_clone(tmp_path, origin, "work") - push_main(work, checker=True, entries={"pull_request:1": "a" * 64}) - result = run_step_on_merge(tmp_path, origin, work, edit_entry) + push_main(work, checker=True, history=LATER_LOCKS) + result = run_step_on_merge(tmp_path, origin, work, put_back_the_earlier_query_lock) assert result.returncode == 1, result.stdout + result.stderr - assert "pull_request:1 changed" in result.stderr + (removed,) = entries_at(QUERY_LOCKS[1], LOCK).keys() - entries_at(QUERY_LOCKS[0], LOCK).keys() + assert f"{removed} was removed" in result.stderr def test_the_step_runs_the_base_checker_not_the_pull_requests(tmp_path: Path, origin: Path) -> None: + """The pull request also deletes its copy of the checker, which the step would then fail to + open, with another exit code than the base copy's verdict.""" work = work_clone(tmp_path, origin, "work") - push_main(work, checker=True, entries={"pull_request:1": "a" * 64}) - result = run_step_on_merge(tmp_path, origin, work, edit_entry_and_relax_checker) + push_main(work, checker=True, history=LATER_LOCKS) + + def put_back_and_delete_the_checker(target: Path) -> None: + put_back_the_earlier_query_lock(target) + (target / "scripts" / "check_query_lock.py").unlink() + + result = run_step_on_merge(tmp_path, origin, work, put_back_and_delete_the_checker) assert result.returncode == 1, result.stdout + result.stderr assert "runner-temp/check_query_lock.py" in result.stdout @@ -720,7 +644,7 @@ def test_the_step_uses_the_pull_requests_checker_where_the_base_has_none( ) -> None: # The first pull request with the lock: the base has neither the checker nor the lock. work = work_clone(tmp_path, origin, "work") - push_main(work, checker=False, entries=None) + push_main(work, checker=False) def add_lock(target: Path) -> None: (target / "scripts").mkdir(exist_ok=True) @@ -735,15 +659,14 @@ def add_lock(target: Path) -> None: def test_the_step_fails_a_pair_removed_from_a_schema_entry(tmp_path: Path, origin: Path) -> None: work = work_clone(tmp_path, origin, "work") - write_schema_lock(work, V1) - push_main(work, checker=True, entries={"pull_request:1": "a" * 64}) + push_main(work, checker=True, history=LATER_LOCKS) - def remove_pair(target: Path) -> None: - write_schema_lock(target, {"1": V1["1"][:1]}) + def put_back_the_earlier_schema_lock(target: Path) -> None: + place(target, SCHEMA_LOCK, SCHEMA_LOCKS[1]) - result = run_step_on_merge(tmp_path, origin, work, remove_pair) + result = run_step_on_merge(tmp_path, origin, work, put_back_the_earlier_schema_lock) assert result.returncode == 1, result.stdout + result.stderr - assert "schema version 1 lost v_pr_reviewers.final_verdict" in result.stderr + assert "schema version 2 lost sync_failures.last_run_id" in result.stderr def test_the_step_fails_without_a_merge_commit(tmp_path: Path, origin: Path) -> None: diff --git a/tests/test_data_dictionary.py b/tests/test_data_dictionary.py index eb59c59..b0938b8 100644 --- a/tests/test_data_dictionary.py +++ b/tests/test_data_dictionary.py @@ -3,12 +3,12 @@ from __future__ import annotations import hashlib -import json import sqlite3 from contextlib import closing from pathlib import Path import pytest +from alembic.script import ScriptDirectory from github_activity_db.db.models import Base from github_activity_db.db.schema import SCHEMA_VERSION @@ -28,9 +28,17 @@ read_lock, view_digest, ) -from tests.db.alembic_support import alembic_config +from tests.db.alembic_support import alembic_config, upgrade from tests.db.schema_support import frozen_views, views +# Earlier versions of repository files, copied unchanged (tests/fixtures/history/README.md). +HISTORY = Path(__file__).resolve().parent / "fixtures" / "history" +STEP3A = "33f8570f6540" +STEP3B1 = "94a52e4f1d57" +LAST_RUN_ID = "2d2ba76b177c" +# The commits that added 94a52e4f1d57 and 2d2ba76b177c, with the dictionary and lock of each. +AT_STEP3B1, AT_LAST_RUN_ID = "21a4267", "77663a7" + ENTRY = """\ ## Tables @@ -84,6 +92,26 @@ def entries() -> dict[str, dict[str, str]]: return read_dictionary(DICTIONARY.read_text()) +@pytest.fixture(scope="module") +def revisions(tmp_path_factory: pytest.TempPathFactory) -> dict[str, dict[str, list[str]]]: + """The tables and views, with their columns, of databases the migrations built at Step 3a, + at Step 3b-1, and at the revision after it.""" + folder = tmp_path_factory.mktemp("revisions") + columns = {} + for revision in (STEP3A, STEP3B1, LAST_RUN_ID): + upgrade(folder / f"{revision}.db", revision) + columns[revision] = database_columns(folder / f"{revision}.db") + return columns + + +def dictionary_at(commit: str) -> dict[str, dict[str, str]]: + return read_dictionary((HISTORY / commit / "docs" / "data-dictionary.md").read_text()) + + +def lock_at(commit: str) -> dict[str, LockEntry]: + return parse_lock((HISTORY / commit / "tests" / "schema_versions.json").read_text()) + + class TestDictionary: def test_every_object_and_column_at_head_has_an_entry_and_none_is_orphaned( self, entries: dict[str, dict[str, str]], head_template: Path @@ -171,21 +199,31 @@ def test_row_ids_are_consumer_exactly_where_consumers_join_on_them( expected = "consumer" if (table.name, column.name) in joins else "internal" assert entries[table.name][column.name] == expected, (table.name, column.name) - @pytest.mark.parametrize( - ("database", "expected"), - [ - ({"t": ["id", "name"], "v_t": ["name"]}, []), - ({"t": ["id", "name", "added"], "v_t": ["name"]}, ["no entry for t.added"]), - ({"t": ["id"], "v_t": ["name"]}, ["t.name is not at head"]), - ({"t": ["id", "name"], "v_t": ["name"], "v_new": ["x"]}, ["no entry for v_new"]), - ({"t": ["id", "name"]}, ["v_t is not at head"]), - ], - ids=["equal", "column at head only", "entry only", "view at head only", "view entry only"], - ) + def test_each_commit_s_dictionary_matched_its_schema( + self, revisions: dict[str, dict[str, list[str]]] + ) -> None: + assert differences(dictionary_at(AT_STEP3B1), revisions[STEP3B1]) == [] + assert differences(dictionary_at(AT_LAST_RUN_ID), revisions[LAST_RUN_ID]) == [] + def test_differences_are_reported_both_ways( - self, database: dict[str, list[str]], expected: list[str] + self, revisions: dict[str, dict[str, list[str]]], head_template: Path ) -> None: - assert differences(read_dictionary(ENTRY), database) == expected + """Each commit's dictionary against another revision's database: a column only one of + them has, and the views Step 3b-1 drops and the tables and views it adds.""" + assert differences(dictionary_at(AT_LAST_RUN_ID), revisions[STEP3B1]) == [ + "sync_failures.last_run_id is not at head" + ] + assert differences(dictionary_at(AT_STEP3B1), revisions[LAST_RUN_ID]) == [ + "no entry for sync_failures.last_run_id" + ] + script = ScriptDirectory.from_config(alembic_config(head_template)) + dropped = script.get_revision(STEP3B1).module.DROPPED_VIEWS + added = revisions[STEP3B1].keys() - revisions[STEP3A].keys() + found = differences(dictionary_at(AT_STEP3B1), revisions[STEP3A]) + assert [line for line in found if "." not in line] == [ + *(f"no entry for {name}" for name in sorted(dropped)), + *(f"{name} is not at head" for name in sorted(added)), + ] @pytest.mark.parametrize( ("text", "message"), @@ -236,7 +274,7 @@ def test_rejects_shapes_it_does_not_model(self, text: str, message: str) -> None def test_database_columns_reads_every_user_object(tmp_path: Path) -> None: """Names that look like SQLite's own are kept, SQLite's own are not, generated columns are listed.""" - path = tmp_path / "planted.db" + path = tmp_path / "scratch.db" with closing(sqlite3.connect(path)) as connection: connection.execute("CREATE TABLE sqliteXstats (x INTEGER)") connection.execute("CREATE TABLE SQLite1 (x INTEGER)") @@ -289,55 +327,30 @@ def test_a_key_listed_twice_is_refused(self, text: str) -> None: def test_the_lock_reads_without_repeated_keys(self) -> None: assert parse_lock('{"1": [["t", "c"]]}') == {"1": ({("t", "c")}, {})} - @pytest.mark.parametrize( - ("before", "after"), - [ - pytest.param( - "CREATE VIEW v AS SELECT 'a b' AS value", - "CREATE VIEW v AS SELECT 'a b' AS value", - id="whitespace in a literal", - ), - pytest.param( - "CREATE VIEW v AS SELECT 1 AS value -- explanation\nWHERE 0", - "CREATE VIEW v AS SELECT 1 AS value -- explanation WHERE 0", - id="the end of a line comment", - ), - ], - ) - def test_a_view_returning_other_rows_fails_the_lock(self, before: str, after: str) -> None: - """Two definitions that differ only in whitespace but return different rows get different - digests, so the checker reports the view as changed.""" - - def rows(sql: str) -> list[tuple[object, ...]]: - with closing(sqlite3.connect(":memory:")) as connection: - connection.execute(sql) - return connection.execute("SELECT * FROM v").fetchall() - - def entry(sql: str) -> dict[str, LockEntry]: - return parse_lock( - json.dumps({"2": {"columns": [["v", "value"]], "views": {"v": view_digest(sql)}}}) - ) - - assert rows(before) != rows(after) - assert lock_checker().schema_problems(entry(before), entry(after)) == [ - "schema version 2 changed view v; a change to a view's rows or derivation needs a " - "new schema version" - ] - - def test_the_digest_is_of_the_exact_sql(self) -> None: + def test_the_digest_is_of_the_exact_sql(self, head_template: Path) -> None: sql = "CREATE VIEW v AS SELECT a FROM t" assert view_digest(sql) == hashlib.sha256(sql.encode()).hexdigest() assert view_digest(sql) != view_digest("CREATE VIEW v AS\n SELECT a\nFROM t") + # Each view at head, its SQL hashed as the revision embeds it, nothing normalized. + frozen = frozen_views(alembic_config(head_template)) + assert len(frozen) == len(views(head_template)) + assert {name: view_digest(sql) for name, sql in frozen.items()} == { + name: hashlib.sha256(sql.encode()).hexdigest() for name, sql in frozen.items() + } - def test_a_listed_pair_that_is_not_consumer_at_head_is_reported(self) -> None: - consumer = consumer_pairs(read_dictionary(ENTRY)) - entry = [*consumer, ["t", "id"], ["v_t", "gone"]] - assert lock_differences(entry, consumer) == [ - "t.id is not a consumer column at head", - "v_t.gone is not a consumer column at head", + def test_each_commit_s_lock_entry_matched_its_dictionary(self) -> None: + for commit in (AT_STEP3B1, AT_LAST_RUN_ID): + entry = lock_columns(lock_at(commit)["2"]) + assert lock_differences(entry, consumer_pairs(dictionary_at(commit))) == [], commit + + def test_lock_differences_are_reported_both_ways(self) -> None: + """Schema version 2 as one commit's lock lists it, against the other commit's consumer + columns: 2d2ba76b177c added sync_failures.last_run_id to both.""" + older = lock_columns(lock_at(AT_STEP3B1)["2"]) + newer = lock_columns(lock_at(AT_LAST_RUN_ID)["2"]) + assert lock_differences(older, consumer_pairs(dictionary_at(AT_LAST_RUN_ID))) == [ + "sync_failures.last_run_id is missing from the entry" + ] + assert lock_differences(newer, consumer_pairs(dictionary_at(AT_STEP3B1))) == [ + "sync_failures.last_run_id is not a consumer column at head" ] - - def test_a_consumer_pair_missing_from_the_entry_is_reported(self) -> None: - consumer = consumer_pairs(read_dictionary(ENTRY)) - assert consumer == [["t", "name"], ["v_t", "name"]] - assert lock_differences([["t", "name"]], consumer) == ["v_t.name is missing from the entry"] diff --git a/tests/test_db_engine.py b/tests/test_db_engine.py index f6564ef..7772084 100644 --- a/tests/test_db_engine.py +++ b/tests/test_db_engine.py @@ -98,34 +98,50 @@ async def test_connect_needs_no_database_lock(self, tmp_path: Path) -> None: assert foreign_keys == 1 - def test_listener_failure_closes_the_connection(self) -> None: - class FailingCursor: + def test_an_interrupt_in_the_listener_closes_the_connection(self) -> None: + """An interrupt at a set point, the listener's first statement: the connection is + closed before the interrupt goes on, since SQLAlchemy would not close it.""" + executed: list[str] = [] + + class InterruptedCursor: + closed = False + def execute(self, statement: str) -> None: - raise sqlite3.OperationalError("database is locked") + executed.append(statement) + raise KeyboardInterrupt def close(self) -> None: - pass + self.closed = True class Connection: closed = False - def cursor(self) -> FailingCursor: - return FailingCursor() + def __init__(self) -> None: + self.cursors: list[InterruptedCursor] = [] + + def cursor(self) -> InterruptedCursor: + self.cursors.append(InterruptedCursor()) + return self.cursors[-1] def close(self) -> None: self.closed = True connection = Connection() - with pytest.raises(sqlite3.OperationalError, match="database is locked"): + with pytest.raises(KeyboardInterrupt): _apply_sqlite_pragmas(connection, None) + assert executed == ["PRAGMA foreign_keys=ON"] + assert [cursor.closed for cursor in connection.cursors] == [True] assert connection.closed async def test_foreign_key_violation_is_rejected(self, test_engine) -> None: - with pytest.raises(IntegrityError, match="FOREIGN KEY constraint failed"): + with pytest.raises(IntegrityError) as error: async with test_engine.begin() as conn: await conn.execute( text("INSERT INTO pr_user_tags (pr_id, user_tag_id) VALUES (999, 999)") ) + driver_error = error.value.orig + assert isinstance(driver_error, sqlite3.IntegrityError), repr(driver_error) + assert driver_error.sqlite_errorname == "SQLITE_CONSTRAINT_FOREIGNKEY" async def test_on_delete_cascade_removes_children(self, test_engine) -> None: async with test_engine.begin() as conn: From 952aa5331c2d97e01b0cb5a111890158909a902b Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 08:00:36 -0600 Subject: [PATCH 03/29] Step 3b-2a-2d: sync tests without planted inputs (D39, D40) tests/sync/test_engine.py - A database error mid-batch, a close that fails, a failed close after step 7, and an interrupted close whose own commit fails now come from another connection holding a read open (SQLite's SHARED lock) through the busy timeout, not from triggers or patched writer calls. A handle_error listener records each error's sqlite_errorname (SQLITE_BUSY) and, where the run goes on to close, ends the read first. The engine's connections wait 100 ms instead of 30 s (db.engine.SQLITE_BUSY_TIMEOUT_MS, a short_busy_timeout fixture). - The run ending after step 7 prunes a real untracked repository; the interrupt is raised at the prune's DELETE, and the fatal case fails the prune's commit on the held read. Both keep the halt reason, and the prune deletes nothing. - The AccountMissing test that patched write_pull_request is replaced by a passing-side test: commit links whose account was sighted by the item, stored earlier, sighted in the batch, found by a lookup, and answered NOT_FOUND, with the linked pr_commits rows compared to the linked commits counted from the scenario model. - The RuntimeError test that patched apply_in_place_updates is removed; the same except arm is held by the recorded 401 and by the database error above. The 401 test checks the error's class field. - A database not at head is a file built by the migrations to the revision before head (MIGRATION_REVISIONS[-2]), not an edited alembic_version. tests/sync/engine_support.py - World.expects_account_missing is removed with the patched test; the world fixture's AccountMissing check has no exemption. tests/sync/test_runs.py - The user_version test is removed; the head check's passing side reads alembic_version and PRAGMA user_version on its own connection and compares them with alembic's script head and the newest key of tests/schema_versions.json. - The claim's head refusal uses a file at the revision before head. - The held-lock claim asserts sqlite_errorname SQLITE_BUSY and the connection's busy_timeout instead of elapsed time; the claim-in- progress test commits on an event set at the claim's BEGIN IMMEDIATE instead of sleeping, and asserts busy_timeout instead of elapsed time. - The refused stale reclaim drops its message substring; blocking is asserted. - git runs with HOME and GIT_CEILING_DIRECTORIES at tmp_path, no global or system config, no inherited GIT_* variables, fixed author and committer dates, and an environment built from PATH and fixed values. tests/sync/test_writer.py - The unchanged-default-branch test records the statements the write sends and asserts none updates pull_requests, over valid rows; the changing side asserts the one update it sends. - Each ItemConflict refusal also asserts the error's number field. - "The controls" in a docstring is reworded. tests/sync/test_snapshot.py - The digest's positive control is removed; test_writes_nothing compares every byte of a file whose size equals page_count * page_size and whose pull requests the reader read. - The half-read cursor control is removed; the read count comes from the statements recorded as they reach the database, one per table, and the probe's refusing side is another connection's open read. tests/sync/test_connection.py - The CHECK-violating insert is replaced by a BEGIN IMMEDIATE that fails on another connection's write lock; lock errors assert sqlite_errorname SQLITE_BUSY instead of message text. - The body's own KeyError no longer says "planted". tests/test_cli_sync.py - A database not at head is a file at the revision before head. - The replay fixture sets GITHUB_SHA and unsets GITHUB_EVENT_NAME, so no git runs and the trigger is fixed; the first sync and the option test assert the trigger and code version, and the run's counters are asserted in sync_runs. - Child processes get an environment of PATH and fixed values (HOME is the test's directory, GITHUB_SHA set); the child setup hook is renamed from plant to setup. tests/test_cli_app.py - The traceback test no longer adds a failing command to the app; it asserts the console script loads the root app and that app's pretty_exceptions_show_locals is False. - The exit-code test apps are renamed from planted to commands. - Installed-script children get PATH and HOME (the test's directory) only. --- tests/sync/engine_support.py | 2 - tests/sync/test_connection.py | 47 ++++-- tests/sync/test_engine.py | 287 +++++++++++++++++++++++----------- tests/sync/test_runs.py | 145 ++++++++++++----- tests/sync/test_snapshot.py | 89 ++++++----- tests/sync/test_writer.py | 82 ++++++++-- tests/test_cli_app.py | 51 +++--- tests/test_cli_sync.py | 94 +++++++---- 8 files changed, 542 insertions(+), 255 deletions(-) diff --git a/tests/sync/engine_support.py b/tests/sync/engine_support.py index 819e910..beaf4c6 100644 --- a/tests/sync/engine_support.py +++ b/tests/sync/engine_support.py @@ -28,8 +28,6 @@ def __init__(self, scenario: Scenario, database: Path) -> None: self.sync_settings: dict[str, Any] = {} self.automation: dict[str, Any] = {} self.floor = FLOOR - # Only the test that fakes the writer's refusal may leave an AccountMissing failure. - self.expects_account_missing = False # Options a parametrized case gives both a dry run and the run. self.options: dict[str, Any] = {} diff --git a/tests/sync/test_connection.py b/tests/sync/test_connection.py index 615f5fb..48b6b43 100644 --- a/tests/sync/test_connection.py +++ b/tests/sync/test_connection.py @@ -7,7 +7,8 @@ from pathlib import Path import pytest -from sqlalchemy import insert, select, text, update +from sqlalchemy import select, text, update +from sqlalchemy.exc import DBAPIError from github_activity_db.db.models import SyncRun from github_activity_db.sync import connection as connection_module @@ -41,7 +42,7 @@ def _write_is_refused(database: Path) -> bool: other.execute("ROLLBACK") return False except sqlite3.OperationalError as e: - assert "locked" in str(e) + assert e.sqlite_errorname == "SQLITE_BUSY" return True finally: other.close() @@ -109,20 +110,41 @@ async def test_the_body_s_exception_rolls_back(migrated_database: Path) -> None: with pytest.raises(KeyError): async with transaction(writer, run_id): await writer.execute(update(SyncRun).values(rest_calls=7)) - raise KeyError("planted") + raise KeyError("the body's own error") assert fetch(migrated_database, "SELECT rest_calls FROM sync_runs") == [{"rest_calls": 0}] -async def test_a_constraint_failure_is_a_local_database_error(migrated_database: Path) -> None: +def _sqlite_errorname(error: LocalDatabaseError) -> str: + """The SQLite error under the ``DBAPIError`` the writer turned into ``error``.""" + cause = error.__cause__ + assert isinstance(cause, DBAPIError) + assert isinstance(cause.orig, sqlite3.Error) + return cause.orig.sqlite_errorname + + +async def test_a_begin_that_fails_on_a_lock_is_a_local_database_error( + migrated_database: Path, +) -> None: + """Another process holds the write lock through the busy timeout: the BEGIN IMMEDIATE fails, + which is a LocalDatabaseError, the body never runs, and the next transaction on the same + connection runs once that process has gone.""" run_id = _run(migrated_database) - async with open_writer(_url(migrated_database)) as writer: - with pytest.raises(LocalDatabaseError, match="IntegrityError"): - async with transaction(writer, run_id): - await writer.execute(update(SyncRun).values(rest_calls=5)) - await writer.execute(insert(SyncRun).values(status="bogus")) + entered: list[str] = [] + async with open_writer(_url(migrated_database), busy_timeout_ms=50) as writer: + holder = sqlite3.connect(migrated_database, isolation_level=None) + holder.execute("BEGIN IMMEDIATE") + try: + with pytest.raises(LocalDatabaseError) as raised: + async with transaction(writer, run_id): + entered.append("body") + finally: + holder.execute("ROLLBACK") + holder.close() async with transaction(writer, run_id): - rest_calls = (await writer.execute(select(SyncRun.rest_calls))).scalar_one() - assert rest_calls == 0 + await writer.execute(update(SyncRun).values(rest_calls=5)) + assert _sqlite_errorname(raised.value) == "SQLITE_BUSY" + assert entered == [] + assert fetch(migrated_database, "SELECT rest_calls FROM sync_runs") == [{"rest_calls": 5}] async def test_a_commit_that_fails_on_a_lock_leaves_the_connection_usable( @@ -137,12 +159,13 @@ async def test_a_commit_that_fails_on_a_lock_leaves_the_connection_usable( reader.execute("BEGIN") assert reader.execute("SELECT count(*) FROM sync_runs").fetchone() == (1,) try: - with pytest.raises(LocalDatabaseError, match="database is locked"): + with pytest.raises(LocalDatabaseError) as raised: async with transaction(writer, run_id): await writer.execute(update(SyncRun).values(rest_calls=3)) finally: reader.execute("ROLLBACK") reader.close() + assert _sqlite_errorname(raised.value) == "SQLITE_BUSY" async with transaction(writer, run_id): await writer.execute(update(SyncRun).values(items_planned=4)) assert fetch(migrated_database, "SELECT rest_calls, items_planned FROM sync_runs") == [ diff --git a/tests/sync/test_engine.py b/tests/sync/test_engine.py index e747944..a1eb4d7 100644 --- a/tests/sync/test_engine.py +++ b/tests/sync/test_engine.py @@ -12,7 +12,8 @@ import hashlib import json import sqlite3 -from collections.abc import Iterator +from collections.abc import Callable, Iterator +from contextlib import contextmanager from datetime import timedelta from pathlib import Path from typing import Any, Final @@ -21,15 +22,18 @@ from sqlalchemy import event from sqlalchemy.engine import Engine +from github_activity_db.db import engine as db_engine +from github_activity_db.db.schema import HEAD_REVISION, MIGRATION_REVISIONS from github_activity_db.source import PULL_REQUEST_HYDRATION_VERSION -from github_activity_db.sync import writer from github_activity_db.sync.deletions import DeletionApproval, parse_allow_deletions from github_activity_db.sync.plan import Reason from github_activity_db.sync.runs import ClaimRefused, HeadCheckError from github_activity_db.sync.writer import Pruned +from tests.db.alembic_support import upgrade from tests.source.replay import synthetic from tests.source.scenario import Scenario from tests.sync.engine_support import World, ended, outcomes, planned, project +from tests.sync.records import fetch @pytest.fixture @@ -38,8 +42,54 @@ def world(scenario: Scenario, migrated_database: Path) -> Iterator[World]: commit links (plan 5.2).""" world = World(scenario, migrated_database) yield world - if not world.expects_account_missing: - assert world.rows("SELECT number FROM sync_failures WHERE last_error LIKE 'Account%'") == [] + assert world.rows("SELECT number FROM sync_failures WHERE last_error LIKE 'Account%'") == [] + + +@pytest.fixture +def short_busy_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + """Connections the engine opens wait 100 ms, not 30 s, on a lock another process holds; a + lock held through the wait fails the statement or commit the same way at either length.""" + monkeypatch.setattr(db_engine, "SQLITE_BUSY_TIMEOUT_MS", 100) + + +class Reader: + """Another process reading the database. Its open read transaction holds SQLite's SHARED + lock, which a writer's commit waits on until the busy timeout (rollback journal, gap 77).""" + + def __init__(self, database: Path) -> None: + self.database = database + self.connection: sqlite3.Connection | None = None + + def hold(self) -> None: + connection = sqlite3.connect(self.database, isolation_level=None) + connection.execute("BEGIN") + connection.execute("SELECT count(*) FROM sync_runs").fetchone() + self.connection = connection + + def release(self) -> None: + if self.connection is not None: + self.connection.execute("ROLLBACK") + self.connection.close() + self.connection = None + + +@contextmanager +def database_errors(after: Callable[[], None] = lambda: None) -> Iterator[list[str]]: + """The ``sqlite_errorname`` of each SQLite error the engine's connections meet, in order; + ``after`` runs once each is recorded, before the error reaches the engine.""" + names: list[str] = [] + + def record(context: Any) -> None: + error = context.original_exception + if isinstance(error, sqlite3.Error): + names.append(error.sqlite_errorname) + after() + + event.listen(Engine, "handle_error", record) + try: + yield names + finally: + event.remove(Engine, "handle_error", record) def batches(scenario: Scenario) -> list[list[int]]: @@ -1061,26 +1111,33 @@ async def test_a_repository_gone_during_hydration_fails_for_the_rest_of_the_run( async def test_a_database_error_mid_batch_is_fatal_and_the_next_run_resumes( - world: World, + world: World, short_busy_timeout: None ) -> None: - repository, _ = project(world, pulls=2) - world.execute( - "CREATE TRIGGER refuse_second BEFORE INSERT ON pull_requests WHEN NEW.number = 2 " - "BEGIN SELECT RAISE(ABORT, 'refused by the test'); END" - ) + """Another process starts reading once the second batch is answered and reads through the + busy timeout: #26's commit fails, which is fatal (5.3 step 4). The close, after that reader + has gone, writes failed; the first batch's items stay, and the next run plans the rest.""" + scenario = world.scenario + repository, _ = project(world, pulls=26) + reader = Reader(world.database) + scenario.after("PullRequestBatch", reader.hold, when=lambda seen: seen.numbers == [26]) - failed = await world.sync() - assert failed.error is not None - assert failed.error.startswith("LocalDatabaseError: IntegrityError: refused by the test") + try: + with database_errors(reader.release) as errors: + failed = await world.sync() + finally: + reader.release() + + assert batches(scenario) == [list(range(1, 26)), [26]] + assert errors == ["SQLITE_BUSY"] + assert failed.error == "LocalDatabaseError: OperationalError: database is locked" assert ended(world, failed) == ("failed", 1, {"prebid/Prebid.js": "failed"}) - assert world.numbers(repository) == [1] + assert world.numbers(repository) == list(range(1, 26)) run = world.run(failed.run_id) - assert (run["items_planned"], run["items_hydrated"]) == (2, 1) + assert (run["items_planned"], run["items_hydrated"]) == (26, 25) - world.execute("DROP TRIGGER refuse_second") resumed = await world.sync() - assert planned(resumed) == [2] - assert world.numbers(repository) == [1, 2] + assert planned(resumed) == [26] + assert world.numbers(repository) == list(range(1, 27)) assert ended(world, resumed) == ("complete", 0, {"prebid/Prebid.js": "complete"}) @@ -1094,47 +1151,51 @@ async def test_a_fatal_source_error_closes_the_run_failed(world: World) -> None: report = await world.sync() - assert report.error is not None and report.error.startswith("Unauthorized") + assert report.error is not None and report.error.split(":")[0] == "Unauthorized" assert ended(world, report) == ("failed", 1, {"prebid/Prebid.js": "failed"}) state = world.rows("SELECT last_index_started_at, index_pr_total FROM repo_sync_state") assert state == [{"last_index_started_at": None, "index_pr_total": None}] -async def test_any_other_exception_after_the_claim_closes_the_run_failed( - world: World, monkeypatch: pytest.MonkeyPatch -) -> None: - project(world) - - async def broken(*args: Any, **kwargs: Any) -> None: - raise RuntimeError("broken in-place update") - - monkeypatch.setattr(writer, "apply_in_place_updates", broken) - - report = await world.sync() - - assert report.error == "RuntimeError: broken in-place update" - assert ended(world, report) == ("failed", 1, {"prebid/Prebid.js": "failed"}) - - -async def test_an_account_missing_refusal_is_an_item_failure( - world: World, monkeypatch: pytest.MonkeyPatch -) -> None: - world.expects_account_missing = True - repository, _ = project(world, pulls=2) - write = writer.write_pull_request - - async def refusing(connection: Any, run_id: int, record: Any, **kwargs: Any) -> Any: - if record.number == 1: - raise writer.AccountMissing(record.number, [7]) - return await write(connection, run_id, record, **kwargs) - - monkeypatch.setattr(writer, "write_pull_request", refusing) +async def test_every_commit_link_reaches_the_writer_with_its_account(world: World) -> None: + """Commit links whose account the item sighted itself, an earlier run stored, another item + of the batch sighted, a lookup found, and a lookup answered NOT_FOUND for: the engine gives + the writer each account it does not store, so every linked commit is written and none fails + with ``AccountMissing`` (plan 5.2). The count of linked commits comes from the model.""" + scenario = world.scenario + repository, author = project(world, pulls=3) + stored = scenario.add_account() + repository.pulls[1].add("reviews", scenario.node("reviews", by=stored)) + await world.sync() + reviewer = scenario.add_account() + bot = scenario.add_account("Bot", login="helper") + gone = scenario.add_account() + scenario.delete_account(gone) + repository.pulls[2].add("reviews", scenario.node("reviews", by=reviewer)) + repository.pulls[3].add( + "commits", + *(scenario.node("commits", by=account) for account in (author, stored, reviewer, bot)), + ) + repository.pulls[3].add("commits", scenario.node("commits", by=gone)) report = await world.sync() - assert world.failures() == [("prebid/Prebid.js", 1, "AccountMissing")] - assert world.numbers(repository) == [2] - assert ended(world, report) == ("complete", 2, {"prebid/Prebid.js": "complete"}) + linked = [ + node["commit"]["author"]["user"]["databaseId"] + for pull in repository.pulls.values() + for node in pull.node["commits"]["nodes"] + if node["commit"]["author"]["user"] is not None + ] + assert len(linked) == 5 + assert batches(scenario)[-1] == [2, 3] + assert requests(scenario, "UserLookup") == 1 + assert world.rows( + "SELECT linked_github_id FROM pr_commits WHERE linked_github_id IS NOT NULL " + "ORDER BY linked_github_id" + ) == [{"linked_github_id": github_id} for github_id in sorted(linked)] + assert report.failures == () + assert world.failures() == [] + assert ended(world, report) == ("complete", 0, {"prebid/Prebid.js": "complete"}) # -------------------------------------------------------------------------------------------- @@ -1506,32 +1567,58 @@ async def test_a_run_another_process_aborted_exits_1_and_stays_aborted(world: Wo @pytest.mark.parametrize("ending", ["fatal", "interrupt"]) async def test_a_run_ending_after_step_7_keeps_its_halt_reason( - world: World, monkeypatch: pytest.MonkeyPatch, ending: str + world: World, short_busy_timeout: None, ending: str ) -> None: + """The prune of an untracked repository comes after step 7. Another process that starts + reading at the prune's delete and reads through the busy timeout fails the prune's commit, + which closes the run failed; an interrupt at that delete closes it aborted. Either close + keeps step 7's halt, and the prune deletes nothing (5.3 step 8).""" repository, _ = project(world, pulls=26) + dropped, _ = project(world, name="dropped") + await world.sync() + world.untrack("prebid/dropped") await world.sync() for pull in repository.pulls.values(): pull.gone = True + reader = Reader(world.database) + deletes: list[str] = [] - async def failing_prune(*args: Any) -> Any: - if ending == "fatal": - raise RuntimeError("prune failed") - raise KeyboardInterrupt + def at_the_prune( + connection: Any, cursor: Any, statement: str, parameters: Any, *args: Any + ) -> None: + if not statement.startswith("DELETE FROM repositories"): + return + deletes.append(statement) + if ending == "interrupt": + raise KeyboardInterrupt + reader.hold() - monkeypatch.setattr(writer, "prune_repositories", failing_prune) + event.listen(Engine, "before_cursor_execute", at_the_prune) + try: + with database_errors(reader.release) as errors: + if ending == "fatal": + report = await world.sync(prune_untracked=True) + assert ended(world, report) == ("failed", 1, {"prebid/Prebid.js": "failed"}) + else: + with pytest.raises(KeyboardInterrupt): + await world.sync(prune_untracked=True) + assert ( + world.value("SELECT status FROM sync_runs ORDER BY id DESC LIMIT 1") + == "aborted" + ) + finally: + event.remove(Engine, "before_cursor_execute", at_the_prune) + reader.release() - if ending == "fatal": - report = await world.sync(prune_untracked=True) - assert ended(world, report) == ("failed", 1, {"prebid/Prebid.js": "failed"}) - else: - with pytest.raises(KeyboardInterrupt): - await world.sync(prune_untracked=True) - assert world.value("SELECT status FROM sync_runs ORDER BY id DESC LIMIT 1") == "aborted" + assert len(deletes) == 1 + assert errors == (["SQLITE_BUSY"] if ending == "fatal" else []) assert halt_reasons(world) == { "prebid/Prebid.js": "26 deletion candidates exceed the delete guard's 25 (26 stored " - "rows before the run)" + "rows before the run)", + "prebid/dropped": None, } assert len(world.numbers(repository)) == 26 + assert world.numbers(dropped) == [1] async def test_the_head_check_comes_before_the_claim_and_any_request( @@ -1549,14 +1636,20 @@ async def test_the_head_check_comes_before_the_claim_and_any_request( assert world.scenario.replay.seen == [] -async def test_a_database_not_at_head_starts_no_run(world: World) -> None: +async def test_a_database_not_at_head_starts_no_run(world: World, tmp_path: Path) -> None: + """A file the migrations built to the revision before head, as the code before it did.""" project(world) - world.execute("UPDATE alembic_version SET version_num = 'older'") - - with pytest.raises(HeadCheckError, match="not the code's head"): + older = tmp_path / "older.db" + upgrade(older, MIGRATION_REVISIONS[-2]) + world.url = f"sqlite+aiosqlite:///{older}" + + with pytest.raises( + HeadCheckError, + match=f"revision {MIGRATION_REVISIONS[-2]}, not the code's head {HEAD_REVISION}", + ): await world.sync() - assert world.rows("SELECT id FROM sync_runs") == [] + assert fetch(older, "SELECT count(*) AS n FROM sync_runs") == [{"n": 0}] assert world.scenario.replay.seen == [] @@ -2223,22 +2316,41 @@ async def test_the_refresh_looks_up_the_oldest_first(world: World) -> None: assert ended(world, report) == ("complete", 0, {"prebid/Prebid.js": "complete"}) -async def test_a_close_that_fails_leaves_the_row_running_and_exits_1(world: World) -> None: - project(world) - world.execute( - "CREATE TRIGGER refuse_close BEFORE UPDATE OF status ON sync_runs " - "BEGIN SELECT RAISE(ABORT, 'close refused by the test'); END" - ) +async def test_a_close_that_fails_leaves_the_row_running_and_exits_1( + world: World, short_busy_timeout: None +) -> None: + """Another process starts reading at the close's update and reads through the busy timeout: + the close's commit fails, the row stays running, and the run exits 1; the row then blocks + the next claim (5.3).""" + repository, _ = project(world) + reader = Reader(world.database) - report = await world.sync() + def at_the_close( + connection: Any, cursor: Any, statement: str, parameters: Any, *args: Any + ) -> None: + if statement.startswith("UPDATE sync_runs SET finished_at") and reader.connection is None: + reader.hold() + + event.listen(Engine, "before_cursor_execute", at_the_close) + try: + with database_errors() as errors: + report = await world.sync() + finally: + event.remove(Engine, "before_cursor_execute", at_the_close) + reader.release() + assert errors == ["SQLITE_BUSY"] run = world.run(report.run_id) assert (run["status"], run["exit_code"]) == ("running", None) assert (report.status, report.exit_code) == ("running", 1) assert report.error == ( f"the close failed, so run {report.run_id} stays running: LocalDatabaseError: " - "IntegrityError: close refused by the test" + "OperationalError: database is locked" ) + assert world.numbers(repository) == [1] + with pytest.raises(ClaimRefused) as refused: + await world.sync() + assert refused.value.blocking == (report.run_id,) async def test_halted_comes_before_time_limited(world: World) -> None: @@ -2637,17 +2749,14 @@ def lookups() -> list[int]: @pytest.mark.parametrize("cleanup", ["second interrupt", "database error"]) async def test_an_interrupted_close_that_does_not_land_leaves_the_row_running( - world: World, cleanup: str + world: World, short_busy_timeout: None, cleanup: str ) -> None: - """A second interrupt stops the interrupted close, and a database error in it is not raised - over the interrupt; either way the row stays running for the stale-row rule or + """A second interrupt stops the interrupted close, and a database error in it (another + process reading from the first interrupt on, through the busy timeout, fails its commit) is + not raised over the interrupt; either way the row stays running for the stale-row rule or --break-lock (5.3).""" project(world) - if cleanup == "database error": - world.execute( - "CREATE TRIGGER refuse_abort BEFORE UPDATE OF status ON sync_runs " - "WHEN NEW.status = 'aborted' BEGIN SELECT RAISE(ABORT, 'refused by the test'); END" - ) + reader = Reader(world.database) interrupted: list[object] = [] def interrupt( @@ -2659,17 +2768,21 @@ def interrupt( if status not in parameters or (status == "aborted" and cleanup != "second interrupt"): return interrupted.append(status) + if cleanup == "database error": + reader.hold() raise KeyboardInterrupt(status) event.listen(Engine, "before_cursor_execute", interrupt) try: - with pytest.raises(KeyboardInterrupt) as raised: + with database_errors() as errors, pytest.raises(KeyboardInterrupt) as raised: await world.sync() finally: event.remove(Engine, "before_cursor_execute", interrupt) + reader.release() expected = ["complete", "aborted"] if cleanup == "second interrupt" else ["complete"] assert interrupted == expected + assert errors == (["SQLITE_BUSY"] if cleanup == "database error" else []) # The interrupt re-raised is the latest one delivered. assert raised.value.args == (expected[-1],) assert world.rows("SELECT status, exit_code FROM sync_runs") == [ diff --git a/tests/sync/test_runs.py b/tests/sync/test_runs.py index b9f79eb..d41f1d3 100644 --- a/tests/sync/test_runs.py +++ b/tests/sync/test_runs.py @@ -3,21 +3,24 @@ from __future__ import annotations import hashlib +import json +import os import shutil import sqlite3 import subprocess import threading -import time from datetime import timedelta from pathlib import Path from typing import Any import pytest +from alembic.script import ScriptDirectory from sqlalchemy import event, text +from sqlalchemy.exc import DBAPIError from github_activity_db.config import Settings -from github_activity_db.db.schema import HEAD_REVISION -from github_activity_db.sync.connection import LocalDatabaseError, open_writer +from github_activity_db.db.schema import HEAD_REVISION, MIGRATION_REVISIONS +from github_activity_db.sync.connection import LocalDatabaseError, open_writer, transaction from github_activity_db.sync.runs import ( STALE_AFTER, ClaimRefused, @@ -36,7 +39,8 @@ trigger, ) from github_activity_db.tracking import read_tracking_config -from tests.db.alembic_support import upgrade +from tests.data_dictionary import SCHEMA_LOCK +from tests.db.alembic_support import alembic_config, upgrade from tests.db.seed import Seed, connect from tests.sync.records import T0, fetch, ts @@ -69,6 +73,18 @@ async def _claim(database: Path, **options: object) -> int: # Head check # -------------------------------------------------------------------------------------------- async def test_the_head_check_passes_at_head(migrated_database: Path) -> None: + """The 2 values the check compares, read on a connection of the test's own, are the chain's + head in ``alembic/versions`` and the newest schema version ``tests/schema_versions.json`` + records.""" + head = ScriptDirectory.from_config(alembic_config(migrated_database)).get_current_head() + newest = max(int(version) for version in json.loads(SCHEMA_LOCK.read_text())) + connection = sqlite3.connect(migrated_database) + try: + revisions = connection.execute("SELECT version_num FROM alembic_version").fetchall() + (user_version,) = connection.execute("PRAGMA user_version").fetchone() + finally: + connection.close() + assert (revisions, user_version) == ([(head,)], newest) await check_head(_url(migrated_database)) @@ -96,14 +112,6 @@ async def test_an_older_revision_is_refused(tmp_path: Path) -> None: await check_head(_url(older)) -async def test_another_user_version_is_refused(migrated_database: Path) -> None: - connection = sqlite3.connect(migrated_database) - connection.execute("PRAGMA user_version = 1") - connection.close() - with pytest.raises(HeadCheckError, match="user_version is 1, not schema_version 2"): - await check_head(_url(migrated_database)) - - @pytest.mark.parametrize( "url", ["sqlite+aiosqlite://", "sqlite+aiosqlite:///:memory:", "postgresql://host/db"] ) @@ -112,13 +120,16 @@ def test_a_url_without_a_file_is_refused(url: str) -> None: database_path(url) -async def test_the_claim_refuses_a_database_not_at_head(migrated_database: Path) -> None: - connection = sqlite3.connect(migrated_database) - connection.execute("PRAGMA user_version = 1") - connection.close() - with pytest.raises(HeadCheckError): - await _claim(migrated_database) - assert fetch(migrated_database, "SELECT count(*) AS n FROM sync_runs") == [{"n": 0}] +async def test_the_claim_refuses_a_database_not_at_head(tmp_path: Path) -> None: + """A file the migrations built to the revision before head, as the code before it did.""" + older = tmp_path / "older.db" + upgrade(older, MIGRATION_REVISIONS[-2]) + with pytest.raises( + HeadCheckError, + match=f"revision {MIGRATION_REVISIONS[-2]}, not the code's head {HEAD_REVISION}", + ): + await _claim(older) + assert fetch(older, "SELECT count(*) AS n FROM sync_runs") == [{"n": 0}] async def test_the_claim_checks_the_head_inside_its_transaction(migrated_database: Path) -> None: @@ -286,7 +297,6 @@ async def test_a_refused_claim_rolls_back_the_stale_reclaim(migrated_database: P with pytest.raises(ClaimRefused) as refused: await _claim(migrated_database) assert refused.value.blocking == (blocking,) - assert "--break-lock" in str(refused.value) assert fetch(migrated_database, "SELECT id, status FROM sync_runs ORDER BY id") == [ {"id": stale, "status": "running"}, {"id": blocking, "status": "running"}, @@ -327,15 +337,21 @@ async def test_break_lock_without_a_blocking_row_proceeds(migrated_database: Pat assert (claim.reclaimed, claim.broken) == ((), ()) +async def _busy_timeout(writer: Any) -> int: + """The writer connection's ``PRAGMA busy_timeout``, in milliseconds.""" + async with transaction(writer): + value: int = (await writer.exec_driver_sql("PRAGMA busy_timeout")).scalar_one() + return value + + async def test_a_lock_held_by_a_separate_connection(migrated_database: Path) -> None: - """An uncommitted BEGIN IMMEDIATE elsewhere: the claim waits out the busy timeout and fails - with SQLite's "database is locked", inserting nothing (plan 5.3, section 7 "Run claim").""" + """An uncommitted BEGIN IMMEDIATE elsewhere, held through the claim's busy timeout: the + claim fails with SQLite's SQLITE_BUSY, inserting nothing (plan 5.3, section 7 "Run claim").""" holder = sqlite3.connect(migrated_database, isolation_level=None) holder.execute("BEGIN IMMEDIATE") try: async with open_writer(_url(migrated_database), busy_timeout_ms=300) as writer: - started = time.monotonic() - with pytest.raises(LocalDatabaseError, match="database is locked"): + with pytest.raises(LocalDatabaseError) as raised: await claim_run( writer, started_at=T0, @@ -345,20 +361,26 @@ async def test_a_lock_held_by_a_separate_connection(migrated_database: Path) -> trigger="manual", code_version="x", ) - waited = time.monotonic() - started + holder.execute("ROLLBACK") + assert await _busy_timeout(writer) == 300 finally: - holder.execute("ROLLBACK") holder.close() - assert waited >= 0.3 + cause = raised.value.__cause__ + assert isinstance(cause, DBAPIError) + assert isinstance(cause.orig, sqlite3.Error) + assert cause.orig.sqlite_errorname == "SQLITE_BUSY" assert fetch(migrated_database, "SELECT count(*) AS n FROM sync_runs") == [{"n": 0}] async def test_a_claim_waits_for_a_claim_in_progress_then_refuses(migrated_database: Path) -> None: - """Another claimant inserts its running row and commits within the busy timeout: this - claim's BEGIN IMMEDIATE waits for that commit, then reads the row and refuses (section 7, - "Run claim").""" + """Another claimant inserts its running row and commits once this claim sends its BEGIN + IMMEDIATE, within the claim's 5 s busy timeout: the claim gets the lock after that commit, + reads the row, and refuses (section 7, "Run claim"). Whether the commit lands before + SQLite runs the BEGIN or while it waits, the claim reads the same committed row.""" holding = threading.Event() + begun = threading.Event() other: list[int] = [] + order: list[str] = [] def claimant() -> None: connection = sqlite3.connect(migrated_database, isolation_level=None) @@ -375,17 +397,23 @@ def claimant() -> None: assert cursor.lastrowid is not None other.append(cursor.lastrowid) holding.set() - time.sleep(0.3) - connection.execute("COMMIT") + if begun.wait(5): + order.append("commit") + connection.execute("COMMIT") finally: connection.close() + def beginning(*args: Any) -> None: + if args[2] == "BEGIN IMMEDIATE" and not begun.is_set(): + order.append("begin") + begun.set() + thread = threading.Thread(target=claimant) thread.start() try: assert holding.wait(5) async with open_writer(_url(migrated_database), busy_timeout_ms=5000) as writer: - started = time.monotonic() + event.listen(writer.sync_engine, "before_cursor_execute", beginning) with pytest.raises(ClaimRefused) as refused: await claim_run( writer, @@ -396,11 +424,13 @@ def claimant() -> None: trigger="manual", code_version="x", ) - waited = time.monotonic() - started + assert await _busy_timeout(writer) == 5000 finally: + begun.set() thread.join(5) + assert not thread.is_alive() + assert order == ["begin", "commit"] assert refused.value.blocking == tuple(other) - assert 0.2 <= waited < 5 assert fetch(migrated_database, "SELECT id, code_version FROM sync_runs") == [ {"id": other[0], "code_version": "other"} ] @@ -849,25 +879,56 @@ def test_trigger(event: str | None, expected: str) -> None: GIT = shutil.which("git") -def _git(directory: Path, *args: str) -> str: +@pytest.fixture +def git_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: + """git reads no configuration and finds no repository outside ``tmp_path``, and commits at a + fixed date. ``code_version`` runs git with this process's environment, so the variables are + set here too, and every other ``GIT_*`` variable is removed; the test's own git calls get + the returned environment, built from these values and ``PATH`` alone.""" + isolated = { + "HOME": str(tmp_path), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CEILING_DIRECTORIES": str(tmp_path), + } + for name in [name for name in os.environ if name.startswith("GIT_")]: + monkeypatch.delenv(name) + for name, value in isolated.items(): + monkeypatch.setenv(name, value) + fixed = "2026-01-01T00:00:00+00:00" + return { + "PATH": os.environ["PATH"], + **isolated, + "GIT_AUTHOR_DATE": fixed, + "GIT_COMMITTER_DATE": fixed, + } + + +def _git(directory: Path, env: dict[str, str], *args: str) -> str: assert GIT is not None, "git is not installed" result = subprocess.run( - [GIT, "-C", str(directory), *args], capture_output=True, text=True, timeout=30, check=True + [GIT, "-C", str(directory), *args], + capture_output=True, + text=True, + env=env, + timeout=30, + check=True, ) return result.stdout.strip() @pytest.fixture -def checkout(tmp_path: Path) -> Path: +def checkout(tmp_path: Path, git_env: dict[str, str]) -> Path: repository = tmp_path / "checkout" package = repository / "github_activity_db" package.mkdir(parents=True) (package / "__init__.py").write_text("") (repository / ".gitignore").write_text("venv/\n") - _git(repository, "init", "-q") - _git(repository, "add", ".") + _git(repository, git_env, "init", "-q") + _git(repository, git_env, "add", ".") _git( repository, + git_env, "-c", "user.name=t", "-c", @@ -884,8 +945,8 @@ def test_code_version_prefers_github_sha(checkout: Path) -> None: assert code_version(checkout / "github_activity_db", {"GITHUB_SHA": "f00"}) == "f00" -def test_code_version_names_the_checkout_commit(checkout: Path) -> None: - head = _git(checkout, "rev-parse", "HEAD") +def test_code_version_names_the_checkout_commit(checkout: Path, git_env: dict[str, str]) -> None: + head = _git(checkout, git_env, "rev-parse", "HEAD") package = checkout / "github_activity_db" assert code_version(package, {}) == head (checkout / "untracked.txt").write_text("x") diff --git a/tests/sync/test_snapshot.py b/tests/sync/test_snapshot.py index 6094613..1d8507a 100644 --- a/tests/sync/test_snapshot.py +++ b/tests/sync/test_snapshot.py @@ -2,17 +2,16 @@ from __future__ import annotations -import hashlib +import re import sqlite3 from pathlib import Path from typing import Any, cast import pytest -from sqlalchemy import select, update +from sqlalchemy import event from sqlalchemy.ext.asyncio import AsyncConnection from github_activity_db.db.engine import build_engine -from github_activity_db.db.models import PullRequest from github_activity_db.sync.snapshot import ( Snapshot, StoredPullRequest, @@ -20,6 +19,7 @@ read_snapshot, ) from tests.db.seed import Seed, connect +from tests.sync.records import fetch from tests.sync.support import ( HYDRATED, LINKED_REPO, @@ -141,38 +141,42 @@ async def test_an_empty_database(migrated_database: Path) -> None: assert await snapshot_of(migrated_database) == Snapshot(repositories=()) -def _digest(database: Path) -> str: - return hashlib.sha256(database.read_bytes()).hexdigest() - - -async def _commit_after(database: Path, statement: Any | None) -> None: - """Run ``read_snapshot`` or ``statement`` on an application connection, then commit, so a - write either made lands in the file.""" +async def _read_then_commit(database: Path) -> Snapshot: + """Run ``read_snapshot`` on an application connection, then commit, so a write it made + would land in the file.""" engine = build_engine(f"sqlite+aiosqlite:///{database}") try: async with engine.connect() as connection: - if statement is None: - await read_snapshot(connection) - else: - await connection.execute(statement) + snapshot = await read_snapshot(connection) await connection.commit() finally: await engine.dispose() + return snapshot -async def test_writes_nothing(migrated_database: Path) -> None: - _seed(migrated_database) - before = _digest(migrated_database) - await _commit_after(migrated_database, None) - assert _digest(migrated_database) == before +def _pages(database: Path) -> int: + """The file's size in bytes as SQLite counts it: its pages times the page size.""" + connection = sqlite3.connect(database) + try: + (count,) = connection.execute("PRAGMA page_count").fetchone() + (size,) = connection.execute("PRAGMA page_size").fetchone() + finally: + connection.close() + return int(count) * int(size) -async def test_the_digest_sees_a_committed_write(migrated_database: Path) -> None: - """Positive control: an UPDATE committed on the same kind of connection changes the file.""" +async def test_writes_nothing(migrated_database: Path) -> None: + """Every byte of the file is unchanged after the reader's statements and a commit, over a + file whose every page is compared and whose pull requests the reader read.""" _seed(migrated_database) - before = _digest(migrated_database) - await _commit_after(migrated_database, update(PullRequest).values(is_draft=False)) - assert _digest(migrated_database) != before + before = migrated_database.read_bytes() + snapshot = await _read_then_commit(migrated_database) + assert migrated_database.read_bytes() == before + assert len(before) == _pages(migrated_database) + assert ( + sum(repository.item_rows for repository in snapshot.repositories) + == fetch(migrated_database, "SELECT count(*) AS n FROM pull_requests")[0]["n"] + ) class _CheckedConnection: @@ -192,8 +196,16 @@ async def execute(self, statement: Any) -> Any: async def test_no_read_is_left_half_consumed(migrated_database: Path) -> None: + """After each statement the reader sends, and after it returns, another connection commits + at once; the statements, one per table, are counted as they reach the database.""" _seed(migrated_database) + sent: list[str] = [] + + def record(connection: Any, cursor: Any, statement: str, *args: Any) -> None: + sent.append(statement) + engine = build_engine(f"sqlite+aiosqlite:///{migrated_database}") + event.listen(engine.sync_engine, "before_cursor_execute", record) try: async with engine.connect() as connection: checked = _CheckedConnection(connection, migrated_database) @@ -201,21 +213,26 @@ async def test_no_read_is_left_half_consumed(migrated_database: Path) -> None: commit_does_not_wait(migrated_database) finally: await engine.dispose() - # labels, closing links, pull requests, failures, pending, repository labels, repositories - assert checked.reads == 7 + tables = [re.search(r"\bFROM (\w+)", statement) for statement in sent] + assert checked.reads == len(sent) > 0 + assert all(statement.startswith("SELECT ") for statement in sent) + assert len({table.group(1) for table in tables if table is not None}) == len(sent) -async def test_the_lock_probe_sees_a_half_consumed_read(migrated_database: Path) -> None: - """Positive control: a server-side cursor left half-read blocks the probe's commit.""" +async def test_the_lock_probe_refuses_while_another_connection_reads( + migrated_database: Path, +) -> None: + """The probe's commit is refused at once while another process holds a read open, and + succeeds once it ends.""" _seed(migrated_database) - engine = build_engine(f"sqlite+aiosqlite:///{migrated_database}") + reader = sqlite3.connect(migrated_database, isolation_level=None) try: - async with engine.connect() as connection: - stream = await connection.stream(select(PullRequest.id)) - assert await stream.fetchone() is not None - with pytest.raises(sqlite3.OperationalError, match="database is locked"): - commit_does_not_wait(migrated_database) - await stream.close() + reader.execute("BEGIN") + reader.execute("SELECT count(*) FROM sync_runs").fetchone() + with pytest.raises(sqlite3.OperationalError) as raised: commit_does_not_wait(migrated_database) + assert raised.value.sqlite_errorname == "SQLITE_BUSY" + reader.execute("ROLLBACK") finally: - await engine.dispose() + reader.close() + commit_does_not_wait(migrated_database) diff --git a/tests/sync/test_writer.py b/tests/sync/test_writer.py index 7b6b32b..4ec7b52 100644 --- a/tests/sync/test_writer.py +++ b/tests/sync/test_writer.py @@ -9,7 +9,8 @@ from typing import Any import pytest -from sqlalchemy import text +from sqlalchemy import event, text +from sqlalchemy.engine import Engine from sqlalchemy.ext.asyncio import AsyncConnection from github_activity_db.source.records import ( @@ -318,25 +319,60 @@ async def test_a_repository_update_keeps_its_row_and_replaces_its_labels(w: Writ ] +async def _sent_during[T](write: Awaitable[T]) -> tuple[T, list[str]]: + """``write``'s result and every statement sent while it ran.""" + sent: list[str] = [] + + def record(connection: Any, cursor: Any, statement: str, *args: Any) -> None: + sent.append(statement) + + event.listen(Engine, "before_cursor_execute", record) + try: + result = await write + finally: + event.remove(Engine, "before_cursor_execute", record) + return result, sent + + +def _updates_of_pull_requests(sent: list[str]) -> list[str]: + return [statement for statement in sent if statement.startswith("UPDATE pull_requests")] + + +def _targets(w: Writer) -> list[dict[str, Any]]: + return w.rows( + "SELECT p.number, p.targets_default_branch, p.base_ref = r.default_branch AS targets " + "FROM pull_requests p JOIN repositories r ON r.id = p.repository_id ORDER BY p.number" + ) + + async def test_a_default_branch_change_recomputes_targets_default_branch(w: Writer) -> None: await w.repository(repository(default_branch="master")) await w.item(pull_request(1, 10, base_ref="master")) await w.item(pull_request(2, 20, base_ref="main")) assert [w.pr(1)["targets_default_branch"], w.pr(2)["targets_default_branch"]] == [1, 0] - result = await w.repository(repository(default_branch="main")) + result, sent = await _sent_during(w.repository(repository(default_branch="main"))) assert result.default_branch_changed + assert len(_updates_of_pull_requests(sent)) == 1 assert [w.pr(1)["targets_default_branch"], w.pr(2)["targets_default_branch"]] == [0, 1] async def test_an_unchanged_default_branch_leaves_targets_default_branch(w: Writer) -> None: - """A planted wrong value stays: only a change of default branch recomputes it.""" + """Only a change of default branch recomputes it: a write with the same default branch + sends no update of the repository's pull requests, whose values stay what their base + branches give.""" await w.repository(repository(default_branch="master")) await w.item(pull_request(1, 10, base_ref="master")) - with connect(w.database) as connection: - connection.execute(text("UPDATE pull_requests SET targets_default_branch = 0")) - result = await w.repository(repository(default_branch="master")) + await w.item(pull_request(2, 20, base_ref="main")) + before = _targets(w) + assert before == [ + {"number": 1, "targets_default_branch": 1, "targets": 1}, + {"number": 2, "targets_default_branch": 0, "targets": 0}, + ] + result, sent = await _sent_during(w.repository(repository(default_branch="master"))) assert not result.default_branch_changed - assert w.pr(1)["targets_default_branch"] == 0 + assert any(statement.startswith("INSERT INTO repositories") for statement in sent) + assert _updates_of_pull_requests(sent) == [] + assert _targets(w) == before @pytest.mark.parametrize("spelling", ["prebid/Prebid.js", "PREBID/prebid.JS"]) @@ -899,8 +935,9 @@ async def test_a_second_node_id_for_a_stored_github_id_keeps_the_stored_row(w: W async def test_a_node_stored_under_another_github_id_fails_the_item(w: Writer) -> None: await w.repository() await w.item(pull_request(1, 10, author=account("alice", 101, node_id="N_X"))) - with pytest.raises(ItemConflict, match="node N_X is stored under github_id 101"): + with pytest.raises(ItemConflict, match="node N_X is stored under github_id 101") as raised: await w.item(pull_request(2, 20, author=account("mallory", 999, node_id="N_X"))) + assert raised.value.number == 2 assert w.rows("SELECT number FROM pull_requests") == [{"number": 1}] @@ -987,14 +1024,15 @@ async def test_a_node_sighted_or_looked_up_under_two_ids_in_one_item_is_refused( if stored is not None: await w.item(pull_request(1, 10, author=stored)) before = w.dump() - with pytest.raises(ItemConflict, match=message): + with pytest.raises(ItemConflict, match=message) as raised: await w.item(pull_request(2, 20, **record_values), links) + assert raised.value.number == 2 assert w.dump() == before async def test_one_account_in_two_shapes_or_under_two_nodes_in_one_item(w: Writer) -> None: - """The controls: one github_id on one node as a Bot and as a User, and one github_id under - a second node id, each resolve to one row.""" + """Accepted: one github_id on one node as a Bot and as a User, and one github_id under a + second node id, each resolve to one row.""" await w.repository() await w.item(pull_request(1, 10, author=A_ON_N_A)) bot = account("dependabot", 49699333, "Bot", node_id="N_D") @@ -1032,8 +1070,9 @@ async def test_a_second_node_id_another_account_holds_fails_the_item(w: Writer) before = w.rows("SELECT * FROM github_users ORDER BY id") with pytest.raises( ItemConflict, match="node N_b is stored under github_id 8, GitHub reports 7" - ): + ) as raised: await w.item(pull_request(2, 20, author=account("a", 7, node_id="N_b"))) + assert raised.value.number == 2 assert w.rows("SELECT number FROM pull_requests") == [{"number": 1}] assert w.rows("SELECT * FROM github_users ORDER BY id") == before @@ -1045,12 +1084,13 @@ async def test_an_enterprise_sighting_of_a_node_stored_with_a_github_id_fails_th await w.item(pull_request(1, 10, author=account("x", 7, node_id="EUA_1"))) with pytest.raises( ItemConflict, match="node EUA_1 is stored as github_id 7, not as an EnterpriseUserAccount" - ): + ) as raised: await w.item( pull_request( 2, 20, author=account("ent", None, "EnterpriseUserAccount", node_id="EUA_1") ) ) + assert raised.value.number == 2 assert w.rows("SELECT number FROM pull_requests") == [{"number": 1}] assert w.rows("SELECT github_id, node_id, type FROM github_users") == [ {"github_id": 7, "node_id": "EUA_1", "type": "User"} @@ -1381,8 +1421,9 @@ async def test_an_id_stored_under_another_pull_request_is_refused(w: Writer, kin await w.item(pull_request(1, 10, reviews=reviews, comments=comments)) with pytest.raises( ItemConflict, match=rf"{kind} ids \[900\] are stored under another pull request" - ): + ) as raised: await w.item(pull_request(2, 20, author=BOB, reviews=reviews, comments=comments)) + assert raised.value.number == 2 assert w.rows("SELECT number FROM pull_requests") == [{"number": 1}] assert w.rows("SELECT count(*) AS n FROM github_users WHERE login = 'bob'") == [{"n": 0}] @@ -1390,15 +1431,21 @@ async def test_an_id_stored_under_another_pull_request_is_refused(w: Writer, kin async def test_a_number_stored_under_another_github_id_is_refused(w: Writer) -> None: await w.repository() await w.item(pull_request(1, 10)) - with pytest.raises(ItemConflict, match="the number is stored under github_id 10"): + with pytest.raises(ItemConflict, match="the number is stored under github_id 10") as raised: await w.item(pull_request(1, 11)) + assert raised.value.number == 1 + assert w.rows("SELECT number, github_id FROM pull_requests") == [{"number": 1, "github_id": 10}] async def test_a_github_id_stored_as_another_pull_request_is_refused(w: Writer) -> None: await w.repository() await w.item(pull_request(1, 10)) - with pytest.raises(ItemConflict, match="github_id 10 is stored as another pull request"): + with pytest.raises( + ItemConflict, match="github_id 10 is stored as another pull request" + ) as raised: await w.item(pull_request(2, 10)) + assert raised.value.number == 2 + assert w.rows("SELECT number, github_id FROM pull_requests") == [{"number": 1, "github_id": 10}] # -------------------------------------------------------------------------------------------- @@ -1493,8 +1540,9 @@ async def test_a_link_given_for_another_account_is_refused(w: Writer, given: str entry: LinkAccount = ( Sighted(BOB, HYDRATED) if given == "sighted" else Found(looked_up(BOB), HYDRATED) ) - with pytest.raises(ItemConflict, match="commit link 101"): + with pytest.raises(ItemConflict, match="commit link 101") as raised: await w.item(pull_request(1, 10, commits=[commit("s1", link(ALICE))]), {101: entry}) + assert raised.value.number == 1 assert w.rows("SELECT count(*) AS n FROM pull_requests") == [{"n": 0}] diff --git a/tests/test_cli_app.py b/tests/test_cli_app.py index 1790c8e..ff64037 100644 --- a/tests/test_cli_app.py +++ b/tests/test_cli_app.py @@ -7,6 +7,7 @@ import subprocess import sys from collections.abc import Iterator +from importlib import metadata from pathlib import Path import pytest @@ -151,19 +152,19 @@ def test_invalid_settings_exit_1( assert "invalid settings" in _plain(result.stderr) def test_a_commands_own_exit_code_passes_through(self) -> None: - planted = typer.Typer(cls=ExitCodeGroup) + commands = typer.Typer(cls=ExitCodeGroup) - @planted.command() + @commands.command() def partial() -> None: raise typer.Exit(2) - @planted.command() + @commands.command() def done() -> None: pass - assert runner.invoke(planted, ["partial"]).exit_code == 2 - assert runner.invoke(planted, ["done"]).exit_code == 0 - assert runner.invoke(planted, ["--nope"]).exit_code == 1 + assert runner.invoke(commands, ["partial"]).exit_code == 2 + assert runner.invoke(commands, ["done"]).exit_code == 0 + assert runner.invoke(commands, ["--nope"]).exit_code == 1 @pytest.mark.parametrize( ("error", "code"), @@ -175,17 +176,17 @@ def done() -> None: def test_an_aborted_command_line(self, error: type[BaseException], code: int) -> None: """click turns end of input into Abort, which exits 1 here; typer turns an interrupt into exit 130 before click sees it.""" - planted = typer.Typer(cls=ExitCodeGroup) + commands = typer.Typer(cls=ExitCodeGroup) - @planted.command() + @commands.command() def ask() -> None: raise error - @planted.command() + @commands.command() def other() -> None: pass - result = runner.invoke(planted, ["ask"]) + result = runner.invoke(commands, ["ask"]) assert result.exit_code == code, result.output assert isinstance(result.exception, SystemExit), result.exception @@ -193,10 +194,9 @@ def other() -> None: def _installed( *args: str, env: dict[str, str], cwd: Path, script: str | None = None ) -> subprocess.CompletedProcess[str]: - """Run the installed console script, or ``script`` with the same interpreter.""" - base = { - key: value for key, value in os.environ.items() if key in ("PATH", "HOME", "SYSTEMROOT") - } + """Run the installed console script, or ``script`` with the same interpreter, in an + environment built from ``env``, ``PATH``, and ``cwd`` as ``HOME``.""" + base = {"PATH": os.environ["PATH"], "HOME": str(cwd)} command = [str(GHACTIVITY), *args] if script is None else [sys.executable, "-I", "-c", script] return subprocess.run( command, capture_output=True, text=True, env={**base, **env}, cwd=cwd, timeout=60 @@ -280,18 +280,11 @@ def test_an_invalid_tracking_config_exits_1_without_a_traceback(self, tmp_path: assert "not valid TOML" in _plain(result.stderr) assert "Traceback" not in result.stderr - def test_a_traceback_shows_no_local_variables(self, tmp_path: Path) -> None: - """The real app's pretty traceback, for a planted command that fails (gap 75).""" - script = ( - "from github_activity_db.cli.app import app\n" - "@app.command()\n" - "def boom() -> None:\n" - " credential = 'sentinel-credential-7f3a'\n" - " raise RuntimeError('boom ' + str(len(credential)))\n" - "app(['boom'], prog_name='ghactivity')\n" - ) - result = _installed(env={}, cwd=tmp_path, script=script) - assert result.returncode == 1 - assert "RuntimeError" in result.stderr - assert "boom 24" in result.stderr - assert "sentinel-credential-7f3a" not in result.stderr + result.stdout + def test_a_traceback_shows_no_local_variables(self) -> None: + """An uncaught exception's traceback is typer's, set up by the app it was called + through: the console script calls the root app, whose setting keeps local variables out + whatever group the command is in (gap 75). The 2 values compared: the script's target + and the setting.""" + (script,) = metadata.entry_points(group="console_scripts", name="ghactivity") + assert script.load() is app + assert app.pretty_exceptions_show_locals is False diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index 5a5e51a..21bdf99 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -33,10 +33,12 @@ from github_activity_db.cli import sync as sync_cli from github_activity_db.cli.app import app from github_activity_db.config import get_settings +from github_activity_db.db.schema import HEAD_REVISION, MIGRATION_REVISIONS from github_activity_db.source import client from github_activity_db.sync import engine from github_activity_db.sync.deletions import parse_allow_deletions from github_activity_db.sync.engine import SyncOptions, SyncReport +from tests.db.alembic_support import upgrade from tests.source.conftest import FakeClock from tests.source.recorder import FIRST_SYNC_REPOSITORY from tests.source.recordings import FIRST_SYNC @@ -106,8 +108,12 @@ def replay( """``ghactivity sync`` reads GitHub through go-gdpr's first-sync recordings, on the fake clock, into a migrated database; the replay must answer every request it gets. ``common.open_source`` and ``GitHubSource`` are the command's own, with the replay as their HTTP transport; the - report prints on a plain console, so ``_ReportConsole`` runs only in the child tests.""" + report prints on a plain console, so ``_ReportConsole`` runs only in the child tests. + ``GITHUB_SHA`` names the code version, so no git runs, and ``GITHUB_EVENT_NAME`` is unset, so + the trigger is ``manual`` whatever the shell or workflow running the tests sets.""" monkeypatch.chdir(tmp_path) + monkeypatch.setenv("GITHUB_SHA", "test") + monkeypatch.delenv("GITHUB_EVENT_NAME", raising=False) monkeypatch.setenv("GITHUB_TOKEN", "test-token") monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{migrated_database}") monkeypatch.setenv("TRACKING_CONFIG_PATH", str(tracking)) @@ -210,14 +216,23 @@ def test_a_missing_database_file_exits_1_and_creates_none( assert replay.seen == [] -def test_a_database_not_at_head_exits_1(replay: Replay, migrated_database: Path) -> None: - _execute(migrated_database, "UPDATE alembic_version SET version_num = 'older'") +def test_a_database_not_at_head_exits_1( + replay: Replay, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A file the migrations built to the revision before head, as the code before it did.""" + older = tmp_path / "older.db" + upgrade(older, MIGRATION_REVISIONS[-2]) + monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{older}") + get_settings.cache_clear() result = _sync() assert result.exit_code == 1 - assert "not the code's head" in _plain(result.output) - assert _runs(migrated_database) == [] + assert ( + f"Sync did not start: the database is at revision {MIGRATION_REVISIONS[-2]}, not the " + f"code's head {HEAD_REVISION}" in _plain(result.output) + ) + assert _runs(older) == [] assert replay.seen == [] @@ -255,6 +270,18 @@ def test_a_first_sync_exits_0_and_prints_its_report( # Resolve, index, the batch, and one account lookup, at the replay's cost of 1 each. assert "Hydrated 11 items; GraphQL points 4, REST calls 0" in output assert _runs(migrated_database) == [{"id": 1, "status": "complete", "exit_code": 0}] + assert fetch( + migrated_database, + "SELECT items_hydrated, graphql_points, rest_calls, trigger, code_version FROM sync_runs", + ) == [ + { + "items_hydrated": 11, + "graphql_points": 4, + "rest_calls": 0, + "trigger": "manual", + "code_version": "test", + } + ] assert fetch(migrated_database, "SELECT count(*) AS n FROM pull_requests") == [{"n": 11}] @@ -311,7 +338,7 @@ def test_repo_names_a_configured_repository_ignoring_case( client.build_github = lambda auth, *, http_transport=None: build_github( auth, http_transport=replay.transport() ) -{plant} +{setup} from github_activity_db.cli.app import app app({args!r}, prog_name="ghactivity") """ @@ -345,23 +372,26 @@ def _child( database: Path, tracking: Path, cwd: Path, - plant: str = "", + setup: str = "", stdout: int = subprocess.PIPE, stderr: int = subprocess.PIPE, preexec_fn: Callable[[], None] | None = None, ) -> subprocess.CompletedProcess[str]: """Run the installed package in an isolated interpreter (``-I``: no user site, no - PYTHONPATH, no working directory on the path), over the recordings; its stdout and stderr - are captured unless ``stdout`` or ``stderr`` names a file descriptor, and ``preexec_fn`` - runs in the child before the interpreter starts.""" - base = {key: value for key, value in os.environ.items() if key in ("PATH", "HOME")} + PYTHONPATH, no working directory on the path), over the recordings, after ``setup``; its + stdout and stderr are captured unless ``stdout`` or ``stderr`` names a file descriptor, and + ``preexec_fn`` runs in the child before the interpreter starts. The environment is built + from fixed values and ``PATH``: ``HOME`` is ``cwd``, and ``GITHUB_SHA`` names the code + version, so no git runs.""" env = { - **base, + "PATH": os.environ["PATH"], + "HOME": str(cwd), + "GITHUB_SHA": "test", "GITHUB_TOKEN": "test-token", "DATABASE_URL": f"sqlite+aiosqlite:///{database}", "TRACKING_CONFIG_PATH": str(tracking), } - script = CHILD.format(root=str(ROOT), plant=plant, args=["sync", *args]) + script = CHILD.format(root=str(ROOT), setup=setup, args=["sync", *args]) return subprocess.run( [sys.executable, "-I", "-c", script], stdout=stdout, @@ -379,10 +409,13 @@ def test_the_installed_command_exits_with_the_engines_code( migrated_database: Path, tracking: Path, tmp_path: Path ) -> None: failed = _child( - [], database=migrated_database, tracking=tracking, cwd=tmp_path, plant=FAIL_ONE_ITEM + [], database=migrated_database, tracking=tracking, cwd=tmp_path, setup=FAIL_ONE_ITEM ) assert failed.returncode == 2, failed.stderr assert "Run 1: complete, exit 2" in _plain(failed.stdout) + assert fetch(migrated_database, "SELECT trigger, code_version FROM sync_runs") == [ + {"trigger": "manual", "code_version": "test"} + ] usage = _child( ["--dry-run", "--break-lock"], database=migrated_database, tracking=tracking, cwd=tmp_path @@ -392,9 +425,9 @@ def test_the_installed_command_exits_with_the_engines_code( assert _runs(migrated_database) == [{"id": 1, "status": "complete", "exit_code": 2}] -@pytest.mark.parametrize(("plant", "code"), [("", 0), (FAIL_ONE_ITEM, 2)], ids=["0", "2"]) +@pytest.mark.parametrize(("setup", "code"), [("", 0), (FAIL_ONE_ITEM, 2)], ids=["0", "2"]) def test_a_closed_stdout_keeps_the_runs_exit_code( - migrated_database: Path, tracking: Path, tmp_path: Path, plant: str, code: int + migrated_database: Path, tracking: Path, tmp_path: Path, setup: str, code: int ) -> None: """A reader that closed the pipe before the report (``ghactivity sync | true``): the report is not printed, stderr says so, and the command exits with the run's code.""" @@ -406,7 +439,7 @@ def test_a_closed_stdout_keeps_the_runs_exit_code( database=migrated_database, tracking=tracking, cwd=tmp_path, - plant=plant, + setup=setup, stdout=write, ) finally: @@ -417,9 +450,9 @@ def test_a_closed_stdout_keeps_the_runs_exit_code( assert _runs(migrated_database) == [{"id": 1, "status": "complete", "exit_code": code}] -@pytest.mark.parametrize(("plant", "code"), [("", 0), (FAIL_ONE_ITEM, 2)], ids=["0", "2"]) +@pytest.mark.parametrize(("setup", "code"), [("", 0), (FAIL_ONE_ITEM, 2)], ids=["0", "2"]) def test_closed_stdout_and_stderr_keep_the_runs_exit_code( - migrated_database: Path, tracking: Path, tmp_path: Path, plant: str, code: int + migrated_database: Path, tracking: Path, tmp_path: Path, setup: str, code: int ) -> None: """``ghactivity sync 2>&1 | true``: neither stream can be written, and the command still exits with the run's code, not the interpreter's 120 for output it could not flush.""" @@ -431,7 +464,7 @@ def test_closed_stdout_and_stderr_keep_the_runs_exit_code( database=migrated_database, tracking=tracking, cwd=tmp_path, - plant=plant, + setup=setup, stdout=write, stderr=write, ) @@ -442,9 +475,9 @@ def test_closed_stdout_and_stderr_keep_the_runs_exit_code( assert _runs(migrated_database) == [{"id": 1, "status": "complete", "exit_code": code}] -@pytest.mark.parametrize(("plant", "code"), [("", 0), (FAIL_ONE_ITEM, 2)], ids=["0", "2"]) +@pytest.mark.parametrize(("setup", "code"), [("", 0), (FAIL_ONE_ITEM, 2)], ids=["0", "2"]) def test_a_stdout_closed_at_start_keeps_the_runs_exit_code( - migrated_database: Path, tracking: Path, tmp_path: Path, plant: str, code: int + migrated_database: Path, tracking: Path, tmp_path: Path, setup: str, code: int ) -> None: """``ghactivity sync >&-``: with fd 1 closed when the interpreter starts, ``sys.stdout`` is None and the report goes nowhere; the command exits with the run's code.""" @@ -453,7 +486,7 @@ def test_a_stdout_closed_at_start_keeps_the_runs_exit_code( database=migrated_database, tracking=tracking, cwd=tmp_path, - plant=plant, + setup=setup, preexec_fn=functools.partial(os.close, 1), ) @@ -462,9 +495,9 @@ def test_a_stdout_closed_at_start_keeps_the_runs_exit_code( assert _runs(migrated_database) == [{"id": 1, "status": "complete", "exit_code": code}] -@pytest.mark.parametrize(("plant", "code"), [("", 0), (FAIL_ONE_ITEM, 2)], ids=["0", "2"]) +@pytest.mark.parametrize(("setup", "code"), [("", 0), (FAIL_ONE_ITEM, 2)], ids=["0", "2"]) def test_a_stdout_file_that_cannot_grow_keeps_the_runs_exit_code( - migrated_database: Path, tracking: Path, tmp_path: Path, plant: str, code: int + migrated_database: Path, tracking: Path, tmp_path: Path, setup: str, code: int ) -> None: """``ghactivity sync > report.txt`` where the file cannot grow (it is at the child's RLIMIT_FSIZE, so a write fails as on a full disk): what the failed write left buffered is @@ -483,7 +516,7 @@ def test_a_stdout_file_that_cannot_grow_keeps_the_runs_exit_code( database=migrated_database, tracking=tracking, cwd=tmp_path, - plant=plant, + setup=setup, stdout=stdout, preexec_fn=functools.partial(resource.setrlimit, resource.RLIMIT_FSIZE, (limit, limit)), ) @@ -500,7 +533,7 @@ def test_a_killed_run_stays_running_and_a_resume_keeps_its_writes( migrated_database: Path, tracking: Path, tmp_path: Path ) -> None: killed = _child( - [], database=migrated_database, tracking=tracking, cwd=tmp_path, plant=KILL_AFTER_THIRD_ITEM + [], database=migrated_database, tracking=tracking, cwd=tmp_path, setup=KILL_AFTER_THIRD_ITEM ) assert killed.returncode == -signal.SIGKILL, killed.stderr assert _runs(migrated_database) == [{"id": 1, "status": "running", "exit_code": None}] @@ -537,7 +570,7 @@ def test_a_killed_run_stays_running_and_a_resume_keeps_its_writes( # A child started with SIGINT ignored (a suite started with & from a non-interactive shell) -# inherits that; Python then installs no handler of its own, so the plant installs Python's +# inherits that; Python then installs no handler of its own, so the setup installs Python's # default, as at a terminal. INTERRUPT_AFTER_THIRD_ITEM = """\ import os, signal @@ -566,7 +599,7 @@ def test_a_sigint_closes_the_run_aborted_and_exits_130( database=migrated_database, tracking=tracking, cwd=tmp_path, - plant=INTERRUPT_AFTER_THIRD_ITEM, + setup=INTERRUPT_AFTER_THIRD_ITEM, ) assert interrupted.returncode == 130, interrupted.stdout + interrupted.stderr assert _runs(migrated_database) == [{"id": 1, "status": "aborted", "exit_code": 130}] @@ -655,8 +688,9 @@ async def spy(options: SyncOptions, **kwargs: Any) -> SyncReport: assert (options.max_minutes, options.prune_untracked) == (30, True) assert options.allow_deletions == parse_allow_deletions(approval, config) assert (options.dry_run, options.break_lock) == (dry_run, not dry_run) - assert (options.trigger, options.database_url) == ( + assert (options.trigger, options.code_version, options.database_url) == ( "manual", + "test", f"sqlite+aiosqlite:///{migrated_database}", ) assert options.config.source == str(tracking) From 993917cfe8a95c47d05d48c0d21d355c5d68a2cc Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 08:01:05 -0600 Subject: [PATCH 04/29] Step 3b-2a-2d: checkers tests without planted inputs (D39, D40) tests/test_check_log_placeholders.py: remove the written-source cases (flagged calls, accepted calls, per-site listing). The run over src/ asserts exit 0, no message found, and the files scanned equal an os.walk count of .py/.pyi files. The empty-directory refusal asserts exit 1 and empty stdout instead of stderr wording. Subprocess env is PATH only. tests/test_count_suppressions.py: remove the written spellings, packages, refused/accepted files, the chmod 000 file, and the non-file entries. src/ at the hook caps: exit 0, empty stderr, files scanned equal to os.walk, counts equal to a second reading with tokenize. Caps at and one below each count over src/: exit 0, then exit 1 listing exactly the sites the second reading finds. scripts/count_suppressions.py, whose docstring holds directive text, is counted as its comments alone. refusal() returns None for every regular file under src/ (os.walk list). identify's read of a file stubbed to raise PermissionError: refusal() reports it and main() exits 1 naming each non-Python file; unstubbed, exit 0. Empty directory: exit 1, empty stdout. tests/test_check_any_aliases.py: remove the written alias, non-alias, and stub cases. Add the run over src/: exit 0, no alias, files scanned equal to os.walk. Empty directory: exit 1, empty stdout. tests/test_suite_rules.py: remove the written-shape test; the scan over tests/ asserts its file count equals os.walk's. tests/test_import_boundaries.py: remove the written spellings, text, and package copy. Both passing sides assert the file count from os.walk and that the (file, line) of every import the scan read equals a tokenize reading of the same files. source/'s githubkit imports and sync/'s SQLAlchemy and db/ imports are found at exactly the statements a text match finds. tests/test_server_side_cursors.py: remove the written calls; the src/ scan asserts its file count equals os.walk's. tests/test_declared_dependencies.py: remove the written package and the distributions installed on disk. src/ against pyproject.toml at ea7d12e (tests/fixtures/history, before httpx was declared) reports exactly each file importing a dependency declared since, by text match and packaging's requirement names. The environment's top-level map, stubbed in memory with sqlalchemy installed by an undeclared sibling, by declared ones, or by none, gives one finding per sqlalchemy importer, none, and one with no provider. undeclared_imports takes an optional owners map. The src/ pass asserts the files scanned equal os.walk's. tests/test_wheel_smoke.py: remove the package copy without tracking.toml. Runs use -I as CI does, an explicit env, and a temporary working directory. The packaged count equals tomllib's count of the source file; refusals assert exit 1, empty stderr, and the value named. An override naming a missing file is refused before any read. tests/test_check_junit.py: reports are pytest's own, or built in memory from one with its counters changed. Assertions are exit codes, the counters the gate prints, the number of problems, and empty stdout for an unreadable report or a refused allowance. Nested pytest gets its own pytest.ini and an explicit env without plugin autoload. tests/fixtures/history/: ea7d12e's pyproject.toml, unchanged, and a README naming its source commit. --- tests/fixtures/history/README.md | 29 +- tests/fixtures/history/ea7d12e/pyproject.toml | 188 ++++++++++ tests/test_check_any_aliases.py | 185 ++-------- tests/test_check_junit.py | 153 +++++--- tests/test_check_log_placeholders.py | 94 ++--- tests/test_count_suppressions.py | 335 +++++++----------- tests/test_declared_dependencies.py | 168 ++++----- tests/test_import_boundaries.py | 162 +++++---- tests/test_server_side_cursors.py | 55 +-- tests/test_suite_rules.py | 45 +-- tests/test_wheel_smoke.py | 99 +++--- 11 files changed, 708 insertions(+), 805 deletions(-) create mode 100644 tests/fixtures/history/ea7d12e/pyproject.toml diff --git a/tests/fixtures/history/README.md b/tests/fixtures/history/README.md index df611bd..2a5c54d 100644 --- a/tests/fixtures/history/README.md +++ b/tests/fixtures/history/README.md @@ -1,15 +1,20 @@ # Earlier versions of repository files -Each file here is a real earlier version of a repository file, copied unchanged from -`git show :` to `/`. CI checks out with depth 1 and has no -history, so the tests read these copies instead. +Each file under `/` is `git show :` copied unchanged to +`/`: a repository file as an earlier commit holds it, which a test reads as input +(redesign plan D39). CI checks out with depth 1 and has no history, so tests read these copies +instead of git history. -| File | Commit | -|------|--------| -| `21a4267/docs/data-dictionary.md` | 21a4267adc84e69327f2795d6f045ea5514881b1 | -| `77663a7/docs/data-dictionary.md` | 77663a7a1039fbf1892fa7fc42ef3ecefa55ea5f | -| `c1f474b/tests/query_versions.json` | c1f474bf86117a38a5b6c45dd5808cf99e3f0da0 | -| `21a4267/tests/query_versions.json` | 21a4267adc84e69327f2795d6f045ea5514881b1 | -| `6fedb05/tests/schema_versions.json` | 6fedb05fde7bd26cc79b0a7eb2dd91d020655a58 | -| `21a4267/tests/schema_versions.json` | 21a4267adc84e69327f2795d6f045ea5514881b1 | -| `77663a7/tests/schema_versions.json` | 77663a7a1039fbf1892fa7fc42ef3ecefa55ea5f | +| File | Source commit | Read by | +|------|---------------|---------| +| `21a4267/docs/data-dictionary.md` | `21a4267adc84e69327f2795d6f045ea5514881b1` | `tests/test_data_dictionary.py` | +| `77663a7/docs/data-dictionary.md` | `77663a7a1039fbf1892fa7fc42ef3ecefa55ea5f` | `tests/test_data_dictionary.py` | +| `c1f474b/tests/query_versions.json` | `c1f474bf86117a38a5b6c45dd5808cf99e3f0da0` | `tests/test_check_query_lock.py` | +| `21a4267/tests/query_versions.json` | `21a4267adc84e69327f2795d6f045ea5514881b1` | `tests/test_check_query_lock.py` | +| `6fedb05/tests/schema_versions.json` | `6fedb05fde7bd26cc79b0a7eb2dd91d020655a58` | `tests/test_check_query_lock.py` | +| `21a4267/tests/schema_versions.json` | `21a4267adc84e69327f2795d6f045ea5514881b1` | `tests/test_check_query_lock.py`, `tests/test_data_dictionary.py` | +| `77663a7/tests/schema_versions.json` | `77663a7a1039fbf1892fa7fc42ef3ecefa55ea5f` | `tests/test_check_query_lock.py`, `tests/test_data_dictionary.py` | +| `ea7d12e/pyproject.toml` | `ea7d12e798fefe1971d1b409523a87f4d8caaf17`, the parent of the commit that declared httpx (gap 76) | `tests/test_declared_dependencies.py` | +| `c1f474b/tests/fixtures/source/recorded/pull_request/prebid__Prebid.js/13481.json` | `c1f474bf86117a38a5b6c45dd5808cf99e3f0da0` | `tests/source/test_recordings.py` | +| `c1f474b/tests/fixtures/source/recorded/overflow/PR_kwDOBUaT8s6WsAjJ/comments/0e3c0d53c5b8c0eb.json` | `c1f474bf86117a38a5b6c45dd5808cf99e3f0da0` | `tests/source/test_recordings.py` | +| `c1f474b/tests/fixtures/source/recorded/resolve/prebid__go-gdpr.json` | `c1f474bf86117a38a5b6c45dd5808cf99e3f0da0` | `tests/source/test_recordings.py` | diff --git a/tests/fixtures/history/ea7d12e/pyproject.toml b/tests/fixtures/history/ea7d12e/pyproject.toml new file mode 100644 index 0000000..d865ae0 --- /dev/null +++ b/tests/fixtures/history/ea7d12e/pyproject.toml @@ -0,0 +1,188 @@ +[project] +name = "github-activity-db" +version = "0.1.0" +description = "Searchable data store for GitHub PR data with custom tagging" +readme = "README.md" +license = { file = "LICENSE" } +authors = [{ name = "agentmoose", email = "phoenixtechnerd@gmail.com" }] +requires-python = ">=3.12" +dependencies = [ + "sqlalchemy>=2.0", + "aiosqlite>=0.20", + "pydantic>=2.0", + "pydantic-settings>=2.0", + "githubkit>=0.12", + "python-dotenv>=1.0", + "alembic>=1.14", + "loguru>=0.7", + "typer>=0.15", + "rich>=13.0", + "greenlet>=3.3.0", +] + +[project.scripts] +ghactivity = "github_activity_db.cli.app:app" + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "pytest-asyncio>=1.4", + "pytest-cov>=5.0", + "mypy>=1.13", + "ruff>=0.8", + "pre-commit>=4.0", + # scripts/count_suppressions.py tags files as pre-commit does. + "identify>=2.6", +] + +[build-system] +requires = ["hatchling==1.32.4"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/github_activity_db"] + +[tool.uv] +# Build dependencies for uv builds, including the editable install `uv sync` makes. Run +# `uv lock` after changing them: `uv sync --frozen` (CI and the documented install) reads +# the copy in uv.lock's [manifest]. build-constraints.txt pins the same versions with hashes +# for the built-wheel CI job; tests/test_build_inputs.py keeps all four pin sites equal. +build-constraint-dependencies = [ + "editables==0.6", + "hatchling==1.32.4", + "packaging==26.3", + "pathspec==1.1.1", + "pluggy==1.7.0", + "tomlkit==0.15.2", + "trove-classifiers==2026.9.21.13", +] + +# ------------------------------------------------------------------------------ +# Ruff - Linting & Formatting +# ------------------------------------------------------------------------------ +[tool.ruff] +target-version = "py312" +line-length = 100 +src = ["src", "tests"] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "UP", # pyupgrade + "ASYNC", # flake8-async + "S", # flake8-bandit (security) + "PTH", # flake8-use-pathlib + "RUF", # ruff-specific rules + # Type checking best practices (non-strict mode) + "TC006", # Quote type annotations in cast() calls + "FA", # flake8-future-annotations + "PGH004", # blanket noqa: a suppression must name its rule codes +] +ignore = [ + "S101", # allow assert in tests + "RUF022", # allow categorized __all__ (not alphabetically sorted) + "ASYNC109", # allow timeout parameter naming in async functions + "ASYNC110", # allow sleep in while loop for queue drain patterns + "FA100", # allow files without `from __future__ import annotations` +] + +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = ["S101", "S105", "S106", "S603"] # assert, test tokens, fixed-argv subprocesses +"scripts/check_junit.py" = ["S314"] # parses the report pytest wrote earlier in the same job +"scripts/check_query_lock.py" = ["S603"] # runs git with a fixed argument list + +[tool.ruff.lint.isort] +known-first-party = ["github_activity_db"] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" + +# ------------------------------------------------------------------------------ +# Mypy - Type Checking +# ------------------------------------------------------------------------------ +[tool.mypy] +python_version = "3.12" +strict = true +warn_return_any = true +warn_unused_ignores = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_configs = true +show_error_codes = true +# Require error codes on type: ignore comments for maintainability +enable_error_code = ["ignore-without-code"] +plugins = ["pydantic.mypy"] +exclude = ["alembic/", "build/", "dist/"] + +[[tool.mypy.overrides]] +module = ["githubkit.*", "aiosqlite.*"] +ignore_missing_imports = true + +[[tool.mypy.overrides]] +# Relaxed strictness for tests - enables mypy checking while allowing +# flexibility for test-specific patterns (mocks, fixtures, etc.) +module = "tests.*" +disallow_untyped_defs = false +disallow_incomplete_defs = false +disallow_untyped_calls = false +# Relax generic type parameter requirements (allows `dict` without `dict[str, Any]`) +disallow_any_generics = false +# Keep important checks enabled +check_untyped_defs = true +warn_return_any = false # Tests often return mock values +warn_unused_ignores = true + +[tool.pydantic-mypy] +init_forbid_extra = true +init_typed = true +warn_required_dynamic_aliases = true + +# ------------------------------------------------------------------------------ +# Pytest - Testing +# ------------------------------------------------------------------------------ +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" +addopts = [ + "-ra", + "-q", + "--strict-markers", + "--strict-config", +] +filterwarnings = [ + "error", + "ignore::DeprecationWarning", +] + +# ------------------------------------------------------------------------------ +# Coverage - Test Coverage +# ------------------------------------------------------------------------------ +[tool.coverage.run] +source = ["src/github_activity_db"] +branch = true +parallel = true +omit = ["*/tests/*", "*/__pycache__/*"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] +show_missing = true diff --git a/tests/test_check_any_aliases.py b/tests/test_check_any_aliases.py index 5d63ddc..2e96786 100644 --- a/tests/test_check_any_aliases.py +++ b/tests/test_check_any_aliases.py @@ -1,179 +1,46 @@ """The pre-commit check that rejects module-level type aliases to Any.""" +import os +import re import subprocess import sys from pathlib import Path -import pytest +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "check_any_aliases.py" +SRC = ROOT / "src" -SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "check_any_aliases.py" - -def _run(tmp_path: Path, source: str, name: str = "module.py") -> subprocess.CompletedProcess[str]: - root = tmp_path / "pkg" - root.mkdir() - (root / name).write_text(source) +def _run(*paths: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( - [sys.executable, str(SCRIPT), str(root)], capture_output=True, text=True, timeout=60 + [sys.executable, str(SCRIPT), *map(str, paths)], + capture_output=True, + text=True, + env={"PATH": os.environ.get("PATH", "")}, + timeout=60, ) -ALIASES = [ - "from typing import Any\n\nFoo = Any\n", - "import typing\n\nFoo = typing.Any\n", - "from typing import Any\n\nFoo = Any | None\n", - "from typing import Any, TypeAlias\n\nFoo: TypeAlias = Any\n", - "import typing\n\nFoo: typing.TypeAlias = typing.Any\n", - "from typing import Any\n\ntype Foo = Any\n", - "import typing\n\ntype Foo = typing.Any\n", - "from typing import Any\n\ntype Foo[T] = Any\n", - "from typing import Any\n\nFoo = None | Any\n", - "from typing import Any, Optional\n\nFoo = Optional[Any]\n", - "import typing\n\nFoo = typing.Union[int, typing.Any]\n", - 'from typing import Annotated, Any\n\nFoo = Annotated[Any, "x"]\n', - "from typing import Any as Dynamic\n\ntype Foo = Dynamic\n", - "from typing import Any, Optional as Maybe\n\nFoo = Maybe[Any]\n", - "from typing import Any, Union as Either\n\nFoo = Either[int, Any]\n", - 'from typing import Annotated as Note, Any\n\nFoo = Note[Any, "x"]\n', - "from typing import Any, TypeAlias as Alias\n\nFoo: Alias = Any\n", - "import typing_extensions as te\n\nFoo = te.Any\n", - "from typing import *\n\nFoo = Any\n", - # Imports in function and class bodies bind other scopes and do not replace this one. - "from typing import Any as Dynamic\n\n\ndef convert(value: object) -> int:\n" - ' from typing import cast as Dynamic\n return Dynamic("int", value)\n\n\n' - "type Foo = Dynamic\n", - "from typing import Any as Dynamic\n\n\nclass C:\n from typing import cast as Dynamic\n\n\n" - "Foo = Dynamic\n", - # Imports in module-level blocks run in module scope. - "from typing import TYPE_CHECKING\n\nif TYPE_CHECKING:\n" - " from typing import Any as Dynamic\n\n" - "Foo = Dynamic\n", - "try:\n from typing import Any as Dynamic\nexcept ImportError:\n pass\n\nFoo = Dynamic\n", - # The latest module-level binding wins. - "class Any:\n pass\n\n\nfrom typing import Any\n\nFoo = Any\n", - # Where branches disagree, a name that may be typing.Any counts. - "from typing import TYPE_CHECKING\n\nif TYPE_CHECKING:\n from typing import Any as JSON\n" - "else:\n JSON = object\n\nFoo = JSON\n", - "import sys\n\nif sys.version_info >= (3, 12):\n from mylib import D\nelse:\n" - " from typing import Any as D\n\nFoo = D\n", - "try:\n from mylib import D\nexcept ImportError:\n" - " from typing import Any as D\n\nFoo = D\n", - "from typing import Any as D\n\nfor D in []:\n pass\n\nFoo = D\n", - "from typing import Any as D\n\nmatch 1:\n case 1:\n D = int\n\nFoo = D\n", - # String forward references where a type is expected. - 'from typing import Any, TypeAlias\n\nFoo: TypeAlias = "Any"\n', - 'from typing import Any\n\ntype Foo = "Any"\n', - 'from typing import Any, Optional\n\nFoo = Optional["Any"]\n', - 'from typing import Any, Optional, TypeAlias\n\nFoo: TypeAlias = "Optional[Any]"\n', - # x[k] = ... and x.attr = ... read x; they do not rebind it. - 'from typing import Any\n\nHANDLERS: dict[object, str] = {}\nHANDLERS[Any] = "any"\n' - "Foo = Any\n", - 'import typing as t\n\nt.__doc__ = "patched"\nFoo = t.Any\n', - "from typing import Any\n\nCOUNTS: dict[object, int] = {}\nCOUNTS[Any] += 1\nFoo = Any\n", - # Aliases in module-level blocks other than an `if TYPE_CHECKING:` body. - "from typing import Any\n\ntry:\n Foo = Any\nexcept ImportError:\n pass\n", - "import contextlib\nfrom typing import Any\n\nwith contextlib.suppress(Exception):\n" - " Foo = Any\n", - "from typing import Any\n\nfor _ in range(1):\n Foo = Any\n", - "from typing import Any\n\nwhile True:\n Foo = Any\n break\n", - "from typing import Any\n\nmatch 1:\n case 1:\n Foo = Any\n", - "import sys\nfrom typing import Any\n\nif sys.version_info >= (3, 12):\n Foo = Any\n", - # Assignments that copy a typing module or object. - "import typing\n\nt = typing\nFoo = t.Any\n", - # Each binding rule. - "from typing import *\n\n\nclass Any:\n pass\n\n\nfrom typing import *\n\nFoo = Any\n", - "from typing import Any as D\n\nwhile False:\n D = int\n\nFoo = D\n", - "try:\n pass\nfinally:\n from typing import Any as D\n\nFoo = D\n", - "try:\n from mylib import D\nexcept* ImportError:\n from typing import Any as D\n\n" - "Foo = D\n", - "from mylib import D\n\nmatch 1:\n case 1:\n from typing import Any as D\n\n" - "Foo = D\n", - "import sys\nfrom typing import *\n\nif sys.version_info >= (3, 12):\n\n class Any:\n" - " pass\n\n\nFoo = Any\n", - # A type checker reads an `if TYPE_CHECKING:` body and the else of `if not TYPE_CHECKING:`. - "from typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n Foo = Any\n", - "from typing import TYPE_CHECKING, Any\n\nif not TYPE_CHECKING:\n pass\nelse:\n" - " Foo = Any\n", - # A quoted TypeAlias annotation, a copied typing object, and a string Annotated argument. - 'from typing import Any, TypeAlias\n\nFoo: "TypeAlias" = Any\n', - "import typing\nfrom typing import Any\n\nO = typing.Optional\nFoo = O[Any]\n", - 'from typing import Annotated, Any\n\nFoo = Annotated["Any", "x"]\n', - # An annotated subscript target reads its names. - "from typing import Any\n\nCOUNTS: dict[object, int] = {}\nCOUNTS[Any]: int = 2\nFoo = Any\n", -] -NOT_ALIASES = [ - "from typing import Any\n\nFoo = dict[str, Any]\n", - 'from typing import Annotated, Any\n\nFoo = Annotated[int, Any, "x"]\n', - "from typing import Any, Optional\n\nFoo = Optional[list[Any]]\n", - "from typing import AnyStr\n\nFoo = AnyStr\n", - "from typing import Any\n\nvalue: Any = 1\n", - "from typing import Any\n\n\ndef f() -> None:\n Foo = Any\n", - "from typing import Any\n\n\nclass C:\n Foo = Any\n", - 'TEXT = "Foo = Any"\n', - "class Any:\n pass\n\n\ntype Foo = Any\n", - "from mylib import Any\n\nFoo = Any\n", - "import mylib\n\nFoo = mylib.Any\n", - # A function-local typing import does not make the module's own class typing.Any. - "class Any:\n pass\n\n\ndef f() -> None:\n from typing import Any\n\n\ntype Foo = Any\n", - # A later module-level binding replaces the typing import. - "from typing import Any\n\n\nclass Any:\n pass\n\n\nFoo = Any\n", - "from typing import Any as Dynamic\n\nDynamic = int\nFoo = Dynamic\n", - "from typing import Any\nfrom mylib import Any\n\nFoo = Any\n", - "import typing as t\nimport mylib as t\n\nFoo = t.Any\n", - "from typing import *\n\n\nclass Any:\n pass\n\n\nFoo = Any\n", - "import contextlib\nfrom typing import Any as D\n\nwith contextlib.nullcontext(1) as D:\n" - " pass\n\nFoo = D\n", - # A string outside a type position is a value, and an unparsable reference is skipped. - 'from typing import Any\n\nFoo = "Any"\n', - 'from typing import Any, Literal\n\nFoo = Literal["Any"]\n', - 'from typing import TypeAlias\n\nFoo: TypeAlias = "not ( valid"\n', - # Each binding rule. - "from .typing import Any\n\nFoo = Any\n", - "from typing import Any as D\n\ntype D = int\nFoo = D\n", - "from typing import Any as D\n\nD += 1\nFoo = D\n", - "from typing import Any as D\n\nD: int = 1\nFoo = D\n", - "from typing import Any as D\n\ntry:\n pass\nexcept Exception as D:\n Foo = D\n", - "from typing import Any\n\nfor Any in [1]:\n Foo = Any\n", - # The branch only the runtime executes is exempt. - "from typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n pass\nelse:\n Foo = Any\n", - "from typing import TYPE_CHECKING, Any\n\nif not TYPE_CHECKING:\n Foo = Any\n", - # The TYPE_CHECKING pattern: a precise type for the checker, Any for the runtime. - "from typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from mylib import Model\n\n" - " Foo = Model\nelse:\n Foo = Any\n", - # The attribute form `typing.TYPE_CHECKING` marks the runtime-only else as well. - "import typing\nfrom typing import Any\n\nif typing.TYPE_CHECKING:\n pass\nelse:\n" - " Foo = Any\n", -] - - -@pytest.mark.parametrize("source", ALIASES) -def test_rejects_an_alias_to_any(tmp_path: Path, source: str) -> None: - result = _run(tmp_path, source) - assert result.returncode == 1 - assert "aliases to Any: 1" in result.stdout - assert "Foo is an alias to Any" in result.stdout +def _python_file_count(root: Path) -> int: + """The .py and .pyi files under ``root``, counted with os.walk rather than the script's glob.""" + return sum( + Path(name).suffix in {".py", ".pyi"} for _, _, names in os.walk(root) for name in names + ) -@pytest.mark.parametrize("source", NOT_ALIASES) -def test_accepts_other_uses_of_any(tmp_path: Path, source: str) -> None: - result = _run(tmp_path, source) +def test_src_has_no_alias_to_any() -> None: + result = _run(SRC) assert result.returncode == 0, result.stdout - assert "aliases to Any: 0" in result.stdout - - -def test_scans_stub_files(tmp_path: Path) -> None: - result = _run(tmp_path, "from typing import Any\n\nFoo = Any\n", name="stub.pyi") - assert result.returncode == 1 - assert result.stdout.startswith("scanned 1 files") + scanned = re.search(r"^scanned (\d+) files", result.stdout, re.MULTILINE) + found = re.search(r"^aliases to Any: (\d+)$", result.stdout, re.MULTILINE) + assert scanned is not None and found is not None, result.stdout + assert int(scanned.group(1)) == _python_file_count(SRC) + assert int(found.group(1)) == 0 def test_no_python_files_is_an_error(tmp_path: Path) -> None: (tmp_path / "empty").mkdir() - result = subprocess.run( - [sys.executable, str(SCRIPT), str(tmp_path / "empty")], - capture_output=True, - text=True, - timeout=60, - ) + result = _run(tmp_path / "empty") assert result.returncode == 1 - assert "no Python files" in result.stderr + # A run that finds an alias also exits 1, but only after reporting what it scanned. + assert result.stdout == "" diff --git a/tests/test_check_junit.py b/tests/test_check_junit.py index cc67461..3478740 100644 --- a/tests/test_check_junit.py +++ b/tests/test_check_junit.py @@ -1,21 +1,68 @@ -"""The CI gate over pytest's JUnit report.""" +"""The CI gate over pytest's JUnit report. +Every report is one pytest wrote, in a run over a test file written for it, or is built in memory +from one with its counters changed into a shape a run can leave: failures, errors, skips, no test +executed, several suites merged, a file cut short, or no suite at all. +""" + +import ast +import copy +import os import subprocess import sys +import xml.etree.ElementTree as ET from pathlib import Path import pytest SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "check_junit.py" +COUNTED = ("tests", "failures", "errors", "skipped") -def _report(tmp_path: Path, *suites: tuple[int, int, int, int]) -> Path: - body = "".join( - f'' - for t, f, e, s in suites +def _pytest(root: Path, test_body: str) -> subprocess.CompletedProcess[str]: + """Run pytest over one test file in ``root``, with its own configuration and only the + environment variables it needs; the report is ``root/junit.xml``.""" + (root / "pytest.ini").write_text("[pytest]\n") + (root / "test_sample.py").write_text(f"import pytest\n\n\n{test_body}") + return subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + f"--junitxml={root / 'junit.xml'}", + ], + cwd=root, + capture_output=True, + text=True, + env={"PATH": os.environ.get("PATH", ""), "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}, + timeout=60, ) + + +@pytest.fixture(scope="module") +def written(tmp_path_factory: pytest.TempPathFactory) -> bytes: + """The report pytest wrote for one passing test.""" + root = tmp_path_factory.mktemp("pytest-run") + run = _pytest(root, "def test_a():\n pass\n") + assert run.returncode == 0, run.stdout + return (root / "junit.xml").read_bytes() + + +def _report(tmp_path: Path, written: bytes, *suites: tuple[int, int, int, int]) -> Path: + """The written report with its once per ``suites``, each with those counters.""" + # The report pytest wrote in this session. + root = ET.fromstring(written) # noqa: S314 + (template,) = root.findall("testsuite") + root.remove(template) + for counts in suites: + suite = copy.deepcopy(template) + suite.attrib.update(zip(COUNTED, map(str, counts), strict=True)) + root.append(suite) path = tmp_path / "junit.xml" - path.write_text(f'{body}') + path.write_bytes(ET.tostring(root, encoding="utf-8", xml_declaration=True)) return path @@ -24,76 +71,92 @@ def _gate(report: Path, *extra: str) -> subprocess.CompletedProcess[str]: [sys.executable, str(SCRIPT), str(report), *extra], capture_output=True, text=True, + env={"PATH": os.environ.get("PATH", "")}, timeout=60, ) +def _printed(result: subprocess.CompletedProcess[str]) -> object: + """The counters the gate read, which it prints before judging them.""" + return ast.literal_eval(result.stdout.strip()) + + @pytest.mark.parametrize( - ("counts", "reason"), + ("counts", "problems"), [ - ((0, 0, 0, 0), "no test executed"), - ((5, 0, 0, 5), "no test executed"), - ((5, 1, 0, 0), "1 failed"), - ((5, 0, 1, 0), "1 errored"), - ((5, 0, 0, 1), "more than the allowed 0"), + ((0, 0, 0, 0), 1), # no test executed + ((5, 0, 0, 5), 2), # no test executed, and more skips than allowed + ((5, 1, 0, 0), 1), + ((5, 0, 1, 0), 1), + ((5, 0, 0, 1), 1), ], ) -def test_rejects(tmp_path: Path, counts: tuple[int, int, int, int], reason: str) -> None: - result = _gate(_report(tmp_path, counts)) +def test_rejects( + tmp_path: Path, written: bytes, counts: tuple[int, int, int, int], problems: int +) -> None: + result = _gate(_report(tmp_path, written, counts)) assert result.returncode == 1 - assert reason in result.stderr + assert _printed(result) == dict(zip(COUNTED, counts, strict=True)) + assert len(result.stderr.splitlines()) == problems -def test_accepts_an_executed_passing_run(tmp_path: Path) -> None: - assert _gate(_report(tmp_path, (5, 0, 0, 0))).returncode == 0 +def test_accepts_an_executed_passing_run(tmp_path: Path, written: bytes) -> None: + report = tmp_path / "junit.xml" + report.write_bytes(written) + result = _gate(report) + assert result.returncode == 0, result.stderr + assert _printed(result) == {"tests": 1, "failures": 0, "errors": 0, "skipped": 0} + assert result.stderr == "" -def test_partial_skips_need_an_explicit_allowance(tmp_path: Path) -> None: - report = _report(tmp_path, (5, 0, 0, 2)) +def test_partial_skips_need_an_explicit_allowance(tmp_path: Path, written: bytes) -> None: + report = _report(tmp_path, written, (5, 0, 0, 2)) assert _gate(report).returncode == 1 + assert _gate(report, "--max-skipped", "1").returncode == 1 assert _gate(report, "--max-skipped", "2").returncode == 0 -def test_sums_every_testsuite(tmp_path: Path) -> None: - result = _gate(_report(tmp_path, (3, 0, 0, 0), (2, 1, 0, 0))) +def test_sums_every_testsuite(tmp_path: Path, written: bytes) -> None: + result = _gate(_report(tmp_path, written, (3, 0, 0, 0), (2, 1, 0, 0))) assert result.returncode == 1 - assert "'tests': 5" in result.stdout - assert "1 failed" in result.stderr + assert _printed(result) == {"tests": 5, "failures": 1, "errors": 0, "skipped": 0} + assert len(result.stderr.splitlines()) == 1 -@pytest.mark.parametrize("content", [None, "']) -def test_unreadable_report_fails(tmp_path: Path, content: str | None) -> None: +@pytest.mark.parametrize("shape", ["missing", "cut short", "no testsuite"]) +def test_unreadable_report_fails(tmp_path: Path, written: bytes, shape: str) -> None: path = tmp_path / "junit.xml" - if content is not None: - path.write_text(content) + if shape == "cut short": + path.write_bytes(written[: len(written) // 2]) + elif shape == "no testsuite": + path = _report(tmp_path, written) result = _gate(path) assert result.returncode == 1 - assert "cannot read JUnit report" in result.stderr + # The counters are printed only after the report is read. + assert result.stdout == "" -def test_negative_allowance_is_refused(tmp_path: Path) -> None: - result = _gate(_report(tmp_path, (5, 0, 0, 0)), "--max-skipped", "-1") +def test_negative_allowance_is_refused(tmp_path: Path, written: bytes) -> None: + report = tmp_path / "junit.xml" + report.write_bytes(written) + result = _gate(report, "--max-skipped", "-1") assert result.returncode == 2 - assert "--max-skipped must be 0 or more" in result.stderr + assert result.stdout == "" @pytest.mark.parametrize( - ("test_body", "gate_exit"), + ("test_body", "skipped", "gate_exit"), [ - ("@pytest.mark.skip\ndef test_a():\n pass\n", 1), - ("def test_a():\n pass\n", 0), + ("@pytest.mark.skip\ndef test_a():\n pass\n", 1, 1), + ("def test_a():\n pass\n", 0, 0), ], ) -def test_report_written_by_pytest(tmp_path: Path, test_body: str, gate_exit: int) -> None: - (tmp_path / "test_sample.py").write_text(f"import pytest\n\n\n{test_body}") - report = tmp_path / "junit.xml" - run = subprocess.run( - [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", f"--junitxml={report}"], - cwd=tmp_path, - capture_output=True, - text=True, - timeout=60, - ) +def test_report_written_by_pytest( + tmp_path: Path, test_body: str, skipped: int, gate_exit: int +) -> None: + run = _pytest(tmp_path, test_body) # pytest exits 0 in both cases, including when every test was skipped. assert run.returncode == 0, run.stdout - assert _gate(report).returncode == gate_exit + result = _gate(tmp_path / "junit.xml") + assert result.returncode == gate_exit + assert _printed(result) == {"tests": 1, "failures": 0, "errors": 0, "skipped": skipped} diff --git a/tests/test_check_log_placeholders.py b/tests/test_check_log_placeholders.py index 745e76a..ac450a1 100644 --- a/tests/test_check_log_placeholders.py +++ b/tests/test_check_log_placeholders.py @@ -2,93 +2,47 @@ from __future__ import annotations +import os +import re import subprocess import sys from pathlib import Path -import pytest +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "check_log_placeholders.py" +SRC = ROOT / "src" -SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "check_log_placeholders.py" - -def _run(tmp_path: Path, source: str) -> subprocess.CompletedProcess[str]: - root = tmp_path / "pkg" - root.mkdir() - (root / "module.py").write_text(source) +def _run(*paths: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( - [sys.executable, str(SCRIPT), str(root)], capture_output=True, text=True, timeout=60 + [sys.executable, str(SCRIPT), *map(str, paths)], + capture_output=True, + text=True, + env={"PATH": os.environ.get("PATH", "")}, + timeout=60, ) -FLAGGED = [ - 'logger.info("found %s items", count)\n', - 'logger.warning("retry %d of %d", attempt, total)\n', - 'log.error("failed: %r", error)\n', - 'logger.opt(depth=1).debug("value %(name)s", values)\n', - 'logger.exception("lost %.2f%%", share)\n', - 'logger.log("INFO", "found %s", count)\n', - 'logger.success("done " "in %ss", seconds)\n', - 'logger.trace("%5d", n)\n', - "def f():\n logger.critical('%s', x)\n", - 'logger.info("found %(n)s items", n=3)\n', # loguru formats keyword arguments too - 'logger.warning("retry %s", attempt=2)\n', - 'logger.log("INFO", "found %s", count=3)\n', -] -ACCEPTED = [ - 'logger.info("found {} items", count)\n', - 'logger.info("found %s items")\n', # no arguments: loguru formats nothing - 'logger.info("100%% done", count)\n', # a literal percent sign, no directive - "logger.info(template, count)\n", # not a literal - 'logger.info(f"found {count} items")\n', - 'console.print("found %s", count)\n', # not a log level - 'logger.log("INFO", "found {}", count)\n', - "DECOY = \"logger.info('found %s', count)\"\n", - 'logger.info("a % b", count)\n', # `% b` is a space flag without a conversion - 'logger.info("found {n} items", n=3)\n', - "logger.info(**fields)\n", # no message at all -] - - -@pytest.mark.parametrize("source", FLAGGED) -def test_flags_a_percent_style_message(tmp_path: Path, source: str) -> None: - result = _run(tmp_path, source) - assert result.returncode == 1, result.stdout - assert "%-style loguru messages: 1" in result.stdout - - -@pytest.mark.parametrize("source", ACCEPTED) -def test_accepts_brace_style_and_other_calls(tmp_path: Path, source: str) -> None: - result = _run(tmp_path, source) - assert result.returncode == 0, result.stdout - assert "%-style loguru messages: 0" in result.stdout - - -def test_reports_each_site(tmp_path: Path) -> None: - result = _run( - tmp_path, 'logger.info("a %s", x)\nlogger.debug("b {}", y)\nlogger.error("c %d", z)\n' +def _python_file_count(root: Path) -> int: + """The .py and .pyi files under ``root``, counted with os.walk rather than the script's glob.""" + return sum( + Path(name).suffix in {".py", ".pyi"} for _, _, names in os.walk(root) for name in names ) - assert result.returncode == 1 - assert "%-style loguru messages: 2" in result.stdout - assert "module.py:1:" in result.stdout - assert "module.py:3:" in result.stdout def test_no_python_files_is_an_error(tmp_path: Path) -> None: (tmp_path / "empty").mkdir() - result = subprocess.run( - [sys.executable, str(SCRIPT), str(tmp_path / "empty")], - capture_output=True, - text=True, - timeout=60, - ) + result = _run(tmp_path / "empty") assert result.returncode == 1 - assert "no Python files" in result.stderr + # A run that finds a site also exits 1, but only after reporting what it scanned. + assert result.stdout == "" def test_src_has_no_percent_style_messages() -> None: - src = Path(__file__).resolve().parents[1] / "src" - result = subprocess.run( - [sys.executable, str(SCRIPT), str(src)], capture_output=True, text=True, timeout=60 - ) + result = _run(SRC) assert result.returncode == 0, result.stdout - assert "%-style loguru messages: 0" in result.stdout + scanned = re.search(r"^scanned (\d+) files", result.stdout, re.MULTILINE) + found = re.search(r"^%-style loguru messages: (\d+)$", result.stdout, re.MULTILINE) + assert scanned is not None and found is not None, result.stdout + assert int(scanned.group(1)) == _python_file_count(SRC) + assert int(found.group(1)) == 0 diff --git a/tests/test_count_suppressions.py b/tests/test_count_suppressions.py index f10ff94..21eeefb 100644 --- a/tests/test_count_suppressions.py +++ b/tests/test_count_suppressions.py @@ -1,15 +1,22 @@ """The suppression-comment caps that pre-commit enforces on src/.""" +import importlib.util +import io import os import re import subprocess import sys +import tokenize from pathlib import Path +from types import ModuleType import pytest -SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "count_suppressions.py" +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "count_suppressions.py" +SRC = ROOT / "src" COUNT_LINE = re.compile(r"^(type: ignore|noqa) comments: (\d+) \(max \d+\)$", re.MULTILINE) +SCANNED_LINE = re.compile(r"^scanned (\d+) files", re.MULTILINE) def _run( @@ -27,6 +34,7 @@ def _run( ], capture_output=True, text=True, + env={"PATH": os.environ.get("PATH", "")}, timeout=60, ) @@ -36,236 +44,133 @@ def _counts(result: subprocess.CompletedProcess[str]) -> tuple[int, int]: return int(found["type: ignore"]), int(found["noqa"]) -def _package(tmp_path: Path, *sources: str) -> Path: - root = tmp_path / "pkg" - (root / "sub").mkdir(parents=True) - for i, source in enumerate(sources): - (root / ("sub" if i % 2 else ".") / f"module_{i}.py").write_text(source, encoding="utf-8") - return root - - -# Spellings counted as suppressions; ruff 0.14.11 and mypy 1.19.1 honor each one except where a -# comment below says otherwise. -NOQA_SPELLINGS = [ - "import os # noqa\n", - "import os # NOQA\n", - "import os #noqa\n", - "import os # noqa:F401\n", - "import os # NoQa: F401\n", - "import os # unused on purpose # noqa: F401\n", - "# ruff: noqa\nimport os\n", - "# ruff: noqa: F401\nimport os\n", - "#ruff:noqa\nimport os\n", - "# flake8: noqa\nimport os\n", - "# flake8: noqa: F401\nimport os\n", - "# isort: skip_file\nimport sys\nimport os\n", - "# isort: off\nimport sys\nimport os\n", - "import sys # isort: skip\nimport os\n", - "# ruff: isort: skip_file\nimport sys\nimport os\n", - "# ruff: isort: off\nimport sys\nimport os\n", - # Range suppressions, which ruff 0.14.11 honors only in preview. - "# ruff: disable[F401]\nimport os\n", - "#ruff:disable[F401, E501]\nimport os\n", - "# ruff: disable [F401]\nimport os\n", - "# ruff : disable[F401]\nimport os\n", - "#\truff:\tdisable\t[F401]\nimport os\n", - "#\u3000ruff:\u3000disable[F401]\nimport os\n", - "# ruff: disable[ F401, ] # reason\nimport os\n", - "# ruff: disable[F401] reason\nimport os\n", - "def f() -> None:\n # ruff: disable[F401]\n import os\n", - "# ruff: disable[F401]\nimport os\n# ruff: enable[F401]\n", - # Not honored by ruff (uppercase), counted so that no honored spelling escapes. - "# RUFF: DISABLE[F401]\nimport os\n", -] -TYPE_IGNORE_SPELLINGS = [ - 'x: int = "a" # type: ignore[assignment]\n', - 'x: int = "a" # type:ignore[assignment]\n', - 'x: int = "a" #type:ignore\n', - 'x: int = "a" # type: ignore\n', - '# type: ignore\nx: int = "a"\n', - '# mypy: ignore-errors\nx: int = "a"\n', - '# mypy: disable-error-code="assignment"\nx: int = "a"\n', - "from typing import no_type_check\n\n\n@no_type_check\ndef f() -> int:\n return 'x'\n", - "import typing\n\n\n@typing.no_type_check\nclass C:\n x: int = 'x'\n", - "from typing import no_type_check\n\n\n@no_type_check\nasync def f() -> int:\n return 'x'\n", - "from typing import no_type_check as ntc\n\n\n@ntc\ndef f() -> int:\n return 'x'\n", - "from typing_extensions import no_type_check as ntc\n\n\n" - "@ntc\ndef f() -> int:\n return 'x'\n", - # A re-export of typing's no_type_check, as mypy follows it. - "from pkg.compat import no_type_check as skip\n\n\n@skip\ndef f() -> int:\n return 'x'\n", -] -NOT_SUPPRESSIONS = [ - 'TEXT = "# noqa: F401, # type: ignore[misc], # ruff: noqa, # mypy: ignore-errors"\n', - '"""Docstring.\n\n# noqa\n# type: ignore\n"""\n', - "# this module needs no noqa and no type checking exceptions\nimport os\n", - "import os\n# isort: on\n", - 'RANGE = "# ruff: disable[F401]"\nimport os\n', - "# ruff: enable[F401]\nimport os\n", - "# ruff: disabled checks are listed in pyproject.toml\nimport os\n", - "def no_type_check() -> None:\n pass\n\n\nno_type_check()\n", -] - - -@pytest.mark.parametrize("source", NOQA_SPELLINGS) -def test_counts_every_noqa_spelling(tmp_path: Path, source: str) -> None: - assert _counts(_run(_package(tmp_path, source))) == (0, 1) - - -@pytest.mark.parametrize("source", TYPE_IGNORE_SPELLINGS) -def test_counts_every_type_ignore_spelling(tmp_path: Path, source: str) -> None: - assert _counts(_run(_package(tmp_path, source))) == (1, 0) - - -@pytest.mark.parametrize("source", NOT_SUPPRESSIONS) -def test_directive_text_outside_a_directive_comment_is_not_counted( - tmp_path: Path, source: str -) -> None: - assert _counts(_run(_package(tmp_path, source))) == (0, 0) +def _scanned(result: subprocess.CompletedProcess[str]) -> int: + match = SCANNED_LINE.search(result.stdout) + assert match is not None, result.stdout + return int(match.group(1)) + + +def _load() -> ModuleType: + """The script as a module, so its functions can be called in process.""" + spec = importlib.util.spec_from_file_location("count_suppressions", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module -def test_counts_stub_files(tmp_path: Path) -> None: - root = tmp_path / "pkg" - root.mkdir() - (root / "stub.pyi").write_text("x: int # type: ignore[misc]\ny: int # noqa: E501\n") - result = _run(root) - assert result.stdout.startswith("scanned 1 files") - assert _counts(result) == (1, 1) +def _walk(root: Path) -> list[Path]: + """Every regular file at or under ``root``, listed with os.walk rather than a glob.""" + if root.is_file(): + return [root] + return sorted( + Path(directory, name) + for directory, _, names in os.walk(root) + for name in names + if Path(directory, name).is_file() + ) -def test_counts_across_files_and_subdirectories(tmp_path: Path) -> None: - root = _package(tmp_path, *NOQA_SPELLINGS[:3], *TYPE_IGNORE_SPELLINGS[:2]) - result = _run(root) - assert _counts(result) == (2, 3) - assert result.stdout.startswith("scanned 5 files") +def _python_files(root: Path) -> list[Path]: + return [path for path in _walk(root) if path.suffix in {".py", ".pyi"}] + + +def _second_reading(root: Path) -> dict[str, list[str]]: + """The suppression sites under ``root``, as the script lists them, read a second way: comment + tokens whose text, without whitespace and in lowercase, holds a directive, and decorator lines + naming no_type_check.""" + found: dict[str, list[str]] = {"type: ignore": [], "noqa": []} + for path in _python_files(root): + source = path.read_bytes() + for token in tokenize.tokenize(io.BytesIO(source).readline): + if token.type != tokenize.COMMENT: + continue + text = "".join(token.string.lower().split()) + if "type:ignore" in text or "mypy:" in text: + found["type: ignore"].append(f"{path}:{token.start[0]}") + if any(word in text for word in ("noqa", "ruff:disable", "isort:skip", "isort:off")): + found["noqa"].append(f"{path}:{token.start[0]}") + found["type: ignore"] += [ + f"{path}:{number}" + for number, line in enumerate(source.decode().splitlines(), start=1) + if line.lstrip().startswith("@") and "no_type_check" in line + ] + return found + + +def test_src_is_within_the_hook_caps() -> None: + result = _run(SRC) + assert result.returncode == 0, result.stdout + assert result.stderr == "" + assert _scanned(result) == len(_python_files(SRC)) + sites = _second_reading(SRC) + assert _counts(result) == (len(sites["type: ignore"]), len(sites["noqa"])) @pytest.mark.parametrize( - ("sources", "caps", "kind"), - [ - (NOQA_SPELLINGS[:3], {"max_noqa": 3}, "noqa"), - (TYPE_IGNORE_SPELLINGS[:5], {"max_type_ignore": 5}, "type: ignore"), - ], + ("kind", "cap"), [("type: ignore", "max_type_ignore"), ("noqa", "max_noqa")] ) -def test_at_the_cap_passes_and_one_more_fails( - tmp_path: Path, sources: list[str], caps: dict[str, int], kind: str -) -> None: - at_cap = _run(_package(tmp_path, *sources), **caps) - assert at_cap.returncode == 0, at_cap.stdout - - over = {name: value - 1 for name, value in caps.items()} - over_cap = _run(tmp_path / "pkg", **over) +def test_at_the_cap_passes_and_one_below_fails(kind: str, cap: str) -> None: + sites = _second_reading(SRC) + at_cap = {"max_type_ignore": len(sites["type: ignore"]), "max_noqa": len(sites["noqa"])} + passing = _run(SRC, **at_cap) + assert passing.returncode == 0, passing.stdout + + # One below the count; -1 where src holds no site of the kind. + over_cap = _run(SRC, **{**at_cap, cap: at_cap[cap] - 1}) assert over_cap.returncode == 1 listed = [line.strip() for line in over_cap.stdout.splitlines() if line.startswith(" ")] - assert len(listed) == len(sources) - assert all(re.fullmatch(r".+\.py:\d+", site) for site in listed) - assert f"{kind} comments: {len(sources)}" in over_cap.stdout - - -# Files tokenize may read differently from ruff, or that pre-commit passes to ruff without a -# lowercase .py/.pyi suffix (identify tags them python). -REFUSED = [ - pytest.param( - "module.py", - "# coding: latin-1\n# ruff:\u3000disable[F401]\nimport os\n".encode(), - False, - id="latin-1 declaration", - ), - pytest.param( - "module.py", - b'# coding: unicode_escape\n# \\x0a"""\n# ruff: noqa: F401\n# \\x0a"""\n', - False, - id="unicode_escape declaration", - ), - pytest.param("module.py", b"# coding: no-such-codec\nimport os\n", False, id="unknown codec"), - pytest.param("module.py", b'"""Doc."""\r# ruff: noqa: F401\rimport os\r', False, id="lone CR"), - pytest.param("module.PY", b"import os # noqa: F401\n", False, id="uppercase suffix"), - pytest.param( - "stub.PYI", b"x: int = undefined_name # noqa: F821\n", False, id="uppercase stub suffix" - ), - pytest.param( - "tool", b"#!/usr/bin/env python3\nimport os # noqa: F401\n", True, id="python shebang" - ), - pytest.param("app.wsgi", b"import os # noqa: F401\n", False, id="wsgi suffix"), - pytest.param( - "tool", - b"#!/usr/bin/env nix-shell\n#! nix-shell -i python3\nimport os # noqa: F401\n", - True, - id="nix-shell shebang", - ), -] -ACCEPTED = [ - pytest.param("module.py", b"# -*- coding: utf-8 -*-\nimport os\n", False, id="utf-8"), - pytest.param("module.py", b"\xef\xbb\xbfimport os\n", False, id="byte-order mark"), - pytest.param("module.py", b"import os\r\nimport sys\r\n", False, id="CRLF"), - pytest.param( - "tool", b"#!/usr/bin/env python3\nimport os\n", False, id="shebang, not executable" - ), - pytest.param("tool.sh", b"#!/bin/sh\necho python\n", True, id="shell script"), - pytest.param( - "tool.sh", b"#!/usr/bin/env python3\nimport os\n", True, id="python shebang, shell suffix" - ), -] - - -@pytest.mark.parametrize(("name", "content", "executable"), REFUSED) -def test_refuses_a_file_it_cannot_read_as_ruff_does( - tmp_path: Path, name: str, content: bytes, executable: bool -) -> None: - root = _package(tmp_path, "import os\n") - path = root / name - path.write_bytes(content) - if executable: - path.chmod(0o755) - result = _run(root) - assert result.returncode == 1 - assert f"{path}: refused:" in result.stderr - assert "comments:" not in result.stdout + assert sorted(listed) == sorted(sites[kind]) + + +def test_directive_text_outside_a_directive_comment_is_not_counted() -> None: + # The script's docstring spells the directives it counts; text in a string is not a comment. + strings = [ + token.string + for token in tokenize.tokenize(io.BytesIO(SCRIPT.read_bytes()).readline) + if token.type == tokenize.STRING + ] + assert any("# noqa" in text and "# type: ignore" in text for text in strings) + result = _run(SCRIPT) + sites = _second_reading(SCRIPT) + assert result.returncode == 0, result.stdout + assert _scanned(result) == 1 + assert _counts(result) == (len(sites["type: ignore"]), len(sites["noqa"])) + + +def test_reads_every_file_under_src_as_ruff_does() -> None: + module = _load() + files = _walk(SRC) + assert list(module.all_files([SRC])) == files + assert [module.refusal(path) for path in files] == [None] * len(files) -@pytest.mark.parametrize(("name", "content", "executable"), ACCEPTED) -def test_accepts_files_it_reads_as_ruff_does( - tmp_path: Path, name: str, content: bytes, executable: bool +def test_refuses_a_file_identify_cannot_read( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: - root = _package(tmp_path, "import os\n") - path = root / name - path.write_bytes(content) - if executable: - path.chmod(0o755) - result = _run(root) - assert result.returncode == 0, result.stderr - assert "refused" not in result.stderr - - -def test_refuses_a_file_identify_cannot_read(tmp_path: Path) -> None: - root = _package(tmp_path, "import os\n") - locked = root / "LOCK" - locked.write_bytes(b"data\n") - locked.chmod(0o000) - try: - readable = os.access(locked, os.R_OK) # True only for root, which ignores the mode - result = _run(root) - finally: - locked.chmod(0o644) - if readable: - assert result.returncode == 0, result.stderr - else: - assert result.returncode == 1 - assert f"{locked}: refused: identify cannot read it" in result.stderr - - -def test_skips_entries_named_py_that_are_not_files(tmp_path: Path) -> None: - root = _package(tmp_path, "import os # noqa: F401\n") - (root / "pkg.py").mkdir() - (root / "pkg.py" / "inner.py").write_text("import sys\n") - (root / "gone.py").symlink_to(root / "missing.py") - result = _run(root) - assert result.returncode == 0, result.stderr - assert result.stdout.startswith("scanned 2 files") - assert _counts(result) == (0, 1) + module = _load() + typed = SRC / "github_activity_db" / "py.typed" + others = [str(path) for path in _walk(SRC) if path.suffix not in {".py", ".pyi"}] + arguments = [str(SRC), "--max-type-ignore", "5", "--max-noqa", "3"] + assert module.refusal(typed) is None + assert module.main(arguments) == 0 + capsys.readouterr() + + def unreadable(path: str) -> set[str]: + # The response the operating system gives a read of a file without read permission. + raise PermissionError(13, "Permission denied", path) + + monkeypatch.setattr(module, "tags_from_path", unreadable) + assert module.refusal(typed) is not None + assert module.main(arguments) == 1 + out, err = capsys.readouterr() + assert out == "" + assert [line.partition(": refused: ")[0] for line in err.splitlines()] == others def test_no_python_files_is_an_error(tmp_path: Path) -> None: (tmp_path / "empty").mkdir() result = _run(tmp_path / "empty") assert result.returncode == 1 - assert "no Python files" in result.stderr + # A run over files reports what it scanned before any cap fails it. + assert result.stdout == "" diff --git a/tests/test_declared_dependencies.py b/tests/test_declared_dependencies.py index e86bb86..fdde461 100644 --- a/tests/test_declared_dependencies.py +++ b/tests/test_declared_dependencies.py @@ -11,16 +11,23 @@ from __future__ import annotations import ast +import os import re import sys import tomllib +from collections.abc import Mapping, Sequence from dataclasses import dataclass from importlib.metadata import packages_distributions from pathlib import Path import pytest +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +# pyproject.toml as it was before httpx was declared (tests/fixtures/history/README.md). +BEFORE_HTTPX = ROOT / "tests" / "fixtures" / "history" / "ea7d12e" / "pyproject.toml" FIRST_PARTY = "github_activity_db" # sys.stdlib_module_names leaves out `__main__` and typeshed's type-checking-only `_typeshed`. STANDARD_LIBRARY = sys.stdlib_module_names | {"__main__", "_typeshed"} @@ -61,10 +68,17 @@ def _imported_top_levels(path: Path) -> set[str]: return names -def undeclared_imports(src: Path, pyproject: Path) -> tuple[int, list[Undeclared]]: - """The number of files scanned, and every import no declared dependency provides.""" +def undeclared_imports( + src: Path, pyproject: Path, owners: Mapping[str, Sequence[str]] | None = None +) -> tuple[int, list[Undeclared]]: + """The number of files scanned, and every import no declared dependency provides. + + ``owners`` maps top-level names to the distributions installing them; by default, the + installed environment's answer. + """ declared = _declared(pyproject) - owners = packages_distributions() + if owners is None: + owners = packages_distributions() files = sorted(src.rglob("*.py")) found: list[Undeclared] = [] for path in files: @@ -86,108 +100,76 @@ def _explain(found: list[Undeclared]) -> str: ) -def test_every_third_party_import_in_src_is_declared() -> None: - scanned, found = undeclared_imports(ROOT / "src", ROOT / "pyproject.toml") - assert scanned > 0 - assert found == [], _explain(found) +def _python_file_count(root: Path) -> int: + """The .py files under ``root``, counted with os.walk rather than the glob.""" + return sum(Path(name).suffix == ".py" for _, _, names in os.walk(root) for name in names) -def test_reports_undeclared_imports_in_a_planted_package(tmp_path: Path) -> None: - (tmp_path / "pyproject.toml").write_text( - '[project]\nname = "planted"\ndependencies = [" rich>=13.0", "Pydantic-Settings>=2"]\n' +def _importers(name: str) -> list[str]: + """The files under src/ with a statement importing ``name``, found in their text.""" + statement = re.compile(rf"^[ \t]*(?:from|import)[ \t]+{re.escape(name)}\b", re.MULTILINE) + return sorted( + path.relative_to(SRC).as_posix() + for path in SRC.rglob("*.py") + if statement.search(path.read_text()) ) - package = tmp_path / "src" / "planted" - package.mkdir(parents=True) - (package / "a.py").write_text( - "from __future__ import annotations\n" - "import json\n" - "import httpx\n" - "from click.exceptions import UsageError\n" - "from pydantic_settings import BaseSettings\n" - "from . import b\n" - "from github_activity_db import config\n" - "import __main__\n" - ) - (package / "b.py").write_text( - "from typing import TYPE_CHECKING\n" - "if TYPE_CHECKING:\n" - " import rich\n" - " import httpx.Timeout\n" - " from _typeshed import StrPath\n" - "def f() -> None:\n" - " import not_an_installed_module_xyz\n" - 'DECOY = "import sqlalchemy"\n' - ) - scanned, found = undeclared_imports(tmp_path / "src", tmp_path / "pyproject.toml") - assert scanned == 2 - assert found == [ - Undeclared("planted/a.py", "click", ("click",)), - Undeclared("planted/a.py", "httpx", ("httpx",)), - Undeclared("planted/b.py", "httpx", ("httpx",)), - Undeclared("planted/b.py", "not_an_installed_module_xyz", ()), - ] -def _install(site: Path, name: str, files: dict[str, str], *, record: bool = True) -> None: - """A distribution as an installer leaves it: its files, METADATA, and RECORD (or, without - RECORD, top_level.txt).""" - for relative, text in files.items(): - (site / relative).parent.mkdir(parents=True, exist_ok=True) - (site / relative).write_text(text) - info = site / f"{name.replace('-', '_')}-1.0.dist-info" - info.mkdir(parents=True) - (info / "METADATA").write_text(f"Metadata-Version: 2.1\nName: {name}\nVersion: 1.0\n") - if record: - rows = [*files, f"{info.name}/METADATA", f"{info.name}/RECORD"] - (info / "RECORD").write_text("".join(f"{row},,\n" for row in rows)) - else: - (info / "top_level.txt").write_text(f"{name.replace('-', '_')}\n") +def _requirement_names(pyproject: Path) -> set[str]: + """The [project].dependencies names, read with packaging rather than this test's pattern.""" + return { + canonicalize_name(Requirement(requirement).name) + for requirement in tomllib.loads(pyproject.read_text())["project"]["dependencies"] + } + + +def test_every_third_party_import_in_src_is_declared() -> None: + scanned, found = undeclared_imports(SRC, ROOT / "pyproject.toml") + assert scanned == _python_file_count(SRC) + assert found == [], _explain(found) + + +def test_reports_what_an_earlier_pyproject_did_not_declare() -> None: + """The dependencies before httpx was declared, read against src/: each file importing a + distribution declared since then is reported.""" + added = _requirement_names(ROOT / "pyproject.toml") - _requirement_names(BEFORE_HTTPX) + assert "httpx" in added + expected = [ + Undeclared(path, name.replace("-", "_"), (name,)) + for name in added + for path in _importers(name.replace("-", "_")) + ] + scanned, found = undeclared_imports(SRC, BEFORE_HTTPX) + assert scanned == _python_file_count(SRC) + assert sorted(found) == sorted(expected) @pytest.mark.parametrize( - ("source", "declared", "expected"), + ("installers", "reported"), [ pytest.param( - "from review_namespace.beta import PROVIDER\n", - '["review-alpha"]', - [Undeclared("app/m.py", "review_namespace", ("review-alpha", "review-beta"))], - id="a declared sibling does not stand in for the provider", + ["SQLAlchemy", "identify"], + ("identify", "sqlalchemy"), + id="a declared sibling does not stand in for an undeclared provider", ), - pytest.param( - "from review_namespace.beta import PROVIDER\n", - '["review-beta"]', - [Undeclared("app/m.py", "review_namespace", ("review-alpha", "review-beta"))], - id="a namespace shared with an undeclared distribution is ambiguous", - ), - pytest.param( - "from review_namespace import beta\nimport review_namespace.alpha\n", - '["review-alpha", "review-beta"]', - [], - id="every installing distribution declared", - ), - pytest.param( - "import legacy_thing.sub\n", - '["review-alpha"]', - [Undeclared("app/m.py", "legacy_thing", ("legacy-thing",))], - id="no RECORD, top_level.txt", - ), - pytest.param("import legacy_thing\n", '["legacy-thing"]', [], id="no RECORD, declared"), + pytest.param(["SQLAlchemy", "alembic"], None, id="every installing distribution declared"), + pytest.param(None, (), id="no installed distribution"), ], ) def test_a_name_several_distributions_install_needs_all_of_them_declared( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - source: str, - declared: str, - expected: list[Undeclared], + installers: list[str] | None, reported: tuple[str, ...] | None ) -> None: - site = tmp_path / "site" - _install(site, "review-alpha", {"review_namespace/alpha/__init__.py": 'PROVIDER = "a"\n'}) - _install(site, "review-beta", {"review_namespace/beta/__init__.py": 'PROVIDER = "b"\n'}) - _install(site, "legacy-thing", {"legacy_thing/__init__.py": ""}, record=False) - monkeypatch.syspath_prepend(str(site)) - (tmp_path / "pyproject.toml").write_text(f'[project]\nname = "x"\ndependencies = {declared}\n') - package = tmp_path / "src" / "app" - package.mkdir(parents=True) - (package / "m.py").write_text(source) - assert undeclared_imports(tmp_path / "src", tmp_path / "pyproject.toml") == (1, expected) + # The installed environment's answer, with sqlalchemy installed by the distributions given, + # as in a namespace several distributions share, or by none. + owners = dict(packages_distributions()) + if installers is None: + del owners["sqlalchemy"] + else: + owners["sqlalchemy"] = installers + scanned, found = undeclared_imports(SRC, ROOT / "pyproject.toml", owners) + assert scanned == _python_file_count(SRC) + importers = _importers("sqlalchemy") + assert importers + assert sorted(found) == ( + [] if reported is None else [Undeclared(path, "sqlalchemy", reported) for path in importers] + ) diff --git a/tests/test_import_boundaries.py b/tests/test_import_boundaries.py index 72b5a27..3aff3ad 100644 --- a/tests/test_import_boundaries.py +++ b/tests/test_import_boundaries.py @@ -16,13 +16,19 @@ from __future__ import annotations import ast +import io +import os +import re +import tokenize from collections.abc import Iterator from pathlib import Path -import pytest - ROOT = Path(__file__).resolve().parents[1] PACKAGE = ROOT / "src" / "github_activity_db" +GITHUBKIT_STATEMENT = re.compile(r"^[ \t]*(?:from|import)[ \t]+githubkit\b", re.MULTILINE) +DATABASE_STATEMENT = re.compile( + r"^[ \t]*(?:from|import)[ \t]+(?:sqlalchemy|github_activity_db\.db)\b", re.MULTILINE +) def _python_files(*roots: Path, excluding: Path | None = None) -> list[Path]: @@ -111,83 +117,93 @@ def database_imports(files: list[Path], src: Path) -> list[str]: ] +def _file_count(root: Path) -> int: + """The .py and .pyi files under ``root``, counted with os.walk rather than a glob.""" + return sum( + Path(name).suffix in {".py", ".pyi"} for _, _, names in os.walk(root) for name in names + ) + + +def _import_lines(path: Path, *, relative: bool) -> set[int]: + """The first line of each import in ``path``, read with tokenize rather than ast: a statement + holds one ``import`` keyword, ``from`` followed by ``.`` starts a relative one, and + ``import_module`` or ``__import__`` is called with a string literal first or as ``name=``.""" + tokens = [ + token + for token in tokenize.tokenize(io.BytesIO(path.read_bytes()).readline) + if token.type not in (tokenize.NL, tokenize.COMMENT) + ] + separators = (tokenize.ENCODING, tokenize.NEWLINE, tokenize.INDENT, tokenize.DEDENT) + lines: set[int] = set() + start = 0 + for index, token in enumerate(tokens): + if token.type in separators or token.string in (";", ":"): + start = index + 1 + elif token.type == tokenize.NAME and token.string == "import": + words = [word.string for word in tokens[start:index]] + if relative or not (words[:1] == ["from"] and words[1].startswith(".")): + lines.add(tokens[start].start[0]) + elif token.type == tokenize.NAME and token.string in ("import_module", "__import__"): + call = [ + "literal" if word.type == tokenize.STRING else word.string + for word in tokens[index + 1 : index + 5] + ] + if call[:2] == ["(", "literal"] or call == ["(", "name", "=", "literal"]: + lines.add(token.start[0]) + return lines + + +def _scanned_lines(files: list[Path], src: Path | None = None) -> set[tuple[Path, int]]: + return {(path, line) for path in files for line, _ in imported_modules(path, src)} + + +def _second_reading(files: list[Path], *, relative: bool) -> set[tuple[Path, int]]: + return {(path, line) for path in files for line in _import_lines(path, relative=relative)} + + +def _sites(found: list[str]) -> set[tuple[Path, int]]: + """The file and line of each finding, ``path:line: module``.""" + sites = set() + for finding in found: + path, _, line = finding.rpartition(": ")[0].rpartition(":") + sites.add((Path(path), int(line))) + return sites + + +def _statement_sites(files: list[Path], statement: re.Pattern[str]) -> set[tuple[Path, int]]: + """The file and line of each statement the pattern matches in the text of ``files``.""" + sites = set() + for path in files: + text = path.read_text() + sites |= { + (path, text.count("\n", 0, match.start()) + 1) for match in statement.finditer(text) + } + return sites + + def test_only_source_imports_githubkit() -> None: - files = _python_files(PACKAGE, ROOT / "scripts", ROOT / "alembic", excluding=PACKAGE / "source") - assert len(files) > 10, f"scanned {len(files)} files" + roots = (PACKAGE, ROOT / "scripts", ROOT / "alembic") + files = _python_files(*roots, excluding=PACKAGE / "source") + assert len(files) == sum(map(_file_count, roots)) - _file_count(PACKAGE / "source") + assert _scanned_lines(files) == _second_reading(files, relative=False) assert githubkit_imports(files) == [] def test_source_does_not_import_the_database() -> None: files = _python_files(PACKAGE / "source") - assert len(files) > 10, f"scanned {len(files)} files" + assert len(files) == _file_count(PACKAGE / "source") + assert _scanned_lines(files, ROOT / "src") == _second_reading(files, relative=True) assert database_imports(files, ROOT / "src") == [] -def test_the_scan_sees_source_importing_githubkit() -> None: - """Positive control: the same matcher finds the imports source/ is allowed to make.""" - assert githubkit_imports(_python_files(PACKAGE / "source")) - - -@pytest.mark.parametrize( - "source", - [ - "import githubkit\n", - "import githubkit.versions as versions\n", - "from githubkit import GitHub\n", - "from githubkit.exception import RequestFailed\n", - "def f():\n from githubkit import GitHub\n", - "from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from githubkit import GitHub\n", - "import importlib\nimportlib.import_module('githubkit')\n", - "from importlib import import_module\nimport_module('githubkit')\n", - "from importlib import import_module as load\nload('githubkit')\n", - "import builtins\nbuiltins.__import__('githubkit')\n", - "from builtins import __import__ as imp\nimp('githubkit')\n", - "import importlib\nimportlib.import_module(name='githubkit')\n", - "__import__(name='githubkit')\n", - "__import__('githubkit.versions')\n", - "class C:\n import githubkit\n", - ], -) -def test_flags_every_githubkit_import_spelling(tmp_path: Path, source: str) -> None: - (tmp_path / "module.py").write_text(source) - assert len(githubkit_imports(_python_files(tmp_path))) >= 1 - - -@pytest.mark.parametrize( - "source", - [ - 'DECOY = "import githubkit"\n', - "import githubkit_extensions\n", - "# import githubkit\n", - "import importlib\nname = 'githubkit'\nimportlib.import_module(name)\n", - "def load(name):\n return name\nload('githubkit')\n", - ], -) -def test_does_not_flag_other_text(tmp_path: Path, source: str) -> None: - (tmp_path / "module.py").write_text(source) - assert githubkit_imports(_python_files(tmp_path)) == [] - - -@pytest.mark.parametrize( - ("name", "source", "flagged"), - [ - ("transport.py", "import sqlalchemy\n", True), - ("transport.py", "from sqlalchemy.orm import Session\n", True), - ("transport.py", "from github_activity_db.db import models\n", True), - ("transport.py", "from github_activity_db.db.engine import get_session\n", True), - ("transport.py", "from ..db import engine\n", True), - ("transport.py", "from .. import db\n", True), - ("transport.py", "from .records import Account\n", False), - ("transport.py", "from ..tracking import load_tracking_config\n", False), - ("transport.py", "import sqlalchemy_utils\n", False), - ], -) -def test_flags_database_imports_from_source( - tmp_path: Path, name: str, source: str, flagged: bool -) -> None: - package = tmp_path / "github_activity_db" / "source" - package.mkdir(parents=True) - (package / "__init__.py").write_text("") - (package / name).write_text(source) - found = database_imports(_python_files(package), tmp_path) - assert bool(found) is flagged, found +def test_the_scan_finds_each_githubkit_import_in_source() -> None: + """source/ is allowed githubkit: the matcher finds each statement that imports it.""" + files = _python_files(PACKAGE / "source") + assert _sites(githubkit_imports(files)) == _statement_sites(files, GITHUBKIT_STATEMENT) + + +def test_the_scan_finds_each_database_import_in_sync() -> None: + """sync/ is allowed SQLAlchemy and db/: the matcher finds each statement that imports them.""" + files = _python_files(PACKAGE / "sync") + found = database_imports(files, ROOT / "src") + assert _sites(found) == _statement_sites(files, DATABASE_STATEMENT) diff --git a/tests/test_server_side_cursors.py b/tests/test_server_side_cursors.py index e2ad954..9731a32 100644 --- a/tests/test_server_side_cursors.py +++ b/tests/test_server_side_cursors.py @@ -14,10 +14,9 @@ from __future__ import annotations import ast +import os from pathlib import Path -import pytest - ROOT = Path(__file__).resolve().parents[1] SRC = ROOT / "src" CALLS = {"stream", "stream_scalars", "yield_per"} @@ -48,53 +47,9 @@ def server_side_cursors(files: list[Path]) -> list[str]: def test_no_server_side_cursor_in_src() -> None: files = sorted(SRC.rglob("*.py")) - assert len(files) > 30, f"scanned {len(files)} files" + # Counted with os.walk rather than the glob, so the scan cannot pass on fewer files. + assert len(files) == sum( + Path(name).suffix == ".py" for _, _, names in os.walk(SRC) for name in names + ) assert SRC / "github_activity_db" / "sync" / "snapshot.py" in files assert server_side_cursors(files) == [] - - -@pytest.mark.parametrize( - "source", - [ - "async def f(c, q):\n return await c.stream(q)\n", - "async def f(s, q):\n return await s.stream_scalars(q)\n", - "def f(r):\n return r.yield_per(100)\n", - "def f(q):\n return q.execution_options(yield_per=100)\n", - "def f(q):\n return q.execution_options(stream_results=True)\n", - "def f(q, flag):\n return q.execution_options(stream_results=flag)\n", - "def f(q):\n return q.execution_options(**{'stream_results': True})\n", - "OPTIONS = {'yield_per': 10}\n", - "class Reader:\n def read(self, c, q):\n return c.stream(q)\n", - ], - ids=[ - "stream", - "stream-scalars", - "yield-per-method", - "yield-per-option", - "stream-results-true", - "stream-results-variable", - "stream-results-dict", - "yield-per-dict", - "in-a-method", - ], -) -def test_flags_each_spelling(tmp_path: Path, source: str) -> None: - (tmp_path / "module.py").write_text(source) - assert len(server_side_cursors([tmp_path / "module.py"])) == 1 - - -@pytest.mark.parametrize( - "source", - [ - "DECOY = 'connection.stream(query)'\n", - "# rows = await connection.stream(query)\n", - "def f(q):\n return q.execution_options(stream_results=False)\n", - "def stream(x):\n return x\nstream(1)\n", - "async def f(c, q):\n return (await c.execute(q)).all()\n", - "OPTIONS = {'stream': True}\n", - ], - ids=["string", "comment", "stream-results-false", "a-function-named-stream", "execute", "key"], -) -def test_does_not_flag_other_code(tmp_path: Path, source: str) -> None: - (tmp_path / "module.py").write_text(source) - assert server_side_cursors([tmp_path / "module.py"]) == [] diff --git a/tests/test_suite_rules.py b/tests/test_suite_rules.py index 98d37e6..9a66b06 100644 --- a/tests/test_suite_rules.py +++ b/tests/test_suite_rules.py @@ -14,10 +14,9 @@ from __future__ import annotations import ast +import os from pathlib import Path -import pytest - TESTS = Path(__file__).resolve().parent WAITING = frozenset({"run", "call", "check_call", "check_output"}) # subprocess names that start no process. @@ -88,42 +87,8 @@ def subprocess_problems(*paths: Path) -> list[str]: def test_every_subprocess_wait_in_the_suite_has_a_timeout() -> None: files = sorted(TESTS.rglob("*.py")) - assert len(files) > 30, f"scanned {len(files)} files" + # Counted with os.walk rather than the glob, so the scan cannot pass on fewer files. + assert len(files) == sum( + Path(name).suffix == ".py" for _, _, names in os.walk(TESTS) for name in names + ) assert subprocess_problems(*files) == [] - - -@pytest.mark.parametrize( - ("source", "flagged"), - [ - ("import subprocess\nsubprocess.run(['true'])\n", True), - ("import subprocess\nsubprocess.check_output(['true'], text=True)\n", True), - ("import subprocess\nsubprocess.run(['true'], timeout=None)\n", True), - ("import subprocess\nsubprocess.run(['true'], **options)\n", True), - ("from subprocess import run\nrun(['true'])\n", True), - ("from subprocess import run as go\ngo(['true'])\n", True), - ("from subprocess import run\nwait = run\n", True), - ("import subprocess as sp\nsp.run(['true'], timeout=60)\n", True), - ("from subprocess import *\nrun(['true'], timeout=60)\n", True), - ("from subprocess import Popen\n", True), - ("import subprocess\nsubprocess.Popen(['true']).communicate()\n", True), - ("import subprocess\nsubprocess.getoutput('true')\n", True), - ("import subprocess\nmodule = subprocess\n", True), - ("import subprocess\nsubprocess.run(['true'], timeout=60)\n", False), - ("from subprocess import run\nrun(['true'], timeout=60)\n", False), - ( - "import subprocess\n" - "def f() -> subprocess.CompletedProcess[str]:\n" - " try:\n" - " return subprocess.run(['true'], timeout=60, stdout=subprocess.PIPE)\n" - " except subprocess.CalledProcessError:\n" - " raise\n", - False, - ), - ("def run(argv):\n return argv\nrun(['true'])\n", False), - ], -) -def test_flags_each_shape_it_does_not_model(tmp_path: Path, source: str, flagged: bool) -> None: - (tmp_path / "pkg").mkdir() - path = tmp_path / "pkg" / "module.py" - path.write_text(source) - assert bool(subprocess_problems(path)) is flagged diff --git a/tests/test_wheel_smoke.py b/tests/test_wheel_smoke.py index ec8b445..227a297 100644 --- a/tests/test_wheel_smoke.py +++ b/tests/test_wheel_smoke.py @@ -1,42 +1,53 @@ """The built-wheel smoke check rejects every way it could read a file other than the packaged one. -The CI job runs the script against an installed wheel. These tests run it in this environment -with a separate directory as the checkout argument, so the import-location check passes, and -use a copy of the package without tracking.toml for the missing-resource cases. +The CI job runs the script isolated (`-I`) against an installed wheel. These tests run it the same +way in this environment, with only the environment variables each test sets, and with a separate +directory as the checkout argument, so the import-location check passes. """ import os -import shutil +import re import subprocess import sys +import tomllib from pathlib import Path import pytest +import github_activity_db + REPO_ROOT = Path(__file__).resolve().parents[1] SCRIPT = REPO_ROOT / "scripts" / "wheel_smoke.py" SOURCE_TOML = REPO_ROOT / "src" / "github_activity_db" / "tracking.toml" SPELLINGS = ["TRACKING_CONFIG_PATH", "tracking_config_path", "Tracking_Config_Path"] +COUNT_LINE = re.compile(r"^tracked repositories from the packaged tracking\.toml: (\d+)$", re.M) def _run( - checkout: Path, - overrides: dict[str, str] | None = None, - package_root: Path | None = None, + checkout: Path, cwd: Path, overrides: dict[str, str] | None = None ) -> subprocess.CompletedProcess[str]: - env = {k: v for k, v in os.environ.items() if k.lower() != "tracking_config_path"} - env.update(overrides or {}) - if package_root is not None: - env["PYTHONPATH"] = str(package_root) return subprocess.run( - [sys.executable, str(SCRIPT), str(SOURCE_TOML), str(checkout)], + [sys.executable, "-I", str(SCRIPT), str(SOURCE_TOML), str(checkout)], capture_output=True, text=True, - env=env, + env={"PATH": os.environ.get("PATH", ""), **(overrides or {})}, + cwd=cwd, timeout=60, ) +def _tracked_count(result: subprocess.CompletedProcess[str]) -> int: + match = COUNT_LINE.search(result.stdout) + assert match is not None, result.stdout + return int(match.group(1)) + + +def _refused_value(result: subprocess.CompletedProcess[str]) -> Path: + """The value a refusal names in parentheses; a refusal prints one line and nothing else.""" + (line,) = result.stdout.splitlines() + return Path(line[line.index("(") + 1 : line.rindex(")")]) + + @pytest.fixture def checkout(tmp_path: Path) -> Path: path = tmp_path / "checkout" @@ -45,53 +56,45 @@ def checkout(tmp_path: Path) -> Path: @pytest.fixture -def package_without_resource(tmp_path: Path) -> Path: - """A copy of the package with tracking.toml removed, importable through PYTHONPATH.""" - root = tmp_path / "site" - shutil.copytree(REPO_ROOT / "src" / "github_activity_db", root / "github_activity_db") - (root / "github_activity_db" / "tracking.toml").unlink() - return root +def source_count() -> int: + """The repositories the source tracking.toml lists, read with tomllib.""" + return len(tomllib.loads(SOURCE_TOML.read_text())["repositories"]) -def test_passes_when_the_packaged_file_is_read(checkout: Path) -> None: - result = _run(checkout) +def test_passes_when_the_packaged_file_is_read(checkout: Path, source_count: int) -> None: + result = _run(checkout, checkout.parent) assert result.returncode == 0, result.stdout - assert "tracked repositories from the packaged tracking.toml: 29" in result.stdout + assert _tracked_count(result) == source_count @pytest.mark.parametrize("spelling", SPELLINGS) def test_rejects_an_override_in_any_letter_case(checkout: Path, spelling: str) -> None: - result = _run(checkout, {spelling: str(SOURCE_TOML)}) + result = _run(checkout, checkout.parent, {spelling: str(SOURCE_TOML)}) assert result.returncode == 1 - assert "tracking_config_path is set" in result.stdout - - -def test_an_empty_override_means_the_packaged_file(checkout: Path) -> None: - result = _run(checkout, {"TRACKING_CONFIG_PATH": ""}) - assert result.returncode == 0, result.stdout + assert _refused_value(result) == SOURCE_TOML + assert result.stderr == "" -def test_fails_when_the_packaged_file_is_missing( - checkout: Path, package_without_resource: Path -) -> None: - result = _run(checkout, package_root=package_without_resource) - output = result.stdout + result.stderr - assert result.returncode != 0 - assert "tracking config not found" in output - # The error names the copy, so the resource-less package is the one that was imported. - assert str(package_without_resource) in output +def test_rejects_an_override_before_reading_it(checkout: Path) -> None: + missing = checkout.parent / "missing.toml" + result = _run(checkout, checkout.parent, {"TRACKING_CONFIG_PATH": str(missing)}) + # Reading a missing file raises; the refusal comes first, and nothing else is printed. + assert result.returncode == 1 + assert _refused_value(result) == missing + assert result.stderr == "" -@pytest.mark.parametrize("spelling", SPELLINGS) -def test_an_override_cannot_hide_a_missing_packaged_file( - checkout: Path, package_without_resource: Path, spelling: str -) -> None: - result = _run(checkout, {spelling: str(SOURCE_TOML)}, package_without_resource) - assert result.returncode == 1 - assert "tracking_config_path is set" in result.stdout +def test_an_empty_override_means_the_packaged_file(checkout: Path, source_count: int) -> None: + result = _run(checkout, checkout.parent, {"TRACKING_CONFIG_PATH": ""}) + assert result.returncode == 0, result.stdout + assert _tracked_count(result) == source_count -def test_rejects_an_import_from_the_checkout() -> None: - result = _run(REPO_ROOT) +def test_rejects_an_import_from_the_checkout(tmp_path: Path) -> None: + # The editable install imports the package from the checkout's src/. + package_dir = Path(github_activity_db.__file__).resolve().parent + assert package_dir.is_relative_to(REPO_ROOT) + result = _run(REPO_ROOT, tmp_path) assert result.returncode == 1 - assert "imported from the checkout" in result.stdout + assert _refused_value(result) == package_dir + assert result.stderr == "" From c37d9a8f73c8c6d1343ab2afbf68350210f79963 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 08:01:14 -0600 Subject: [PATCH 05/29] Step 3b-2a-2d: source tests without planted inputs (D39, D40) tests/source/recorder.py: record() and record_first_sync() date units from the run's clock (SystemClock by default) instead of datetime.now. PrivateRepository carries full_name; RecordingFailed carries the failed items, the failed lookups, and the first sync's report. tests/source/test_recorder.py: every record() and first sync runs on a FakeClock and asserts recorded_at from it. Refusals assert structured fields: PrivateRepository.full_name, (number, CompareRefused, 404) for a failed item, ServerError causes for every looked-up id, and the first sync report's exit code, status, error and reason classes, and failure numbers. main() runs over the real recorder and the replay: an earlier recording at one clock, a later one at another, compared by file bytes. A scenario no longer listed leaves no file; a private resolve leaves the earlier recording; --only rate-limits replaces only the readings; --only documents drops a scenario's GraphQL units, keeps compare pages and readings, and adds the compare pages an earlier recording lacked. The patched MAX_FILE_BYTES is replaced by the largest recorded compare page with its first commit's message lengthened past the limit, for main() and --only first-sync. The '{}' and 'stale' placeholder files are replaced by recorder output. tests/source/test_recordings.py: the edited batch and continuation documents and the replay refusals of edited bindings are replaced. Refusing side: units the recorder wrote at Step 2 (c1f474b), committed unchanged under tests/fixtures/history/, are refused by the replay when the real source requests them (batch alias, comments continuation, resolve). Passing side: batch stamps at every size 1..PR_BATCH_SIZE equal the whole one-alias document, wrapper and rateLimit included; continuation stamps equal the one-alias document; every batch size and every continuation request size the hydrator sends passes the binding checks, with the aliases and variables counted; the compare requests for #8858 find exactly the recorded compare units. The no-unit refusal asserts the whole problem list. tests/source/test_scenario.py: the edited Resolve, index, and batch documents and the endCursor rewritten to a cursor never served are replaced: the model's stamps equal the current documents' for every kind and continued connection in the recordings' layout; a missing repository is answered null for the current index and batch documents; the repository-gone tests count the requests answered; each cursor the source sends is one the model served, one continuation per further page. The unanswered() test sends a request the replay cannot answer instead of appending to problems. tests/source/test_hydrate.py: an assignee listed twice is the same node again (first page and overflow); the github_id comparison is tested on its passing side over one page and two. PagingStalled asserts its fields instead of its message. tests/source/test_resolve_index_users.py: IndexOrderViolation asserts full_name, connection, number (from the recorded index page), and problem. tests/source/test_transport.py: the TypeError and ValueError raised from inside a client and the incompatible client are replaced by a check that the rate-limit reader names are methods of the installed githubkit; the JWT client failure asserts its RequestError cause; the system clock's jitter runs on a seeded random.Random. tests/test_logging.py: no sleep; level selection reads the console handler's lines; file, stdlib, and bound-context tests assert loguru records' level, message, and extra. tests/test_config.py: the docs/development.md and .env.example wording test is replaced by Settings reading a .env copied from .env.example; get_settings runs in a temporary working directory with its cache cleared. tests/test_tracking_config.py: the loader's refusals assert the cause type. --- .../comments/0e3c0d53c5b8c0eb.json | 115 +++++ .../pull_request/prebid__Prebid.js/13481.json | 196 +++++++ .../recorded/resolve/prebid__go-gdpr.json | 73 +++ tests/source/recorder.py | 64 ++- tests/source/test_hydrate.py | 53 +- tests/source/test_recorder.py | 479 ++++++++++++------ tests/source/test_recordings.py | 252 ++++----- tests/source/test_resolve_index_users.py | 21 +- tests/source/test_scenario.py | 153 ++++-- tests/source/test_transport.py | 70 +-- tests/test_config.py | 40 +- tests/test_logging.py | 207 ++++---- tests/test_tracking_config.py | 13 +- 13 files changed, 1194 insertions(+), 542 deletions(-) create mode 100644 tests/fixtures/history/c1f474b/tests/fixtures/source/recorded/overflow/PR_kwDOBUaT8s6WsAjJ/comments/0e3c0d53c5b8c0eb.json create mode 100644 tests/fixtures/history/c1f474b/tests/fixtures/source/recorded/pull_request/prebid__Prebid.js/13481.json create mode 100644 tests/fixtures/history/c1f474b/tests/fixtures/source/recorded/resolve/prebid__go-gdpr.json diff --git a/tests/fixtures/history/c1f474b/tests/fixtures/source/recorded/overflow/PR_kwDOBUaT8s6WsAjJ/comments/0e3c0d53c5b8c0eb.json b/tests/fixtures/history/c1f474b/tests/fixtures/source/recorded/overflow/PR_kwDOBUaT8s6WsAjJ/comments/0e3c0d53c5b8c0eb.json new file mode 100644 index 0000000..5314ac3 --- /dev/null +++ b/tests/fixtures/history/c1f474b/tests/fixtures/source/recorded/overflow/PR_kwDOBUaT8s6WsAjJ/comments/0e3c0d53c5b8c0eb.json @@ -0,0 +1,115 @@ +{ + "data": { + "comments": { + "nodes": [ + { + "author": { + "__typename": "User", + "databaseId": 5030553, + "id": "MDQ6VXNlcjUwMzA1NTM=", + "login": "foxart" + }, + "fullDatabaseId": "5668701353" + }, + { + "author": { + "__typename": "Bot", + "databaseId": 41898282, + "id": "MDM6Qm90NDE4OTgyODI=", + "login": "github-actions" + }, + "fullDatabaseId": "5712770698" + }, + { + "author": { + "__typename": "User", + "databaseId": 5030553, + "id": "MDQ6VXNlcjUwMzA1NTM=", + "login": "foxart" + }, + "fullDatabaseId": "5714404712" + }, + { + "author": { + "__typename": "Bot", + "databaseId": 41898282, + "id": "MDM6Qm90NDE4OTgyODI=", + "login": "github-actions" + }, + "fullDatabaseId": "5725445429" + }, + { + "author": { + "__typename": "Bot", + "databaseId": 41898282, + "id": "MDM6Qm90NDE4OTgyODI=", + "login": "github-actions" + }, + "fullDatabaseId": "5755731896" + }, + { + "author": { + "__typename": "User", + "databaseId": 5030553, + "id": "MDQ6VXNlcjUwMzA1NTM=", + "login": "foxart" + }, + "fullDatabaseId": "5799575501" + }, + { + "author": { + "__typename": "Bot", + "databaseId": 41898282, + "id": "MDM6Qm90NDE4OTgyODI=", + "login": "github-actions" + }, + "fullDatabaseId": "5809962526" + }, + { + "author": { + "__typename": "User", + "databaseId": 5030553, + "id": "MDQ6VXNlcjUwMzA1NTM=", + "login": "foxart" + }, + "fullDatabaseId": "5867743001" + }, + { + "author": { + "__typename": "Bot", + "databaseId": 41898282, + "id": "MDM6Qm90NDE4OTgyODI=", + "login": "github-actions" + }, + "fullDatabaseId": "5952640992" + } + ], + "pageInfo": { + "endCursor": "Y3Vyc29yOnYyOpHPAAAAAWLOF-A=", + "hasNextPage": false + }, + "totalCount": 109 + }, + "headRefOid": "c0dd6b96762622636d52f1241dca4a4ad1bb6305", + "updatedAt": "2026-10-02T14:07:24Z" + }, + "document_sha256": "b890a54b57991076fe5706269e44334f7dcbe150ff8f9f228f4a326011367c9d", + "errors": [], + "extra": {}, + "headers": { + "date": "Sat, 17 Oct 2026 12:09:31 GMT", + "x-ratelimit-limit": "5000", + "x-ratelimit-remaining": "4983", + "x-ratelimit-reset": "1792242556", + "x-ratelimit-resource": "graphql", + "x-ratelimit-used": "17" + }, + "key": { + "after": "Y3Vyc29yOnYyOpHPAAAAAU7JU7k=", + "connection": "comments", + "id": "PR_kwDOBUaT8s6WsAjJ" + }, + "kind": "overflow", + "recorded_at": "2026-10-17T12:09:19Z", + "source": "live" +} diff --git a/tests/fixtures/history/c1f474b/tests/fixtures/source/recorded/pull_request/prebid__Prebid.js/13481.json b/tests/fixtures/history/c1f474b/tests/fixtures/source/recorded/pull_request/prebid__Prebid.js/13481.json new file mode 100644 index 0000000..7152413 --- /dev/null +++ b/tests/fixtures/history/c1f474b/tests/fixtures/source/recorded/pull_request/prebid__Prebid.js/13481.json @@ -0,0 +1,196 @@ +{ + "data": { + "additions": 141, + "author": { + "__typename": "User", + "databaseId": 10151381, + "id": "MDQ6VXNlcjEwMTUxMzgx", + "login": "dgirardi" + }, + "authorAssociation": "MEMBER", + "baseRefName": "master", + "baseRefOid": "55cbaa384a204334a72c264dc3344593dd13de2e", + "body": "- **bump coveralls**\n- **make symlink**\n- **Fix symlink**\n\n\n\n## Type of change\n\n- [ ] Bugfix\n- [ ] Feature\n- [ ] New bidder adapter \n- [ ] Updated bidder adapter \n- [ ] Code style update (formatting, local variables)\n- [ ] Refactoring (no functional changes, no api changes)\n- [ ] Build related changes\n- [ ] CI related changes\n\n- [ ] Does this change affect user-facing APIs or examples documented on http://prebid.org?\n- [ ] Other\n\n## Description of change\n\n\n\n\n\n## Other information\n\n", + "changedFiles": 2, + "closingIssuesReferences": { + "nodes": [], + "pageInfo": { + "endCursor": null, + "hasNextPage": false + }, + "totalCount": 0 + }, + "comments": { + "nodes": [], + "pageInfo": { + "endCursor": null, + "hasNextPage": false + }, + "totalCount": 0 + }, + "commits": { + "nodes": [ + { + "commit": { + "author": { + "email": "patmmccann@gmail.com", + "name": "Patrick McCann", + "user": { + "databaseId": 1683175, + "id": "MDQ6VXNlcjE2ODMxNzU=", + "login": "patmmccann" + } + }, + "authoredDate": "2025-06-11T12:45:31Z", + "oid": "54466d46dcda742a8678fe975a844a7e9daa26af" + } + }, + { + "commit": { + "author": { + "email": "patmmccann@gmail.com", + "name": "Patrick McCann", + "user": { + "databaseId": 1683175, + "id": "MDQ6VXNlcjE2ODMxNzU=", + "login": "patmmccann" + } + }, + "authoredDate": "2025-07-01T14:07:34Z", + "oid": "c0d7d2cff7989243964e5b0c6f2e6dc81c2e50df" + } + }, + { + "commit": { + "author": { + "email": "dgirardi@prebid.org", + "name": "Demetrio Girardi", + "user": { + "databaseId": 10151381, + "id": "MDQ6VXNlcjEwMTUxMzgx", + "login": "dgirardi" + } + }, + "authoredDate": "2025-07-01T14:50:50Z", + "oid": "8ee0fc1bbc99bedd370697b93a63001a097ffd46" + } + }, + { + "commit": { + "author": { + "email": "dgirardi@prebid.org", + "name": "Demetrio Girardi", + "user": { + "databaseId": 10151381, + "id": "MDQ6VXNlcjEwMTUxMzgx", + "login": "dgirardi" + } + }, + "authoredDate": "2025-07-01T14:52:41Z", + "oid": "cfa0b27d26e3f4bdd0cce3750ec0290b7ecfaf12" + } + } + ], + "pageInfo": { + "endCursor": "NA", + "hasNextPage": false + }, + "totalCount": 4 + }, + "createdAt": "2025-07-01T14:51:49Z", + "deletions": 381, + "files": { + "nodes": [ + { + "additions": 0, + "changeType": "DELETED", + "deletions": 1, + "path": "CLAUDE.md" + }, + { + "additions": 1, + "changeType": "ADDED", + "deletions": 0, + "path": "CLAUDE.md" + }, + { + "additions": 140, + "changeType": "MODIFIED", + "deletions": 380, + "path": "package-lock.json" + } + ], + "pageInfo": { + "endCursor": "Mw", + "hasNextPage": false + }, + "totalCount": 3 + }, + "fullDatabaseId": "2632763795", + "headRefOid": "cfa0b27d26e3f4bdd0cce3750ec0290b7ecfaf12", + "id": "PR_kwDOAlf9v86c7MmT", + "isCrossRepository": false, + "isDraft": false, + "labels": { + "nodes": [ + { + "name": "maintenance" + } + ], + "pageInfo": { + "endCursor": "MQ", + "hasNextPage": false + }, + "totalCount": 1 + }, + "mergedAt": "2025-07-01T14:55:24Z", + "mergedBy": { + "__typename": "User", + "databaseId": 1683175, + "id": "MDQ6VXNlcjE2ODMxNzU=", + "login": "patmmccann" + }, + "number": 13481, + "reviews": { + "nodes": [ + { + "author": { + "__typename": "User", + "databaseId": 1683175, + "id": "MDQ6VXNlcjE2ODMxNzU=", + "login": "patmmccann" + }, + "fullDatabaseId": "2975840076", + "state": "APPROVED" + } + ], + "pageInfo": { + "endCursor": "Y3Vyc29yOnYyOpO0MjAyNS0wNy0wMVQxNDo1NDo0N1q0MjAyNS0wNy0wMVQxNDo1NDo0N1rOsV-3TA==", + "hasNextPage": false + }, + "totalCount": 1 + }, + "state": "MERGED", + "title": "link CLAUDE.md to AGENTS.md", + "updatedAt": "2025-07-01T14:55:26Z" + }, + "document_sha256": "39fd2b5d7744d57be4fd58ba38b2fcf50319c5bd19e1c3796ae4771203af3ea0", + "errors": [], + "extra": {}, + "headers": { + "date": "Sat, 17 Oct 2026 12:09:24 GMT", + "x-ratelimit-limit": "5000", + "x-ratelimit-remaining": "4929", + "x-ratelimit-reset": "1792241340", + "x-ratelimit-resource": "graphql", + "x-ratelimit-used": "71" + }, + "key": { + "name": "Prebid.js", + "number": 13481, + "owner": "prebid" + }, + "kind": "pull_request", + "recorded_at": "2026-10-17T12:09:19Z", + "source": "live" +} diff --git a/tests/fixtures/history/c1f474b/tests/fixtures/source/recorded/resolve/prebid__go-gdpr.json b/tests/fixtures/history/c1f474b/tests/fixtures/source/recorded/resolve/prebid__go-gdpr.json new file mode 100644 index 0000000..e77b170 --- /dev/null +++ b/tests/fixtures/history/c1f474b/tests/fixtures/source/recorded/resolve/prebid__go-gdpr.json @@ -0,0 +1,73 @@ +{ + "data": { + "databaseId": 132915235, + "defaultBranchRef": { + "name": "master" + }, + "id": "MDEwOlJlcG9zaXRvcnkxMzI5MTUyMzU=", + "isArchived": false, + "isPrivate": false, + "labels": { + "nodes": [ + { + "name": "bug" + }, + { + "name": "duplicate" + }, + { + "name": "enhancement" + }, + { + "name": "good first issue" + }, + { + "name": "help wanted" + }, + { + "name": "invalid" + }, + { + "name": "question" + }, + { + "name": "wontfix" + }, + { + "name": "dependencies" + }, + { + "name": "go" + } + ], + "pageInfo": { + "endCursor": "MTA", + "hasNextPage": false + }, + "totalCount": 10 + }, + "name": "go-gdpr", + "nameWithOwner": "prebid/go-gdpr", + "owner": { + "login": "prebid" + } + }, + "document_sha256": "035aef891393ccd81c36161af0b0d2e10034e582cac4bae68e62c9e5b035790e", + "errors": [], + "extra": {}, + "headers": { + "date": "Sat, 17 Oct 2026 12:09:20 GMT", + "x-ratelimit-limit": "5000", + "x-ratelimit-remaining": "4998", + "x-ratelimit-reset": "1792242556", + "x-ratelimit-resource": "graphql", + "x-ratelimit-used": "2" + }, + "key": { + "name": "go-gdpr", + "owner": "prebid" + }, + "kind": "resolve", + "recorded_at": "2026-10-17T12:09:19Z", + "source": "live" +} diff --git a/tests/source/recorder.py b/tests/source/recorder.py index 9fbab4c..472dcb5 100644 --- a/tests/source/recorder.py +++ b/tests/source/recorder.py @@ -27,14 +27,21 @@ import sys import tempfile from dataclasses import dataclass -from datetime import UTC, datetime +from datetime import datetime from pathlib import Path import httpx from github_activity_db.cli.sync import report_lines -from github_activity_db.source import Clock, Failed, GitHubSource, RepositoryRecord, SystemClock -from github_activity_db.sync.engine import SyncOptions, sync +from github_activity_db.source import ( + Clock, + Failed, + GitHubSource, + RepositoryRecord, + SystemClock, + UserLookupFailed, +) +from github_activity_db.sync.engine import SyncOptions, SyncReport, sync from github_activity_db.tracking import TrackingConfig, TrackingConfigFile, load_tracking_config from tests.db.alembic_support import upgrade from tests.source.recordings import FIRST_SYNC, RECORDED, Unit, split_graphql, split_rest @@ -108,7 +115,11 @@ class Scenario: class PrivateRepository(Exception): - pass + """``full_name`` is private: listed in ``PRIVATE``, or answered private by its resolve.""" + + def __init__(self, full_name: str) -> None: + super().__init__(full_name) + self.full_name = full_name def _refuse_listed(owner: str, name: str) -> None: @@ -134,7 +145,22 @@ async def resolve(self, owner: str, name: str) -> RepositoryRecord: class RecordingFailed(Exception): - """An item or a user lookup failed, so the recording would be incomplete.""" + """An item or a user lookup failed, or a first sync did not exit 0, so the recording would be + incomplete. ``failed`` holds the failed items, ``lookup_failed`` the failed lookups by node + id, and ``report`` the first sync's report.""" + + def __init__( + self, + message: str, + *, + failed: tuple[Failed, ...] = (), + lookup_failed: dict[str, UserLookupFailed] | None = None, + report: SyncReport | None = None, + ) -> None: + super().__init__(message) + self.failed = failed + self.lookup_failed = dict(lookup_failed or {}) + self.report = report class RecordingTransport(httpx.AsyncBaseTransport): @@ -195,11 +221,13 @@ async def record( The resolve's requests (its first response, then any further label pages) are the first ones for a repository, so a private one raises after them and before ``main`` writes - anything. + anything. ``clock`` is the run's clock, which also dates the units: the system clock by + default. """ - recorded_at = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + run_clock = clock or SystemClock() + recorded_at = run_clock.now().strftime("%Y-%m-%dT%H:%M:%SZ") recording = RecordingTransport(inner or httpx.AsyncHTTPTransport(), recorded_at) - async with PublicOnlySource(token, http_transport=recording, clock=clock) as source: + async with PublicOnlySource(token, http_transport=recording, clock=run_clock) as source: for scenario in scenarios: full_name = f"{scenario.owner}/{scenario.name}" _refuse_listed(scenario.owner, scenario.name) @@ -208,13 +236,15 @@ async def record( await source.index_pull_requests(repository) if scenario.numbers: async for outcomes in source.hydrate_pull_requests(repository, scenario.numbers): - failed = [outcome for outcome in outcomes if isinstance(outcome, Failed)] + failed = tuple(outcome for outcome in outcomes if isinstance(outcome, Failed)) if failed: - raise RecordingFailed(f"{full_name}: {failed}") + raise RecordingFailed(f"{full_name}: {list(failed)}", failed=failed) if users: async for lookup in source.lookup_users(users): if lookup.failed: - raise RecordingFailed(f"user lookup: {lookup.failed}") + raise RecordingFailed( + f"user lookup: {lookup.failed}", lookup_failed=dict(lookup.failed) + ) if rate_limits: await source.rate_limits() print(f"usage: {source.usage}", file=sys.stderr) @@ -241,13 +271,13 @@ async def record_first_sync( (its label pages included) and before any other request; a failed item or a run that does not exit 0 stops it too. - ``clock`` is the run's clock, the system clock by default. A replay of recorded units passes a - fixed one: the sync's 24-hour refresh measures each account from its recorded ``Date``, so on - the system clock a replay asks for lookups the recording does not hold once a day has - passed.""" + ``clock`` is the run's clock, which also dates the units: the system clock by default. A + replay of recorded units passes a fixed one: the sync's 24-hour refresh measures each account + from its recorded ``Date``, so on the system clock a replay asks for lookups the recording + does not hold once a day has passed.""" _refuse_listed(owner, name) run_clock = clock or SystemClock() - recorded_at = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + recorded_at = run_clock.now().strftime("%Y-%m-%dT%H:%M:%SZ") recording = RecordingTransport(inner or httpx.AsyncHTTPTransport(), recorded_at) opened: list[PublicOnlySource] = [] @@ -277,7 +307,7 @@ def open_source(deadline: datetime | None) -> PublicOnlySource: raise PrivateRepository(refused[0]) if report.exit_code != 0 or report.failures: lines = "; ".join(report_lines(report, seconds=0)) - raise RecordingFailed(f"first sync exited {report.exit_code}: {lines}") + raise RecordingFailed(f"first sync exited {report.exit_code}: {lines}", report=report) print( f"first sync: {report.items_hydrated} items, {report.graphql_points} GraphQL points", file=sys.stderr, diff --git a/tests/source/test_hydrate.py b/tests/source/test_hydrate.py index 494b4d7..8e06e15 100644 --- a/tests/source/test_hydrate.py +++ b/tests/source/test_hydrate.py @@ -859,48 +859,38 @@ async def test_a_short_page_on_an_added_connection(open_source, replay: Replay, @pytest.mark.parametrize("overflow", [False, True], ids=["first-page", "overflow"]) -async def test_an_assignee_node_id_listed_twice_with_two_github_ids( - open_source, replay: Replay, overflow: bool -) -> None: - # The github_ids differ, so only the node-id comparison can refuse this; its key is the - # node id, where the github_id comparison's is a number. +async def test_an_assignee_listed_twice(open_source, replay: Replay, overflow: bool) -> None: + # The same assignee again, in place of another or on a later page, as when the list shifts + # between pages. The node-id comparison refuses it first: its key is the node id, where the + # github_id comparison's is a number. unit = pr_unit(replay, 15771) first = dict(unit.data["assignedActors"]["nodes"][0]) - second = {**first, "databaseId": first["databaseId"] + 999999} if overflow: - second_page(replay, unit, "assignedActors", second) + second_page(replay, unit, "assignedActors", dict(first)) else: - unit.data["assignedActors"]["nodes"][1] = second + unit.data["assignedActors"]["nodes"][1] = dict(first) replay.add(unit) error = await failure(open_source, PREBID_JS, 15771) assert type(error) is DuplicateEntry assert (error.connection, error.key) == ("assignedActors", first["id"]) -async def test_an_assignee_github_id_listed_twice_under_two_node_ids( - open_source, replay: Replay -) -> None: - # pr_assignees keys an assignee by its account, which plan 5.2 resolves by databaseId, so - # two node ids for one databaseId cannot both be stored. - unit = pr_unit(replay, 15771) - nodes = unit.data["assignedActors"]["nodes"] - nodes[1] = {**nodes[0], "id": "U_another_node_id"} - replay.add(unit) - error = await failure(open_source, PREBID_JS, 15771) - assert isinstance(error, DuplicateEntry) - assert (error.connection, error.key) == ("assignedActors", nodes[0]["databaseId"]) - - -async def test_an_assignee_github_id_listed_again_on_a_later_page( - open_source, replay: Replay +@pytest.mark.parametrize("overflow", [False, True], ids=["first-page", "overflow"]) +async def test_each_assignee_github_id_is_distinct( + open_source, replay: Replay, overflow: bool ) -> None: + # pr_assignees keys an assignee by its account, which plan 5.2 resolves by databaseId: the + # github_ids compared, on one page or two, are each assignee's own. unit = pr_unit(replay, 15771) - first = unit.data["assignedActors"]["nodes"][0] - second_page(replay, unit, "assignedActors", {**first, "id": "U_another_node_id"}) - replay.add(unit) - error = await failure(open_source, PREBID_JS, 15771) - assert isinstance(error, DuplicateEntry) - assert (error.connection, error.key) == ("assignedActors", first["databaseId"]) + nodes = list(unit.data["assignedActors"]["nodes"]) + if overflow: + second_page(replay, unit, "assignedActors", EXTRA_ASSIGNEE) + replay.add(unit) + nodes.append(EXTRA_ASSIGNEE) + record = await hydrate_one(open_source, PREBID_JS, 15771) + github_ids = [assignee.github_id for assignee in record.assignees] + assert github_ids == [node["databaseId"] for node in nodes] + assert len(set(github_ids)) == len(nodes) async def test_a_duplicate_review_request(open_source, replay: Replay) -> None: @@ -1410,8 +1400,7 @@ def empty(body: dict[str, Any]) -> None: replay.modify(empty, when=is_overflow, times=2) error = await failure(open_source, PREBID_JS, 11155) assert isinstance(error, PagingStalled) - assert (error.cursor, error.entries) == ("fresh", 0) - assert "0 entries, cursor 'fresh'" in str(error) + assert (error.connection, error.cursor, error.entries) == ("commits", "fresh", 0) async def test_an_error_outside_every_alias_fails_the_items( self, open_source, replay: Replay diff --git a/tests/source/test_recorder.py b/tests/source/test_recorder.py index af89084..2ebeae0 100644 --- a/tests/source/test_recorder.py +++ b/tests/source/test_recorder.py @@ -5,33 +5,52 @@ import asyncio import dataclasses import json +import shutil from collections.abc import Callable +from datetime import timedelta from pathlib import Path from typing import Any import httpx import pytest +from github_activity_db.source.errors import CompareRefused, ServerError from github_activity_db.source.limits import RATE_LIMIT_FALLBACK_WAIT, SERVER_RETRY_DELAYS from tests.source import recorder -from tests.source.conftest import FakeClock +from tests.source.conftest import START, FakeClock from tests.source.recorder import ( FIRST_SYNC_REPOSITORY, + SCENARIOS, PrivateRepository, RecordingFailed, Scenario, record, record_first_sync, + write_units, ) from tests.source.recordings import FIRST_SYNC, RECORDED, Unit from tests.source.replay import Replay, synthetic +# An earlier recording's clock; the later one runs on FakeClock's default, START. +EARLIER = START - timedelta(hours=1) +# The scenario a later recording no longer lists: its units are its resolve and its one PR. +DROPPED = next(scenario for scenario in SCENARIOS if scenario.name == "prebid-cache") +# Prebid.js #8858 (254 commits) is the recorded compare fallback, the only one (SCENARIOS). +COMPARED = 8858 + + +def stamp(clock: FakeClock) -> str: + """``recorded_at`` of a unit recorded on ``clock``.""" + return clock.now().strftime("%Y-%m-%dT%H:%M:%SZ") + async def test_recording_the_replay_gives_back_every_unit() -> None: """Split (recorder) after assemble (replay) is the identity on the fixtures, per operation.""" replay = Replay.from_fixtures() - units = await record("test-token", inner=replay.transport()) + clock = FakeClock() + units = await record("test-token", inner=replay.transport(), clock=clock) assert replay.problems == [] + assert clock.sleeps == [] requests = {(s.operation, json.dumps(s.variables, sort_keys=True), s.path) for s in replay.seen} assert len(requests) == len(replay.seen), "a request was sent twice" produced = {(unit.kind, json.dumps(unit.key, sort_keys=True)): unit for unit in units} @@ -45,6 +64,7 @@ async def test_recording_the_replay_gives_back_every_unit() -> None: original.extra, original.headers, ), key + assert {unit.recorded_at for unit in units} == {stamp(clock)} async def test_a_private_repository_stops_the_run_after_its_resolve() -> None: @@ -56,8 +76,9 @@ async def test_a_private_repository_stops_the_run_after_its_resolve() -> None: Scenario("prebid", "prebid-cache-java", index=True), Scenario("prebid", "go-gdpr", index=True), ) - with pytest.raises(PrivateRepository, match="prebid/go-gdpr"): - await record("test-token", inner=replay.transport(), scenarios=scenarios) + with pytest.raises(PrivateRepository) as raised: + await record("test-token", inner=replay.transport(), scenarios=scenarios, clock=FakeClock()) + assert raised.value.full_name == "prebid/go-gdpr" operations = [seen.operation for seen in replay.seen] # prebid-cache-java: resolve and two index pages; go-gdpr: its resolve only; no users. assert operations == ["Resolve", "PullRequestIndex", "PullRequestIndex", "Resolve"] @@ -70,9 +91,17 @@ async def test_a_failed_item_stops_the_recording() -> None: when=lambda seen: seen.operation == "rest", times=None, ) - scenarios = (Scenario("prebid", "Prebid.js", numbers=(8858,)),) - with pytest.raises(RecordingFailed, match="compare answered 404"): - await record("test-token", inner=replay.transport(), scenarios=scenarios, users=()) + scenarios = (Scenario("prebid", "Prebid.js", numbers=(COMPARED,)),) + with pytest.raises(RecordingFailed) as raised: + await record( + "test-token", inner=replay.transport(), scenarios=scenarios, users=(), clock=FakeClock() + ) + failed = [ + (outcome.number, type(outcome.error), getattr(outcome.error, "status", None)) + for outcome in raised.value.failed + ] + assert failed == [(COMPARED, CompareRefused, 404)] + assert (raised.value.lookup_failed, raised.value.report) == ({}, None) async def test_a_failed_user_lookup_stops_the_recording() -> None: @@ -83,8 +112,11 @@ async def test_a_failed_user_lookup_stops_the_recording() -> None: times=None, ) clock = FakeClock() - with pytest.raises(RecordingFailed, match="user lookup"): + with pytest.raises(RecordingFailed) as raised: await record("test-token", inner=replay.transport(), scenarios=(), clock=clock) + causes = {node_id: type(error.cause) for node_id, error in raised.value.lookup_failed.items()} + assert causes == dict.fromkeys(recorder.USERS, ServerError) + assert raised.value.failed == () assert clock.sleeps == list(SERVER_RETRY_DELAYS) @@ -111,8 +143,11 @@ async def test_a_private_answer_comes_after_the_resolves_label_pages() -> None: unit.data["isPrivate"] = True replay.add(unit) scenarios = (Scenario("golang", "go", index=True),) - with pytest.raises(PrivateRepository, match="golang/go"): - await record("test-token", inner=replay.transport(), scenarios=scenarios, users=()) + with pytest.raises(PrivateRepository) as raised: + await record( + "test-token", inner=replay.transport(), scenarios=scenarios, users=(), clock=FakeClock() + ) + assert raised.value.full_name == "golang/go" assert [seen.operation for seen in replay.seen] == ["Resolve", "RepositoryLabels"] @@ -122,148 +157,232 @@ async def test_a_private_answer_comes_after_the_resolves_label_pages() -> None: async def test_a_listed_private_repository_is_never_requested(owner: str, name: str) -> None: replay = Replay.from_fixtures() scenarios = (Scenario(owner, name, index=True),) - with pytest.raises(PrivateRepository): - await record("test-token", inner=replay.transport(), scenarios=scenarios) - with pytest.raises(PrivateRepository): + with pytest.raises(PrivateRepository) as recording: + await record("test-token", inner=replay.transport(), scenarios=scenarios, clock=FakeClock()) + with pytest.raises(PrivateRepository) as first_sync: await record_first_sync( "test-token", owner, name, inner=replay.transport(), clock=FakeClock() ) + assert recording.value.full_name == first_sync.value.full_name == f"{owner}/{name}" assert replay.seen == [] -def test_a_failed_recording_writes_nothing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - recorded = tmp_path / "recorded" - recorded.mkdir() - (recorded / "kept.json").write_text("{}") +# -------------------------------------------------------------------------------------------- +# Writing: main() over the replay +# -------------------------------------------------------------------------------------------- +@dataclasses.dataclass +class Replayed: + """``record`` as ``main`` calls it, over ``replay`` on ``clock``: ``scenarios`` unless ``main`` + names its own. ``calls`` holds the arguments ``main`` passed, ``units`` what was recorded.""" + + replay: Replay + clock: FakeClock + scenarios: tuple[Scenario, ...] = SCENARIOS + calls: list[dict[str, Any]] = dataclasses.field(default_factory=list) + units: list[Unit] = dataclasses.field(default_factory=list) + + async def __call__(self, token: str, **kwargs: Any) -> list[Unit]: + self.calls.append(kwargs) + arguments = {"scenarios": self.scenarios, **kwargs} + self.units = await record( + token, inner=self.replay.transport(), clock=self.clock, **arguments + ) + return self.units - async def refuse(token: str, **kwargs: Any) -> list[Any]: - raise PrivateRepository("prebid/private") - monkeypatch.setattr(recorder, "RECORDED", recorded) - monkeypatch.setattr(recorder, "record", refuse) - monkeypatch.setenv("GITHUB_TOKEN", "test-token") - with pytest.raises(PrivateRepository): - recorder.main([]) - assert [path.name for path in tmp_path.iterdir()] == ["recorded"] - assert [path.name for path in recorded.iterdir()] == ["kept.json"] +def written(units: list[Unit]) -> dict[str, bytes]: + """The files ``units`` are written to, by path, with their text.""" + return {str(unit.path()): unit.dumps().encode() for unit in units} -def recorded_units() -> list[Any]: - """The units the recorder produces from the replay, as a live recording would.""" - return asyncio.run(record("test-token", inner=Replay.from_fixtures().transport())) +def snapshot(directory: Path) -> dict[str, bytes]: + """Every file under ``directory``, by relative path, with its bytes.""" + return { + str(path.relative_to(directory)): path.read_bytes() + for path in sorted(directory.rglob("*")) + if path.is_file() + } -def test_a_recording_writes_one_file_per_unit( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - units = recorded_units() +def of_repository(units: list[Unit], scenario: Scenario) -> set[str]: + """The files of ``units`` keyed by ``scenario``'s repository.""" + return { + str(unit.path()) + for unit in units + if (unit.key.get("owner"), unit.key.get("name")) == (scenario.owner, scenario.name) + } - async def replayed(token: str, **kwargs: Any) -> list[Any]: - return units - target = tmp_path / "recorded" - target.mkdir() - (target / "stale.json").write_text("{}") - monkeypatch.setattr(recorder, "RECORDED", target) - monkeypatch.setattr(recorder, "record", replayed) +def oversized(replay: Replay) -> Unit: + """The largest recorded compare page, with its first commit's message repeated until the page + is over ``MAX_FILE_BYTES``: compare returns each commit's message, which git does not bound.""" + unit = max( + (unit for unit in replay.units.values() if unit.kind == "compare"), + key=lambda unit: len(unit.dumps().encode()), + ) + unit = replay.unit("compare", **unit.key) + commit = unit.data["commits"][0]["commit"] + short = recorder.MAX_FILE_BYTES - len(unit.dumps().encode()) + commit["message"] = "\n".join([commit["message"]] * (short // len(commit["message"]) + 2)) + assert len(unit.dumps().encode()) > recorder.MAX_FILE_BYTES + return unit + + +@pytest.fixture +def target(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + """Where ``main`` writes the recordings: ``recorded`` in the test's directory.""" + directory = tmp_path / "recorded" + monkeypatch.setattr(recorder, "RECORDED", directory) monkeypatch.setenv("GITHUB_TOKEN", "test-token") - assert recorder.main([]) == 0 - written = sorted(str(path.relative_to(target)) for path in target.rglob("*.json")) + return directory + + +def run(monkeypatch: pytest.MonkeyPatch, replayed: Replayed, argv: list[str]) -> int: + monkeypatch.setattr(recorder, "record", replayed) + return recorder.main(argv) + + +def test_a_recording_writes_one_file_per_unit( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, target: Path +) -> None: + """Every unit to its own file, the directory replaced whole: a scenario no longer recorded + leaves no file.""" + earlier = Replayed(Replay.from_fixtures(), FakeClock(EARLIER)) + assert run(monkeypatch, earlier, []) == 0 committed = sorted(str(path.relative_to(RECORDED)) for path in RECORDED.rglob("*.json")) - assert written == committed # every unit to its own file; the stale file is gone - for unit in units: - assert (target / unit.path()).read_text() == unit.dumps() + assert sorted(snapshot(target)) == committed + assert snapshot(target) == written(earlier.units) + + later = Replayed( + Replay.from_fixtures(), + FakeClock(), + scenarios=tuple(scenario for scenario in SCENARIOS if scenario != DROPPED), + ) + assert run(monkeypatch, later, []) == 0 + gone = set(written(earlier.units)) - set(written(later.units)) + assert gone == of_repository(earlier.units, DROPPED) != set() + assert snapshot(target) == written(later.units) + assert {unit.recorded_at for unit in later.units} == {stamp(later.clock)} assert [path.name for path in tmp_path.iterdir()] == ["recorded"] -def test_only_rate_limits_adds_them_and_keeps_every_other_unit( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path +def test_a_failed_recording_writes_nothing( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, target: Path ) -> None: - units = [unit for unit in recorded_units() if unit.kind in ("rate_limit", "rest_rate_limit")] - assert sorted(unit.kind for unit in units) == ["rate_limit", "rest_rate_limit"] - calls: list[dict[str, Any]] = [] + earlier = Replayed(Replay.from_fixtures(), FakeClock(EARLIER)) + assert run(monkeypatch, earlier, []) == 0 + before = snapshot(target) + assert len(before) == len(earlier.units) - async def replayed(token: str, **kwargs: Any) -> list[Any]: - calls.append(kwargs) - return units + replay = Replay.from_fixtures() + unit = replay.unit("resolve", owner="prebid", name="go-gdpr") + unit.data["isPrivate"] = True + replay.add(unit) + with pytest.raises(PrivateRepository) as raised: + run(monkeypatch, Replayed(replay, FakeClock()), []) + assert raised.value.full_name == "prebid/go-gdpr" + assert snapshot(target) == before + assert [path.name for path in tmp_path.iterdir()] == ["recorded"] - target = tmp_path / "recorded" - (target / "rate_limit").mkdir(parents=True) - (target / "kept.json").write_text("{}") - (target / "rate_limit" / "rest.json").write_text("stale") - monkeypatch.setattr(recorder, "RECORDED", target) - monkeypatch.setattr(recorder, "record", replayed) - monkeypatch.setenv("GITHUB_TOKEN", "test-token") - assert recorder.main(["--only", "rate-limits"]) == 0 - assert calls == [{"scenarios": (), "users": ()}] - assert (target / "kept.json").read_text() == "{}" - for unit in units: - assert (target / unit.path()).read_text() == unit.dumps() + +def test_only_rate_limits_adds_them_and_keeps_every_other_unit( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, target: Path +) -> None: + earlier = Replayed(Replay.from_fixtures(), FakeClock(EARLIER)) + assert run(monkeypatch, earlier, []) == 0 + before = snapshot(target) + + readings = Replayed(Replay.from_fixtures(), FakeClock()) + assert run(monkeypatch, readings, ["--only", "rate-limits"]) == 0 + assert readings.calls == [{"scenarios": (), "users": ()}] + assert sorted(unit.kind for unit in readings.units) == ["rate_limit", "rest_rate_limit"] + replaced = written(readings.units) + # The readings are dated by the later clock, so each file changed. + assert [path for path in replaced if before[path] == replaced[path]] == [] + after = snapshot(target) + assert after == {**before, **replaced} + assert len(after) == len(earlier.units) assert [path.name for path in tmp_path.iterdir()] == ["recorded"] def test_only_documents_replaces_graphql_units_and_keeps_compare_and_rate_limits( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, target: Path ) -> None: - units = recorded_units() + """The GraphQL units are recorded again, so a scenario no longer listed leaves no file; the + compare pages and readings stay as the earlier recording wrote them.""" + earlier = Replayed(Replay.from_fixtures(), FakeClock(EARLIER)) + assert run(monkeypatch, earlier, []) == 0 + before = snapshot(target) + + documents = Replayed( + Replay.from_fixtures(), + FakeClock(), + scenarios=tuple(scenario for scenario in SCENARIOS if scenario != DROPPED), + ) + assert run(monkeypatch, documents, ["--only", "documents"]) == 0 + assert documents.calls == [{"rate_limits": False}] kept_kinds = recorder.KEPT_BY_DOCUMENTS - compare = [unit for unit in units if unit.kind == "compare"] - assert len(compare) > 1 and any(unit.kind == "rate_limit" for unit in units) - missing = compare[0] - target = tmp_path / "recorded" - - def place(unit: Unit) -> Path: - path = target / unit.path() - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(unit.dumps()) - return path - - # The existing recordings: every compare page but one and both readings, marked so a - # replacement shows, and a GraphQL unit no scenario records any more. - for unit in units: - if unit.kind in kept_kinds and unit is not missing: - place(dataclasses.replace(unit, recorded_at="kept")) - stale = next(unit for unit in units if unit.kind == "resolve") - stale_path = place(dataclasses.replace(stale, key={"owner": "prebid", "name": "retired"})) - calls: list[dict[str, Any]] = [] - - async def replayed(token: str, **kwargs: Any) -> list[Any]: - calls.append(kwargs) - return [unit for unit in units if unit.kind not in ("rate_limit", "rest_rate_limit")] - - monkeypatch.setattr(recorder, "RECORDED", target) - monkeypatch.setattr(recorder, "record", replayed) - monkeypatch.setenv("GITHUB_TOKEN", "test-token") - assert recorder.main(["--only", "documents"]) == 0 - assert calls == [{"rate_limits": False}] - assert not stale_path.exists() - written = sorted(str(path.relative_to(target)) for path in target.rglob("*.json")) - assert written == sorted(str(unit.path()) for unit in units) - for unit in units: - text = (target / unit.path()).read_text() - if unit.kind in kept_kinds and unit is not missing: - assert json.loads(text)["recorded_at"] == "kept", unit.path() - else: - assert text == unit.dumps(), unit.path() + kept = { + path: before[path] + for path in written([unit for unit in earlier.units if unit.kind in kept_kinds]) + } + assert len(kept) == sum( + 1 for unit in documents.replay.units.values() if unit.kind in kept_kinds + ) + graphql = written([unit for unit in documents.units if unit.kind not in kept_kinds]) + assert snapshot(target) == {**kept, **graphql} + assert of_repository(earlier.units, DROPPED) & set(snapshot(target)) == set() + assert of_repository(earlier.units, DROPPED) != set() assert [path.name for path in tmp_path.iterdir()] == ["recorded"] -def test_an_oversized_unit_writes_nothing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - units = recorded_units() +def test_only_documents_adds_a_compare_page_the_recordings_lack( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, target: Path +) -> None: + without_compare = tuple( + dataclasses.replace( + scenario, numbers=tuple(number for number in scenario.numbers if number != COMPARED) + ) + for scenario in SCENARIOS + ) + earlier = Replayed(Replay.from_fixtures(), FakeClock(EARLIER), scenarios=without_compare) + assert run(monkeypatch, earlier, []) == 0 + assert [unit for unit in earlier.units if unit.kind == "compare"] == [] + before = snapshot(target) + + documents = Replayed(Replay.from_fixtures(), FakeClock()) + assert run(monkeypatch, documents, ["--only", "documents"]) == 0 + compare = written([unit for unit in documents.units if unit.kind == "compare"]) + assert len(compare) == sum( + 1 for unit in documents.replay.units.values() if unit.kind == "compare" + ) + readings = { + path: before[path] + for path in written( + [unit for unit in earlier.units if unit.kind in ("rate_limit", "rest_rate_limit")] + ) + } + assert snapshot(target) == {**readings, **written(documents.units)} + assert set(compare) <= set(snapshot(target)) + assert [path.name for path in tmp_path.iterdir()] == ["recorded"] - async def replayed(token: str, **kwargs: Any) -> list[Any]: - return units - target = tmp_path / "recorded" - target.mkdir() - (target / "kept.json").write_text("{}") - monkeypatch.setattr(recorder, "RECORDED", target) - monkeypatch.setattr(recorder, "record", replayed) - monkeypatch.setattr(recorder, "MAX_FILE_BYTES", 10) - monkeypatch.setenv("GITHUB_TOKEN", "test-token") - assert recorder.main([]) == 1 +def test_an_oversized_unit_writes_nothing( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, target: Path +) -> None: + earlier = Replayed(Replay.from_fixtures(), FakeClock(EARLIER)) + assert run(monkeypatch, earlier, []) == 0 + before = snapshot(target) + assert len(before) == len(earlier.units) + + replay = Replay.from_fixtures() + replay.add(oversized(replay)) + later = Replayed(replay, FakeClock()) + assert run(monkeypatch, later, []) == 1 + sizes = [len(unit.dumps().encode()) for unit in later.units] + assert [size > recorder.MAX_FILE_BYTES for size in sizes].count(True) == 1 + assert snapshot(target) == before assert [path.name for path in tmp_path.iterdir()] == ["recorded"] - assert [path.name for path in target.iterdir()] == ["kept.json"] # -------------------------------------------------------------------------------------------- @@ -280,8 +399,9 @@ async def test_recording_a_first_sync_of_its_recordings_gives_them_back() -> Non """The first sync asks for exactly the recorded units, and records each as it was.""" replay = Replay.from_directory(FIRST_SYNC) owner, name = FIRST_SYNC_REPOSITORY + clock = FakeClock() units = await record_first_sync( - "test-token", owner, name, inner=replay.transport(), clock=FakeClock() + "test-token", owner, name, inner=replay.transport(), clock=clock ) assert replay.problems == [] # One resolve, the sync's own: the resolution the privacy check reads is the one it uses. @@ -301,6 +421,7 @@ async def test_recording_a_first_sync_of_its_recordings_gives_them_back() -> Non original.errors, original.headers, ), key + assert {unit.recorded_at for unit in units} == {stamp(clock)} async def test_a_private_repository_stops_a_first_sync_after_its_resolve() -> None: @@ -309,10 +430,11 @@ async def test_a_private_repository_stops_a_first_sync_after_its_resolve() -> No unit = replay.unit("resolve", owner=owner, name=name) unit.data["isPrivate"] = True replay.add(unit) - with pytest.raises(PrivateRepository, match=f"{owner}/{name}"): + with pytest.raises(PrivateRepository) as raised: await record_first_sync( "test-token", owner, name, inner=replay.transport(), clock=FakeClock() ) + assert raised.value.full_name == f"{owner}/{name}" assert [seen.operation for seen in replay.seen] == ["Resolve"] @@ -328,7 +450,7 @@ async def test_a_retried_item_records_one_unit_per_file(first_sync: bool) -> Non "test-token", owner, name, inner=replay.transport(), clock=FakeClock() ) else: - units = await record("test-token", inner=replay.transport()) + units = await record("test-token", inner=replay.transport(), clock=FakeClock()) batches = [seen.numbers for seen in replay.seen if seen.operation == "PullRequestBatch"] assert batches[1] == batches[0][:1] # the solo retry assert len(units) == len(replay.units) @@ -345,36 +467,38 @@ def _label_count_mismatch(body: dict[str, Any]) -> None: body["data"]["repository"]["labels"]["totalCount"] += 1 +def _class(text: str | None) -> str | None: + """The class an error's text names: ``error_text`` writes ": ".""" + return None if text is None else text.partition(": ")[0] + + @pytest.mark.parametrize( - ("change", "message"), + ("change", "error", "reason"), [ pytest.param( lambda replay: replay.fault( lambda request: httpx.Response(401, json={"message": "Bad credentials"}), times=None, ), - "first sync exited 1: Run 1: failed, exit 1; prebid/go-gdpr: no outcome: not " - "resolved; Hydrated 0 items; GraphQL points 0, REST calls 0, 0 s; Error: Unauthorized: " - "GitHub answered 401: Bad credentials", + "Unauthorized", + None, id="401", ), pytest.param( lambda replay: replay.modify( _label_count_mismatch, when=lambda seen: seen.operation == "Resolve" ), - "first sync exited 1: Run 1: failed, exit 1; prebid/go-gdpr: no outcome: " - "LabelCountMismatch: prebid/go-gdpr: fetched 10 labels, totalCount 11; Hydrated 0 " - "items; GraphQL points 1, REST calls 0, 0 s; Error: NoRepositoryResolved: no " - "configured repository resolved", + "NoRepositoryResolved", + "LabelCountMismatch", id="label count mismatch", ), ], ) async def test_a_first_sync_that_fails_names_why( - change: Callable[[Replay], None], message: str + change: Callable[[Replay], None], error: str, reason: str | None ) -> None: """A run that fails at its resolve raises with the run's report, which names the cause: the - run's error, or the repository's when only the report has it.""" + run's error, and the repository's when its resolve failed.""" replay = Replay.from_directory(FIRST_SYNC) change(replay) owner, name = FIRST_SYNC_REPOSITORY @@ -382,7 +506,20 @@ async def test_a_first_sync_that_fails_names_why( await record_first_sync( "test-token", owner, name, inner=replay.transport(), clock=FakeClock() ) - assert str(raised.value) == message + report = raised.value.report + assert report is not None + assert (report.exit_code, report.status, report.items_hydrated, report.failures) == ( + 1, + "failed", + 0, + (), + ) + assert _class(report.error) == error + assert [ + (repository.configured.full_name, repository.record, repository.outcome) + for repository in report.repositories + ] == [(f"{owner}/{name}", None, None)] + assert [_class(repository.reason) for repository in report.repositories] == [reason] assert [seen.operation for seen in replay.seen] == ["Resolve"] @@ -392,55 +529,75 @@ async def test_a_first_sync_that_does_not_exit_0_stops_the_recording() -> None: _batch_alias_error, when=lambda seen: seen.operation == "PullRequestBatch", times=2 ) owner, name = FIRST_SYNC_REPOSITORY - with pytest.raises(RecordingFailed, match="first sync exited 2"): + with pytest.raises(RecordingFailed) as raised: await record_first_sync( "test-token", owner, name, inner=replay.transport(), clock=FakeClock() ) + report = raised.value.report + assert report is not None + first = next(seen for seen in replay.seen if seen.operation == "PullRequestBatch").numbers[0] + assert report.exit_code == 2 + assert [(failure.number, _class(failure.error)) for failure in report.failures] == [ + (first, "AliasError") + ] -def _first_sync_units() -> list[Any]: - owner, name = FIRST_SYNC_REPOSITORY - replay = Replay.from_directory(FIRST_SYNC) - return asyncio.run( - record_first_sync("test-token", owner, name, inner=replay.transport(), clock=FakeClock()) - ) - - -@pytest.mark.parametrize("oversized", [False, True]) +@pytest.mark.parametrize("too_large", [False, True], ids=["written", "oversized"]) def test_only_first_sync_writes_its_directory_alone( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, oversized: bool + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, too_large: bool ) -> None: """``--only first-sync`` replaces the first-sync directory with every unit, or with an oversized unit writes nothing; the other recordings are never touched.""" - units = _first_sync_units() - - async def replayed(token: str, owner: str, name: str, **kwargs: Any) -> list[Any]: - return units - + owner, name = FIRST_SYNC_REPOSITORY target = tmp_path / "first_sync" - target.mkdir() - (target / "stale.json").write_text("{}") + # What the directory held before: an earlier recording of go-gdpr, which also has a pull + # request the first sync does not ask for. + earlier = asyncio.run( + record( + "test-token", + inner=Replay.from_fixtures().transport(), + scenarios=(Scenario(owner, name, index=True, numbers=(21,)),), + users=(), + rate_limits=False, + clock=FakeClock(EARLIER), + ) + ) + assert write_units(earlier, target) == 0 + before = snapshot(target) others = tmp_path / "recorded" - others.mkdir() - (others / "kept.json").write_text("{}") + shutil.copytree(RECORDED, others) + recordings = snapshot(others) + assert len(recordings) == len(list(RECORDED.rglob("*.json"))) + extra = [oversized(Replay.from_fixtures())] if too_large else [] + units: list[Unit] = [] + + async def replayed(token: str, owner: str, name: str, **kwargs: Any) -> list[Unit]: + assert kwargs == {} + replay = Replay.from_directory(FIRST_SYNC) + recorded = await record_first_sync( + token, owner, name, inner=replay.transport(), clock=FakeClock() + ) + # A first sync that reads a pull request's compare pages records them too. + units.extend([*recorded, *extra]) + return units + monkeypatch.setattr(recorder, "FIRST_SYNC", target) monkeypatch.setattr(recorder, "RECORDED", others) monkeypatch.setattr(recorder, "record_first_sync", replayed) - if oversized: - monkeypatch.setattr(recorder, "MAX_FILE_BYTES", 10) monkeypatch.setenv("GITHUB_TOKEN", "test-token") code = recorder.main(["--only", "first-sync"]) assert sorted(path.name for path in tmp_path.iterdir()) == ["first_sync", "recorded"] - assert [path.name for path in others.iterdir()] == ["kept.json"] - if oversized: + assert snapshot(others) == recordings + if too_large: assert code == 1 - assert [path.name for path in target.iterdir()] == ["stale.json"] + assert snapshot(target) == before return assert code == 0 - written = sorted(str(path.relative_to(target)) for path in target.rglob("*.json")) + assert snapshot(target) == written(units) committed = sorted(str(path.relative_to(FIRST_SYNC)) for path in FIRST_SYNC.rglob("*.json")) - assert written == committed - for unit in units: - assert (target / unit.path()).read_text() == unit.dumps() + assert sorted(snapshot(target)) == committed + # The earlier recording's pull request is gone with the directory it was in. + pull_requests = set(written([unit for unit in earlier if unit.kind == "pull_request"])) + assert set(written(earlier)) - set(written(units)) == pull_requests != set() diff --git a/tests/source/test_recordings.py b/tests/source/test_recordings.py index 9b9e50d..7497171 100644 --- a/tests/source/test_recordings.py +++ b/tests/source/test_recordings.py @@ -5,18 +5,22 @@ import json import re from pathlib import Path +from typing import Any import httpx import pytest -from github_activity_db.source import documents -from tests.source import recordings +from github_activity_db.source import GitHubSource, Hydrated, PullRequestOutcome, documents +from github_activity_db.source.errors import UnexpectedResponse +from github_activity_db.source.limits import OVERFLOW_ALIASES_PER_REQUEST, PR_BATCH_SIZE +from tests.source.conftest import FakeClock, all_outcomes from tests.source.recordings import ( COMPARE_STAMP, RECORDED, REST_RATE_LIMIT_STAMP, Unit, canonical_batch, + canonical_overflow, compare_key, expected_overflow_stamp, expected_pull_request_stamp, @@ -26,6 +30,7 @@ whole_stamp, ) from tests.source.replay import Replay, unused_faults +from tests.source.test_hydrate import SECOND_PAGES, copied, copies, with_second_pages UNITS = sorted(RECORDED.rglob("*.json")) WHOLE_DOCUMENTS = { @@ -35,6 +40,11 @@ "user": documents.USER_LOOKUP, "rate_limit": documents.RATE_LIMIT_QUERY, } +# Units the recorder wrote at Step 2 (c1f474b), for the documents of hydration_version 1, +# committed unchanged (tests/fixtures/history/README.md). +STEP_2 = ( + Path(__file__).resolve().parents[1] / "fixtures/history/c1f474b/tests/fixtures/source/recorded" +) def test_fixtures_were_recorded() -> None: @@ -82,150 +92,158 @@ def test_unit_keeps_no_request_headers(path: Path) -> None: assert not re.search(r"\bgh[pousr]_[A-Za-z0-9]{30,}|\bgithub_pat_[A-Za-z0-9_]{30,}", text) -@pytest.mark.parametrize("size", [1, 2, 7, 25]) +@pytest.mark.parametrize("size", range(1, PR_BATCH_SIZE + 1)) def test_a_batch_stamp_does_not_depend_on_its_size(size: int) -> None: + """At every size the hydrator sends, the stamp hashes the whole one-alias document, its + repository wrapper and rateLimit included.""" batch = documents.pull_request_batch(size) - assert canonical_batch(batch) == documents.normalize(documents.pull_request_batch(1)) + canonical = canonical_batch(batch) + assert canonical == documents.normalize(documents.pull_request_batch(1)) + assert "repository(owner: $owner, name: $name)" in canonical + assert documents.normalize(documents.RATE_LIMIT) in canonical assert pull_request_stamp(batch) == expected_pull_request_stamp() @pytest.mark.parametrize("connection", documents.PULL_REQUEST_CONNECTIONS, ids=lambda c: c.key) def test_a_continuation_stamp_does_not_depend_on_its_neighbours(connection) -> None: + """The stamp hashes the one-alias continuation: its block, the rateLimit it selects, and the + fragment it uses.""" query = documents.overflow_continuation([documents.LABELS, documents.REVIEWS, connection]) + alone = documents.overflow_continuation([connection]) + assert canonical_overflow(query, 2, connection.key) == documents.normalize(alone) assert overflow_stamp(query, 2, connection.key) == expected_overflow_stamp(connection.key) +async def _hydrate( + replay: Replay, owner: str, name: str, numbers: list[int] +) -> list[PullRequestOutcome]: + """The real source over ``replay``: resolve ``owner/name``, then hydrate ``numbers``.""" + source = GitHubSource("test-token", http_transport=replay.transport(), clock=FakeClock()) + async with source: + repository = await source.resolve(owner, name) + return await all_outcomes(source, repository, numbers) + + @pytest.mark.parametrize( - ("document", "edit"), + ("path", "lookup", "expected"), [ - ( - documents.pull_request_batch(2), - ("repository(owner: $owner, name: $name)", "repository(name: $name, owner: $owner)"), + pytest.param( + "pull_request/prebid__Prebid.js/13481.json", + {"owner": "prebid", "name": "Prebid.js", "number": 13481}, + expected_pull_request_stamp(), + id="pull_request", + ), + pytest.param( + "overflow/PR_kwDOBUaT8s6WsAjJ/comments/0e3c0d53c5b8c0eb.json", + { + "id": "PR_kwDOBUaT8s6WsAjJ", + "connection": "comments", + "after": "Y3Vyc29yOnYyOpHPAAAAAU7JU7k=", + }, + expected_overflow_stamp("comments"), + id="overflow", + ), + pytest.param( + "resolve/prebid__go-gdpr.json", + {"owner": "prebid", "name": "go-gdpr"}, + whole_stamp(documents.RESOLVE), + id="resolve", ), - (documents.pull_request_batch(2), ("rateLimit { cost remaining resetAt }", "")), - (documents.pull_request_batch(2), ("additions deletions changedFiles", "additions")), ], - ids=["batch-wrapper", "batch-rate-limit", "batch-fields"], -) -def test_a_changed_batch_document_changes_the_stamp(document: str, edit: tuple[str, str]) -> None: - assert edit[0] in document - assert pull_request_stamp(document.replace(*edit)) != expected_pull_request_stamp() - - -@pytest.mark.parametrize( - "edit", - [("headRefOid updatedAt ", ""), ("rateLimit { cost remaining resetAt }", "")], - ids=["continuation-head", "continuation-rate-limit"], ) -def test_a_changed_continuation_document_changes_the_stamp(edit: tuple[str, str]) -> None: - query = documents.overflow_continuation([documents.FILES]) - assert edit[0] in query - assert overflow_stamp(query.replace(*edit), 0, "files") != expected_overflow_stamp("files") - - -def test_the_replay_refuses_a_request_for_another_document() -> None: +async def test_a_unit_recorded_for_an_earlier_document_is_refused( + path: str, lookup: dict[str, Any], expected: str +) -> None: + """A unit the recorder wrote for Step 2's documents, in place of the current recording: the + replay refuses it, and the source stops on the answer, so a document changed without + re-recording fails the suite.""" + earlier = Unit.load(STEP_2 / path) + current = Unit.load(RECORDED / path) + assert (earlier.kind, earlier.key) == (current.kind, current.key) + assert current.document_sha256 == expected != earlier.document_sha256 replay = Replay.from_fixtures() - query = documents.pull_request_batch(1).replace("additions deletions changedFiles", "additions") - payload = {"query": query, "variables": {"owner": "prebid", "name": "Prebid.js", "n0": 13481}} - request = httpx.Request("POST", "https://api.github.com/graphql", json=payload) - response = replay.transport().handle_request(request) - assert response.status_code == 418 - assert "re-record" in replay.problems[0] - - -def _refused(replay: Replay, payload: dict) -> str: + replay.add(earlier) + with pytest.raises(UnexpectedResponse) as raised: + match earlier.kind: + case "resolve": + await _hydrate(replay, lookup["owner"], lookup["name"], []) + case "pull_request": + await _hydrate(replay, "prebid", "Prebid.js", [lookup["number"]]) + case _: + await _hydrate(replay, "prebid", "prebid-server", [4343]) # 109 comments + assert raised.value.status == 418 + assert replay.problems == [ + f"{earlier.kind} unit {lookup} was recorded for another query document: re-record with " + "scripts/record_fixtures.py" + ] + + +def _refused(replay: Replay, payload: dict) -> list[str]: request = httpx.Request("POST", "https://api.github.com/graphql", json=payload) response = replay.transport().handle_request(request) assert response.status_code == 418 - return replay.problems[0] - - -def test_the_replay_refuses_a_batch_alias_bound_to_another_number() -> None: - query = documents.pull_request_batch(2).replace( - "p1: pullRequest(number: $n1)", "p1: pullRequest(number: $n0)" - ) - assert "pullRequest(number: $n0)" in query - payload = { - "query": query, - "variables": {"owner": "prebid", "name": "Prebid.js", "n0": 13481, "n1": 15724}, - } - assert "bind" in _refused(Replay.from_fixtures(), payload) - - -def test_the_replay_refuses_a_continuation_bound_to_another_cursor() -> None: - connections = [documents.CONNECTIONS_BY_KEY["commits"]] * 2 - query = documents.overflow_continuation(connections) - block = recordings.extract_block(query, "o1_commits: node(") - query = query.replace(block, block.replace("$after1", "$after0")) - payload = { - "query": query, - "variables": {"id0": "PR_a", "after0": "x", "id1": "PR_b", "after1": "y"}, - } - assert "binds" in _refused(Replay.from_fixtures(), payload) - - -def test_the_replay_refuses_a_batch_declaring_an_unsent_number() -> None: - query = documents.pull_request_batch(1).replace("$n0: Int!", "$n0: Int! $n1: Int!") - payload = {"query": query, "variables": {"owner": "prebid", "name": "Prebid.js", "n0": 13481}} - assert "declares" in _refused(Replay.from_fixtures(), payload) - - -def test_the_replay_refuses_a_continuation_with_an_unbound_variable() -> None: - query = documents.overflow_continuation([documents.CONNECTIONS_BY_KEY["commits"]]) - payload = {"query": query, "variables": {"id0": "PR_a", "after0": "x", "id1": "PR_b"}} - assert "variables" in _refused(Replay.from_fixtures(), payload) - - -def test_the_replay_refuses_batch_aliases_out_of_order() -> None: - query = documents.pull_request_batch(2) - first = "p0: pullRequest(number: $n0) { ...PullRequestFields }" - second = "p1: pullRequest(number: $n1) { ...PullRequestFields }" - query = query.replace(f"{first} {second}", f"{second} {first}") - assert query.index("p1:") < query.index("p0:") - payload = { - "query": query, - "variables": {"owner": "prebid", "name": "Prebid.js", "n0": 13481, "n1": 15724}, - } - assert "bind" in _refused(Replay.from_fixtures(), payload) - - -def test_the_replay_refuses_a_continuation_reading_another_alias_id() -> None: - connections = [documents.CONNECTIONS_BY_KEY["commits"]] * 2 - query = documents.overflow_continuation(connections).replace( - "o1_commits: node(id: $id1)", "o1_commits: node(id: $id0)" - ) - payload = { - "query": query, - "variables": {"id0": "PR_a", "after0": "x", "id1": "PR_b", "after1": "y"}, - } - assert "binds" in _refused(Replay.from_fixtures(), payload) - - -def test_the_replay_refuses_a_continuation_declaring_an_unused_alias() -> None: - query = documents.overflow_continuation([documents.CONNECTIONS_BY_KEY["commits"]]).replace( - "$id0: ID! $after0: String!", "$id0: ID! $after0: String! $id1: ID! $after1: String!" - ) - payload = {"query": query, "variables": {"id0": "PR_a", "after0": "x"}} - assert "declarations" in _refused(Replay.from_fixtures(), payload) + return replay.problems def test_the_replay_refuses_a_request_it_has_no_unit_for() -> None: + key = {"owner": "prebid", "name": "Prebid.js", "number": 1} payload = { "query": documents.pull_request_batch(1), "variables": {"owner": "prebid", "name": "Prebid.js", "n0": 1}, } - assert "no pull_request unit" in _refused(Replay.from_fixtures(), payload) + assert _refused(Replay.from_fixtures(), payload) == [ + f"no pull_request unit for {key}: record it with scripts/record_fixtures.py" + ] -@pytest.mark.parametrize( - "query", - ["per_page=100&page=1&sha=main", "per_page=100", "per_page=100&page=1&page=2"], - ids=["extra-parameter", "missing-page", "repeated-page"], -) -def test_a_compare_request_with_other_parameters_finds_no_unit(query: str) -> None: - base = "/repos/prebid/Prebid.js/compare/" + "a" * 40 + "..." + "b" * 40 - assert compare_key(f"https://api.github.com{base}?{query}") is None - assert compare_key(f"https://api.github.com{base}?per_page=100&page=1") is not None +@pytest.mark.parametrize("size", range(1, PR_BATCH_SIZE + 1)) +async def test_each_batch_the_hydrator_sends_binds_its_own_numbers(size: int) -> None: + """Alias ``p`` reads ``$n``, in order, and the document declares exactly the numbers + sent: the replay's check passes every batch size the hydrator sends.""" + replay = Replay.from_fixtures() + numbers = copies(replay, size) + outcomes = await _hydrate(replay, "prebid", "Prebid.js", numbers) + assert replay.problems == [] + assert [o.number for o in outcomes if isinstance(o, Hydrated)] == numbers + batches = [seen for seen in replay.seen if seen.operation == "PullRequestBatch"] + assert [seen.numbers for seen in batches] == [numbers] + assert [len(seen.variables) for seen in batches] == [size + 2] # and owner and name + + +@pytest.mark.parametrize("aliases", range(1, OVERFLOW_ALIASES_PER_REQUEST + 1)) +async def test_each_continuation_request_binds_its_own_variables(aliases: int) -> None: + """Alias ``o_*`` reads ``$id`` and ``$after`` only, each declared once and sent: the + replay's check passes every continuation request size, mixing connections.""" + replay = Replay.from_fixtures() + numbers = [900_001 + i for i in range(aliases)] + connections = [list(SECOND_PAGES)[i % len(SECOND_PAGES)] for i in range(aliases)] + for number, connection in zip(numbers, connections, strict=True): + with_second_pages(replay, copied(replay, 12975, number), (connection,)) + outcomes = await _hydrate(replay, "prebid", "Prebid.js", numbers) + assert replay.problems == [] + records = [outcome.record for outcome in outcomes if isinstance(outcome, Hydrated)] + assert [record.number for record in records] == numbers + # Each item's second page reached it: one more entry than the recorded first page holds. + assert [ + len(record.labels if connection == "labels" else record.closing_issues) + for record, connection in zip(records, connections, strict=True) + ] == [2] * aliases + overflow = [seen for seen in replay.seen if seen.operation == "Overflow"] + assert [len(seen.variables) for seen in overflow] == [2 * aliases] + + +async def test_the_compare_requests_the_source_sends_find_their_units() -> None: + """A compare request carries exactly ``page`` and ``per_page``, the parameters + ``COMPARE_STAMP`` names, so each one the source sends finds its unit.""" + replay = Replay.from_fixtures() + (outcome,) = await _hydrate(replay, "prebid", "Prebid.js", [8858]) # 254 commits + assert isinstance(outcome, Hydrated) + assert outcome.record.commits_source == "compare" + requested = [compare_key(seen.path) for seen in replay.seen if seen.operation == "rest"] + recorded = [unit.key for unit in replay.units.values() if unit.kind == "compare"] + assert sorted(requested, key=json.dumps) == sorted(recorded, key=json.dumps) + assert len(requested) == len(recorded) + assert replay.problems == [] def test_a_fault_never_served_or_not_used_up_is_reported() -> None: diff --git a/tests/source/test_resolve_index_users.py b/tests/source/test_resolve_index_users.py index 4d6f6e2..18946cb 100644 --- a/tests/source/test_resolve_index_users.py +++ b/tests/source/test_resolve_index_users.py @@ -599,10 +599,18 @@ def duplicate(body: dict) -> None: nodes[1] = dict(nodes[0]) replay.modify(duplicate, when=operation("PullRequestIndex")) + first = replay.unit("index", owner="prebid", name="go-gdpr", after=None).data["nodes"][0] async with open_source() as source: repository = await source.resolve("prebid", "go-gdpr") - with pytest.raises(IndexOrderViolation, match="listed twice"): + with pytest.raises(IndexOrderViolation) as raised: await source.index_pull_requests(repository) + violation = raised.value + assert (violation.full_name, violation.connection, violation.number, violation.problem) == ( + "prebid/go-gdpr", + "pullRequests", + first["number"], + "listed twice", + ) async def test_creation_order_is_checked(self, open_source, replay: Replay) -> None: def reorder(body: dict) -> None: @@ -610,10 +618,19 @@ def reorder(body: dict) -> None: nodes[0], nodes[1] = nodes[1], nodes[0] replay.modify(reorder, when=operation("PullRequestIndex")) + # The first recorded entry, moved second, was created before the one now ahead of it. + first = replay.unit("index", owner="prebid", name="go-gdpr", after=None).data["nodes"][0] async with open_source() as source: repository = await source.resolve("prebid", "go-gdpr") - with pytest.raises(IndexOrderViolation, match="created before"): + with pytest.raises(IndexOrderViolation) as raised: await source.index_pull_requests(repository) + violation = raised.value + assert (violation.full_name, violation.connection, violation.number, violation.problem) == ( + "prebid/go-gdpr", + "pullRequests", + first["number"], + "created before the previous entry", + ) @pytest.mark.parametrize( "change", diff --git a/tests/source/test_scenario.py b/tests/source/test_scenario.py index 87be089..6ffbfed 100644 --- a/tests/source/test_scenario.py +++ b/tests/source/test_scenario.py @@ -9,6 +9,7 @@ import copy import json +import math from collections.abc import Callable from dataclasses import replace from datetime import UTC, datetime, timedelta, timezone @@ -30,9 +31,17 @@ RepositoryNotFound, RepositoryShapeError, ) +from github_activity_db.source.limits import PAGE_SIZE from github_activity_db.source.models import ComparePage from tests.source.conftest import START, FakeClock, all_lookups, all_outcomes -from tests.source.recordings import Unit +from tests.source.recordings import ( + COMPARE_STAMP, + RECORDED, + Unit, + expected_overflow_stamp, + expected_pull_request_stamp, + whole_stamp, +) from tests.source.replay import Replay, loaded_units, synthetic, unanswered from tests.source.scenario import ( ORDERS, @@ -41,6 +50,7 @@ Repository, Scenario, ScenarioError, + _templates, iso, recorded_units, ) @@ -485,6 +495,14 @@ async def test_a_repository_gone(scenario: Scenario) -> None: await source.index_pull_requests(resolved) with pytest.raises(RepositoryNotFound): await all_outcomes(source, resolved, [1]) + # Each index and batch request after the removal was answered with a null repository. + assert [seen.operation for seen in scenario.replay.seen] == [ + "Resolve", + "Resolve", + "PullRequestIndex", + "PullRequestBatch", + ] + assert scenario.replay.problems == [] async def test_a_repository_gone_between_its_label_pages(scenario: Scenario) -> None: @@ -583,38 +601,61 @@ def post(world: Scenario, query: str, variables: dict[str, Any]) -> httpx.Respon return client.post("/graphql", json={"query": query, "variables": variables}) -def test_a_document_the_model_was_not_checked_against_is_refused() -> None: +WHOLE_DOCUMENTS = { + "resolve": documents.RESOLVE, + "repository_labels": documents.REPOSITORY_LABELS, + "index": documents.PULL_REQUEST_INDEX, + "user": documents.USER_LOOKUP, +} + + +def current_stamp(kind: str, connection: str | None) -> str: + """The stamp of the document the source sends now for ``kind`` (and continued + ``connection``).""" + if kind == "pull_request": + return expected_pull_request_stamp() + if kind == "overflow" and connection is not None: + return expected_overflow_stamp(connection) + if kind == "compare": + return whole_stamp(COMPARE_STAMP) + return whole_stamp(WHOLE_DOCUMENTS[kind]) + + +def test_the_model_stamps_each_kind_as_the_current_documents_do() -> None: + """Rendered units carry the recordings' stamps, one per kind and continued connection, each + the current document's, so the replay accepts what the model renders. A unit recorded for an + earlier document is refused (test_recordings.py).""" + layout = { + (parts[0], parts[2] if parts[0] == "overflow" else None) + for parts in (path.relative_to(RECORDED).parts for path in RECORDED.rglob("*.json")) + if parts[0] in RENDERED + } + assert {kind for kind, _ in layout} == set(RENDERED) + assert _templates().stamps == {key: current_stamp(*key) for key in layout} world = Scenario(FakeClock()) world.add_repository("example", "project") variables = {"owner": "example", "name": "project"} - changed = documents.RESOLVE.replace("isPrivate", "isPrivate isFork") - assert changed != documents.RESOLVE assert post(world, documents.RESOLVE, variables).status_code == 200 - assert post(world, changed, variables).status_code == 418 - assert world.replay.problems == [ - f"resolve unit {variables} was recorded for another query document: re-record with " - "scripts/record_fixtures.py" - ] + assert world.replay.problems == [] -def test_a_missing_repository_is_answered_only_for_a_checked_document() -> None: +def test_a_missing_repository_is_answered_null_for_the_current_documents() -> None: + """The model answers an index or batch request for a repository it does not hold with a null + repository, stamped as the recordings of that request are.""" world = Scenario(FakeClock()) - variables = {"owner": "example", "name": "absent", "after": None} - changed = documents.PULL_REQUEST_INDEX.replace("isDraft", "isDraft title") - answered = post(world, documents.PULL_REQUEST_INDEX, variables) - assert (answered.status_code, answered.json()["data"]["repository"]) == (200, None) - assert post(world, changed, variables).status_code == 418 - batch = documents.pull_request_batch(1) - changed_batch = batch.replace("additions deletions", "additions") - assert changed_batch != batch - numbered = {"owner": "example", "name": "absent", "n0": 1} - assert post(world, batch, numbered).json()["data"]["repository"] is None - assert post(world, changed_batch, numbered).status_code == 418 - assert world.replay.problems == [ - f"{kind} unit {{'owner': 'example', 'name': 'absent', 'for': '{kind}'}} was recorded for " - "another query document: re-record with scripts/record_fixtures.py" - for kind in ("index", "pull_request") + index = post( + world, documents.PULL_REQUEST_INDEX, {"owner": "example", "name": "absent", "after": None} + ) + batch = post( + world, documents.pull_request_batch(1), {"owner": "example", "name": "absent", "n0": 1} + ) + assert [ + (answer.status_code, answer.json()["data"]["repository"]) for answer in (index, batch) + ] == [ + (200, None), + (200, None), ] + assert world.replay.problems == [] async def test_a_continuation_no_recording_covers_is_left_unanswered() -> None: @@ -636,37 +677,43 @@ async def test_a_continuation_no_recording_covers_is_left_unanswered() -> None: @pytest.mark.parametrize( - ("connection", "cursor"), - [("comments", "unserved"), ("comments", "scenario:[1]"), ("commits", "NTA")], + ("connection", "nodes"), + [("comments", 101), ("comments", 250), ("commits", 250)], ) -async def test_a_cursor_the_model_never_served_is_reported(connection: str, cursor: str) -> None: - """A cursor in the model's own form, or GitHub's offset form (``NTA``, base64 of 50), that the - model did not serve names no page; the failure is the replay's problem, not an error inside - the transport.""" +async def test_each_cursor_the_source_sends_is_one_the_model_served( + connection: str, nodes: int +) -> None: + """A page's ``endCursor`` is one the model serves: in its own form for a connection whose + cursors name a node, or GitHub's offset form for one whose cursors name a position. The source + sends each back, one continuation per further page.""" world = Scenario(FakeClock()) repository, author = project(world) - repository.pulls[1].add(connection, *(world.node(connection, by=author) for _ in range(101))) - - def forged(body: dict[str, Any]) -> None: - body["data"]["repository"]["p0"][connection]["pageInfo"]["endCursor"] = cursor - - world.replay.modify(forged, when=lambda seen: seen.operation == "PullRequestBatch") + pull = repository.pulls[1] + pull.add(connection, *(world.node(connection, by=author) for _ in range(nodes))) async with world.source() as source: resolved = await source.resolve("example", "project") - with pytest.raises(FatalSourceError): - await all_outcomes(source, resolved, [1]) - key = {"id": repository.pulls[1].node["id"], "connection": connection, "after": cursor} - message = f"cursor {cursor!r} names no page the model served" - assert world.replay.problems == [ - f"the provider failed on overflow {key}: ScenarioError({message!r})" - ] + (outcome,) = await all_outcomes(source, resolved, [1]) + assert len(getattr(hydrated(outcome), connection)) == nodes + sent = [seen.variables["after0"] for seen in world.replay.seen if seen.operation == "Overflow"] + assert len(sent) == math.ceil(nodes / PAGE_SIZE) - 1 + assert set(sent) <= set(pull.cursors[connection].known) + assert unanswered(world.replay) == [] def test_unanswered_lists_problems_then_unused_faults() -> None: - replay = Replay() - replay.fault(lambda request: synthetic("server_error")) - replay.problems.append("no unit") - assert unanswered(replay) == ["no unit", "fault 0: served 0, 1 left"] + replay = Replay.from_fixtures() + replay.fault( + lambda request: synthetic("server_error"), when=lambda seen: seen.operation == "Resolve" + ) + # Prebid.js #1 is not recorded: the replay cannot answer it. + payload = { + "query": documents.pull_request_batch(1), + "variables": {"owner": "prebid", "name": "Prebid.js", "n0": 1}, + } + with httpx.Client(transport=replay.transport(), base_url="https://api.github.com") as client: + assert client.post("/graphql", json=payload).status_code == 418 + assert len(replay.problems) == 1 + assert unanswered(replay) == [replay.problems[0], "fault 0: served 0, 1 left"] async def test_a_bot_commit_links_its_bot_login(scenario: Scenario) -> None: @@ -786,7 +833,15 @@ def remove() -> None: resolved = await source.resolve("example", "project") with pytest.raises(RepositoryNotFound): await all_outcomes(source, resolved, [1]) - assert "Overflow" in [seen.operation for seen in scenario.replay.seen] + # The continuation finds no node, and the item's solo retry is answered with a null + # repository. + assert [seen.operation for seen in scenario.replay.seen] == [ + "Resolve", + "PullRequestBatch", + "Overflow", + "PullRequestBatch", + ] + assert scenario.replay.problems == [] async def test_what_the_model_leaves_unanswered() -> None: diff --git a/tests/source/test_transport.py b/tests/source/test_transport.py index 12d98ce..3eb4d93 100644 --- a/tests/source/test_transport.py +++ b/tests/source/test_transport.py @@ -3,17 +3,21 @@ from __future__ import annotations import asyncio +import inspect +import random import time from collections.abc import Callable, Iterator from datetime import UTC, datetime, timedelta from email.utils import format_datetime -from typing import Any, cast +from typing import Any import httpx import pytest +from githubkit.core import GitHubCore from githubkit.exception import ( AuthCredentialError, AuthExpiredError, + RequestError, RequestFailed, SecondaryRateLimitExceeded, ) @@ -45,7 +49,12 @@ RESET_MARGIN, SERVER_RETRY_DELAYS, ) -from github_activity_db.source.transport import RestRefused, Transport, _Failure +from github_activity_db.source.transport import ( + _GITHUBKIT_RATE_LIMIT_READERS, + RestRefused, + Transport, + _Failure, +) from tests.source.conftest import START, FakeClock from tests.source.replay import RATE_LIMIT, Replay, Seen, synthetic @@ -648,30 +657,31 @@ def limited(request: httpx.Request) -> httpx.Response: await source.resolve(**RESOLVE) assert clock.sleeps == [300 + RESET_MARGIN] - @pytest.mark.parametrize( - "error", - [TypeError("not from githubkit's rate-limit readers"), ValueError("bad key data")], - ids=["type-error", "value-error"], - ) - async def test_a_defect_inside_the_client_is_fatal( - self, open_source, replay: Replay, clock: FakeClock, error: Exception - ) -> None: - replay.fault(self.raising(error), when=resolves) - async with open_source() as source: - with pytest.raises(ClientFailure): - await source.resolve(**RESOLVE) - assert clock.sleeps == [] + def test_the_rate_limit_readers_are_githubkits(self) -> None: + """An error raised inside the client is a rate limit githubkit could not read only when + one of these frames raised it (TestOutOfRangeValues, TestMalformedHeaders); anything + else is a ClientFailure or propagates. Each name is a method of the installed githubkit's + client.""" + found = sorted( + name + for name in _GITHUBKIT_RATE_LIMIT_READERS + if inspect.isfunction(getattr(GitHubCore, name, None)) + ) + assert found == sorted(_GITHUBKIT_RATE_LIMIT_READERS) + assert len(found) == len(_GITHUBKIT_RATE_LIMIT_READERS) > 0 async def test_a_client_failure_outside_the_network_is_fatal( self, open_source, replay: Replay, clock: FakeClock ) -> None: - replay.fault( - self.raising(RuntimeError("JWT support for GitHub APP should be installed")), - when=resolves, - ) + # githubkit's App auth raises this when PyJWT is not installed. + error = RuntimeError("JWT support for GitHub APP should be installed") + replay.fault(self.raising(error), when=resolves) async with open_source() as source: - with pytest.raises(ClientFailure, match="JWT support"): + with pytest.raises(ClientFailure) as raised: await source.resolve(**RESOLVE) + cause = raised.value.__cause__ + assert isinstance(cause, RequestError) + assert cause.exc is error assert clock.sleeps == [] assert len(replay.seen) == 1 @@ -727,18 +737,6 @@ async def test_a_403_with_malformed_errors_is_forbidden( with pytest.raises(Forbidden, match="Resource not accessible"): await source.resolve(**RESOLVE) - async def test_a_defect_in_the_request_path_is_not_a_rate_limit(self, clock: FakeClock) -> None: - # A client whose arequest no longer accepts the transport's arguments: the TypeError is - # not raised by githubkit's rate-limit readers, so it propagates instead of waiting. - class Incompatible: - async def arequest(self, *args: object, **kwargs: object) -> object: - raise TypeError("arequest() got an unexpected keyword argument 'content'") - - transport = Transport(cast("Any", Incompatible()), clock=clock) - with pytest.raises(TypeError, match="unexpected keyword"): - await transport.graphql("query Resolve { x }", {}, splittable=False) - assert clock.sleeps == [] - async def test_a_403_with_an_html_body_is_forbidden(self, open_source, replay: Replay) -> None: replay.fault( lambda request: httpx.Response(403, text="blocked"), when=resolves @@ -1144,11 +1142,15 @@ async def counting(request: httpx.Request) -> httpx.Response: assert len(replay.seen) == 4 -def test_the_system_clock_jitters_within_its_fraction() -> None: +def test_the_system_clock_jitters_within_its_fraction(monkeypatch: pytest.MonkeyPatch) -> None: + # The clock's source of randomness, seeded: the same draws on every run. clock = SystemClock() + monkeypatch.setattr(clock, "_random", random.Random(0)) # noqa: S311 - repeatable, not a secret samples = [clock.jitter(8.0) for _ in range(200)] + draws = random.Random(0) # noqa: S311 - repeatable, not a secret + assert samples == [8.0 + draws.uniform(0.0, 8.0 * JITTER_FRACTION) for _ in range(200)] assert all(8.0 <= sample <= 8.0 * (1 + JITTER_FRACTION) for sample in samples) - assert len(set(samples)) > 1 + assert len(set(samples)) == len(samples) async def test_retry_delays_go_through_the_clock_jitter(replay: Replay) -> None: diff --git a/tests/test_config.py b/tests/test_config.py index 75d83bd..1af1d6c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,7 @@ """Tests for configuration settings.""" -import re +import shutil +from collections.abc import Iterator from pathlib import Path import pytest @@ -124,34 +125,43 @@ def test_removed_pipeline_settings_are_gone(self): assert fields.isdisjoint({"rate_limit", "pacing", "sync"}) -class TestDocumentedDatabaseUrl: - """The documented DATABASE_URL lines show the default, which names the local file (D26).""" - - @pytest.mark.parametrize("document", [".env.example", "docs/development.md"]) - def test_names_the_local_database(self, document: str) -> None: - lines = re.findall(r"^#? ?DATABASE_URL=(.*)$", (REPO_ROOT / document).read_text(), re.M) - assert lines == [LOCAL_DATABASE_URL] +class TestDefaultDatabaseUrl: + """The default DATABASE_URL names the local file, never the tracked one (D26).""" def test_the_field_default_is_the_local_database(self) -> None: assert Settings.model_fields["database_url"].default == LOCAL_DATABASE_URL + def test_a_env_copied_from_the_example_reads_the_local_database( + self, monkeypatch, tmp_path: Path + ) -> None: + """Setup copies .env.example to .env, which Settings reads from the working directory.""" + shutil.copyfile(REPO_ROOT / ".env.example", tmp_path / ".env") + monkeypatch.chdir(tmp_path) + assert Settings().database_url == LOCAL_DATABASE_URL + + +@pytest.fixture +def no_env_file(monkeypatch, tmp_path: Path) -> Iterator[None]: + """get_settings() reads .env from the working directory: a test directory without one, and + a fresh cache.""" + monkeypatch.chdir(tmp_path) + get_settings.cache_clear() + yield + get_settings.cache_clear() + class TestGetSettings: """Tests for get_settings function.""" - def test_get_settings_returns_settings(self): + def test_get_settings_returns_settings(self, no_env_file: None): """Test that get_settings returns a Settings instance.""" - # Clear cache to ensure fresh settings - get_settings.cache_clear() - settings = get_settings() assert isinstance(settings, Settings) + assert settings.database_url == LOCAL_DATABASE_URL - def test_get_settings_cached(self): + def test_get_settings_cached(self, no_env_file: None): """Test that get_settings returns cached instance.""" - get_settings.cache_clear() - settings1 = get_settings() settings2 = get_settings() diff --git a/tests/test_logging.py b/tests/test_logging.py index 605bc8a..98b4939 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -2,7 +2,9 @@ from __future__ import annotations +import ast import logging +import re from pathlib import Path from typing import TYPE_CHECKING @@ -20,7 +22,9 @@ ) if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import Generator, Iterator + + from loguru import Record @pytest.fixture(autouse=True) @@ -31,6 +35,35 @@ def _reset_logging_state() -> Generator[None, None, None]: reset_logging() +_ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;]*m") + + +def emitted(capsys: pytest.CaptureFixture[str]) -> list[tuple[str, str, str]]: + """Log a DEBUG, an INFO, and a WARNING message from a named logger: the level, name, and + message of each line the console handler wrote.""" + named = logger.bind(name="test") + named.debug("debug message") + named.info("info message") + named.warning("warning message") + logger.complete() + parsed: list[tuple[str, str, str]] = [] + # time | level | name - message + for line in _ANSI_ESCAPE.sub("", capsys.readouterr().err).splitlines(): + _, level, rest = line.split(" | ", 2) + name, message = rest.split(" - ", 1) + parsed.append((level.strip(), name, message)) + return parsed + + +@pytest.fixture +def records() -> Iterator[list[Record]]: + """Every record logged during the test, as loguru hands it to a sink.""" + collected: list[Record] = [] + handler_id = logger.add(lambda message: collected.append(message.record), level="DEBUG") + yield collected + logger.remove(handler_id) + + class TestSetupLogging: """Tests for setup_logging function.""" @@ -39,46 +72,28 @@ def test_setup_logging_default_level(self) -> None: setup_logging(level="INFO") assert is_configured() - def test_setup_logging_verbose_overrides_level(self) -> None: + def test_setup_logging_verbose_overrides_level( + self, capsys: pytest.CaptureFixture[str] + ) -> None: """Test that verbose flag sets DEBUG level.""" - messages: list[str] = [] setup_logging(level="WARNING", verbose=True) + assert emitted(capsys) == [ + ("DEBUG", "test", "debug message"), + ("INFO", "test", "info message"), + ("WARNING", "test", "warning message"), + ] - handler_id = logger.add(lambda msg: messages.append(str(msg))) - try: - logger.bind(name="test").debug("debug message") - assert any("debug message" in msg for msg in messages) - finally: - logger.remove(handler_id) - - def test_setup_logging_quiet_overrides_level(self) -> None: + def test_setup_logging_quiet_overrides_level(self, capsys: pytest.CaptureFixture[str]) -> None: """Test that quiet flag sets WARNING level.""" - messages: list[str] = [] setup_logging(level="DEBUG", quiet=True) + assert emitted(capsys) == [("WARNING", "test", "warning message")] - handler_id = logger.add(lambda msg: messages.append(str(msg)), level="DEBUG") - try: - # INFO should be filtered by the setup handlers - # but our test handler captures everything - logger.bind(name="test").info("info message") - # The main handlers filter INFO when quiet=True - # We're just verifying setup_logging doesn't crash - assert is_configured() - finally: - logger.remove(handler_id) - - def test_setup_logging_verbose_takes_precedence(self) -> None: + def test_setup_logging_verbose_takes_precedence( + self, capsys: pytest.CaptureFixture[str] + ) -> None: """Test verbose takes precedence over quiet when both set.""" - messages: list[str] = [] setup_logging(level="INFO", verbose=True, quiet=True) - - handler_id = logger.add(lambda msg: messages.append(str(msg))) - try: - logger.bind(name="test").debug("debug message") - # Verbose wins, so DEBUG should be captured - assert any("debug message" in msg for msg in messages) - finally: - logger.remove(handler_id) + assert [level for level, _, _ in emitted(capsys)] == ["DEBUG", "INFO", "WARNING"] def test_setup_logging_with_file(self, tmp_path: Path) -> None: """Test file logging setup.""" @@ -86,15 +101,21 @@ def test_setup_logging_with_file(self, tmp_path: Path) -> None: setup_logging(level="INFO", log_file=log_file) logger.bind(name="test").info("Test file message") + logger.complete() - # Give loguru a moment to flush - import time - - time.sleep(0.1) - - assert log_file.exists() - content = log_file.read_text() - assert "Test file message" in content + # time | level | name:function:line | extra | message + fields = [line.split(" | ") for line in log_file.read_text().splitlines()] + assert [ + (level.strip(), where.split(":")[:2], ast.literal_eval(extra), message) + for _, level, where, extra, message in fields + ] == [ + ( + "INFO", + ["test", "test_setup_logging_with_file"], + {"name": "test"}, + "Test file message", + ) + ] @pytest.mark.parametrize("bound", [True, False], ids=["named logger", "stdlib or unnamed"]) def test_a_logged_traceback_shows_no_local_values( @@ -129,20 +150,18 @@ class TestInterceptHandler: """Tests for stdlib logging interception.""" def test_intercept_stdlib_logging(self) -> None: - """Test that stdlib logging is routed to loguru.""" - messages: list[str] = [] + """Test that stdlib logging is routed to loguru at its level.""" setup_logging(level="DEBUG") - - handler_id = logger.add(lambda msg: messages.append(str(msg))) + records: list[Record] = [] + handler_id = logger.add(lambda message: records.append(message.record)) try: - # Create a stdlib logger stdlib_logger = logging.getLogger("test_stdlib_intercept") stdlib_logger.warning("Hello from stdlib") - - # Should be captured by loguru - assert any("Hello from stdlib" in msg for msg in messages) finally: logger.remove(handler_id) + assert [(record["level"].name, record["message"]) for record in records] == [ + ("WARNING", "Hello from stdlib") + ] def test_sqlalchemy_logging_controlled(self) -> None: """Test SQLAlchemy logger level is controlled.""" @@ -164,77 +183,43 @@ def test_get_logger_returns_logger(self) -> None: assert hasattr(test_logger, "debug") assert hasattr(test_logger, "error") - def test_get_logger_binds_name(self) -> None: + def test_get_logger_binds_name(self, records: list[Record]) -> None: """Test that get_logger binds the module name.""" - messages: list[str] = [] - setup_logging(level="DEBUG") - - handler_id = logger.add( - lambda msg: messages.append(str(msg)), - format="{extra} | {message}", - ) - try: - test_logger = get_logger("my_test_module") - test_logger.info("Test message") - assert any("my_test_module" in msg for msg in messages) - finally: - logger.remove(handler_id) + test_logger = get_logger("my_test_module") + test_logger.info("Test message") + assert [(record["extra"], record["message"]) for record in records] == [ + ({"name": "my_test_module"}, "Test message") + ] class TestContextBinding: """Tests for context binding helpers.""" - def test_bind_repo(self) -> None: + def test_bind_repo(self, records: list[Record]) -> None: """Test bind_repo adds repo context.""" - messages: list[str] = [] - setup_logging(level="DEBUG") - - handler_id = logger.add( - lambda msg: messages.append(str(msg)), - format="{extra} | {message}", - ) - try: - repo_logger = bind_repo("prebid", "prebid-server") - repo_logger.info("Test repo message") - assert any("prebid/prebid-server" in msg for msg in messages) - finally: - logger.remove(handler_id) + repo_logger = bind_repo("prebid", "prebid-server") + repo_logger.info("Test repo message") + assert [record["extra"] for record in records] == [ + {"name": "sync", "repo": "prebid/prebid-server"} + ] - def test_bind_pr(self) -> None: + def test_bind_pr(self, records: list[Record]) -> None: """Test bind_pr adds repo and PR context.""" - messages: list[str] = [] - setup_logging(level="DEBUG") - - handler_id = logger.add( - lambda msg: messages.append(str(msg)), - format="{extra} | {message}", - ) - try: - pr_logger = bind_pr("prebid", "prebid-server", 123) - pr_logger.info("Test PR message") - output = "".join(messages) - assert "prebid/prebid-server" in output - assert "123" in output - finally: - logger.remove(handler_id) - - def test_log_context_manager(self) -> None: - """Test LogContext context manager.""" - messages: list[str] = [] - setup_logging(level="DEBUG") - - handler_id = logger.add( - lambda msg: messages.append(str(msg)), - format="{extra} | {message}", - ) - try: - with LogContext(custom_key="custom_value"): - logger.info("Inside context") - - # Inside should have context - assert any("custom_value" in msg for msg in messages) - finally: - logger.remove(handler_id) + pr_logger = bind_pr("prebid", "prebid-server", 123) + pr_logger.info("Test PR message") + assert [record["extra"] for record in records] == [ + {"name": "sync", "repo": "prebid/prebid-server", "pr": 123} + ] + + def test_log_context_manager(self, records: list[Record]) -> None: + """Test LogContext context manager: the context is bound inside it and gone after.""" + with LogContext(custom_key="custom_value"): + logger.info("Inside context") + logger.info("After context") + assert [(record["message"], record["extra"]) for record in records] == [ + ("Inside context", {"custom_key": "custom_value"}), + ("After context", {}), + ] class TestLogLevels: diff --git a/tests/test_tracking_config.py b/tests/test_tracking_config.py index d6652ba..55e876c 100644 --- a/tests/test_tracking_config.py +++ b/tests/test_tracking_config.py @@ -1,5 +1,6 @@ """Tests for the tracked-repository configuration.""" +import tomllib from datetime import UTC, datetime from pathlib import Path @@ -114,23 +115,27 @@ def test_settings_reads_the_configured_path(self, tmp_path: Path, monkeypatch) - class TestValidation: def test_missing_file(self, tmp_path: Path) -> None: missing = tmp_path / "absent.toml" - with pytest.raises(TrackingConfigError, match="not found"): + with pytest.raises(TrackingConfigError) as raised: load_tracking_config(missing) + assert type(raised.value.__cause__) is FileNotFoundError def test_unreadable_path(self, tmp_path: Path) -> None: # A directory raises IsADirectoryError, an OSError other than FileNotFoundError. - with pytest.raises(TrackingConfigError, match="cannot read tracking config"): + with pytest.raises(TrackingConfigError) as raised: load_tracking_config(tmp_path) + assert type(raised.value.__cause__) is IsADirectoryError def test_invalid_toml(self, tmp_path: Path) -> None: - with pytest.raises(TrackingConfigError, match="not valid TOML"): + with pytest.raises(TrackingConfigError) as raised: load_tracking_config(_write(tmp_path, "[[repositories]\n")) + assert type(raised.value.__cause__) is tomllib.TOMLDecodeError def test_non_utf8_file(self, tmp_path: Path) -> None: path = tmp_path / "tracking.toml" path.write_bytes(b"\xff\xfe") - with pytest.raises(TrackingConfigError, match="not valid TOML"): + with pytest.raises(TrackingConfigError) as raised: load_tracking_config(path) + assert type(raised.value.__cause__) is UnicodeDecodeError def test_empty_repository_list(self, tmp_path: Path) -> None: assert _errors(tmp_path, "repositories = []\n" + COVERAGE) == [NO_REPOSITORY] From 8f721e03df8f7a48ab24a8d9ef94be8655cab205 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 08:05:40 -0600 Subject: [PATCH 06/29] Step 3b-2a-2d: time the sync report on the run's clock; a uid-free .env test - cli/sync.py measured the report's seconds with time.monotonic(), a wall clock in every code path a CLI sync test runs. It now reads the clock the run reads (common.clock()), and a test advances the fake clock seven seconds during the resolve and finds "7 s" in the report. - The unreadable .env test made the file mode 000, which a root user can still read. The OS's refusal is now stubbed at the dotenv read (PermissionError), whatever uid runs the tests; the not-UTF-8 case stays a real file through the installed command. - D39's inputs name a timing setting a test shortens (SQLite's busy timeout), which changes how long a wait lasts and no outcome. --- CLAUDE.md | 2 +- CONTRIBUTING.md | 2 +- docs/redesign-plan.md | 4 +-- docs/testing.md | 2 +- src/github_activity_db/cli/sync.py | 6 ++-- tests/test_cli_app.py | 45 +++++++++++++++++++----------- tests/test_cli_sync.py | 13 +++++++++ 7 files changed, 49 insertions(+), 25 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a2a8614..57c23bf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,7 +49,7 @@ or an acceptance's inputs are GitHub's recorded responses; stubs of a response G environment can send (a status, a body, a timeout, an interrupt, a failing `git`, a clock), which a test may build in memory from a recorded unit changed into that shape; calls the code under test must refuse, and configuration it must refuse or that names a state GitHub can produce (a repository -that does not resolve, one no longer listed); valid rows a test writes (through the models with +that does not resolve, one no longer listed); a timing setting a test shortens (SQLite's busy timeout), which changes how long a wait lasts and no outcome; valid rows a test writes (through the models with `tests/db/seed.py`, or in SQL); and databases and files this project's own code wrote, at any revision. Each test asserts the exact result. A check over stored data, files, or source that only a defect of this code could fail is tested on its passing side, with the exact count of the rows, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 34e8942..c4f0953 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -254,7 +254,7 @@ measures how many such flaws are found. A test's or an acceptance's inputs are G responses; stubs of a response GitHub or the environment can send (a status, a body, a timeout, an interrupt, a failing `git`, a clock), which a test may build in memory from a recorded unit changed into that shape; calls the code under test must refuse, and configuration it must refuse or that -names a state GitHub can produce (a repository that does not resolve, one no longer listed); valid +names a state GitHub can produce (a repository that does not resolve, one no longer listed); a timing setting a test shortens (SQLite's busy timeout), which changes how long a wait lasts and no outcome; valid rows a test writes (through the models with `tests/db/seed.py`, or in SQL); and databases and files this project's own code wrote, at any revision. Each test asserts the exact result. A check over stored data, files, or source that only a defect of this code could fail is tested on its passing diff --git a/docs/redesign-plan.md b/docs/redesign-plan.md index 2b043cf..e8a58ed 100644 --- a/docs/redesign-plan.md +++ b/docs/redesign-plan.md @@ -93,7 +93,7 @@ Within scope the database collects and distinguishes data and never interprets i | D36 | Verify decisions of 2026-10-21: Step 3b-2b is two pull requests, the REST sampler and then verify with the live acceptance (section 6). Verify writes nothing to the database and claims no run: before any request it reads `sync_runs` and exits 1 while a row holds the lock by the claim's rule, so a stale or crashed run stays for the next claim or `sync --break-lock` (5.3). `--repo` limits verify to the repositories it names, as it limits a sync (D27). A verify that never starts writes no report; once it starts, `--report` is written on every exit verify's own code handles, and one that cannot be written turns an exit 0 or 2 into 1, because Step 6a reads it, while an interrupt exits 130 either way; verify's printed output follows sync's rule (5.3 step 8). Every repository failure 5.3 gives a resolve, an index, or a hydration batch, a repository error during a timeline lookup, and from Step 4b a log or history read verify cannot complete make a repository unverified, and a verify in which every repository it attempts is unverified, the deadline having stopped none, is fatal. verify reads each repository's stored rows in two statements before indexing it, and from Step 4b its stored events and default-branch commits in one statement each before the first request of its log and history reads, and its report records its own points, REST calls, and time (5.4, gap 94). | | D37 | Decision of 2026-10-21 (the maintainer): an error is better than a silent failure or silently wrong data. Where the code cannot check something, or cannot accept what GitHub returned, it reports a failure that stops publication or turns the run red, never a pass. For verify: an unchecked item and an unverified repository fail verify in both modes, not only `--exact`, and a lookup whose answer verify cannot accept is an error (5.4, 5.8, gap 97), so D25 does not publish a database verify could not check. | | D38 | Decisions of 2026-10-21 applying D37 to the sync and to the steps not yet built (gaps 98-110). The sync exits 2 when a 24-hour account refresh fails, when the run reaches its deadline, and when a hydration contradicts the run's own index after one retry (Step 3b-2a-3); the failures a batch made final before a deadline are recorded, not left pending. An `accepted_failures` entry accepts one error class and one cause, the first line of the failure's message, of an item, an account's refresh, or from Step 4b a compare, so a permanent failure can be accepted without turning every run red while a new cause is not. A PR with more closing links than the index carries is hydrated every run. Every node id a sighting, a lookup, or a commit link reports for a stored account is stored, and `sync_runs` records the `--repo` filter, each configured entry's `github_id`, and its first commit-link lookups and those GitHub answered `NOT_FOUND`. The source refuses what it cannot read (gap 104), and the run records refuse what they cannot write and mark `-unknown` what they cannot determine (gap 105). Step 4b marks a compare file list at GitHub's 300-file cap and counts it in verify's report and the manifest, completes an activity read only at the floor or at the log's first event, as a read in ascending order finds it, fails a history read whose count still differs after a rescan, and records a compare that still fails, which an entry can accept. Step 5 measures which edits leave `updatedAt` unchanged. Step 6 compares the App's reads with the token's before any App run publishes to `db`, pins the database file the job checked, tests that every exit-code `if:` compares a non-empty string, and ties a snapshot to the job's own sync. A schedule that stops is the one silent case this repository cannot see: no run starts, so nothing turns red; 5.6 states the staleness rule a consumer applies, and building it belongs to the dashboard's integration (section 9). Left as they are, because the stored data already says what happened: a commit-linked account a lookup answers `NOT_FOUND` (D28; now counted per run and in the manifest, and from Step 6a a run whose `links_not_found` exceeds the share of its `link_lookups` Step 5 sets, with at least Step 5's minimum number of lookups, exits 2 (an account an earlier lookup answered `NOT_FOUND` counted in neither), since a lookup form GitHub stopped resolving would look the same per link), a compare at GitHub's 300-file cap and one GitHub refuses (marked by `files_at_cap` and `http_status`, counted in verify's report and the manifest), a REST git author without a name or email (stored null, as GitHub reports none), deletions recorded as a count (`items_deleted`, and from Step 4b `events_deleted` and `history_commits_deleted`; the snapshots retained for 7 days hold the identities; older ones are not kept), which accounts failed a refresh (printed; the count is stored, the run is red unless an account entry accepts the failure, and the accounts are retried), an `author_association` difference (verify reports it as drift with its own count: GitHub computes association when read and changes it without moving `updatedAt`, section 7, so no read tells a change from a defect), and the merge state as of `hydrated_at` (D32). New tables and columns of GitHub-derived data on a populated file follow gap 110's rule (5.1). | -| D39 | Decision of 2026-10-21 (the maintainer). Nothing is planted or tampered, no flaw is seeded, and there is no mutation testing, under any name. No test, review, or acceptance changes a file or a copy of one (other than a stub or configuration the inputs below allow), or writes a file, a row, a revision, a call site, or a decoy holding a flaw, to see whether something catches it, and none measures how many such flaws are found. A test's or an acceptance's inputs are GitHub's recorded responses; stubs of a response GitHub or the environment can send (a status, a body, a timeout, an interrupt, a failing `git`, a clock), which a test may build in memory from a recorded unit changed into that shape; calls the code under test must refuse, and configuration it must refuse or that names a state GitHub can produce (a repository that does not resolve, one no longer listed); valid rows a test writes (through the models with `tests/db/seed.py`, or in SQL); and databases and files this project's own code wrote, at any revision. Each test asserts the exact result. A check over stored data, files, or source that only a defect of this code could fail is tested on its passing side, with the exact count of the rows, files, or values it compared, so it cannot pass on nothing; a check that GitHub, the environment, or a caller can fail is tested on both sides. A fix comes with a regression test that fails against the parent commit's code. Tests merged before D39 that change a file or a copy of one, or write a file, a revision, a call, a connection, or a decoy holding a flaw, to see whether a check catches it are replaced in Step 3b-2a-2d. | +| D39 | Decision of 2026-10-21 (the maintainer). Nothing is planted or tampered, no flaw is seeded, and there is no mutation testing, under any name. No test, review, or acceptance changes a file or a copy of one (other than a stub or configuration the inputs below allow), or writes a file, a row, a revision, a call site, or a decoy holding a flaw, to see whether something catches it, and none measures how many such flaws are found. A test's or an acceptance's inputs are GitHub's recorded responses; stubs of a response GitHub or the environment can send (a status, a body, a timeout, an interrupt, a failing `git`, a clock), which a test may build in memory from a recorded unit changed into that shape; calls the code under test must refuse, and configuration it must refuse or that names a state GitHub can produce (a repository that does not resolve, one no longer listed); a timing setting a test shortens (SQLite's busy timeout), which changes how long a wait lasts and no outcome; valid rows a test writes (through the models with `tests/db/seed.py`, or in SQL); and databases and files this project's own code wrote, at any revision. Each test asserts the exact result. A check over stored data, files, or source that only a defect of this code could fail is tested on its passing side, with the exact count of the rows, files, or values it compared, so it cannot pass on nothing; a check that GitHub, the environment, or a caller can fail is tested on both sides. A fix comes with a regression test that fails against the parent commit's code. Tests merged before D39 that change a file or a copy of one, or write a file, a revision, a call, a connection, or a decoy holding a flaw, to see whether a check catches it are replaced in Step 3b-2a-2d. | | D40 | Decision of 2026-10-22 (the maintainer). Tests are software that gives the same result every time. The same code gives every test the same outcome on any machine, on any date, and in any run order: a test, and every code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state outside its own temporary directory, the repository, and the session's database template, and bounds every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on printed wording, and no test checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode), never a planted file (D39). Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. `tests/test_suite_rules.py` enforces the rules it can model, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test (Step 3b-2a-2d). | ## 3. Coverage rule @@ -1590,7 +1590,7 @@ such flaws are found. A test's or an acceptance's inputs are GitHub's recorded r response GitHub or the environment can send (a status, a body, a timeout, an interrupt, a failing `git`, a clock), which a test may build in memory from a recorded unit changed into that shape; calls the code under test must refuse, and configuration it must refuse or that names a state GitHub -can produce (a repository that does not resolve, one no longer listed); valid rows a test writes +can produce (a repository that does not resolve, one no longer listed); a timing setting a test shortens (SQLite's busy timeout), which changes how long a wait lasts and no outcome; valid rows a test writes (through the models with `tests/db/seed.py`, or in SQL); and databases and files this project's own code wrote, at any revision. Each test asserts the exact result. A check over stored data, files, or source that only a defect of this code could fail is tested on its passing side, with the exact diff --git a/docs/testing.md b/docs/testing.md index 0428569..d9c385e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -19,7 +19,7 @@ How the test suite is built and what it counts as evidence. The plan of record i timeout, an interrupt, a failing `git`, a clock), which a test may build in memory from a recorded unit changed into that shape; calls the code under test must refuse, and configuration it must refuse or that names a state GitHub can produce (a repository that does not resolve, one no longer - listed); valid rows a test writes (through the models with `tests/db/seed.py`, or in SQL); and + listed); a timing setting a test shortens (SQLite's busy timeout), which changes how long a wait lasts and no outcome; valid rows a test writes (through the models with `tests/db/seed.py`, or in SQL); and databases and files this project's own code wrote, at any revision. Each test asserts the exact result. A check over stored data, files, or source that only a defect of this code could fail is tested on its passing side, with the exact count of the rows, files, or values it compared, so it diff --git a/src/github_activity_db/cli/sync.py b/src/github_activity_db/cli/sync.py index 3c0a099..af77917 100644 --- a/src/github_activity_db/cli/sync.py +++ b/src/github_activity_db/cli/sync.py @@ -12,7 +12,6 @@ import errno import os import sys -import time from collections.abc import Iterator from pathlib import Path from typing import Annotated, TextIO @@ -105,7 +104,7 @@ def sync( prune_untracked=prune_untracked, dry_run=dry_run, ) - started = time.monotonic() + started = clock.now() report = run_async_command( engine.sync( options, @@ -115,7 +114,8 @@ def sync( error_prefix="Sync did not start", ) try: - print_report(console, report, seconds=time.monotonic() - started, break_lock=break_lock) + seconds = (clock.now() - started).total_seconds() + print_report(console, report, seconds=seconds, break_lock=break_lock) except (UnicodeEncodeError, OSError) as e: # The run is recorded; an output that cannot take the report (an encoding without one of # its characters, a closed pipe, a full disk) never changes the exit code. diff --git a/tests/test_cli_app.py b/tests/test_cli_app.py index ff64037..4617c35 100644 --- a/tests/test_cli_app.py +++ b/tests/test_cli_app.py @@ -10,6 +10,7 @@ from importlib import metadata from pathlib import Path +import pydantic_settings.sources.providers.dotenv as dotenv_source import pytest import typer from typer.testing import CliRunner @@ -222,27 +223,37 @@ def test_unparsable_settings_exit_1_without_a_traceback(self, tmp_path: Path) -> assert "invalid settings" in _words(result.stderr) assert "Traceback" not in result.stderr - @pytest.mark.parametrize( - ("dotenv", "mode"), - [ - pytest.param(b"LOG_LEVEL=\xff\n", 0o644, id="not UTF-8"), - pytest.param(b"", 0o000, id="unreadable"), - ], - ) - def test_an_unreadable_env_file_exits_1_without_a_traceback( - self, tmp_path: Path, dotenv: bytes, mode: int - ) -> None: - env_file = tmp_path / ".env" - env_file.write_bytes(dotenv) - env_file.chmod(mode) - try: - result = _installed("github", "rate-limit", env={}, cwd=tmp_path) - finally: - env_file.chmod(0o644) + def test_an_env_file_that_is_not_utf8_exits_1_without_a_traceback(self, tmp_path: Path) -> None: + (tmp_path / ".env").write_bytes(b"LOG_LEVEL=\xff\n") + result = _installed("github", "rate-limit", env={}, cwd=tmp_path) assert result.returncode == 1 assert "invalid settings" in _words(result.stderr) assert "Traceback" not in result.stderr + +def test_an_env_file_the_os_refuses_to_read_exits_1( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A .env the OS refuses to read: the read answers PermissionError, as it does for a file + the user may not read, whatever uid runs the tests (D40), and the command exits 1.""" + (tmp_path / ".env").write_text("LOG_LEVEL=INFO\n") + read: list[Path] = [] + + def refused(path: Path, **_: object) -> dict[str, str | None]: + read.append(Path(path)) + raise PermissionError(13, "Permission denied", str(path)) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(dotenv_source, "dotenv_values", refused) + get_settings.cache_clear() + try: + result = runner.invoke(app, ["github", "rate-limit"]) + finally: + get_settings.cache_clear() + assert read == [Path(".env")] + assert result.exit_code == 1, result.output + assert isinstance(result.exception, SystemExit), result.exception + @pytest.mark.parametrize( "env", [ diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index 21bdf99..832f7b6 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -285,6 +285,19 @@ def test_a_first_sync_exits_0_and_prints_its_report( assert fetch(migrated_database, "SELECT count(*) AS n FROM pull_requests") == [{"n": 11}] +def test_the_report_times_the_run_on_its_clock(replay: Replay, migrated_database: Path) -> None: + """The report's seconds come from the clock the run reads (D40): seven seconds pass on the + fake clock while the resolve is answered, and the report says 7 s.""" + clock = common.clock() + assert isinstance(clock, FakeClock) # the replay fixture's + replay.modify(lambda body: clock.advance(7), when=lambda seen: seen.operation == "Resolve") + + result = _sync() + + assert result.exit_code == 0, result.output + assert "Hydrated 11 items; GraphQL points 4, REST calls 0, 7 s" in _plain(result.output) + + def test_an_item_failure_exits_2_and_is_reported(replay: Replay, migrated_database: Path) -> None: replay.modify(_alias_error, when=lambda seen: seen.operation == "PullRequestBatch", times=2) From 045471216bb4edfe4365d3064c7272322c8e0ca6 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 08:20:23 -0600 Subject: [PATCH 07/29] Step 3b-2a-2d: collect the installed entry point's tests again The module-level .env test sat inside TestInstalledEntryPoint, so the class's last four methods (six test ids) became functions nested in it, which pytest never collects. The test now follows the class. --- tests/test_cli_app.py | 48 +++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/test_cli_app.py b/tests/test_cli_app.py index 4617c35..4da96bf 100644 --- a/tests/test_cli_app.py +++ b/tests/test_cli_app.py @@ -230,30 +230,6 @@ def test_an_env_file_that_is_not_utf8_exits_1_without_a_traceback(self, tmp_path assert "invalid settings" in _words(result.stderr) assert "Traceback" not in result.stderr - -def test_an_env_file_the_os_refuses_to_read_exits_1( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """A .env the OS refuses to read: the read answers PermissionError, as it does for a file - the user may not read, whatever uid runs the tests (D40), and the command exits 1.""" - (tmp_path / ".env").write_text("LOG_LEVEL=INFO\n") - read: list[Path] = [] - - def refused(path: Path, **_: object) -> dict[str, str | None]: - read.append(Path(path)) - raise PermissionError(13, "Permission denied", str(path)) - - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(dotenv_source, "dotenv_values", refused) - get_settings.cache_clear() - try: - result = runner.invoke(app, ["github", "rate-limit"]) - finally: - get_settings.cache_clear() - assert read == [Path(".env")] - assert result.exit_code == 1, result.output - assert isinstance(result.exception, SystemExit), result.exception - @pytest.mark.parametrize( "env", [ @@ -299,3 +275,27 @@ def test_a_traceback_shows_no_local_variables(self) -> None: (script,) = metadata.entry_points(group="console_scripts", name="ghactivity") assert script.load() is app assert app.pretty_exceptions_show_locals is False + + +def test_an_env_file_the_os_refuses_to_read_exits_1( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A .env the OS refuses to read: the read answers PermissionError, as it does for a file + the user may not read, whatever uid runs the tests (D40), and the command exits 1.""" + (tmp_path / ".env").write_text("LOG_LEVEL=INFO\n") + read: list[Path] = [] + + def refused(path: Path, **_: object) -> dict[str, str | None]: + read.append(Path(path)) + raise PermissionError(13, "Permission denied", str(path)) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(dotenv_source, "dotenv_values", refused) + get_settings.cache_clear() + try: + result = runner.invoke(app, ["github", "rate-limit"]) + finally: + get_settings.cache_clear() + assert read == [Path(".env")] + assert result.exit_code == 1, result.output + assert isinstance(result.exception, SystemExit), result.exception From a02a867db9e006c6430b50d29d8c5f530c454d49 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 08:20:37 -0600 Subject: [PATCH 08/29] Step 3b-2a-2d: refuse the network and the real clock in every test (D40) tests/determinism.py, installed for the session by the conftest, refuses a network connection other than a Unix socket, every host-name lookup, and SystemClock's now, sleep, and unseeded random source. A refusal raises NotDeterministic (a BaseException) and is recorded; a test that reached one fails after it runs even where the code caught it. tests/test_suite_rules.py reads source: no test file reads the wall clock, sleeps on it, or draws unseeded random numbers; the package reads them only in SystemClock and db.types.utc_now; and no test function is defined inside another function. scripts/check_junit.py --same-as is tested on both sides over reports pytest wrote. The plan's 3b-2a-2d scope names the one src change (the sync report's timing), and CLAUDE.md, the testing guide, and CONTRIBUTING.md describe what enforces D40. --- CLAUDE.md | 20 ++-- CONTRIBUTING.md | 2 +- docs/redesign-plan.md | 26 +++-- docs/testing.md | 32 ++++- tests/conftest.py | 19 +++ tests/determinism.py | 74 ++++++++++++ tests/test_check_junit.py | 70 ++++++++++- tests/test_suite_rules.py | 240 +++++++++++++++++++++++++++++++++++++- 8 files changed, 454 insertions(+), 29 deletions(-) create mode 100644 tests/determinism.py diff --git a/CLAUDE.md b/CLAUDE.md index 57c23bf..87f6f19 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,7 +56,7 @@ defect of this code could fail is tested on its passing side, with the exact cou files, or values it compared, so it cannot pass on nothing; a check that GitHub, the environment, or a caller can fail is tested on both sides. A fix comes with a regression test that fails against the parent commit's code. Tests merged before D39 that change a file or a copy of one, or write a file, -a revision, a call, a connection, or a decoy holding a flaw, to see whether a check catches it are replaced in Step 3b-2a-2d. See [Testing](docs/testing.md). +a revision, a call, a connection, or a decoy holding a flaw, to see whether a check catches it were replaced in Step 3b-2a-2d. See [Testing](docs/testing.md). ## Deterministic Tests @@ -72,9 +72,10 @@ checks the wording of a document. Where a test needs an input the code cannot ea project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode), never a planted file (D39). Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a -review finds work to do but is never acceptance. `tests/test_suite_rules.py` enforces the rules it -can model, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test (Step -3b-2a-2d). +review finds work to do but is never acceptance. `tests/determinism.py` refuses the network, the +real clock, and its unseeded random source in every test, `tests/test_suite_rules.py` enforces the +rules it can model in source, and `scripts/check_junit.py --same-as` compares two runs of the suite +test by test (Step 3b-2a-2d). ## Current Scope @@ -97,14 +98,15 @@ can model, and `scripts/check_junit.py --same-as` compares two runs of the suite planner, and the writer, and `--dry-run` - `ghactivity sync` (Step 3b-2a-2c, `cli/sync.py`): the engine's run with the options of plan 5.7; a run that never starts exits 1, and a run that starts exits with the engine's 0, 2, or 1 +- Tests without planted inputs, deterministic (Step 3b-2a-2d, D39, D40): `tests/determinism.py` + refuses the network, the real clock, and its unseeded random source in every test; + `tests/test_suite_rules.py`'s source rules; `scripts/check_junit.py --same-as` -**Not implemented yet:** tests without planted inputs (Step 3b-2a-2d, D39), D37 in the sync -(Step 3b-2a-3, D38), the REST -sampler (Step 3b-2b-1), and `verify` (Step 3b-2b-2). The tracked `github_activity.db` stays +**Not implemented yet:** D37 in the sync (Step 3b-2a-3, D38), the REST sampler (Step 3b-2b-1), +and `verify` (Step 3b-2b-2). The tracked `github_activity.db` stays frozen at the first schema (D26). -**Planned in the redesign plan:** tests without planted inputs (Step 3b-2a-2d), D37 in the sync -(Step 3b-2a-3), the REST sampler (Step +**Planned in the redesign plan:** D37 in the sync (Step 3b-2a-3), the REST sampler (Step 3b-2b-1), and verify (Step 3b-2b-2); GitHub Issues, closing-issue links, and default-branch activity (Step 4); the full bootstrap (Step 5); scheduled sync and release publication (Step 6). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c4f0953..bc27522 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -262,7 +262,7 @@ side, with the exact count of the rows, files, or values it compared, so it cann a check that GitHub, the environment, or a caller can fail is tested on both sides. A fix comes with a regression test that fails against the parent commit's code. Tests merged before D39 that change a file or a copy of one, or write a file, a revision, a call, a connection, or a decoy holding a flaw, -to see whether a check catches it are replaced in Step 3b-2a-2d. See the [Testing +to see whether a check catches it were replaced in Step 3b-2a-2d. See the [Testing Guide](docs/testing.md#principles). **4. Write tests that give the same result every time** (redesign plan D40). The same code gives diff --git a/docs/redesign-plan.md b/docs/redesign-plan.md index e8a58ed..c3a10e6 100644 --- a/docs/redesign-plan.md +++ b/docs/redesign-plan.md @@ -94,7 +94,7 @@ Within scope the database collects and distinguishes data and never interprets i | D37 | Decision of 2026-10-21 (the maintainer): an error is better than a silent failure or silently wrong data. Where the code cannot check something, or cannot accept what GitHub returned, it reports a failure that stops publication or turns the run red, never a pass. For verify: an unchecked item and an unverified repository fail verify in both modes, not only `--exact`, and a lookup whose answer verify cannot accept is an error (5.4, 5.8, gap 97), so D25 does not publish a database verify could not check. | | D38 | Decisions of 2026-10-21 applying D37 to the sync and to the steps not yet built (gaps 98-110). The sync exits 2 when a 24-hour account refresh fails, when the run reaches its deadline, and when a hydration contradicts the run's own index after one retry (Step 3b-2a-3); the failures a batch made final before a deadline are recorded, not left pending. An `accepted_failures` entry accepts one error class and one cause, the first line of the failure's message, of an item, an account's refresh, or from Step 4b a compare, so a permanent failure can be accepted without turning every run red while a new cause is not. A PR with more closing links than the index carries is hydrated every run. Every node id a sighting, a lookup, or a commit link reports for a stored account is stored, and `sync_runs` records the `--repo` filter, each configured entry's `github_id`, and its first commit-link lookups and those GitHub answered `NOT_FOUND`. The source refuses what it cannot read (gap 104), and the run records refuse what they cannot write and mark `-unknown` what they cannot determine (gap 105). Step 4b marks a compare file list at GitHub's 300-file cap and counts it in verify's report and the manifest, completes an activity read only at the floor or at the log's first event, as a read in ascending order finds it, fails a history read whose count still differs after a rescan, and records a compare that still fails, which an entry can accept. Step 5 measures which edits leave `updatedAt` unchanged. Step 6 compares the App's reads with the token's before any App run publishes to `db`, pins the database file the job checked, tests that every exit-code `if:` compares a non-empty string, and ties a snapshot to the job's own sync. A schedule that stops is the one silent case this repository cannot see: no run starts, so nothing turns red; 5.6 states the staleness rule a consumer applies, and building it belongs to the dashboard's integration (section 9). Left as they are, because the stored data already says what happened: a commit-linked account a lookup answers `NOT_FOUND` (D28; now counted per run and in the manifest, and from Step 6a a run whose `links_not_found` exceeds the share of its `link_lookups` Step 5 sets, with at least Step 5's minimum number of lookups, exits 2 (an account an earlier lookup answered `NOT_FOUND` counted in neither), since a lookup form GitHub stopped resolving would look the same per link), a compare at GitHub's 300-file cap and one GitHub refuses (marked by `files_at_cap` and `http_status`, counted in verify's report and the manifest), a REST git author without a name or email (stored null, as GitHub reports none), deletions recorded as a count (`items_deleted`, and from Step 4b `events_deleted` and `history_commits_deleted`; the snapshots retained for 7 days hold the identities; older ones are not kept), which accounts failed a refresh (printed; the count is stored, the run is red unless an account entry accepts the failure, and the accounts are retried), an `author_association` difference (verify reports it as drift with its own count: GitHub computes association when read and changes it without moving `updatedAt`, section 7, so no read tells a change from a defect), and the merge state as of `hydrated_at` (D32). New tables and columns of GitHub-derived data on a populated file follow gap 110's rule (5.1). | | D39 | Decision of 2026-10-21 (the maintainer). Nothing is planted or tampered, no flaw is seeded, and there is no mutation testing, under any name. No test, review, or acceptance changes a file or a copy of one (other than a stub or configuration the inputs below allow), or writes a file, a row, a revision, a call site, or a decoy holding a flaw, to see whether something catches it, and none measures how many such flaws are found. A test's or an acceptance's inputs are GitHub's recorded responses; stubs of a response GitHub or the environment can send (a status, a body, a timeout, an interrupt, a failing `git`, a clock), which a test may build in memory from a recorded unit changed into that shape; calls the code under test must refuse, and configuration it must refuse or that names a state GitHub can produce (a repository that does not resolve, one no longer listed); a timing setting a test shortens (SQLite's busy timeout), which changes how long a wait lasts and no outcome; valid rows a test writes (through the models with `tests/db/seed.py`, or in SQL); and databases and files this project's own code wrote, at any revision. Each test asserts the exact result. A check over stored data, files, or source that only a defect of this code could fail is tested on its passing side, with the exact count of the rows, files, or values it compared, so it cannot pass on nothing; a check that GitHub, the environment, or a caller can fail is tested on both sides. A fix comes with a regression test that fails against the parent commit's code. Tests merged before D39 that change a file or a copy of one, or write a file, a revision, a call, a connection, or a decoy holding a flaw, to see whether a check catches it are replaced in Step 3b-2a-2d. | -| D40 | Decision of 2026-10-22 (the maintainer). Tests are software that gives the same result every time. The same code gives every test the same outcome on any machine, on any date, and in any run order: a test, and every code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state outside its own temporary directory, the repository, and the session's database template, and bounds every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on printed wording, and no test checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode), never a planted file (D39). Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. `tests/test_suite_rules.py` enforces the rules it can model, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test (Step 3b-2a-2d). | +| D40 | Decision of 2026-10-22 (the maintainer). Tests are software that gives the same result every time. The same code gives every test the same outcome on any machine, on any date, and in any run order: a test, and every code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state outside its own temporary directory, the repository, and the session's database template, and bounds every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on printed wording, and no test checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode), never a planted file (D39). Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. `tests/determinism.py` refuses the network, the real clock, and its unseeded random source in every test, `tests/test_suite_rules.py` enforces the rules it can model in source, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test (Step 3b-2a-2d). | ## 3. Coverage rule @@ -1943,11 +1943,18 @@ check without a test that targets it; CI passes on the exact PR head. Prerequisite: Step 3b-2a-2c has merged, and D39 is on `main`. -Scope: tests, `docs/testing.md`, and `scripts/check_junit.py`'s `--same-as`; nothing under `src/` -or `alembic/` changes, and no rule loses its test. Every test meets D40, and -`tests/test_suite_rules.py` gains the determinism rules it can model: no wall-clock read, no real -network transport, and no unseeded randomness in a test file, each rule tested on its passing side -over every file under `tests/`, the files it scanned counted against `git ls-files`. Every test merged before D39 that changes a file or a copy of one, or writes a +Scope: tests, `docs/testing.md`, `scripts/check_junit.py`'s `--same-as`, and one change under +`src/`: `ghactivity sync` times its report on the run's clock, not `time.monotonic` (D40). Nothing +under `alembic/` changes, and no rule loses its test. Every test meets D40. +`tests/determinism.py` refuses, for every test in the pytest process, a network connection, a +host-name lookup, and this project's real clock and unseeded random source (`SystemClock`), and +records each refusal, so a test fails even where the code under test caught it; each refusal has a +test. `tests/test_suite_rules.py` gains the rules it can model in source: no test file reads the +wall clock, sleeps on it, or draws unseeded random numbers; the package reads them only in its two +real clocks (`SystemClock` and `db.types.utc_now`); and no test function is defined inside another +function, where pytest never collects it. Each is tested on its passing side over every file under +`tests/` or `src/github_activity_db/`, the files it scanned counted by a second walk of the +directory, and the package scan with the exact list of reads it found. Every test merged before D39 that changes a file or a copy of one, or writes a file, a revision, a call, a connection, or a decoy holding a flaw, to see whether a check catches it, is replaced under D39: a check that GitHub, the environment, or a caller can fail keeps a refusing side built from D39's inputs, and a check that only a defect of this code could fail is tested on @@ -1987,9 +1994,10 @@ environment, or a caller can fail it, or on its passing side with the exact coun pull request lists each test it replaces or removes, with the rule it held and the test that now holds it. -Acceptance: `tests/test_suite_rules.py`'s determinism and subprocess rules pass over every test file -and report the files they scanned; two runs of the full suite give the same outcome for every test, -compared by `scripts/check_junit.py --same-as`; CI passes on the exact PR head. +Acceptance: `tests/test_suite_rules.py`'s rules pass over every test file and package module and +count the files they scanned; each refusal of `tests/determinism.py` has a test; two runs of the +full suite give the same outcome for every test, compared by `scripts/check_junit.py --same-as`; +CI passes on the exact PR head. #### Step 3b-2a-3: D37 in the sync diff --git a/docs/testing.md b/docs/testing.md index d9c385e..7e7f130 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -26,7 +26,7 @@ How the test suite is built and what it counts as evidence. The plan of record i cannot pass on nothing; a check that GitHub, the environment, or a caller can fail is tested on both sides. A fix comes with a regression test that fails against the parent commit's code. Tests merged before D39 that change a file or a copy of one, or write a file, a revision, a call, a - connection, or a decoy holding a flaw, to see whether a check catches it are replaced in Step 3b-2a-2d. + connection, or a decoy holding a flaw, to see whether a check catches it were replaced in Step 3b-2a-2d. - **The same result every time** (redesign plan D40). The same code gives every test the same outcome on any machine, on any date, and in any run order: a test, and every code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no network (GitHub @@ -39,9 +39,10 @@ How the test suite is built and what it counts as evidence. The plan of record i builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode), never a planted file (D39). Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review - finds work to do but is never acceptance. `tests/test_suite_rules.py` enforces the rules it can - model, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test (Step - 3b-2a-2d). + finds work to do but is never acceptance. `tests/determinism.py` refuses the network, the real + clock, and its unseeded random source in every test, `tests/test_suite_rules.py` enforces the + rules it can model in source, and `scripts/check_junit.py --same-as` compares two runs of the + suite test by test (Step 3b-2a-2d). - **No skipped tests** (D22). CI runs `scripts/check_junit.py --max-skipped 0`, which fails a run where any test was skipped (pytest reports `xfail` as skipped) or none ran. - **Time.** Source tests pass a `FakeClock`, so waits and retries are asserted, never slept. @@ -50,6 +51,18 @@ How the test suite is built and what it counts as evidence. The plan of record i The module reached without an import statement (`importlib.import_module`, `__import__`, `sys.modules`) and process APIs outside `subprocess` (`os.system`, `asyncio` subprocesses) are outside its scan. +- **What enforces D40.** In the pytest process, `tests/determinism.py` (installed for the session + by `tests/conftest.py`) refuses a network connection other than a Unix socket, every host-name + lookup, and `SystemClock`'s `now`, `sleep`, and unseeded random source, by raising + `NotDeterministic`; the refusal is also recorded, and a test that reached one fails after it + runs even where the code under test caught the exception. `tests/test_suite_rules.py` reads + source: no test file reads the wall clock, sleeps on it, or draws unseeded random numbers + (`random.Random` takes a seed, `asyncio.sleep` takes `0`); the package reads them only in its + two real clocks, `SystemClock` and `db.types.utc_now`; and no test function is defined inside + another function, where pytest never collects it. Processes a test starts run without the + run-time refusals, and the order of a set of strings changes with `PYTHONHASHSEED` from one + process to the next: two runs compared by `scripts/check_junit.py --same-as` show whether an + outcome depends on either. --- @@ -58,6 +71,7 @@ How the test suite is built and what it counts as evidence. The plan of record i ``` tests/ ├── conftest.py # The session template database and its copies; the scenario fixture +├── determinism.py # Network, real clock, and unseeded randomness refused in every test ├── db/ │ ├── alembic_support.py # Running the real migrations; test-only revisions after head │ ├── schema_support.py # Structural schema readings for the model comparison @@ -303,6 +317,16 @@ uv run pytest --junitxml="${TMPDIR:-/tmp}/junit.xml" \ && uv run python scripts/check_junit.py "${TMPDIR:-/tmp}/junit.xml" --max-skipped 0 ``` +Two runs of the suite give the same outcome for every test (D40). To compare them: + +```bash +uv run pytest --junitxml="${TMPDIR:-/tmp}/first.xml" +uv run pytest --junitxml="${TMPDIR:-/tmp}/second.xml" +uv run python scripts/check_junit.py "${TMPDIR:-/tmp}/first.xml" --same-as "${TMPDIR:-/tmp}/second.xml" +``` + +The comparison fails on each test whose outcome differs, or that only one run holds. + --- ## CI/CD Integration diff --git a/tests/conftest.py b/tests/conftest.py index f9eb177..66ac3e7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,7 @@ from github_activity_db.config import Settings from github_activity_db.db.engine import build_engine +from tests import determinism from tests.db.alembic_support import upgrade if TYPE_CHECKING: @@ -32,6 +33,24 @@ SETTINGS_VARIABLES = frozenset(name.upper() for name in Settings.model_fields) +@pytest.fixture(autouse=True, scope="session") +def no_network_or_wall_clock() -> Iterator[None]: + """Every test runs with the network, the real clock, and its random source refused + (``tests/determinism.py``, redesign plan D40).""" + with pytest.MonkeyPatch.context() as patch: + determinism.install(patch) + yield + + +@pytest.fixture(autouse=True) +def nothing_refused(no_network_or_wall_clock: None) -> Iterator[None]: + """A test fails when it reached something ``tests/determinism.py`` refuses, even where the code + under test caught the refusal.""" + determinism.REFUSED.clear() + yield + assert [(refused.what, refused.detail) for refused in determinism.REFUSED] == [] + + @pytest.fixture(autouse=True) def no_settings_from_the_shell(monkeypatch: pytest.MonkeyPatch) -> None: """Every variable Settings reads is unset, whatever the developer's shell exports; a test diff --git a/tests/determinism.py b/tests/determinism.py new file mode 100644 index 0000000..a626846 --- /dev/null +++ b/tests/determinism.py @@ -0,0 +1,74 @@ +"""What every test runs under (redesign plan D40), installed for the session by the conftest. + +Inside the test process, a connection to anything but a Unix socket and a host-name lookup are +refused, and so are this project's real clock (``SystemClock.now`` and ``sleep``) and its unseeded +random source (``SystemClock._random``, which an instance may replace with a seeded one). A refusal +raises ``NotDeterministic``, a ``BaseException`` so that no ``except Exception`` in the code under +test can turn it into a handled failure, and is recorded in ``REFUSED``, which the conftest checks +after every test. Out of scope: processes a test starts (they run without these patches), the +standard library's clocks and random modules called directly (``tests/test_suite_rules.py`` reads +test files for those), and ``db.types.utc_now``, the default of columns no producer writes. +""" + +from __future__ import annotations + +import socket +from typing import Any, NoReturn + +import pytest + +from github_activity_db.source.clock import SystemClock + + +class NotDeterministic(BaseException): + """A test reached the network, the wall clock, or unseeded randomness.""" + + def __init__(self, what: str, detail: str) -> None: + super().__init__(f"{what}: {detail}") + self.what = what + self.detail = detail + + +REFUSED: list[NotDeterministic] = [] + + +def refuse(what: str, detail: str) -> NoReturn: + refusal = NotDeterministic(what, detail) + REFUSED.append(refusal) + raise refusal + + +class _NoSystemRandom: + def uniform(self, low: float, high: float) -> NoReturn: + refuse("randomness", f"SystemClock._random.uniform({low}, {high})") + + +def install(patch: pytest.MonkeyPatch) -> None: + connect = socket.socket.connect + connect_ex = socket.socket.connect_ex + + def connecting(self: socket.socket, address: Any) -> None: + if self.family != socket.AF_UNIX: + refuse("network", f"connect {address!r}") + connect(self, address) + + def connecting_ex(self: socket.socket, address: Any) -> int: + if self.family != socket.AF_UNIX: + refuse("network", f"connect {address!r}") + return connect_ex(self, address) + + def looking_up(host: Any, port: Any, *args: Any, **kwargs: Any) -> NoReturn: + refuse("network", f"getaddrinfo {host!r} {port!r}") + + def now(self: SystemClock) -> NoReturn: + refuse("wall clock", "SystemClock.now()") + + async def sleep(self: SystemClock, seconds: float) -> NoReturn: + refuse("wall clock", f"SystemClock.sleep({seconds})") + + patch.setattr(socket.socket, "connect", connecting) + patch.setattr(socket.socket, "connect_ex", connecting_ex) + patch.setattr(socket, "getaddrinfo", looking_up) + patch.setattr(SystemClock, "now", now) + patch.setattr(SystemClock, "sleep", sleep) + patch.setattr(SystemClock, "_random", _NoSystemRandom()) diff --git a/tests/test_check_junit.py b/tests/test_check_junit.py index 3478740..6477c75 100644 --- a/tests/test_check_junit.py +++ b/tests/test_check_junit.py @@ -2,7 +2,8 @@ Every report is one pytest wrote, in a run over a test file written for it, or is built in memory from one with its counters changed into a shape a run can leave: failures, errors, skips, no test -executed, several suites merged, a file cut short, or no suite at all. +executed, several suites merged, a file cut short, or no suite at all. ``--same-as`` compares two +reports pytest wrote, in runs over test files written for them. """ import ast @@ -160,3 +161,70 @@ def test_report_written_by_pytest( result = _gate(tmp_path / "junit.xml") assert result.returncode == gate_exit assert _printed(result) == {"tests": 1, "failures": 0, "errors": 0, "skipped": skipped} + + +def _written_report(root: Path, test_body: str) -> Path: + """The report pytest wrote for one run over ``test_body``.""" + root.mkdir() + _pytest(root, test_body) + return root / "junit.xml" + + +PASSES = "def test_a():\n pass\n" + + +def test_two_runs_with_the_same_outcomes_agree(tmp_path: Path) -> None: + body = PASSES + "\n\ndef test_b():\n pass\n" + first = _written_report(tmp_path / "first", body) + second = _written_report(tmp_path / "second", body) + result = _gate(first, "--same-as", str(second)) + assert result.returncode == 0, result.stderr + assert _printed(result) == {"tests": 2, "failures": 0, "errors": 0, "skipped": 0} + assert result.stderr == "" + + +@pytest.mark.parametrize( + ("second_body", "difference"), + [ + ("@pytest.mark.skip\ndef test_a():\n pass\n", "passed vs skipped"), + ("def test_a():\n assert False\n", "passed vs failed"), + ( + "@pytest.fixture\ndef broken():\n raise RuntimeError\n\n\n" + "def test_a(broken):\n pass\n", + "passed vs error", + ), + ("def test_c():\n pass\n", "passed vs absent"), + ], + ids=["skipped", "failed", "error", "absent"], +) +def test_a_test_whose_outcome_differs_between_runs_fails( + tmp_path: Path, second_body: str, difference: str +) -> None: + """The first run passes, so each failure printed is a difference between the runs.""" + first = _written_report(tmp_path / "first", PASSES) + second = _written_report(tmp_path / "second", second_body) + result = _gate(first, "--same-as", str(second)) + assert result.returncode == 1 + assert _printed(result) == {"tests": 1, "failures": 0, "errors": 0, "skipped": 0} + expected = [f"FAIL: differs: test_sample::test_a: {difference}"] + if difference == "passed vs absent": + expected.append("FAIL: differs: test_sample::test_c: absent vs passed") + assert result.stderr.splitlines() == expected + + +@pytest.mark.parametrize("shape", ["missing", "a test twice"]) +def test_an_unreadable_second_report_fails(tmp_path: Path, written: bytes, shape: str) -> None: + """A second report that is missing, or that holds a test twice (two suites merged from the + same run), cannot be compared.""" + first = tmp_path / "first.xml" + first.write_bytes(written) + second = tmp_path / "missing.xml" + problem = f"[Errno 2] No such file or directory: '{second}'" + if shape == "a test twice": + second = _report(tmp_path, written, (1, 0, 0, 0), (1, 0, 0, 0)) + problem = f"test_sample::test_a appears twice in {second}" + result = _gate(first, "--same-as", str(second)) + assert result.returncode == 1 + # The counters are printed only after both reports are read. + assert result.stdout == "" + assert result.stderr == f"cannot read JUnit report: {problem}\n" diff --git a/tests/test_suite_rules.py b/tests/test_suite_rules.py index 9a66b06..33b9162 100644 --- a/tests/test_suite_rules.py +++ b/tests/test_suite_rules.py @@ -9,15 +9,40 @@ value. Out of scope: the module reached without an import statement (``importlib.import_module``, ``__import__``, ``sys.modules``), and process APIs outside ``subprocess`` (``os.system``, ``os.popen``, ``asyncio``'s subprocesses, ``multiprocessing``). + +No test file reads the wall clock or draws unseeded random numbers (D40). The scan models these +modules imported under their own names: ``time`` is used only for ``tzset``, ``random`` only for +``Random`` called with a seed, and ``secrets`` not at all; ``datetime.now``, ``utcnow``, and +``today`` and ``date.today`` are not read, through the module or the classes ``from datetime +import ...`` binds; ``asyncio.sleep`` is called only with ``0``; and ``os.urandom``, +``os.getrandom``, ``uuid.uuid1``, and ``uuid.uuid4`` are not used. An alias, a star import, or one +of these modules or names passed as a value is refused rather than read. Out of scope: the modules +reached without an import statement, the event loop's ``time()``, file times, and the order of a +set of strings, which ``PYTHONHASHSEED`` changes from one process to the next (two runs compared by +``scripts/check_junit.py --same-as`` show that). The code a test runs is held to the same rules at +run time by ``tests/determinism.py``. + +Every test function is one pytest collects: none is defined inside another function, where it +would never run. """ from __future__ import annotations import ast import os +import socket from pathlib import Path +import pytest + +from github_activity_db.source import GitHubSource +from github_activity_db.source.clock import SystemClock +from tests import determinism +from tests.determinism import NotDeterministic +from tests.source.conftest import FakeClock + TESTS = Path(__file__).resolve().parent +SRC = TESTS.parent / "src" / "github_activity_db" WAITING = frozenset({"run", "call", "check_call", "check_output"}) # subprocess names that start no process. INERT = frozenset( @@ -85,10 +110,215 @@ def subprocess_problems(*paths: Path) -> list[str]: return found -def test_every_subprocess_wait_in_the_suite_has_a_timeout() -> None: - files = sorted(TESTS.rglob("*.py")) - # Counted with os.walk rather than the glob, so the scan cannot pass on fewer files. +# Modules a test file uses only through these names; the rest of each reads the wall clock, sleeps, +# or draws unseeded random numbers. +ONLY = { + "time": frozenset({"tzset"}), + "random": frozenset({"Random"}), + "secrets": frozenset(), +} +# Names in other modules that read the wall clock, sleep, or draw unseeded random numbers. +NEVER = { + "asyncio": frozenset({"sleep"}), + "os": frozenset({"urandom", "getrandom"}), + "uuid": frozenset({"uuid1", "uuid4"}), +} +# The datetime classes' methods that read the wall clock. +NOW = {"datetime": frozenset({"now", "utcnow", "today"}), "date": frozenset({"today"})} + + +def _sleeps_zero(call: ast.Call) -> bool: + return ( + len(call.args) == 1 + and not call.keywords + and isinstance(call.args[0], ast.Constant) + and call.args[0].value == 0 + ) + + +def determinism_problems(*paths: Path) -> list[str]: + found = [] + for path in paths: + tree = ast.parse(path.read_bytes()) + nodes = list(ast.walk(tree)) + where = path.relative_to(path.parents[1]) + modules: set[str] = set() + bound: dict[str, str] = {} # a name ``from random import Random`` binds, to its module + classes: set[str] = set() # the datetime classes ``from datetime import ...`` binds + for node in nodes: + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name in {*ONLY, *NEVER, "datetime"}: + if alias.asname not in (None, alias.name): + found.append( + f"{where}:{node.lineno}: {alias.name} imported as {alias.asname}" + ) + modules.add(alias.name) + elif isinstance(node, ast.ImportFrom) and node.module in {*ONLY, *NEVER, "datetime"}: + for alias in node.names: + name = f"{node.module}.{alias.name}" + if alias.asname not in (None, alias.name): + found.append(f"{where}:{node.lineno}: {name} imported as {alias.asname}") + elif node.module in ONLY and alias.name not in ONLY[node.module]: + found.append(f"{where}:{node.lineno}: imports {name}") + elif node.module in NEVER and alias.name in NEVER[node.module] | {"*"}: + found.append(f"{where}:{node.lineno}: imports {name}") + elif node.module == "datetime" and alias.name == "*": + found.append(f"{where}:{node.lineno}: imports {name}") + elif node.module in ONLY: + bound[alias.name] = node.module + elif node.module == "datetime" and alias.name in NOW: + classes.add(alias.name) + called = {id(node.func): node for node in nodes if isinstance(node, ast.Call)} + bases = {id(node.value) for node in nodes if isinstance(node, ast.Attribute)} + # A name in an annotation is a type, never called. + typed = {id(part) for annotation in _annotations(nodes) for part in ast.walk(annotation)} + for node in nodes: + if id(node) in typed: + continue + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): + module, attr = node.value.id, node.attr + if module in modules and module in ONLY: + if attr not in ONLY[module]: + found.append(f"{where}:{node.lineno}: {module}.{attr}") + elif module == "random" and not _seeded(called.get(id(node))): + found.append(f"{where}:{node.lineno}: random.Random without a seed") + elif module in modules and attr in NEVER.get(module, ()): + call = called.get(id(node)) + if not (module == "asyncio" and call is not None and _sleeps_zero(call)): + found.append(f"{where}:{node.lineno}: {module}.{attr}") + elif module in classes and attr in NOW[module]: + found.append(f"{where}:{node.lineno}: {module}.{attr}") + elif ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Attribute) + and isinstance(node.value.value, ast.Name) + and node.value.value.id == "datetime" + and "datetime" in modules + and node.value.attr in NOW + and node.attr in NOW[node.value.attr] + ): + found.append(f"{where}:{node.lineno}: datetime.{node.value.attr}.{node.attr}") + elif isinstance(node, ast.Name) and id(node) not in bases: + if node.id in modules and node.id in ONLY: + found.append(f"{where}:{node.lineno}: the {node.id} module used as a value") + elif bound.get(node.id) == "random" and not _seeded(called.get(id(node))): + found.append(f"{where}:{node.lineno}: Random without a seed") + return found + + +def _annotations(nodes: list[ast.AST]) -> list[ast.expr]: + found: list[ast.expr | None] = [] + for node in nodes: + if isinstance(node, ast.arg | ast.AnnAssign): + found.append(node.annotation) + elif isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + found.append(node.returns) + return [annotation for annotation in found if annotation is not None] + + +def _seeded(call: ast.Call | None) -> bool: + return call is not None and bool(call.args or call.keywords) + + +def uncollected_tests(*paths: Path) -> list[str]: + """Test functions and classes defined inside a function, where pytest never collects them.""" + found = [] + for path in paths: + where = path.relative_to(path.parents[1]) + for outer in ast.walk(ast.parse(path.read_bytes())): + if not isinstance(outer, ast.FunctionDef | ast.AsyncFunctionDef): + continue + for node in ast.walk(outer): + if node is not outer and ( + ( + isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) + and node.name.startswith("test") + ) + or (isinstance(node, ast.ClassDef) and node.name.startswith("Test")) + ): + found.append(f"{where}:{node.lineno}: {node.name} inside {outer.name}") + return found + + +def _python_files(root: Path) -> list[Path]: + files = sorted(root.rglob("*.py")) + # Counted with os.walk rather than the glob, so a scan cannot pass on fewer files. assert len(files) == sum( - Path(name).suffix == ".py" for _, _, names in os.walk(TESTS) for name in names + Path(name).suffix == ".py" for _, _, names in os.walk(root) for name in names ) - assert subprocess_problems(*files) == [] + return files + + +def _test_files() -> list[Path]: + return _python_files(TESTS) + + +def test_every_subprocess_wait_in_the_suite_has_a_timeout() -> None: + assert subprocess_problems(*_test_files()) == [] + + +def test_no_test_file_reads_the_wall_clock_or_draws_unseeded_random_numbers() -> None: + assert determinism_problems(*_test_files()) == [] + + +def test_the_package_reads_the_wall_clock_and_randomness_only_in_its_real_clocks() -> None: + """Over every module of the package, the scan finds the reads of the two real clocks, and no + other: everything else takes a clock it is given, which a test fixes.""" + assert determinism_problems(*_python_files(SRC)) == [ + "db/types.py:15: datetime.now", + "source/clock.py:32: secrets.SystemRandom", + "source/clock.py:35: datetime.now", + "source/clock.py:38: asyncio.sleep", + ] + + +def test_pytest_collects_every_test_function_in_the_suite() -> None: + assert uncollected_tests(*_test_files()) == [] + + +def _refused(refusal: pytest.ExceptionInfo[NotDeterministic]) -> tuple[str, str]: + """The refusal's fields, once it is the only one recorded; the record is then cleared, so the + conftest's check after the test passes.""" + assert determinism.REFUSED == [refusal.value] + determinism.REFUSED.clear() + return refusal.value.what, refusal.value.detail + + +def test_a_connection_to_a_host_is_refused() -> None: + with pytest.raises(NotDeterministic) as refusal, socket.socket() as connection: + connection.connect(("192.0.2.1", 443)) + assert _refused(refusal) == ("network", "connect ('192.0.2.1', 443)") + + +def test_a_host_name_lookup_is_refused() -> None: + with pytest.raises(NotDeterministic) as refusal: + socket.getaddrinfo("api.github.com", 443) + assert _refused(refusal) == ("network", "getaddrinfo 'api.github.com' 443") + + +async def test_the_source_without_a_recorded_transport_is_refused_the_network() -> None: + """The source as a command builds it, with githubkit's own transport: its first request is + refused at the host-name lookup.""" + async with GitHubSource("test-token", clock=FakeClock()) as source: + with pytest.raises(NotDeterministic) as refusal: + await source.resolve("prebid", "Prebid.js") + assert _refused(refusal) == ("network", "getaddrinfo b'api.github.com' 443") + + +def test_the_real_clock_is_refused() -> None: + with pytest.raises(NotDeterministic) as refusal: + SystemClock().now() + assert _refused(refusal) == ("wall clock", "SystemClock.now()") + + +async def test_the_real_clocks_sleep_is_refused() -> None: + with pytest.raises(NotDeterministic) as refusal: + await SystemClock().sleep(2.0) + assert _refused(refusal) == ("wall clock", "SystemClock.sleep(2.0)") + + +def test_the_real_clocks_unseeded_jitter_is_refused() -> None: + with pytest.raises(NotDeterministic) as refusal: + SystemClock().jitter(8.0) + assert _refused(refusal) == ("randomness", "SystemClock._random.uniform(0.0, 2.0)") From 3ae6831e46d4b09723fc43c3bb81fe1910318324 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 08:52:43 -0600 Subject: [PATCH 09/29] Step 3b-2a-2d: close the gaps the review found in the D40 guard - tests/determinism.py sets no_proxy to *, so the source built without a recorded transport is refused at GitHub's host whatever proxy the environment or the macOS settings name; it refuses every host-name lookup the socket module offers, and connect_ex reports its own name. The conftest installs it at configure time and checks the record after every test and at the end of the session, without clearing it first. - Tests for connect_ex, each lookup, getfqdn, the Unix-socket pass-through of connect and connect_ex, and the source under four proxy settings. - The source scan registers `import os.path` as os, counts Random(None) as unseeded, and reads annotations that hold a call or Annotated metadata; its docstring names what it reads and what is out of scope. - check_junit --same-as reads a test pytest reports in two elements (a failure, then a teardown error) as one outcome, error+failed, instead of refusing the report. - docs/testing.md no longer describes the replaced tests. --- docs/redesign-plan.md | 9 +-- docs/testing.md | 38 ++++++------ scripts/check_junit.py | 15 ++--- tests/conftest.py | 33 ++++++---- tests/determinism.py | 46 ++++++++++---- tests/test_check_junit.py | 33 +++++++--- tests/test_suite_rules.py | 123 +++++++++++++++++++++++++++++--------- 7 files changed, 208 insertions(+), 89 deletions(-) diff --git a/docs/redesign-plan.md b/docs/redesign-plan.md index c3a10e6..de03f56 100644 --- a/docs/redesign-plan.md +++ b/docs/redesign-plan.md @@ -1946,10 +1946,11 @@ Prerequisite: Step 3b-2a-2c has merged, and D39 is on `main`. Scope: tests, `docs/testing.md`, `scripts/check_junit.py`'s `--same-as`, and one change under `src/`: `ghactivity sync` times its report on the run's clock, not `time.monotonic` (D40). Nothing under `alembic/` changes, and no rule loses its test. Every test meets D40. -`tests/determinism.py` refuses, for every test in the pytest process, a network connection, a -host-name lookup, and this project's real clock and unseeded random source (`SystemClock`), and -records each refusal, so a test fails even where the code under test caught it; each refusal has a -test. `tests/test_suite_rules.py` gains the rules it can model in source: no test file reads the +`tests/determinism.py` refuses, in the pytest process from collection on, a network connection +other than a Unix socket, every host-name lookup the `socket` module offers, and this project's +real clock and unseeded random source (`SystemClock`), with `no_proxy` set so the machine's proxy +settings change nothing, and records each refusal, so a test fails even where the code under test +caught it; each refusal, and the Unix socket it lets through, has a test. `tests/test_suite_rules.py` gains the rules it can model in source: no test file reads the wall clock, sleeps on it, or draws unseeded random numbers; the package reads them only in its two real clocks (`SystemClock` and `db.types.utc_now`); and no test function is defined inside another function, where pytest never collects it. Each is tested on its passing side over every file under diff --git a/docs/testing.md b/docs/testing.md index 7e7f130..16e8d17 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -51,18 +51,20 @@ How the test suite is built and what it counts as evidence. The plan of record i The module reached without an import statement (`importlib.import_module`, `__import__`, `sys.modules`) and process APIs outside `subprocess` (`os.system`, `asyncio` subprocesses) are outside its scan. -- **What enforces D40.** In the pytest process, `tests/determinism.py` (installed for the session - by `tests/conftest.py`) refuses a network connection other than a Unix socket, every host-name - lookup, and `SystemClock`'s `now`, `sleep`, and unseeded random source, by raising - `NotDeterministic`; the refusal is also recorded, and a test that reached one fails after it - runs even where the code under test caught the exception. `tests/test_suite_rules.py` reads - source: no test file reads the wall clock, sleeps on it, or draws unseeded random numbers - (`random.Random` takes a seed, `asyncio.sleep` takes `0`); the package reads them only in its - two real clocks, `SystemClock` and `db.types.utc_now`; and no test function is defined inside - another function, where pytest never collects it. Processes a test starts run without the - run-time refusals, and the order of a set of strings changes with `PYTHONHASHSEED` from one - process to the next: two runs compared by `scripts/check_junit.py --same-as` show whether an - outcome depends on either. +- **What enforces D40.** In the pytest process, from collection on, `tests/determinism.py` + (installed by `tests/conftest.py`) refuses a network connection other than a Unix socket, every + host-name lookup the `socket` module offers, and `SystemClock`'s `now`, `sleep`, and unseeded + random source, by raising `NotDeterministic`. It sets `no_proxy` to `*`, so a client that reads + proxies from the environment or the system settings is refused at GitHub's own host on every + machine. Each refusal is also recorded and checked after every test and at the end of the + session, so a test fails even where the code under test caught the exception (a refusal in a + wider fixture's teardown fails the next check). `tests/test_suite_rules.py` reads source: no test + file reads the wall clock, sleeps on it, or draws unseeded random numbers (`random.Random` takes + a seed, `asyncio.sleep` takes `0`); the package reads them only in its two real clocks, + `SystemClock` and `db.types.utc_now`; and no test function is defined inside another function, + where pytest never collects it. Processes a test starts run without the run-time refusals, and + the order of a set of strings changes with `PYTHONHASHSEED` from one process to the next: two + runs compared by `scripts/check_junit.py --same-as` show whether an outcome depends on either. --- @@ -116,16 +118,17 @@ tests/ as the other side of the model comparison. - **The real environment.** Migration tests run the repository's `alembic.ini`, `env.py`, and revisions. `write_revision` adds a test-only revision after the head through - `version_locations`, which is how a planted failure, a foreign-key violation, and a batch alter - under the views are run through `env.py` (tests from before D39, which Step 3b-2a-2d replaces). `env.py` is loaded fresh by every command, so its + `version_locations`: one records the settings of the connection `env.py` gives a revision, and + one runs a batch alter under the views. Earlier schemas come from the real revisions + (`upgrade(path, "")`). `env.py` is loaded fresh by every command, so its `MigrationRefused` is matched by class name. - **Seeding.** `tests/db/seed.py` writes rows through the models' tables with every column a test does not care about filled with a valid default; GitHub ids come from one process-wide counter, so separate transactions never collide. - **Schema comparison.** `schema_of` reads columns (`PRAGMA table_xinfo`), indexes, foreign keys, and every item of each table's definition, which is where constraint names and CHECK text - live; each reading is tested against planted differences (tests from before D39, which Step 3b-2a-2d - replaces). + live; the readings are tested on a scratch database of valid SQL holding what no pragma + reports, at head against the models' declarations, and between real revisions. - **Dictionary and schema version.** `tests/test_data_dictionary.py` compares `docs/data-dictionary.md` with a migrated database in both directions. `tests/schema_versions.json` maps each schema version (`PRAGMA user_version`) to its consumer @@ -271,8 +274,7 @@ run must see GitHub as the test left it. template, and the `tracking.toml` entries a test adds; `World.sync()` runs the engine over them, first moving the clock past the unsettled window (`settle=False` keeps it). Each case checks the stored rows, the `sync_runs` row, the `repo_sync_state` rows the run wrote (`ended`), and the - exit code. The `world` fixture fails a test that leaves an `AccountMissing` failure, except the - one that fakes the writer's refusal. + exit code. The `world` fixture fails a test that leaves an `AccountMissing` failure. - **Seeded runs**: `tests/sync/test_seeded_runs.py` gives each of a few fixed seeds a small world and five runs, with random changes through the model's methods and random clock gaps between them. After each run `tests/sync/expected_state.py` computes the database the plan requires from diff --git a/scripts/check_junit.py b/scripts/check_junit.py index aaa7721..9a4d43e 100644 --- a/scripts/check_junit.py +++ b/scripts/check_junit.py @@ -3,7 +3,9 @@ pytest exits 0 when every collected test is skipped. This gate reads the report instead: at least one test executed, none failed or errored, and no more skips than allowed. With ``--same-as``, it also fails unless a second report holds the same tests, each with the -same outcome, so two runs of the suite can be shown to agree (redesign plan D40). +same outcome, so two runs of the suite can be shown to agree (redesign plan D40). A test pytest +reports in more than one (it writes a second one for a teardown error after a failure) +has the outcomes of all of them, sorted and joined: ``error+failed``. Usage: python scripts/check_junit.py REPORT.xml [--max-skipped N] [--same-as OTHER.xml] """ @@ -28,24 +30,23 @@ def read_counts(report: Path) -> dict[str, int]: def read_outcomes(report: Path) -> dict[str, str]: - """Each test's id (classname::name) and its outcome: passed, failed, error, or skipped.""" + """Each test's id (classname::name) and its outcome: passed, failed, error, or skipped, or + the outcomes of each that holds the test, joined: ``error+failed``.""" root = ET.parse(report).getroot() suites = [root] if root.tag == "testsuite" else root.findall("testsuite") if not suites: raise ValueError(f"no element in {report}") - outcomes: dict[str, str] = {} + outcomes: dict[str, set[str]] = {} for suite in suites: for case in suite.iter("testcase"): test = f"{case.get('classname', '')}::{case.get('name', '')}" - if test in outcomes: - raise ValueError(f"{test} appears twice in {report}") outcome = "passed" for kind in ("failure", "error", "skipped"): if case.find(kind) is not None: outcome = "failed" if kind == "failure" else kind break - outcomes[test] = outcome - return outcomes + outcomes.setdefault(test, set()).add(outcome) + return {test: "+".join(sorted(found)) for test, found in outcomes.items()} def differences(first: dict[str, str], second: dict[str, str]) -> list[str]: diff --git a/tests/conftest.py b/tests/conftest.py index 66ac3e7..531983f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,22 +33,33 @@ SETTINGS_VARIABLES = frozenset(name.upper() for name in Settings.model_fields) +_GUARD = pytest.MonkeyPatch() + + +def pytest_configure(config: pytest.Config) -> None: + """The network, the real clock, and its random source are refused from collection to the end + of the session (``tests/determinism.py``, redesign plan D40).""" + determinism.install(_GUARD) + + +def pytest_unconfigure(config: pytest.Config) -> None: + _GUARD.undo() + + @pytest.fixture(autouse=True, scope="session") -def no_network_or_wall_clock() -> Iterator[None]: - """Every test runs with the network, the real clock, and its random source refused - (``tests/determinism.py``, redesign plan D40).""" - with pytest.MonkeyPatch.context() as patch: - determinism.install(patch) - yield +def nothing_refused_after_the_last_test() -> Iterator[None]: + """The refusals recorded after the last test's check, in fixtures torn down after it.""" + yield + assert determinism.drain() == [] @pytest.fixture(autouse=True) -def nothing_refused(no_network_or_wall_clock: None) -> Iterator[None]: - """A test fails when it reached something ``tests/determinism.py`` refuses, even where the code - under test caught the refusal.""" - determinism.REFUSED.clear() +def nothing_refused(nothing_refused_after_the_last_test: None) -> Iterator[None]: + """A test fails when anything since the previous test's check (collection, a wider fixture's + setup or teardown, or the test itself) reached something ``tests/determinism.py`` refuses, even + where the code under test caught the refusal.""" yield - assert [(refused.what, refused.detail) for refused in determinism.REFUSED] == [] + assert determinism.drain() == [] @pytest.fixture(autouse=True) diff --git a/tests/determinism.py b/tests/determinism.py index a626846..e3c9e53 100644 --- a/tests/determinism.py +++ b/tests/determinism.py @@ -1,13 +1,17 @@ -"""What every test runs under (redesign plan D40), installed for the session by the conftest. - -Inside the test process, a connection to anything but a Unix socket and a host-name lookup are -refused, and so are this project's real clock (``SystemClock.now`` and ``sleep``) and its unseeded -random source (``SystemClock._random``, which an instance may replace with a seeded one). A refusal -raises ``NotDeterministic``, a ``BaseException`` so that no ``except Exception`` in the code under -test can turn it into a handled failure, and is recorded in ``REFUSED``, which the conftest checks -after every test. Out of scope: processes a test starts (they run without these patches), the +"""What every test runs under (redesign plan D40), installed by the conftest from collection on. + +Inside the test process, a connection to anything but a Unix socket and every host-name lookup the +``socket`` module offers are refused, and so are this project's real clock (``SystemClock.now`` and +``sleep``) and its unseeded random source (``SystemClock._random``, which an instance may replace +with a seeded one). ``no_proxy`` is ``*``, so an HTTP client that reads proxies from the +environment, or on macOS from the system settings, connects straight to the host it was given and +is refused there, whatever the machine's proxy settings. A refusal raises ``NotDeterministic``, a +``BaseException`` so that no ``except Exception`` in the code under test can turn it into a handled +failure, and is recorded in ``REFUSED``, which the conftest checks after every test and at the end +of the session. Out of scope: processes a test starts (they run without these patches), the standard library's clocks and random modules called directly (``tests/test_suite_rules.py`` reads -test files for those), and ``db.types.utc_now``, the default of columns no producer writes. +test files for those, and the package for its own reads), and ``db.types.utc_now``, the default of +columns no producer writes. """ from __future__ import annotations @@ -19,6 +23,9 @@ from github_activity_db.source.clock import SystemClock +# The host-name lookups the socket module offers; getfqdn goes through gethostbyaddr. +LOOKUPS = ("getaddrinfo", "gethostbyname", "gethostbyname_ex", "gethostbyaddr", "getnameinfo") + class NotDeterministic(BaseException): """A test reached the network, the wall clock, or unseeded randomness.""" @@ -38,6 +45,13 @@ def refuse(what: str, detail: str) -> NoReturn: raise refusal +def drain() -> list[tuple[str, str]]: + """Each refusal recorded since the last drain, as (what, detail), and an empty record.""" + refused = [(refusal.what, refusal.detail) for refusal in REFUSED] + REFUSED.clear() + return refused + + class _NoSystemRandom: def uniform(self, low: float, high: float) -> NoReturn: refuse("randomness", f"SystemClock._random.uniform({low}, {high})") @@ -54,11 +68,14 @@ def connecting(self: socket.socket, address: Any) -> None: def connecting_ex(self: socket.socket, address: Any) -> int: if self.family != socket.AF_UNIX: - refuse("network", f"connect {address!r}") + refuse("network", f"connect_ex {address!r}") return connect_ex(self, address) - def looking_up(host: Any, port: Any, *args: Any, **kwargs: Any) -> NoReturn: - refuse("network", f"getaddrinfo {host!r} {port!r}") + def looking_up(name: str) -> Any: + def lookup(looked_up: Any, *args: Any, **kwargs: Any) -> NoReturn: + refuse("network", f"{name} {looked_up!r}") + + return lookup def now(self: SystemClock) -> NoReturn: refuse("wall clock", "SystemClock.now()") @@ -66,9 +83,12 @@ def now(self: SystemClock) -> NoReturn: async def sleep(self: SystemClock, seconds: float) -> NoReturn: refuse("wall clock", f"SystemClock.sleep({seconds})") + patch.setenv("no_proxy", "*") + patch.setenv("NO_PROXY", "*") patch.setattr(socket.socket, "connect", connecting) patch.setattr(socket.socket, "connect_ex", connecting_ex) - patch.setattr(socket, "getaddrinfo", looking_up) + for name in LOOKUPS: + patch.setattr(socket, name, looking_up(name)) patch.setattr(SystemClock, "now", now) patch.setattr(SystemClock, "sleep", sleep) patch.setattr(SystemClock, "_random", _NoSystemRandom()) diff --git a/tests/test_check_junit.py b/tests/test_check_junit.py index 6477c75..3f44075 100644 --- a/tests/test_check_junit.py +++ b/tests/test_check_junit.py @@ -1,8 +1,9 @@ """The CI gate over pytest's JUnit report. Every report is one pytest wrote, in a run over a test file written for it, or is built in memory -from one with its counters changed into a shape a run can leave: failures, errors, skips, no test -executed, several suites merged, a file cut short, or no suite at all. ``--same-as`` compares two +from one with its counters changed (the gate's verdict reads only the counters; the test cases are +left as pytest wrote them): failures, errors, skips, no test executed, several suites merged, a +file cut short, or no suite at all. ``--same-as`` compares two reports pytest wrote, in runs over test files written for them. """ @@ -173,6 +174,16 @@ def _written_report(root: Path, test_body: str) -> Path: PASSES = "def test_a():\n pass\n" +def test_a_test_in_two_suites_has_one_outcome(tmp_path: Path, written: bytes) -> None: + """Two suites merged from the same run hold test_a twice, with one outcome.""" + first = tmp_path / "first.xml" + first.write_bytes(written) + second = _report(tmp_path, written, (1, 0, 0, 0), (1, 0, 0, 0)) + result = _gate(first, "--same-as", str(second)) + assert result.returncode == 0, result.stderr + assert result.stderr == "" + + def test_two_runs_with_the_same_outcomes_agree(tmp_path: Path) -> None: body = PASSES + "\n\ndef test_b():\n pass\n" first = _written_report(tmp_path / "first", body) @@ -193,9 +204,14 @@ def test_two_runs_with_the_same_outcomes_agree(tmp_path: Path) -> None: "def test_a(broken):\n pass\n", "passed vs error", ), + ( + "@pytest.fixture\ndef breaks_after():\n yield\n raise RuntimeError\n\n\n" + "def test_a(breaks_after):\n assert False\n", + "passed vs error+failed", + ), ("def test_c():\n pass\n", "passed vs absent"), ], - ids=["skipped", "failed", "error", "absent"], + ids=["skipped", "failed", "error", "failed then error", "absent"], ) def test_a_test_whose_outcome_differs_between_runs_fails( tmp_path: Path, second_body: str, difference: str @@ -212,17 +228,16 @@ def test_a_test_whose_outcome_differs_between_runs_fails( assert result.stderr.splitlines() == expected -@pytest.mark.parametrize("shape", ["missing", "a test twice"]) +@pytest.mark.parametrize("shape", ["missing", "no testsuite"]) def test_an_unreadable_second_report_fails(tmp_path: Path, written: bytes, shape: str) -> None: - """A second report that is missing, or that holds a test twice (two suites merged from the - same run), cannot be compared.""" + """A second report that is missing, or holds no suite, cannot be compared.""" first = tmp_path / "first.xml" first.write_bytes(written) second = tmp_path / "missing.xml" problem = f"[Errno 2] No such file or directory: '{second}'" - if shape == "a test twice": - second = _report(tmp_path, written, (1, 0, 0, 0), (1, 0, 0, 0)) - problem = f"test_sample::test_a appears twice in {second}" + if shape == "no testsuite": + second = _report(tmp_path, written) + problem = f"no element in {second}" result = _gate(first, "--same-as", str(second)) assert result.returncode == 1 # The counters are printed only after both reports are read. diff --git a/tests/test_suite_rules.py b/tests/test_suite_rules.py index 33b9162..ffb3089 100644 --- a/tests/test_suite_rules.py +++ b/tests/test_suite_rules.py @@ -15,12 +15,17 @@ ``Random`` called with a seed, and ``secrets`` not at all; ``datetime.now``, ``utcnow``, and ``today`` and ``date.today`` are not read, through the module or the classes ``from datetime import ...`` binds; ``asyncio.sleep`` is called only with ``0``; and ``os.urandom``, -``os.getrandom``, ``uuid.uuid1``, and ``uuid.uuid4`` are not used. An alias, a star import, or one -of these modules or names passed as a value is refused rather than read. Out of scope: the modules -reached without an import statement, the event loop's ``time()``, file times, and the order of a -set of strings, which ``PYTHONHASHSEED`` changes from one process to the next (two runs compared by -``scripts/check_junit.py --same-as`` show that). The code a test runs is held to the same rules at -run time by ``tests/determinism.py``. +``os.getrandom``, ``uuid.uuid1``, and ``uuid.uuid4`` are not used. ``Random(None)`` draws its seed +from the OS and counts as unseeded. An alias or a star import of these modules or names is refused +rather than read, and so are ``time``, ``random``, ``secrets``, and the names listed passed as +values; ``asyncio``, ``os``, ``uuid``, and ``datetime`` are read only through the names listed, so +one of them passed as a value (to ``getattr``, say) is out of scope. A name in an annotation is a +type and is not read, unless the annotation holds a call or ``Annotated`` metadata, which code reads +at run time. Out of scope as well: the modules reached without an import statement, the event loop's +``time()``, file times, and the order of a set of strings, which ``PYTHONHASHSEED`` changes from one +process to the next (two runs compared by ``scripts/check_junit.py --same-as`` show that). The same +scan over the package finds its reads only in its two real clocks; at run time +``tests/determinism.py`` refuses one of them, ``SystemClock``, and the network, in every test. Every test function is one pytest collects: none is defined inside another function, where it would never run. @@ -154,6 +159,9 @@ def determinism_problems(*paths: Path) -> list[str]: f"{where}:{node.lineno}: {alias.name} imported as {alias.asname}" ) modules.add(alias.name) + elif alias.asname is None and alias.name.split(".")[0] in {*ONLY, *NEVER}: + # ``import os.path`` binds ``os``. + modules.add(alias.name.split(".")[0]) elif isinstance(node, ast.ImportFrom) and node.module in {*ONLY, *NEVER, "datetime"}: for alias in node.names: name = f"{node.module}.{alias.name}" @@ -171,8 +179,7 @@ def determinism_problems(*paths: Path) -> list[str]: classes.add(alias.name) called = {id(node.func): node for node in nodes if isinstance(node, ast.Call)} bases = {id(node.value) for node in nodes if isinstance(node, ast.Attribute)} - # A name in an annotation is a type, never called. - typed = {id(part) for annotation in _annotations(nodes) for part in ast.walk(annotation)} + typed = _types(nodes) for node in nodes: if id(node) in typed: continue @@ -207,18 +214,37 @@ def determinism_problems(*paths: Path) -> list[str]: return found -def _annotations(nodes: list[ast.AST]) -> list[ast.expr]: - found: list[ast.expr | None] = [] +def _types(nodes: list[ast.AST]) -> set[int]: + """The ids of the nodes in annotations that only name types. An annotation holding a call or + ``Annotated`` metadata is left out: code reads those at run time.""" + annotations: list[ast.expr | None] = [] for node in nodes: if isinstance(node, ast.arg | ast.AnnAssign): - found.append(node.annotation) + annotations.append(node.annotation) elif isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): - found.append(node.returns) - return [annotation for annotation in found if annotation is not None] + annotations.append(node.returns) + typed: set[int] = set() + for annotation in annotations: + parts = list(ast.walk(annotation)) if annotation is not None else [] + if not any( + isinstance(part, ast.Call) + or (isinstance(part, ast.Name) and part.id == "Annotated") + or (isinstance(part, ast.Attribute) and part.attr == "Annotated") + for part in parts + ): + typed.update(id(part) for part in parts) + return typed def _seeded(call: ast.Call | None) -> bool: - return call is not None and bool(call.args or call.keywords) + """``Random`` called with a seed: its first argument, or ``x``, given and not ``None``.""" + if call is None: + return False + seed = call.args[0] if call.args else None + for keyword in call.keywords: + if keyword.arg in ("x", None): # None: ``**kwargs``, which the scan cannot read + seed = keyword.value if keyword.arg == "x" else None + return seed is not None and not (isinstance(seed, ast.Constant) and seed.value is None) def uncollected_tests(*paths: Path) -> list[str]: @@ -285,25 +311,68 @@ def _refused(refusal: pytest.ExceptionInfo[NotDeterministic]) -> tuple[str, str] return refusal.value.what, refusal.value.detail -def test_a_connection_to_a_host_is_refused() -> None: +@pytest.mark.parametrize("method", ["connect", "connect_ex"]) +def test_a_connection_to_a_host_is_refused(method: str) -> None: with pytest.raises(NotDeterministic) as refusal, socket.socket() as connection: - connection.connect(("192.0.2.1", 443)) - assert _refused(refusal) == ("network", "connect ('192.0.2.1', 443)") - - -def test_a_host_name_lookup_is_refused() -> None: + getattr(connection, method)(("192.0.2.1", 443)) + assert _refused(refusal) == ("network", f"{method} ('192.0.2.1', 443)") + + +@pytest.mark.parametrize("method", ["connect", "connect_ex"]) +def test_a_connection_to_a_unix_socket_goes_through( + method: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # A relative path, since macOS limits a Unix socket's path to 104 bytes. + monkeypatch.chdir(tmp_path) + with socket.socket(socket.AF_UNIX) as listening, socket.socket(socket.AF_UNIX) as connection: + listening.bind("s") + listening.listen(1) + assert getattr(connection, method)("s") in (None, 0) + assert connection.getpeername() == "s" + assert determinism.REFUSED == [] + + +@pytest.mark.parametrize( + ("lookup", "argument", "detail"), + [ + ("getaddrinfo", ("api.github.com", 443), "getaddrinfo 'api.github.com'"), + ("gethostbyname", ("api.github.com",), "gethostbyname 'api.github.com'"), + ("gethostbyname_ex", ("api.github.com",), "gethostbyname_ex 'api.github.com'"), + ("gethostbyaddr", ("192.0.2.1",), "gethostbyaddr '192.0.2.1'"), + ("getnameinfo", (("192.0.2.1", 443), 0), "getnameinfo ('192.0.2.1', 443)"), + ("getfqdn", ("api.github.com",), "gethostbyaddr 'api.github.com'"), + ], + ids=[*determinism.LOOKUPS, "getfqdn"], +) +def test_every_host_name_lookup_is_refused( + lookup: str, argument: tuple[object, ...], detail: str +) -> None: with pytest.raises(NotDeterministic) as refusal: - socket.getaddrinfo("api.github.com", 443) - assert _refused(refusal) == ("network", "getaddrinfo 'api.github.com' 443") - - -async def test_the_source_without_a_recorded_transport_is_refused_the_network() -> None: + getattr(socket, lookup)(*argument) + assert _refused(refusal) == ("network", detail) + + +@pytest.mark.parametrize( + "proxy", + [ + {}, + {"HTTPS_PROXY": "http://192.0.2.10:3128"}, + {"https_proxy": "http://192.0.2.10:3128"}, + {"ALL_PROXY": "http://192.0.2.10:3128"}, + ], + ids=["none", "HTTPS_PROXY", "https_proxy", "ALL_PROXY"], +) +async def test_the_source_without_a_recorded_transport_is_refused_the_network( + proxy: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: """The source as a command builds it, with githubkit's own transport: its first request is - refused at the host-name lookup.""" + refused at the lookup of GitHub's host, whatever proxy the environment names.""" + for name, value in proxy.items(): + monkeypatch.setenv(name, value) async with GitHubSource("test-token", clock=FakeClock()) as source: with pytest.raises(NotDeterministic) as refusal: await source.resolve("prebid", "Prebid.js") - assert _refused(refusal) == ("network", "getaddrinfo b'api.github.com' 443") + assert _refused(refusal) == ("network", "getaddrinfo b'api.github.com'") def test_the_real_clock_is_refused() -> None: From 132adf3946a94eb81a96680d0d073113226c910d Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 08:55:02 -0600 Subject: [PATCH 10/29] Step 3b-2a-2d: refusing sides for a client error outside the rate-limit readers and an assignee databaseId under two node ids test_transport.py: a ValueError raised inside the HTTP client by anything other than githubkit's rate-limit readers is a ClientFailure with no wait. The input is githubkit's own OAuth token refresh, run inside client.send, parsing a 200 from github.com/login/oauth/access_token whose body is the endpoint's form-encoded default format: response.json() raises JSONDecodeError. The test asserts the ClientFailure, its RequestError cause, the JSONDecodeError's msg, doc, and pos, no sleep, and that the token request was the only request sent. test_hydrate.py: databaseId 175728472 under node ids BOT_kgDOCnlnWA and MDM6Qm90MTc1NzI4NDcy among #15771's assignees, on one page and across an overflow page, fails the item with DuplicateEntry(15771, "assignedActors", 175728472); the node-id comparison passes both entries. --- tests/source/test_hydrate.py | 30 ++++++++++++++++++++++++++++++ tests/source/test_transport.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/tests/source/test_hydrate.py b/tests/source/test_hydrate.py index 8e06e15..f1cfbdc 100644 --- a/tests/source/test_hydrate.py +++ b/tests/source/test_hydrate.py @@ -893,6 +893,36 @@ async def test_each_assignee_github_id_is_distinct( assert len(set(github_ids)) == len(nodes) +COPILOT = {"__typename": "Bot", "databaseId": 175728472, "id": "BOT_kgDOCnlnWA", "login": "Copilot"} +# The same databaseId under its second node id, in the legacy format (gap 103). +COPILOT_SECOND_NODE = {**COPILOT, "id": "MDM6Qm90MTc1NzI4NDcy"} + + +@pytest.mark.parametrize("overflow", [False, True], ids=["first-page", "overflow"]) +async def test_an_assignee_github_id_under_two_node_ids( + open_source, replay: Replay, overflow: bool +) -> None: + # Synthetic: recorded #15771 with its second assignee replaced by Copilot and Copilot's + # second node id added after it, on the same page or a second one. pr_assignees keys an + # assignee by its account, which plan 5.2 resolves by databaseId, so both cannot be stored. + # The node ids differ, so the node-id comparison passes them. + unit = pr_unit(replay, 15771) + page = unit.data["assignedActors"] + page["nodes"][1] = dict(COPILOT) + if overflow: + second_page(replay, unit, "assignedActors", dict(COPILOT_SECOND_NODE)) + else: + page["nodes"].append(dict(COPILOT_SECOND_NODE)) + page["totalCount"] += 1 + replay.add(unit) + error = await failure(open_source, PREBID_JS, 15771) + assert type(error) is DuplicateEntry + assert (error.number, error.connection, error.key) == (15771, "assignedActors", 175728472) + # The item fails, then once more when retried alone. + attempt = ["PullRequestBatch", "Overflow"] if overflow else ["PullRequestBatch"] + assert [seen.operation for seen in replay.seen] == ["Resolve", *attempt, *attempt] + + async def test_a_duplicate_review_request(open_source, replay: Replay) -> None: # A request listed again in place of a missing one would pass the count check. unit = pr_unit(replay, 15771) diff --git a/tests/source/test_transport.py b/tests/source/test_transport.py index 3eb4d93..1d49085 100644 --- a/tests/source/test_transport.py +++ b/tests/source/test_transport.py @@ -4,6 +4,7 @@ import asyncio import inspect +import json import random import time from collections.abc import Callable, Iterator @@ -13,6 +14,7 @@ import httpx import pytest +from githubkit import OAuthTokenAuthStrategy from githubkit.core import GitHubCore from githubkit.exception import ( AuthCredentialError, @@ -685,6 +687,38 @@ async def test_a_client_failure_outside_the_network_is_fatal( assert clock.sleeps == [] assert len(replay.seen) == 1 + async def test_a_token_refresh_githubkit_cannot_parse_is_fatal( + self, replay: Replay, clock: FakeClock + ) -> None: + """A ValueError raised inside the client outside githubkit's rate-limit readers. + + githubkit's OAuth token auth refreshes a missing token inside the client and parses the + token endpoint's answer with response.json(). Here the endpoint answers 200 with a body in + its documented default format, form-encoded, rather than JSON. + """ + endpoint = "https://github.com/login/oauth/access_token" + body = "access_token=test-token&scope=repo&token_type=bearer" + replay.fault( + lambda request: httpx.Response( + 200, + content=body.encode(), + headers={"content-type": "application/x-www-form-urlencoded; charset=utf-8"}, + ), + when=lambda seen: seen.path == endpoint, + ) + auth = OAuthTokenAuthStrategy("test-client", "test-secret", refresh_token="test-refresh") + async with GitHubSource(auth, http_transport=replay.transport(), clock=clock) as source: + with pytest.raises(ClientFailure) as raised: + await source.resolve(**RESOLVE) + assert type(raised.value) is ClientFailure + cause = raised.value.__cause__ + assert type(cause) is RequestError + error = cause.exc + assert type(error) is json.JSONDecodeError + assert (error.msg, error.doc, error.pos) == ("Expecting value", body, 0) + assert clock.sleeps == [] + assert [seen.path for seen in replay.seen] == [endpoint] + class TestFatal: async def test_401_is_unauthorized(self, open_source, replay: Replay) -> None: From 63cd3adc3db61b083772eae391d010194756795e Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 08:57:10 -0600 Subject: [PATCH 11/29] alembic/env.py: an interrupt during an upgrade propagates; test the statement cache SQLAlchemy invalidates the migration connection when an interrupt ends a statement. env.py's handler then wrote Connection.info, which revalidates the connection and raised PendingRollbackError in place of the interrupt. The handler now writes the authorizer's mapping, held before the upgrade runs, so the original exception propagates and the failed state is still recorded. - test_an_interrupt_after_the_second_revision_applies_none: the KeyboardInterrupt raised at the set point is the exception the upgrade ends with, the pool invalidated the connection once for it, and the file keeps its revision and contents. - test_a_revision_that_refuses_leaves_env_py_only_its_rollback: after the real chain's refusal of a repositories row, the authorizer allows ROLLBACK and refuses BEGIN and COMMIT. - test_the_driver_keeps_no_prepared_statements: the driver receives cached_statements=0 whether or not the caller's URL carries ?cached_statements=128. --- alembic/env.py | 6 ++- tests/db/test_migrations.py | 101 ++++++++++++++++++++++++++++++------ 2 files changed, 90 insertions(+), 17 deletions(-) diff --git a/alembic/env.py b/alembic/env.py index bdcfadd..535c5a1 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -271,12 +271,16 @@ def run_migrations_online() -> None: transactional_ddl=True, render_as_batch=True, # autogenerate: SQLite alters tables through batch mode ) + # The mapping the authorizer reads, held before the upgrade runs: SQLAlchemy + # invalidates the connection when an interrupt, or an error it reads as a disconnect, + # ends a statement, and Connection.info then raises in place of that exception. + state = connection.info with context.begin_transaction(): try: context.run_migrations() check_foreign_keys(connection) except BaseException: - connection.info[TRANSACTION_STATE] = "failed" + state[TRANSACTION_STATE] = "failed" raise finally: engine.dispose() diff --git a/tests/db/test_migrations.py b/tests/db/test_migrations.py index 2d26b93..360e768 100644 --- a/tests/db/test_migrations.py +++ b/tests/db/test_migrations.py @@ -26,7 +26,8 @@ from alembic.script import ScriptDirectory from sqlalchemy import create_engine, event from sqlalchemy.engine import Engine -from sqlalchemy.exc import OperationalError, PendingRollbackError +from sqlalchemy.exc import OperationalError +from sqlalchemy.pool import Pool from github_activity_db.config import get_settings from github_activity_db.db.models import Base @@ -110,13 +111,15 @@ def foreign_key_rows(database: Path) -> dict[str, int]: @contextmanager -def listening(name: str, listener: Callable[..., None]) -> Iterator[None]: - """``listener`` on the ``name`` event of every engine, env.py's own included.""" - event.listen(Engine, name, listener) +def listening( + name: str, listener: Callable[..., None], target: type[Engine | Pool] = Engine +) -> Iterator[None]: + """``listener`` on the ``name`` event of every engine, or every pool, env.py's own included.""" + event.listen(target, name, listener) try: yield finally: - event.remove(Engine, name, listener) + event.remove(target, name, listener) @pytest.fixture(scope="module") @@ -493,27 +496,32 @@ def table_definitions(database: Path) -> dict[str, str]: class TestOneTransaction: def test_an_interrupt_after_the_second_revision_applies_none(self, first_schema: Path) -> None: - """An interrupt at a set point, once the second of the revisions after the first schema - has run: the file stays at the revision the upgrade started from.""" + """An interrupt at a set point inside a statement, once the second of the revisions after + the first schema has run: the interrupt itself ends the upgrade, and the file stays at the + revision the upgrade started from.""" second = AFTER_THE_FIRST_SCHEMA[1] + raised = KeyboardInterrupt() + invalidated: list[BaseException | None] = [] def interrupt(_connection: Any, _cursor: Any, statement: str, *_rest: Any) -> None: if statement.startswith(f"UPDATE alembic_version SET version_num='{second}'"): - raise KeyboardInterrupt + raise raised + + def record(_dbapi_connection: Any, _record: Any, exception: BaseException | None) -> None: + invalidated.append(exception) before = snapshot(first_schema) - # Both caught, so an interrupt that reached here would fail this test, not end the run. + # Caught here, so a test that fails another way still does not end the run. with ( listening("after_cursor_execute", interrupt), - pytest.raises((PendingRollbackError, KeyboardInterrupt)) as error, + listening("invalidate", record, Pool), + pytest.raises(KeyboardInterrupt) as error, ): upgrade(first_schema) - # SQLAlchemy invalidates the connection for the interrupt; env.py's handler then reads - # connection.info, which raises on it with the interrupt as its context. - assert [type(error.value), type(error.value.__context__)] == [ - PendingRollbackError, - KeyboardInterrupt, - ] + assert type(error.value) is KeyboardInterrupt + assert error.value is raised + # Inside a statement, so SQLAlchemy invalidated the connection for it, once. + assert invalidated == [raised] assert snapshot(first_schema) == before assert revision_of(first_schema) == FIRST_SCHEMA_HEAD @@ -661,6 +669,40 @@ def ask(_connection: Any, _cursor: Any, statement: str, *_rest: Any) -> None: "ROLLBACK": sqlite3.SQLITE_OK, } + def test_a_revision_that_refuses_leaves_env_py_only_its_rollback( + self, step3a_database: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The real chain's refusal of a repositories row, raised between statements on a + connection still usable: that error ends the upgrade, and the authorizer allows + env.py's ROLLBACK and no other transaction statement.""" + connections: list[RecordingConnection] = [] + real_connect = sqlite3.dbapi2.connect + + def connect(*args: Any, **kwargs: Any) -> sqlite3.Connection: + connection = real_connect(*args, factory=RecordingConnection, **kwargs) + connections.append(connection) + return connection + + with closing(sqlite3.connect(step3a_database)) as connection, connection: + connection.execute(STEP3A_ROWS["repositories"]) + before = snapshot(step3a_database) + monkeypatch.setattr(sqlite3.dbapi2, "connect", connect) + with pytest.raises(RuntimeError, match=r"rows would be lost \(repositories 1\)") as error: + upgrade(step3a_database) + monkeypatch.undo() + assert type(error.value) is RuntimeError + (recorded,) = connections + assert { + name: recorded.authorizer(*REFUSED_WHILE_OPEN[name]) + for name in ("BEGIN", "COMMIT", "ROLLBACK") + } == { + "BEGIN": sqlite3.SQLITE_DENY, + "COMMIT": sqlite3.SQLITE_DENY, + "ROLLBACK": sqlite3.SQLITE_OK, + } + assert snapshot(step3a_database) == before + assert revision_of(step3a_database) == STEP3A + def test_a_failed_commit_rolls_back_cleanly(self, step3b1_database: Path) -> None: """A reader's lock makes the COMMIT fail once the check has run; the rollback that follows is env.py's own, so the error is the commit's, not the authorizer's.""" @@ -716,6 +758,33 @@ def record(_connection: Any, _cursor: Any, statement: str, *_rest: Any) -> None: assert snapshot(step3b1_database) == before assert revision_of(step3b1_database) == STEP3B1 + @pytest.mark.parametrize( + "url_query", + [ + pytest.param("", id="no statement cache asked for"), + pytest.param("?cached_statements=128", id="statement cache asked for by the URL"), + ], + ) + def test_the_driver_keeps_no_prepared_statements( + self, first_schema: Path, monkeypatch: pytest.MonkeyPatch, url_query: str + ) -> None: + """SQLite authorizes a statement when it prepares it, so the driver is asked for no + statement cache, whatever the caller's URL asks for, and the real chain runs on it.""" + received: list[object] = [] + real_connect = sqlite3.dbapi2.connect + + def connect(*args: Any, **kwargs: Any) -> sqlite3.Connection: + received.append(kwargs.get("cached_statements")) + return real_connect(*args, **kwargs) + + config = alembic_config(first_schema) + config.attributes["database_url"] += url_query + monkeypatch.setattr(sqlite3.dbapi2, "connect", connect) + command.upgrade(config, "head") + monkeypatch.undo() + assert received == [0] + assert revision_of(first_schema) == HEAD_REVISION + def _batch_alter_revision(views_sql: tuple[tuple[str, str], ...]) -> str: """A revision that batch-recreates pull_requests, dropping and recreating the views reading From 5f5b0e39691e61dbbb6d89cfd9e4d7a6e44b1100 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 09:09:40 -0600 Subject: [PATCH 12/29] Step 3b-2a-2d: ground the token-refresh input; assert no invalidation - The token-refresh test's non-JSON answer is something between the client and GitHub (a captive portal or a proxy's block page) answering 200 with HTML; githubkit asks GitHub for JSON, so GitHub's own form-encoded format is not what that request gets. - The refusing-revision test asserts SQLAlchemy never invalidated the connection, the premise its docstring states. - The plan's 3b-2a-2d scope names the env.py change, and the prepared-statement evidence row says what the test holds. --- docs/redesign-plan.md | 9 ++++++--- tests/db/test_migrations.py | 11 ++++++++++- tests/source/test_transport.py | 10 ++++------ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/redesign-plan.md b/docs/redesign-plan.md index de03f56..6f2d205 100644 --- a/docs/redesign-plan.md +++ b/docs/redesign-plan.md @@ -1944,8 +1944,11 @@ check without a test that targets it; CI passes on the exact PR head. Prerequisite: Step 3b-2a-2c has merged, and D39 is on `main`. Scope: tests, `docs/testing.md`, `scripts/check_junit.py`'s `--same-as`, and one change under -`src/`: `ghactivity sync` times its report on the run's clock, not `time.monotonic` (D40). Nothing -under `alembic/` changes, and no rule loses its test. Every test meets D40. +`src/`: `ghactivity sync` times its report on the run's clock, not `time.monotonic` (D40); and one +under `alembic/`: `env.py` records a failed upgrade's state without asking SQLAlchemy for the +connection again, so an interrupt inside a statement, which SQLAlchemy answers by invalidating the +connection, propagates instead of a `PendingRollbackError`. No rule loses its test. Every test meets +D40. `tests/determinism.py` refuses, in the pytest process from collection on, a network connection other than a Unix socket, every host-name lookup the `socket` module offers, and this project's real clock and unseeded random source (`SystemClock`), with `no_proxy` set so the machine's proxy @@ -2598,7 +2601,7 @@ Measured 2026-10-16 unless noted. "Index" means the created-order enumeration in | URI filenames | Python 3.13's `sqlite3` with SQLite 3.45.3 (no `SQLITE_USE_URI` compile option) opens `file:target.db`, without `uri=True`, as a new file of that name, not as `target.db`. Other builds, CI's included, are not checked; `alembic/env.py` refuses `uri=true` URLs and checks the file SQLite opened, so either reading is covered | probe | | Path from the URL | SQLAlchemy 2.0.45's pysqlite dialect opens `os.path.abspath(database)`, which drops `..` as text, while `Path.resolve()` follows symlinks first: `/tmp/..` before the tracked path (on macOS `/tmp` is a symlink) and `/../github_activity.db` name the tracked file to the driver and a missing file to `resolve()`. An env.py that checked the resolved URL path migrated a stand-in of the tracked file through the second spelling; `PRAGMA database_list` on the opened connection names the file SQLite opened | review probe at `f90eee9`, on PR #11, 2026-10-17; `tests/db/test_migrations.py` | | Commit inside a revision | Under `transactional_ddl=True`, a revision's `op.execute("COMMIT")`, `op.execute("END")`, or `autocommit_block()` committed its writes before a later failure, leaving the file at head with a foreign-key violation. An SQLite authorizer that denies `SQLITE_TRANSACTION` statements other than the upgrade's own refuses each, and `connection.commit()` too | review probe at `f90eee9`, on PR #11; `tests/db/test_migrations.py` | -| Prepared statements | Python's `sqlite3` reuses a prepared statement by its SQL text (128 by default), and SQLite runs the authorizer only when it prepares one. SQLite's own rollback (`INSERT OR ROLLBACK`, `RAISE(ROLLBACK)`) expires prepared statements only when the transaction changed the schema, so in a data-only upgrade a statement the revision repeated after a swallowed rollback ran in autocommit: its row was committed with a foreign-key violation, and a repeated `SAVEPOINT` reopened a transaction env.py then committed, the earlier writes lost. With `cached_statements=0` every execute is prepared and refused | review probes at `4af764b`, on PR #11, 2026-10-17, SQLite 3.45.3 and 3.47.1; `tests/db/test_migrations.py` | +| Prepared statements | Python's `sqlite3` reuses a prepared statement by its SQL text (128 by default), and SQLite runs the authorizer only when it prepares one. SQLite's own rollback (`INSERT OR ROLLBACK`, `RAISE(ROLLBACK)`) expires prepared statements only when the transaction changed the schema, so in a data-only upgrade a statement the revision repeated after a swallowed rollback ran in autocommit: its row was committed with a foreign-key violation, and a repeated `SAVEPOINT` reopened a transaction env.py then committed, the earlier writes lost. With `cached_statements=0` every execute is prepared and refused | review probes at `4af764b`, on PR #11, 2026-10-17, SQLite 3.45.3 and 3.47.1; `tests/db/test_migrations.py` holds that the migration connection's driver receives `cached_statements=0`, also when the URL asks for a cache (Step 3b-2a-2d) | | Pragmas in a revision | Inside the upgrade's `BEGIN IMMEDIATE` and before its first write, `PRAGMA journal_mode=OFF` takes effect: pages a large revision then spilled reached the file unjournaled, and a failure after them left it unreadable ("malformed database schema"). `legacy_alter_table=ON` let a batch alter under the views pass, and `ignore_check_constraints=ON` committed a row a named CHECK refuses | review probes at `4af764b`, on PR #11, 2026-10-17, SQLite 3.45.3 and 3.47.1; `tests/db/test_migrations.py` | | Revision module code | alembic loads each revision module inside env.py's transaction, so `op.execute` at a module's top level runs in `current`, `check`, and `stamp`, which run no revision. Such code's `DELETE` was refused only when the check ran on rows changed (`total_changes`); its `DROP TABLE github_users` changes no row, left one `pull_requests.author_id` orphaned, and committed. The check now runs before every commit | review probes at `459393f`, on PR #11, 2026-10-17, SQLite 3.45.3 and 3.47.1; `tests/db/test_migrations.py` | | View lookups | With only single-column indexes on `pr_reviews` and `pr_comments`, `EXPLAIN QUERY PLAN` for `v_pr_reviewers` and `v_pr_commenters` searched the verdict, comment-count, and reviewer lookups by the account alone (`reviewer_id=?`, `author_id=?`); with the PR index created first, the planner took the account index, created later. With one account reviewing and commenting on every PR, four times the PRs cost about 15 times the SQLite virtual-machine instructions for each view, against about 4 times with `(pr_id, reviewer_id, github_id)` and `(pr_id, author_id)`, with the same rows | review probe at `459393f`, on PR #11, and local probes, 2026-10-17, SQLite 3.45.3 and 3.47.1; `tests/db/test_views.py` regenerates the ratio | diff --git a/tests/db/test_migrations.py b/tests/db/test_migrations.py index 360e768..f03f3b5 100644 --- a/tests/db/test_migrations.py +++ b/tests/db/test_migrations.py @@ -676,6 +676,7 @@ def test_a_revision_that_refuses_leaves_env_py_only_its_rollback( connection still usable: that error ends the upgrade, and the authorizer allows env.py's ROLLBACK and no other transaction statement.""" connections: list[RecordingConnection] = [] + invalidated: list[BaseException | None] = [] real_connect = sqlite3.dbapi2.connect def connect(*args: Any, **kwargs: Any) -> sqlite3.Connection: @@ -683,14 +684,22 @@ def connect(*args: Any, **kwargs: Any) -> sqlite3.Connection: connections.append(connection) return connection + def record(_dbapi_connection: Any, _record: Any, exception: BaseException | None) -> None: + invalidated.append(exception) + with closing(sqlite3.connect(step3a_database)) as connection, connection: connection.execute(STEP3A_ROWS["repositories"]) before = snapshot(step3a_database) monkeypatch.setattr(sqlite3.dbapi2, "connect", connect) - with pytest.raises(RuntimeError, match=r"rows would be lost \(repositories 1\)") as error: + with ( + listening("invalidate", record, Pool), + pytest.raises(RuntimeError, match=r"rows would be lost \(repositories 1\)") as error, + ): upgrade(step3a_database) monkeypatch.undo() assert type(error.value) is RuntimeError + # Between statements, so SQLAlchemy never invalidated the connection. + assert invalidated == [] (recorded,) = connections assert { name: recorded.authorizer(*REFUSED_WHILE_OPEN[name]) diff --git a/tests/source/test_transport.py b/tests/source/test_transport.py index 1d49085..c939d1e 100644 --- a/tests/source/test_transport.py +++ b/tests/source/test_transport.py @@ -693,16 +693,14 @@ async def test_a_token_refresh_githubkit_cannot_parse_is_fatal( """A ValueError raised inside the client outside githubkit's rate-limit readers. githubkit's OAuth token auth refreshes a missing token inside the client and parses the - token endpoint's answer with response.json(). Here the endpoint answers 200 with a body in - its documented default format, form-encoded, rather than JSON. + token endpoint's answer with response.json(). Here something between the client and + GitHub, a captive portal or a proxy's block page, answers the refresh 200 with HTML. """ endpoint = "https://github.com/login/oauth/access_token" - body = "access_token=test-token&scope=repo&token_type=bearer" + body = "Sign in to this network to continue." replay.fault( lambda request: httpx.Response( - 200, - content=body.encode(), - headers={"content-type": "application/x-www-form-urlencoded; charset=utf-8"}, + 200, content=body.encode(), headers={"content-type": "text/html; charset=utf-8"} ), when=lambda seen: seen.path == endpoint, ) From 93ca39a6d756598ed0425e0d1392dbc8861d2d66 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 09:14:17 -0600 Subject: [PATCH 13/29] Step 3b-2a-2d: record the step in the plan's status table (PR #30) --- docs/redesign-plan.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/redesign-plan.md b/docs/redesign-plan.md index 6f2d205..9d4341d 100644 --- a/docs/redesign-plan.md +++ b/docs/redesign-plan.md @@ -35,7 +35,7 @@ audited for silent failures and silently wrong data under D37, and D38 records t |------|-------|--------| | 1 | Guardrails | done (PR #5) | | 2 | GitHub source layer | done (PR #7) | -| 3 | New schema, PR sync engine, cutover (pull requests 3a, 3b-1, 3b-2a-1, 3b-2a-2a, 3b-2a-2b-1, 3b-2a-2b-2, 3b-2a-2c, 3b-2a-2d, 3b-2a-3, 3b-2b-1, and 3b-2b-2) | in progress: 3a done (PR #11), 3b-1 done (PR #14), 3b-2a-1 done (PR #19), 3b-2a-2a done (PR #21), 3b-2a-2b-1 done (PR #24), 3b-2a-2b-2 done (PR #25), 3b-2a-2c done (PR #26) | +| 3 | New schema, PR sync engine, cutover (pull requests 3a, 3b-1, 3b-2a-1, 3b-2a-2a, 3b-2a-2b-1, 3b-2a-2b-2, 3b-2a-2c, 3b-2a-2d, 3b-2a-3, 3b-2b-1, and 3b-2b-2) | in progress: 3a done (PR #11), 3b-1 done (PR #14), 3b-2a-1 done (PR #19), 3b-2a-2a done (PR #21), 3b-2a-2b-1 done (PR #24), 3b-2a-2b-2 done (PR #25), 3b-2a-2c done (PR #26), 3b-2a-2d done (PR #30) | | 4 | Issues, closing-issue links, and default-branch activity (pull requests 4a and 4b) | not started | | 5 | Full bootstrap and verification (pull request 5, and 5b when Step 5 does not show that every measured edit kind changes `updatedAt`, or finds an index contradiction that recurs) | not started | | 6 | Scheduled sync and publication | not started | From a822d5c59df4a70af7a1969bb2500867e6d10aae Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 09:27:53 -0600 Subject: [PATCH 14/29] D39 clarified: negative tests with inputs the test builds are expected D39 bans mutation testing (changing the code under test or a check to see whether a test notices), false content planted in the project's real files or documents, and scoring a reviewer on seeded flaws. Its first wording also banned ordinary negative tests; the clarified wording expects them: a test builds its own invalid or failing input and feeds it to the unchanged code, and every check is tested on both sides. - The rule in the plan's D39 row, its Step 3 restatement, CLAUDE.md, docs/testing.md, and CONTRIBUTING.md; D40's tail in each copy. - Step 3b-2a-2d keeps the negative tests on main, beside the tests it adds over real revisions and history files. - The REST sampler's one mismatch per field, verify's stored-row checks on a copy with a row deleted, and snapshot's refusals on a changed copy return to their steps. --- CLAUDE.md | 32 ++++++------- CONTRIBUTING.md | 34 ++++++-------- docs/redesign-plan.md | 101 +++++++++++++----------------------------- docs/testing.md | 33 ++++++-------- 4 files changed, 70 insertions(+), 130 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 87f6f19..c84d0d0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,21 +42,17 @@ uv run pytest # CI also runs Python 3.12, `alembic check`, ## Test Evidence -Nothing is planted or tampered, no flaw is seeded, and there is no mutation testing, under any name -(redesign plan D39). No test, review, or acceptance changes a file or a copy of one (other than a stub or configuration the inputs below allow), or writes a file, a row, a revision, a call site, or a decoy holding a -flaw, to see whether something catches it, and none measures how many such flaws are found. A test's -or an acceptance's inputs are GitHub's recorded responses; stubs of a response GitHub or the -environment can send (a status, a body, a timeout, an interrupt, a failing `git`, a clock), which a -test may build in memory from a recorded unit changed into that shape; calls the code under test -must refuse, and configuration it must refuse or that names a state GitHub can produce (a repository -that does not resolve, one no longer listed); a timing setting a test shortens (SQLite's busy timeout), which changes how long a wait lasts and no outcome; valid rows a test writes (through the models with -`tests/db/seed.py`, or in SQL); and databases and files this project's own code wrote, at any -revision. Each test asserts the exact result. A check over stored data, files, or source that only a -defect of this code could fail is tested on its passing side, with the exact count of the rows, -files, or values it compared, so it cannot pass on nothing; a check that GitHub, the environment, or -a caller can fail is tested on both sides. A fix comes with a regression test that fails against the -parent commit's code. Tests merged before D39 that change a file or a copy of one, or write a file, -a revision, a call, a connection, or a decoy holding a flaw, to see whether a check catches it were replaced in Step 3b-2a-2d. See [Testing](docs/testing.md). +No mutation testing, under any name: the code under test and the checks are never changed, on disk +or in memory, to see whether a test notices. No false content is planted in the project's real files +or documents (the plan, a document, review material, a tracked lock or database) to see whether a +check or a reviewer catches it, and no reviewer or agent is scored on how many seeded flaws it +finds. Negative tests are expected: a test builds its own invalid or failing input on purpose (a +sample test that fails, an invalid configuration, a malformed or edited copy of a file or database +in its temporary directory, a test-only revision that raises, a stub or fake that raises, a recorded +GitHub response changed in memory) and feeds it to the unchanged code. Each test asserts the exact +result. Every check is tested on both sides, and its passing side states the exact count of the +rows, files, or values it compared, so it cannot pass on nothing. A fix comes with a regression test +that fails against the parent commit's code (redesign plan D39). See [Testing](docs/testing.md). ## Deterministic Tests @@ -68,9 +64,7 @@ outside its own temporary directory, the repository, and the session's database every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on printed wording, and no test -checks the wording of a document. Where a test needs an input the code cannot easily be given, the -project builds software that produces it (a fake clock, the scenario model, the replay, a recorder -mode), never a planted file (D39). Evidence is what software reproduces: an acceptance criterion +checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test changes the project's real files. Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. `tests/determinism.py` refuses the network, the real clock, and its unseeded random source in every test, `tests/test_suite_rules.py` enforces the @@ -98,7 +92,7 @@ test by test (Step 3b-2a-2d). planner, and the writer, and `--dry-run` - `ghactivity sync` (Step 3b-2a-2c, `cli/sync.py`): the engine's run with the options of plan 5.7; a run that never starts exits 1, and a run that starts exits with the engine's 0, 2, or 1 -- Tests without planted inputs, deterministic (Step 3b-2a-2d, D39, D40): `tests/determinism.py` +- Deterministic tests, with D39's negative tests (Step 3b-2a-2d, D39, D40): `tests/determinism.py` refuses the network, the real clock, and its unseeded random source in every test; `tests/test_suite_rules.py`'s source rules; `scripts/check_junit.py --same-as` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bc27522..852a6c8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -246,24 +246,18 @@ def test_reviewers(migrated_database: Path) -> None: # a copy of the alembic-bu pr = seed.pull_request(seed.repository("prebid/Prebid.js"), 13481, seed.user("author")) ``` -**3. Never plant, tamper, or seed a flaw** (redesign plan D39). Nothing is planted or tampered, no -flaw is seeded, and there is no mutation testing, under any name. No test, review, or acceptance -changes a file or a copy of one (other than a stub or configuration the inputs below allow), or writes a file, a row, -a revision, a call site, or a decoy holding a flaw, to see whether something catches it, and none -measures how many such flaws are found. A test's or an acceptance's inputs are GitHub's recorded -responses; stubs of a response GitHub or the environment can send (a status, a body, a timeout, an -interrupt, a failing `git`, a clock), which a test may build in memory from a recorded unit changed -into that shape; calls the code under test must refuse, and configuration it must refuse or that -names a state GitHub can produce (a repository that does not resolve, one no longer listed); a timing setting a test shortens (SQLite's busy timeout), which changes how long a wait lasts and no outcome; valid -rows a test writes (through the models with `tests/db/seed.py`, or in SQL); and databases and files -this project's own code wrote, at any revision. Each test asserts the exact result. A check over -stored data, files, or source that only a defect of this code could fail is tested on its passing -side, with the exact count of the rows, files, or values it compared, so it cannot pass on nothing; -a check that GitHub, the environment, or a caller can fail is tested on both sides. A fix comes with -a regression test that fails against the parent commit's code. Tests merged before D39 that change a -file or a copy of one, or write a file, a revision, a call, a connection, or a decoy holding a flaw, -to see whether a check catches it were replaced in Step 3b-2a-2d. See the [Testing -Guide](docs/testing.md#principles). +**3. Test both sides with inputs the test builds; never mutate the code** (redesign plan D39). No +mutation testing, under any name: the code under test and the checks are never changed, on disk or +in memory, to see whether a test notices. No false content is planted in the project's real files or +documents (the plan, a document, review material, a tracked lock or database) to see whether a check +or a reviewer catches it, and no reviewer or agent is scored on how many seeded flaws it finds. +Negative tests are expected: a test builds its own invalid or failing input on purpose (a sample +test that fails, an invalid configuration, a malformed or edited copy of a file or database in its +temporary directory, a test-only revision that raises, a stub or fake that raises, a recorded GitHub +response changed in memory) and feeds it to the unchanged code. Each test asserts the exact result. +Every check is tested on both sides, and its passing side states the exact count of the rows, files, +or values it compared, so it cannot pass on nothing. A fix comes with a regression test that fails +against the parent commit's code. See the [Testing Guide](docs/testing.md#principles). **4. Write tests that give the same result every time** (redesign plan D40). The same code gives every test the same outcome on any machine, on any date, and in any run order: a test, and every @@ -273,9 +267,7 @@ outside its own temporary directory, the repository, and the session's database every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on printed wording, and no test -checks the wording of a document. Where a test needs an input the code cannot easily be given, the -project builds software that produces it (a fake clock, the scenario model, the replay, a recorder -mode), never a planted file (D39). Evidence is what software reproduces: an acceptance criterion +checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test changes the project's real files. Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. diff --git a/docs/redesign-plan.md b/docs/redesign-plan.md index 9d4341d..77ab417 100644 --- a/docs/redesign-plan.md +++ b/docs/redesign-plan.md @@ -93,8 +93,8 @@ Within scope the database collects and distinguishes data and never interprets i | D36 | Verify decisions of 2026-10-21: Step 3b-2b is two pull requests, the REST sampler and then verify with the live acceptance (section 6). Verify writes nothing to the database and claims no run: before any request it reads `sync_runs` and exits 1 while a row holds the lock by the claim's rule, so a stale or crashed run stays for the next claim or `sync --break-lock` (5.3). `--repo` limits verify to the repositories it names, as it limits a sync (D27). A verify that never starts writes no report; once it starts, `--report` is written on every exit verify's own code handles, and one that cannot be written turns an exit 0 or 2 into 1, because Step 6a reads it, while an interrupt exits 130 either way; verify's printed output follows sync's rule (5.3 step 8). Every repository failure 5.3 gives a resolve, an index, or a hydration batch, a repository error during a timeline lookup, and from Step 4b a log or history read verify cannot complete make a repository unverified, and a verify in which every repository it attempts is unverified, the deadline having stopped none, is fatal. verify reads each repository's stored rows in two statements before indexing it, and from Step 4b its stored events and default-branch commits in one statement each before the first request of its log and history reads, and its report records its own points, REST calls, and time (5.4, gap 94). | | D37 | Decision of 2026-10-21 (the maintainer): an error is better than a silent failure or silently wrong data. Where the code cannot check something, or cannot accept what GitHub returned, it reports a failure that stops publication or turns the run red, never a pass. For verify: an unchecked item and an unverified repository fail verify in both modes, not only `--exact`, and a lookup whose answer verify cannot accept is an error (5.4, 5.8, gap 97), so D25 does not publish a database verify could not check. | | D38 | Decisions of 2026-10-21 applying D37 to the sync and to the steps not yet built (gaps 98-110). The sync exits 2 when a 24-hour account refresh fails, when the run reaches its deadline, and when a hydration contradicts the run's own index after one retry (Step 3b-2a-3); the failures a batch made final before a deadline are recorded, not left pending. An `accepted_failures` entry accepts one error class and one cause, the first line of the failure's message, of an item, an account's refresh, or from Step 4b a compare, so a permanent failure can be accepted without turning every run red while a new cause is not. A PR with more closing links than the index carries is hydrated every run. Every node id a sighting, a lookup, or a commit link reports for a stored account is stored, and `sync_runs` records the `--repo` filter, each configured entry's `github_id`, and its first commit-link lookups and those GitHub answered `NOT_FOUND`. The source refuses what it cannot read (gap 104), and the run records refuse what they cannot write and mark `-unknown` what they cannot determine (gap 105). Step 4b marks a compare file list at GitHub's 300-file cap and counts it in verify's report and the manifest, completes an activity read only at the floor or at the log's first event, as a read in ascending order finds it, fails a history read whose count still differs after a rescan, and records a compare that still fails, which an entry can accept. Step 5 measures which edits leave `updatedAt` unchanged. Step 6 compares the App's reads with the token's before any App run publishes to `db`, pins the database file the job checked, tests that every exit-code `if:` compares a non-empty string, and ties a snapshot to the job's own sync. A schedule that stops is the one silent case this repository cannot see: no run starts, so nothing turns red; 5.6 states the staleness rule a consumer applies, and building it belongs to the dashboard's integration (section 9). Left as they are, because the stored data already says what happened: a commit-linked account a lookup answers `NOT_FOUND` (D28; now counted per run and in the manifest, and from Step 6a a run whose `links_not_found` exceeds the share of its `link_lookups` Step 5 sets, with at least Step 5's minimum number of lookups, exits 2 (an account an earlier lookup answered `NOT_FOUND` counted in neither), since a lookup form GitHub stopped resolving would look the same per link), a compare at GitHub's 300-file cap and one GitHub refuses (marked by `files_at_cap` and `http_status`, counted in verify's report and the manifest), a REST git author without a name or email (stored null, as GitHub reports none), deletions recorded as a count (`items_deleted`, and from Step 4b `events_deleted` and `history_commits_deleted`; the snapshots retained for 7 days hold the identities; older ones are not kept), which accounts failed a refresh (printed; the count is stored, the run is red unless an account entry accepts the failure, and the accounts are retried), an `author_association` difference (verify reports it as drift with its own count: GitHub computes association when read and changes it without moving `updatedAt`, section 7, so no read tells a change from a defect), and the merge state as of `hydrated_at` (D32). New tables and columns of GitHub-derived data on a populated file follow gap 110's rule (5.1). | -| D39 | Decision of 2026-10-21 (the maintainer). Nothing is planted or tampered, no flaw is seeded, and there is no mutation testing, under any name. No test, review, or acceptance changes a file or a copy of one (other than a stub or configuration the inputs below allow), or writes a file, a row, a revision, a call site, or a decoy holding a flaw, to see whether something catches it, and none measures how many such flaws are found. A test's or an acceptance's inputs are GitHub's recorded responses; stubs of a response GitHub or the environment can send (a status, a body, a timeout, an interrupt, a failing `git`, a clock), which a test may build in memory from a recorded unit changed into that shape; calls the code under test must refuse, and configuration it must refuse or that names a state GitHub can produce (a repository that does not resolve, one no longer listed); a timing setting a test shortens (SQLite's busy timeout), which changes how long a wait lasts and no outcome; valid rows a test writes (through the models with `tests/db/seed.py`, or in SQL); and databases and files this project's own code wrote, at any revision. Each test asserts the exact result. A check over stored data, files, or source that only a defect of this code could fail is tested on its passing side, with the exact count of the rows, files, or values it compared, so it cannot pass on nothing; a check that GitHub, the environment, or a caller can fail is tested on both sides. A fix comes with a regression test that fails against the parent commit's code. Tests merged before D39 that change a file or a copy of one, or write a file, a revision, a call, a connection, or a decoy holding a flaw, to see whether a check catches it are replaced in Step 3b-2a-2d. | -| D40 | Decision of 2026-10-22 (the maintainer). Tests are software that gives the same result every time. The same code gives every test the same outcome on any machine, on any date, and in any run order: a test, and every code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state outside its own temporary directory, the repository, and the session's database template, and bounds every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on printed wording, and no test checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode), never a planted file (D39). Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. `tests/determinism.py` refuses the network, the real clock, and its unseeded random source in every test, `tests/test_suite_rules.py` enforces the rules it can model in source, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test (Step 3b-2a-2d). | +| D39 | Decision of 2026-10-21 (the maintainer), clarified on 2026-10-22: the first wording also banned negative tests, and Step 3b-2a-2d removed some before the clarification restored them. No mutation testing, under any name: the code under test and the checks are never changed, on disk or in memory, to see whether a test notices. No false content is planted in the project's real files or documents (the plan, a document, review material, a tracked lock or database) to see whether a check or a reviewer catches it, and no reviewer or agent is scored on how many seeded flaws it finds. Negative tests are expected: a test builds its own invalid or failing input on purpose (a sample test that fails, an invalid configuration, a malformed or edited copy of a file or database in its temporary directory, a test-only revision that raises, a stub or fake that raises, a recorded GitHub response changed in memory) and feeds it to the unchanged code. Each test asserts the exact result. Every check is tested on both sides, and its passing side states the exact count of the rows, files, or values it compared, so it cannot pass on nothing. A fix comes with a regression test that fails against the parent commit's code. | +| D40 | Decision of 2026-10-22 (the maintainer). Tests are software that gives the same result every time. The same code gives every test the same outcome on any machine, on any date, and in any run order: a test, and every code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state outside its own temporary directory, the repository, and the session's database template, and bounds every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on printed wording, and no test checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test changes the project's real files. Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. `tests/determinism.py` refuses the network, the real clock, and its unseeded random source in every test, `tests/test_suite_rules.py` enforces the rules it can model in source, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test (Step 3b-2a-2d). | ## 3. Coverage rule @@ -1553,7 +1553,7 @@ configuration it reads; 3b-2a-2a adds the writer and the run records on that sch is built against the final tables; 3b-2a-2b-1 adds the scenario model the engine's tests run against; 3b-2a-2b-2 adds the sync engine over the planner and the writer, `--dry-run` included; 3b-2a-2c adds `ghactivity sync`, the seeded runs, and the live acceptance; 3b-2a-2d -replaces the tests that planted inputs (D39); 3b-2a-3 applies D37 to the sync (D38); 3b-2b-1 adds +makes the tests deterministic and keeps D39's negative tests (D40); 3b-2a-3 applies D37 to the sync (D38); 3b-2b-1 adds the REST sampler that checks what the sync wrote; and 3b-2b-2 adds verify, whose live acceptance runs the sampler. "Step 3b-2" elsewhere in this plan means 3b-2a-1, the parts of 3b-2a-2, 3b-2a-3, and the two parts of 3b-2b together; "Step 3b-2a" means 3b-2a-1, 3b-2a-2, and 3b-2a-3; "Step @@ -1567,7 +1567,7 @@ file's two comment changes are 3b-2a-1's; `[automation]` and `[sync].accepted_fa loading, every write the sync makes, run records, the run lock, the head check, and the busy-timeout comment are 3b-2a-2a's; the scenario model and its round-trip test are 3b-2a-2b-1's; the command with its usage errors and the runs that never start, the seeded runs, the go-gdpr recordings, and -the live acceptance are 3b-2a-2c's; the D39 test replacements are 3b-2a-2d's; the changes D38 +the live acceptance are 3b-2a-2c's; the determinism rules and the restored negative tests are 3b-2a-2d's; the changes D38 makes to the sync, its exit code included, are 3b-2a-3's; and the rest, `--dry-run` and the exit code of a run that starts included, is 3b-2a-2b-2's. Until 3b-2a-2c `main` has no `sync` command and the tracked database @@ -1582,24 +1582,18 @@ Every Step 3 test of a GitHub interaction runs through the real `GitHubSource` o schema comparison. Evidence is direct, with inputs at and on each side of every boundary the design states (the coverage floor, the 10-minute unsettled window, 10 minutes past `deadline_at` for the lock, the 24-hour refresh, the delete guard's max(25, 2%)) and regression tests that fail against -the parent's code. D39 and D40 govern every test. Nothing is planted or tampered, no flaw is seeded, and -there is no mutation testing, under any name. No test, review, or acceptance changes a file or a -copy of one (other than a stub or configuration the inputs below allow), or writes a file, a row, a revision, a call -site, or a decoy holding a flaw, to see whether something catches it, and none measures how many -such flaws are found. A test's or an acceptance's inputs are GitHub's recorded responses; stubs of a -response GitHub or the environment can send (a status, a body, a timeout, an interrupt, a failing -`git`, a clock), which a test may build in memory from a recorded unit changed into that shape; -calls the code under test must refuse, and configuration it must refuse or that names a state GitHub -can produce (a repository that does not resolve, one no longer listed); a timing setting a test shortens (SQLite's busy timeout), which changes how long a wait lasts and no outcome; valid rows a test writes -(through the models with `tests/db/seed.py`, or in SQL); and databases and files this project's own -code wrote, at any revision. Each test asserts the exact result. A check over stored data, files, or -source that only a defect of this code could fail is tested on its passing side, with the exact -count of the rows, files, or values it compared, so it cannot pass on nothing; a check that GitHub, -the environment, or a caller can fail is tested on both sides. A fix comes with a regression test -that fails against the parent commit's code. Tests merged before D39 that change a file or a copy of -one, or write a file, a revision, a call, a connection, or a decoy holding a flaw, to see whether a -check catches it are replaced in Step 3b-2a-2d. Each wait on a thread or subprocess has an explicit -timeout. +the parent's code. D39 and D40 govern every test. No mutation testing, under any name: the code under test and the +checks are never changed, on disk or in memory, to see whether a test notices. No false content is +planted in the project's real files or documents (the plan, a document, review material, a tracked +lock or database) to see whether a check or a reviewer catches it, and no reviewer or agent is +scored on how many seeded flaws it finds. Negative tests are expected: a test builds its own invalid +or failing input on purpose (a sample test that fails, an invalid configuration, a malformed or +edited copy of a file or database in its temporary directory, a test-only revision that raises, a +stub or fake that raises, a recorded GitHub response changed in memory) and feeds it to the +unchanged code. Each test asserts the exact result. Every check is tested on both sides, and its +passing side states the exact count of the rows, files, or values it compared, so it cannot pass on +nothing. A fix comes with a regression test that fails against the parent commit's code. Each wait +on a thread or subprocess has an explicit timeout. #### Step 3a: New schema and cutover @@ -1939,7 +1933,7 @@ syncs, the pull request records that PR's `updatedAt` and `mergeStateStatus` fro 8). An adversarial review of the change, by reviewers who did not write it, finds no invariant check without a test that targets it; CI passes on the exact PR head. -#### Step 3b-2a-2d: Tests without planted inputs, deterministic (D39, D40) +#### Step 3b-2a-2d: Deterministic tests, with D39's negative tests (D39, D40) Prerequisite: Step 3b-2a-2c has merged, and D39 is on `main`. @@ -1956,47 +1950,17 @@ settings change nothing, and records each refusal, so a test fails even where th caught it; each refusal, and the Unix socket it lets through, has a test. `tests/test_suite_rules.py` gains the rules it can model in source: no test file reads the wall clock, sleeps on it, or draws unseeded random numbers; the package reads them only in its two real clocks (`SystemClock` and `db.types.utc_now`); and no test function is defined inside another -function, where pytest never collects it. Each is tested on its passing side over every file under -`tests/` or `src/github_activity_db/`, the files it scanned counted by a second walk of the -directory, and the package scan with the exact list of reads it found. Every test merged before D39 that changes a file or a copy of one, or writes a -file, a revision, a call, a connection, or a decoy holding a flaw, to see whether a check catches it, -is replaced under D39: a check that GitHub, the environment, or a caller can fail keeps a refusing -side built from D39's inputs, and a check that only a defect of this code could fail is tested on -its passing side, with the exact count of what it compared. Among them, by file: -`tests/db/test_schema.py` (changed copies of a migrated database); `tests/db/test_migrations.py` (revisions holding a failure, a foreign-key violation, a `COMMIT` or -`END`, a statement the authorizer refuses, an `ATTACH`, or a batch alter that leaves a view in -place; the revision-rule checker's planted files; the connection put inside a transaction; the -copies changed for the definition reading); `tests/db/test_tracked_database.py` (the frozen file's -changed copy); `tests/db/test_views.py` (indexes dropped on a copy, and a row id projected under a consumer name); -`tests/sync/test_snapshot.py` (a server-side cursor left half-read); `tests/sync/test_engine.py` (a -writer patched to raise `AccountMissing` or a `RuntimeError`, a prune patched to raise a `RuntimeError`, and -`alembic_version` set to a revision that does not exist); `tests/sync/test_runs.py` (`user_version` -set on a migrated copy); `tests/sync/test_writer.py` (a wrong `targets_default_branch` written into -a row); `tests/test_cli_sync.py` (`alembic_version` set to `'older'`); `tests/test_cli_app.py` (the -command written to raise `boom`); the schema-version lock, the query lock, and the CI lock step on -edited or removed entries (`tests/test_data_dictionary.py`, `tests/test_check_query_lock.py`, and -`tests/source/test_recordings.py`'s edited query documents); and the planted calls, spellings, -decoys, alias files, subprocess calls, and package copies of the checkers' tests -(`tests/test_check_log_placeholders.py`, `tests/test_count_suppressions.py`, -`tests/test_check_any_aliases.py`, `tests/test_suite_rules.py`, `tests/test_import_boundaries.py`, -`tests/test_server_side_cursors.py`, `tests/test_declared_dependencies.py`, and -`tests/test_wheel_smoke.py`). A file at an older schema comes from a real earlier revision, as -`tests/sync/test_runs.py` builds one at `33f8570f6540`. env.py's refusal of a file that already -holds a foreign-key violation is tested on its passing side: a command over a file of seeded rows -succeeds, and the test counts the rows of that file the check read. These are D39 inputs and keep -their tests, renamed where a name says planted: a recorded unit `replay.modify` changes into a shape -GitHub can send (`FAIL_ONE_ITEM`), a kill or an interrupt at a set point, an exception a caller -raises in a transaction body (`tests/sync/test_connection.py`), the CLI's exit-code test commands, -valid tables in a scratch database (`tests/test_data_dictionary.py`'s `planted.db`), the tracked-file -check's WAL, non-database, symlink, and sidecar inputs. `tests/source/test_recorder.py`'s file-size -limit patched to a small value is replaced by a recorded unit given, in memory, a body over -`MAX_FILE_BYTES`. -`docs/testing.md` drops its descriptions of the replaced tests. - -Tests: each rule a replaced test held has a test D39 allows, on both sides where GitHub, the -environment, or a caller can fail it, or on its passing side with the exact count it compared; the -pull request lists each test it replaces or removes, with the rule it held and the test that now -holds it. +function, where pytest never collects it. Each is tested on both sides: on sample source the test writes holding each +spelling it refuses, and over every file under `tests/` or `src/github_activity_db/`, the files it scanned counted by a second walk of the +directory, and the package scan with the exact list of reads it found. Every test on `main` before this step whose input the test built itself (a test-only revision that +fails, a changed copy of a database or file in its temporary directory, an edited lock entry, a +sample file holding each spelling a checker flags, a stub or fake that raises) is kept, under D39 as +clarified, beside the tests this step adds that read real earlier revisions (`upgrade(path, +"")`) and real earlier files (`tests/fixtures/history/`). A test this step removes is one +D40 refuses: it read the wall clock or depended on the uid, or it checked a document's wording. + +Tests: every test on `main` before this step is at head under its name, or the pull request names +the test that replaced it and the reason; each check has a test on both sides. Acceptance: `tests/test_suite_rules.py`'s rules pass over every test file and package module and count the files they scanned; each refusal of `tests/determinism.py` has a test; two runs of the @@ -2133,8 +2097,7 @@ Scope: tests, `scripts/`, and `docs/testing.md`; `Transport.rest_get` already ta query, so nothing under `src/` changes. The REST sampler 3b-2b-2's acceptance uses, laid out like the recorder: comparison logic in a module under `tests/source/`, tested offline on REST and GraphQL units recorded from the same PRs, -each compared field matching, with the number of fields compared per PR asserted so a comparison -that reads nothing fails (D39), REST's `ghost` matching a null reference or a stored reference to an +each compared field matching, with the number of fields compared per PR asserted so a comparison that reads nothing fails, and one mismatch per compared field built in memory from those units (D39), REST's `ghost` matching a null reference or a stored reference to an account a lookup now returns `NOT_FOUND` for (the sampler looks it up), REST's 250 commits of a PR above 250 compared as a subset of the stored commits with equal attribution, a PR whose REST `updated_at` differs from the stored `github_updated_at` reported as changed and not compared, and @@ -2365,8 +2328,7 @@ verify's own log read failing and ending early, and its history count differing each making the repository unverified; `activity_halt_reason` set by a halt and cleared by the next run that does not halt, `halt_reason` unchanged; `--allow-deletions` applying N to events and history commits separately; a history replace keeping a commit's stored author link when GitHub -returns none (D28); and each verify check and each stored-row check on the recordings, each -stored-row check reporting the rows it compared (D39). +returns none (D28); and each verify check and each stored-row check on the recordings, each stored-row check reporting the rows it compared, and each one failing on a copy of that database with one of its rows deleted (D39). Acceptance: on the Step 3b-2 acceptance repositories and on `prebid/github-activity-db`, the stored events equal the log and the stored commits equal the history; each repository's first logged event @@ -2420,8 +2382,7 @@ the per-repository counts it compared, the revision it read, and the answers of and `foreign_key_check`, each equal to what the file holds; halting with exit 1 and naming its check on an asset digest that names another file (a stub of GitHub's release-asset response) and on one snapshot's gz given with another snapshot's manifest, both written by `snapshot`, its -decompression, `integrity_check`, and `foreign_key_check` refusals, which once the digest matches -only a defect of `snapshot` could reach, tested on their passing side (D39), and its unknown-revision +decompression, `integrity_check`, and `foreign_key_check` refusals, tested on both sides, the refusing side on a copy of a file `snapshot` wrote that the test changes (D39), and its unknown-revision refusal on a file `snapshot` wrote after an upgrade through a test-only revision past head that holds no flaw, as newer code writes one, halting with exit 1 and naming the revision check; a release with no complete snapshot pair halting with exit 1; `snapshot --verify-report` exiting 1 on a missing report, on a file that is not a verify report (the gz diff --git a/docs/testing.md b/docs/testing.md index 16e8d17..4c2f180 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -10,23 +10,18 @@ How the test suite is built and what it counts as evidence. The plan of record i - **Real components, faked boundaries.** GitHub is the only fake: the real `GitHubSource` runs over `httpx.MockTransport`, answering from responses recorded from public repositories. Databases are real SQLite files built by `alembic upgrade head`. -- **Direct evidence, nothing planted** (redesign plan D39). Nothing is planted or tampered, no flaw - is seeded, and there is no mutation testing, under any name. No test, review, or acceptance - changes a file or a copy of one (other than a stub or configuration the inputs below allow), or writes a file, a - row, a revision, a call site, or a decoy holding a flaw, to see whether something catches it, and - none measures how many such flaws are found. A test's or an acceptance's inputs are GitHub's - recorded responses; stubs of a response GitHub or the environment can send (a status, a body, a - timeout, an interrupt, a failing `git`, a clock), which a test may build in memory from a recorded - unit changed into that shape; calls the code under test must refuse, and configuration it must - refuse or that names a state GitHub can produce (a repository that does not resolve, one no longer - listed); a timing setting a test shortens (SQLite's busy timeout), which changes how long a wait lasts and no outcome; valid rows a test writes (through the models with `tests/db/seed.py`, or in SQL); and - databases and files this project's own code wrote, at any revision. Each test asserts the exact - result. A check over stored data, files, or source that only a defect of this code could fail is - tested on its passing side, with the exact count of the rows, files, or values it compared, so it - cannot pass on nothing; a check that GitHub, the environment, or a caller can fail is tested on - both sides. A fix comes with a regression test that fails against the parent commit's code. Tests - merged before D39 that change a file or a copy of one, or write a file, a revision, a call, a - connection, or a decoy holding a flaw, to see whether a check catches it were replaced in Step 3b-2a-2d. +- **Negative tests, never mutation** (redesign plan D39). No mutation testing, under any name: the + code under test and the checks are never changed, on disk or in memory, to see whether a test + notices. No false content is planted in the project's real files or documents (the plan, a + document, review material, a tracked lock or database) to see whether a check or a reviewer + catches it, and no reviewer or agent is scored on how many seeded flaws it finds. Negative tests + are expected: a test builds its own invalid or failing input on purpose (a sample test that fails, + an invalid configuration, a malformed or edited copy of a file or database in its temporary + directory, a test-only revision that raises, a stub or fake that raises, a recorded GitHub + response changed in memory) and feeds it to the unchanged code. Each test asserts the exact + result. Every check is tested on both sides, and its passing side states the exact count of the + rows, files, or values it compared, so it cannot pass on nothing. A fix comes with a regression + test that fails against the parent commit's code. - **The same result every time** (redesign plan D40). The same code gives every test the same outcome on any machine, on any date, and in any run order: a test, and every code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no network (GitHub @@ -35,9 +30,7 @@ How the test suite is built and what it counts as evidence. The plan of record i wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on printed wording, and no test checks the - wording of a document. Where a test needs an input the code cannot easily be given, the project - builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode), - never a planted file (D39). Evidence is what software reproduces: an acceptance criterion names a + wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test changes the project's real files. Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. `tests/determinism.py` refuses the network, the real clock, and its unseeded random source in every test, `tests/test_suite_rules.py` enforces the From 1f1223ad6d2e00048bf2d8d18f481a00a3ad6ef8 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 10:22:59 -0600 Subject: [PATCH 15/29] Step 3b-2a-2d: restore the checker and config negative tests (D39) The negative tests main had for the source and lock checkers, the wheel smoke check, the suppression counter, the import boundaries, the server-side cursor scan, the suite rules, the data dictionary's lock, the engine's connect listener, and the settings default, back as they were, beside the tests this step added. - test_check_any_aliases, test_check_log_placeholders: _run_on_source writes the sample module and runs the check over it with head's _run. - test_wheel_smoke: _run_package runs the check with -s and PYTHONPATH, since -I ignores PYTHONPATH, for the copy without tracking.toml. - test_check_query_lock: V3 follows V2, which it extends. - Not restored: test_config's check of the documented DATABASE_URL lines (a document's wording, D40); two query-lock cases that test_the_verdict_does_not_depend_on_how_the_checker_runs holds. --- tests/test_check_any_aliases.py | 162 +++++++++++++++++++++++ tests/test_check_log_placeholders.py | 64 +++++++++ tests/test_check_query_lock.py | 165 +++++++++++++++++++++++ tests/test_config.py | 7 + tests/test_count_suppressions.py | 191 +++++++++++++++++++++++++++ tests/test_data_dictionary.py | 49 +++++++ tests/test_db_engine.py | 22 +++ tests/test_declared_dependencies.py | 36 +++++ tests/test_import_boundaries.py | 72 ++++++++++ tests/test_server_side_cursors.py | 49 +++++++ tests/test_suite_rules.py | 37 ++++++ tests/test_wheel_smoke.py | 53 +++++++- 12 files changed, 906 insertions(+), 1 deletion(-) diff --git a/tests/test_check_any_aliases.py b/tests/test_check_any_aliases.py index 2e96786..43eb24b 100644 --- a/tests/test_check_any_aliases.py +++ b/tests/test_check_any_aliases.py @@ -6,11 +6,142 @@ import sys from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts" / "check_any_aliases.py" SRC = ROOT / "src" +ALIASES = [ + "from typing import Any\n\nFoo = Any\n", + "import typing\n\nFoo = typing.Any\n", + "from typing import Any\n\nFoo = Any | None\n", + "from typing import Any, TypeAlias\n\nFoo: TypeAlias = Any\n", + "import typing\n\nFoo: typing.TypeAlias = typing.Any\n", + "from typing import Any\n\ntype Foo = Any\n", + "import typing\n\ntype Foo = typing.Any\n", + "from typing import Any\n\ntype Foo[T] = Any\n", + "from typing import Any\n\nFoo = None | Any\n", + "from typing import Any, Optional\n\nFoo = Optional[Any]\n", + "import typing\n\nFoo = typing.Union[int, typing.Any]\n", + 'from typing import Annotated, Any\n\nFoo = Annotated[Any, "x"]\n', + "from typing import Any as Dynamic\n\ntype Foo = Dynamic\n", + "from typing import Any, Optional as Maybe\n\nFoo = Maybe[Any]\n", + "from typing import Any, Union as Either\n\nFoo = Either[int, Any]\n", + 'from typing import Annotated as Note, Any\n\nFoo = Note[Any, "x"]\n', + "from typing import Any, TypeAlias as Alias\n\nFoo: Alias = Any\n", + "import typing_extensions as te\n\nFoo = te.Any\n", + "from typing import *\n\nFoo = Any\n", + # Imports in function and class bodies bind other scopes and do not replace this one. + "from typing import Any as Dynamic\n\n\ndef convert(value: object) -> int:\n" + ' from typing import cast as Dynamic\n return Dynamic("int", value)\n\n\n' + "type Foo = Dynamic\n", + "from typing import Any as Dynamic\n\n\nclass C:\n from typing import cast as Dynamic\n\n\n" + "Foo = Dynamic\n", + # Imports in module-level blocks run in module scope. + "from typing import TYPE_CHECKING\n\nif TYPE_CHECKING:\n" + " from typing import Any as Dynamic\n\n" + "Foo = Dynamic\n", + "try:\n from typing import Any as Dynamic\nexcept ImportError:\n pass\n\nFoo = Dynamic\n", + # The latest module-level binding wins. + "class Any:\n pass\n\n\nfrom typing import Any\n\nFoo = Any\n", + # Where branches disagree, a name that may be typing.Any counts. + "from typing import TYPE_CHECKING\n\nif TYPE_CHECKING:\n from typing import Any as JSON\n" + "else:\n JSON = object\n\nFoo = JSON\n", + "import sys\n\nif sys.version_info >= (3, 12):\n from mylib import D\nelse:\n" + " from typing import Any as D\n\nFoo = D\n", + "try:\n from mylib import D\nexcept ImportError:\n" + " from typing import Any as D\n\nFoo = D\n", + "from typing import Any as D\n\nfor D in []:\n pass\n\nFoo = D\n", + "from typing import Any as D\n\nmatch 1:\n case 1:\n D = int\n\nFoo = D\n", + # String forward references where a type is expected. + 'from typing import Any, TypeAlias\n\nFoo: TypeAlias = "Any"\n', + 'from typing import Any\n\ntype Foo = "Any"\n', + 'from typing import Any, Optional\n\nFoo = Optional["Any"]\n', + 'from typing import Any, Optional, TypeAlias\n\nFoo: TypeAlias = "Optional[Any]"\n', + # x[k] = ... and x.attr = ... read x; they do not rebind it. + 'from typing import Any\n\nHANDLERS: dict[object, str] = {}\nHANDLERS[Any] = "any"\n' + "Foo = Any\n", + 'import typing as t\n\nt.__doc__ = "patched"\nFoo = t.Any\n', + "from typing import Any\n\nCOUNTS: dict[object, int] = {}\nCOUNTS[Any] += 1\nFoo = Any\n", + # Aliases in module-level blocks other than an `if TYPE_CHECKING:` body. + "from typing import Any\n\ntry:\n Foo = Any\nexcept ImportError:\n pass\n", + "import contextlib\nfrom typing import Any\n\nwith contextlib.suppress(Exception):\n" + " Foo = Any\n", + "from typing import Any\n\nfor _ in range(1):\n Foo = Any\n", + "from typing import Any\n\nwhile True:\n Foo = Any\n break\n", + "from typing import Any\n\nmatch 1:\n case 1:\n Foo = Any\n", + "import sys\nfrom typing import Any\n\nif sys.version_info >= (3, 12):\n Foo = Any\n", + # Assignments that copy a typing module or object. + "import typing\n\nt = typing\nFoo = t.Any\n", + # Each binding rule. + "from typing import *\n\n\nclass Any:\n pass\n\n\nfrom typing import *\n\nFoo = Any\n", + "from typing import Any as D\n\nwhile False:\n D = int\n\nFoo = D\n", + "try:\n pass\nfinally:\n from typing import Any as D\n\nFoo = D\n", + "try:\n from mylib import D\nexcept* ImportError:\n from typing import Any as D\n\n" + "Foo = D\n", + "from mylib import D\n\nmatch 1:\n case 1:\n from typing import Any as D\n\n" + "Foo = D\n", + "import sys\nfrom typing import *\n\nif sys.version_info >= (3, 12):\n\n class Any:\n" + " pass\n\n\nFoo = Any\n", + # A type checker reads an `if TYPE_CHECKING:` body and the else of `if not TYPE_CHECKING:`. + "from typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n Foo = Any\n", + "from typing import TYPE_CHECKING, Any\n\nif not TYPE_CHECKING:\n pass\nelse:\n" + " Foo = Any\n", + # A quoted TypeAlias annotation, a copied typing object, and a string Annotated argument. + 'from typing import Any, TypeAlias\n\nFoo: "TypeAlias" = Any\n', + "import typing\nfrom typing import Any\n\nO = typing.Optional\nFoo = O[Any]\n", + 'from typing import Annotated, Any\n\nFoo = Annotated["Any", "x"]\n', + # An annotated subscript target reads its names. + "from typing import Any\n\nCOUNTS: dict[object, int] = {}\nCOUNTS[Any]: int = 2\nFoo = Any\n", +] + +NOT_ALIASES = [ + "from typing import Any\n\nFoo = dict[str, Any]\n", + 'from typing import Annotated, Any\n\nFoo = Annotated[int, Any, "x"]\n', + "from typing import Any, Optional\n\nFoo = Optional[list[Any]]\n", + "from typing import AnyStr\n\nFoo = AnyStr\n", + "from typing import Any\n\nvalue: Any = 1\n", + "from typing import Any\n\n\ndef f() -> None:\n Foo = Any\n", + "from typing import Any\n\n\nclass C:\n Foo = Any\n", + 'TEXT = "Foo = Any"\n', + "class Any:\n pass\n\n\ntype Foo = Any\n", + "from mylib import Any\n\nFoo = Any\n", + "import mylib\n\nFoo = mylib.Any\n", + # A function-local typing import does not make the module's own class typing.Any. + "class Any:\n pass\n\n\ndef f() -> None:\n from typing import Any\n\n\ntype Foo = Any\n", + # A later module-level binding replaces the typing import. + "from typing import Any\n\n\nclass Any:\n pass\n\n\nFoo = Any\n", + "from typing import Any as Dynamic\n\nDynamic = int\nFoo = Dynamic\n", + "from typing import Any\nfrom mylib import Any\n\nFoo = Any\n", + "import typing as t\nimport mylib as t\n\nFoo = t.Any\n", + "from typing import *\n\n\nclass Any:\n pass\n\n\nFoo = Any\n", + "import contextlib\nfrom typing import Any as D\n\nwith contextlib.nullcontext(1) as D:\n" + " pass\n\nFoo = D\n", + # A string outside a type position is a value, and an unparsable reference is skipped. + 'from typing import Any\n\nFoo = "Any"\n', + 'from typing import Any, Literal\n\nFoo = Literal["Any"]\n', + 'from typing import TypeAlias\n\nFoo: TypeAlias = "not ( valid"\n', + # Each binding rule. + "from .typing import Any\n\nFoo = Any\n", + "from typing import Any as D\n\ntype D = int\nFoo = D\n", + "from typing import Any as D\n\nD += 1\nFoo = D\n", + "from typing import Any as D\n\nD: int = 1\nFoo = D\n", + "from typing import Any as D\n\ntry:\n pass\nexcept Exception as D:\n Foo = D\n", + "from typing import Any\n\nfor Any in [1]:\n Foo = Any\n", + # The branch only the runtime executes is exempt. + "from typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n pass\nelse:\n Foo = Any\n", + "from typing import TYPE_CHECKING, Any\n\nif not TYPE_CHECKING:\n Foo = Any\n", + # The TYPE_CHECKING pattern: a precise type for the checker, Any for the runtime. + "from typing import TYPE_CHECKING, Any\n\nif TYPE_CHECKING:\n from mylib import Model\n\n" + " Foo = Model\nelse:\n Foo = Any\n", + # The attribute form `typing.TYPE_CHECKING` marks the runtime-only else as well. + "import typing\nfrom typing import Any\n\nif typing.TYPE_CHECKING:\n pass\nelse:\n" + " Foo = Any\n", +] + + def _run(*paths: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( [sys.executable, str(SCRIPT), *map(str, paths)], @@ -21,6 +152,16 @@ def _run(*paths: Path) -> subprocess.CompletedProcess[str]: ) +def _run_on_source( + tmp_path: Path, source: str, name: str = "module.py" +) -> subprocess.CompletedProcess[str]: + """The check over a package of one module holding ``source``.""" + root = tmp_path / "pkg" + root.mkdir() + (root / name).write_text(source) + return _run(root) + + def _python_file_count(root: Path) -> int: """The .py and .pyi files under ``root``, counted with os.walk rather than the script's glob.""" return sum( @@ -44,3 +185,24 @@ def test_no_python_files_is_an_error(tmp_path: Path) -> None: assert result.returncode == 1 # A run that finds an alias also exits 1, but only after reporting what it scanned. assert result.stdout == "" + + +@pytest.mark.parametrize("source", ALIASES) +def test_rejects_an_alias_to_any(tmp_path: Path, source: str) -> None: + result = _run_on_source(tmp_path, source) + assert result.returncode == 1 + assert "aliases to Any: 1" in result.stdout + assert "Foo is an alias to Any" in result.stdout + + +@pytest.mark.parametrize("source", NOT_ALIASES) +def test_accepts_other_uses_of_any(tmp_path: Path, source: str) -> None: + result = _run_on_source(tmp_path, source) + assert result.returncode == 0, result.stdout + assert "aliases to Any: 0" in result.stdout + + +def test_scans_stub_files(tmp_path: Path) -> None: + result = _run_on_source(tmp_path, "from typing import Any\n\nFoo = Any\n", name="stub.pyi") + assert result.returncode == 1 + assert result.stdout.startswith("scanned 1 files") diff --git a/tests/test_check_log_placeholders.py b/tests/test_check_log_placeholders.py index ac450a1..b1b0b8c 100644 --- a/tests/test_check_log_placeholders.py +++ b/tests/test_check_log_placeholders.py @@ -8,11 +8,43 @@ import sys from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts" / "check_log_placeholders.py" SRC = ROOT / "src" +FLAGGED = [ + 'logger.info("found %s items", count)\n', + 'logger.warning("retry %d of %d", attempt, total)\n', + 'log.error("failed: %r", error)\n', + 'logger.opt(depth=1).debug("value %(name)s", values)\n', + 'logger.exception("lost %.2f%%", share)\n', + 'logger.log("INFO", "found %s", count)\n', + 'logger.success("done " "in %ss", seconds)\n', + 'logger.trace("%5d", n)\n', + "def f():\n logger.critical('%s', x)\n", + 'logger.info("found %(n)s items", n=3)\n', # loguru formats keyword arguments too + 'logger.warning("retry %s", attempt=2)\n', + 'logger.log("INFO", "found %s", count=3)\n', +] + +ACCEPTED = [ + 'logger.info("found {} items", count)\n', + 'logger.info("found %s items")\n', # no arguments: loguru formats nothing + 'logger.info("100%% done", count)\n', # a literal percent sign, no directive + "logger.info(template, count)\n", # not a literal + 'logger.info(f"found {count} items")\n', + 'console.print("found %s", count)\n', # not a log level + 'logger.log("INFO", "found {}", count)\n', + "DECOY = \"logger.info('found %s', count)\"\n", + 'logger.info("a % b", count)\n', # `% b` is a space flag without a conversion + 'logger.info("found {n} items", n=3)\n', + "logger.info(**fields)\n", # no message at all +] + + def _run(*paths: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( [sys.executable, str(SCRIPT), *map(str, paths)], @@ -23,6 +55,14 @@ def _run(*paths: Path) -> subprocess.CompletedProcess[str]: ) +def _run_on_source(tmp_path: Path, source: str) -> subprocess.CompletedProcess[str]: + """The check over a package of one module holding ``source``.""" + root = tmp_path / "pkg" + root.mkdir() + (root / "module.py").write_text(source) + return _run(root) + + def _python_file_count(root: Path) -> int: """The .py and .pyi files under ``root``, counted with os.walk rather than the script's glob.""" return sum( @@ -46,3 +86,27 @@ def test_src_has_no_percent_style_messages() -> None: assert scanned is not None and found is not None, result.stdout assert int(scanned.group(1)) == _python_file_count(SRC) assert int(found.group(1)) == 0 + + +@pytest.mark.parametrize("source", FLAGGED) +def test_flags_a_percent_style_message(tmp_path: Path, source: str) -> None: + result = _run_on_source(tmp_path, source) + assert result.returncode == 1, result.stdout + assert "%-style loguru messages: 1" in result.stdout + + +@pytest.mark.parametrize("source", ACCEPTED) +def test_accepts_brace_style_and_other_calls(tmp_path: Path, source: str) -> None: + result = _run_on_source(tmp_path, source) + assert result.returncode == 0, result.stdout + assert "%-style loguru messages: 0" in result.stdout + + +def test_reports_each_site(tmp_path: Path) -> None: + result = _run_on_source( + tmp_path, 'logger.info("a %s", x)\nlogger.debug("b {}", y)\nlogger.error("c %d", z)\n' + ) + assert result.returncode == 1 + assert "%-style loguru messages: 2" in result.stdout + assert "module.py:1:" in result.stdout + assert "module.py:3:" in result.stdout diff --git a/tests/test_check_query_lock.py b/tests/test_check_query_lock.py index f47fa3a..a2cab40 100644 --- a/tests/test_check_query_lock.py +++ b/tests/test_check_query_lock.py @@ -29,6 +29,10 @@ SCHEMA_LOCKS = ("6fedb05", "21a4267", "77663a7") +def edit_entry(work: Path) -> None: + write_lock(work, {"pull_request:1": "c" * 64}) + + def environment(directory: Path) -> dict[str, str]: """A git, checker, or step process's whole environment: the directory it runs in, inside the test's own, as its home; no global or system git configuration; fixed commit dates.""" @@ -281,6 +285,16 @@ def write_schema_lock(repo: Path, entries: dict[str, Any]) -> None: } +# Two entries with views, so a view can be edited in one that is not the newest at the base. +V3: dict[str, Any] = { + **V2, + "3": { + "columns": V2["2"]["columns"], + "views": {"v_pr_reviews": "c" * 64, "v_github_users": "b" * 64}, + }, +} + + def with_views(views: dict[str, str]) -> dict[str, Any]: """V2 with version 2's views replaced.""" return {**V2, "2": {**V2["2"], "views": views}} @@ -497,6 +511,140 @@ def test_the_repository_locks_are_read_by_their_rules(self, repo: Path) -> None: entries = len(json.loads((root / lock).read_text())) assert f"{lock}: {entries} entries at main kept; {entries} entries now" in result.stdout + def test_a_pair_added_to_a_superseded_entry_fails(self, repo: Path) -> None: + """Version 1 describes databases that never had a column version 2 added.""" + write_schema_lock(repo, V1) + commit(repo) + write_schema_lock( + repo, + {"1": [*V1["1"], ["pull_requests", "title"]], "2": [["pull_requests", "number"]]}, + ) + result = check(repo) + assert result.returncode == 1 + assert ( + "schema version 1 gained pull_requests.title; only the newest version, 2, gains " + "columns" in result.stderr + ) + + def test_a_pair_removed_from_a_base_entry_fails(self, repo: Path) -> None: + write_schema_lock(repo, V1) + commit(repo) + write_schema_lock(repo, {"1": [["pull_requests", "number"]]}) + result = check(repo) + assert result.returncode == 1 + assert "schema version 1 lost v_pr_reviewers.final_verdict" in result.stderr + + def test_a_renamed_pair_fails(self, repo: Path) -> None: + write_schema_lock(repo, V1) + commit(repo) + write_schema_lock(repo, {"1": [["pull_requests", "pr_number"], V1["1"][1]]}) + result = check(repo) + assert result.returncode == 1 + assert "schema version 1 lost pull_requests.number" in result.stderr + + def test_a_pair_added_to_a_superseded_object_entry_fails(self, repo: Path) -> None: + write_schema_lock(repo, V2) + commit(repo) + columns = [*V2["2"]["columns"], ["pull_requests", "title"]] + write_schema_lock(repo, {**V2, "2": {**V2["2"], "columns": columns}, "3": V2["2"]}) + result = check(repo) + assert result.returncode == 1 + assert ( + "schema version 2 gained pull_requests.title; only the newest version, 3, gains " + "columns" in result.stderr + ) + + @pytest.mark.parametrize( + ("current", "message"), + [ + pytest.param( + {**with_views({**V2["2"]["views"], "v_pr_reviews": "c" * 64}), "3": V2["2"]}, + "schema version 2 changed view v_pr_reviews", + id="changed digest", + ), + pytest.param( + {**with_views({"v_pr_reviews": "a" * 64}), "3": V2["2"]}, + "schema version 2 lost view v_github_users", + id="removed view", + ), + pytest.param( + {**V2, "1": [["pull_requests", "number"]], "3": V2["2"]}, + "schema version 1 lost v_pr_reviewers.final_verdict", + id="removed pair", + ), + ], + ) + def test_an_edit_to_a_superseded_entry_fails( + self, repo: Path, current: dict[str, Any], message: str + ) -> None: + """An entry stays fixed once a newer one is added.""" + write_schema_lock(repo, V2) + commit(repo) + write_schema_lock(repo, current) + result = check(repo) + assert result.returncode == 1 + assert message in result.stderr + + @pytest.mark.parametrize( + ("views", "message"), + [ + pytest.param( + {**V2["2"]["views"], "v_pr_reviews": "d" * 64}, + "schema version 2 changed view v_pr_reviews", + id="changed digest", + ), + pytest.param( + {"v_pr_reviews": "a" * 64}, + "schema version 2 lost view v_github_users", + id="removed view", + ), + ], + ) + def test_a_view_edit_below_the_newest_entry_at_the_base_fails( + self, repo: Path, views: dict[str, str], message: str + ) -> None: + """Views are kept in every base entry, not only in the base's newest one.""" + write_schema_lock(repo, V3) + commit(repo) + write_schema_lock(repo, {**V3, "2": {**V3["2"], "views": views}}) + result = check(repo) + assert result.returncode == 1 + assert message in result.stderr + + def test_a_changed_view_digest_fails(self, repo: Path) -> None: + write_schema_lock(repo, V2) + commit(repo) + write_schema_lock(repo, with_views({**V2["2"]["views"], "v_pr_reviews": "c" * 64})) + result = check(repo) + assert result.returncode == 1 + assert "schema version 2 changed view v_pr_reviews" in result.stderr + + def test_a_removed_view_fails(self, repo: Path) -> None: + write_schema_lock(repo, V2) + commit(repo) + write_schema_lock(repo, with_views({"v_pr_reviews": "a" * 64})) + result = check(repo) + assert result.returncode == 1 + assert "schema version 2 lost view v_github_users" in result.stderr + + def test_an_entry_rewritten_as_a_list_loses_its_views(self, repo: Path) -> None: + write_schema_lock(repo, V2) + commit(repo) + write_schema_lock(repo, {**V2, "2": V2["2"]["columns"]}) + result = check(repo) + assert result.returncode == 1 + assert "schema version 2 lost view v_github_users" in result.stderr + assert "schema version 2 lost view v_pr_reviews" in result.stderr + + def test_a_view_added_to_a_superseded_entry_fails(self, repo: Path) -> None: + write_schema_lock(repo, V2) + commit(repo) + superseded = {"columns": V1["1"], "views": {"v_late": "d" * 64}} + write_schema_lock(repo, {**V2, "1": superseded, "3": V2["2"]}) + result = check(repo) + assert result.returncode == 1 + assert "schema version 1 gained view v_late; only the newest version, 3" in result.stderr + # -------------------------------------------------------------------------------------------- # The CI step itself, run against shallow merge checkouts like the one actions/checkout makes @@ -676,3 +824,20 @@ def test_the_step_fails_without_a_merge_commit(tmp_path: Path, origin: Path) -> git(tmp_path, "clone", "-q", "--depth=1", f"file://{origin}", str(checkout)) result = run_step(checkout, tmp_path) assert result.returncode != 0 + + +def test_a_changed_entry_fails(repo: Path) -> None: + write_lock(repo, {"pull_request:1": "a" * 64}) + commit(repo) + write_lock(repo, {"pull_request:1": "c" * 64}) + result = check(repo) + assert result.returncode == 1 + assert "pull_request:1 changed" in result.stderr + + +def test_the_step_fails_an_edited_entry(tmp_path: Path, origin: Path) -> None: + work = work_clone(tmp_path, origin, "work") + push_main(work, checker=True, entries={"pull_request:1": "a" * 64}) + result = run_step_on_merge(tmp_path, origin, work, edit_entry) + assert result.returncode == 1, result.stdout + result.stderr + assert "pull_request:1 changed" in result.stderr diff --git a/tests/test_config.py b/tests/test_config.py index 1af1d6c..8451de4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -167,3 +167,10 @@ def test_get_settings_cached(self, no_env_file: None): # Should be the same object (cached) assert settings1 is settings2 + + +class TestDocumentedDatabaseUrl: + """The documented DATABASE_URL lines show the default, which names the local file (D26).""" + + def test_the_field_default_is_the_local_database(self) -> None: + assert Settings.model_fields["database_url"].default == LOCAL_DATABASE_URL diff --git a/tests/test_count_suppressions.py b/tests/test_count_suppressions.py index 21eeefb..678d0f5 100644 --- a/tests/test_count_suppressions.py +++ b/tests/test_count_suppressions.py @@ -19,6 +19,109 @@ SCANNED_LINE = re.compile(r"^scanned (\d+) files", re.MULTILINE) +def _package(tmp_path: Path, *sources: str) -> Path: + root = tmp_path / "pkg" + (root / "sub").mkdir(parents=True) + for i, source in enumerate(sources): + (root / ("sub" if i % 2 else ".") / f"module_{i}.py").write_text(source, encoding="utf-8") + return root + + +NOQA_SPELLINGS = [ + "import os # noqa\n", + "import os # NOQA\n", + "import os #noqa\n", + "import os # noqa:F401\n", + "import os # NoQa: F401\n", + "import os # unused on purpose # noqa: F401\n", + "# ruff: noqa\nimport os\n", + "# ruff: noqa: F401\nimport os\n", + "#ruff:noqa\nimport os\n", + "# flake8: noqa\nimport os\n", + "# flake8: noqa: F401\nimport os\n", + "# isort: skip_file\nimport sys\nimport os\n", + "# isort: off\nimport sys\nimport os\n", + "import sys # isort: skip\nimport os\n", + "# ruff: isort: skip_file\nimport sys\nimport os\n", + "# ruff: isort: off\nimport sys\nimport os\n", + # Range suppressions, which ruff 0.14.11 honors only in preview. + "# ruff: disable[F401]\nimport os\n", + "#ruff:disable[F401, E501]\nimport os\n", + "# ruff: disable [F401]\nimport os\n", + "# ruff : disable[F401]\nimport os\n", + "#\truff:\tdisable\t[F401]\nimport os\n", + "#\u3000ruff:\u3000disable[F401]\nimport os\n", + "# ruff: disable[ F401, ] # reason\nimport os\n", + "# ruff: disable[F401] reason\nimport os\n", + "def f() -> None:\n # ruff: disable[F401]\n import os\n", + "# ruff: disable[F401]\nimport os\n# ruff: enable[F401]\n", + # Not honored by ruff (uppercase), counted so that no honored spelling escapes. + "# RUFF: DISABLE[F401]\nimport os\n", +] + +TYPE_IGNORE_SPELLINGS = [ + 'x: int = "a" # type: ignore[assignment]\n', + 'x: int = "a" # type:ignore[assignment]\n', + 'x: int = "a" #type:ignore\n', + 'x: int = "a" # type: ignore\n', + '# type: ignore\nx: int = "a"\n', + '# mypy: ignore-errors\nx: int = "a"\n', + '# mypy: disable-error-code="assignment"\nx: int = "a"\n', + "from typing import no_type_check\n\n\n@no_type_check\ndef f() -> int:\n return 'x'\n", + "import typing\n\n\n@typing.no_type_check\nclass C:\n x: int = 'x'\n", + "from typing import no_type_check\n\n\n@no_type_check\nasync def f() -> int:\n return 'x'\n", + "from typing import no_type_check as ntc\n\n\n@ntc\ndef f() -> int:\n return 'x'\n", + "from typing_extensions import no_type_check as ntc\n\n\n" + "@ntc\ndef f() -> int:\n return 'x'\n", + # A re-export of typing's no_type_check, as mypy follows it. + "from pkg.compat import no_type_check as skip\n\n\n@skip\ndef f() -> int:\n return 'x'\n", +] + +REFUSED = [ + pytest.param( + "module.py", + "# coding: latin-1\n# ruff:\u3000disable[F401]\nimport os\n".encode(), + False, + id="latin-1 declaration", + ), + pytest.param( + "module.py", + b'# coding: unicode_escape\n# \\x0a"""\n# ruff: noqa: F401\n# \\x0a"""\n', + False, + id="unicode_escape declaration", + ), + pytest.param("module.py", b"# coding: no-such-codec\nimport os\n", False, id="unknown codec"), + pytest.param("module.py", b'"""Doc."""\r# ruff: noqa: F401\rimport os\r', False, id="lone CR"), + pytest.param("module.PY", b"import os # noqa: F401\n", False, id="uppercase suffix"), + pytest.param( + "stub.PYI", b"x: int = undefined_name # noqa: F821\n", False, id="uppercase stub suffix" + ), + pytest.param( + "tool", b"#!/usr/bin/env python3\nimport os # noqa: F401\n", True, id="python shebang" + ), + pytest.param("app.wsgi", b"import os # noqa: F401\n", False, id="wsgi suffix"), + pytest.param( + "tool", + b"#!/usr/bin/env nix-shell\n#! nix-shell -i python3\nimport os # noqa: F401\n", + True, + id="nix-shell shebang", + ), +] + +ACCEPTED = [ + pytest.param("module.py", b"# -*- coding: utf-8 -*-\nimport os\n", False, id="utf-8"), + pytest.param("module.py", b"\xef\xbb\xbfimport os\n", False, id="byte-order mark"), + pytest.param("module.py", b"import os\r\nimport sys\r\n", False, id="CRLF"), + pytest.param( + "tool", b"#!/usr/bin/env python3\nimport os\n", False, id="shebang, not executable" + ), + pytest.param("tool.sh", b"#!/bin/sh\necho python\n", True, id="shell script"), + pytest.param( + "tool.sh", b"#!/usr/bin/env python3\nimport os\n", True, id="python shebang, shell suffix" + ), +] + + def _run( root: Path, *, max_type_ignore: int = 5, max_noqa: int = 3 ) -> subprocess.CompletedProcess[str]: @@ -174,3 +277,91 @@ def test_no_python_files_is_an_error(tmp_path: Path) -> None: assert result.returncode == 1 # A run over files reports what it scanned before any cap fails it. assert result.stdout == "" + + +@pytest.mark.parametrize("source", NOQA_SPELLINGS) +def test_counts_every_noqa_spelling(tmp_path: Path, source: str) -> None: + assert _counts(_run(_package(tmp_path, source))) == (0, 1) + + +@pytest.mark.parametrize("source", TYPE_IGNORE_SPELLINGS) +def test_counts_every_type_ignore_spelling(tmp_path: Path, source: str) -> None: + assert _counts(_run(_package(tmp_path, source))) == (1, 0) + + +def test_counts_stub_files(tmp_path: Path) -> None: + root = tmp_path / "pkg" + root.mkdir() + (root / "stub.pyi").write_text("x: int # type: ignore[misc]\ny: int # noqa: E501\n") + result = _run(root) + assert result.stdout.startswith("scanned 1 files") + assert _counts(result) == (1, 1) + + +def test_counts_across_files_and_subdirectories(tmp_path: Path) -> None: + root = _package(tmp_path, *NOQA_SPELLINGS[:3], *TYPE_IGNORE_SPELLINGS[:2]) + result = _run(root) + assert _counts(result) == (2, 3) + assert result.stdout.startswith("scanned 5 files") + + +@pytest.mark.parametrize( + ("sources", "caps", "kind"), + [ + (NOQA_SPELLINGS[:3], {"max_noqa": 3}, "noqa"), + (TYPE_IGNORE_SPELLINGS[:5], {"max_type_ignore": 5}, "type: ignore"), + ], +) +def test_at_the_cap_passes_and_one_more_fails( + tmp_path: Path, sources: list[str], caps: dict[str, int], kind: str +) -> None: + at_cap = _run(_package(tmp_path, *sources), **caps) + assert at_cap.returncode == 0, at_cap.stdout + + over = {name: value - 1 for name, value in caps.items()} + over_cap = _run(tmp_path / "pkg", **over) + assert over_cap.returncode == 1 + listed = [line.strip() for line in over_cap.stdout.splitlines() if line.startswith(" ")] + assert len(listed) == len(sources) + assert all(re.fullmatch(r".+\.py:\d+", site) for site in listed) + assert f"{kind} comments: {len(sources)}" in over_cap.stdout + + +@pytest.mark.parametrize(("name", "content", "executable"), REFUSED) +def test_refuses_a_file_it_cannot_read_as_ruff_does( + tmp_path: Path, name: str, content: bytes, executable: bool +) -> None: + root = _package(tmp_path, "import os\n") + path = root / name + path.write_bytes(content) + if executable: + path.chmod(0o755) + result = _run(root) + assert result.returncode == 1 + assert f"{path}: refused:" in result.stderr + assert "comments:" not in result.stdout + + +@pytest.mark.parametrize(("name", "content", "executable"), ACCEPTED) +def test_accepts_files_it_reads_as_ruff_does( + tmp_path: Path, name: str, content: bytes, executable: bool +) -> None: + root = _package(tmp_path, "import os\n") + path = root / name + path.write_bytes(content) + if executable: + path.chmod(0o755) + result = _run(root) + assert result.returncode == 0, result.stderr + assert "refused" not in result.stderr + + +def test_skips_entries_named_py_that_are_not_files(tmp_path: Path) -> None: + root = _package(tmp_path, "import os # noqa: F401\n") + (root / "pkg.py").mkdir() + (root / "pkg.py" / "inner.py").write_text("import sys\n") + (root / "gone.py").symlink_to(root / "missing.py") + result = _run(root) + assert result.returncode == 0, result.stderr + assert result.stdout.startswith("scanned 2 files") + assert _counts(result) == (0, 1) diff --git a/tests/test_data_dictionary.py b/tests/test_data_dictionary.py index b0938b8..a715427 100644 --- a/tests/test_data_dictionary.py +++ b/tests/test_data_dictionary.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import json import sqlite3 from contextlib import closing from pathlib import Path @@ -354,3 +355,51 @@ def test_lock_differences_are_reported_both_ways(self) -> None: assert lock_differences(newer, consumer_pairs(dictionary_at(AT_STEP3B1))) == [ "sync_failures.last_run_id is not a consumer column at head" ] + + @pytest.mark.parametrize( + ("before", "after"), + [ + pytest.param( + "CREATE VIEW v AS SELECT 'a b' AS value", + "CREATE VIEW v AS SELECT 'a b' AS value", + id="whitespace in a literal", + ), + pytest.param( + "CREATE VIEW v AS SELECT 1 AS value -- explanation\nWHERE 0", + "CREATE VIEW v AS SELECT 1 AS value -- explanation WHERE 0", + id="the end of a line comment", + ), + ], + ) + def test_a_view_returning_other_rows_fails_the_lock(self, before: str, after: str) -> None: + """Two definitions that differ only in whitespace but return different rows get different + digests, so the checker reports the view as changed.""" + + def rows(sql: str) -> list[tuple[object, ...]]: + with closing(sqlite3.connect(":memory:")) as connection: + connection.execute(sql) + return connection.execute("SELECT * FROM v").fetchall() + + def entry(sql: str) -> dict[str, LockEntry]: + return parse_lock( + json.dumps({"2": {"columns": [["v", "value"]], "views": {"v": view_digest(sql)}}}) + ) + + assert rows(before) != rows(after) + assert lock_checker().schema_problems(entry(before), entry(after)) == [ + "schema version 2 changed view v; a change to a view's rows or derivation needs a " + "new schema version" + ] + + def test_a_listed_pair_that_is_not_consumer_at_head_is_reported(self) -> None: + consumer = consumer_pairs(read_dictionary(ENTRY)) + entry = [*consumer, ["t", "id"], ["v_t", "gone"]] + assert lock_differences(entry, consumer) == [ + "t.id is not a consumer column at head", + "v_t.gone is not a consumer column at head", + ] + + def test_a_consumer_pair_missing_from_the_entry_is_reported(self) -> None: + consumer = consumer_pairs(read_dictionary(ENTRY)) + assert consumer == [["t", "name"], ["v_t", "name"]] + assert lock_differences([["t", "name"]], consumer) == ["v_t.name is missing from the entry"] diff --git a/tests/test_db_engine.py b/tests/test_db_engine.py index 7772084..86a9acd 100644 --- a/tests/test_db_engine.py +++ b/tests/test_db_engine.py @@ -168,3 +168,25 @@ def seed(sync: Connection) -> None: ).all() assert [tuple(row) for row in pull_requests] == [(5, None)] + + def test_listener_failure_closes_the_connection(self) -> None: + class FailingCursor: + def execute(self, statement: str) -> None: + raise sqlite3.OperationalError("database is locked") + + def close(self) -> None: + pass + + class Connection: + closed = False + + def cursor(self) -> FailingCursor: + return FailingCursor() + + def close(self) -> None: + self.closed = True + + connection = Connection() + with pytest.raises(sqlite3.OperationalError, match="database is locked"): + _apply_sqlite_pragmas(connection, None) + assert connection.closed diff --git a/tests/test_declared_dependencies.py b/tests/test_declared_dependencies.py index fdde461..ae6305e 100644 --- a/tests/test_declared_dependencies.py +++ b/tests/test_declared_dependencies.py @@ -173,3 +173,39 @@ def test_a_name_several_distributions_install_needs_all_of_them_declared( assert sorted(found) == ( [] if reported is None else [Undeclared(path, "sqlalchemy", reported) for path in importers] ) + + +def test_reports_undeclared_imports_in_a_planted_package(tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text( + '[project]\nname = "planted"\ndependencies = [" rich>=13.0", "Pydantic-Settings>=2"]\n' + ) + package = tmp_path / "src" / "planted" + package.mkdir(parents=True) + (package / "a.py").write_text( + "from __future__ import annotations\n" + "import json\n" + "import httpx\n" + "from click.exceptions import UsageError\n" + "from pydantic_settings import BaseSettings\n" + "from . import b\n" + "from github_activity_db import config\n" + "import __main__\n" + ) + (package / "b.py").write_text( + "from typing import TYPE_CHECKING\n" + "if TYPE_CHECKING:\n" + " import rich\n" + " import httpx.Timeout\n" + " from _typeshed import StrPath\n" + "def f() -> None:\n" + " import not_an_installed_module_xyz\n" + 'DECOY = "import sqlalchemy"\n' + ) + scanned, found = undeclared_imports(tmp_path / "src", tmp_path / "pyproject.toml") + assert scanned == 2 + assert found == [ + Undeclared("planted/a.py", "click", ("click",)), + Undeclared("planted/a.py", "httpx", ("httpx",)), + Undeclared("planted/b.py", "httpx", ("httpx",)), + Undeclared("planted/b.py", "not_an_installed_module_xyz", ()), + ] diff --git a/tests/test_import_boundaries.py b/tests/test_import_boundaries.py index 3aff3ad..a2175f8 100644 --- a/tests/test_import_boundaries.py +++ b/tests/test_import_boundaries.py @@ -23,6 +23,8 @@ from collections.abc import Iterator from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] PACKAGE = ROOT / "src" / "github_activity_db" GITHUBKIT_STATEMENT = re.compile(r"^[ \t]*(?:from|import)[ \t]+githubkit\b", re.MULTILINE) @@ -207,3 +209,73 @@ def test_the_scan_finds_each_database_import_in_sync() -> None: files = _python_files(PACKAGE / "sync") found = database_imports(files, ROOT / "src") assert _sites(found) == _statement_sites(files, DATABASE_STATEMENT) + + +def test_the_scan_sees_source_importing_githubkit() -> None: + """Positive control: the same matcher finds the imports source/ is allowed to make.""" + assert githubkit_imports(_python_files(PACKAGE / "source")) + + +@pytest.mark.parametrize( + "source", + [ + "import githubkit\n", + "import githubkit.versions as versions\n", + "from githubkit import GitHub\n", + "from githubkit.exception import RequestFailed\n", + "def f():\n from githubkit import GitHub\n", + "from typing import TYPE_CHECKING\nif TYPE_CHECKING:\n from githubkit import GitHub\n", + "import importlib\nimportlib.import_module('githubkit')\n", + "from importlib import import_module\nimport_module('githubkit')\n", + "from importlib import import_module as load\nload('githubkit')\n", + "import builtins\nbuiltins.__import__('githubkit')\n", + "from builtins import __import__ as imp\nimp('githubkit')\n", + "import importlib\nimportlib.import_module(name='githubkit')\n", + "__import__(name='githubkit')\n", + "__import__('githubkit.versions')\n", + "class C:\n import githubkit\n", + ], +) +def test_flags_every_githubkit_import_spelling(tmp_path: Path, source: str) -> None: + (tmp_path / "module.py").write_text(source) + assert len(githubkit_imports(_python_files(tmp_path))) >= 1 + + +@pytest.mark.parametrize( + "source", + [ + 'DECOY = "import githubkit"\n', + "import githubkit_extensions\n", + "# import githubkit\n", + "import importlib\nname = 'githubkit'\nimportlib.import_module(name)\n", + "def load(name):\n return name\nload('githubkit')\n", + ], +) +def test_does_not_flag_other_text(tmp_path: Path, source: str) -> None: + (tmp_path / "module.py").write_text(source) + assert githubkit_imports(_python_files(tmp_path)) == [] + + +@pytest.mark.parametrize( + ("name", "source", "flagged"), + [ + ("transport.py", "import sqlalchemy\n", True), + ("transport.py", "from sqlalchemy.orm import Session\n", True), + ("transport.py", "from github_activity_db.db import models\n", True), + ("transport.py", "from github_activity_db.db.engine import get_session\n", True), + ("transport.py", "from ..db import engine\n", True), + ("transport.py", "from .. import db\n", True), + ("transport.py", "from .records import Account\n", False), + ("transport.py", "from ..tracking import load_tracking_config\n", False), + ("transport.py", "import sqlalchemy_utils\n", False), + ], +) +def test_flags_database_imports_from_source( + tmp_path: Path, name: str, source: str, flagged: bool +) -> None: + package = tmp_path / "github_activity_db" / "source" + package.mkdir(parents=True) + (package / "__init__.py").write_text("") + (package / name).write_text(source) + found = database_imports(_python_files(package), tmp_path) + assert bool(found) is flagged, found diff --git a/tests/test_server_side_cursors.py b/tests/test_server_side_cursors.py index 9731a32..e38240b 100644 --- a/tests/test_server_side_cursors.py +++ b/tests/test_server_side_cursors.py @@ -17,6 +17,8 @@ import os from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] SRC = ROOT / "src" CALLS = {"stream", "stream_scalars", "yield_per"} @@ -53,3 +55,50 @@ def test_no_server_side_cursor_in_src() -> None: ) assert SRC / "github_activity_db" / "sync" / "snapshot.py" in files assert server_side_cursors(files) == [] + + +@pytest.mark.parametrize( + "source", + [ + "async def f(c, q):\n return await c.stream(q)\n", + "async def f(s, q):\n return await s.stream_scalars(q)\n", + "def f(r):\n return r.yield_per(100)\n", + "def f(q):\n return q.execution_options(yield_per=100)\n", + "def f(q):\n return q.execution_options(stream_results=True)\n", + "def f(q, flag):\n return q.execution_options(stream_results=flag)\n", + "def f(q):\n return q.execution_options(**{'stream_results': True})\n", + "OPTIONS = {'yield_per': 10}\n", + "class Reader:\n def read(self, c, q):\n return c.stream(q)\n", + ], + ids=[ + "stream", + "stream-scalars", + "yield-per-method", + "yield-per-option", + "stream-results-true", + "stream-results-variable", + "stream-results-dict", + "yield-per-dict", + "in-a-method", + ], +) +def test_flags_each_spelling(tmp_path: Path, source: str) -> None: + (tmp_path / "module.py").write_text(source) + assert len(server_side_cursors([tmp_path / "module.py"])) == 1 + + +@pytest.mark.parametrize( + "source", + [ + "DECOY = 'connection.stream(query)'\n", + "# rows = await connection.stream(query)\n", + "def f(q):\n return q.execution_options(stream_results=False)\n", + "def stream(x):\n return x\nstream(1)\n", + "async def f(c, q):\n return (await c.execute(q)).all()\n", + "OPTIONS = {'stream': True}\n", + ], + ids=["string", "comment", "stream-results-false", "a-function-named-stream", "execute", "key"], +) +def test_does_not_flag_other_code(tmp_path: Path, source: str) -> None: + (tmp_path / "module.py").write_text(source) + assert server_side_cursors([tmp_path / "module.py"]) == [] diff --git a/tests/test_suite_rules.py b/tests/test_suite_rules.py index ffb3089..bc0a3a2 100644 --- a/tests/test_suite_rules.py +++ b/tests/test_suite_rules.py @@ -391,3 +391,40 @@ def test_the_real_clocks_unseeded_jitter_is_refused() -> None: with pytest.raises(NotDeterministic) as refusal: SystemClock().jitter(8.0) assert _refused(refusal) == ("randomness", "SystemClock._random.uniform(0.0, 2.0)") + + +@pytest.mark.parametrize( + ("source", "flagged"), + [ + ("import subprocess\nsubprocess.run(['true'])\n", True), + ("import subprocess\nsubprocess.check_output(['true'], text=True)\n", True), + ("import subprocess\nsubprocess.run(['true'], timeout=None)\n", True), + ("import subprocess\nsubprocess.run(['true'], **options)\n", True), + ("from subprocess import run\nrun(['true'])\n", True), + ("from subprocess import run as go\ngo(['true'])\n", True), + ("from subprocess import run\nwait = run\n", True), + ("import subprocess as sp\nsp.run(['true'], timeout=60)\n", True), + ("from subprocess import *\nrun(['true'], timeout=60)\n", True), + ("from subprocess import Popen\n", True), + ("import subprocess\nsubprocess.Popen(['true']).communicate()\n", True), + ("import subprocess\nsubprocess.getoutput('true')\n", True), + ("import subprocess\nmodule = subprocess\n", True), + ("import subprocess\nsubprocess.run(['true'], timeout=60)\n", False), + ("from subprocess import run\nrun(['true'], timeout=60)\n", False), + ( + "import subprocess\n" + "def f() -> subprocess.CompletedProcess[str]:\n" + " try:\n" + " return subprocess.run(['true'], timeout=60, stdout=subprocess.PIPE)\n" + " except subprocess.CalledProcessError:\n" + " raise\n", + False, + ), + ("def run(argv):\n return argv\nrun(['true'])\n", False), + ], +) +def test_flags_each_shape_it_does_not_model(tmp_path: Path, source: str, flagged: bool) -> None: + (tmp_path / "pkg").mkdir() + path = tmp_path / "pkg" / "module.py" + path.write_text(source) + assert bool(subprocess_problems(path)) is flagged diff --git a/tests/test_wheel_smoke.py b/tests/test_wheel_smoke.py index 227a297..78dd8c1 100644 --- a/tests/test_wheel_smoke.py +++ b/tests/test_wheel_smoke.py @@ -2,11 +2,14 @@ The CI job runs the script isolated (`-I`) against an installed wheel. These tests run it the same way in this environment, with only the environment variables each test sets, and with a separate -directory as the checkout argument, so the import-location check passes. +directory as the checkout argument, so the import-location check passes. The tests that import a +copy of the package without its tracking.toml run it with ``-s`` instead, since ``-I`` ignores +PYTHONPATH. """ import os import re +import shutil import subprocess import sys import tomllib @@ -23,6 +26,15 @@ COUNT_LINE = re.compile(r"^tracked repositories from the packaged tracking\.toml: (\d+)$", re.M) +@pytest.fixture +def package_without_resource(tmp_path: Path) -> Path: + """A copy of the package with tracking.toml removed, importable through PYTHONPATH.""" + root = tmp_path / "site" + shutil.copytree(REPO_ROOT / "src" / "github_activity_db", root / "github_activity_db") + (root / "github_activity_db" / "tracking.toml").unlink() + return root + + def _run( checkout: Path, cwd: Path, overrides: dict[str, str] | None = None ) -> subprocess.CompletedProcess[str]: @@ -36,6 +48,25 @@ def _run( ) +def _run_package( + checkout: Path, package_root: Path, overrides: dict[str, str] | None = None +) -> subprocess.CompletedProcess[str]: + """The check importing the package from ``package_root``: not isolated, since ``-I`` ignores + PYTHONPATH, but with no user site directory and only the variables it names.""" + return subprocess.run( + [sys.executable, "-s", str(SCRIPT), str(SOURCE_TOML), str(checkout)], + capture_output=True, + text=True, + env={ + "PATH": os.environ.get("PATH", ""), + "PYTHONPATH": str(package_root), + **(overrides or {}), + }, + cwd=checkout.parent, + timeout=60, + ) + + def _tracked_count(result: subprocess.CompletedProcess[str]) -> int: match = COUNT_LINE.search(result.stdout) assert match is not None, result.stdout @@ -98,3 +129,23 @@ def test_rejects_an_import_from_the_checkout(tmp_path: Path) -> None: assert result.returncode == 1 assert _refused_value(result) == package_dir assert result.stderr == "" + + +def test_fails_when_the_packaged_file_is_missing( + checkout: Path, package_without_resource: Path +) -> None: + result = _run_package(checkout, package_without_resource) + output = result.stdout + result.stderr + assert result.returncode != 0 + assert "tracking config not found" in output + # The error names the copy, so the resource-less package is the one that was imported. + assert str(package_without_resource) in output + + +@pytest.mark.parametrize("spelling", SPELLINGS) +def test_an_override_cannot_hide_a_missing_packaged_file( + checkout: Path, package_without_resource: Path, spelling: str +) -> None: + result = _run_package(checkout, package_without_resource, {spelling: str(SOURCE_TOML)}) + assert result.returncode == 1 + assert "tracking_config_path is set" in result.stdout From 10104676e4748ee790399bcfc73ab0ad5e21acd0 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 10:24:19 -0600 Subject: [PATCH 16/29] Step 3b-2a-2d: restore the source and sync negative tests (D39) The negative tests main had for the replay's refusals of edited query documents and requests, the recordings' stamps, the scenario model's refusals, the transport's client defects, the hydrator's assignee listed twice with two databaseIds, the writer's connection on a constraint failure, the engine with the writer raising, the run claim on another user_version, and the snapshot digest and lock probe, back as they were. - test_recordings: _first_refused reads the first problem the replay records, as main's _refused returned it. - The engine's world fixture and World.expects_account_missing let the test that fakes the writer's AccountMissing refusal leave that failure. - Not restored: two assignee tests that test_an_assignee_github_id_under_two_node_ids holds with real ids. --- docs/testing.md | 3 +- tests/source/test_hydrate.py | 19 +++++ tests/source/test_recordings.py | 124 ++++++++++++++++++++++++++++++++ tests/source/test_scenario.py | 61 ++++++++++++++++ tests/source/test_transport.py | 28 +++++++- tests/sync/engine_support.py | 2 + tests/sync/test_connection.py | 14 +++- tests/sync/test_engine.py | 41 ++++++++++- tests/sync/test_runs.py | 8 +++ tests/sync/test_snapshot.py | 47 +++++++++++- 10 files changed, 342 insertions(+), 5 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 4c2f180..5efa2ab 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -267,7 +267,8 @@ run must see GitHub as the test left it. template, and the `tracking.toml` entries a test adds; `World.sync()` runs the engine over them, first moving the clock past the unsettled window (`settle=False` keeps it). Each case checks the stored rows, the `sync_runs` row, the `repo_sync_state` rows the run wrote (`ended`), and the - exit code. The `world` fixture fails a test that leaves an `AccountMissing` failure. + exit code. The `world` fixture fails a test that leaves an `AccountMissing` failure, except the + one that fakes the writer's refusal. - **Seeded runs**: `tests/sync/test_seeded_runs.py` gives each of a few fixed seeds a small world and five runs, with random changes through the model's methods and random clock gaps between them. After each run `tests/sync/expected_state.py` computes the database the plan requires from diff --git a/tests/source/test_hydrate.py b/tests/source/test_hydrate.py index f1cfbdc..88db998 100644 --- a/tests/source/test_hydrate.py +++ b/tests/source/test_hydrate.py @@ -2068,3 +2068,22 @@ async def test_a_failed_item_gets_no_more_continuations( assert all(isinstance(outcome, Hydrated) for outcome in outcomes[:-1]) # The first round, then the solo retry's two continuations. assert overflow_sizes(replay) == [OVERFLOW_ALIASES_PER_REQUEST, 2] + + +@pytest.mark.parametrize("overflow", [False, True], ids=["first-page", "overflow"]) +async def test_an_assignee_node_id_listed_twice_with_two_github_ids( + open_source, replay: Replay, overflow: bool +) -> None: + # The github_ids differ, so only the node-id comparison can refuse this; its key is the + # node id, where the github_id comparison's is a number. + unit = pr_unit(replay, 15771) + first = dict(unit.data["assignedActors"]["nodes"][0]) + second = {**first, "databaseId": first["databaseId"] + 999999} + if overflow: + second_page(replay, unit, "assignedActors", second) + else: + unit.data["assignedActors"]["nodes"][1] = second + replay.add(unit) + error = await failure(open_source, PREBID_JS, 15771) + assert type(error) is DuplicateEntry + assert (error.connection, error.key) == ("assignedActors", first["id"]) diff --git a/tests/source/test_recordings.py b/tests/source/test_recordings.py index 7497171..74a2893 100644 --- a/tests/source/test_recordings.py +++ b/tests/source/test_recordings.py @@ -13,6 +13,7 @@ from github_activity_db.source import GitHubSource, Hydrated, PullRequestOutcome, documents from github_activity_db.source.errors import UnexpectedResponse from github_activity_db.source.limits import OVERFLOW_ALIASES_PER_REQUEST, PR_BATCH_SIZE +from tests.source import recordings from tests.source.conftest import FakeClock, all_outcomes from tests.source.recordings import ( COMPARE_STAMP, @@ -185,6 +186,11 @@ def _refused(replay: Replay, payload: dict) -> list[str]: return replay.problems +def _first_refused(replay: Replay, payload: dict) -> str: + """The first problem the replay records for a request it answers 418.""" + return _refused(replay, payload)[0] + + def test_the_replay_refuses_a_request_it_has_no_unit_for() -> None: key = {"owner": "prebid", "name": "Prebid.js", "number": 1} payload = { @@ -288,3 +294,121 @@ def test_splitting_a_replayed_batch_gives_back_its_units() -> None: ] assert unit.document_sha256 == original.document_sha256 assert replay.problems == [] + + +@pytest.mark.parametrize( + ("document", "edit"), + [ + ( + documents.pull_request_batch(2), + ("repository(owner: $owner, name: $name)", "repository(name: $name, owner: $owner)"), + ), + (documents.pull_request_batch(2), ("rateLimit { cost remaining resetAt }", "")), + (documents.pull_request_batch(2), ("additions deletions changedFiles", "additions")), + ], + ids=["batch-wrapper", "batch-rate-limit", "batch-fields"], +) +def test_a_changed_batch_document_changes_the_stamp(document: str, edit: tuple[str, str]) -> None: + assert edit[0] in document + assert pull_request_stamp(document.replace(*edit)) != expected_pull_request_stamp() + + +@pytest.mark.parametrize( + "edit", + [("headRefOid updatedAt ", ""), ("rateLimit { cost remaining resetAt }", "")], + ids=["continuation-head", "continuation-rate-limit"], +) +def test_a_changed_continuation_document_changes_the_stamp(edit: tuple[str, str]) -> None: + query = documents.overflow_continuation([documents.FILES]) + assert edit[0] in query + assert overflow_stamp(query.replace(*edit), 0, "files") != expected_overflow_stamp("files") + + +def test_the_replay_refuses_a_request_for_another_document() -> None: + replay = Replay.from_fixtures() + query = documents.pull_request_batch(1).replace("additions deletions changedFiles", "additions") + payload = {"query": query, "variables": {"owner": "prebid", "name": "Prebid.js", "n0": 13481}} + request = httpx.Request("POST", "https://api.github.com/graphql", json=payload) + response = replay.transport().handle_request(request) + assert response.status_code == 418 + assert "re-record" in replay.problems[0] + + +def test_the_replay_refuses_a_batch_alias_bound_to_another_number() -> None: + query = documents.pull_request_batch(2).replace( + "p1: pullRequest(number: $n1)", "p1: pullRequest(number: $n0)" + ) + assert "pullRequest(number: $n0)" in query + payload = { + "query": query, + "variables": {"owner": "prebid", "name": "Prebid.js", "n0": 13481, "n1": 15724}, + } + assert "bind" in _first_refused(Replay.from_fixtures(), payload) + + +def test_the_replay_refuses_a_continuation_bound_to_another_cursor() -> None: + connections = [documents.CONNECTIONS_BY_KEY["commits"]] * 2 + query = documents.overflow_continuation(connections) + block = recordings.extract_block(query, "o1_commits: node(") + query = query.replace(block, block.replace("$after1", "$after0")) + payload = { + "query": query, + "variables": {"id0": "PR_a", "after0": "x", "id1": "PR_b", "after1": "y"}, + } + assert "binds" in _first_refused(Replay.from_fixtures(), payload) + + +def test_the_replay_refuses_a_batch_declaring_an_unsent_number() -> None: + query = documents.pull_request_batch(1).replace("$n0: Int!", "$n0: Int! $n1: Int!") + payload = {"query": query, "variables": {"owner": "prebid", "name": "Prebid.js", "n0": 13481}} + assert "declares" in _first_refused(Replay.from_fixtures(), payload) + + +def test_the_replay_refuses_a_continuation_with_an_unbound_variable() -> None: + query = documents.overflow_continuation([documents.CONNECTIONS_BY_KEY["commits"]]) + payload = {"query": query, "variables": {"id0": "PR_a", "after0": "x", "id1": "PR_b"}} + assert "variables" in _first_refused(Replay.from_fixtures(), payload) + + +def test_the_replay_refuses_batch_aliases_out_of_order() -> None: + query = documents.pull_request_batch(2) + first = "p0: pullRequest(number: $n0) { ...PullRequestFields }" + second = "p1: pullRequest(number: $n1) { ...PullRequestFields }" + query = query.replace(f"{first} {second}", f"{second} {first}") + assert query.index("p1:") < query.index("p0:") + payload = { + "query": query, + "variables": {"owner": "prebid", "name": "Prebid.js", "n0": 13481, "n1": 15724}, + } + assert "bind" in _first_refused(Replay.from_fixtures(), payload) + + +def test_the_replay_refuses_a_continuation_reading_another_alias_id() -> None: + connections = [documents.CONNECTIONS_BY_KEY["commits"]] * 2 + query = documents.overflow_continuation(connections).replace( + "o1_commits: node(id: $id1)", "o1_commits: node(id: $id0)" + ) + payload = { + "query": query, + "variables": {"id0": "PR_a", "after0": "x", "id1": "PR_b", "after1": "y"}, + } + assert "binds" in _first_refused(Replay.from_fixtures(), payload) + + +def test_the_replay_refuses_a_continuation_declaring_an_unused_alias() -> None: + query = documents.overflow_continuation([documents.CONNECTIONS_BY_KEY["commits"]]).replace( + "$id0: ID! $after0: String!", "$id0: ID! $after0: String! $id1: ID! $after1: String!" + ) + payload = {"query": query, "variables": {"id0": "PR_a", "after0": "x"}} + assert "declarations" in _first_refused(Replay.from_fixtures(), payload) + + +@pytest.mark.parametrize( + "query", + ["per_page=100&page=1&sha=main", "per_page=100", "per_page=100&page=1&page=2"], + ids=["extra-parameter", "missing-page", "repeated-page"], +) +def test_a_compare_request_with_other_parameters_finds_no_unit(query: str) -> None: + base = "/repos/prebid/Prebid.js/compare/" + "a" * 40 + "..." + "b" * 40 + assert compare_key(f"https://api.github.com{base}?{query}") is None + assert compare_key(f"https://api.github.com{base}?per_page=100&page=1") is not None diff --git a/tests/source/test_scenario.py b/tests/source/test_scenario.py index 6ffbfed..bf8eaca 100644 --- a/tests/source/test_scenario.py +++ b/tests/source/test_scenario.py @@ -1157,3 +1157,64 @@ def test_recorded_lists_are_in_the_order_cursors_name(unit: Unit) -> None: for connection, order in ORDERS.items(): keys = [order(node) for node in unit.data[connection]["nodes"]] assert keys == sorted(keys), connection + + +def test_a_document_the_model_was_not_checked_against_is_refused() -> None: + world = Scenario(FakeClock()) + world.add_repository("example", "project") + variables = {"owner": "example", "name": "project"} + changed = documents.RESOLVE.replace("isPrivate", "isPrivate isFork") + assert changed != documents.RESOLVE + assert post(world, documents.RESOLVE, variables).status_code == 200 + assert post(world, changed, variables).status_code == 418 + assert world.replay.problems == [ + f"resolve unit {variables} was recorded for another query document: re-record with " + "scripts/record_fixtures.py" + ] + + +def test_a_missing_repository_is_answered_only_for_a_checked_document() -> None: + world = Scenario(FakeClock()) + variables = {"owner": "example", "name": "absent", "after": None} + changed = documents.PULL_REQUEST_INDEX.replace("isDraft", "isDraft title") + answered = post(world, documents.PULL_REQUEST_INDEX, variables) + assert (answered.status_code, answered.json()["data"]["repository"]) == (200, None) + assert post(world, changed, variables).status_code == 418 + batch = documents.pull_request_batch(1) + changed_batch = batch.replace("additions deletions", "additions") + assert changed_batch != batch + numbered = {"owner": "example", "name": "absent", "n0": 1} + assert post(world, batch, numbered).json()["data"]["repository"] is None + assert post(world, changed_batch, numbered).status_code == 418 + assert world.replay.problems == [ + f"{kind} unit {{'owner': 'example', 'name': 'absent', 'for': '{kind}'}} was recorded for " + "another query document: re-record with scripts/record_fixtures.py" + for kind in ("index", "pull_request") + ] + + +@pytest.mark.parametrize( + ("connection", "cursor"), + [("comments", "unserved"), ("comments", "scenario:[1]"), ("commits", "NTA")], +) +async def test_a_cursor_the_model_never_served_is_reported(connection: str, cursor: str) -> None: + """A cursor in the model's own form, or GitHub's offset form (``NTA``, base64 of 50), that the + model did not serve names no page; the failure is the replay's problem, not an error inside + the transport.""" + world = Scenario(FakeClock()) + repository, author = project(world) + repository.pulls[1].add(connection, *(world.node(connection, by=author) for _ in range(101))) + + def forged(body: dict[str, Any]) -> None: + body["data"]["repository"]["p0"][connection]["pageInfo"]["endCursor"] = cursor + + world.replay.modify(forged, when=lambda seen: seen.operation == "PullRequestBatch") + async with world.source() as source: + resolved = await source.resolve("example", "project") + with pytest.raises(FatalSourceError): + await all_outcomes(source, resolved, [1]) + key = {"id": repository.pulls[1].node["id"], "connection": connection, "after": cursor} + message = f"cursor {cursor!r} names no page the model served" + assert world.replay.problems == [ + f"the provider failed on overflow {key}: ScenarioError({message!r})" + ] diff --git a/tests/source/test_transport.py b/tests/source/test_transport.py index c939d1e..d55568c 100644 --- a/tests/source/test_transport.py +++ b/tests/source/test_transport.py @@ -10,7 +10,7 @@ from collections.abc import Callable, Iterator from datetime import UTC, datetime, timedelta from email.utils import format_datetime -from typing import Any +from typing import Any, cast import httpx import pytest @@ -717,6 +717,20 @@ async def test_a_token_refresh_githubkit_cannot_parse_is_fatal( assert clock.sleeps == [] assert [seen.path for seen in replay.seen] == [endpoint] + @pytest.mark.parametrize( + "error", + [TypeError("not from githubkit's rate-limit readers"), ValueError("bad key data")], + ids=["type-error", "value-error"], + ) + async def test_a_defect_inside_the_client_is_fatal( + self, open_source, replay: Replay, clock: FakeClock, error: Exception + ) -> None: + replay.fault(self.raising(error), when=resolves) + async with open_source() as source: + with pytest.raises(ClientFailure): + await source.resolve(**RESOLVE) + assert clock.sleeps == [] + class TestFatal: async def test_401_is_unauthorized(self, open_source, replay: Replay) -> None: @@ -786,6 +800,18 @@ async def test_an_unhandled_status_is_unexpected(self, open_source, replay: Repl await source.resolve(**RESOLVE) assert raised.value.status == 409 + async def test_a_defect_in_the_request_path_is_not_a_rate_limit(self, clock: FakeClock) -> None: + # A client whose arequest no longer accepts the transport's arguments: the TypeError is + # not raised by githubkit's rate-limit readers, so it propagates instead of waiting. + class Incompatible: + async def arequest(self, *args: object, **kwargs: object) -> object: + raise TypeError("arequest() got an unexpected keyword argument 'content'") + + transport = Transport(cast("Any", Incompatible()), clock=clock) + with pytest.raises(TypeError, match="unexpected keyword"): + await transport.graphql("query Resolve { x }", {}, splittable=False) + assert clock.sleeps == [] + class TestServerErrors: """An unsplittable request (resolve) retries after 2, 4, and 8 s, then fails its repository.""" diff --git a/tests/sync/engine_support.py b/tests/sync/engine_support.py index beaf4c6..819e910 100644 --- a/tests/sync/engine_support.py +++ b/tests/sync/engine_support.py @@ -28,6 +28,8 @@ def __init__(self, scenario: Scenario, database: Path) -> None: self.sync_settings: dict[str, Any] = {} self.automation: dict[str, Any] = {} self.floor = FLOOR + # Only the test that fakes the writer's refusal may leave an AccountMissing failure. + self.expects_account_missing = False # Options a parametrized case gives both a dry run and the run. self.options: dict[str, Any] = {} diff --git a/tests/sync/test_connection.py b/tests/sync/test_connection.py index 48b6b43..6fdadad 100644 --- a/tests/sync/test_connection.py +++ b/tests/sync/test_connection.py @@ -7,7 +7,7 @@ from pathlib import Path import pytest -from sqlalchemy import select, text, update +from sqlalchemy import insert, select, text, update from sqlalchemy.exc import DBAPIError from github_activity_db.db.models import SyncRun @@ -200,3 +200,15 @@ async def test_a_reader_takes_no_lock(migrated_database: Path) -> None: async with transaction(reader): await reader.execute(select(SyncRun.id)) assert not _write_is_refused(migrated_database) + + +async def test_a_constraint_failure_is_a_local_database_error(migrated_database: Path) -> None: + run_id = _run(migrated_database) + async with open_writer(_url(migrated_database)) as writer: + with pytest.raises(LocalDatabaseError, match="IntegrityError"): + async with transaction(writer, run_id): + await writer.execute(update(SyncRun).values(rest_calls=5)) + await writer.execute(insert(SyncRun).values(status="bogus")) + async with transaction(writer, run_id): + rest_calls = (await writer.execute(select(SyncRun.rest_calls))).scalar_one() + assert rest_calls == 0 diff --git a/tests/sync/test_engine.py b/tests/sync/test_engine.py index a1eb4d7..0817004 100644 --- a/tests/sync/test_engine.py +++ b/tests/sync/test_engine.py @@ -25,6 +25,7 @@ from github_activity_db.db import engine as db_engine from github_activity_db.db.schema import HEAD_REVISION, MIGRATION_REVISIONS from github_activity_db.source import PULL_REQUEST_HYDRATION_VERSION +from github_activity_db.sync import writer from github_activity_db.sync.deletions import DeletionApproval, parse_allow_deletions from github_activity_db.sync.plan import Reason from github_activity_db.sync.runs import ClaimRefused, HeadCheckError @@ -42,7 +43,8 @@ def world(scenario: Scenario, migrated_database: Path) -> Iterator[World]: commit links (plan 5.2).""" world = World(scenario, migrated_database) yield world - assert world.rows("SELECT number FROM sync_failures WHERE last_error LIKE 'Account%'") == [] + if not world.expects_account_missing: + assert world.rows("SELECT number FROM sync_failures WHERE last_error LIKE 'Account%'") == [] @pytest.fixture @@ -2904,3 +2906,40 @@ def failed_again(body: dict[str, Any]) -> None: assert world.numbers(repository) == [1, 2, 3] assert world.pending(repository) == [] assert ended(world, finished) == ("complete", 0, {"prebid/Prebid.js": "complete"}) + + +async def test_any_other_exception_after_the_claim_closes_the_run_failed( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + project(world) + + async def broken(*args: Any, **kwargs: Any) -> None: + raise RuntimeError("broken in-place update") + + monkeypatch.setattr(writer, "apply_in_place_updates", broken) + + report = await world.sync() + + assert report.error == "RuntimeError: broken in-place update" + assert ended(world, report) == ("failed", 1, {"prebid/Prebid.js": "failed"}) + + +async def test_an_account_missing_refusal_is_an_item_failure( + world: World, monkeypatch: pytest.MonkeyPatch +) -> None: + world.expects_account_missing = True + repository, _ = project(world, pulls=2) + write = writer.write_pull_request + + async def refusing(connection: Any, run_id: int, record: Any, **kwargs: Any) -> Any: + if record.number == 1: + raise writer.AccountMissing(record.number, [7]) + return await write(connection, run_id, record, **kwargs) + + monkeypatch.setattr(writer, "write_pull_request", refusing) + + report = await world.sync() + + assert world.failures() == [("prebid/Prebid.js", 1, "AccountMissing")] + assert world.numbers(repository) == [2] + assert ended(world, report) == ("complete", 2, {"prebid/Prebid.js": "complete"}) diff --git a/tests/sync/test_runs.py b/tests/sync/test_runs.py index d41f1d3..1a79c00 100644 --- a/tests/sync/test_runs.py +++ b/tests/sync/test_runs.py @@ -979,3 +979,11 @@ def test_code_version_outside_a_checkout_is_the_package_version( ignored = checkout / "venv" / "github_activity_db" shutil.copytree(alone, ignored) assert code_version(ignored, {}) == metadata.version("github-activity-db") + + +async def test_another_user_version_is_refused(migrated_database: Path) -> None: + connection = sqlite3.connect(migrated_database) + connection.execute("PRAGMA user_version = 1") + connection.close() + with pytest.raises(HeadCheckError, match="user_version is 1, not schema_version 2"): + await check_head(_url(migrated_database)) diff --git a/tests/sync/test_snapshot.py b/tests/sync/test_snapshot.py index 1d8507a..48875f5 100644 --- a/tests/sync/test_snapshot.py +++ b/tests/sync/test_snapshot.py @@ -2,16 +2,18 @@ from __future__ import annotations +import hashlib import re import sqlite3 from pathlib import Path from typing import Any, cast import pytest -from sqlalchemy import event +from sqlalchemy import event, select, update from sqlalchemy.ext.asyncio import AsyncConnection from github_activity_db.db.engine import build_engine +from github_activity_db.db.models import PullRequest from github_activity_db.sync.snapshot import ( Snapshot, StoredPullRequest, @@ -30,6 +32,25 @@ ) +def _digest(database: Path) -> str: + return hashlib.sha256(database.read_bytes()).hexdigest() + + +async def _commit_after(database: Path, statement: Any | None) -> None: + """Run ``read_snapshot`` or ``statement`` on an application connection, then commit, so a + write either made lands in the file.""" + engine = build_engine(f"sqlite+aiosqlite:///{database}") + try: + async with engine.connect() as connection: + if statement is None: + await read_snapshot(connection) + else: + await connection.execute(statement) + await connection.commit() + finally: + await engine.dispose() + + def _seed(database: Path) -> dict[str, int]: """Two repositories, one with rows in every table the reader reads, one with none.""" with connect(database) as connection: @@ -236,3 +257,27 @@ async def test_the_lock_probe_refuses_while_another_connection_reads( finally: reader.close() commit_does_not_wait(migrated_database) + + +async def test_the_digest_sees_a_committed_write(migrated_database: Path) -> None: + """Positive control: an UPDATE committed on the same kind of connection changes the file.""" + _seed(migrated_database) + before = _digest(migrated_database) + await _commit_after(migrated_database, update(PullRequest).values(is_draft=False)) + assert _digest(migrated_database) != before + + +async def test_the_lock_probe_sees_a_half_consumed_read(migrated_database: Path) -> None: + """Positive control: a server-side cursor left half-read blocks the probe's commit.""" + _seed(migrated_database) + engine = build_engine(f"sqlite+aiosqlite:///{migrated_database}") + try: + async with engine.connect() as connection: + stream = await connection.stream(select(PullRequest.id)) + assert await stream.fetchone() is not None + with pytest.raises(sqlite3.OperationalError, match="database is locked"): + commit_does_not_wait(migrated_database) + await stream.close() + commit_does_not_wait(migrated_database) + finally: + await engine.dispose() From d9bec82578067af2259cc67cf8668e9d8395eec7 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 10:26:13 -0600 Subject: [PATCH 17/29] Step 3b-2a-2d: restore the database negative tests (D39) The negative tests main had for env.py (a revision that fails, one that leaves a foreign-key violation, one that ends the transaction, attaches a database, or runs an action the authorizer does not list, a batch alter under a view, a connection already inside a transaction, a file holding a violation), the revision-rule checker, the schema comparison and plan rules on changed copies, the frozen file's changed copy, and the participation views' growth with single-column indexes, back as they were, beside the tests this step added over real revisions. - _batch_alter_revision takes main's form again, which also builds the revision without the view statements. - create_all_uses returns the uses and the files read; the restored caller reads the uses. - The participation test builds its copies with the single-column indexes, then measures them with head's growth. --- docs/testing.md | 8 +- tests/db/test_migrations.py | 410 +++++++++++++++++++++++++++++- tests/db/test_schema.py | 296 +++++++++++++++++++++ tests/db/test_tracked_database.py | 13 + tests/db/test_views.py | 22 ++ 5 files changed, 739 insertions(+), 10 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 5efa2ab..7f8ca00 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -111,8 +111,9 @@ tests/ as the other side of the model comparison. - **The real environment.** Migration tests run the repository's `alembic.ini`, `env.py`, and revisions. `write_revision` adds a test-only revision after the head through - `version_locations`: one records the settings of the connection `env.py` gives a revision, and - one runs a batch alter under the views. Earlier schemas come from the real revisions + `version_locations`: revisions that fail, leave a foreign-key violation, try to end the + transaction, or batch-alter a table under the views show what `env.py` refuses and rolls back, + and one records the settings of the connection `env.py` gives a revision. Earlier schemas come from the real revisions (`upgrade(path, "")`). `env.py` is loaded fresh by every command, so its `MigrationRefused` is matched by class name. - **Seeding.** `tests/db/seed.py` writes rows through the models' tables with every column a @@ -121,7 +122,8 @@ tests/ - **Schema comparison.** `schema_of` reads columns (`PRAGMA table_xinfo`), indexes, foreign keys, and every item of each table's definition, which is where constraint names and CHECK text live; the readings are tested on a scratch database of valid SQL holding what no pragma - reports, at head against the models' declarations, and between real revisions. + reports, at head against the models' declarations, between real revisions, and on changed + copies of a migrated database, where each difference is reported. - **Dictionary and schema version.** `tests/test_data_dictionary.py` compares `docs/data-dictionary.md` with a migrated database in both directions. `tests/schema_versions.json` maps each schema version (`PRAGMA user_version`) to its consumer diff --git a/tests/db/test_migrations.py b/tests/db/test_migrations.py index f03f3b5..48644fc 100644 --- a/tests/db/test_migrations.py +++ b/tests/db/test_migrations.py @@ -68,6 +68,62 @@ ) +PLANTED_LABEL = "INSERT INTO pr_labels (pr_id, name, label_node_id) VALUES (999, {!r}, 'x')" + +ORPHAN = f"op.execute({PLANTED_LABEL.format('orphan')!r})" + +SWALLOWED_ROLLBACK = ( + "try:\n" + ' op.execute("INSERT OR ROLLBACK INTO alembic_version "\n' + f" \"VALUES ('{HEAD_REVISION}')\")\n" + "except Exception:\n" + " pass\n" +) + +STOP_TRIGGER = ( + "CREATE TRIGGER stop BEFORE INSERT ON pr_labels WHEN NEW.name = 'stop' " + "BEGIN SELECT RAISE(ROLLBACK, 'stop'); END" +) + +STOPPED_INSERT = ( + f"try:\n op.execute({PLANTED_LABEL.format('stop')!r})\nexcept Exception:\n pass\n" +) + +BOUND_INSERT = ( + 'op.get_bind().exec_driver_sql("INSERT INTO pr_labels (pr_id, name, label_node_id) ' + "VALUES (?, ?, 'LA_bound')\", (999, {!r}))" +) + + +def revision_source_problems(path: Path) -> list[str]: + """Plan 5.1's rules for a revision file: it imports only the standard library, alembic, and + SQLAlchemy (no application code, no relative import), and its ``VIEWS`` is a literal.""" + tree = ast.parse(path.read_bytes()) + problems = [ + f"{path.name}:{line}: imports {module}" + for line, module in imported_modules(path) + if module.split(".")[0] not in REVISION_IMPORTS | sys.stdlib_module_names + ] + problems += [ + f"{path.name}:{node.lineno}: relative import" + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.level > 0 + ] + for node in tree.body: + if isinstance(node, ast.Assign): + targets, value = node.targets, node.value + elif isinstance(node, ast.AnnAssign) and node.value is not None: + targets, value = [node.target], node.value + else: + continue + if any(isinstance(t, ast.Name) and t.id == "VIEWS" for t in targets): + try: + ast.literal_eval(value) + except ValueError: + problems.append(f"{path.name}:{node.lineno}: VIEWS is not a literal") + return problems + + def snapshot(database: Path) -> tuple[Any, ...]: """Everything an upgrade could change: the schema, every table's rows, and user_version.""" with closing(sqlite3.connect(database)) as connection: @@ -229,6 +285,78 @@ def test_an_in_memory_database_is_migrated( command.upgrade(config, "head") assert list(tmp_path.iterdir()) == [] + @pytest.mark.parametrize( + "run", + [ + pytest.param(command.current, id="current"), + pytest.param(command.check, id="check"), + pytest.param(lambda config: command.stamp(config, "head"), id="stamp head"), + pytest.param(lambda config: command.upgrade(config, "head"), id="upgrade at head"), + ], + ) + def test_every_command_refuses_a_file_holding_a_foreign_key_violation( + self, migrated_database: Path, run: Any + ) -> None: + """No command commits a file that fails the check, whether it ran a revision or not.""" + with closing(sqlite3.connect(migrated_database)) as connection, connection: + connection.execute(PLANTED_LABEL.format("orphan")) + before = snapshot(migrated_database) + with pytest.raises(RuntimeError) as error: + run(alembic_config(migrated_database)) + assert "1 foreign-key violations in the file before the commit" in refusal(error) + assert snapshot(migrated_database) == before + + @pytest.mark.parametrize( + ("run", "statement"), + [ + *( + pytest.param(run, statement, id=f"{change}, {name}") + for change, statement in ( + ("a row", ORPHAN), + # The seeded pull request's author row goes with the table. + ("the schema", 'op.execute("DROP TABLE github_users")'), + ) + for name, run in ( + ("current", command.current), + ("stamp head", lambda config: command.stamp(config, "head")), + ("upgrade at head", lambda config: command.upgrade(config, "head")), + ) + ), + # With the table gone, check's own comparison fails before the commit instead. + pytest.param(command.check, ORPHAN, id="a row, check"), + ], + ) + def test_revision_module_code_that_breaks_a_reference_is_refused( + self, migrated_database: Path, tmp_path: Path, run: Any, statement: str + ) -> None: + """alembic loads each revision module inside the transaction, so code at a module's top + level runs even when no upgrade() does.""" + with connect(migrated_database) as seeded: + seed = Seed(seeded) + seed.pull_request(seed.repository(), 1, seed.user("author")) + versions = tmp_path / "versions" + versions.mkdir() + (versions / f"{TEST_REVISION}_module_level.py").write_text( + "from alembic import op\n\n" + f"revision = {TEST_REVISION!r}\n" + f"down_revision = {HEAD_REVISION!r}\n" + "branch_labels = None\n" + "depends_on = None\n\n" + f"{statement}\n\n\n" + "def upgrade() -> None:\n" + " pass\n\n\n" + "def downgrade() -> None:\n" + " raise RuntimeError('test-only revision')\n" + ) + # At the test revision already, so no command runs a revision or finds the file behind. + with closing(sqlite3.connect(migrated_database)) as connection, connection: + connection.execute("UPDATE alembic_version SET version_num = ?", (TEST_REVISION,)) + before = snapshot(migrated_database) + with pytest.raises(RuntimeError) as error: + run(alembic_config(migrated_database, test_versions=versions)) + assert "1 foreign-key violations" in refusal(error) + assert snapshot(migrated_database) == before + class TestStep3Migration: def test_upgrades_a_seeded_first_schema_database(self, first_schema: Path) -> None: @@ -426,6 +554,42 @@ def test_the_refusal_covers_every_table_the_revision_recreates( "account_profiles", } + def test_the_definition_reading_sees_a_changed_table( + self, step3a_database: Path, tmp_path: Path + ) -> None: + """Positive control: a column added to a table is a changed definition.""" + changed = tmp_path / "changed.db" + shutil.copyfile(step3a_database, changed) + with closing(sqlite3.connect(changed)) as connection, connection: + connection.execute("ALTER TABLE pr_files ADD COLUMN probe INTEGER") + assert ( + table_definitions(changed)["pr_files"] != table_definitions(step3a_database)["pr_files"] + ) + + def test_the_definition_reading_sees_a_literal_s_whitespace( + self, step3a_database: Path, tmp_path: Path + ) -> None: + """Whitespace inside a literal is data: a CHECK on 'a b' and one on 'a b' differ.""" + + def defined(literal: str, name: str) -> str: + copy = tmp_path / f"{name}.db" + shutil.copyfile(step3a_database, copy) + with closing(sqlite3.connect(copy)) as connection, connection: + ((sql,),) = connection.execute( + "SELECT sql FROM sqlite_master WHERE name = 'pr_files'" + ).fetchall() + assert "path TEXT NOT NULL" in sql + connection.execute("DROP TABLE pr_files") + connection.execute( + sql.replace( + "path TEXT NOT NULL", f"path TEXT NOT NULL CHECK (path != {literal})" + ) + ) + return table_definitions(copy)["pr_files"] + + assert defined("'a b'", "one") == defined("'a b'", "again") + assert defined("'a b'", "two") != defined("'a b'", "one") + class TestSyncFailuresLastRunIdMigration: FAILURE = ( @@ -568,6 +732,72 @@ def test_the_migration_connection_has_foreign_keys_off_inside_one_transaction( upgrade(migrated_database, test_versions=versions) assert query(migrated_database, "SELECT * FROM probe") == [(0, 0, 1, 1)] + def test_a_failure_in_the_second_revision_applies_neither( + self, first_schema: Path, tmp_path: Path + ) -> None: + versions = tmp_path / "versions" + write_revision( + versions, TEST_REVISION, HEAD_REVISION, "raise RuntimeError('planted failure')" + ) + before = snapshot(first_schema) + with pytest.raises(RuntimeError, match="planted failure"): + upgrade(first_schema, test_versions=versions) + assert snapshot(first_schema) == before + assert revision_of(first_schema) == FIRST_SCHEMA_HEAD + + def test_a_foreign_key_violation_fails_the_upgrade_before_commit( + self, migrated_database: Path, tmp_path: Path + ) -> None: + versions = tmp_path / "versions" + write_revision( + versions, + TEST_REVISION, + HEAD_REVISION, + ORPHAN, + ) + before = snapshot(migrated_database) + with pytest.raises(RuntimeError) as error: + upgrade(migrated_database, test_versions=versions) + assert "1 foreign-key violations in the file before the commit" in refusal(error) + assert "('pr_labels', 1, 'pull_requests', 0)" in refusal(error) + assert snapshot(migrated_database) == before + + @pytest.mark.parametrize( + ("foreign_keys", "message"), + [ + pytest.param("ON", "PRAGMA foreign_keys reads 1 after turning it off", id="fk on"), + pytest.param("OFF", "already inside a transaction", id="fk off"), + ], + ) + def test_a_connection_already_inside_a_transaction_is_refused( + self, + first_schema: Path, + monkeypatch: pytest.MonkeyPatch, + foreign_keys: str, + message: str, + ) -> None: + """Planted through env.py's own engine: the DBAPI connect function SQLAlchemy calls.""" + real_connect = sqlite3.dbapi2.connect + planted: list[sqlite3.Connection] = [] + + def connect_inside_a_transaction(*args: Any, **kwargs: Any) -> sqlite3.Connection: + connection = real_connect(*args, **kwargs) + connection.execute(f"PRAGMA foreign_keys={foreign_keys}") + connection.execute("BEGIN") + planted.append(connection) + return connection + + before = snapshot(first_schema) + monkeypatch.setattr(sqlite3.dbapi2, "connect", connect_inside_a_transaction) + with pytest.raises(RuntimeError) as error: + upgrade(first_schema) + monkeypatch.undo() + assert message in refusal(error) + assert len(planted) == 1 + with pytest.raises(sqlite3.ProgrammingError, match="closed database"): + planted[0].execute("SELECT 1") + assert snapshot(first_schema) == before + Authorizer = Callable[[int, str | None, str | None, str | None, str | None], int] @@ -794,15 +1024,148 @@ def connect(*args: Any, **kwargs: Any) -> sqlite3.Connection: assert received == [0] assert revision_of(first_schema) == HEAD_REVISION + @pytest.mark.parametrize( + "body", + [ + pytest.param(f"{ORPHAN}\nop.execute('COMMIT')", id="COMMIT"), + pytest.param(f"{ORPHAN}\nop.execute('END')", id="END"), + pytest.param(f"{ORPHAN}\nop.execute('ROLLBACK')\n{ORPHAN}", id="ROLLBACK"), + pytest.param(f"op.execute('BEGIN')\n{ORPHAN}", id="BEGIN"), + pytest.param( + f"with op.get_context().autocommit_block():\n {ORPHAN}", id="autocommit_block" + ), + pytest.param( + "op.execute('CREATE TABLE early (x INTEGER)')\n" + f"{SWALLOWED_ROLLBACK}" + f"op.execute('CREATE TABLE late (x INTEGER)')\n{ORPHAN}", + id="INSERT OR ROLLBACK", + ), + pytest.param( + "op.execute('CREATE TRIGGER stop BEFORE INSERT ON pr_labels " + "BEGIN SELECT RAISE(ROLLBACK, \\'stop\\'); END')\n" + "try:\n" + f" {ORPHAN}\n" + "except Exception:\n" + " pass\n" + "op.execute('CREATE TABLE late (x INTEGER)')", + id="RAISE(ROLLBACK)", + ), + ], + ) + def test_a_revision_cannot_end_the_transaction( + self, first_schema: Path, tmp_path: Path, body: str + ) -> None: + versions = tmp_path / "versions" + write_revision(versions, TEST_REVISION, HEAD_REVISION, body) + before = snapshot(first_schema) + with pytest.raises(Exception, match="not authorized"): + upgrade(first_schema, test_versions=versions) + assert snapshot(first_schema) == before + + @pytest.mark.parametrize( + ("body", "url_query"), + [ + pytest.param(f"{ORPHAN}\n{SWALLOWED_ROLLBACK}{ORPHAN}", "", id="INSERT OR ROLLBACK"), + pytest.param( + f"{ORPHAN}\n{STOPPED_INSERT}{ORPHAN}", + "", + id="RAISE(ROLLBACK) of a trigger in the file", + ), + pytest.param( + f"{BOUND_INSERT.format('a')}\n{SWALLOWED_ROLLBACK}{BOUND_INSERT.format('b')}", + "", + id="bound parameters", + ), + pytest.param( + f"{ORPHAN}\n{SWALLOWED_ROLLBACK}{ORPHAN}", + "?cached_statements=128", + id="statement cache asked for by the URL", + ), + ], + ) + def test_a_statement_run_before_sqlite_ended_the_transaction_is_refused_after( + self, migrated_database: Path, tmp_path: Path, body: str, url_query: str + ) -> None: + """SQLite authorizes a statement when it prepares it. A data-only upgrade changes no + schema, so SQLite's own rollback expires nothing the driver could reuse by its text.""" + with closing(sqlite3.connect(migrated_database)) as connection, connection: + connection.execute(STOP_TRIGGER) + versions = tmp_path / "versions" + write_revision(versions, TEST_REVISION, HEAD_REVISION, body) + config = alembic_config(migrated_database, test_versions=versions) + config.attributes["database_url"] += url_query + before = snapshot(migrated_database) + with pytest.raises(Exception, match="not authorized"): + command.upgrade(config, "head") + assert snapshot(migrated_database) == before + + @pytest.mark.parametrize( + "statement", + [ + # The rollback of a failed upgrade could not restore pages spilled to the file. + "PRAGMA journal_mode=OFF", + "PRAGMA main.JOURNAL_MODE=OFF", + # A batch alter under a view would pass (gap 71). + "PRAGMA legacy_alter_table=ON", + # Rows the named CHECK constraints refuse would be committed. + "PRAGMA ignore_check_constraints=ON", + "PRAGMA writable_schema=ON", + "SAVEPOINT early", + "CREATE TEMP TABLE scratch (x INTEGER)", + # Schema-qualified, these reach SQLite as the main database's action codes. + "CREATE TABLE temp.scratch (x INTEGER)", + "CREATE VIEW temp.scratch AS SELECT 1 AS x", + "CREATE TRIGGER temp.drop_labels BEFORE INSERT ON pr_labels " + "BEGIN SELECT RAISE(IGNORE); END", + "ANALYZE", + ], + ) + def test_a_revision_runs_only_the_actions_the_authorizer_lists( + self, migrated_database: Path, tmp_path: Path, statement: str + ) -> None: + versions = tmp_path / "versions" + write_revision(versions, TEST_REVISION, HEAD_REVISION, f"op.execute({statement!r})") + before = snapshot(migrated_database) + with pytest.raises(Exception, match="not authorized"): + upgrade(migrated_database, test_versions=versions) + assert snapshot(migrated_database) == before + + def test_a_revision_cannot_attach_another_database( + self, migrated_database: Path, tmp_path: Path + ) -> None: + """ATTACH would let a revision write a file the refusals never saw, such as another + checkout's github_activity.db.""" + other = tmp_path / "other-checkout" / "github_activity.db" + other.parent.mkdir() + versions = tmp_path / "versions" + write_revision( + versions, + TEST_REVISION, + HEAD_REVISION, + f'op.execute("ATTACH DATABASE {str(other)!r} AS other")\n' + "op.execute('CREATE TABLE other.probe (x INTEGER)')", + ) + before = snapshot(migrated_database) + with pytest.raises(Exception, match="not authorized"): + upgrade(migrated_database, test_versions=versions) + assert snapshot(migrated_database) == before + assert not other.exists() + -def _batch_alter_revision(views_sql: tuple[tuple[str, str], ...]) -> str: - """A revision that batch-recreates pull_requests, dropping and recreating the views reading - it from ``views_sql``, a frozen copy, as plan 5.1 requires.""" +def _batch_alter_revision(views_sql: tuple[tuple[str, str], ...] | None) -> str: + """A revision that batch-recreates pull_requests; with ``views_sql`` it drops and recreates + the views reading it, from that frozen copy, as plan 5.1 requires.""" + drop = recreate = "" + if views_sql is not None: + drop = ( + f"for name in {VIEWS_READING_PULL_REQUESTS!r}:\n op.execute(f'DROP VIEW {{name}}')\n" + ) + recreate = f"for _name, sql in {views_sql!r}:\n op.execute(sql)\n" return ( - f"for name in {VIEWS_READING_PULL_REQUESTS!r}:\n op.execute(f'DROP VIEW {{name}}')\n" - "with op.batch_alter_table('pull_requests', recreate='always') as batch:\n" - " batch.add_column(sa.Column('probe', sa.Integer(), nullable=True))\n" - f"for _name, sql in {views_sql!r}:\n op.execute(sql)\n" + drop + + "with op.batch_alter_table('pull_requests', recreate='always') as batch:\n" + + " batch.add_column(sa.Column('probe', sa.Integer(), nullable=True))\n" + + recreate ) @@ -912,6 +1275,17 @@ def test_a_batch_alter_of_pull_requests_keeps_child_rows_and_constraints( } <= constraints.keys() assert query(migrated_database, PULL_REQUEST_INDEXES) == indexes + def test_a_batch_alter_under_a_view_fails_and_applies_nothing( + self, migrated_database: Path, tmp_path: Path + ) -> None: + """legacy_alter_table stays off, so a view left in place fails the migration (gap 71).""" + versions = tmp_path / "versions" + write_revision(versions, TEST_REVISION, HEAD_REVISION, _batch_alter_revision(None)) + before = snapshot(migrated_database) + with pytest.raises(Exception, match=r"error in view v_\w+"): + upgrade(migrated_database, test_versions=versions) + assert snapshot(migrated_database) == before + class TestRefusals: @pytest.fixture(autouse=True) @@ -1257,6 +1631,28 @@ def defines_views(path: Path) -> bool: ) assert newest.name.startswith(STEP3B1) + @pytest.mark.parametrize( + ("source", "flagged"), + [ + ("import json\nimport sqlalchemy as sa\nfrom alembic import op\n", False), + ("VIEWS = (('v', 'CREATE VIEW v AS SELECT 1'),)\n", False), + ("from typing import Final\nVIEWS: Final = (('v', 'SELECT 1'),)\n", False), + ("from github_activity_db.db import models\n", True), + ("import githubkit\n", True), + ("from . import shared\n", True), + ("import importlib\nimportlib.import_module('github_activity_db')\n", True), + ("VIEWS = SHARED_VIEWS\n", True), + ("VIEWS: tuple = tuple(view for view in ())\n", True), + ("VIEWS = (('v', 'CREATE VIEW v AS SELECT ' + '1'),)\n", True), + ], + ) + def test_flags_each_rule_on_planted_files( + self, tmp_path: Path, source: str, flagged: bool + ) -> None: + path = tmp_path / "f00000000001_planted.py" + path.write_text(source) + assert bool(revision_source_problems(path)) is flagged + def test_a_generated_revision_raises_in_downgrade_and_follows_the_rules(tmp_path: Path) -> None: """``alembic revision`` writes from alembic/script.py.mako.""" diff --git a/tests/db/test_schema.py b/tests/db/test_schema.py index bf7e9eb..bd75e42 100644 --- a/tests/db/test_schema.py +++ b/tests/db/test_schema.py @@ -12,6 +12,7 @@ import ast import os import re +import shutil import sqlite3 import tokenize from collections.abc import Callable @@ -59,6 +60,22 @@ SQLITE = sqlite.dialect() +def _tampered(source: Path, target: Path, *statements: str) -> Path: + shutil.copyfile(source, target) + with closing(sqlite3.connect(target)) as connection, connection: + for statement in statements: + connection.execute(statement) + return target + + +def _redefined(database: Path, table: str, old: str, new: str) -> tuple[str, str]: + """Statements that recreate ``table`` with ``old`` replaced by ``new`` in its definition.""" + with closing(sqlite3.connect(database)) as connection: + ((sql,),) = connection.execute("SELECT sql FROM sqlite_master WHERE name = ?", (table,)) + assert old in sql, (old, sql) + return f"DROP TABLE {table}", sql.replace(old, new) + + @pytest.fixture(scope="module") def models_database(tmp_path_factory: pytest.TempPathFactory) -> Path: path = tmp_path_factory.mktemp("models") / "models.db" @@ -298,6 +315,145 @@ def test_a_check_s_changed_text_is_read(self, revisions: dict[str, Path]) -> Non assert old["github_users"]["definition"] != new["github_users"]["definition"] assert old["pull_requests"]["definition"] != new["pull_requests"]["definition"] + @pytest.mark.parametrize( + ("table", "old", "new"), + [ + pytest.param( + "automation_emails", + "CONSTRAINT ck_automation_emails_email_lowercase CHECK", + "CONSTRAINT ck_automation_emails_lower CHECK", + id="renamed CHECK", + ), + pytest.param( + "automation_emails", + "email = lower(email)", + "email = upper(email)", + id="changed CHECK text", + ), + pytest.param( + "automation_emails", + "CONSTRAINT ck_automation_emails_email_lowercase ", + "", + id="unnamed CHECK", + ), + pytest.param("pr_files", "path TEXT NOT NULL", "path TEXT", id="nullability"), + pytest.param( + "pr_commits", + "authored_at DATETIME NOT NULL", + "authored_at TEXT NOT NULL", + id="column type", + ), + pytest.param("pr_labels", " ON DELETE CASCADE", "", id="foreign key action"), + pytest.param( + "pr_labels", + "CONSTRAINT fk_pr_labels_pr_id_pull_requests", + "CONSTRAINT fk_pr_labels_pr", + id="foreign key name", + ), + pytest.param( + "pr_reviews", + "CONSTRAINT uq_pr_reviews_github_id UNIQUE", + "UNIQUE", + id="unnamed UNIQUE", + ), + ], + ) + def test_a_planted_difference_is_reported( + self, + head_template: Path, + models_database: Path, + tmp_path: Path, + table: str, + old: str, + new: str, + ) -> None: + tampered = _tampered( + head_template, tmp_path / "tampered.db", *_redefined(head_template, table, old, new) + ) + assert schema_of(tampered)[table] != schema_of(models_database)[table] + + def test_a_missing_index_is_reported( + self, head_template: Path, models_database: Path, tmp_path: Path + ) -> None: + tampered = _tampered( + head_template, tmp_path / "tampered.db", "DROP INDEX ix_pr_reviews_reviewer_id" + ) + assert schema_of(tampered)["pr_reviews"] != schema_of(models_database)["pr_reviews"] + + @pytest.mark.parametrize( + ("first", "second"), + [ + pytest.param( + ["CREATE INDEX ix_probe ON pr_labels (name) WHERE name <> 'a'"], + ["CREATE INDEX ix_probe ON pr_labels (name) WHERE name <> 'b'"], + id="partial index WHERE", + ), + pytest.param( + ["CREATE INDEX ix_probe ON pr_labels (lower(name))"], + ["CREATE INDEX ix_probe ON pr_labels (upper(name))"], + id="expression index", + ), + pytest.param( + [], + ["CREATE TRIGGER tr_probe AFTER DELETE ON pr_labels BEGIN SELECT 1; END"], + id="trigger", + ), + ], + ) + def test_index_sql_and_triggers_are_compared( + self, head_template: Path, tmp_path: Path, first: list[str], second: list[str] + ) -> None: + """Shapes no pragma reports: a partial index's WHERE, an expression, a trigger.""" + one = _tampered(head_template, tmp_path / "one.db", *first) + two = _tampered(head_template, tmp_path / "two.db", *second) + assert schema_of(one)["pr_labels"] != schema_of(two)["pr_labels"] + + @pytest.mark.parametrize("kind", ["check", "index", "trigger"]) + def test_whitespace_inside_a_literal_is_a_difference( + self, head_template: Path, tmp_path: Path, kind: str + ) -> None: + """Two schemas whose SQL differs only in a literal's whitespace read differently.""" + + def read(literal: str, name: str) -> dict[str, Any]: + statements: tuple[str, ...] + if kind == "check": + statements = _redefined( + head_template, + "pr_files", + "path TEXT NOT NULL", + f"path TEXT NOT NULL CHECK (path != {literal})", + ) + elif kind == "index": + statements = (f"CREATE INDEX ix_probe ON pr_files (path) WHERE path != {literal}",) + else: + statements = ( + "CREATE TRIGGER tr_probe BEFORE INSERT ON pr_files " + f"BEGIN SELECT RAISE(ABORT, {literal}); END", + ) + tampered = _tampered(head_template, tmp_path / f"{name}.db", *statements) + return schema_of(tampered)["pr_files"] + + one, again, two = read("'a b'", "one"), read("'a b'", "again"), read("'a b'", "two") + assert one == again + # The difference is read where the kind's SQL is: a table's items, an index, a trigger. + assert [key for key in one if one[key] != two[key]] == [ + {"check": "definition", "index": "indexes", "trigger": "triggers"}[kind] + ] + + @pytest.mark.parametrize("option", ["STRICT", "WITHOUT ROWID"]) + def test_a_table_option_is_reported( + self, head_template: Path, models_database: Path, tmp_path: Path, option: str + ) -> None: + tampered = _tampered( + head_template, + tmp_path / "tampered.db", + *_redefined(head_template, "pr_labels", "\n)", f"\n) {option}"), + ) + assert ( + schema_of(tampered)["pr_labels"]["flags"] + != schema_of(models_database)["pr_labels"]["flags"] + ) + class TestViews: def test_views_match_the_newest_frozen_sql(self, head_template: Path) -> None: @@ -327,6 +483,37 @@ def test_an_earlier_revision_s_views_are_reported(self, revisions: dict[str, Pat ] assert view_differences(revisions[STEP3B1], at_step3b1) == [] + def test_a_changed_view_is_reported(self, head_template: Path, tmp_path: Path) -> None: + tampered = _tampered( + head_template, + tmp_path / "tampered.db", + "DROP VIEW v_github_users", + "CREATE VIEW v_github_users AS SELECT github_id, login FROM github_users", + "CREATE VIEW v_extra AS SELECT 1 AS one", + "DROP VIEW v_commit_authors", + ) + assert view_differences(tampered, frozen_views(alembic_config(head_template))) == [ + "missing v_commit_authors", + "not frozen v_extra", + "differs v_github_users", + ] + + def test_whitespace_inside_a_literal_is_a_difference( + self, head_template: Path, tmp_path: Path + ) -> None: + tampered = _tampered( + head_template, + tmp_path / "tampered.db", + "CREATE VIEW v_literal AS SELECT 'a b' AS value", + "CREATE VIEW v_kept AS SELECT 'a b' AS value", + ) + frozen = { + **frozen_views(alembic_config(head_template)), + "v_literal": "\nCREATE VIEW v_literal AS SELECT 'a b' AS value\n", + "v_kept": "\nCREATE VIEW v_kept AS SELECT 'a b' AS value\n", + } + assert view_differences(tampered, frozen) == ["differs v_literal"] + class TestObjectsOutsideTheComparison: """schema_of reads tables and the indexes and triggers on them, view_differences the views; @@ -358,6 +545,23 @@ def test_the_views_a_later_revision_drops_are_reported( ("view", name) for name in sorted(dropped) ] + def test_each_planted_object_is_reported(self, head_template: Path, tmp_path: Path) -> None: + tampered = _tampered( + head_template, + tmp_path / "tampered.db", + "CREATE TRIGGER tr_view INSTEAD OF DELETE ON v_github_users BEGIN SELECT 1; END", + "CREATE INDEX ix_version ON alembic_version (version_num)", + "CREATE TRIGGER tr_version AFTER UPDATE ON alembic_version BEGIN SELECT 1; END", + "ANALYZE", + ) + frozen = frozen_views(alembic_config(head_template)) + assert objects_outside_the_comparison(tampered, frozen) == [ + ("index", "ix_version"), + ("table", "sqlite_stat1"), + ("trigger", "tr_version"), + ("trigger", "tr_view"), + ] + # Plan 5.1, transcribed: the comparison above holds the models and the migration to each other, # and these hold both to the plan. @@ -602,6 +806,75 @@ def test_the_plan_readers_read_each_column_and_key(self, tmp_path: Path) -> None ("c", "a", "q", "SET NULL"), } + def test_a_changed_rule_is_reported(self, head_template: Path, tmp_path: Path) -> None: + tampered = _tampered( + head_template, + tmp_path / "tampered.db", + *_redefined(head_template, "pr_files", "path TEXT NOT NULL", "path TEXT"), + *_redefined( + head_template, + "pr_reviews", + "REFERENCES github_users (id) ON DELETE SET NULL, \n\t" + "CONSTRAINT uq_pr_reviews_github_id UNIQUE (github_id)", + "REFERENCES github_users (id) ON DELETE CASCADE", + ), + *_redefined( + head_template, + "sync_failures", + "REFERENCES sync_runs (id), \n\tCONSTRAINT uq_sync_failures", + "REFERENCES sync_runs (id) ON DELETE CASCADE, \n\tCONSTRAINT uq_sync_failures", + ), + ) + assert nullable_columns(tampered) - NULLABLE == {("pr_files", "path")} + assert unique_sets(tampered)["pr_reviews"] == set() + assert delete_rule_problems(tampered) == ( + {("pr_reviews", "editor_id", "CASCADE"), ("sync_failures", "last_run_id", "CASCADE")}, + OUTSIDE_THE_DELETE_RULE, + ) + + def test_a_primary_key_reference_set_null_is_reported( + self, head_template: Path, tmp_path: Path + ) -> None: + """The cascade exception is read per column: SET NULL on pr_assignees.account_id breaks + it.""" + tampered = _tampered( + head_template, + tmp_path / "tampered.db", + *_redefined( + head_template, + "pr_assignees", + "REFERENCES github_users (id) ON DELETE CASCADE", + "REFERENCES github_users (id) ON DELETE SET NULL", + ), + ) + assert delete_rule_problems(tampered) == ( + {("pr_assignees", "account_id", "SET NULL")}, + OUTSIDE_THE_DELETE_RULE, + ) + + def test_a_primary_key_column_and_a_second_reference_are_read( + self, head_template: Path, tmp_path: Path + ) -> None: + """SQLite holds a composite primary key's columns NOT NULL only as declared, and one + column can carry two foreign keys.""" + tampered = _tampered( + head_template, + tmp_path / "tampered.db", + *_redefined(head_template, "pr_labels", "name TEXT NOT NULL", "name TEXT"), + *_redefined( + head_template, + "pr_reviews", + "ON DELETE SET NULL, \n\t", + "ON DELETE SET NULL, \n\t" + "FOREIGN KEY(reviewer_id) REFERENCES github_users (id) ON DELETE CASCADE, \n\t", + ), + ) + assert nullable_columns(tampered) - NULLABLE == {("pr_labels", "name")} + assert delete_rule_problems(tampered) == ( + {("pr_reviews", "reviewer_id", "CASCADE")}, + OUTSIDE_THE_DELETE_RULE, + ) + class TestForeignKeyIndexes: # Kept from the first schema unchanged (plan 5.1). @@ -629,6 +902,12 @@ def test_a_composite_primary_key_and_the_rowid_count(self, head_template: Path) if entry[0] in ("pr_labels", "repo_sync_state") } + def test_a_missing_index_is_reported(self, head_template: Path, tmp_path: Path) -> None: + tampered = _tampered( + head_template, tmp_path / "tampered.db", "DROP INDEX ix_pr_comments_author_id" + ) + assert unindexed_foreign_keys(tampered) == self.EXEMPT | {("pr_comments", ("author_id",))} + def _sync_run(seed: Seed, **values: object) -> int: return seed.sync_run(**values) @@ -857,3 +1136,20 @@ def test_create_all_is_used_only_by_the_comparison(self) -> None: assert len(uses) == names assert texts assert texts.isdisjoint(uses) + + @pytest.mark.parametrize( + ("source", "flagged"), + [ + ("Base.metadata.create_all(engine)\n", True), + ("conn.run_sync(Base.metadata.create_all)\n", True), + ("def f():\n metadata.create_all(bind=engine)\n", True), + ('DECOY = "Base.metadata.create_all(engine)"\n', False), + ("# Base.metadata.create_all(engine)\n", False), + ("Base.metadata.drop_all(engine)\n", False), + ], + ) + def test_flags_every_use_and_no_text(self, tmp_path: Path, source: str, flagged: bool) -> None: + package = tmp_path / "pkg" + package.mkdir() + (package / "module.py").write_text(source) + assert bool(create_all_uses(package)[0]) is flagged diff --git a/tests/db/test_tracked_database.py b/tests/db/test_tracked_database.py index dd3d784..9f3eacc 100644 --- a/tests/db/test_tracked_database.py +++ b/tests/db/test_tracked_database.py @@ -159,6 +159,19 @@ def test_git_stores_the_tracked_file_as_a_regular_file(self, tmp_path: Path) -> mode, _blob, stage_and_path = entry.split(" ") assert (mode, stage_and_path) == ("100644", "0\tgithub_activity.db") + def test_a_changed_copy_fails(self, tmp_path: Path) -> None: + copy = tmp_path / "changed.db" + shutil.copyfile(REPO_ROOT / "github_activity.db", copy) + connection = sqlite3.connect(copy) + try: + connection.execute("PRAGMA user_version = 1") + finally: + connection.close() + result = _frozen(REPO_ROOT / "github_activity.db", copy) + assert result.returncode == 1 + assert f"{copy}: sha256 " in result.stdout + assert "differs from the pinned" in result.stdout + class TestGitignore: """Only the repository-root github_activity.db escapes the *.db rule (D26).""" diff --git a/tests/db/test_views.py b/tests/db/test_views.py index 193cbb5..2a63e69 100644 --- a/tests/db/test_views.py +++ b/tests/db/test_views.py @@ -27,6 +27,14 @@ ] +SINGLE_COLUMN_INDEXES = ( + "DROP INDEX ix_pr_reviews_pr_id_reviewer_id_github_id", + "DROP INDEX ix_pr_reviews_reviewer_id", + "CREATE INDEX ix_pr_reviews_pr_id ON pr_reviews (pr_id)", + "CREATE INDEX ix_pr_reviews_reviewer_id ON pr_reviews (reviewer_id)", +) + + def account(prefix: str) -> list[str]: return [ f"{prefix}_github_id", @@ -1272,3 +1280,17 @@ def test_work_grows_with_the_pull_requests(self, databases: dict[int, Path], vie (self.rows_read(prs)[view],) ], prs assert self.growth(databases, view) < 6 + + def test_a_lookup_by_the_account_alone_is_reported( + self, databases: dict[int, Path], tmp_path: Path + ) -> None: + """With single-column indexes in place of the (PR, account) one, the same measure grows + with the square of the pull requests.""" + altered = {} + for prs in self.SIZES: + altered[prs] = tmp_path / f"{prs}.db" + shutil.copyfile(databases[prs], altered[prs]) + with closing(sqlite3.connect(altered[prs])) as connection, connection: + for statement in SINGLE_COLUMN_INDEXES: + connection.execute(statement) + assert self.growth(altered, "v_pr_comments") > 10 From f83a06a848eb799a5bd3395acba6c0b3d1785bb3 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 10:27:02 -0600 Subject: [PATCH 18/29] Step 3b-2a-2d: test the determinism and collection scans on both sides Each spelling the determinism scan refuses (the wall clock through datetime, date, and time; sleeps; unseeded random, secrets, os.urandom, and uuid4; aliases, from-imports, a module as a value, and a call in an annotation) is reported with its line, and the spellings it allows (tzset, a seeded Random, an annotation, sleep(0), uuid5, a datetime built from its fields, another object's now) are not. The collection scan reports a test function or class nested in a function, and not one at module or class level. --- tests/test_suite_rules.py | 67 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/test_suite_rules.py b/tests/test_suite_rules.py index bc0a3a2..c1faa35 100644 --- a/tests/test_suite_rules.py +++ b/tests/test_suite_rules.py @@ -428,3 +428,70 @@ def test_flags_each_shape_it_does_not_model(tmp_path: Path, source: str, flagged path = tmp_path / "pkg" / "module.py" path.write_text(source) assert bool(subprocess_problems(path)) is flagged + + +def _module(tmp_path: Path, source: str) -> Path: + (tmp_path / "pkg").mkdir() + path = tmp_path / "pkg" / "module.py" + path.write_text(source) + return path + + +@pytest.mark.parametrize( + ("source", "problems"), + [ + ("from datetime import datetime\nnow = datetime.now()\n", ["2: datetime.now"]), + ("import datetime\nnow = datetime.datetime.utcnow()\n", ["2: datetime.datetime.utcnow"]), + ("from datetime import date\ntoday = date.today()\n", ["2: date.today"]), + ("from datetime import datetime as dt\n", ["1: datetime.datetime imported as dt"]), + ("import time\nstarted = time.monotonic()\n", ["2: time.monotonic"]), + ("from time import sleep\n", ["1: imports time.sleep"]), + ("import time as clock\n", ["1: time imported as clock"]), + ("import time\nclock = time\n", ["2: the time module used as a value"]), + ("import random\nvalue = random.random()\n", ["2: random.random"]), + ("import random\nrng = random.Random()\n", ["2: random.Random without a seed"]), + ("from random import Random\nrng = Random(None)\n", ["2: Random without a seed"]), + ("import secrets\ntoken = secrets.token_hex()\n", ["2: secrets.token_hex"]), + ("import asyncio\n\n\nasync def f():\n await asyncio.sleep(1)\n", ["5: asyncio.sleep"]), + ("from asyncio import sleep\n", ["1: imports asyncio.sleep"]), + ("import os.path\nkey = os.urandom(8)\n", ["2: os.urandom"]), + ("import uuid\nkey = uuid.uuid4()\n", ["2: uuid.uuid4"]), + ( + "import time\nfrom typing import Annotated\n\n\n" + "def f(x: Annotated[float, time.time] = 0.0) -> None:\n pass\n", + ["5: time.time"], + ), + ("import time\ntime.tzset()\n", []), + ("import random\nrng = random.Random(7)\n", []), + ("import random\n\n\ndef f(rng: random.Random) -> None:\n pass\n", []), + ("import asyncio\n\n\nasync def f():\n await asyncio.sleep(0)\n", []), + ("import uuid\nkey = uuid.uuid5(uuid.NAMESPACE_URL, 'x')\n", []), + ("from datetime import UTC, datetime\nwhen = datetime(2026, 1, 1, tzinfo=UTC)\n", []), + ("clock = make_clock()\nclock.now()\n", []), + ], +) +def test_the_determinism_scan_reports_each_spelling( + tmp_path: Path, source: str, problems: list[str] +) -> None: + assert determinism_problems(_module(tmp_path, source)) == [ + f"pkg/module.py:{problem}" for problem in problems + ] + + +@pytest.mark.parametrize( + ("source", "problems"), + [ + ( + "def test_outer():\n def test_inner():\n pass\n", + ["2: test_inner inside test_outer"], + ), + ("def helper():\n class TestNested:\n pass\n", ["2: TestNested inside helper"]), + ("def test_a():\n pass\n\n\nclass TestB:\n def test_c(self):\n pass\n", []), + ], +) +def test_the_collection_scan_reports_each_nested_test( + tmp_path: Path, source: str, problems: list[str] +) -> None: + assert uncollected_tests(_module(tmp_path, source)) == [ + f"pkg/module.py:{problem}" for problem in problems + ] From 79e835f74dfdbb15ab5c38c47cb65d96ec37adea Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 12:28:38 -0600 Subject: [PATCH 19/29] alembic/env.py: an upgrade runs on one driver connection env.py held the authorizer's state mapping before the upgrade, so after SQLAlchemy invalidated the connection and a revision called rollback(), the next statement reconnected on a second driver connection whose own mapping still read "open". env.py's ROLLBACK of that connection was then refused, and the upgrade ended with DatabaseError "not authorized" in place of the revision's error; a revision that went on instead returned with nothing committed and the file behind. Once the upgrade's connection is open, a connect listener now closes and refuses any other, so the statement fails with StatementError wrapping MigrationRefused, and a second connection a revision asks the engine for is refused as well. Tests (tests/db/test_migrations.py): - test_no_statement_reconnects_after_the_connection_is_invalidated (the revision goes on / raises) and test_a_revision_gets_no_second_connection fail at f83a06a and pass here. - The interrupt test also runs SystemExit(3), and test_an_error_read_as_a_disconnect_applies_none covers the disconnect route the env.py comment names. - TestRevisionSources.test_flags_each_rule_on_planted_files runs revision_source, the checker the real revision files go through, and asserts its exact problems and counts; the copy revision_source_problems is removed. - The authorizer tests assert DatabaseError and SQLITE_AUTH instead of the message text. - test_aborts_while_a_tag_table_holds_rows has main's inputs back, including the pr_user_tags row whose tag is missing; the valid-file case is test_aborts_while_a_pull_request_is_tagged_in_a_valid_file. tests/db/test_schema.py's module docstring names the changed copies of the head template among the schemas its readings tell apart. --- alembic/env.py | 20 ++- tests/db/test_migrations.py | 275 ++++++++++++++++++++++++++---------- tests/db/test_schema.py | 3 +- 3 files changed, 225 insertions(+), 73 deletions(-) diff --git a/alembic/env.py b/alembic/env.py index 535c5a1..ed56075 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -36,6 +36,11 @@ SQLite ending one by itself (``INSERT OR ROLLBACK``, ``RAISE(ROLLBACK)``). SQLite authorizes a statement when it prepares it, so the connection keeps no prepared statements: each execute is prepared, and checked, again. +- An upgrade runs on one driver connection, which holds its transaction and the state the + authorizer reads. Once it is open, a connect listener closes and refuses any other before + anything runs on it, so a statement after SQLAlchemy invalidated the connection cannot + reconnect outside both. SQLAlchemy raises that statement's refusal wrapped in + ``StatementError``. - Nothing runs on the connection before ``context.configure()``: alembic would treat the transaction as the caller's and never commit it. - ``legacy_alter_table`` stays off, so batch-altering a table a view reads fails unless the @@ -231,6 +236,17 @@ def configure_connection(dbapi_connection: sqlite3.Connection, record: Connectio raise +def refuse_another_connection(dbapi_connection: sqlite3.Connection, _: ConnectionPoolEntry) -> None: + """Once the upgrade's connection is open, no other is: closed and refused before anything + runs on it. Without this, a statement after SQLAlchemy invalidated the upgrade's connection + and a ``rollback()`` reconnects, outside the transaction and the authorizer's state.""" + dbapi_connection.close() + raise MigrationRefused( + "the upgrade runs on one driver connection: no other is opened, after SQLAlchemy " + "invalidated it or for a second connection to the same engine" + ) + + def begin_immediate(connection: Connection) -> None: """Open the upgrade's transaction, taking the write lock at once.""" connection.exec_driver_sql("BEGIN IMMEDIATE") @@ -265,6 +281,7 @@ def run_migrations_online() -> None: event.listen(engine, "connect", configure_connection, insert=True) event.listen(engine, "begin", begin_immediate) with engine.connect() as connection: + event.listen(engine, "connect", refuse_another_connection, insert=True) context.configure( connection=connection, target_metadata=Base.metadata, @@ -273,7 +290,8 @@ def run_migrations_online() -> None: ) # The mapping the authorizer reads, held before the upgrade runs: SQLAlchemy # invalidates the connection when an interrupt, or an error it reads as a disconnect, - # ends a statement, and Connection.info then raises in place of that exception. + # ends a statement, and Connection.info then raises in place of that exception. No + # other connection is opened, so this is the live connection's mapping. state = connection.info with context.begin_transaction(): try: diff --git a/tests/db/test_migrations.py b/tests/db/test_migrations.py index 48644fc..5f656c9 100644 --- a/tests/db/test_migrations.py +++ b/tests/db/test_migrations.py @@ -26,7 +26,13 @@ from alembic.script import ScriptDirectory from sqlalchemy import create_engine, event from sqlalchemy.engine import Engine -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import ( + DatabaseError, + DBAPIError, + OperationalError, + ProgrammingError, + StatementError, +) from sqlalchemy.pool import Pool from github_activity_db.config import get_settings @@ -94,34 +100,9 @@ "VALUES (?, ?, 'LA_bound')\", (999, {!r}))" ) - -def revision_source_problems(path: Path) -> list[str]: - """Plan 5.1's rules for a revision file: it imports only the standard library, alembic, and - SQLAlchemy (no application code, no relative import), and its ``VIEWS`` is a literal.""" - tree = ast.parse(path.read_bytes()) - problems = [ - f"{path.name}:{line}: imports {module}" - for line, module in imported_modules(path) - if module.split(".")[0] not in REVISION_IMPORTS | sys.stdlib_module_names - ] - problems += [ - f"{path.name}:{node.lineno}: relative import" - for node in ast.walk(tree) - if isinstance(node, ast.ImportFrom) and node.level > 0 - ] - for node in tree.body: - if isinstance(node, ast.Assign): - targets, value = node.targets, node.value - elif isinstance(node, ast.AnnAssign) and node.value is not None: - targets, value = [node.target], node.value - else: - continue - if any(isinstance(t, ast.Name) and t.id == "VIEWS" for t in targets): - try: - ast.literal_eval(value) - except ValueError: - problems.append(f"{path.name}:{node.lineno}: VIEWS is not a literal") - return problems +# A revision's first lines: the driver connection under SQLAlchemy closed, so the next statement +# fails with an error the pysqlite dialect reads as a disconnect. +CLOSED_CONNECTION = "bind = op.get_bind()\nbind.connection.dbapi_connection.close()\n" def snapshot(database: Path) -> tuple[Any, ...]: @@ -149,13 +130,22 @@ def refusal(error: pytest.ExceptionInfo[BaseException]) -> str: return str(error.value) -def sqlite_error_name(error: pytest.ExceptionInfo[OperationalError]) -> str: +def sqlite_error_name(error: pytest.ExceptionInfo[DBAPIError]) -> str: """The SQLite result code of the driver error SQLAlchemy's error wraps.""" driver_error = error.value.orig assert isinstance(driver_error, sqlite3.Error), repr(driver_error) return str(driver_error.sqlite_errorname) +def is_closed(connection: sqlite3.Connection) -> bool: + """Whether the driver connection is closed: a closed one runs nothing.""" + try: + connection.execute("SELECT 1") + except sqlite3.ProgrammingError: + return True + return False + + def foreign_key_rows(database: Path) -> dict[str, int]: """The rows of each table that has a foreign key: what ``PRAGMA foreign_key_check`` reads.""" counted = {} @@ -397,40 +387,51 @@ def test_upgrades_a_scratch_copy_of_the_tracked_file(self, tmp_path: Path) -> No assert digest(TRACKED) == tracked_digest @pytest.mark.parametrize( - ("inserts", "held"), + ("insert", "held"), [ pytest.param( - ( - "INSERT INTO user_tags (name, created_at) " - "VALUES ('keep', '2026-01-01 00:00:00')", - ), + "INSERT INTO user_tags (name, created_at) VALUES ('keep', '2026-01-01 00:00:00')", "user_tags 1, pr_user_tags 0", id="user_tags", ), pytest.param( - ( - "INSERT INTO user_tags (id, name, created_at) " - "VALUES (1, 'keep', '2026-01-01 00:00:00')", - "INSERT INTO user_tags (id, name, created_at) " - "VALUES (2, 'other', '2026-01-01 00:00:00')", - # The template's pull request 1, tagged. - "INSERT INTO pr_user_tags (pr_id, user_tag_id) VALUES (1, 1)", - ), - "user_tags 2, pr_user_tags 1", + # Foreign keys are off on this connection: a link whose tag row is missing. + "INSERT INTO pr_user_tags (pr_id, user_tag_id) VALUES (1, 99)", + "user_tags 0, pr_user_tags 1", id="pr_user_tags", ), ], ) def test_aborts_while_a_tag_table_holds_rows( - self, first_schema: Path, inserts: tuple[str, ...], held: str + self, first_schema: Path, insert: str, held: str ) -> None: + with closing(sqlite3.connect(first_schema)) as connection, connection: + connection.execute(insert) + before = snapshot(first_schema) + with pytest.raises(RuntimeError, match=rf"tag rows would be lost \({held}\)"): + upgrade(first_schema) + assert snapshot(first_schema) == before + + def test_aborts_while_a_pull_request_is_tagged_in_a_valid_file( + self, first_schema: Path + ) -> None: + """Two tags and a link from the template's pull request to one, every reference intact: + the file passes the foreign-key check, and the upgrade refuses it for its tag rows.""" with closing(sqlite3.connect(first_schema)) as connection, connection: connection.execute("PRAGMA foreign_keys=ON") - for insert in inserts: + for insert in ( + "INSERT INTO user_tags (id, name, created_at) " + "VALUES (1, 'keep', '2026-01-01 00:00:00')", + "INSERT INTO user_tags (id, name, created_at) " + "VALUES (2, 'other', '2026-01-01 00:00:00')", + "INSERT INTO pr_user_tags (pr_id, user_tag_id) VALUES (1, 1)", + ): connection.execute(insert) assert query(first_schema, "PRAGMA foreign_key_check") == [] before = snapshot(first_schema) - with pytest.raises(RuntimeError, match=rf"tag rows would be lost \({held}\)"): + with pytest.raises( + RuntimeError, match=r"tag rows would be lost \(user_tags 2, pr_user_tags 1\)" + ): upgrade(first_schema) assert snapshot(first_schema) == before @@ -659,12 +660,20 @@ def table_definitions(database: Path) -> dict[str, str]: class TestOneTransaction: - def test_an_interrupt_after_the_second_revision_applies_none(self, first_schema: Path) -> None: - """An interrupt at a set point inside a statement, once the second of the revisions after - the first schema has run: the interrupt itself ends the upgrade, and the file stays at the - revision the upgrade started from.""" + @pytest.mark.parametrize( + "raised", + [ + pytest.param(KeyboardInterrupt(), id="KeyboardInterrupt"), + pytest.param(SystemExit(3), id="SystemExit"), + ], + ) + def test_an_interrupt_after_the_second_revision_applies_none( + self, first_schema: Path, raised: BaseException + ) -> None: + """An interrupt, or a SystemExit, at a set point inside a statement, once the second of + the revisions after the first schema has run: that exception itself ends the upgrade, + and the file stays at the revision the upgrade started from.""" second = AFTER_THE_FIRST_SCHEMA[1] - raised = KeyboardInterrupt() invalidated: list[BaseException | None] = [] def interrupt(_connection: Any, _cursor: Any, statement: str, *_rest: Any) -> None: @@ -679,16 +688,113 @@ def record(_dbapi_connection: Any, _record: Any, exception: BaseException | None with ( listening("after_cursor_execute", interrupt), listening("invalidate", record, Pool), - pytest.raises(KeyboardInterrupt) as error, + pytest.raises(type(raised)) as error, ): upgrade(first_schema) - assert type(error.value) is KeyboardInterrupt assert error.value is raised # Inside a statement, so SQLAlchemy invalidated the connection for it, once. assert invalidated == [raised] assert snapshot(first_schema) == before assert revision_of(first_schema) == FIRST_SCHEMA_HEAD + def test_an_error_read_as_a_disconnect_applies_none( + self, migrated_database: Path, tmp_path: Path + ) -> None: + """A test-only revision closes its driver connection and runs a statement: SQLAlchemy + reads the driver's error as a disconnect and invalidates the connection, once, and that + error ends the upgrade.""" + versions = tmp_path / "versions" + write_revision( + versions, TEST_REVISION, HEAD_REVISION, f"{CLOSED_CONNECTION}op.execute('SELECT 1')" + ) + invalidated: list[BaseException | None] = [] + + def record(_dbapi_connection: Any, _record: Any, exception: BaseException | None) -> None: + invalidated.append(exception) + + before = snapshot(migrated_database) + with listening("invalidate", record, Pool), pytest.raises(ProgrammingError) as error: + upgrade(migrated_database, test_versions=versions) + assert type(error.value) is ProgrammingError + assert type(error.value.orig) is sqlite3.ProgrammingError + assert error.value.statement == "SELECT 1" + assert error.value.connection_invalidated is True + assert invalidated == [error.value.orig] + assert snapshot(migrated_database) == before + assert revision_of(migrated_database) == HEAD_REVISION + + @pytest.mark.parametrize( + "after", + [ + pytest.param("", id="the revision goes on"), + pytest.param("raise RuntimeError('after the reconnect')", id="the revision raises"), + ], + ) + def test_no_statement_reconnects_after_the_connection_is_invalidated( + self, migrated_database: Path, tmp_path: Path, after: str + ) -> None: + """After SQLAlchemy invalidated the upgrade's connection, a rollback() would let the next + statement reconnect on a second connection, outside the upgrade's transaction and the + state its authorizer reads. env.py refuses that connection and closes it: the statement + fails with the refusal, which SQLAlchemy wraps as it wraps any error raised while it + reconnects, and nothing is applied.""" + versions = tmp_path / "versions" + write_revision( + versions, + TEST_REVISION, + HEAD_REVISION, + f"{CLOSED_CONNECTION}" + "try:\n op.execute('SELECT 1')\nexcept Exception:\n pass\n" + "bind.rollback()\n" + f"op.execute('CREATE TABLE late (x INTEGER)')\n{after}", + ) + connections: list[sqlite3.Connection] = [] + invalidated: list[BaseException | None] = [] + + def connected(dbapi_connection: sqlite3.Connection, _record: Any) -> None: + connections.append(dbapi_connection) + + def record(_dbapi_connection: Any, _record: Any, exception: BaseException | None) -> None: + invalidated.append(exception) + + before = snapshot(migrated_database) + with ( + listening("connect", connected, Pool), + listening("invalidate", record, Pool), + pytest.raises(StatementError) as error, + ): + upgrade(migrated_database, test_versions=versions) + assert type(error.value) is StatementError + assert error.value.statement == "CREATE TABLE late (x INTEGER)" + assert error.value.orig is error.value.__cause__ + assert type(error.value.orig).__name__ == "MigrationRefused" + assert "runs on one driver connection" in str(error.value.orig) + assert [type(exception) for exception in invalidated] == [sqlite3.ProgrammingError] + # The upgrade's connection, closed by the revision, and the refused one, closed by env.py. + assert [is_closed(connection) for connection in connections] == [True, True] + assert snapshot(migrated_database) == before + assert revision_of(migrated_database) == HEAD_REVISION + + def test_a_revision_gets_no_second_connection( + self, migrated_database: Path, tmp_path: Path + ) -> None: + """A second connection a revision asks the engine for would hold no part of the + upgrade's transaction: env.py refuses it before anything runs on it, and closes it.""" + versions = tmp_path / "versions" + write_revision(versions, TEST_REVISION, HEAD_REVISION, "op.get_bind().engine.connect()") + connections: list[sqlite3.Connection] = [] + + def connected(dbapi_connection: sqlite3.Connection, _record: Any) -> None: + connections.append(dbapi_connection) + + before = snapshot(migrated_database) + with listening("connect", connected, Pool), pytest.raises(RuntimeError) as error: + upgrade(migrated_database, test_versions=versions) + assert "runs on one driver connection" in refusal(error) + assert [is_closed(connection) for connection in connections] == [True, True] + assert snapshot(migrated_database) == before + assert revision_of(migrated_database) == HEAD_REVISION + def test_the_chain_commits_a_file_of_seeded_rows(self, step3b1_database: Path) -> None: """The foreign-key check before the upgrade's commit passes a file of valid rows, and each table it reads holds the rows the seed wrote.""" @@ -1058,8 +1164,10 @@ def test_a_revision_cannot_end_the_transaction( versions = tmp_path / "versions" write_revision(versions, TEST_REVISION, HEAD_REVISION, body) before = snapshot(first_schema) - with pytest.raises(Exception, match="not authorized"): + with pytest.raises(DatabaseError) as error: upgrade(first_schema, test_versions=versions) + assert type(error.value) is DatabaseError + assert sqlite_error_name(error) == "SQLITE_AUTH" assert snapshot(first_schema) == before @pytest.mark.parametrize( @@ -1095,8 +1203,10 @@ def test_a_statement_run_before_sqlite_ended_the_transaction_is_refused_after( config = alembic_config(migrated_database, test_versions=versions) config.attributes["database_url"] += url_query before = snapshot(migrated_database) - with pytest.raises(Exception, match="not authorized"): + with pytest.raises(DatabaseError) as error: command.upgrade(config, "head") + assert type(error.value) is DatabaseError + assert sqlite_error_name(error) == "SQLITE_AUTH" assert snapshot(migrated_database) == before @pytest.mark.parametrize( @@ -1126,8 +1236,10 @@ def test_a_revision_runs_only_the_actions_the_authorizer_lists( versions = tmp_path / "versions" write_revision(versions, TEST_REVISION, HEAD_REVISION, f"op.execute({statement!r})") before = snapshot(migrated_database) - with pytest.raises(Exception, match="not authorized"): + with pytest.raises(DatabaseError) as error: upgrade(migrated_database, test_versions=versions) + assert type(error.value) is DatabaseError + assert sqlite_error_name(error) == "SQLITE_AUTH" assert snapshot(migrated_database) == before def test_a_revision_cannot_attach_another_database( @@ -1146,8 +1258,10 @@ def test_a_revision_cannot_attach_another_database( "op.execute('CREATE TABLE other.probe (x INTEGER)')", ) before = snapshot(migrated_database) - with pytest.raises(Exception, match="not authorized"): + with pytest.raises(DatabaseError) as error: upgrade(migrated_database, test_versions=versions) + assert type(error.value) is DatabaseError + assert sqlite_error_name(error) == "SQLITE_AUTH" assert snapshot(migrated_database) == before assert not other.exists() @@ -1632,26 +1746,45 @@ def defines_views(path: Path) -> bool: assert newest.name.startswith(STEP3B1) @pytest.mark.parametrize( - ("source", "flagged"), + ("source", "problems", "imports", "views"), [ - ("import json\nimport sqlalchemy as sa\nfrom alembic import op\n", False), - ("VIEWS = (('v', 'CREATE VIEW v AS SELECT 1'),)\n", False), - ("from typing import Final\nVIEWS: Final = (('v', 'SELECT 1'),)\n", False), - ("from github_activity_db.db import models\n", True), - ("import githubkit\n", True), - ("from . import shared\n", True), - ("import importlib\nimportlib.import_module('github_activity_db')\n", True), - ("VIEWS = SHARED_VIEWS\n", True), - ("VIEWS: tuple = tuple(view for view in ())\n", True), - ("VIEWS = (('v', 'CREATE VIEW v AS SELECT ' + '1'),)\n", True), + ("import json\nimport sqlalchemy as sa\nfrom alembic import op\n", [], 3, 0), + ("VIEWS = (('v', 'CREATE VIEW v AS SELECT 1'),)\n", [], 0, 1), + ("from typing import Final\nVIEWS: Final = (('v', 'SELECT 1'),)\n", [], 1, 1), + ( + "from github_activity_db.db import models\n", + ["1: imports github_activity_db.db", "1: imports github_activity_db.db.models"], + 1, + 0, + ), + ("import githubkit\n", ["1: imports githubkit"], 1, 0), + ("from . import shared\n", ["1: relative import"], 1, 0), + ( + "import importlib\nimportlib.import_module('github_activity_db')\n", + ["2: imports github_activity_db"], + 1, + 0, + ), + ("VIEWS = SHARED_VIEWS\n", ["1: VIEWS is not a literal"], 0, 1), + ("VIEWS: tuple = tuple(view for view in ())\n", ["1: VIEWS is not a literal"], 0, 1), + ( + "VIEWS = (('v', 'CREATE VIEW v AS SELECT ' + '1'),)\n", + ["1: VIEWS is not a literal"], + 0, + 1, + ), ], ) def test_flags_each_rule_on_planted_files( - self, tmp_path: Path, source: str, flagged: bool + self, tmp_path: Path, source: str, problems: list[str], imports: int, views: int ) -> None: + """The checker the real revisions run through, on a file the test writes: the exact + problems, and the import statements and ``VIEWS`` assignments it read.""" path = tmp_path / "f00000000001_planted.py" path.write_text(source) - assert bool(revision_source_problems(path)) is flagged + assert revision_source(path) == RevisionSource( + [f"{path.name}:{problem}" for problem in problems], imports, views + ) def test_a_generated_revision_raises_in_downgrade_and_follows_the_rules(tmp_path: Path) -> None: diff --git a/tests/db/test_schema.py b/tests/db/test_schema.py index bd75e42..cce49cd 100644 --- a/tests/db/test_schema.py +++ b/tests/db/test_schema.py @@ -3,7 +3,8 @@ ``alembic check`` compares neither views nor CHECK constraints, so these tests compare a database migrated to head with one ``metadata.create_all`` builds, structurally, and each view with the newest migration's frozen SQL. Where the readings must tell two schemas apart, the two -are databases the migrations built at neighbouring revisions; a shape no revision has is read +are databases the migrations built at neighbouring revisions, or the head template and a copy of +it that a test changed with its own statements (``_tampered``); a shape no revision has is read from a scratch database of valid SQL. """ From 66fb157126614a81458e50166bb45b1efe170e8f Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 12:21:49 -0600 Subject: [PATCH 20/29] Step 3b-2a-2d: keep main's sync negative tests beside the lock-based ones (D39) Main's bodies return under main's names, each with the input the test built in its own database or a stub that raises: - test_engine: a trigger refusing #2's insert mid-batch, a stub prune that raises RuntimeError or KeyboardInterrupt, an alembic_version edited to 'older', a trigger refusing the close, and a trigger refusing the aborted close. - test_runs: PRAGMA user_version = 1 at the claim. - test_writer: a wrong targets_default_branch an unchanged default branch leaves in place. Exact results where main's asserted part of one: the run's error string, and the HeadCheckError message for the edited revision and user_version. The lock-based and real-revision bodies keep their cases under new names: test_an_item_commit_that_fails_on_a_lock_is_fatal_and_the_next_run_resumes, test_a_run_ending_at_the_prune_s_delete_keeps_step_7_s_halt_reason, test_a_database_at_the_previous_revision_starts_no_run, test_a_close_whose_commit_fails_on_a_lock_leaves_the_row_running_and_exits_1, test_an_interrupted_close_whose_commit_fails_on_a_lock_leaves_the_row_running (the lock case only; main's test holds the second-interrupt case), test_the_claim_refuses_a_database_at_the_previous_revision, and test_an_unchanged_default_branch_sends_no_update_of_pull_requests. test_snapshot: test_writes_nothing and its positive control share one helper, which reads the snapshot, runs the control's UPDATE when given, and commits; both compare the file's bytes, and the control asserts the rows it changed. The reader's statements are 7, one on each of the 7 tables named. The half-consumed-read probe and the constraint failure assert sqlite_errorname (SQLITE_BUSY, SQLITE_CONSTRAINT_NOTNULL) and the cause's class instead of message text. --- tests/sync/test_connection.py | 7 +- tests/sync/test_engine.py | 150 +++++++++++++++++++++++++++++++--- tests/sync/test_runs.py | 11 ++- tests/sync/test_snapshot.py | 63 +++++++------- tests/sync/test_writer.py | 11 +++ 5 files changed, 196 insertions(+), 46 deletions(-) diff --git a/tests/sync/test_connection.py b/tests/sync/test_connection.py index 6fdadad..8bf4970 100644 --- a/tests/sync/test_connection.py +++ b/tests/sync/test_connection.py @@ -8,7 +8,7 @@ import pytest from sqlalchemy import insert, select, text, update -from sqlalchemy.exc import DBAPIError +from sqlalchemy.exc import DBAPIError, IntegrityError from github_activity_db.db.models import SyncRun from github_activity_db.sync import connection as connection_module @@ -205,10 +205,13 @@ async def test_a_reader_takes_no_lock(migrated_database: Path) -> None: async def test_a_constraint_failure_is_a_local_database_error(migrated_database: Path) -> None: run_id = _run(migrated_database) async with open_writer(_url(migrated_database)) as writer: - with pytest.raises(LocalDatabaseError, match="IntegrityError"): + with pytest.raises(LocalDatabaseError) as raised: async with transaction(writer, run_id): await writer.execute(update(SyncRun).values(rest_calls=5)) await writer.execute(insert(SyncRun).values(status="bogus")) async with transaction(writer, run_id): rest_calls = (await writer.execute(select(SyncRun.rest_calls))).scalar_one() + assert isinstance(raised.value.__cause__, IntegrityError) + # The row has no started_at, whose NOT NULL SQLite tests before the status CHECK. + assert _sqlite_errorname(raised.value) == "SQLITE_CONSTRAINT_NOTNULL" assert rest_calls == 0 diff --git a/tests/sync/test_engine.py b/tests/sync/test_engine.py index 0817004..ccfa52f 100644 --- a/tests/sync/test_engine.py +++ b/tests/sync/test_engine.py @@ -1113,6 +1113,29 @@ async def test_a_repository_gone_during_hydration_fails_for_the_rest_of_the_run( async def test_a_database_error_mid_batch_is_fatal_and_the_next_run_resumes( + world: World, +) -> None: + repository, _ = project(world, pulls=2) + world.execute( + "CREATE TRIGGER refuse_second BEFORE INSERT ON pull_requests WHEN NEW.number = 2 " + "BEGIN SELECT RAISE(ABORT, 'refused by the test'); END" + ) + + failed = await world.sync() + assert failed.error == "LocalDatabaseError: IntegrityError: refused by the test" + assert ended(world, failed) == ("failed", 1, {"prebid/Prebid.js": "failed"}) + assert world.numbers(repository) == [1] + run = world.run(failed.run_id) + assert (run["items_planned"], run["items_hydrated"]) == (2, 1) + + world.execute("DROP TRIGGER refuse_second") + resumed = await world.sync() + assert planned(resumed) == [2] + assert world.numbers(repository) == [1, 2] + assert ended(world, resumed) == ("complete", 0, {"prebid/Prebid.js": "complete"}) + + +async def test_an_item_commit_that_fails_on_a_lock_is_fatal_and_the_next_run_resumes( world: World, short_busy_timeout: None ) -> None: """Another process starts reading once the second batch is answered and reads through the @@ -1569,6 +1592,36 @@ async def test_a_run_another_process_aborted_exits_1_and_stays_aborted(world: Wo @pytest.mark.parametrize("ending", ["fatal", "interrupt"]) async def test_a_run_ending_after_step_7_keeps_its_halt_reason( + world: World, monkeypatch: pytest.MonkeyPatch, ending: str +) -> None: + repository, _ = project(world, pulls=26) + await world.sync() + for pull in repository.pulls.values(): + pull.gone = True + + async def failing_prune(*args: Any) -> Any: + if ending == "fatal": + raise RuntimeError("prune failed") + raise KeyboardInterrupt + + monkeypatch.setattr(writer, "prune_repositories", failing_prune) + + if ending == "fatal": + report = await world.sync(prune_untracked=True) + assert ended(world, report) == ("failed", 1, {"prebid/Prebid.js": "failed"}) + else: + with pytest.raises(KeyboardInterrupt): + await world.sync(prune_untracked=True) + assert world.value("SELECT status FROM sync_runs ORDER BY id DESC LIMIT 1") == "aborted" + assert halt_reasons(world) == { + "prebid/Prebid.js": "26 deletion candidates exceed the delete guard's 25 (26 stored " + "rows before the run)" + } + assert len(world.numbers(repository)) == 26 + + +@pytest.mark.parametrize("ending", ["fatal", "interrupt"]) +async def test_a_run_ending_at_the_prune_s_delete_keeps_step_7_s_halt_reason( world: World, short_busy_timeout: None, ending: str ) -> None: """The prune of an untracked repository comes after step 7. Another process that starts @@ -1638,7 +1691,22 @@ async def test_the_head_check_comes_before_the_claim_and_any_request( assert world.scenario.replay.seen == [] -async def test_a_database_not_at_head_starts_no_run(world: World, tmp_path: Path) -> None: +async def test_a_database_not_at_head_starts_no_run(world: World) -> None: + project(world) + world.execute("UPDATE alembic_version SET version_num = 'older'") + + with pytest.raises( + HeadCheckError, match=f"at revision older, not the code's head {HEAD_REVISION}" + ): + await world.sync() + + assert world.rows("SELECT id FROM sync_runs") == [] + assert world.scenario.replay.seen == [] + + +async def test_a_database_at_the_previous_revision_starts_no_run( + world: World, tmp_path: Path +) -> None: """A file the migrations built to the revision before head, as the code before it did.""" project(world) older = tmp_path / "older.db" @@ -2318,7 +2386,25 @@ async def test_the_refresh_looks_up_the_oldest_first(world: World) -> None: assert ended(world, report) == ("complete", 0, {"prebid/Prebid.js": "complete"}) -async def test_a_close_that_fails_leaves_the_row_running_and_exits_1( +async def test_a_close_that_fails_leaves_the_row_running_and_exits_1(world: World) -> None: + project(world) + world.execute( + "CREATE TRIGGER refuse_close BEFORE UPDATE OF status ON sync_runs " + "BEGIN SELECT RAISE(ABORT, 'close refused by the test'); END" + ) + + report = await world.sync() + + run = world.run(report.run_id) + assert (run["status"], run["exit_code"]) == ("running", None) + assert (report.status, report.exit_code) == ("running", 1) + assert report.error == ( + f"the close failed, so run {report.run_id} stays running: LocalDatabaseError: " + "IntegrityError: close refused by the test" + ) + + +async def test_a_close_whose_commit_fails_on_a_lock_leaves_the_row_running_and_exits_1( world: World, short_busy_timeout: None ) -> None: """Another process starts reading at the close's update and reads through the busy timeout: @@ -2751,14 +2837,17 @@ def lookups() -> list[int]: @pytest.mark.parametrize("cleanup", ["second interrupt", "database error"]) async def test_an_interrupted_close_that_does_not_land_leaves_the_row_running( - world: World, short_busy_timeout: None, cleanup: str + world: World, cleanup: str ) -> None: - """A second interrupt stops the interrupted close, and a database error in it (another - process reading from the first interrupt on, through the busy timeout, fails its commit) is - not raised over the interrupt; either way the row stays running for the stale-row rule or + """A second interrupt stops the interrupted close, and a database error in it is not raised + over the interrupt; either way the row stays running for the stale-row rule or --break-lock (5.3).""" project(world) - reader = Reader(world.database) + if cleanup == "database error": + world.execute( + "CREATE TRIGGER refuse_abort BEFORE UPDATE OF status ON sync_runs " + "WHEN NEW.status = 'aborted' BEGIN SELECT RAISE(ABORT, 'refused by the test'); END" + ) interrupted: list[object] = [] def interrupt( @@ -2770,21 +2859,17 @@ def interrupt( if status not in parameters or (status == "aborted" and cleanup != "second interrupt"): return interrupted.append(status) - if cleanup == "database error": - reader.hold() raise KeyboardInterrupt(status) event.listen(Engine, "before_cursor_execute", interrupt) try: - with database_errors() as errors, pytest.raises(KeyboardInterrupt) as raised: + with pytest.raises(KeyboardInterrupt) as raised: await world.sync() finally: event.remove(Engine, "before_cursor_execute", interrupt) - reader.release() expected = ["complete", "aborted"] if cleanup == "second interrupt" else ["complete"] assert interrupted == expected - assert errors == (["SQLITE_BUSY"] if cleanup == "database error" else []) # The interrupt re-raised is the latest one delivered. assert raised.value.args == (expected[-1],) assert world.rows("SELECT status, exit_code FROM sync_runs") == [ @@ -2794,6 +2879,47 @@ def interrupt( await world.sync() +async def test_an_interrupted_close_whose_commit_fails_on_a_lock_leaves_the_row_running( + world: World, short_busy_timeout: None +) -> None: + """Another process reads from the interrupt on, through the busy timeout, so the commit of + the interrupted close fails with SQLITE_BUSY; that database error is not raised over the + interrupt, and the row stays running for the stale-row rule or --break-lock (5.3).""" + project(world) + reader = Reader(world.database) + interrupted: list[object] = [] + + def interrupt( + connection: Any, cursor: Any, statement: str, parameters: Any, *args: Any + ) -> None: + if ( + not statement.startswith("UPDATE sync_runs SET finished_at") + or "complete" not in parameters + or interrupted + ): + return + interrupted.append("complete") + reader.hold() + raise KeyboardInterrupt("complete") + + event.listen(Engine, "before_cursor_execute", interrupt) + try: + with database_errors() as errors, pytest.raises(KeyboardInterrupt) as raised: + await world.sync() + finally: + event.remove(Engine, "before_cursor_execute", interrupt) + reader.release() + + assert interrupted == ["complete"] + assert errors == ["SQLITE_BUSY"] + assert raised.value.args == ("complete",) + (run,) = world.rows("SELECT id, status, exit_code FROM sync_runs") + assert (run["status"], run["exit_code"]) == ("running", None) + with pytest.raises(ClaimRefused) as refused: + await world.sync() + assert refused.value.blocking == (run["id"],) + + def alias_error(alias: str) -> Any: """A batch response change: ``alias`` is null with an error of its own, an item error the hydrator retries alone.""" diff --git a/tests/sync/test_runs.py b/tests/sync/test_runs.py index 1a79c00..034b5b1 100644 --- a/tests/sync/test_runs.py +++ b/tests/sync/test_runs.py @@ -120,7 +120,16 @@ def test_a_url_without_a_file_is_refused(url: str) -> None: database_path(url) -async def test_the_claim_refuses_a_database_not_at_head(tmp_path: Path) -> None: +async def test_the_claim_refuses_a_database_not_at_head(migrated_database: Path) -> None: + connection = sqlite3.connect(migrated_database) + connection.execute("PRAGMA user_version = 1") + connection.close() + with pytest.raises(HeadCheckError, match="user_version is 1, not schema_version 2"): + await _claim(migrated_database) + assert fetch(migrated_database, "SELECT count(*) AS n FROM sync_runs") == [{"n": 0}] + + +async def test_the_claim_refuses_a_database_at_the_previous_revision(tmp_path: Path) -> None: """A file the migrations built to the revision before head, as the code before it did.""" older = tmp_path / "older.db" upgrade(older, MIGRATION_REVISIONS[-2]) diff --git a/tests/sync/test_snapshot.py b/tests/sync/test_snapshot.py index 48875f5..353e711 100644 --- a/tests/sync/test_snapshot.py +++ b/tests/sync/test_snapshot.py @@ -2,14 +2,13 @@ from __future__ import annotations -import hashlib import re import sqlite3 from pathlib import Path from typing import Any, cast import pytest -from sqlalchemy import event, select, update +from sqlalchemy import Executable, event, select, update from sqlalchemy.ext.asyncio import AsyncConnection from github_activity_db.db.engine import build_engine @@ -32,25 +31,6 @@ ) -def _digest(database: Path) -> str: - return hashlib.sha256(database.read_bytes()).hexdigest() - - -async def _commit_after(database: Path, statement: Any | None) -> None: - """Run ``read_snapshot`` or ``statement`` on an application connection, then commit, so a - write either made lands in the file.""" - engine = build_engine(f"sqlite+aiosqlite:///{database}") - try: - async with engine.connect() as connection: - if statement is None: - await read_snapshot(connection) - else: - await connection.execute(statement) - await connection.commit() - finally: - await engine.dispose() - - def _seed(database: Path) -> dict[str, int]: """Two repositories, one with rows in every table the reader reads, one with none.""" with connect(database) as connection: @@ -162,13 +142,15 @@ async def test_an_empty_database(migrated_database: Path) -> None: assert await snapshot_of(migrated_database) == Snapshot(repositories=()) -async def _read_then_commit(database: Path) -> Snapshot: - """Run ``read_snapshot`` on an application connection, then commit, so a write it made - would land in the file.""" +async def _read_then_commit(database: Path, write: Executable | None = None) -> Snapshot: + """Run ``read_snapshot`` on an application connection, then ``write`` when given, then + commit, so a write either made would land in the file.""" engine = build_engine(f"sqlite+aiosqlite:///{database}") try: async with engine.connect() as connection: snapshot = await read_snapshot(connection) + if write is not None: + await connection.execute(write) await connection.commit() finally: await engine.dispose() @@ -235,9 +217,17 @@ def record(connection: Any, cursor: Any, statement: str, *args: Any) -> None: finally: await engine.dispose() tables = [re.search(r"\bFROM (\w+)", statement) for statement in sent] - assert checked.reads == len(sent) > 0 + assert checked.reads == len(sent) == 7 assert all(statement.startswith("SELECT ") for statement in sent) - assert len({table.group(1) for table in tables if table is not None}) == len(sent) + assert sorted(table.group(1) for table in tables if table is not None) == [ + "pr_closing_issues", + "pr_labels", + "pull_requests", + "repo_labels", + "repositories", + "sync_failures", + "sync_pending", + ] async def test_the_lock_probe_refuses_while_another_connection_reads( @@ -260,11 +250,21 @@ async def test_the_lock_probe_refuses_while_another_connection_reads( async def test_the_digest_sees_a_committed_write(migrated_database: Path) -> None: - """Positive control: an UPDATE committed on the same kind of connection changes the file.""" + """Positive control for ``test_writes_nothing``: through the same helper, an UPDATE sent + after the reader's statements is committed, and the file's bytes change.""" _seed(migrated_database) - before = _digest(migrated_database) - await _commit_after(migrated_database, update(PullRequest).values(is_draft=False)) - assert _digest(migrated_database) != before + query = "SELECT number, is_draft FROM pull_requests ORDER BY number" + assert fetch(migrated_database, query) == [ + {"number": 2, "is_draft": 0}, + {"number": 7, "is_draft": 1}, + ] + before = migrated_database.read_bytes() + await _read_then_commit(migrated_database, update(PullRequest).values(is_draft=False)) + assert migrated_database.read_bytes() != before + assert fetch(migrated_database, query) == [ + {"number": 2, "is_draft": 0}, + {"number": 7, "is_draft": 0}, + ] async def test_the_lock_probe_sees_a_half_consumed_read(migrated_database: Path) -> None: @@ -275,8 +275,9 @@ async def test_the_lock_probe_sees_a_half_consumed_read(migrated_database: Path) async with engine.connect() as connection: stream = await connection.stream(select(PullRequest.id)) assert await stream.fetchone() is not None - with pytest.raises(sqlite3.OperationalError, match="database is locked"): + with pytest.raises(sqlite3.OperationalError) as raised: commit_does_not_wait(migrated_database) + assert raised.value.sqlite_errorname == "SQLITE_BUSY" await stream.close() commit_does_not_wait(migrated_database) finally: diff --git a/tests/sync/test_writer.py b/tests/sync/test_writer.py index 4ec7b52..36aa06f 100644 --- a/tests/sync/test_writer.py +++ b/tests/sync/test_writer.py @@ -357,6 +357,17 @@ async def test_a_default_branch_change_recomputes_targets_default_branch(w: Writ async def test_an_unchanged_default_branch_leaves_targets_default_branch(w: Writer) -> None: + """A planted wrong value stays: only a change of default branch recomputes it.""" + await w.repository(repository(default_branch="master")) + await w.item(pull_request(1, 10, base_ref="master")) + with connect(w.database) as connection: + connection.execute(text("UPDATE pull_requests SET targets_default_branch = 0")) + result = await w.repository(repository(default_branch="master")) + assert not result.default_branch_changed + assert w.pr(1)["targets_default_branch"] == 0 + + +async def test_an_unchanged_default_branch_sends_no_update_of_pull_requests(w: Writer) -> None: """Only a change of default branch recomputes it: a write with the same default branch sends no update of the repository's pull requests, whose values stay what their base branches give.""" From f5bf4e2f88856d4ecded7e4c34f5553d19162a4c Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 12:23:44 -0600 Subject: [PATCH 21/29] Step 3b-2a-2d: restore the CLI traceback and head-check tests (D39) - test_cli_app: test_a_traceback_shows_no_local_variables runs a test-only command that raises with a credential-like local through the root app in a child interpreter again, and asserts the local is not printed. A second test does the same for a command in the github group, whose own Typer keeps typer's default of showing locals. The entry-point and setting check is kept as test_the_console_script_calls_the_root_app_which_hides_local_variables. - test_cli_app: the refused .env read is run with a token set and a recording source, and asserts no settings were cached and no source was opened; a second test reads the .env and asserts the source is opened once with the token and the settings are cached. - test_cli_sync: test_a_database_not_at_head_exits_1 edits its own database's alembic_version to an unknown revision again, asserting the exact refusal; the real-revision case is kept as test_a_database_at_the_revision_before_head_exits_1. --- tests/test_cli_app.py | 98 +++++++++++++++++++++++++++++++++++++----- tests/test_cli_sync.py | 16 ++++++- 2 files changed, 103 insertions(+), 11 deletions(-) diff --git a/tests/test_cli_app.py b/tests/test_cli_app.py index 4da96bf..612883c 100644 --- a/tests/test_cli_app.py +++ b/tests/test_cli_app.py @@ -6,13 +6,14 @@ import re import subprocess import sys -from collections.abc import Iterator +from collections.abc import Callable, Iterator from importlib import metadata from pathlib import Path import pydantic_settings.sources.providers.dotenv as dotenv_source import pytest import typer +from click.testing import Result from typer.testing import CliRunner from github_activity_db.cli import common @@ -267,7 +268,44 @@ def test_an_invalid_tracking_config_exits_1_without_a_traceback(self, tmp_path: assert "not valid TOML" in _plain(result.stderr) assert "Traceback" not in result.stderr - def test_a_traceback_shows_no_local_variables(self) -> None: + def test_a_traceback_shows_no_local_variables(self, tmp_path: Path) -> None: + """The real app's pretty traceback, for a test-only command that fails (gap 75).""" + script = ( + "from github_activity_db.cli.app import app\n" + "@app.command()\n" + "def boom() -> None:\n" + " credential = 'sentinel-credential-7f3a'\n" + " raise RuntimeError('boom ' + str(len(credential)))\n" + "app(['boom'], prog_name='ghactivity')\n" + ) + result = _installed(env={}, cwd=tmp_path, script=script) + assert result.returncode == 1 + assert "RuntimeError" in result.stderr + assert "boom 24" in result.stderr + assert "sentinel-credential-7f3a" not in result.stderr + result.stdout + + def test_a_github_group_commands_traceback_shows_no_local_variables( + self, tmp_path: Path + ) -> None: + """A test-only command in the ``github`` group, called through the root app: the group's + own Typer keeps typer's default of showing local variables, and the root app's setting + is the one that applies (gap 75).""" + script = ( + "from github_activity_db.cli import github\n" + "from github_activity_db.cli.app import app\n" + "@github.app.command()\n" + "def boom() -> None:\n" + " credential = 'sentinel-credential-7f3a'\n" + " raise RuntimeError('boom ' + str(len(credential)))\n" + "app(['github', 'boom'], prog_name='ghactivity')\n" + ) + result = _installed(env={}, cwd=tmp_path, script=script) + assert result.returncode == 1 + assert "RuntimeError" in result.stderr + assert "boom 24" in result.stderr + assert "sentinel-credential-7f3a" not in result.stderr + result.stdout + + def test_the_console_script_calls_the_root_app_which_hides_local_variables(self) -> None: """An uncaught exception's traceback is typer's, set up by the app it was called through: the console script calls the root app, whose setting keeps local variables out whatever group the command is in (gap 75). The 2 values compared: the script's target @@ -277,25 +315,65 @@ def test_a_traceback_shows_no_local_variables(self) -> None: assert app.pretty_exceptions_show_locals is False -def test_an_env_file_the_os_refuses_to_read_exits_1( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """A .env the OS refuses to read: the read answers PermissionError, as it does for a file - the user may not read, whatever uid runs the tests (D40), and the command exits 1.""" +def _env_file_command( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, read_env: Callable[..., dict[str, str | None]] +) -> tuple[Result, list[Path], list[str], int]: + """``github rate-limit`` with a token set and a .env whose read ``read_env`` answers: the + result, the paths read, the tokens a source was opened with, and the settings cached when + the command returned. The source opens nothing: it records the token and raises.""" (tmp_path / ".env").write_text("LOG_LEVEL=INFO\n") read: list[Path] = [] + opened: list[str] = [] - def refused(path: Path, **_: object) -> dict[str, str | None]: + def reading(path: Path, **kwargs: object) -> dict[str, str | None]: read.append(Path(path)) - raise PermissionError(13, "Permission denied", str(path)) + return read_env(path, **kwargs) + + def open_source(token: str, **_: object) -> object: + opened.append(token) + raise RuntimeError("the test's source opens nothing") monkeypatch.chdir(tmp_path) - monkeypatch.setattr(dotenv_source, "dotenv_values", refused) + monkeypatch.setenv("GITHUB_TOKEN", "test-token") + monkeypatch.setattr(common, "open_source", open_source) + monkeypatch.setattr(dotenv_source, "dotenv_values", reading) get_settings.cache_clear() try: result = runner.invoke(app, ["github", "rate-limit"]) + cached = get_settings.cache_info().currsize finally: get_settings.cache_clear() + return result, read, opened, cached + + +def test_an_env_file_the_os_refuses_to_read_exits_1( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A .env the OS refuses to read: the read answers PermissionError, as it does for a file + the user may not read, whatever uid runs the tests (D40), and the command exits 1 at the + settings, with a token set: no settings are cached and no source is opened.""" + + def refused(path: Path, **_: object) -> dict[str, str | None]: + raise PermissionError(13, "Permission denied", str(path)) + + result, read, opened, cached = _env_file_command(monkeypatch, tmp_path, refused) + assert read == [Path(".env")] + assert (opened, cached) == ([], 0) + assert result.exit_code == 1, result.output + assert isinstance(result.exception, SystemExit), result.exception + + +def test_an_env_file_the_os_reads_reaches_the_source( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The same command line with the .env read: the settings are cached and the command opens + its source once, with the token (which here raises, so the command exits 1).""" + + def readable(path: Path, **_: object) -> dict[str, str | None]: + return {"LOG_LEVEL": "INFO"} + + result, read, opened, cached = _env_file_command(monkeypatch, tmp_path, readable) assert read == [Path(".env")] + assert (opened, cached) == (["test-token"], 1) assert result.exit_code == 1, result.output assert isinstance(result.exception, SystemExit), result.exception diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index 832f7b6..03f1b7b 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -216,7 +216,21 @@ def test_a_missing_database_file_exits_1_and_creates_none( assert replay.seen == [] -def test_a_database_not_at_head_exits_1( +def test_a_database_not_at_head_exits_1(replay: Replay, migrated_database: Path) -> None: + _execute(migrated_database, "UPDATE alembic_version SET version_num = 'older'") + + result = _sync() + + assert result.exit_code == 1 + assert ( + "Sync did not start: the database is at revision older, not the code's head " + f"{HEAD_REVISION}" in _plain(result.output) + ) + assert _runs(migrated_database) == [] + assert replay.seen == [] + + +def test_a_database_at_the_revision_before_head_exits_1( replay: Replay, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: """A file the migrations built to the revision before head, as the code before it did.""" From 9cd9d7a3ed45bc964e7bbf47201d124efb6eb602 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 12:54:23 -0600 Subject: [PATCH 22/29] Step 3b-2a-2d: keep main's checker tests beside the history-based ones; record the history copies' blob ids Main's tests whose input the test built (edited lock entries, a relaxed checker copy in the pull request, written databases, installed sample distributions, non-directive comments) are back under main's names; the bodies that read tests/fixtures/history keep their own names: - test_check_query_lock: test_a_removed_entry_fails, TestSchemaLock's test_a_removed_entry_fails, test_a_pair_removed_from_an_object_entry_fails, test_both_locks_are_checked_by_default, test_lock_selects_one, test_the_step_fails_a_pair_removed_from_a_schema_entry, and test_the_step_runs_the_base_checker_not_the_pull_requests. The history-based refusals assert the exact problem lines. - test_count_suppressions: NOT_SUPPRESSIONS and test_directive_text_outside_a_directive_comment_is_not_counted; the docstring case is test_the_scripts_own_docstring_is_not_counted. The src cap test runs noqa only and refuses an empty site list. - test_data_dictionary: the in-memory test_differences_are_reported_both_ways. - test_declared_dependencies: test_a_name_several_distributions_install_needs_all_of_them_declared over distributions written to a temporary site directory. The empty-input refusals of the alias, placeholder, and suppression checkers assert their one stderr line. The import-boundary scans assert their site counts (5 and 20). test_config keeps TestDocumentedDatabaseUrl's node id and drops the identical TestDefaultDatabaseUrl test. tests/fixtures/history/README.md records each copy's git blob id, and tests/test_history_fixtures.py recomputes it for every tracked copy. --- tests/fixtures/history/README.md | 34 +++-- tests/test_check_any_aliases.py | 4 +- tests/test_check_log_placeholders.py | 4 +- tests/test_check_query_lock.py | 125 +++++++++++++++-- tests/test_config.py | 12 +- tests/test_count_suppressions.py | 40 ++++-- tests/test_data_dictionary.py | 16 +++ tests/test_declared_dependencies.py | 67 ++++++++- tests/test_history_fixtures.py | 202 +++++++++++++++++++++++++++ tests/test_import_boundaries.py | 15 +- 10 files changed, 468 insertions(+), 51 deletions(-) create mode 100644 tests/test_history_fixtures.py diff --git a/tests/fixtures/history/README.md b/tests/fixtures/history/README.md index 2a5c54d..12b863a 100644 --- a/tests/fixtures/history/README.md +++ b/tests/fixtures/history/README.md @@ -5,16 +5,24 @@ Each file under `/` is `git show :` copied unchanged to (redesign plan D39). CI checks out with depth 1 and has no history, so tests read these copies instead of git history. -| File | Source commit | Read by | -|------|---------------|---------| -| `21a4267/docs/data-dictionary.md` | `21a4267adc84e69327f2795d6f045ea5514881b1` | `tests/test_data_dictionary.py` | -| `77663a7/docs/data-dictionary.md` | `77663a7a1039fbf1892fa7fc42ef3ecefa55ea5f` | `tests/test_data_dictionary.py` | -| `c1f474b/tests/query_versions.json` | `c1f474bf86117a38a5b6c45dd5808cf99e3f0da0` | `tests/test_check_query_lock.py` | -| `21a4267/tests/query_versions.json` | `21a4267adc84e69327f2795d6f045ea5514881b1` | `tests/test_check_query_lock.py` | -| `6fedb05/tests/schema_versions.json` | `6fedb05fde7bd26cc79b0a7eb2dd91d020655a58` | `tests/test_check_query_lock.py` | -| `21a4267/tests/schema_versions.json` | `21a4267adc84e69327f2795d6f045ea5514881b1` | `tests/test_check_query_lock.py`, `tests/test_data_dictionary.py` | -| `77663a7/tests/schema_versions.json` | `77663a7a1039fbf1892fa7fc42ef3ecefa55ea5f` | `tests/test_check_query_lock.py`, `tests/test_data_dictionary.py` | -| `ea7d12e/pyproject.toml` | `ea7d12e798fefe1971d1b409523a87f4d8caaf17`, the parent of the commit that declared httpx (gap 76) | `tests/test_declared_dependencies.py` | -| `c1f474b/tests/fixtures/source/recorded/pull_request/prebid__Prebid.js/13481.json` | `c1f474bf86117a38a5b6c45dd5808cf99e3f0da0` | `tests/source/test_recordings.py` | -| `c1f474b/tests/fixtures/source/recorded/overflow/PR_kwDOBUaT8s6WsAjJ/comments/0e3c0d53c5b8c0eb.json` | `c1f474bf86117a38a5b6c45dd5808cf99e3f0da0` | `tests/source/test_recordings.py` | -| `c1f474b/tests/fixtures/source/recorded/resolve/prebid__go-gdpr.json` | `c1f474bf86117a38a5b6c45dd5808cf99e3f0da0` | `tests/source/test_recordings.py` | +The Blob column is the file's git blob id: the SHA-1 of `blob `, a NUL byte, and the file's +bytes. `tests/test_history_fixtures.py` recomputes it from each copy and compares it with the row, +and requires exactly one row for every file git tracks under this directory other than this +README. With history, `git ls-tree ` prints the mode, `blob`, the blob id +the source commit holds for ``, and the path; the copy is unchanged when that id is the +row's. For example, `git ls-tree ea7d12e798fefe1971d1b409523a87f4d8caaf17 pyproject.toml` prints +the blob id `d865ae0d2390d2dbdf626f4f91fca7c1034045f7`, the row of `ea7d12e/pyproject.toml`. + +| File | Source commit | Blob | Read by | +|------|---------------|------|---------| +| `21a4267/docs/data-dictionary.md` | `21a4267adc84e69327f2795d6f045ea5514881b1` | `c4e778a39ab2e98dc0ece20d27b80252dccd284c` | `tests/test_data_dictionary.py` | +| `77663a7/docs/data-dictionary.md` | `77663a7a1039fbf1892fa7fc42ef3ecefa55ea5f` | `f639e61164fde84407cc88049b8e227a1772b171` | `tests/test_data_dictionary.py` | +| `c1f474b/tests/query_versions.json` | `c1f474bf86117a38a5b6c45dd5808cf99e3f0da0` | `6269da133a613b36d1c419994f675be54839786a` | `tests/test_check_query_lock.py` | +| `21a4267/tests/query_versions.json` | `21a4267adc84e69327f2795d6f045ea5514881b1` | `bd342e4e9242249ebff54fc1a8c7838f2a6fb8f2` | `tests/test_check_query_lock.py` | +| `6fedb05/tests/schema_versions.json` | `6fedb05fde7bd26cc79b0a7eb2dd91d020655a58` | `30f862a867e675a17f76a7ad9ab0b54fee408de7` | `tests/test_check_query_lock.py` | +| `21a4267/tests/schema_versions.json` | `21a4267adc84e69327f2795d6f045ea5514881b1` | `067e5a8c6ec870ce6da995afa4a9f8d651f4f96e` | `tests/test_check_query_lock.py`, `tests/test_data_dictionary.py` | +| `77663a7/tests/schema_versions.json` | `77663a7a1039fbf1892fa7fc42ef3ecefa55ea5f` | `bc40c9c73ddce85f01a11dbc177fd2f0306e5fb1` | `tests/test_check_query_lock.py`, `tests/test_data_dictionary.py` | +| `ea7d12e/pyproject.toml` | `ea7d12e798fefe1971d1b409523a87f4d8caaf17`, the parent of the commit that declared httpx (gap 76) | `d865ae0d2390d2dbdf626f4f91fca7c1034045f7` | `tests/test_declared_dependencies.py` | +| `c1f474b/tests/fixtures/source/recorded/pull_request/prebid__Prebid.js/13481.json` | `c1f474bf86117a38a5b6c45dd5808cf99e3f0da0` | `71524134cbf78e867460ee4eff36995bb506eda1` | `tests/source/test_recordings.py` | +| `c1f474b/tests/fixtures/source/recorded/overflow/PR_kwDOBUaT8s6WsAjJ/comments/0e3c0d53c5b8c0eb.json` | `c1f474bf86117a38a5b6c45dd5808cf99e3f0da0` | `5314ac3ff708182c8b2a6005c0d1e8a8e0b9d733` | `tests/source/test_recordings.py` | +| `c1f474b/tests/fixtures/source/recorded/resolve/prebid__go-gdpr.json` | `c1f474bf86117a38a5b6c45dd5808cf99e3f0da0` | `e77b1704c546ac63854505b882f133cb77445e1e` | `tests/source/test_recordings.py` | diff --git a/tests/test_check_any_aliases.py b/tests/test_check_any_aliases.py index 43eb24b..9097df0 100644 --- a/tests/test_check_any_aliases.py +++ b/tests/test_check_any_aliases.py @@ -183,8 +183,10 @@ def test_no_python_files_is_an_error(tmp_path: Path) -> None: (tmp_path / "empty").mkdir() result = _run(tmp_path / "empty") assert result.returncode == 1 - # A run that finds an alias also exits 1, but only after reporting what it scanned. + # A run that finds an alias also exits 1, but only after reporting what it scanned; an + # uncaught exception would print a traceback instead of the one refusal line. assert result.stdout == "" + assert result.stderr.splitlines() == [f"no Python files under {tmp_path / 'empty'}"] @pytest.mark.parametrize("source", ALIASES) diff --git a/tests/test_check_log_placeholders.py b/tests/test_check_log_placeholders.py index b1b0b8c..7dcc834 100644 --- a/tests/test_check_log_placeholders.py +++ b/tests/test_check_log_placeholders.py @@ -74,8 +74,10 @@ def test_no_python_files_is_an_error(tmp_path: Path) -> None: (tmp_path / "empty").mkdir() result = _run(tmp_path / "empty") assert result.returncode == 1 - # A run that finds a site also exits 1, but only after reporting what it scanned. + # A run that finds a site also exits 1, but only after reporting what it scanned; an + # uncaught exception would print a traceback instead of the one refusal line. assert result.stdout == "" + assert result.stderr.splitlines() == [f"no Python files under {tmp_path / 'empty'}"] def test_src_has_no_percent_style_messages() -> None: diff --git a/tests/test_check_query_lock.py b/tests/test_check_query_lock.py index a2cab40..c0ed9b8 100644 --- a/tests/test_check_query_lock.py +++ b/tests/test_check_query_lock.py @@ -1,7 +1,8 @@ """The append-only check on tests/query_versions.json and tests/schema_versions.json. -A lock a pull request changes or removes entries of is a real earlier version of the lock file -(tests/fixtures/history), placed over a base that holds a later one. +A lock a pull request changes or removes entries of is either written by the test (lock entries +it edits, removes, or adds, over a base lock it also writes) or a real earlier version of the lock +file (tests/fixtures/history), placed over a base that holds a later one. """ from __future__ import annotations @@ -118,6 +119,24 @@ def kept(lock: str, before: str, now: str) -> str: ) +def removed_entry(key: str) -> str: + """The line the checker prints for a query lock entry the pull request removed.""" + return f"{LOCK}: {key} was removed; lock entries are append-only" + + +def removed_version(version: str) -> str: + """The line the checker prints for a schema version the pull request removed.""" + return f"{SCHEMA_LOCK}: schema version {version} was removed; lock entries are append-only" + + +def lock_lines(stderr: str) -> list[str]: + """The lines of ``stderr`` that name a lock, one per problem the checker found; the CI step's + stderr also holds git's own output.""" + return [ + line for line in stderr.splitlines() if line.startswith((f"{LOCK}: ", f"{SCHEMA_LOCK}: ")) + ] + + def test_a_new_entry_passes(repo: Path) -> None: write_lock(repo, {"pull_request:1": "a" * 64}) commit(repo) @@ -146,13 +165,22 @@ def test_each_change_the_history_made_passes(repo: Path, lock: str, before: str, def test_a_removed_entry_fails(repo: Path) -> None: + write_lock(repo, {"pull_request:1": "a" * 64, "pull_request:2": "b" * 64}) + commit(repo) + write_lock(repo, {"pull_request:2": "b" * 64}) + result = check(repo) + assert result.returncode == 1 + assert "pull_request:1 was removed" in result.stderr + + +def test_an_earlier_query_lock_over_a_later_base_fails(repo: Path) -> None: place(repo, LOCK, QUERY_LOCKS[1]) commit(repo) place(repo, LOCK, QUERY_LOCKS[0]) result = check(repo) assert result.returncode == 1 (removed,) = entries_at(QUERY_LOCKS[1], LOCK).keys() - entries_at(QUERY_LOCKS[0], LOCK).keys() - assert f"{removed} was removed" in result.stderr + assert result.stderr.splitlines() == [removed_entry(removed)] def test_a_lock_absent_at_the_base_passes(repo: Path) -> None: @@ -174,8 +202,9 @@ def test_a_deleted_lock_removes_every_entry(repo: Path) -> None: (repo / LOCK).unlink() result = check(repo) assert result.returncode == 1 - for key in entries_at(QUERY_LOCKS[0], LOCK): - assert f"{key} was removed" in result.stderr + entries = sorted(entries_at(QUERY_LOCKS[0], LOCK)) + assert entries + assert result.stderr.splitlines() == [removed_entry(key) for key in entries] @pytest.mark.parametrize("text", ["{not json", "[1, 2]", '{"pull_request:1": 1}']) @@ -212,7 +241,7 @@ def test_the_verdict_does_not_depend_on_how_the_checker_runs( result = run_checker(repo, "--base", "main", cwd=repo / directory, isolated=isolated) assert result.returncode == 1, result.stdout + result.stderr (removed,) = entries_at(QUERY_LOCKS[1], LOCK).keys() - entries_at(QUERY_LOCKS[0], LOCK).keys() - assert f"{removed} was removed" in result.stderr + assert result.stderr.splitlines() == [removed_entry(removed)] @pytest.mark.parametrize( @@ -321,6 +350,14 @@ def test_an_added_version_passes(self, repo: Path) -> None: assert "tests/schema_versions.json: 1 entries at main kept; 2 entries now" in result.stdout def test_a_removed_entry_fails(self, repo: Path) -> None: + write_schema_lock(repo, {**V1, "2": V1["1"]}) + commit(repo) + write_schema_lock(repo, {"2": V1["1"]}) + result = check(repo) + assert result.returncode == 1 + assert "schema version 1 was removed" in result.stderr + + def test_an_earlier_schema_lock_without_the_newest_version_fails(self, repo: Path) -> None: place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[1]) commit(repo) place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[0]) @@ -330,7 +367,7 @@ def test_a_removed_entry_fails(self, repo: Path) -> None: entries_at(SCHEMA_LOCKS[1], SCHEMA_LOCK).keys() - entries_at(SCHEMA_LOCKS[0], SCHEMA_LOCK).keys() ) - assert f"schema version {removed} was removed" in result.stderr + assert result.stderr.splitlines() == [removed_version(removed)] def test_a_deleted_lock_removes_every_entry(self, repo: Path) -> None: place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[0]) @@ -338,10 +375,19 @@ def test_a_deleted_lock_removes_every_entry(self, repo: Path) -> None: (repo / SCHEMA_LOCK).unlink() result = check(repo) assert result.returncode == 1 - for version in entries_at(SCHEMA_LOCKS[0], SCHEMA_LOCK): - assert f"schema version {version} was removed" in result.stderr + versions = sorted(entries_at(SCHEMA_LOCKS[0], SCHEMA_LOCK), key=int) + assert versions + assert result.stderr.splitlines() == [removed_version(version) for version in versions] def test_a_pair_removed_from_an_object_entry_fails(self, repo: Path) -> None: + write_schema_lock(repo, V2) + commit(repo) + write_schema_lock(repo, {**V2, "2": {**V2["2"], "columns": [["pull_requests", "number"]]}}) + result = check(repo) + assert result.returncode == 1 + assert "schema version 2 lost v_pr_reviews.state" in result.stderr + + def test_an_earlier_schema_lock_without_a_newer_pair_fails(self, repo: Path) -> None: place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[2]) commit(repo) place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[1]) @@ -350,7 +396,10 @@ def test_a_pair_removed_from_an_object_entry_fails(self, repo: Path) -> None: before = entries_at(SCHEMA_LOCKS[2], SCHEMA_LOCK)["2"]["columns"] now = entries_at(SCHEMA_LOCKS[1], SCHEMA_LOCK)["2"]["columns"] ((table, column),) = {tuple(pair) for pair in before} - {tuple(pair) for pair in now} - assert f"schema version 2 lost {table}.{column}" in result.stderr + assert result.stderr.splitlines() == [ + f"{SCHEMA_LOCK}: schema version 2 lost {table}.{column}; removing or renaming a " + "consumer column needs a new schema version" + ] def test_a_pair_added_to_the_newest_object_entry_passes(self, repo: Path) -> None: write_schema_lock(repo, V2) @@ -476,6 +525,17 @@ def test_a_key_listed_twice_is_an_error_not_a_pass( assert "is not JSON" not in result.stderr def test_both_locks_are_checked_by_default(self, repo: Path) -> None: + write_lock(repo, {"pull_request:1": "a" * 64}) + write_schema_lock(repo, V1) + commit(repo) + write_lock(repo, {"pull_request:1": "c" * 64}) + write_schema_lock(repo, {"1": []}) + result = check(repo) + assert result.returncode == 1 + assert "pull_request:1 changed" in result.stderr + assert "schema version 1 lost pull_requests.number" in result.stderr + + def test_both_earlier_locks_are_checked_by_default(self, repo: Path) -> None: place(repo, LOCK, QUERY_LOCKS[1]) place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[2]) commit(repo) @@ -487,6 +547,15 @@ def test_both_locks_are_checked_by_default(self, repo: Path) -> None: assert [line.split(":")[0] for line in stderr] == [LOCK, SCHEMA_LOCK] def test_lock_selects_one(self, repo: Path) -> None: + write_lock(repo, {"pull_request:1": "a" * 64}) + write_schema_lock(repo, V1) + commit(repo) + write_lock(repo, {"pull_request:1": "c" * 64}) + result = run_checker(repo, "--base", "main", "--lock", SCHEMA_LOCK) + assert result.returncode == 0, result.stderr + assert "query_versions" not in result.stdout + result.stderr + + def test_lock_selects_one_of_two_earlier_locks(self, repo: Path) -> None: place(repo, LOCK, QUERY_LOCKS[1]) place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[2]) commit(repo) @@ -752,16 +821,32 @@ def put_back_the_earlier_query_lock(work: Path) -> None: place(work, LOCK, QUERY_LOCKS[0]) +def edit_entry_and_relax_checker(work: Path) -> None: + edit_entry(work) + checker = work / "scripts" / "check_query_lock.py" + checker.write_text("raise SystemExit(0)\n") + + def test_the_step_fails_a_removed_entry(tmp_path: Path, origin: Path) -> None: work = work_clone(tmp_path, origin, "work") push_main(work, checker=True, history=LATER_LOCKS) result = run_step_on_merge(tmp_path, origin, work, put_back_the_earlier_query_lock) assert result.returncode == 1, result.stdout + result.stderr (removed,) = entries_at(QUERY_LOCKS[1], LOCK).keys() - entries_at(QUERY_LOCKS[0], LOCK).keys() - assert f"{removed} was removed" in result.stderr + assert lock_lines(result.stderr) == [removed_entry(removed)] def test_the_step_runs_the_base_checker_not_the_pull_requests(tmp_path: Path, origin: Path) -> None: + work = work_clone(tmp_path, origin, "work") + push_main(work, checker=True, entries={"pull_request:1": "a" * 64}) + result = run_step_on_merge(tmp_path, origin, work, edit_entry_and_relax_checker) + assert result.returncode == 1, result.stdout + result.stderr + assert "runner-temp/check_query_lock.py" in result.stdout + + +def test_the_step_runs_the_base_checker_where_the_pull_request_deletes_its_copy( + tmp_path: Path, origin: Path +) -> None: """The pull request also deletes its copy of the checker, which the step would then fail to open, with another exit code than the base copy's verdict.""" work = work_clone(tmp_path, origin, "work") @@ -806,6 +891,19 @@ def add_lock(target: Path) -> None: def test_the_step_fails_a_pair_removed_from_a_schema_entry(tmp_path: Path, origin: Path) -> None: + work = work_clone(tmp_path, origin, "work") + write_schema_lock(work, V1) + push_main(work, checker=True, entries={"pull_request:1": "a" * 64}) + + def remove_pair(target: Path) -> None: + write_schema_lock(target, {"1": V1["1"][:1]}) + + result = run_step_on_merge(tmp_path, origin, work, remove_pair) + assert result.returncode == 1, result.stdout + result.stderr + assert "schema version 1 lost v_pr_reviewers.final_verdict" in result.stderr + + +def test_the_step_fails_an_earlier_schema_lock(tmp_path: Path, origin: Path) -> None: work = work_clone(tmp_path, origin, "work") push_main(work, checker=True, history=LATER_LOCKS) @@ -814,7 +912,10 @@ def put_back_the_earlier_schema_lock(target: Path) -> None: result = run_step_on_merge(tmp_path, origin, work, put_back_the_earlier_schema_lock) assert result.returncode == 1, result.stdout + result.stderr - assert "schema version 2 lost sync_failures.last_run_id" in result.stderr + assert lock_lines(result.stderr) == [ + f"{SCHEMA_LOCK}: schema version 2 lost sync_failures.last_run_id; removing or renaming " + "a consumer column needs a new schema version" + ] def test_the_step_fails_without_a_merge_commit(tmp_path: Path, origin: Path) -> None: diff --git a/tests/test_config.py b/tests/test_config.py index 8451de4..93348ca 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -125,8 +125,9 @@ def test_removed_pipeline_settings_are_gone(self): assert fields.isdisjoint({"rate_limit", "pacing", "sync"}) -class TestDefaultDatabaseUrl: - """The default DATABASE_URL names the local file, never the tracked one (D26).""" +class TestDocumentedDatabaseUrl: + """The field's default DATABASE_URL, and the one a .env copied from .env.example gives, name + the local file, never the tracked one (D26).""" def test_the_field_default_is_the_local_database(self) -> None: assert Settings.model_fields["database_url"].default == LOCAL_DATABASE_URL @@ -167,10 +168,3 @@ def test_get_settings_cached(self, no_env_file: None): # Should be the same object (cached) assert settings1 is settings2 - - -class TestDocumentedDatabaseUrl: - """The documented DATABASE_URL lines show the default, which names the local file (D26).""" - - def test_the_field_default_is_the_local_database(self) -> None: - assert Settings.model_fields["database_url"].default == LOCAL_DATABASE_URL diff --git a/tests/test_count_suppressions.py b/tests/test_count_suppressions.py index 678d0f5..3a64242 100644 --- a/tests/test_count_suppressions.py +++ b/tests/test_count_suppressions.py @@ -76,6 +76,16 @@ def _package(tmp_path: Path, *sources: str) -> Path: # A re-export of typing's no_type_check, as mypy follows it. "from pkg.compat import no_type_check as skip\n\n\n@skip\ndef f() -> int:\n return 'x'\n", ] +NOT_SUPPRESSIONS = [ + 'TEXT = "# noqa: F401, # type: ignore[misc], # ruff: noqa, # mypy: ignore-errors"\n', + '"""Docstring.\n\n# noqa\n# type: ignore\n"""\n', + "# this module needs no noqa and no type checking exceptions\nimport os\n", + "import os\n# isort: on\n", + 'RANGE = "# ruff: disable[F401]"\nimport os\n', + "# ruff: enable[F401]\nimport os\n", + "# ruff: disabled checks are listed in pyproject.toml\nimport os\n", + "def no_type_check() -> None:\n pass\n\n\nno_type_check()\n", +] REFUSED = [ pytest.param( @@ -210,23 +220,24 @@ def test_src_is_within_the_hook_caps() -> None: assert _counts(result) == (len(sites["type: ignore"]), len(sites["noqa"])) -@pytest.mark.parametrize( - ("kind", "cap"), [("type: ignore", "max_type_ignore"), ("noqa", "max_noqa")] -) -def test_at_the_cap_passes_and_one_below_fails(kind: str, cap: str) -> None: +def test_src_at_its_noqa_count_passes_and_one_below_lists_each_site() -> None: + """src's noqa sites, read a second way, are the ones the script lists when the cap is one + below their count. src holds no type: ignore site, so that cap is run on written sites + (test_at_the_cap_passes_and_one_more_fails).""" sites = _second_reading(SRC) + assert sites["noqa"], "src holds no noqa site to list" at_cap = {"max_type_ignore": len(sites["type: ignore"]), "max_noqa": len(sites["noqa"])} passing = _run(SRC, **at_cap) assert passing.returncode == 0, passing.stdout + assert _counts(passing) == (len(sites["type: ignore"]), len(sites["noqa"])) - # One below the count; -1 where src holds no site of the kind. - over_cap = _run(SRC, **{**at_cap, cap: at_cap[cap] - 1}) + over_cap = _run(SRC, **{**at_cap, "max_noqa": len(sites["noqa"]) - 1}) assert over_cap.returncode == 1 listed = [line.strip() for line in over_cap.stdout.splitlines() if line.startswith(" ")] - assert sorted(listed) == sorted(sites[kind]) + assert sorted(listed) == sorted(sites["noqa"]) -def test_directive_text_outside_a_directive_comment_is_not_counted() -> None: +def test_the_scripts_own_docstring_is_not_counted() -> None: # The script's docstring spells the directives it counts; text in a string is not a comment. strings = [ token.string @@ -275,8 +286,10 @@ def test_no_python_files_is_an_error(tmp_path: Path) -> None: (tmp_path / "empty").mkdir() result = _run(tmp_path / "empty") assert result.returncode == 1 - # A run over files reports what it scanned before any cap fails it. + # A run over files reports what it scanned before any cap fails it; an uncaught exception + # would print a traceback instead of the one refusal line. assert result.stdout == "" + assert result.stderr.splitlines() == [f"no Python files under {tmp_path / 'empty'}"] @pytest.mark.parametrize("source", NOQA_SPELLINGS) @@ -289,6 +302,15 @@ def test_counts_every_type_ignore_spelling(tmp_path: Path, source: str) -> None: assert _counts(_run(_package(tmp_path, source))) == (1, 0) +@pytest.mark.parametrize("source", NOT_SUPPRESSIONS) +def test_directive_text_outside_a_directive_comment_is_not_counted( + tmp_path: Path, source: str +) -> None: + result = _run(_package(tmp_path, source)) + assert _scanned(result) == 1 + assert _counts(result) == (0, 0) + + def test_counts_stub_files(tmp_path: Path) -> None: root = tmp_path / "pkg" root.mkdir() diff --git a/tests/test_data_dictionary.py b/tests/test_data_dictionary.py index a715427..2b5b952 100644 --- a/tests/test_data_dictionary.py +++ b/tests/test_data_dictionary.py @@ -206,7 +206,23 @@ def test_each_commit_s_dictionary_matched_its_schema( assert differences(dictionary_at(AT_STEP3B1), revisions[STEP3B1]) == [] assert differences(dictionary_at(AT_LAST_RUN_ID), revisions[LAST_RUN_ID]) == [] + @pytest.mark.parametrize( + ("database", "expected"), + [ + ({"t": ["id", "name"], "v_t": ["name"]}, []), + ({"t": ["id", "name", "added"], "v_t": ["name"]}, ["no entry for t.added"]), + ({"t": ["id"], "v_t": ["name"]}, ["t.name is not at head"]), + ({"t": ["id", "name"], "v_t": ["name"], "v_new": ["x"]}, ["no entry for v_new"]), + ({"t": ["id", "name"]}, ["v_t is not at head"]), + ], + ids=["equal", "column at head only", "entry only", "view at head only", "view entry only"], + ) def test_differences_are_reported_both_ways( + self, database: dict[str, list[str]], expected: list[str] + ) -> None: + assert differences(read_dictionary(ENTRY), database) == expected + + def test_a_commit_s_dictionary_against_another_revision_s_database_differs( self, revisions: dict[str, dict[str, list[str]]], head_template: Path ) -> None: """Each commit's dictionary against another revision's database: a column only one of diff --git a/tests/test_declared_dependencies.py b/tests/test_declared_dependencies.py index ae6305e..44beee0 100644 --- a/tests/test_declared_dependencies.py +++ b/tests/test_declared_dependencies.py @@ -156,7 +156,7 @@ def test_reports_what_an_earlier_pyproject_did_not_declare() -> None: pytest.param(None, (), id="no installed distribution"), ], ) -def test_a_name_several_distributions_install_needs_all_of_them_declared( +def test_src_s_imports_of_a_name_several_distributions_install_need_all_of_them_declared( installers: list[str] | None, reported: tuple[str, ...] | None ) -> None: # The installed environment's answer, with sqlalchemy installed by the distributions given, @@ -209,3 +209,68 @@ def test_reports_undeclared_imports_in_a_planted_package(tmp_path: Path) -> None Undeclared("planted/b.py", "httpx", ("httpx",)), Undeclared("planted/b.py", "not_an_installed_module_xyz", ()), ] + + +def _install(site: Path, name: str, files: dict[str, str], *, record: bool = True) -> None: + """A distribution as an installer leaves it: its files, METADATA, and RECORD (or, without + RECORD, top_level.txt).""" + for relative, text in files.items(): + (site / relative).parent.mkdir(parents=True, exist_ok=True) + (site / relative).write_text(text) + info = site / f"{name.replace('-', '_')}-1.0.dist-info" + info.mkdir(parents=True) + (info / "METADATA").write_text(f"Metadata-Version: 2.1\nName: {name}\nVersion: 1.0\n") + if record: + rows = [*files, f"{info.name}/METADATA", f"{info.name}/RECORD"] + (info / "RECORD").write_text("".join(f"{row},,\n" for row in rows)) + else: + (info / "top_level.txt").write_text(f"{name.replace('-', '_')}\n") + + +@pytest.mark.parametrize( + ("source", "declared", "expected"), + [ + pytest.param( + "from review_namespace.beta import PROVIDER\n", + '["review-alpha"]', + [Undeclared("app/m.py", "review_namespace", ("review-alpha", "review-beta"))], + id="a declared sibling does not stand in for the provider", + ), + pytest.param( + "from review_namespace.beta import PROVIDER\n", + '["review-beta"]', + [Undeclared("app/m.py", "review_namespace", ("review-alpha", "review-beta"))], + id="a namespace shared with an undeclared distribution is ambiguous", + ), + pytest.param( + "from review_namespace import beta\nimport review_namespace.alpha\n", + '["review-alpha", "review-beta"]', + [], + id="every installing distribution declared", + ), + pytest.param( + "import legacy_thing.sub\n", + '["review-alpha"]', + [Undeclared("app/m.py", "legacy_thing", ("legacy-thing",))], + id="no RECORD, top_level.txt", + ), + pytest.param("import legacy_thing\n", '["legacy-thing"]', [], id="no RECORD, declared"), + ], +) +def test_a_name_several_distributions_install_needs_all_of_them_declared( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + source: str, + declared: str, + expected: list[Undeclared], +) -> None: + site = tmp_path / "site" + _install(site, "review-alpha", {"review_namespace/alpha/__init__.py": 'PROVIDER = "a"\n'}) + _install(site, "review-beta", {"review_namespace/beta/__init__.py": 'PROVIDER = "b"\n'}) + _install(site, "legacy-thing", {"legacy_thing/__init__.py": ""}, record=False) + monkeypatch.syspath_prepend(str(site)) + (tmp_path / "pyproject.toml").write_text(f'[project]\nname = "x"\ndependencies = {declared}\n') + package = tmp_path / "src" / "app" + package.mkdir(parents=True) + (package / "m.py").write_text(source) + assert undeclared_imports(tmp_path / "src", tmp_path / "pyproject.toml") == (1, expected) diff --git a/tests/test_history_fixtures.py b/tests/test_history_fixtures.py new file mode 100644 index 0000000..90ee6d2 --- /dev/null +++ b/tests/test_history_fixtures.py @@ -0,0 +1,202 @@ +"""tests/fixtures/history: each copy is the git blob its README row records. + +The README's table gives each copy's path under the directory, its source commit, and its git blob +id. The copies compared are the files git tracks under the directory, so a cache a tool writes +there, which git ignores, is not one of them. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + +HISTORY = Path(__file__).resolve().parent / "fixtures" / "history" +README = "README.md" +GIT = shutil.which("git") +HEADER = "| File | Source commit | Blob | Read by |" +SEPARATOR = re.compile(r"\|(?:-+\|){4}") +ROW = re.compile( + r"\| `(?P[^`|]+)` \| `[0-9a-f]{40}`[^|]* \| `(?P[0-9a-f]{40})` \| [^|]+ \|" +) +# The README's rows, one per tracked copy. +COPIES = 11 + + +class TableError(ValueError): + """A README line the table reader does not model; ``line`` is its number, or None when the + README holds no table or more than one.""" + + def __init__(self, line: int | None, reason: str) -> None: + super().__init__(reason if line is None else f"README line {line}: {reason}") + self.line = line + + +def read_rows(text: str) -> list[tuple[str, str]]: + """Each row's file and blob id, in order. Every line starting with ``|`` is the header, the + separator under it, or a row.""" + lines = text.splitlines() + if lines.count(HEADER) != 1: + raise TableError(None, f"{lines.count(HEADER)} tables, not 1") + header = lines.index(HEADER) + rows = [] + for number, line in enumerate(lines, start=1): + if not line.startswith("|") or number == header + 1: + continue + if number == header + 2 and SEPARATOR.fullmatch(line): + continue + match = ROW.fullmatch(line) + if match is None: + raise TableError(number, "not a `| file | commit | blob | read by |` row") + rows.append((match["file"], match["blob"])) + return rows + + +def blob_id(content: bytes) -> str: + """The id git gives a file holding ``content``: the SHA-1 of ``blob ``, a NUL byte, and + the bytes.""" + return hashlib.sha1(b"blob %d\0" % len(content) + content, usedforsecurity=False).hexdigest() + + +def copy_problems(directory: Path, files: list[str], rows: list[tuple[str, str]]) -> list[str]: + """Each of ``files`` (paths under ``directory``) without exactly one row, each row that names + none of them, and each file whose content is not the blob its row records.""" + recorded: dict[str, list[str]] = {} + for file, blob in rows: + recorded.setdefault(file, []).append(blob) + problems = [] + for file in sorted(set(files) | recorded.keys()): + blobs = recorded.get(file, []) + if file not in files: + problems.append(f"{file}: a row but no file") + elif len(blobs) != 1: + problems.append(f"{file}: {len(blobs)} rows") + elif (found := blob_id((directory / file).read_bytes())) != blobs[0]: + problems.append(f"{file}: blob {found}, the row records {blobs[0]}") + return problems + + +def tracked_files(directory: Path, home: Path) -> list[str]: + """The files git tracks under ``directory``, relative to it, other than the README; git runs + with a home of the test's own and no global or system configuration.""" + assert GIT is not None + result = subprocess.run( + [GIT, "ls-files", "-z", "--", "."], + cwd=directory, + capture_output=True, + check=True, + env={ + "PATH": os.environ["PATH"], + "HOME": str(home), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + }, + timeout=60, + ) + names = result.stdout.decode().split("\0") + return sorted(name for name in names if name and name != README) + + +def test_each_copy_is_the_blob_its_row_records(tmp_path: Path) -> None: + rows = read_rows((HISTORY / README).read_text()) + files = tracked_files(HISTORY, tmp_path) + assert len(files) == len(rows) == COPIES + assert copy_problems(HISTORY, files, rows) == [] + + +# Sample copies the tests below write, with the ids `git hash-object` gives their content. +X, X_BLOB = b'{"a": 1}\n', "cb5b2f69babc53f9145b45ad20b9a075d48717ef" +X_EDITED_BLOB = "3c27e19a80d4780c181bc216b4456841d86d74f3" # b'{"a": 2}\n' +Y, Y_BLOB = b"# y\n", "29dfab31929d7957cdd6997e17ed38642047d7ed" + + +def _readme(rows: list[tuple[str, str]]) -> str: + lines = [HEADER, "|------|---------------|------|---------|"] + lines += [f"| `{file}` | `{'1' * 40}` | `{blob}` | `tests/test_x.py` |" for file, blob in rows] + return "# Earlier versions\n\nText.\n\n" + "\n".join(lines) + "\n" + + +@pytest.fixture +def sample(tmp_path: Path) -> Path: + directory = tmp_path / "history" + (directory / "a").mkdir(parents=True) + (directory / "b").mkdir() + (directory / "a" / "x.json").write_bytes(X) + (directory / "b" / "y.md").write_bytes(Y) + return directory + + +def test_blob_id_is_gits() -> None: + assert blob_id(b"") == "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" + assert [blob_id(X), blob_id(b'{"a": 2}\n'), blob_id(Y)] == [X_BLOB, X_EDITED_BLOB, Y_BLOB] + + +def test_copies_their_rows_record_pass(sample: Path) -> None: + rows = read_rows(_readme([("a/x.json", X_BLOB), ("b/y.md", Y_BLOB)])) + assert rows == [("a/x.json", X_BLOB), ("b/y.md", Y_BLOB)] + assert copy_problems(sample, ["a/x.json", "b/y.md"], rows) == [] + + +@pytest.mark.parametrize( + ("rows", "files", "problems"), + [ + pytest.param( + [("a/x.json", X_EDITED_BLOB), ("b/y.md", Y_BLOB)], + ["a/x.json", "b/y.md"], + [f"a/x.json: blob {X_BLOB}, the row records {X_EDITED_BLOB}"], + id="content other than the row's blob", + ), + pytest.param( + [("a/x.json", X_BLOB)], + ["a/x.json", "b/y.md"], + ["b/y.md: 0 rows"], + id="a file without a row", + ), + pytest.param( + [("a/x.json", X_BLOB), ("a/x.json", X_BLOB), ("b/y.md", Y_BLOB)], + ["a/x.json", "b/y.md"], + ["a/x.json: 2 rows"], + id="a file with two rows", + ), + pytest.param( + [("a/x.json", X_BLOB), ("b/y.md", Y_BLOB), ("c/z.json", X_BLOB)], + ["a/x.json", "b/y.md"], + ["c/z.json: a row but no file"], + id="a row naming no file", + ), + ], +) +def test_a_copy_without_exactly_one_matching_row_is_reported( + sample: Path, rows: list[tuple[str, str]], files: list[str], problems: list[str] +) -> None: + assert copy_problems(sample, files, read_rows(_readme(rows))) == problems + + +@pytest.mark.parametrize( + ("text", "line"), + [ + pytest.param(_readme([("a/x.json", X_BLOB[:39])]), 7, id="short blob id"), + pytest.param( + _readme([("a/x.json", X_BLOB)]).replace(" | `tests/test_x.py` |", " |"), + 7, + id="no reader column", + ), + pytest.param(_readme([("a/x.json", X_BLOB)]) + "\n| x |\n", 9, id="a line after it"), + pytest.param( + _readme([("a/x.json", X_BLOB)]).replace("|------|", "|------|------|"), + 6, + id="a separator of five columns", + ), + pytest.param("# Earlier versions\n\nNo table.\n", None, id="no table"), + pytest.param(_readme([]) + _readme([]), None, id="two tables"), + ], +) +def test_a_table_line_it_does_not_model_is_refused(text: str, line: int | None) -> None: + with pytest.raises(TableError) as raised: + read_rows(text) + assert raised.value.line == line diff --git a/tests/test_import_boundaries.py b/tests/test_import_boundaries.py index a2175f8..d2dd6e8 100644 --- a/tests/test_import_boundaries.py +++ b/tests/test_import_boundaries.py @@ -199,16 +199,21 @@ def test_source_does_not_import_the_database() -> None: def test_the_scan_finds_each_githubkit_import_in_source() -> None: - """source/ is allowed githubkit: the matcher finds each statement that imports it.""" + """source/ is allowed githubkit: the matcher finds each statement that imports it, and + source/ holds 5, so the comparison cannot pass on none.""" files = _python_files(PACKAGE / "source") - assert _sites(githubkit_imports(files)) == _statement_sites(files, GITHUBKIT_STATEMENT) + sites = _sites(githubkit_imports(files)) + assert sites == _statement_sites(files, GITHUBKIT_STATEMENT) + assert len(sites) == 5 def test_the_scan_finds_each_database_import_in_sync() -> None: - """sync/ is allowed SQLAlchemy and db/: the matcher finds each statement that imports them.""" + """sync/ is allowed SQLAlchemy and db/: the matcher finds each statement that imports them, + and sync/ holds 20, so the comparison cannot pass on none.""" files = _python_files(PACKAGE / "sync") - found = database_imports(files, ROOT / "src") - assert _sites(found) == _statement_sites(files, DATABASE_STATEMENT) + sites = _sites(database_imports(files, ROOT / "src")) + assert sites == _statement_sites(files, DATABASE_STATEMENT) + assert len(sites) == 20 def test_the_scan_sees_source_importing_githubkit() -> None: From cfd18b9df13b75811271d7a3fa200f8a9f1ac621 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 13:00:10 -0600 Subject: [PATCH 23/29] Test the determinism guard's failing sides; run the suite reversed in CI The after-test refusal check: a pytest run over sample tests under the conftest's hooks and fixtures, each catching a refusal at collection, in a test, in a module fixture's setup or teardown, or in a session fixture's teardown, gives the exact outcome of each test in its JUnit report. Run order: tests/conftest.py takes --reverse-order, which reverses the collected items after every other plugin has ordered them. CI runs the suite a second time with it, under PYTHONHASHSEED 2 (the first run sets 1), and compares the two JUnit reports with check_junit.py --max-skipped 0 --same-as. A sample order-dependent pair shows the option and the comparison on both sides. The guard: - db.types.utc_now reads the wall clock through _wall_clock, which tests/determinism.py refuses for the session; replacing it leaves the datetime name UtcDateTime's type check reads alone. test_types replaces the seam and writes an explicit timestamp beside the default; the two test_db_engine tests that wrote UserTag without created_at pass one. - SystemClock's now and sleep are refused at the names source/clock.py reads (datetime, asyncio), so a test gives SystemClock a fixed datetime or a recording sleep and checks its bodies. - A host-name lookup called with keyword arguments, and any method of SystemClock._random, are refused and recorded. The scans: the determinism scan also reads alembic/, its file count checked against MIGRATION_REVISIONS, and treats a starred argument to Random as unseeded; the collection scan also reports a test function or class inside a class whose name does not start with Test. Sample source covers star imports of each modelled module and annotations holding a call. Every scan refuses an empty file list. test_unreadable_report_fails calls check_junit.main in process and asserts it returns 1 and the exact stderr line; the Unix-socket test asserts connect's None and connect_ex's 0 exactly. --- .github/workflows/check.yml | 12 ++ src/github_activity_db/db/types.py | 10 +- tests/conftest.py | 21 +- tests/db/test_types.py | 42 ++-- tests/determinism.py | 87 +++++--- tests/test_check_junit.py | 27 ++- tests/test_db_engine.py | 10 +- tests/test_suite_rules.py | 307 +++++++++++++++++++++++++++-- 8 files changed, 447 insertions(+), 69 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 61bf00b..875d5b9 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -76,9 +76,21 @@ jobs: - name: Python and SQLite versions run: uv run --no-sync python -c "import sqlite3, sys; print(sys.version, sqlite3.sqlite_version)" - name: pytest + env: + PYTHONHASHSEED: "1" run: uv run --no-sync pytest --junitxml="$RUNNER_TEMP/junit.xml" - name: Require an executed, passing test run with no skips run: uv run --no-sync python scripts/check_junit.py "$RUNNER_TEMP/junit.xml" --max-skipped 0 + # D40: the same outcome in another run order and under another hash seed. Each pair of tests + # runs in the other order; a dependence shows only where it changes an outcome. + - name: pytest again, last to first, under another hash seed + env: + PYTHONHASHSEED: "2" + run: uv run --no-sync pytest --reverse-order --junitxml="$RUNNER_TEMP/junit-reversed.xml" + - name: Require the two runs to agree test by test + run: | + uv run --no-sync python scripts/check_junit.py "$RUNNER_TEMP/junit.xml" --max-skipped 0 \ + --same-as "$RUNNER_TEMP/junit-reversed.xml" - name: Migrations upgrade a fresh database and match the models env: DATABASE_URL: sqlite+aiosqlite:///${{ runner.temp }}/migrations.db diff --git a/src/github_activity_db/db/types.py b/src/github_activity_db/db/types.py index 3bd81da..1321041 100644 --- a/src/github_activity_db/db/types.py +++ b/src/github_activity_db/db/types.py @@ -2,7 +2,7 @@ from __future__ import annotations -from datetime import UTC, datetime +from datetime import UTC, datetime, tzinfo from typing import Any from sqlalchemy import DateTime @@ -10,9 +10,15 @@ from sqlalchemy.types import TypeDecorator +def _wall_clock(tz: tzinfo) -> datetime: + """The current time in ``tz``: this module's one read of the wall clock, kept apart from + ``datetime`` so that replacing it leaves ``UtcDateTime``'s type check alone.""" + return datetime.now(tz) + + def utc_now() -> datetime: """The Python-side default of every timestamp column.""" - return datetime.now(UTC) + return _wall_clock(UTC) class UtcDateTime(TypeDecorator[datetime]): diff --git a/tests/conftest.py b/tests/conftest.py index 531983f..fdddb71 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,8 +37,9 @@ def pytest_configure(config: pytest.Config) -> None: - """The network, the real clock, and its random source are refused from collection to the end - of the session (``tests/determinism.py``, redesign plan D40).""" + """What ``tests/determinism.py`` refuses (connections, host-name lookups, the package's real + clocks, and its random source) is refused from collection to the end of the session + (redesign plan D40).""" determinism.install(_GUARD) @@ -46,6 +47,22 @@ def pytest_unconfigure(config: pytest.Config) -> None: _GUARD.undo() +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--reverse-order", + action="store_true", + help="run the collected tests last to first (CI compares such a run with one in order)", + ) + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """With ``--reverse-order``, the tests run last to first, after every other plugin has + ordered them: each pair of tests runs in the other order.""" + if config.getoption("--reverse-order"): + items.reverse() + + @pytest.fixture(autouse=True, scope="session") def nothing_refused_after_the_last_test() -> Iterator[None]: """The refusals recorded after the last test's check, in fixtures torn down after it.""" diff --git a/tests/db/test_types.py b/tests/db/test_types.py index aafaabb..f1b3fe7 100644 --- a/tests/db/test_types.py +++ b/tests/db/test_types.py @@ -41,11 +41,6 @@ def _stored_refreshed_at(database: Path) -> str: return str(text) -def _stored_created_at(database: Path) -> str: - ((text,),) = query(database, "SELECT created_at FROM user_tags") - return str(text) - - def test_no_column_uses_a_bare_datetime() -> None: types = { f"{table.name}.{column.name}": column.type @@ -118,23 +113,32 @@ def test_null_passes_through(migrated_database: Path) -> None: NOW = datetime(2026, 1, 2, 3, 4, 5, 678901, tzinfo=UTC) -class FixedClock(datetime): - """datetime as types.py sees it while a test runs: ``now`` is NOW in the zone asked for.""" - - @classmethod - def now(cls, tz: tzinfo | None = None) -> FixedClock: - assert tz is UTC - return cls.combine(NOW.date(), NOW.timetz()) - - def test_the_default_is_now_in_utc( migrated_database: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setattr(types, "datetime", FixedClock) + """The column default over a fixed wall clock, which answers NOW in the zone asked for; the + clock alone is replaced, so an explicit timestamp written beside the default is still a + datetime to UtcDateTime.""" + asked: list[tzinfo] = [] + + def wall_clock(tz: tzinfo) -> datetime: + asked.append(tz) + return NOW.astimezone(tz) + + monkeypatch.setattr(types, "_wall_clock", wall_clock) assert utc_now() == NOW + assert asked == [UTC] + explicit = datetime(2026, 1, 1, 5, 30, tzinfo=KOLKATA) with connect(migrated_database) as connection: connection.execute(insert(UserTag).values(name="kept")) - created_at = connection.execute(select(UserTag.created_at)).scalar_one() - assert created_at == NOW - assert created_at.tzinfo is UTC - assert _stored_created_at(migrated_database) == "2026-01-02 03:04:05.678901" + connection.execute(insert(UserTag).values(name="given", created_at=explicit)) + created = connection.execute( + select(UserTag.name, UserTag.created_at).order_by(UserTag.id) + ).all() + assert [tuple(row) for row in created] == [("kept", NOW), ("given", explicit)] + assert [when.tzinfo for _, when in created] == [UTC, UTC] + assert asked == [UTC, UTC] + assert query(migrated_database, "SELECT created_at FROM user_tags ORDER BY id") == [ + ("2026-01-02 03:04:05.678901",), + ("2026-01-01 00:00:00.000000",), + ] diff --git a/tests/determinism.py b/tests/determinism.py index e3c9e53..58fa4eb 100644 --- a/tests/determinism.py +++ b/tests/determinism.py @@ -1,26 +1,33 @@ """What every test runs under (redesign plan D40), installed by the conftest from collection on. Inside the test process, a connection to anything but a Unix socket and every host-name lookup the -``socket`` module offers are refused, and so are this project's real clock (``SystemClock.now`` and -``sleep``) and its unseeded random source (``SystemClock._random``, which an instance may replace -with a seeded one). ``no_proxy`` is ``*``, so an HTTP client that reads proxies from the -environment, or on macOS from the system settings, connects straight to the host it was given and -is refused there, whatever the machine's proxy settings. A refusal raises ``NotDeterministic``, a -``BaseException`` so that no ``except Exception`` in the code under test can turn it into a handled -failure, and is recorded in ``REFUSED``, which the conftest checks after every test and at the end -of the session. Out of scope: processes a test starts (they run without these patches), the -standard library's clocks and random modules called directly (``tests/test_suite_rules.py`` reads -test files for those, and the package for its own reads), and ``db.types.utc_now``, the default of -columns no producer writes. +``socket`` module offers are refused, called with positional or keyword arguments; so are the +package's two real clocks: ``source/clock.py``'s ``datetime.now`` and ``asyncio.sleep`` (behind +``SystemClock.now`` and ``sleep``), replaced at those names so a test can give ``SystemClock`` a +fixed clock of its own, and ``db.types._wall_clock`` (behind ``utc_now``, the default of the +timestamp columns); and so is every draw from the unseeded random source ``SystemClock._random``, +which an instance may replace with a seeded one. ``no_proxy`` is ``*``, so an HTTP client that reads +proxies from the environment, or on macOS from the system settings, connects straight to the host +it was given and is refused there, whatever the machine's proxy settings. A refusal raises +``NotDeterministic``, a ``BaseException`` so that no ``except Exception`` in the code under test can +turn it into a handled failure, and is recorded in ``REFUSED``, which the conftest checks after +every test and at the end of the session. Out of scope: processes a test starts (they run without +these patches), the socket's other ways to reach a host (``sendto``, ``sendmsg``, ``bind``, +``listen``, and sockets made in C), and the standard library's clocks and random modules called +directly (``tests/test_suite_rules.py`` reads test files, the package, and ``alembic/`` for those). """ from __future__ import annotations +import asyncio import socket +from datetime import datetime, tzinfo from typing import Any, NoReturn import pytest +from github_activity_db.db import types +from github_activity_db.source import clock from github_activity_db.source.clock import SystemClock # The host-name lookups the socket module offers; getfqdn goes through gethostbyaddr. @@ -52,9 +59,47 @@ def drain() -> list[tuple[str, str]]: return refused +class _NoWallClock(datetime): + """``datetime`` as ``source/clock.py`` sees it: a datetime, except that reading the time is + refused.""" + + @classmethod + def now(cls, tz: tzinfo | None = None) -> NoReturn: + refuse("wall clock", f"source.clock: datetime.now({tz})") + + @classmethod + def utcnow(cls) -> NoReturn: + refuse("wall clock", "source.clock: datetime.utcnow()") + + @classmethod + def today(cls) -> NoReturn: + refuse("wall clock", "source.clock: datetime.today()") + + +class _NoSleep: + """``asyncio`` as ``source/clock.py`` sees it: asyncio, except that sleeping is refused.""" + + @staticmethod + async def sleep(delay: float, result: Any = None) -> NoReturn: + refuse("wall clock", f"source.clock: asyncio.sleep({delay})") + + def __getattr__(self, name: str) -> Any: + return getattr(asyncio, name) + + class _NoSystemRandom: - def uniform(self, low: float, high: float) -> NoReturn: - refuse("randomness", f"SystemClock._random.uniform({low}, {high})") + """The random source as ``SystemClock`` sees it: a call of any of its methods is refused. A + dunder name is not answered, so ``copy`` and ``pickle`` find what an object has.""" + + def __getattr__(self, name: str) -> Any: + if name.startswith("__"): + raise AttributeError(name) + + def draw(*args: Any, **kwargs: Any) -> NoReturn: + given = [repr(arg) for arg in args] + [f"{key}={kwargs[key]!r}" for key in kwargs] + refuse("randomness", f"SystemClock._random.{name}({', '.join(given)})") + + return draw def install(patch: pytest.MonkeyPatch) -> None: @@ -72,16 +117,13 @@ def connecting_ex(self: socket.socket, address: Any) -> int: return connect_ex(self, address) def looking_up(name: str) -> Any: - def lookup(looked_up: Any, *args: Any, **kwargs: Any) -> NoReturn: - refuse("network", f"{name} {looked_up!r}") + def lookup(*args: Any, **kwargs: Any) -> NoReturn: + refuse("network", f"{name} {(args[0] if args else kwargs)!r}") return lookup - def now(self: SystemClock) -> NoReturn: - refuse("wall clock", "SystemClock.now()") - - async def sleep(self: SystemClock, seconds: float) -> NoReturn: - refuse("wall clock", f"SystemClock.sleep({seconds})") + def wall_clock(tz: tzinfo) -> NoReturn: + refuse("wall clock", f"db.types: datetime.now({tz})") patch.setenv("no_proxy", "*") patch.setenv("NO_PROXY", "*") @@ -89,6 +131,7 @@ async def sleep(self: SystemClock, seconds: float) -> NoReturn: patch.setattr(socket.socket, "connect_ex", connecting_ex) for name in LOOKUPS: patch.setattr(socket, name, looking_up(name)) - patch.setattr(SystemClock, "now", now) - patch.setattr(SystemClock, "sleep", sleep) + patch.setattr(clock, "datetime", _NoWallClock) + patch.setattr(clock, "asyncio", _NoSleep()) patch.setattr(SystemClock, "_random", _NoSystemRandom()) + patch.setattr(types, "_wall_clock", wall_clock) diff --git a/tests/test_check_junit.py b/tests/test_check_junit.py index 3f44075..eb4c3c3 100644 --- a/tests/test_check_junit.py +++ b/tests/test_check_junit.py @@ -9,11 +9,13 @@ import ast import copy +import importlib.util import os import subprocess import sys import xml.etree.ElementTree as ET from pathlib import Path +from types import ModuleType import pytest @@ -125,17 +127,34 @@ def test_sums_every_testsuite(tmp_path: Path, written: bytes) -> None: assert len(result.stderr.splitlines()) == 1 +def _load() -> ModuleType: + """The script as a module, so its ``main`` can be called in process.""" + spec = importlib.util.spec_from_file_location("check_junit", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + @pytest.mark.parametrize("shape", ["missing", "cut short", "no testsuite"]) -def test_unreadable_report_fails(tmp_path: Path, written: bytes, shape: str) -> None: +def test_unreadable_report_fails( + tmp_path: Path, written: bytes, shape: str, capsys: pytest.CaptureFixture[str] +) -> None: + """The gate returns 1 rather than raising, and names why: the parser's error for a report + cut short.""" path = tmp_path / "junit.xml" + problem = f"[Errno 2] No such file or directory: '{path}'" if shape == "cut short": path.write_bytes(written[: len(written) // 2]) + with pytest.raises(ET.ParseError) as parse_error: + ET.parse(path) # noqa: S314 - the report pytest wrote in this session, cut short + problem = str(parse_error.value) elif shape == "no testsuite": path = _report(tmp_path, written) - result = _gate(path) - assert result.returncode == 1 + problem = f"no element in {path}" + assert _load().main([str(path)]) == 1 # The counters are printed only after the report is read. - assert result.stdout == "" + assert capsys.readouterr() == ("", f"cannot read JUnit report: {problem}\n") def test_negative_allowance_is_refused(tmp_path: Path, written: bytes) -> None: diff --git a/tests/test_db_engine.py b/tests/test_db_engine.py index 86a9acd..7b5f398 100644 --- a/tests/test_db_engine.py +++ b/tests/test_db_engine.py @@ -1,6 +1,7 @@ """Tests for database engine and session management.""" import sqlite3 +from datetime import UTC, datetime from pathlib import Path import pytest @@ -16,6 +17,9 @@ from github_activity_db.db.models import UserTag from tests.db.seed import Seed +# Written to each tag's created_at, so no write reads the wall clock behind the column default. +CREATED_AT = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC) + class TestDatabaseEngine: """Sessions over the application engine, on a database migrated to head.""" @@ -25,7 +29,7 @@ async def test_session_commits_on_success(self, test_engine): test_engine, class_=AsyncSession, expire_on_commit=False ) async with session_factory() as session: - tag = UserTag(name="committed") + tag = UserTag(name="committed", created_at=CREATED_AT) session.add(tag) await session.commit() tag_id = tag.id @@ -33,7 +37,7 @@ async def test_session_commits_on_success(self, test_engine): async with session_factory() as session: stored = await session.get(UserTag, tag_id) assert stored is not None - assert stored.name == "committed" + assert (stored.name, stored.created_at) == ("committed", CREATED_AT) async def test_session_rollbacks_on_error(self, test_engine): session_factory = async_sessionmaker( @@ -41,7 +45,7 @@ async def test_session_rollbacks_on_error(self, test_engine): ) with pytest.raises(ValueError, match="Simulated error"): async with session_factory() as session: - session.add(UserTag(name="rolled back")) + session.add(UserTag(name="rolled back", created_at=CREATED_AT)) await session.flush() # written to the database, not committed raise ValueError("Simulated error") diff --git a/tests/test_suite_rules.py b/tests/test_suite_rules.py index c1faa35..ee3bdf4 100644 --- a/tests/test_suite_rules.py +++ b/tests/test_suite_rules.py @@ -23,24 +23,37 @@ type and is not read, unless the annotation holds a call or ``Annotated`` metadata, which code reads at run time. Out of scope as well: the modules reached without an import statement, the event loop's ``time()``, file times, and the order of a set of strings, which ``PYTHONHASHSEED`` changes from one -process to the next (two runs compared by ``scripts/check_junit.py --same-as`` show that). The same -scan over the package finds its reads only in its two real clocks; at run time -``tests/determinism.py`` refuses one of them, ``SystemClock``, and the network, in every test. - -Every test function is one pytest collects: none is defined inside another function, where it -would never run. +process to the next. CI runs the suite twice, the second time last to first (``--reverse-order``, +``tests/conftest.py``) under another ``PYTHONHASHSEED``, and compares the two runs with +``scripts/check_junit.py --same-as``: that shows a dependence on run order or on the seed only where +it changed an outcome between those two runs. The same scan over the package finds its reads only in +its two real clocks, and over ``alembic/``, which every database test runs in process, finds none; +at run time ``tests/determinism.py`` refuses both clocks, network connections other than Unix +sockets, and the ``socket`` module's host-name lookups, in every test. + +No test function or test class is defined inside a function, or inside a class whose name does not +start with ``Test``, where pytest never collects it. Out of scope: a module pytest's file pattern +does not match, a ``Test`` class with an ``__init__``, and pytest's naming options changed in its +configuration. """ from __future__ import annotations import ast +import importlib.util import os import socket +import subprocess +import sys +from datetime import UTC, datetime, tzinfo from pathlib import Path +from types import ModuleType import pytest -from github_activity_db.source import GitHubSource +from github_activity_db.db import MIGRATION_REVISIONS +from github_activity_db.db.types import utc_now +from github_activity_db.source import GitHubSource, clock from github_activity_db.source.clock import SystemClock from tests import determinism from tests.determinism import NotDeterministic @@ -48,6 +61,8 @@ TESTS = Path(__file__).resolve().parent SRC = TESTS.parent / "src" / "github_activity_db" +ALEMBIC = TESTS.parent / "alembic" +CHECK_JUNIT = TESTS.parent / "scripts" / "check_junit.py" WAITING = frozenset({"run", "call", "check_call", "check_output"}) # subprocess names that start no process. INERT = frozenset( @@ -241,6 +256,8 @@ def _seeded(call: ast.Call | None) -> bool: if call is None: return False seed = call.args[0] if call.args else None + if isinstance(seed, ast.Starred): # ``*args``, which the scan cannot read + seed = None for keyword in call.keywords: if keyword.arg in ("x", None): # None: ``**kwargs``, which the scan cannot read seed = keyword.value if keyword.arg == "x" else None @@ -248,12 +265,17 @@ def _seeded(call: ast.Call | None) -> bool: def uncollected_tests(*paths: Path) -> list[str]: - """Test functions and classes defined inside a function, where pytest never collects them.""" + """Test functions and classes defined inside a function, or inside a class whose name does not + start with ``Test``, where pytest never collects them; each is listed once for every such + function or class around it.""" found = [] for path in paths: where = path.relative_to(path.parents[1]) for outer in ast.walk(ast.parse(path.read_bytes())): - if not isinstance(outer, ast.FunctionDef | ast.AsyncFunctionDef): + if not ( + isinstance(outer, ast.FunctionDef | ast.AsyncFunctionDef) + or (isinstance(outer, ast.ClassDef) and not outer.name.startswith("Test")) + ): continue for node in ast.walk(outer): if node is not outer and ( @@ -269,10 +291,11 @@ def uncollected_tests(*paths: Path) -> list[str]: def _python_files(root: Path) -> list[Path]: files = sorted(root.rglob("*.py")) - # Counted with os.walk rather than the glob, so a scan cannot pass on fewer files. + # Counted with os.walk rather than the glob, so a scan cannot pass on fewer files, or on none. assert len(files) == sum( Path(name).suffix == ".py" for _, _, names in os.walk(root) for name in names ) + assert files, f"no Python file under {root}" return files @@ -292,14 +315,27 @@ def test_the_package_reads_the_wall_clock_and_randomness_only_in_its_real_clocks """Over every module of the package, the scan finds the reads of the two real clocks, and no other: everything else takes a clock it is given, which a test fixes.""" assert determinism_problems(*_python_files(SRC)) == [ - "db/types.py:15: datetime.now", + "db/types.py:16: datetime.now", "source/clock.py:32: secrets.SystemRandom", "source/clock.py:35: datetime.now", "source/clock.py:38: asyncio.sleep", ] -def test_pytest_collects_every_test_function_in_the_suite() -> None: +def test_the_migrations_read_no_wall_clock_or_randomness() -> None: + """Over ``alembic/env.py`` and one file for each revision of the chain, which every database + test runs in process, the scan finds nothing.""" + files = _python_files(ALEMBIC) + names = [path.relative_to(ALEMBIC).as_posix() for path in files] + assert names[0] == "env.py" + assert sorted(name.removeprefix("versions/").split("_")[0] for name in names[1:]) == sorted( + MIGRATION_REVISIONS + ) + assert len(files) == 1 + len(MIGRATION_REVISIONS) + assert determinism_problems(*files) == [] + + +def test_no_test_is_defined_inside_a_function_or_a_non_test_class() -> None: assert uncollected_tests(*_test_files()) == [] @@ -318,16 +354,16 @@ def test_a_connection_to_a_host_is_refused(method: str) -> None: assert _refused(refusal) == ("network", f"{method} ('192.0.2.1', 443)") -@pytest.mark.parametrize("method", ["connect", "connect_ex"]) +@pytest.mark.parametrize(("method", "returned"), [("connect", None), ("connect_ex", 0)]) def test_a_connection_to_a_unix_socket_goes_through( - method: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + method: str, returned: int | None, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: # A relative path, since macOS limits a Unix socket's path to 104 bytes. monkeypatch.chdir(tmp_path) with socket.socket(socket.AF_UNIX) as listening, socket.socket(socket.AF_UNIX) as connection: listening.bind("s") listening.listen(1) - assert getattr(connection, method)("s") in (None, 0) + assert getattr(connection, method)("s") == returned assert connection.getpeername() == "s" assert determinism.REFUSED == [] @@ -352,6 +388,12 @@ def test_every_host_name_lookup_is_refused( assert _refused(refusal) == ("network", detail) +def test_a_lookup_by_keyword_is_refused() -> None: + with pytest.raises(NotDeterministic) as refusal: + socket.getaddrinfo(host="api.github.com", port=443) + assert _refused(refusal) == ("network", "getaddrinfo {'host': 'api.github.com', 'port': 443}") + + @pytest.mark.parametrize( "proxy", [ @@ -378,13 +420,13 @@ async def test_the_source_without_a_recorded_transport_is_refused_the_network( def test_the_real_clock_is_refused() -> None: with pytest.raises(NotDeterministic) as refusal: SystemClock().now() - assert _refused(refusal) == ("wall clock", "SystemClock.now()") + assert _refused(refusal) == ("wall clock", "source.clock: datetime.now(UTC)") async def test_the_real_clocks_sleep_is_refused() -> None: with pytest.raises(NotDeterministic) as refusal: await SystemClock().sleep(2.0) - assert _refused(refusal) == ("wall clock", "SystemClock.sleep(2.0)") + assert _refused(refusal) == ("wall clock", "source.clock: asyncio.sleep(2.0)") def test_the_real_clocks_unseeded_jitter_is_refused() -> None: @@ -393,6 +435,206 @@ def test_the_real_clocks_unseeded_jitter_is_refused() -> None: assert _refused(refusal) == ("randomness", "SystemClock._random.uniform(0.0, 2.0)") +@pytest.mark.parametrize( + ("method", "args", "kwargs", "detail"), + [ + ("random", (), {}, "random()"), + ("getrandbits", (8,), {}, "getrandbits(8)"), + ("randbytes", (4,), {}, "randbytes(4)"), + ("choice", (["a", "b"],), {}, "choice(['a', 'b'])"), + ("gauss", (), {"mu": 0.0, "sigma": 1.0}, "gauss(mu=0.0, sigma=1.0)"), + ], + ids=["random", "getrandbits", "randbytes", "choice", "gauss"], +) +def test_every_draw_from_the_real_clocks_random_source_is_refused( + method: str, args: tuple[object, ...], kwargs: dict[str, object], detail: str +) -> None: + with pytest.raises(NotDeterministic) as refusal: + getattr(SystemClock._random, method)(*args, **kwargs) + assert _refused(refusal) == ("randomness", f"SystemClock._random.{detail}") + + +def test_the_real_clocks_random_source_answers_no_dunder_name() -> None: + """``copy`` and ``pickle`` look up dunder names on an object; none is a draw.""" + assert not hasattr(SystemClock._random, "__deepcopy__") + assert determinism.REFUSED == [] + + +def test_the_column_defaults_clock_is_refused() -> None: + with pytest.raises(NotDeterministic) as refusal: + utc_now() + assert _refused(refusal) == ("wall clock", "db.types: datetime.now(UTC)") + + +# A fixed instant for the real clock's tests. +INSTANT = datetime(2026, 1, 2, 3, 4, 5, 678901, tzinfo=UTC) + + +def test_the_real_clock_reads_the_time_in_utc(monkeypatch: pytest.MonkeyPatch) -> None: + """``SystemClock.now`` over a fixed ``datetime``: INSTANT in the zone it asks for, or naive + when it asks for none.""" + + class Fixed(datetime): + @classmethod + def now(cls, tz: tzinfo | None = None) -> Fixed: + moment = INSTANT.astimezone(tz) if tz is not None else INSTANT.replace(tzinfo=None) + return cls.combine(moment.date(), moment.timetz()) + + monkeypatch.setattr(clock, "datetime", Fixed) + now = SystemClock().now() + assert (now, now.tzinfo) == (INSTANT, UTC) + + +async def test_the_real_clock_sleeps_the_seconds_it_is_given( + monkeypatch: pytest.MonkeyPatch, +) -> None: + slept: list[float] = [] + + class Recording: + @staticmethod + async def sleep(seconds: float) -> None: + slept.append(seconds) + + monkeypatch.setattr(clock, "asyncio", Recording) + await SystemClock().sleep(2.5) + assert slept == [2.5] + + +def _check_junit() -> ModuleType: + """scripts/check_junit.py, loaded from its file: CI runs it standalone.""" + spec = importlib.util.spec_from_file_location("check_junit", CHECK_JUNIT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pytest_over( + root: Path, files: dict[str, str], *options: str +) -> tuple[int, list[tuple[str, str]]]: + """pytest run over ``files``, written to ``root`` beside a configuration of their own, with this + repository importable: its exit code, and each test's outcome in the order its JUnit report + lists the tests, which is the order they ran in.""" + root.mkdir() + for name, text in {"pytest.ini": "[pytest]\n", **files}.items(): + (root / name).write_text(text) + report = root / "junit.xml" + run = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + f"--junitxml={report}", + *options, + ], + cwd=root, + capture_output=True, + text=True, + env={ + "PATH": os.environ.get("PATH", ""), + "PYTHONPATH": os.pathsep.join([str(TESTS.parent / "src"), str(TESTS.parent)]), + "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", + }, + timeout=120, + ) + assert report.is_file(), run.stdout + run.stderr + return run.returncode, list(_check_junit().read_outcomes(report).items()) + + +# Catches the refusal of the real clock, as code under test may. +SWALLOW = ( + "from github_activity_db.source.clock import SystemClock\n\n\n" + "def swallow():\n" + " try:\n" + " SystemClock().now()\n" + " except BaseException:\n" + " pass\n" +) + + +def test_a_refusal_the_code_caught_fails_the_next_check(tmp_path: Path) -> None: + """Sample tests under this conftest's guard and checks: each refusal caught where it was + raised fails the first check after it, the one after each test or the one at the end of the + session, and the next check passes again.""" + files = { + "conftest.py": ( + "from tests.conftest import (\n" + " nothing_refused,\n" + " nothing_refused_after_the_last_test,\n" + " pytest_configure,\n" + " pytest_unconfigure,\n" + ")\n" + ), + "swallow.py": SWALLOW, + "test_a_collection.py": "from swallow import swallow\n\nswallow()\n\n\n" + "def test_first():\n pass\n", + "test_b_test.py": "from swallow import swallow\n\n\n" + "def test_swallows():\n swallow()\n\n\n" + "def test_after_it():\n pass\n", + "test_c_module.py": "import pytest\nfrom swallow import swallow\n\n\n" + "@pytest.fixture(scope='module')\ndef at_setup():\n swallow()\n\n\n" + "@pytest.fixture(scope='module')\ndef at_teardown():\n yield\n swallow()\n\n\n" + "def test_with_a_setup_that_swallows(at_setup):\n pass\n\n\n" + "def test_with_a_teardown_that_swallows(at_teardown):\n pass\n", + "test_d_next.py": "def test_after_the_module_teardown():\n pass\n", + "test_e_last.py": "import pytest\nfrom swallow import swallow\n\n\n" + "@pytest.fixture(scope='session')\ndef at_the_end():\n yield\n swallow()\n\n\n" + "def test_last(at_the_end):\n pass\n", + } + assert _pytest_over(tmp_path / "run", files) == ( + 1, + [ + ("test_a_collection::test_first", "error"), + ("test_b_test::test_swallows", "error"), + ("test_b_test::test_after_it", "passed"), + ("test_c_module::test_with_a_setup_that_swallows", "error"), + ("test_c_module::test_with_a_teardown_that_swallows", "passed"), + ("test_d_next::test_after_the_module_teardown", "error"), + ("test_e_last::test_last", "error"), + ], + ) + + +def test_reverse_order_runs_the_tests_last_to_first( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Two runs of sample tests, the second with ``--reverse-order``, compared as CI compares its + two runs: test_b passes only after test_a, so only test_b differs.""" + files = { + "conftest.py": ( + "from tests.conftest import pytest_addoption, pytest_collection_modifyitems\n" + ), + "test_one.py": "SEEN = []\n\n\ndef test_a():\n SEEN.append('a')\n\n\n" + "def test_b():\n assert SEEN == ['a']\n", + "test_two.py": "def test_c():\n pass\n", + } + assert _pytest_over(tmp_path / "forward", files) == ( + 0, + [ + ("test_one::test_a", "passed"), + ("test_one::test_b", "passed"), + ("test_two::test_c", "passed"), + ], + ) + assert _pytest_over(tmp_path / "backward", files, "--reverse-order") == ( + 1, + [ + ("test_two::test_c", "passed"), + ("test_one::test_b", "failed"), + ("test_one::test_a", "passed"), + ], + ) + forward, backward = (str(tmp_path / run / "junit.xml") for run in ("forward", "backward")) + assert _check_junit().main([forward, "--max-skipped", "0", "--same-as", backward]) == 1 + assert capsys.readouterr() == ( + "{'tests': 3, 'failures': 0, 'errors': 0, 'skipped': 0}\n", + "FAIL: differs: test_one::test_b: passed vs failed\n", + ) + + @pytest.mark.parametrize( ("source", "flagged"), [ @@ -461,8 +703,26 @@ def _module(tmp_path: Path, source: str) -> Path: "def f(x: Annotated[float, time.time] = 0.0) -> None:\n pass\n", ["5: time.time"], ), + ("from time import *\n", ["1: imports time.*"]), + ("from random import *\n", ["1: imports random.*"]), + ("from secrets import *\n", ["1: imports secrets.*"]), + ("from asyncio import *\n", ["1: imports asyncio.*"]), + ("from os import *\n", ["1: imports os.*"]), + ("from uuid import *\n", ["1: imports uuid.*"]), + ("from datetime import *\n", ["1: imports datetime.*"]), + ("import time\n\n\ndef f(x: g(time.time()) = 0) -> None:\n pass\n", ["4: time.time"]), + ("import time\n\n\ndef f() -> g(time.time()):\n pass\n", ["4: time.time"]), + ("import time\nx: g(time.time()) = 0\n", ["2: time.time"]), + ( + "import random\nseeds = []\nrng = random.Random(*seeds)\n", + ["3: random.Random without a seed"], + ), + ("from random import Random\nrng = Random(*())\n", ["2: Random without a seed"]), + ("import random\nrng = random.Random(**{})\n", ["2: random.Random without a seed"]), ("import time\ntime.tzset()\n", []), + ("import time\n\n\ndef f(x: time.struct_time) -> None:\n pass\n", []), ("import random\nrng = random.Random(7)\n", []), + ("import random\nrng = random.Random(x=7)\n", []), ("import random\n\n\ndef f(rng: random.Random) -> None:\n pass\n", []), ("import asyncio\n\n\nasync def f():\n await asyncio.sleep(0)\n", []), ("import uuid\nkey = uuid.uuid5(uuid.NAMESPACE_URL, 'x')\n", []), @@ -486,7 +746,20 @@ def test_the_determinism_scan_reports_each_spelling( ["2: test_inner inside test_outer"], ), ("def helper():\n class TestNested:\n pass\n", ["2: TestNested inside helper"]), + ( + "class Helper:\n def test_never_collected(self):\n pass\n", + ["2: test_never_collected inside Helper"], + ), + ( + "class Helper:\n class TestInner:\n def test_x(self):\n pass\n", + ["2: TestInner inside Helper", "3: test_x inside Helper"], + ), ("def test_a():\n pass\n\n\nclass TestB:\n def test_c(self):\n pass\n", []), + ( + "class TestOuter:\n class TestInner:\n def test_x(self):\n pass\n", + [], + ), + ("class Helper:\n def check(self):\n pass\n", []), ], ) def test_the_collection_scan_reports_each_nested_test( From ba7c183582cb154c847fbc46f1273c3fe595e797 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 13:22:00 -0600 Subject: [PATCH 24/29] Step 3b-2a-2d: exact results where the restored and rewritten tests had partial ones - The head-check refusals in the engine and run tests compare the whole message, the only result HeadCheckError gives. - The interrupted close records the database errors each ending meets: none for a second interrupt, the trigger's refusal for the other. - The first-sync failure states the run's error, GraphQL points, REST calls, and the repository's reason in full. - The restored query-lock bodies compare each problem line the checker prints, built from its message templates. - The history fixtures: a repository the test builds shows only tracked files are compared. - The guard: the other clock readings are refused, and asyncio answers every name but sleep; the scan reads scripts/; an upgrade from the first schema opens one driver connection. - tests/pytest_runs.py: the one pytest runner and check_junit loader the two test modules shared as copies. - ruff skips tests/fixtures/history, whose pyproject.toml is a copy. --- pyproject.toml | 3 ++ tests/db/test_migrations.py | 12 ++++++ tests/pytest_runs.py | 56 +++++++++++++++++++++++++++ tests/source/test_recorder.py | 16 ++++---- tests/sync/test_engine.py | 28 +++++++++----- tests/sync/test_runs.py | 30 ++++++++++----- tests/test_check_junit.py | 37 +++--------------- tests/test_check_query_lock.py | 34 +++++++++++++---- tests/test_history_fixtures.py | 28 ++++++++++++++ tests/test_suite_rules.py | 69 ++++++++++++++-------------------- 10 files changed, 208 insertions(+), 105 deletions(-) create mode 100644 tests/pytest_runs.py diff --git a/pyproject.toml b/pyproject.toml index 77c0394..490ac12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,9 @@ build-constraint-dependencies = [ target-version = "py312" line-length = 100 src = ["src", "tests"] +# Earlier versions of repository files, copied unchanged; their pyproject.toml is not this +# project's configuration. +extend-exclude = ["tests/fixtures/history"] [tool.ruff.lint] select = [ diff --git a/tests/db/test_migrations.py b/tests/db/test_migrations.py index 5f656c9..eec264a 100644 --- a/tests/db/test_migrations.py +++ b/tests/db/test_migrations.py @@ -775,6 +775,18 @@ def record(_dbapi_connection: Any, _record: Any, exception: BaseException | None assert snapshot(migrated_database) == before assert revision_of(migrated_database) == HEAD_REVISION + def test_an_upgrade_runs_on_one_driver_connection(self, first_schema: Path) -> None: + """The chain from the first schema to head opens one driver connection, and closes it.""" + connections: list[sqlite3.Connection] = [] + + def connected(dbapi_connection: sqlite3.Connection, _record: Any) -> None: + connections.append(dbapi_connection) + + with listening("connect", connected, Pool): + upgrade(first_schema) + assert [is_closed(connection) for connection in connections] == [True] + assert revision_of(first_schema) == HEAD_REVISION + def test_a_revision_gets_no_second_connection( self, migrated_database: Path, tmp_path: Path ) -> None: diff --git a/tests/pytest_runs.py b/tests/pytest_runs.py new file mode 100644 index 0000000..0fdce8a --- /dev/null +++ b/tests/pytest_runs.py @@ -0,0 +1,56 @@ +"""pytest run over test files a test writes, and scripts/check_junit.py loaded to read its report. + +Shared by the tests of check_junit.py and of the suite's own rules, so the two run pytest the same +way: its own configuration, only the environment variables it needs, and a bounded wait. +""" + +from __future__ import annotations + +import importlib.util +import os +import subprocess +import sys +from pathlib import Path +from types import ModuleType + +ROOT = Path(__file__).resolve().parents[1] +CHECK_JUNIT = ROOT / "scripts" / "check_junit.py" + + +def load_check_junit() -> ModuleType: + """scripts/check_junit.py, loaded from its file: CI runs it standalone, not as a package.""" + spec = importlib.util.spec_from_file_location("check_junit", CHECK_JUNIT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def run_pytest( + root: Path, files: dict[str, str], *options: str, importable: bool = False +) -> subprocess.CompletedProcess[str]: + """pytest over ``files``, written to ``root`` beside a ``pytest.ini`` of their own; the report + is ``root/junit.xml``. With ``importable``, this repository's package and tests import.""" + root.mkdir(exist_ok=True) + for name, text in {"pytest.ini": "[pytest]\n", **files}.items(): + (root / name).write_text(text) + environment = {"PATH": os.environ.get("PATH", ""), "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"} + if importable: + environment["PYTHONPATH"] = os.pathsep.join([str(ROOT / "src"), str(ROOT)]) + return subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + f"--junitxml={root / 'junit.xml'}", + *options, + ], + cwd=root, + capture_output=True, + text=True, + env=environment, + timeout=120, + ) diff --git a/tests/source/test_recorder.py b/tests/source/test_recorder.py index 2ebeae0..4a671b2 100644 --- a/tests/source/test_recorder.py +++ b/tests/source/test_recorder.py @@ -473,29 +473,31 @@ def _class(text: str | None) -> str | None: @pytest.mark.parametrize( - ("change", "error", "reason"), + ("change", "error", "reason", "points"), [ pytest.param( lambda replay: replay.fault( lambda request: httpx.Response(401, json={"message": "Bad credentials"}), times=None, ), - "Unauthorized", + "Unauthorized: GitHub answered 401: Bad credentials", None, + 0, id="401", ), pytest.param( lambda replay: replay.modify( _label_count_mismatch, when=lambda seen: seen.operation == "Resolve" ), - "NoRepositoryResolved", - "LabelCountMismatch", + "NoRepositoryResolved: no configured repository resolved", + "LabelCountMismatch: prebid/go-gdpr: fetched 10 labels, totalCount 11", + 1, id="label count mismatch", ), ], ) async def test_a_first_sync_that_fails_names_why( - change: Callable[[Replay], None], error: str, reason: str | None + change: Callable[[Replay], None], error: str, reason: str | None, points: int ) -> None: """A run that fails at its resolve raises with the run's report, which names the cause: the run's error, and the repository's when its resolve failed.""" @@ -514,12 +516,12 @@ async def test_a_first_sync_that_fails_names_why( 0, (), ) - assert _class(report.error) == error + assert (report.error, report.graphql_points, report.rest_calls) == (error, points, 0) assert [ (repository.configured.full_name, repository.record, repository.outcome) for repository in report.repositories ] == [(f"{owner}/{name}", None, None)] - assert [_class(repository.reason) for repository in report.repositories] == [reason] + assert [repository.reason for repository in report.repositories] == [reason] assert [seen.operation for seen in replay.seen] == ["Resolve"] diff --git a/tests/sync/test_engine.py b/tests/sync/test_engine.py index ccfa52f..ae289ae 100644 --- a/tests/sync/test_engine.py +++ b/tests/sync/test_engine.py @@ -1684,8 +1684,11 @@ async def test_the_head_check_comes_before_the_claim_and_any_request( world.url = f"sqlite+aiosqlite:///{missing}" for dry_run in (False, True): - with pytest.raises(HeadCheckError, match="database file not found"): + with pytest.raises(HeadCheckError) as refused: await world.sync(dry_run=dry_run) + assert str(refused.value) == ( + f"database file not found: {missing}; build it with `alembic upgrade head`" + ) assert not missing.exists() assert world.scenario.replay.seen == [] @@ -1695,11 +1698,14 @@ async def test_a_database_not_at_head_starts_no_run(world: World) -> None: project(world) world.execute("UPDATE alembic_version SET version_num = 'older'") - with pytest.raises( - HeadCheckError, match=f"at revision older, not the code's head {HEAD_REVISION}" - ): + with pytest.raises(HeadCheckError) as refused: await world.sync() + assert str(refused.value) == ( + f"the database is at revision older, not the code's head {HEAD_REVISION}; " + "run `alembic upgrade head`" + ) + assert world.rows("SELECT id FROM sync_runs") == [] assert world.scenario.replay.seen == [] @@ -1713,12 +1719,14 @@ async def test_a_database_at_the_previous_revision_starts_no_run( upgrade(older, MIGRATION_REVISIONS[-2]) world.url = f"sqlite+aiosqlite:///{older}" - with pytest.raises( - HeadCheckError, - match=f"revision {MIGRATION_REVISIONS[-2]}, not the code's head {HEAD_REVISION}", - ): + with pytest.raises(HeadCheckError) as refused: await world.sync() + assert str(refused.value) == ( + f"the database is at revision {MIGRATION_REVISIONS[-2]}, not the code's head " + f"{HEAD_REVISION}; run `alembic upgrade head`" + ) + assert fetch(older, "SELECT count(*) AS n FROM sync_runs") == [{"n": 0}] assert world.scenario.replay.seen == [] @@ -2863,13 +2871,15 @@ def interrupt( event.listen(Engine, "before_cursor_execute", interrupt) try: - with pytest.raises(KeyboardInterrupt) as raised: + with database_errors() as errors, pytest.raises(KeyboardInterrupt) as raised: await world.sync() finally: event.remove(Engine, "before_cursor_execute", interrupt) expected = ["complete", "aborted"] if cleanup == "second interrupt" else ["complete"] assert interrupted == expected + # The second interrupt meets no SQLite error; the trigger refuses the aborted close. + assert errors == ([] if cleanup == "second interrupt" else ["SQLITE_CONSTRAINT_TRIGGER"]) # The interrupt re-raised is the latest one delivered. assert raised.value.args == (expected[-1],) assert world.rows("SELECT status, exit_code FROM sync_runs") == [ diff --git a/tests/sync/test_runs.py b/tests/sync/test_runs.py index 034b5b1..17db5e8 100644 --- a/tests/sync/test_runs.py +++ b/tests/sync/test_runs.py @@ -90,26 +90,34 @@ async def test_the_head_check_passes_at_head(migrated_database: Path) -> None: async def test_a_missing_path_is_refused_without_creating_it(tmp_path: Path) -> None: missing = tmp_path / "absent.db" - with pytest.raises(HeadCheckError, match="database file not found"): + with pytest.raises(HeadCheckError) as refused: await check_head(_url(missing)) + assert str(refused.value) == ( + f"database file not found: {missing}; build it with `alembic upgrade head`" + ) assert not missing.exists() async def test_an_empty_file_is_refused(tmp_path: Path) -> None: empty = tmp_path / "empty.db" empty.touch() - with pytest.raises(HeadCheckError, match="no migration revision"): + with pytest.raises(HeadCheckError) as refused: await check_head(_url(empty)) + assert str(refused.value) == ( + "the database has no migration revision; build it with `alembic upgrade head`" + ) assert empty.stat().st_size == 0 async def test_an_older_revision_is_refused(tmp_path: Path) -> None: older = tmp_path / "older.db" upgrade(older, "33f8570f6540") - with pytest.raises( - HeadCheckError, match=f"revision 33f8570f6540, not the code's head {HEAD_REVISION}" - ): + with pytest.raises(HeadCheckError) as refused: await check_head(_url(older)) + assert str(refused.value) == ( + f"the database is at revision 33f8570f6540, not the code's head {HEAD_REVISION}; " + "run `alembic upgrade head`" + ) @pytest.mark.parametrize( @@ -124,8 +132,9 @@ async def test_the_claim_refuses_a_database_not_at_head(migrated_database: Path) connection = sqlite3.connect(migrated_database) connection.execute("PRAGMA user_version = 1") connection.close() - with pytest.raises(HeadCheckError, match="user_version is 1, not schema_version 2"): + with pytest.raises(HeadCheckError) as refused: await _claim(migrated_database) + assert str(refused.value) == "PRAGMA user_version is 1, not schema_version 2" assert fetch(migrated_database, "SELECT count(*) AS n FROM sync_runs") == [{"n": 0}] @@ -133,11 +142,12 @@ async def test_the_claim_refuses_a_database_at_the_previous_revision(tmp_path: P """A file the migrations built to the revision before head, as the code before it did.""" older = tmp_path / "older.db" upgrade(older, MIGRATION_REVISIONS[-2]) - with pytest.raises( - HeadCheckError, - match=f"revision {MIGRATION_REVISIONS[-2]}, not the code's head {HEAD_REVISION}", - ): + with pytest.raises(HeadCheckError) as refused: await _claim(older) + assert str(refused.value) == ( + f"the database is at revision {MIGRATION_REVISIONS[-2]}, not the code's head " + f"{HEAD_REVISION}; run `alembic upgrade head`" + ) assert fetch(older, "SELECT count(*) AS n FROM sync_runs") == [{"n": 0}] diff --git a/tests/test_check_junit.py b/tests/test_check_junit.py index eb4c3c3..9a98733 100644 --- a/tests/test_check_junit.py +++ b/tests/test_check_junit.py @@ -9,41 +9,23 @@ import ast import copy -import importlib.util import os import subprocess import sys import xml.etree.ElementTree as ET from pathlib import Path -from types import ModuleType import pytest +from tests.pytest_runs import load_check_junit, run_pytest + SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "check_junit.py" COUNTED = ("tests", "failures", "errors", "skipped") def _pytest(root: Path, test_body: str) -> subprocess.CompletedProcess[str]: - """Run pytest over one test file in ``root``, with its own configuration and only the - environment variables it needs; the report is ``root/junit.xml``.""" - (root / "pytest.ini").write_text("[pytest]\n") - (root / "test_sample.py").write_text(f"import pytest\n\n\n{test_body}") - return subprocess.run( - [ - sys.executable, - "-m", - "pytest", - "-q", - "-p", - "no:cacheprovider", - f"--junitxml={root / 'junit.xml'}", - ], - cwd=root, - capture_output=True, - text=True, - env={"PATH": os.environ.get("PATH", ""), "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}, - timeout=60, - ) + """Run pytest over one test file in ``root``; the report is ``root/junit.xml``.""" + return run_pytest(root, {"test_sample.py": f"import pytest\n\n\n{test_body}"}) @pytest.fixture(scope="module") @@ -127,15 +109,6 @@ def test_sums_every_testsuite(tmp_path: Path, written: bytes) -> None: assert len(result.stderr.splitlines()) == 1 -def _load() -> ModuleType: - """The script as a module, so its ``main`` can be called in process.""" - spec = importlib.util.spec_from_file_location("check_junit", SCRIPT) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - @pytest.mark.parametrize("shape", ["missing", "cut short", "no testsuite"]) def test_unreadable_report_fails( tmp_path: Path, written: bytes, shape: str, capsys: pytest.CaptureFixture[str] @@ -152,7 +125,7 @@ def test_unreadable_report_fails( elif shape == "no testsuite": path = _report(tmp_path, written) problem = f"no element in {path}" - assert _load().main([str(path)]) == 1 + assert load_check_junit().main([str(path)]) == 1 # The counters are printed only after the report is read. assert capsys.readouterr() == ("", f"cannot read JUnit report: {problem}\n") diff --git a/tests/test_check_query_lock.py b/tests/test_check_query_lock.py index c0ed9b8..381af49 100644 --- a/tests/test_check_query_lock.py +++ b/tests/test_check_query_lock.py @@ -129,6 +129,19 @@ def removed_version(version: str) -> str: return f"{SCHEMA_LOCK}: schema version {version} was removed; lock entries are append-only" +def changed_entry(key: str, before: str, now: str) -> str: + """The line the checker prints for a query lock entry whose digest the pull request changed.""" + return f"{LOCK}: {key} changed from {before} to {now}; add a version instead" + + +def lost_pair(version: str, table: str, column: str) -> str: + """The line the checker prints for a consumer pair a schema version lost.""" + return ( + f"{SCHEMA_LOCK}: schema version {version} lost {table}.{column}; removing or renaming a " + "consumer column needs a new schema version" + ) + + def lock_lines(stderr: str) -> list[str]: """The lines of ``stderr`` that name a lock, one per problem the checker found; the CI step's stderr also holds git's own output.""" @@ -170,7 +183,7 @@ def test_a_removed_entry_fails(repo: Path) -> None: write_lock(repo, {"pull_request:2": "b" * 64}) result = check(repo) assert result.returncode == 1 - assert "pull_request:1 was removed" in result.stderr + assert result.stderr.splitlines() == [removed_entry("pull_request:1")] def test_an_earlier_query_lock_over_a_later_base_fails(repo: Path) -> None: @@ -355,7 +368,7 @@ def test_a_removed_entry_fails(self, repo: Path) -> None: write_schema_lock(repo, {"2": V1["1"]}) result = check(repo) assert result.returncode == 1 - assert "schema version 1 was removed" in result.stderr + assert result.stderr.splitlines() == [removed_version("1")] def test_an_earlier_schema_lock_without_the_newest_version_fails(self, repo: Path) -> None: place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[1]) @@ -385,7 +398,7 @@ def test_a_pair_removed_from_an_object_entry_fails(self, repo: Path) -> None: write_schema_lock(repo, {**V2, "2": {**V2["2"], "columns": [["pull_requests", "number"]]}}) result = check(repo) assert result.returncode == 1 - assert "schema version 2 lost v_pr_reviews.state" in result.stderr + assert result.stderr.splitlines() == [lost_pair("2", "v_pr_reviews", "state")] def test_an_earlier_schema_lock_without_a_newer_pair_fails(self, repo: Path) -> None: place(repo, SCHEMA_LOCK, SCHEMA_LOCKS[2]) @@ -532,8 +545,11 @@ def test_both_locks_are_checked_by_default(self, repo: Path) -> None: write_schema_lock(repo, {"1": []}) result = check(repo) assert result.returncode == 1 - assert "pull_request:1 changed" in result.stderr - assert "schema version 1 lost pull_requests.number" in result.stderr + assert result.stderr.splitlines() == [ + changed_entry("pull_request:1", "a" * 64, "c" * 64), + lost_pair("1", "pull_requests", "number"), + lost_pair("1", "v_pr_reviewers", "final_verdict"), + ] def test_both_earlier_locks_are_checked_by_default(self, repo: Path) -> None: place(repo, LOCK, QUERY_LOCKS[1]) @@ -553,7 +569,10 @@ def test_lock_selects_one(self, repo: Path) -> None: write_lock(repo, {"pull_request:1": "c" * 64}) result = run_checker(repo, "--base", "main", "--lock", SCHEMA_LOCK) assert result.returncode == 0, result.stderr - assert "query_versions" not in result.stdout + result.stderr + assert result.stdout.splitlines() == [ + f"{SCHEMA_LOCK}: 1 entries at main kept; 1 entries now" + ] + assert result.stderr == "" def test_lock_selects_one_of_two_earlier_locks(self, repo: Path) -> None: place(repo, LOCK, QUERY_LOCKS[1]) @@ -842,6 +861,7 @@ def test_the_step_runs_the_base_checker_not_the_pull_requests(tmp_path: Path, or result = run_step_on_merge(tmp_path, origin, work, edit_entry_and_relax_checker) assert result.returncode == 1, result.stdout + result.stderr assert "runner-temp/check_query_lock.py" in result.stdout + assert lock_lines(result.stderr) == [changed_entry("pull_request:1", "a" * 64, "c" * 64)] def test_the_step_runs_the_base_checker_where_the_pull_request_deletes_its_copy( @@ -900,7 +920,7 @@ def remove_pair(target: Path) -> None: result = run_step_on_merge(tmp_path, origin, work, remove_pair) assert result.returncode == 1, result.stdout + result.stderr - assert "schema version 1 lost v_pr_reviewers.final_verdict" in result.stderr + assert lock_lines(result.stderr) == [lost_pair("1", "v_pr_reviewers", "final_verdict")] def test_the_step_fails_an_earlier_schema_lock(tmp_path: Path, origin: Path) -> None: diff --git a/tests/test_history_fixtures.py b/tests/test_history_fixtures.py index 90ee6d2..8783983 100644 --- a/tests/test_history_fixtures.py +++ b/tests/test_history_fixtures.py @@ -200,3 +200,31 @@ def test_a_table_line_it_does_not_model_is_refused(text: str, line: int | None) with pytest.raises(TableError) as raised: read_rows(text) assert raised.value.line == line + + +def test_only_the_files_git_tracks_are_compared(tmp_path: Path) -> None: + """A repository the test builds: the README and one copy committed, one file never added, and + one git ignores. Only the committed copy is compared.""" + assert GIT is not None + repo, home = tmp_path / "repo", tmp_path / "home" + copies = repo / "history" + (copies / "abc1234").mkdir(parents=True) + home.mkdir() + (copies / README).write_text("rows\n") + (copies / "abc1234" / "kept.toml").write_text("kept = 1\n") + (copies / ".gitignore").write_text("ignored.txt\n") + environment = { + "PATH": os.environ["PATH"], + "HOME": str(home), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + } + for args in ( + ["init", "-q"], + ["add", "history/README.md", "history/abc1234/kept.toml"], + ["-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "copies"], + ): + subprocess.run([GIT, *args], cwd=repo, env=environment, check=True, timeout=60) + (copies / "untracked.toml").write_text("untracked = 1\n") + (copies / "ignored.txt").write_text("ignored\n") + assert tracked_files(copies, home) == ["abc1234/kept.toml"] diff --git a/tests/test_suite_rules.py b/tests/test_suite_rules.py index ee3bdf4..8854774 100644 --- a/tests/test_suite_rules.py +++ b/tests/test_suite_rules.py @@ -40,14 +40,11 @@ from __future__ import annotations import ast -import importlib.util +import asyncio import os import socket -import subprocess -import sys from datetime import UTC, datetime, tzinfo from pathlib import Path -from types import ModuleType import pytest @@ -57,12 +54,13 @@ from github_activity_db.source.clock import SystemClock from tests import determinism from tests.determinism import NotDeterministic +from tests.pytest_runs import load_check_junit, run_pytest from tests.source.conftest import FakeClock TESTS = Path(__file__).resolve().parent SRC = TESTS.parent / "src" / "github_activity_db" ALEMBIC = TESTS.parent / "alembic" -CHECK_JUNIT = TESTS.parent / "scripts" / "check_junit.py" +SCRIPTS = TESTS.parent / "scripts" WAITING = frozenset({"run", "call", "check_call", "check_output"}) # subprocess names that start no process. INERT = frozenset( @@ -335,6 +333,13 @@ def test_the_migrations_read_no_wall_clock_or_randomness() -> None: assert determinism_problems(*files) == [] +def test_the_scripts_read_no_wall_clock_or_randomness() -> None: + """Over ``scripts/``, whose checks the suite also runs in process, the scan finds nothing.""" + files = _python_files(SCRIPTS) + assert [path.name for path in files if path.name in ("check_junit.py", "count_suppressions.py")] + assert determinism_problems(*files) == [] + + def test_no_test_is_defined_inside_a_function_or_a_non_test_class() -> None: assert uncollected_tests(*_test_files()) == [] @@ -423,6 +428,22 @@ def test_the_real_clock_is_refused() -> None: assert _refused(refusal) == ("wall clock", "source.clock: datetime.now(UTC)") +@pytest.mark.parametrize("reading", ["utcnow", "today"]) +def test_the_real_clocks_other_readings_are_refused(reading: str) -> None: + """The datetime source/clock.py sees refuses each way to read the time, not only ``now``.""" + with pytest.raises(NotDeterministic) as refusal: + getattr(vars(clock)["datetime"], reading)() + assert _refused(refusal) == ("wall clock", f"source.clock: datetime.{reading}()") + + +def test_the_real_clocks_asyncio_answers_everything_but_sleep() -> None: + """The asyncio source/clock.py sees is asyncio for every name but ``sleep``.""" + seen = vars(clock)["asyncio"] + assert seen.get_running_loop is asyncio.get_running_loop + assert seen.sleep.__qualname__ == "_NoSleep.sleep" + assert determinism.REFUSED == [] + + async def test_the_real_clocks_sleep_is_refused() -> None: with pytest.raises(NotDeterministic) as refusal: await SystemClock().sleep(2.0) @@ -500,48 +521,16 @@ async def sleep(seconds: float) -> None: assert slept == [2.5] -def _check_junit() -> ModuleType: - """scripts/check_junit.py, loaded from its file: CI runs it standalone.""" - spec = importlib.util.spec_from_file_location("check_junit", CHECK_JUNIT) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - def _pytest_over( root: Path, files: dict[str, str], *options: str ) -> tuple[int, list[tuple[str, str]]]: """pytest run over ``files``, written to ``root`` beside a configuration of their own, with this repository importable: its exit code, and each test's outcome in the order its JUnit report lists the tests, which is the order they ran in.""" - root.mkdir() - for name, text in {"pytest.ini": "[pytest]\n", **files}.items(): - (root / name).write_text(text) + run = run_pytest(root, files, *options, importable=True) report = root / "junit.xml" - run = subprocess.run( - [ - sys.executable, - "-m", - "pytest", - "-q", - "-p", - "no:cacheprovider", - f"--junitxml={report}", - *options, - ], - cwd=root, - capture_output=True, - text=True, - env={ - "PATH": os.environ.get("PATH", ""), - "PYTHONPATH": os.pathsep.join([str(TESTS.parent / "src"), str(TESTS.parent)]), - "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", - }, - timeout=120, - ) assert report.is_file(), run.stdout + run.stderr - return run.returncode, list(_check_junit().read_outcomes(report).items()) + return run.returncode, list(load_check_junit().read_outcomes(report).items()) # Catches the refusal of the real clock, as code under test may. @@ -628,7 +617,7 @@ def test_reverse_order_runs_the_tests_last_to_first( ], ) forward, backward = (str(tmp_path / run / "junit.xml") for run in ("forward", "backward")) - assert _check_junit().main([forward, "--max-skipped", "0", "--same-as", backward]) == 1 + assert load_check_junit().main([forward, "--max-skipped", "0", "--same-as", backward]) == 1 assert capsys.readouterr() == ( "{'tests': 3, 'failures': 0, 'errors': 0, 'skipped': 0}\n", "FAIL: differs: test_one::test_b: passed vs failed\n", From 7bd4f64e369a36a3b9e3c5d455e934c8912684d7 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 13:24:01 -0600 Subject: [PATCH 25/29] Step 3b-2a-2d: the documents describe the tests and guard as they are - D40 in its four copies: no result a test asserts comes from the wall clock; a timeout that bounds a wait and a log record's time are not results. What determinism.py refuses is named: connections, lookups, the real clocks, the unseeded random source. - Step 3b-2a-2d: its scope (the CI run in reverse order, db.types' clock seam, env.py's one connection, ruff's exclusion), what it keeps and why six tests are not kept, and its acceptance as CI checks it. - Step 3b-2b-2: verify's createdAt filter is tested on both sides. - docs/testing.md: what enforces D40, the layout, the two-run command as CI runs it, the CI job; docs/development.md: the CI job; docs/database.md: an upgrade runs on one driver connection. --- CLAUDE.md | 32 ++++++++------- CONTRIBUTING.md | 19 +++++---- docs/database.md | 4 ++ docs/development.md | 7 ++-- docs/redesign-plan.md | 75 +++++++++++++++++++++------------- docs/testing.md | 86 +++++++++++++++++++++++---------------- tests/test_suite_rules.py | 3 +- 7 files changed, 135 insertions(+), 91 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c84d0d0..6819c71 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,19 +57,22 @@ that fails against the parent commit's code (redesign plan D39). See [Testing](d ## Deterministic Tests Tests are software that gives the same result every time (redesign plan D40). The same code gives -every test the same outcome on any machine, on any date, and in any run order: a test, and every -code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no -network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state -outside its own temporary directory, the repository, and the session's database template, and bounds -every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or -exits with: where the code gives a structured value (an exit code, rows, parsed fields, an -exception's class and fields), the test asserts on it rather than on printed wording, and no test -checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test changes the project's real files. Evidence is what software reproduces: an acceptance criterion +every test the same outcome on any machine, on any date, and in any run order: no result a test +asserts comes from the wall clock (a fixed clock such as `FakeClock` stands in; a timeout that +bounds a wait and a log record's time are not results), the network (GitHub answers from recordings +and stubs), unseeded randomness, or state outside the test's temporary directory, the repository, +and the session's database template, and every wait is bounded by a timeout. A test exercises code +and asserts on what the code returns, writes, or exits with: where the code gives a structured value +(an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather +than on printed wording, and no test checks the wording of a document. Where a test needs an input +the code cannot easily be given, the project builds software that produces it (a fake clock, the +scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test +changes the project's real files. Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a -review finds work to do but is never acceptance. `tests/determinism.py` refuses the network, the -real clock, and its unseeded random source in every test, `tests/test_suite_rules.py` enforces the -rules it can model in source, and `scripts/check_junit.py --same-as` compares two runs of the suite -test by test (Step 3b-2a-2d). +review finds work to do but is never acceptance. `tests/determinism.py` refuses network connections, +host-name lookups, the real clocks, and the unseeded random source in every test, +`tests/test_suite_rules.py` enforces the rules it can model in source, and `scripts/check_junit.py +--same-as` compares two runs of the suite test by test (Step 3b-2a-2d). ## Current Scope @@ -93,8 +96,9 @@ test by test (Step 3b-2a-2d). - `ghactivity sync` (Step 3b-2a-2c, `cli/sync.py`): the engine's run with the options of plan 5.7; a run that never starts exits 1, and a run that starts exits with the engine's 0, 2, or 1 - Deterministic tests, with D39's negative tests (Step 3b-2a-2d, D39, D40): `tests/determinism.py` - refuses the network, the real clock, and its unseeded random source in every test; - `tests/test_suite_rules.py`'s source rules; `scripts/check_junit.py --same-as` + refuses network connections, host-name lookups, the real clocks, and the unseeded random source + in every test; `tests/test_suite_rules.py`'s source rules; `scripts/check_junit.py --same-as` + over a run in order and one reversed under another hash seed, in CI **Not implemented yet:** D37 in the sync (Step 3b-2a-3, D38), the REST sampler (Step 3b-2b-1), and `verify` (Step 3b-2b-2). The tracked `github_activity.db` stays diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 852a6c8..62d4fdd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -260,14 +260,17 @@ or values it compared, so it cannot pass on nothing. A fix comes with a regressi against the parent commit's code. See the [Testing Guide](docs/testing.md#principles). **4. Write tests that give the same result every time** (redesign plan D40). The same code gives -every test the same outcome on any machine, on any date, and in any run order: a test, and every -code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no -network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state -outside its own temporary directory, the repository, and the session's database template, and bounds -every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or -exits with: where the code gives a structured value (an exit code, rows, parsed fields, an -exception's class and fields), the test asserts on it rather than on printed wording, and no test -checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test changes the project's real files. Evidence is what software reproduces: an acceptance criterion +every test the same outcome on any machine, on any date, and in any run order: no result a test +asserts comes from the wall clock (a fixed clock such as `FakeClock` stands in; a timeout that +bounds a wait and a log record's time are not results), the network (GitHub answers from recordings +and stubs), unseeded randomness, or state outside the test's temporary directory, the repository, +and the session's database template, and every wait is bounded by a timeout. A test exercises code +and asserts on what the code returns, writes, or exits with: where the code gives a structured value +(an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather +than on printed wording, and no test checks the wording of a document. Where a test needs an input +the code cannot easily be given, the project builds software that produces it (a fake clock, the +scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test +changes the project's real files. Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. diff --git a/docs/database.md b/docs/database.md index d54cf29..4676a59 100644 --- a/docs/database.md +++ b/docs/database.md @@ -67,6 +67,10 @@ converted to UTC, and returns aware UTC values. Defaults are `utc_now()`, never `user_version` and the pragmas reflection reads with an argument (`table_info`, `table_xinfo`, `index_list`, `index_info`, `foreign_key_list`), so `journal_mode`, `legacy_alter_table`, and `ignore_check_constraints` keep the values the connection opened with. +- An upgrade runs on one driver connection. Once it is open, env.py refuses and closes any + other the engine would open: a second connection a revision asks for, or the one SQLAlchemy + would open after it invalidated the first for an interrupt or an error it reads as a + disconnect. The upgrade then ends with that error, and the file keeps its revision. - Refused: offline `--sql` mode, databases other than SQLite, SQLite URI filenames (`?uri=true`), a caller-supplied connection or `sqlalchemy.url`, and the tracked `github_activity.db`. The connect listener asks SQLite which file it opened, so the tracked diff --git a/docs/development.md b/docs/development.md index 3144e3b..e1e5052 100644 --- a/docs/development.md +++ b/docs/development.md @@ -245,9 +245,10 @@ ruff and mypy run through `uv run --no-sync`, so hooks never change the environm one, was removed or the digest changed, or pairs or views were added to an entry that is not the newest; a lock it cannot read as its rule expects (a malformed entry or digest, or a key listed twice) also fails the step; -- pytest on Python 3.12 and 3.13, then `scripts/check_junit.py --max-skipped 0`, which fails - unless at least one test ran and none failed, errored, or was skipped (pytest reports `xfail` - as skipped); +- pytest on Python 3.12 and 3.13, then `scripts/check_junit.py --max-skipped 0`, which fails unless + at least one test ran and none failed, errored, or was skipped (pytest reports `xfail` as + skipped); then pytest again with the tests in reverse order under another `PYTHONHASHSEED`, and + `check_junit.py --same-as`, which fails on each test whose outcome differs between the two runs; - `alembic upgrade head` plus `alembic check` against a fresh database; - a built wheel (build dependencies hash-checked against `build-constraints.txt`), installed into a new environment after the `uv.lock` dependencies (with hash checking, then diff --git a/docs/redesign-plan.md b/docs/redesign-plan.md index 77ab417..2a5d97b 100644 --- a/docs/redesign-plan.md +++ b/docs/redesign-plan.md @@ -94,7 +94,7 @@ Within scope the database collects and distinguishes data and never interprets i | D37 | Decision of 2026-10-21 (the maintainer): an error is better than a silent failure or silently wrong data. Where the code cannot check something, or cannot accept what GitHub returned, it reports a failure that stops publication or turns the run red, never a pass. For verify: an unchecked item and an unverified repository fail verify in both modes, not only `--exact`, and a lookup whose answer verify cannot accept is an error (5.4, 5.8, gap 97), so D25 does not publish a database verify could not check. | | D38 | Decisions of 2026-10-21 applying D37 to the sync and to the steps not yet built (gaps 98-110). The sync exits 2 when a 24-hour account refresh fails, when the run reaches its deadline, and when a hydration contradicts the run's own index after one retry (Step 3b-2a-3); the failures a batch made final before a deadline are recorded, not left pending. An `accepted_failures` entry accepts one error class and one cause, the first line of the failure's message, of an item, an account's refresh, or from Step 4b a compare, so a permanent failure can be accepted without turning every run red while a new cause is not. A PR with more closing links than the index carries is hydrated every run. Every node id a sighting, a lookup, or a commit link reports for a stored account is stored, and `sync_runs` records the `--repo` filter, each configured entry's `github_id`, and its first commit-link lookups and those GitHub answered `NOT_FOUND`. The source refuses what it cannot read (gap 104), and the run records refuse what they cannot write and mark `-unknown` what they cannot determine (gap 105). Step 4b marks a compare file list at GitHub's 300-file cap and counts it in verify's report and the manifest, completes an activity read only at the floor or at the log's first event, as a read in ascending order finds it, fails a history read whose count still differs after a rescan, and records a compare that still fails, which an entry can accept. Step 5 measures which edits leave `updatedAt` unchanged. Step 6 compares the App's reads with the token's before any App run publishes to `db`, pins the database file the job checked, tests that every exit-code `if:` compares a non-empty string, and ties a snapshot to the job's own sync. A schedule that stops is the one silent case this repository cannot see: no run starts, so nothing turns red; 5.6 states the staleness rule a consumer applies, and building it belongs to the dashboard's integration (section 9). Left as they are, because the stored data already says what happened: a commit-linked account a lookup answers `NOT_FOUND` (D28; now counted per run and in the manifest, and from Step 6a a run whose `links_not_found` exceeds the share of its `link_lookups` Step 5 sets, with at least Step 5's minimum number of lookups, exits 2 (an account an earlier lookup answered `NOT_FOUND` counted in neither), since a lookup form GitHub stopped resolving would look the same per link), a compare at GitHub's 300-file cap and one GitHub refuses (marked by `files_at_cap` and `http_status`, counted in verify's report and the manifest), a REST git author without a name or email (stored null, as GitHub reports none), deletions recorded as a count (`items_deleted`, and from Step 4b `events_deleted` and `history_commits_deleted`; the snapshots retained for 7 days hold the identities; older ones are not kept), which accounts failed a refresh (printed; the count is stored, the run is red unless an account entry accepts the failure, and the accounts are retried), an `author_association` difference (verify reports it as drift with its own count: GitHub computes association when read and changes it without moving `updatedAt`, section 7, so no read tells a change from a defect), and the merge state as of `hydrated_at` (D32). New tables and columns of GitHub-derived data on a populated file follow gap 110's rule (5.1). | | D39 | Decision of 2026-10-21 (the maintainer), clarified on 2026-10-22: the first wording also banned negative tests, and Step 3b-2a-2d removed some before the clarification restored them. No mutation testing, under any name: the code under test and the checks are never changed, on disk or in memory, to see whether a test notices. No false content is planted in the project's real files or documents (the plan, a document, review material, a tracked lock or database) to see whether a check or a reviewer catches it, and no reviewer or agent is scored on how many seeded flaws it finds. Negative tests are expected: a test builds its own invalid or failing input on purpose (a sample test that fails, an invalid configuration, a malformed or edited copy of a file or database in its temporary directory, a test-only revision that raises, a stub or fake that raises, a recorded GitHub response changed in memory) and feeds it to the unchanged code. Each test asserts the exact result. Every check is tested on both sides, and its passing side states the exact count of the rows, files, or values it compared, so it cannot pass on nothing. A fix comes with a regression test that fails against the parent commit's code. | -| D40 | Decision of 2026-10-22 (the maintainer). Tests are software that gives the same result every time. The same code gives every test the same outcome on any machine, on any date, and in any run order: a test, and every code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state outside its own temporary directory, the repository, and the session's database template, and bounds every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on printed wording, and no test checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test changes the project's real files. Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. `tests/determinism.py` refuses the network, the real clock, and its unseeded random source in every test, `tests/test_suite_rules.py` enforces the rules it can model in source, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test (Step 3b-2a-2d). | +| D40 | Decision of 2026-10-22 (the maintainer). Tests are software that gives the same result every time. The same code gives every test the same outcome on any machine, on any date, and in any run order: no result a test asserts comes from the wall clock (a fixed clock such as `FakeClock` stands in; a timeout that bounds a wait and a log record's time are not results), the network (GitHub answers from recordings and stubs), unseeded randomness, or state outside the test's temporary directory, the repository, and the session's database template, and every wait is bounded by a timeout. A test exercises code and asserts on what the code returns, writes, or exits with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on printed wording, and no test checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test changes the project's real files. Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. `tests/determinism.py` refuses network connections, host-name lookups, the real clocks, and the unseeded random source in every test, `tests/test_suite_rules.py` enforces the rules it can model in source, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test, which CI runs in two orders under two hash seeds (Step 3b-2a-2d). | ## 3. Coverage rule @@ -1937,35 +1937,53 @@ check without a test that targets it; CI passes on the exact PR head. Prerequisite: Step 3b-2a-2c has merged, and D39 is on `main`. -Scope: tests, `docs/testing.md`, `scripts/check_junit.py`'s `--same-as`, and one change under -`src/`: `ghactivity sync` times its report on the run's clock, not `time.monotonic` (D40); and one -under `alembic/`: `env.py` records a failed upgrade's state without asking SQLAlchemy for the -connection again, so an interrupt inside a statement, which SQLAlchemy answers by invalidating the -connection, propagates instead of a `PendingRollbackError`. No rule loses its test. Every test meets -D40. -`tests/determinism.py` refuses, in the pytest process from collection on, a network connection -other than a Unix socket, every host-name lookup the `socket` module offers, and this project's -real clock and unseeded random source (`SystemClock`), with `no_proxy` set so the machine's proxy -settings change nothing, and records each refusal, so a test fails even where the code under test -caught it; each refusal, and the Unix socket it lets through, has a test. `tests/test_suite_rules.py` gains the rules it can model in source: no test file reads the -wall clock, sleeps on it, or draws unseeded random numbers; the package reads them only in its two -real clocks (`SystemClock` and `db.types.utc_now`); and no test function is defined inside another -function, where pytest never collects it. Each is tested on both sides: on sample source the test writes holding each -spelling it refuses, and over every file under `tests/` or `src/github_activity_db/`, the files it scanned counted by a second walk of the -directory, and the package scan with the exact list of reads it found. Every test on `main` before this step whose input the test built itself (a test-only revision that -fails, a changed copy of a database or file in its temporary directory, an edited lock entry, a -sample file holding each spelling a checker flags, a stub or fake that raises) is kept, under D39 as -clarified, beside the tests this step adds that read real earlier revisions (`upgrade(path, -"")`) and real earlier files (`tests/fixtures/history/`). A test this step removes is one -D40 refuses: it read the wall clock or depended on the uid, or it checked a document's wording. +Scope: tests; `docs/testing.md`, `docs/database.md`, `docs/development.md`, `CLAUDE.md`, and +`CONTRIBUTING.md`; `scripts/check_junit.py`'s `--same-as`; the tests job of +`.github/workflows/check.yml`, which runs the suite a second time in reverse order under another +`PYTHONHASHSEED` and compares the two runs; `pyproject.toml`, where ruff skips +`tests/fixtures/history/`; two changes under `src/`: `ghactivity sync` times its report on the run's +clock, not `time.monotonic`, and `db.types.utc_now` reads the time through `_wall_clock`, which a +test replaces or refuses (D40); and one under `alembic/`: `env.py` runs an upgrade on one driver +connection and records a failed upgrade's state on that connection's mapping without asking +SQLAlchemy for the connection again, so an interrupt, a `SystemExit`, or an error SQLAlchemy reads +as a disconnect propagates itself instead of a `PendingRollbackError`. No rule loses its test. Every +test meets D40. + +`tests/determinism.py` refuses, in the pytest process from collection on, a connection other than to +a Unix socket, each host-name lookup function the `socket` module offers, the package's two real +clocks (`source/clock.py`'s readings and sleep, and `db.types.utc_now`'s wall clock), and every draw +from `SystemClock`'s unseeded random source, with `no_proxy` set so the machine's proxy settings +change nothing, and records each refusal, so a test fails even where the code under test caught it; +each refusal, the Unix socket it lets through, and the after-test check's failing side have tests. +`tests/test_suite_rules.py` gains the rules it can model in source: no test file reads the wall +clock, sleeps on it, or draws unseeded random numbers; the package reads them only in its two real +clocks, and `alembic/` and `scripts/` read none; and no test function is defined inside a function +or a class whose name does not start with `Test`, where pytest never collects it. Each is tested on +both sides: on sample source the test writes holding each spelling it refuses, and over every file +under `tests/`, `src/github_activity_db/`, `alembic/`, or `scripts/`, the files it scanned counted +by a second walk of the directory, and the package scan with the exact list of reads it found. + +Every test on `main` before this step whose input the test built itself (a test-only revision that +fails, a changed copy of a database or file in its temporary directory, a trigger in its own +database, an edited lock entry, a sample file holding each spelling a checker flags, a stub or fake +that raises) is kept under its name, under D39 as clarified. Where this step had rewritten such a +test under its name, main's body is back, and the rewrite keeps a name of its own where it holds +more (a real earlier file or revision, an exact count). The tests this step adds read real earlier +revisions (`upgrade(path, "")`) and real earlier files (`tests/fixtures/history/`, each +copy's git blob id in its README, checked by `tests/test_history_fixtures.py`). Six tests on `main` +are not kept: one depended on the uid and one checked a document's wording (D40), and four are held +by tests over real inputs (two assignee tests by `test_an_assignee_github_id_under_two_node_ids`, +with GitHub's ids for one bot; two query-lock cases by +`test_the_verdict_does_not_depend_on_how_the_checker_runs`). Tests: every test on `main` before this step is at head under its name, or the pull request names -the test that replaced it and the reason; each check has a test on both sides. +the test that holds its rule and the reason; each check has a test on both sides. -Acceptance: `tests/test_suite_rules.py`'s rules pass over every test file and package module and -count the files they scanned; each refusal of `tests/determinism.py` has a test; two runs of the -full suite give the same outcome for every test, compared by `scripts/check_junit.py --same-as`; -CI passes on the exact PR head. +Acceptance: `tests/test_suite_rules.py`'s rules pass over every file under `tests/`, +`src/github_activity_db/`, `alembic/`, and `scripts/`, and count the files they scanned; each +refusal of `tests/determinism.py` has a test; CI runs the full suite in order and in reverse order +under two hash seeds, and `scripts/check_junit.py --same-as` finds every outcome the same; CI passes +on the exact PR head. #### Step 3b-2a-3: D37 in the sync @@ -2179,8 +2197,7 @@ another issue, a link difference made by a closing keyword with no event, a diff link totals alone with enough connected events, with only a disconnected event, and with too few, a synthetic disconnected event, an event at the boundary counted and one a second before it not (under the measured `since` -and whole-second times, verify's own `createdAt` filter drops no entry the lookup returns, so it -is tested on its passing side, every returned entry counted), a stored link no longer indexed on a PR whose index lists every link matched only by an +and whole-second times, verify's own `createdAt` filter drops no entry the lookup returns; it is tested on its passing side, every returned entry counted, and on a lookup answer the test changes in memory to hold an entry before `since`, which the filter drops), a stored link no longer indexed on a PR whose index lists every link matched only by an event naming it, a filtered page whose `totalCount` differs from its entries, counted by its entries, a lookup whose entries differ from its `filteredCount` after its retry and one whose response the typed models reject, each an error, one that differs once and matches on its retry, classed by its events, and a diff --git a/docs/testing.md b/docs/testing.md index 7f8ca00..bba7ea5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -23,19 +23,22 @@ How the test suite is built and what it counts as evidence. The plan of record i rows, files, or values it compared, so it cannot pass on nothing. A fix comes with a regression test that fails against the parent commit's code. - **The same result every time** (redesign plan D40). The same code gives every test the same - outcome on any machine, on any date, and in any run order: a test, and every code path it runs, - reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no network (GitHub - answers from recordings and stubs), uses no unseeded randomness, depends on no state outside its - own temporary directory, the repository, and the session's database template, and bounds every - wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits - with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's - class and fields), the test asserts on it rather than on printed wording, and no test checks the - wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test changes the project's real files. Evidence is what software reproduces: an acceptance criterion names a - test, a check, or a command whose output shows it, and a search of text, a reading, or a review - finds work to do but is never acceptance. `tests/determinism.py` refuses the network, the real - clock, and its unseeded random source in every test, `tests/test_suite_rules.py` enforces the - rules it can model in source, and `scripts/check_junit.py --same-as` compares two runs of the - suite test by test (Step 3b-2a-2d). + outcome on any machine, on any date, and in any run order: no result a test asserts comes from the + wall clock (a fixed clock such as `FakeClock` stands in; a timeout that bounds a wait and a log + record's time are not results), the network (GitHub answers from recordings and stubs), unseeded + randomness, or state outside the test's temporary directory, the repository, and the session's + database template, and every wait is bounded by a timeout. A test exercises code and asserts on + what the code returns, writes, or exits with: where the code gives a structured value (an exit + code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on + printed wording, and no test checks the wording of a document. Where a test needs an input the + code cannot easily be given, the project builds software that produces it (a fake clock, the + scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test + changes the project's real files. Evidence is what software reproduces: an acceptance criterion + names a test, a check, or a command whose output shows it, and a search of text, a reading, or a + review finds work to do but is never acceptance. `tests/determinism.py` refuses network + connections, host-name lookups, the real clocks, and the unseeded random source in every test, + `tests/test_suite_rules.py` enforces the rules it can model in source, and `scripts/check_junit.py + --same-as` compares two runs of the suite test by test (Step 3b-2a-2d). - **No skipped tests** (D22). CI runs `scripts/check_junit.py --max-skipped 0`, which fails a run where any test was skipped (pytest reports `xfail` as skipped) or none ran. - **Time.** Source tests pass a `FakeClock`, so waits and retries are asserted, never slept. @@ -45,19 +48,24 @@ How the test suite is built and what it counts as evidence. The plan of record i `sys.modules`) and process APIs outside `subprocess` (`os.system`, `asyncio` subprocesses) are outside its scan. - **What enforces D40.** In the pytest process, from collection on, `tests/determinism.py` - (installed by `tests/conftest.py`) refuses a network connection other than a Unix socket, every - host-name lookup the `socket` module offers, and `SystemClock`'s `now`, `sleep`, and unseeded - random source, by raising `NotDeterministic`. It sets `no_proxy` to `*`, so a client that reads - proxies from the environment or the system settings is refused at GitHub's own host on every - machine. Each refusal is also recorded and checked after every test and at the end of the - session, so a test fails even where the code under test caught the exception (a refusal in a - wider fixture's teardown fails the next check). `tests/test_suite_rules.py` reads source: no test - file reads the wall clock, sleeps on it, or draws unseeded random numbers (`random.Random` takes - a seed, `asyncio.sleep` takes `0`); the package reads them only in its two real clocks, - `SystemClock` and `db.types.utc_now`; and no test function is defined inside another function, - where pytest never collects it. Processes a test starts run without the run-time refusals, and - the order of a set of strings changes with `PYTHONHASHSEED` from one process to the next: two - runs compared by `scripts/check_junit.py --same-as` show whether an outcome depends on either. + (installed by `tests/conftest.py`) refuses a connection other than to a Unix socket, each + host-name lookup function the `socket` module offers, the package's two real clocks + (`source/clock.py`'s readings and `asyncio.sleep`, behind `SystemClock`, and `db.types.utc_now`'s + wall clock), and every draw from `SystemClock`'s unseeded random source, by raising + `NotDeterministic`. It sets `no_proxy` to `*`, so a client that reads proxies from the environment + or the system settings is refused at GitHub's own host on every machine. Each refusal is also + recorded and checked after every test and at the end of the session, so a test fails even where + the code under test caught the exception (a refusal in a wider fixture's teardown fails the next + check); each refusal and that check's failing side have tests. Not refused: a socket's other ways + to reach a host (`sendto`, `bind`), and processes a test starts, which run without these patches. + `tests/test_suite_rules.py` reads source: no test file reads the wall clock, sleeps on it, or + draws unseeded random numbers (`random.Random` takes a seed, `asyncio.sleep` takes `0`); the + package reads them only in its two real clocks, and `alembic/` and `scripts/` read none; and no + test function is defined inside a function or in a class whose name does not start with `Test`, + where pytest never collects it. CI runs the suite a second time with the tests in reverse order + under another `PYTHONHASHSEED` and compares the two runs with `scripts/check_junit.py --same-as`: + a dependence on run order or on the order of a set of strings shows where it changes an outcome + between those two runs, and two runs that agree do not show that no such dependence exists. --- @@ -66,7 +74,8 @@ How the test suite is built and what it counts as evidence. The plan of record i ``` tests/ ├── conftest.py # The session template database and its copies; the scenario fixture -├── determinism.py # Network, real clock, and unseeded randomness refused in every test +├── determinism.py # Connections, lookups, real clocks, unseeded randomness refused +├── pytest_runs.py # pytest over files a test writes; check_junit loaded to read it ├── db/ │ ├── alembic_support.py # Running the real migrations; test-only revisions after head │ ├── schema_support.py # Structural schema readings for the model comparison @@ -88,6 +97,7 @@ tests/ │ ├── test_first_sync.py # go-gdpr's first sync from its recordings, without the model │ ├── test_seeded_runs.py # Seeded random changes over several runs, each checked against... │ └── expected_state.py # ...the database the model and its history give, never the writer +├── fixtures/history/ # Repository files as earlier commits hold them; README records blobs ├── fixtures/source/ │ ├── recorded/ # Units recorded from public repositories (scripts/record_fixtures.py) │ ├── first_sync/ # Every unit go-gdpr's first sync asked for (--only first-sync) @@ -315,15 +325,18 @@ uv run pytest --junitxml="${TMPDIR:-/tmp}/junit.xml" \ && uv run python scripts/check_junit.py "${TMPDIR:-/tmp}/junit.xml" --max-skipped 0 ``` -Two runs of the suite give the same outcome for every test (D40). To compare them: +Two runs of the suite give the same outcome for every test (D40). CI compares a run in order +with one in reverse order under another hash seed: ```bash -uv run pytest --junitxml="${TMPDIR:-/tmp}/first.xml" -uv run pytest --junitxml="${TMPDIR:-/tmp}/second.xml" -uv run python scripts/check_junit.py "${TMPDIR:-/tmp}/first.xml" --same-as "${TMPDIR:-/tmp}/second.xml" +PYTHONHASHSEED=1 uv run pytest --junitxml="${TMPDIR:-/tmp}/first.xml" +PYTHONHASHSEED=2 uv run pytest --reverse-order --junitxml="${TMPDIR:-/tmp}/second.xml" +uv run python scripts/check_junit.py "${TMPDIR:-/tmp}/first.xml" --max-skipped 0 \ + --same-as "${TMPDIR:-/tmp}/second.xml" ``` -The comparison fails on each test whose outcome differs, or that only one run holds. +The comparison fails on each test whose outcome differs, or that only one run holds. A test +pytest reports twice (a failure, then a teardown error) has the outcome `error+failed`. --- @@ -349,10 +362,11 @@ uv run pre-commit install all files; on pull requests, the base commit's `scripts/check_query_lock.py` (the pull request's own copy where the base has none) against the hydration-version and schema-version locks. -- `tests (Python 3.12 / 3.13)`: pytest with JUnit output; `scripts/check_junit.py --max-skipped 0`, - which fails unless at least one test ran and none failed, errored, or was skipped (pytest - reports `xfail` as skipped, so an `xfail` also fails the job); and `alembic upgrade head` plus - `alembic check` on a fresh database. +- `tests (Python 3.12 / 3.13)`: pytest with JUnit output under `PYTHONHASHSEED=1`; + `scripts/check_junit.py --max-skipped 0`, which fails unless at least one test ran and none + failed, errored, or was skipped (pytest reports `xfail` as skipped, so an `xfail` also fails the + job); pytest again with `--reverse-order` under `PYTHONHASHSEED=2`, and `check_junit.py --same-as` + over the two reports; and `alembic upgrade head` plus `alembic check` on a fresh database. - `built wheel`: builds the wheel with build dependencies hash-checked against `build-constraints.txt`, installs the `uv.lock` dependencies with hash checking and then the wheel into a new environment, runs `uv pip check`, and runs `scripts/wheel_smoke.py` diff --git a/tests/test_suite_rules.py b/tests/test_suite_rules.py index 8854774..eec1131 100644 --- a/tests/test_suite_rules.py +++ b/tests/test_suite_rules.py @@ -27,7 +27,8 @@ ``tests/conftest.py``) under another ``PYTHONHASHSEED``, and compares the two runs with ``scripts/check_junit.py --same-as``: that shows a dependence on run order or on the seed only where it changed an outcome between those two runs. The same scan over the package finds its reads only in -its two real clocks, and over ``alembic/``, which every database test runs in process, finds none; +its two real clocks, and over ``alembic/`` and ``scripts/``, which the suite runs in process, finds +none; at run time ``tests/determinism.py`` refuses both clocks, network connections other than Unix sockets, and the ``socket`` module's host-name lookups, in every test. From b45933fe5956f7cd466103b6c859d300528f7e00 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 13:50:17 -0600 Subject: [PATCH 26/29] Step 3b-2a-2d: close the round's remaining findings - alembic/env.py: a refused second connection is recorded, and an upgrade whose revision caught the refusal and went on fails before the commit; the file keeps its revision and contents. - CI: the two-run comparison also runs after a failing reversed run, so each test whose outcome differs is named. - D40 keeps the maintainer's wording in its four copies; what the guard does not reach (a socket's sendto and bind, child processes, the clocks behind a timeout, a log record's time) is listed as its limits in docs/testing.md. - Exact results: the step-7 run's error and the aborted row's exit code; the run that blocks the claim after an interrupted close; the scripts the scan read. - The plan: the six tests not kept and the recorder rewrites that keep main's names; the child-count checks of verify and Step 4a on a copy with a row deleted; the createdAt case's filteredCount; the report seconds under a wall-clock step. docs/testing.md and docs/development.md name tests/fixtures/history. --- .github/workflows/check.yml | 3 +++ CLAUDE.md | 32 +++++++++++++-------------- CONTRIBUTING.md | 26 +++++++++++----------- alembic/env.py | 38 +++++++++++++++++++++++--------- docs/database.md | 3 ++- docs/development.md | 1 + docs/redesign-plan.md | 22 +++++++------------ docs/testing.md | 43 +++++++++++++++++++++---------------- tests/db/test_migrations.py | 30 ++++++++++++++++++++++++++ tests/sync/test_engine.py | 8 +++++-- tests/test_suite_rules.py | 7 +++++- 11 files changed, 137 insertions(+), 76 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 875d5b9..338dfd5 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -84,10 +84,13 @@ jobs: # D40: the same outcome in another run order and under another hash seed. Each pair of tests # runs in the other order; a dependence shows only where it changes an outcome. - name: pytest again, last to first, under another hash seed + id: reversed env: PYTHONHASHSEED: "2" run: uv run --no-sync pytest --reverse-order --junitxml="$RUNNER_TEMP/junit-reversed.xml" + # Also after a failing reversed run, so each test whose outcome differs is named. - name: Require the two runs to agree test by test + if: ${{ !cancelled() && steps.reversed.outcome != 'skipped' }} run: | uv run --no-sync python scripts/check_junit.py "$RUNNER_TEMP/junit.xml" --max-skipped 0 \ --same-as "$RUNNER_TEMP/junit-reversed.xml" diff --git a/CLAUDE.md b/CLAUDE.md index 6819c71..96fa590 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,22 +57,22 @@ that fails against the parent commit's code (redesign plan D39). See [Testing](d ## Deterministic Tests Tests are software that gives the same result every time (redesign plan D40). The same code gives -every test the same outcome on any machine, on any date, and in any run order: no result a test -asserts comes from the wall clock (a fixed clock such as `FakeClock` stands in; a timeout that -bounds a wait and a log record's time are not results), the network (GitHub answers from recordings -and stubs), unseeded randomness, or state outside the test's temporary directory, the repository, -and the session's database template, and every wait is bounded by a timeout. A test exercises code -and asserts on what the code returns, writes, or exits with: where the code gives a structured value -(an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather -than on printed wording, and no test checks the wording of a document. Where a test needs an input -the code cannot easily be given, the project builds software that produces it (a fake clock, the -scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test -changes the project's real files. Evidence is what software reproduces: an acceptance criterion -names a test, a check, or a command whose output shows it, and a search of text, a reading, or a -review finds work to do but is never acceptance. `tests/determinism.py` refuses network connections, -host-name lookups, the real clocks, and the unseeded random source in every test, -`tests/test_suite_rules.py` enforces the rules it can model in source, and `scripts/check_junit.py ---same-as` compares two runs of the suite test by test (Step 3b-2a-2d). +every test the same outcome on any machine, on any date, and in any run order: a test, and every +code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no +network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state +outside its own temporary directory, the repository, and the session's database template, and bounds +every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or +exits with: where the code gives a structured value (an exit code, rows, parsed fields, an +exception's class and fields), the test asserts on it rather than on printed wording, and no test +checks the wording of a document. Where a test needs an input the code cannot easily be given, the +project builds software that produces it (a fake clock, the scenario model, the replay, a recorder +mode) or the test builds the input itself (D39); no test changes the project's real files. Evidence +is what software reproduces: an acceptance criterion names a test, a check, or a command whose +output shows it, and a search of text, a reading, or a review finds work to do but is never +acceptance. `tests/determinism.py` refuses network connections, host-name lookups, the real clocks, +and the unseeded random source in every test, `tests/test_suite_rules.py` enforces the rules it can +model in source, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test +(Step 3b-2a-2d). ## Current Scope diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 62d4fdd..0eb4299 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -260,19 +260,19 @@ or values it compared, so it cannot pass on nothing. A fix comes with a regressi against the parent commit's code. See the [Testing Guide](docs/testing.md#principles). **4. Write tests that give the same result every time** (redesign plan D40). The same code gives -every test the same outcome on any machine, on any date, and in any run order: no result a test -asserts comes from the wall clock (a fixed clock such as `FakeClock` stands in; a timeout that -bounds a wait and a log record's time are not results), the network (GitHub answers from recordings -and stubs), unseeded randomness, or state outside the test's temporary directory, the repository, -and the session's database template, and every wait is bounded by a timeout. A test exercises code -and asserts on what the code returns, writes, or exits with: where the code gives a structured value -(an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather -than on printed wording, and no test checks the wording of a document. Where a test needs an input -the code cannot easily be given, the project builds software that produces it (a fake clock, the -scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test -changes the project's real files. Evidence is what software reproduces: an acceptance criterion -names a test, a check, or a command whose output shows it, and a search of text, a reading, or a -review finds work to do but is never acceptance. +every test the same outcome on any machine, on any date, and in any run order: a test, and every +code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no +network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state +outside its own temporary directory, the repository, and the session's database template, and bounds +every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or +exits with: where the code gives a structured value (an exit code, rows, parsed fields, an +exception's class and fields), the test asserts on it rather than on printed wording, and no test +checks the wording of a document. Where a test needs an input the code cannot easily be given, the +project builds software that produces it (a fake clock, the scenario model, the replay, a recorder +mode) or the test builds the input itself (D39); no test changes the project's real files. Evidence +is what software reproduces: an acceptance criterion names a test, a check, or a command whose +output shows it, and a search of text, a reading, or a review finds work to do but is never +acceptance. ### Running Tests diff --git a/alembic/env.py b/alembic/env.py index ed56075..307bea2 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -236,15 +236,27 @@ def configure_connection(dbapi_connection: sqlite3.Connection, record: Connectio raise -def refuse_another_connection(dbapi_connection: sqlite3.Connection, _: ConnectionPoolEntry) -> None: - """Once the upgrade's connection is open, no other is: closed and refused before anything - runs on it. Without this, a statement after SQLAlchemy invalidated the upgrade's connection - and a ``rollback()`` reconnects, outside the transaction and the authorizer's state.""" - dbapi_connection.close() - raise MigrationRefused( - "the upgrade runs on one driver connection: no other is opened, after SQLAlchemy " - "invalidated it or for a second connection to the same engine" - ) +ONE_CONNECTION = ( + "the upgrade runs on one driver connection: no other is opened, after SQLAlchemy " + "invalidated it or for a second connection to the same engine" +) + + +def refuse_another_connection( + refused: list[str], +) -> Callable[[sqlite3.Connection, ConnectionPoolEntry], None]: + """The connect listener for once the upgrade's connection is open: any other is closed and + refused before anything runs on it, and the refusal is recorded in ``refused``, so a revision + that catches it still fails the upgrade before the commit. Without it, a statement after + SQLAlchemy invalidated the upgrade's connection and a ``rollback()`` reconnects, outside the + transaction and the authorizer's state.""" + + def refuse(dbapi_connection: sqlite3.Connection, _: ConnectionPoolEntry) -> None: + dbapi_connection.close() + refused.append(ONE_CONNECTION) + raise MigrationRefused(ONE_CONNECTION) + + return refuse def begin_immediate(connection: Connection) -> None: @@ -281,7 +293,8 @@ def run_migrations_online() -> None: event.listen(engine, "connect", configure_connection, insert=True) event.listen(engine, "begin", begin_immediate) with engine.connect() as connection: - event.listen(engine, "connect", refuse_another_connection, insert=True) + refused: list[str] = [] + event.listen(engine, "connect", refuse_another_connection(refused), insert=True) context.configure( connection=connection, target_metadata=Base.metadata, @@ -296,6 +309,11 @@ def run_migrations_online() -> None: with context.begin_transaction(): try: context.run_migrations() + if refused: + raise MigrationRefused( + f"{refused[0]}; a revision went on after the refusal, so nothing was " + "committed" + ) check_foreign_keys(connection) except BaseException: state[TRANSACTION_STATE] = "failed" diff --git a/docs/database.md b/docs/database.md index 4676a59..50395e9 100644 --- a/docs/database.md +++ b/docs/database.md @@ -70,7 +70,8 @@ converted to UTC, and returns aware UTC values. Defaults are `utc_now()`, never - An upgrade runs on one driver connection. Once it is open, env.py refuses and closes any other the engine would open: a second connection a revision asks for, or the one SQLAlchemy would open after it invalidated the first for an interrupt or an error it reads as a - disconnect. The upgrade then ends with that error, and the file keeps its revision. + disconnect. The upgrade then fails, even when a revision catches the refusal and goes on, and + nothing is committed. - Refused: offline `--sql` mode, databases other than SQLite, SQLite URI filenames (`?uri=true`), a caller-supplied connection or `sqlalchemy.url`, and the tracked `github_activity.db`. The connect listener asks SQLite which file it opened, so the tracked diff --git a/docs/development.md b/docs/development.md index e1e5052..cdb1d2a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -317,6 +317,7 @@ tests/ ├── db/ # Migrations, schema comparison, views, timestamp type, tracked file ├── source/ # Source layer against recorded GitHub responses ├── sync/ # Planner, deletions, and writer over seeded databases; the engine over the scenario model +├── fixtures/history/ # Repository files as earlier commits hold them ├── fixtures/source/ # Recorded and synthetic GitHub responses ├── data_dictionary.py # Reader for docs/data-dictionary.md ├── query_versions.json # Hydration-version lock diff --git a/docs/redesign-plan.md b/docs/redesign-plan.md index 2a5d97b..7ca4621 100644 --- a/docs/redesign-plan.md +++ b/docs/redesign-plan.md @@ -93,8 +93,8 @@ Within scope the database collects and distinguishes data and never interprets i | D36 | Verify decisions of 2026-10-21: Step 3b-2b is two pull requests, the REST sampler and then verify with the live acceptance (section 6). Verify writes nothing to the database and claims no run: before any request it reads `sync_runs` and exits 1 while a row holds the lock by the claim's rule, so a stale or crashed run stays for the next claim or `sync --break-lock` (5.3). `--repo` limits verify to the repositories it names, as it limits a sync (D27). A verify that never starts writes no report; once it starts, `--report` is written on every exit verify's own code handles, and one that cannot be written turns an exit 0 or 2 into 1, because Step 6a reads it, while an interrupt exits 130 either way; verify's printed output follows sync's rule (5.3 step 8). Every repository failure 5.3 gives a resolve, an index, or a hydration batch, a repository error during a timeline lookup, and from Step 4b a log or history read verify cannot complete make a repository unverified, and a verify in which every repository it attempts is unverified, the deadline having stopped none, is fatal. verify reads each repository's stored rows in two statements before indexing it, and from Step 4b its stored events and default-branch commits in one statement each before the first request of its log and history reads, and its report records its own points, REST calls, and time (5.4, gap 94). | | D37 | Decision of 2026-10-21 (the maintainer): an error is better than a silent failure or silently wrong data. Where the code cannot check something, or cannot accept what GitHub returned, it reports a failure that stops publication or turns the run red, never a pass. For verify: an unchecked item and an unverified repository fail verify in both modes, not only `--exact`, and a lookup whose answer verify cannot accept is an error (5.4, 5.8, gap 97), so D25 does not publish a database verify could not check. | | D38 | Decisions of 2026-10-21 applying D37 to the sync and to the steps not yet built (gaps 98-110). The sync exits 2 when a 24-hour account refresh fails, when the run reaches its deadline, and when a hydration contradicts the run's own index after one retry (Step 3b-2a-3); the failures a batch made final before a deadline are recorded, not left pending. An `accepted_failures` entry accepts one error class and one cause, the first line of the failure's message, of an item, an account's refresh, or from Step 4b a compare, so a permanent failure can be accepted without turning every run red while a new cause is not. A PR with more closing links than the index carries is hydrated every run. Every node id a sighting, a lookup, or a commit link reports for a stored account is stored, and `sync_runs` records the `--repo` filter, each configured entry's `github_id`, and its first commit-link lookups and those GitHub answered `NOT_FOUND`. The source refuses what it cannot read (gap 104), and the run records refuse what they cannot write and mark `-unknown` what they cannot determine (gap 105). Step 4b marks a compare file list at GitHub's 300-file cap and counts it in verify's report and the manifest, completes an activity read only at the floor or at the log's first event, as a read in ascending order finds it, fails a history read whose count still differs after a rescan, and records a compare that still fails, which an entry can accept. Step 5 measures which edits leave `updatedAt` unchanged. Step 6 compares the App's reads with the token's before any App run publishes to `db`, pins the database file the job checked, tests that every exit-code `if:` compares a non-empty string, and ties a snapshot to the job's own sync. A schedule that stops is the one silent case this repository cannot see: no run starts, so nothing turns red; 5.6 states the staleness rule a consumer applies, and building it belongs to the dashboard's integration (section 9). Left as they are, because the stored data already says what happened: a commit-linked account a lookup answers `NOT_FOUND` (D28; now counted per run and in the manifest, and from Step 6a a run whose `links_not_found` exceeds the share of its `link_lookups` Step 5 sets, with at least Step 5's minimum number of lookups, exits 2 (an account an earlier lookup answered `NOT_FOUND` counted in neither), since a lookup form GitHub stopped resolving would look the same per link), a compare at GitHub's 300-file cap and one GitHub refuses (marked by `files_at_cap` and `http_status`, counted in verify's report and the manifest), a REST git author without a name or email (stored null, as GitHub reports none), deletions recorded as a count (`items_deleted`, and from Step 4b `events_deleted` and `history_commits_deleted`; the snapshots retained for 7 days hold the identities; older ones are not kept), which accounts failed a refresh (printed; the count is stored, the run is red unless an account entry accepts the failure, and the accounts are retried), an `author_association` difference (verify reports it as drift with its own count: GitHub computes association when read and changes it without moving `updatedAt`, section 7, so no read tells a change from a defect), and the merge state as of `hydrated_at` (D32). New tables and columns of GitHub-derived data on a populated file follow gap 110's rule (5.1). | -| D39 | Decision of 2026-10-21 (the maintainer), clarified on 2026-10-22: the first wording also banned negative tests, and Step 3b-2a-2d removed some before the clarification restored them. No mutation testing, under any name: the code under test and the checks are never changed, on disk or in memory, to see whether a test notices. No false content is planted in the project's real files or documents (the plan, a document, review material, a tracked lock or database) to see whether a check or a reviewer catches it, and no reviewer or agent is scored on how many seeded flaws it finds. Negative tests are expected: a test builds its own invalid or failing input on purpose (a sample test that fails, an invalid configuration, a malformed or edited copy of a file or database in its temporary directory, a test-only revision that raises, a stub or fake that raises, a recorded GitHub response changed in memory) and feeds it to the unchanged code. Each test asserts the exact result. Every check is tested on both sides, and its passing side states the exact count of the rows, files, or values it compared, so it cannot pass on nothing. A fix comes with a regression test that fails against the parent commit's code. | -| D40 | Decision of 2026-10-22 (the maintainer). Tests are software that gives the same result every time. The same code gives every test the same outcome on any machine, on any date, and in any run order: no result a test asserts comes from the wall clock (a fixed clock such as `FakeClock` stands in; a timeout that bounds a wait and a log record's time are not results), the network (GitHub answers from recordings and stubs), unseeded randomness, or state outside the test's temporary directory, the repository, and the session's database template, and every wait is bounded by a timeout. A test exercises code and asserts on what the code returns, writes, or exits with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on printed wording, and no test checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test changes the project's real files. Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. `tests/determinism.py` refuses network connections, host-name lookups, the real clocks, and the unseeded random source in every test, `tests/test_suite_rules.py` enforces the rules it can model in source, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test, which CI runs in two orders under two hash seeds (Step 3b-2a-2d). | +| D39 | Decision of 2026-10-21 (the maintainer), clarified on 2026-10-22: the first wording also banned negative tests, and Step 3b-2a-2d removed some; under the clarification it keeps them, except the six it names. No mutation testing, under any name: the code under test and the checks are never changed, on disk or in memory, to see whether a test notices. No false content is planted in the project's real files or documents (the plan, a document, review material, a tracked lock or database) to see whether a check or a reviewer catches it, and no reviewer or agent is scored on how many seeded flaws it finds. Negative tests are expected: a test builds its own invalid or failing input on purpose (a sample test that fails, an invalid configuration, a malformed or edited copy of a file or database in its temporary directory, a test-only revision that raises, a stub or fake that raises, a recorded GitHub response changed in memory) and feeds it to the unchanged code. Each test asserts the exact result. Every check is tested on both sides, and its passing side states the exact count of the rows, files, or values it compared, so it cannot pass on nothing. A fix comes with a regression test that fails against the parent commit's code. | +| D40 | Decision of 2026-10-22 (the maintainer). Tests are software that gives the same result every time. The same code gives every test the same outcome on any machine, on any date, and in any run order: a test, and every code path it runs, reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no network (GitHub answers from recordings and stubs), uses no unseeded randomness, depends on no state outside its own temporary directory, the repository, and the session's database template, and bounds every wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on printed wording, and no test checks the wording of a document. Where a test needs an input the code cannot easily be given, the project builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test changes the project's real files. Evidence is what software reproduces: an acceptance criterion names a test, a check, or a command whose output shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. `tests/determinism.py` refuses network connections, host-name lookups, the real clocks, and the unseeded random source in every test, `tests/test_suite_rules.py` enforces the rules it can model in source, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test, which CI runs in two orders under two hash seeds (Step 3b-2a-2d). | ## 3. Coverage rule @@ -1941,8 +1941,7 @@ Scope: tests; `docs/testing.md`, `docs/database.md`, `docs/development.md`, `CLA `CONTRIBUTING.md`; `scripts/check_junit.py`'s `--same-as`; the tests job of `.github/workflows/check.yml`, which runs the suite a second time in reverse order under another `PYTHONHASHSEED` and compares the two runs; `pyproject.toml`, where ruff skips -`tests/fixtures/history/`; two changes under `src/`: `ghactivity sync` times its report on the run's -clock, not `time.monotonic`, and `db.types.utc_now` reads the time through `_wall_clock`, which a +`tests/fixtures/history/`; two changes under `src/`: `ghactivity sync` times its report on the run's clock, not `time.monotonic` (a wall-clock step during a run shows in the printed seconds), and `db.types.utc_now` reads the time through `_wall_clock`, which a test replaces or refuses (D40); and one under `alembic/`: `env.py` runs an upgrade on one driver connection and records a failed upgrade's state on that connection's mapping without asking SQLAlchemy for the connection again, so an interrupt, a `SystemExit`, or an error SQLAlchemy reads @@ -1966,14 +1965,11 @@ by a second walk of the directory, and the package scan with the exact list of r Every test on `main` before this step whose input the test built itself (a test-only revision that fails, a changed copy of a database or file in its temporary directory, a trigger in its own database, an edited lock entry, a sample file holding each spelling a checker flags, a stub or fake -that raises) is kept under its name, under D39 as clarified. Where this step had rewritten such a -test under its name, main's body is back, and the rewrite keeps a name of its own where it holds -more (a real earlier file or revision, an exact count). The tests this step adds read real earlier +that raises) is kept under its name, under D39 as clarified. Where this step had rewritten such a test under its name, main's body is back, and the rewrite keeps a name of its own where it holds more (a real earlier file or revision, an exact count); in `tests/source/test_recorder.py` the rewrites keep main's names, since each drives the same refusal through the recorder's real path (a recorded unit changed in memory) where main replaced the recorder's function or limit. The tests this step adds read real earlier revisions (`upgrade(path, "")`) and real earlier files (`tests/fixtures/history/`, each copy's git blob id in its README, checked by `tests/test_history_fixtures.py`). Six tests on `main` are not kept: one depended on the uid and one checked a document's wording (D40), and four are held -by tests over real inputs (two assignee tests by `test_an_assignee_github_id_under_two_node_ids`, -with GitHub's ids for one bot; two query-lock cases by +by tests over real inputs (two assignee tests by `test_an_assignee_github_id_under_two_node_ids`, a recorded unit changed in memory to list one bot's two GitHub node ids; two query-lock cases by `test_the_verdict_does_not_depend_on_how_the_checker_runs`). Tests: every test on `main` before this step is at head under its name, or the pull request names @@ -2170,8 +2166,7 @@ failure and failing `--exact`, and one whose cause matches, passing it, a stale repository, a stored row no configured entry names, a renamed repository, and a renamed repository whose configured entry fails to resolve, reported unverified and not untracked, and an entry that resolves to a `github_id` other than its stored row's, with no stored `is_tracked` row reported -untracked), its child-count checks, each reporting the rows it compared, equal to the counts the -scenario model gives (D39), an unverified repository for +untracked), its child-count checks, each reporting the rows it compared, equal to the counts the scenario model gives, and each one failing on a copy of that database with one of its rows deleted (D39), an unverified repository for each failure 5.4 lists, one made unverified by a lookup by number and one by a timeline lookup, each keeping the classes it already had and leaving unchecked the items no lookup had answered, those of the batch the error ended included, an unchecked item and an unverified repository each making a @@ -2197,7 +2192,7 @@ another issue, a link difference made by a closing keyword with no event, a diff link totals alone with enough connected events, with only a disconnected event, and with too few, a synthetic disconnected event, an event at the boundary counted and one a second before it not (under the measured `since` -and whole-second times, verify's own `createdAt` filter drops no entry the lookup returns; it is tested on its passing side, every returned entry counted, and on a lookup answer the test changes in memory to hold an entry before `since`, which the filter drops), a stored link no longer indexed on a PR whose index lists every link matched only by an +and whole-second times, verify's own `createdAt` filter drops no entry the lookup returns; it is tested on its passing side, every returned entry counted, and on a lookup answer the test changes in memory to hold an entry before `since`, its `filteredCount` counting it, which the filter drops), a stored link no longer indexed on a PR whose index lists every link matched only by an event naming it, a filtered page whose `totalCount` differs from its entries, counted by its entries, a lookup whose entries differ from its `filteredCount` after its retry and one whose response the typed models reject, each an error, one that differs once and matches on its retry, classed by its events, and a @@ -2252,8 +2247,7 @@ issues; an issue's stored author kept when GitHub returns it as null (D28); issu assignees written and replaced; a transfer arriving with a new id; an old number the run's index listed that hydration answers `NOT_FOUND`, an item failure until an index that omits it removes it (5.3 step 4); a snapshot one migration behind head upgrades and syncs; verify's -child-count checks for `issue_labels`, `issue_comments`, and `issue_assignees`, each reporting the -rows it compared, equal to the stored totals (D39). +child-count checks for `issue_labels`, `issue_comments`, and `issue_assignees`, each reporting the rows it compared, equal to the stored totals, and each one failing on a copy of that database with one of its rows deleted (D39). Acceptance: `verify --exact` for issues on the same repositories; a REST sample of issues matches field by field. diff --git a/docs/testing.md b/docs/testing.md index bba7ea5..60ed324 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -23,22 +23,22 @@ How the test suite is built and what it counts as evidence. The plan of record i rows, files, or values it compared, so it cannot pass on nothing. A fix comes with a regression test that fails against the parent commit's code. - **The same result every time** (redesign plan D40). The same code gives every test the same - outcome on any machine, on any date, and in any run order: no result a test asserts comes from the - wall clock (a fixed clock such as `FakeClock` stands in; a timeout that bounds a wait and a log - record's time are not results), the network (GitHub answers from recordings and stubs), unseeded - randomness, or state outside the test's temporary directory, the repository, and the session's - database template, and every wait is bounded by a timeout. A test exercises code and asserts on - what the code returns, writes, or exits with: where the code gives a structured value (an exit - code, rows, parsed fields, an exception's class and fields), the test asserts on it rather than on - printed wording, and no test checks the wording of a document. Where a test needs an input the - code cannot easily be given, the project builds software that produces it (a fake clock, the - scenario model, the replay, a recorder mode) or the test builds the input itself (D39); no test - changes the project's real files. Evidence is what software reproduces: an acceptance criterion - names a test, a check, or a command whose output shows it, and a search of text, a reading, or a - review finds work to do but is never acceptance. `tests/determinism.py` refuses network - connections, host-name lookups, the real clocks, and the unseeded random source in every test, - `tests/test_suite_rules.py` enforces the rules it can model in source, and `scripts/check_junit.py - --same-as` compares two runs of the suite test by test (Step 3b-2a-2d). + outcome on any machine, on any date, and in any run order: a test, and every code path it runs, + reads no wall clock (a fixed clock such as `FakeClock` stands in), reaches no network (GitHub + answers from recordings and stubs), uses no unseeded randomness, depends on no state outside its + own temporary directory, the repository, and the session's database template, and bounds every + wait with a timeout. A test exercises code and asserts on what the code returns, writes, or exits + with: where the code gives a structured value (an exit code, rows, parsed fields, an exception's + class and fields), the test asserts on it rather than on printed wording, and no test checks the + wording of a document. Where a test needs an input the code cannot easily be given, the project + builds software that produces it (a fake clock, the scenario model, the replay, a recorder mode) + or the test builds the input itself (D39); no test changes the project's real files. Evidence is + what software reproduces: an acceptance criterion names a test, a check, or a command whose output + shows it, and a search of text, a reading, or a review finds work to do but is never acceptance. + `tests/determinism.py` refuses network connections, host-name lookups, the real clocks, and the + unseeded random source in every test, `tests/test_suite_rules.py` enforces the rules it can model + in source, and `scripts/check_junit.py --same-as` compares two runs of the suite test by test + (Step 3b-2a-2d). - **No skipped tests** (D22). CI runs `scripts/check_junit.py --max-skipped 0`, which fails a run where any test was skipped (pytest reports `xfail` as skipped) or none ran. - **Time.** Source tests pass a `FakeClock`, so waits and retries are asserted, never slept. @@ -56,8 +56,10 @@ How the test suite is built and what it counts as evidence. The plan of record i or the system settings is refused at GitHub's own host on every machine. Each refusal is also recorded and checked after every test and at the end of the session, so a test fails even where the code under test caught the exception (a refusal in a wider fixture's teardown fails the next - check); each refusal and that check's failing side have tests. Not refused: a socket's other ways - to reach a host (`sendto`, `bind`), and processes a test starts, which run without these patches. + check); each refusal and that check's failing side have tests. Not refused, and so not enforced at + run time: a socket's other ways to reach a host (`sendto`, `bind`); processes a test starts, which + run without these patches; the event loop's and the standard library's own clocks behind a + timeout; and the time loguru gives a log record, which no test asserts. `tests/test_suite_rules.py` reads source: no test file reads the wall clock, sleeps on it, or draws unseeded random numbers (`random.Random` takes a seed, `asyncio.sleep` takes `0`); the package reads them only in its two real clocks, and `alembic/` and `scripts/` read none; and no @@ -142,7 +144,10 @@ tests/ from the views at head, and on pull requests the CI step that checks the hydration-version lock rejects a removed entry, a pair removed from one, and a view digest removed or changed in one, so removing or renaming a consumer column, or changing a view, needs a new schema - version. Pairs and views may be added to the newest entry only. + version. Pairs and views may be added to the newest entry only. CI checks out one commit + with no history, so the lock, dictionary, recordings, and dependency tests read earlier + versions of repository files from `tests/fixtures/history/`, each the git blob its README + row records (`tests/test_history_fixtures.py`). ## GitHub Source Layer: Recorded Responses diff --git a/tests/db/test_migrations.py b/tests/db/test_migrations.py index eec264a..e51531c 100644 --- a/tests/db/test_migrations.py +++ b/tests/db/test_migrations.py @@ -807,6 +807,36 @@ def connected(dbapi_connection: sqlite3.Connection, _record: Any) -> None: assert snapshot(migrated_database) == before assert revision_of(migrated_database) == HEAD_REVISION + def test_a_revision_that_catches_the_refusal_still_fails_the_upgrade( + self, migrated_database: Path, tmp_path: Path + ) -> None: + """A revision that catches the refusal of a second connection and goes on: the upgrade + fails before the commit, and the file keeps its revision and contents.""" + versions = tmp_path / "versions" + write_revision( + versions, + TEST_REVISION, + HEAD_REVISION, + """ + try: + op.get_bind().engine.connect() + except RuntimeError: + pass + op.execute("CREATE TABLE after_the_refusal (x INTEGER)") + """, + ) + before = snapshot(migrated_database) + with pytest.raises(RuntimeError) as error: + upgrade(migrated_database, test_versions=versions) + assert type(error.value).__name__ == "MigrationRefused" + assert str(error.value) == ( + "the upgrade runs on one driver connection: no other is opened, after SQLAlchemy " + "invalidated it or for a second connection to the same engine; a revision went on " + "after the refusal, so nothing was committed" + ) + assert snapshot(migrated_database) == before + assert revision_of(migrated_database) == HEAD_REVISION + def test_the_chain_commits_a_file_of_seeded_rows(self, step3b1_database: Path) -> None: """The foreign-key check before the upgrade's commit passes a file of valid rows, and each table it reads holds the rows the seed wrote.""" diff --git a/tests/sync/test_engine.py b/tests/sync/test_engine.py index ae289ae..054f451 100644 --- a/tests/sync/test_engine.py +++ b/tests/sync/test_engine.py @@ -1609,10 +1609,13 @@ async def failing_prune(*args: Any) -> Any: if ending == "fatal": report = await world.sync(prune_untracked=True) assert ended(world, report) == ("failed", 1, {"prebid/Prebid.js": "failed"}) + assert report.error == "RuntimeError: prune failed" else: with pytest.raises(KeyboardInterrupt): await world.sync(prune_untracked=True) - assert world.value("SELECT status FROM sync_runs ORDER BY id DESC LIMIT 1") == "aborted" + assert world.rows("SELECT status, exit_code FROM sync_runs ORDER BY id DESC LIMIT 1") == [ + {"status": "aborted", "exit_code": 130} + ] assert halt_reasons(world) == { "prebid/Prebid.js": "26 deletion candidates exceed the delete guard's 25 (26 stored " "rows before the run)" @@ -2885,8 +2888,9 @@ def interrupt( assert world.rows("SELECT status, exit_code FROM sync_runs") == [ {"status": "running", "exit_code": None} ] - with pytest.raises(ClaimRefused): + with pytest.raises(ClaimRefused) as refused: await world.sync() + assert refused.value.blocking == (world.value("SELECT id FROM sync_runs"),) async def test_an_interrupted_close_whose_commit_fails_on_a_lock_leaves_the_row_running( diff --git a/tests/test_suite_rules.py b/tests/test_suite_rules.py index eec1131..1a7f6d4 100644 --- a/tests/test_suite_rules.py +++ b/tests/test_suite_rules.py @@ -337,7 +337,12 @@ def test_the_migrations_read_no_wall_clock_or_randomness() -> None: def test_the_scripts_read_no_wall_clock_or_randomness() -> None: """Over ``scripts/``, whose checks the suite also runs in process, the scan finds nothing.""" files = _python_files(SCRIPTS) - assert [path.name for path in files if path.name in ("check_junit.py", "count_suppressions.py")] + assert [path.name for path in files] == sorted( + path.name for path in SCRIPTS.iterdir() if path.suffix == ".py" + ) + assert {"check_junit.py", "count_suppressions.py", "record_fixtures.py"} <= { + path.name for path in files + } assert determinism_problems(*files) == [] From 2e762af5f0171a85701ad38a352e95372733ce80 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 14:06:10 -0600 Subject: [PATCH 27/29] Tests: one home for the shared walk, error-name, git, listener, and revision helpers tests/support.py holds the helpers several test packages repeated: - files_under(root, suffixes): the files os.walk lists under a directory, by suffix (None for every file), sorted. It replaces the counters in test_check_any_aliases, test_check_log_ placeholders, test_declared_dependencies, test_import_boundaries, test_server_side_cursors, test_suite_rules, and db/test_schema, and the walk in test_count_suppressions' _walk, which keeps its file-as-root case and regular-file filter. db/test_schema keeps its name-ending match, which rglob("*.py") also applies. - sqlite_error_name(error): the result code of the sqlite3 error under SQLAlchemy's DBAPIError, for db/test_migrations, sync/test_connection (through __cause__), and db/test_schema's failed_check, which keeps its IntegrityError and CHECK-name assertions. - git_environment(home, **extra): PATH, a home of the test's own, and no global or system configuration, plus the variables each caller names: fixed dates in test_check_query_lock, the ceiling directory and fixed dates in sync/test_runs' git_env (which still sets them in the process environment), and nothing more in db/test_tracked_database and test_history_fixtures. - listening(name, listener, target): moved from db/test_migrations; it replaces the event.listen/event.remove pairs in sync/test_engine (database_errors and five tests; the three whose finally also called reader.release() keep it there, after the listener's removal) and sync/test_writer's _sent_during. STEP3A, STEP3B1, and LAST_RUN_ID are defined once, in tests/db/alembic_support.py; sync/test_runs uses STEP3A for its older revision. tests/conftest.py's session-scoped revision_template builds the read-only database at a revision once per session; the module-scoped revisions fixtures of db/test_schema and test_data_dictionary read the three revisions from it. The collected node ids are unchanged. --- docs/testing.md | 8 ++-- tests/conftest.py | 25 ++++++++++-- tests/db/alembic_support.py | 5 +++ tests/db/test_migrations.py | 46 ++++++--------------- tests/db/test_schema.py | 25 ++++-------- tests/db/test_tracked_database.py | 12 +----- tests/support.py | 60 ++++++++++++++++++++++++++++ tests/sync/test_connection.py | 17 +++----- tests/sync/test_engine.py | 44 +++++++++----------- tests/sync/test_runs.py | 26 +++++------- tests/sync/test_writer.py | 9 ++--- tests/test_check_any_aliases.py | 11 ++--- tests/test_check_log_placeholders.py | 11 ++--- tests/test_check_query_lock.py | 13 ++---- tests/test_count_suppressions.py | 11 ++--- tests/test_data_dictionary.py | 18 ++++----- tests/test_declared_dependencies.py | 14 +++---- tests/test_history_fixtures.py | 17 ++------ tests/test_import_boundaries.py | 8 ++-- tests/test_server_side_cursors.py | 8 ++-- tests/test_suite_rules.py | 7 +--- 21 files changed, 186 insertions(+), 209 deletions(-) create mode 100644 tests/support.py diff --git a/docs/testing.md b/docs/testing.md index 60ed324..d1fbcd4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -75,9 +75,10 @@ How the test suite is built and what it counts as evidence. The plan of record i ``` tests/ -├── conftest.py # The session template database and its copies; the scenario fixture +├── conftest.py # The session template databases and copies; the scenario fixture ├── determinism.py # Connections, lookups, real clocks, unseeded randomness refused ├── pytest_runs.py # pytest over files a test writes; check_junit loaded to read it +├── support.py # Files by os.walk, SQLite error names, git environments, listeners ├── db/ │ ├── alembic_support.py # Running the real migrations; test-only revisions after head │ ├── schema_support.py # Structural schema readings for the model comparison @@ -119,8 +120,9 @@ tests/ - **One template per session.** `head_template` (`tests/conftest.py`) is built once by `alembic upgrade head` and made read-only; `migrated_database` is a writable copy per test, and `test_engine` the application engine (`build_engine`) over it, so foreign keys and the - busy timeout match production. `metadata.create_all` appears only in `tests/db/test_schema.py`, - as the other side of the model comparison. + busy timeout match production. `revision_template(revision)` gives a read-only database at an + earlier revision, built once per session the first time a test asks for it. `metadata.create_all` + appears only in `tests/db/test_schema.py`, as the other side of the model comparison. - **The real environment.** Migration tests run the repository's `alembic.ini`, `env.py`, and revisions. `write_revision` adds a test-only revision after the head through `version_locations`: revisions that fail, leave a foreign-key violation, try to end the diff --git a/tests/conftest.py b/tests/conftest.py index fdddb71..b458cf6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,15 +1,16 @@ """Shared test fixtures. Databases come from one template per test session, built by ``alembic upgrade head`` and copied -for each test (redesign plan Step 3); ``metadata.create_all`` is used only by the schema -comparison in ``tests/db/test_schema.py``. +for each test (redesign plan Step 3), and from read-only templates at earlier revisions, each +built once; ``metadata.create_all`` is used only by the schema comparison in +``tests/db/test_schema.py``. """ from __future__ import annotations import os import shutil -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Callable, Iterator from pathlib import Path from typing import TYPE_CHECKING @@ -97,6 +98,24 @@ def head_template(tmp_path_factory: pytest.TempPathFactory) -> Path: return path +@pytest.fixture(scope="session") +def revision_template(tmp_path_factory: pytest.TempPathFactory) -> Callable[[str], Path]: + """The database migrated to a revision, built the first time a test asks for that revision + and read-only, like ``head_template``.""" + folder = tmp_path_factory.mktemp("revisions") + built: dict[str, Path] = {} + + def template(revision: str) -> Path: + if revision not in built: + path = folder / f"{revision}.db" + upgrade(path, revision) + path.chmod(0o444) + built[revision] = path + return built[revision] + + return template + + @pytest.fixture def migrated_database(head_template: Path, tmp_path: Path) -> Path: """A writable copy of the head template.""" diff --git a/tests/db/alembic_support.py b/tests/db/alembic_support.py index 1b31688..4df18ec 100644 --- a/tests/db/alembic_support.py +++ b/tests/db/alembic_support.py @@ -15,6 +15,11 @@ REPO_ROOT = Path(__file__).resolve().parents[2] VERSIONS = REPO_ROOT / "alembic" / "versions" FIRST_SCHEMA_HEAD = "b3c4d5e6f7g8" +# The revisions of Step 3a, of Step 3b-1, and the one after it, which adds +# sync_failures.last_run_id. +STEP3A = "33f8570f6540" +STEP3B1 = "94a52e4f1d57" +LAST_RUN_ID = "2d2ba76b177c" def alembic_config( diff --git a/tests/db/test_migrations.py b/tests/db/test_migrations.py index e51531c..c0ecf5a 100644 --- a/tests/db/test_migrations.py +++ b/tests/db/test_migrations.py @@ -16,7 +16,7 @@ import sys import tokenize from collections.abc import Callable, Iterator -from contextlib import closing, contextmanager +from contextlib import closing from dataclasses import dataclass from pathlib import Path from typing import Any @@ -24,11 +24,9 @@ import pytest from alembic import command from alembic.script import ScriptDirectory -from sqlalchemy import create_engine, event -from sqlalchemy.engine import Engine +from sqlalchemy import create_engine from sqlalchemy.exc import ( DatabaseError, - DBAPIError, OperationalError, ProgrammingError, StatementError, @@ -40,7 +38,10 @@ from github_activity_db.db.schema import HEAD_REVISION, MIGRATION_REVISIONS, SCHEMA_VERSION from tests.db.alembic_support import ( FIRST_SCHEMA_HEAD, + LAST_RUN_ID, REPO_ROOT, + STEP3A, + STEP3B1, alembic_config, query, revision_of, @@ -56,6 +57,7 @@ views, ) from tests.db.seed import T0, Seed, connect +from tests.support import listening, sqlite_error_name from tests.test_import_boundaries import imported_modules TRACKED = REPO_ROOT / "github_activity.db" @@ -130,13 +132,6 @@ def refusal(error: pytest.ExceptionInfo[BaseException]) -> str: return str(error.value) -def sqlite_error_name(error: pytest.ExceptionInfo[DBAPIError]) -> str: - """The SQLite result code of the driver error SQLAlchemy's error wraps.""" - driver_error = error.value.orig - assert isinstance(driver_error, sqlite3.Error), repr(driver_error) - return str(driver_error.sqlite_errorname) - - def is_closed(connection: sqlite3.Connection) -> bool: """Whether the driver connection is closed: a closed one runs nothing.""" try: @@ -156,18 +151,6 @@ def foreign_key_rows(database: Path) -> dict[str, int]: return counted -@contextmanager -def listening( - name: str, listener: Callable[..., None], target: type[Engine | Pool] = Engine -) -> Iterator[None]: - """``listener`` on the ``name`` event of every engine, or every pool, env.py's own included.""" - event.listen(target, name, listener) - try: - yield - finally: - event.remove(target, name, listener) - - @pytest.fixture(scope="module") def first_schema_template(tmp_path_factory: pytest.TempPathFactory) -> Path: """A database at the first schema's head holding a repository, a PR, and a sync failure.""" @@ -436,7 +419,6 @@ def test_aborts_while_a_pull_request_is_tagged_in_a_valid_file( assert snapshot(first_schema) == before -STEP3A = "33f8570f6540" STEP3A_ROWS = { "repositories": ( "INSERT INTO repositories (id, github_id, owner, name, full_name, repo_group, is_private, " @@ -475,10 +457,6 @@ def step3a_database(tmp_path: Path) -> Path: return database -STEP3B1 = "94a52e4f1d57" -LAST_RUN_ID = "2d2ba76b177c" - - @pytest.fixture def step3b1_database(tmp_path: Path) -> Path: """A database at the Step 3b-1 revision, which no sync has written.""" @@ -1112,7 +1090,7 @@ def record(_connection: Any, _cursor: Any, statement: str, *_rest: Any) -> None: command.upgrade(config, "head") finally: reader.close() - assert sqlite_error_name(error) == "SQLITE_BUSY" + assert sqlite_error_name(error.value) == "SQLITE_BUSY" assert statements[-1] == "PRAGMA foreign_key_check" assert snapshot(step3b1_database) == before assert revision_of(step3b1_database) == STEP3B1 @@ -1140,7 +1118,7 @@ def record(_connection: Any, _cursor: Any, statement: str, *_rest: Any) -> None: command.upgrade(config, "head") finally: holder.close() - assert sqlite_error_name(error) == "SQLITE_BUSY" + assert sqlite_error_name(error.value) == "SQLITE_BUSY" assert statements == ["BEGIN IMMEDIATE"] assert snapshot(step3b1_database) == before assert revision_of(step3b1_database) == STEP3B1 @@ -1209,7 +1187,7 @@ def test_a_revision_cannot_end_the_transaction( with pytest.raises(DatabaseError) as error: upgrade(first_schema, test_versions=versions) assert type(error.value) is DatabaseError - assert sqlite_error_name(error) == "SQLITE_AUTH" + assert sqlite_error_name(error.value) == "SQLITE_AUTH" assert snapshot(first_schema) == before @pytest.mark.parametrize( @@ -1248,7 +1226,7 @@ def test_a_statement_run_before_sqlite_ended_the_transaction_is_refused_after( with pytest.raises(DatabaseError) as error: command.upgrade(config, "head") assert type(error.value) is DatabaseError - assert sqlite_error_name(error) == "SQLITE_AUTH" + assert sqlite_error_name(error.value) == "SQLITE_AUTH" assert snapshot(migrated_database) == before @pytest.mark.parametrize( @@ -1281,7 +1259,7 @@ def test_a_revision_runs_only_the_actions_the_authorizer_lists( with pytest.raises(DatabaseError) as error: upgrade(migrated_database, test_versions=versions) assert type(error.value) is DatabaseError - assert sqlite_error_name(error) == "SQLITE_AUTH" + assert sqlite_error_name(error.value) == "SQLITE_AUTH" assert snapshot(migrated_database) == before def test_a_revision_cannot_attach_another_database( @@ -1303,7 +1281,7 @@ def test_a_revision_cannot_attach_another_database( with pytest.raises(DatabaseError) as error: upgrade(migrated_database, test_versions=versions) assert type(error.value) is DatabaseError - assert sqlite_error_name(error) == "SQLITE_AUTH" + assert sqlite_error_name(error.value) == "SQLITE_AUTH" assert snapshot(migrated_database) == before assert not other.exists() diff --git a/tests/db/test_schema.py b/tests/db/test_schema.py index cce49cd..b1fa816 100644 --- a/tests/db/test_schema.py +++ b/tests/db/test_schema.py @@ -11,7 +11,6 @@ from __future__ import annotations import ast -import os import re import shutil import sqlite3 @@ -35,7 +34,7 @@ SyncPending, ) from github_activity_db.source.records import AccountType, ReviewState -from tests.db.alembic_support import alembic_config, query, upgrade +from tests.db.alembic_support import LAST_RUN_ID, STEP3A, STEP3B1, alembic_config, query from tests.db.schema_support import ( ALEMBIC_OBJECTS, check_constraints, @@ -54,10 +53,8 @@ views, ) from tests.db.seed import T0, Seed, connect +from tests.support import files_under, sqlite_error_name -STEP3A = "33f8570f6540" -STEP3B1 = "94a52e4f1d57" -LAST_RUN_ID = "2d2ba76b177c" SQLITE = sqlite.dialect() @@ -89,16 +86,10 @@ def models_database(tmp_path_factory: pytest.TempPathFactory) -> Path: @pytest.fixture(scope="module") -def revisions(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Path]: +def revisions(revision_template: Callable[[str], Path]) -> dict[str, Path]: """Databases the migrations built at Step 3a, at Step 3b-1, and at the revision after it; read-only, so no test changes one.""" - folder = tmp_path_factory.mktemp("revisions") - built = {} - for revision in (STEP3A, STEP3B1, LAST_RUN_ID): - built[revision] = folder / f"{revision}.db" - upgrade(built[revision], revision) - built[revision].chmod(0o444) - return built + return {revision: revision_template(revision) for revision in (STEP3A, STEP3B1, LAST_RUN_ID)} def revision_module(database: Path, revision: str) -> Any: @@ -957,7 +948,7 @@ def failed_check(error: pytest.ExceptionInfo[IntegrityError]) -> str: constraint's name is in SQLite's fixed ``CHECK constraint failed: `` form.""" driver_error = error.value.orig assert isinstance(driver_error, sqlite3.IntegrityError), repr(driver_error) - assert driver_error.sqlite_errorname == "SQLITE_CONSTRAINT_CHECK" + assert sqlite_error_name(error.value) == "SQLITE_CONSTRAINT_CHECK" named = re.fullmatch(r"CHECK constraint failed: (\w+)", str(driver_error)) assert named is not None, str(driver_error) return named[1] @@ -1125,11 +1116,9 @@ class TestBuiltByMigrations: def test_create_all_is_used_only_by_the_comparison(self) -> None: roots = [ROOT / name for name in ROOTS] uses, files = create_all_uses(*roots) + # By the name's ending, as rglob("*.py") matches: a file named ".py" has no suffix. assert files == sum( - name.endswith(".py") - for root in roots - for _directory, _subdirectories, names in os.walk(root) - for name in names + path.name.endswith(".py") for root in roots for path in files_under(root, None) ) assert [use.split(":")[0] for use in uses] == ["tests/db/test_schema.py"], uses # Each use is a name the tokenizer reads too; the text in strings and comments is none. diff --git a/tests/db/test_tracked_database.py b/tests/db/test_tracked_database.py index 9f3eacc..f843c45 100644 --- a/tests/db/test_tracked_database.py +++ b/tests/db/test_tracked_database.py @@ -16,6 +16,7 @@ import pytest from tests.db.alembic_support import FIRST_SCHEMA_HEAD, upgrade +from tests.support import git_environment REPO_ROOT = Path(__file__).resolve().parents[2] TRACKED = REPO_ROOT / "github_activity.db" @@ -24,17 +25,6 @@ SCRIPT_ENVIRONMENT = {"PATH": os.environ["PATH"]} -def git_environment(home: Path) -> dict[str, str]: - """A git or pre-commit process's whole environment: a home of the test's own, so no - user-level ignore file applies, and no global or system configuration.""" - return { - "PATH": os.environ["PATH"], - "HOME": str(home), - "GIT_CONFIG_GLOBAL": os.devnull, - "GIT_CONFIG_NOSYSTEM": "1", - } - - def _check(path: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( [sys.executable, str(SCRIPT), str(path)], diff --git a/tests/support.py b/tests/support.py new file mode 100644 index 0000000..57b5965 --- /dev/null +++ b/tests/support.py @@ -0,0 +1,60 @@ +"""Helpers that test modules in several packages share: files listed with os.walk, the SQLite +error under SQLAlchemy's, a git process's environment, and a listener on every engine or pool.""" + +from __future__ import annotations + +import os +import sqlite3 +from collections.abc import Callable, Collection, Iterator +from contextlib import contextmanager +from pathlib import Path + +from sqlalchemy import event +from sqlalchemy.engine import Engine +from sqlalchemy.exc import DBAPIError +from sqlalchemy.pool import Pool + + +def files_under(root: Path, suffixes: Collection[str] | None) -> list[Path]: + """The files under the directory ``root`` whose suffix is one of ``suffixes``, or every file + when it is None, sorted. Listed with os.walk rather than a glob, so a scan that reads the + files a glob finds cannot pass on fewer of them.""" + return sorted( + Path(directory, name) + for directory, _, names in os.walk(root) + for name in names + if suffixes is None or Path(name).suffix in suffixes + ) + + +def sqlite_error_name(error: BaseException | None) -> str: + """The SQLite result code of the driver error that ``error``, SQLAlchemy's error, wraps.""" + assert isinstance(error, DBAPIError), repr(error) + driver_error = error.orig + assert isinstance(driver_error, sqlite3.Error), repr(driver_error) + return driver_error.sqlite_errorname + + +def git_environment(home: Path, **extra: str) -> dict[str, str]: + """A git process's whole environment: this process's ``PATH``; ``home`` as its home, so no + user-level configuration or ignore file applies; no global or system configuration; and the + ``extra`` variables the caller names.""" + return { + "PATH": os.environ["PATH"], + "HOME": str(home), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + **extra, + } + + +@contextmanager +def listening( + name: str, listener: Callable[..., None], target: type[Engine | Pool] = Engine +) -> Iterator[None]: + """``listener`` on the ``name`` event of every engine, or every pool, env.py's own included.""" + event.listen(target, name, listener) + try: + yield + finally: + event.remove(target, name, listener) diff --git a/tests/sync/test_connection.py b/tests/sync/test_connection.py index 8bf4970..61e9a77 100644 --- a/tests/sync/test_connection.py +++ b/tests/sync/test_connection.py @@ -8,7 +8,7 @@ import pytest from sqlalchemy import insert, select, text, update -from sqlalchemy.exc import DBAPIError, IntegrityError +from sqlalchemy.exc import IntegrityError from github_activity_db.db.models import SyncRun from github_activity_db.sync import connection as connection_module @@ -21,6 +21,7 @@ ) from github_activity_db.sync.snapshot import read_snapshot from tests.db.seed import Seed, connect +from tests.support import sqlite_error_name from tests.sync.records import fetch from tests.sync.support import commit_does_not_wait @@ -114,14 +115,6 @@ async def test_the_body_s_exception_rolls_back(migrated_database: Path) -> None: assert fetch(migrated_database, "SELECT rest_calls FROM sync_runs") == [{"rest_calls": 0}] -def _sqlite_errorname(error: LocalDatabaseError) -> str: - """The SQLite error under the ``DBAPIError`` the writer turned into ``error``.""" - cause = error.__cause__ - assert isinstance(cause, DBAPIError) - assert isinstance(cause.orig, sqlite3.Error) - return cause.orig.sqlite_errorname - - async def test_a_begin_that_fails_on_a_lock_is_a_local_database_error( migrated_database: Path, ) -> None: @@ -142,7 +135,7 @@ async def test_a_begin_that_fails_on_a_lock_is_a_local_database_error( holder.close() async with transaction(writer, run_id): await writer.execute(update(SyncRun).values(rest_calls=5)) - assert _sqlite_errorname(raised.value) == "SQLITE_BUSY" + assert sqlite_error_name(raised.value.__cause__) == "SQLITE_BUSY" assert entered == [] assert fetch(migrated_database, "SELECT rest_calls FROM sync_runs") == [{"rest_calls": 5}] @@ -165,7 +158,7 @@ async def test_a_commit_that_fails_on_a_lock_leaves_the_connection_usable( finally: reader.execute("ROLLBACK") reader.close() - assert _sqlite_errorname(raised.value) == "SQLITE_BUSY" + assert sqlite_error_name(raised.value.__cause__) == "SQLITE_BUSY" async with transaction(writer, run_id): await writer.execute(update(SyncRun).values(items_planned=4)) assert fetch(migrated_database, "SELECT rest_calls, items_planned FROM sync_runs") == [ @@ -213,5 +206,5 @@ async def test_a_constraint_failure_is_a_local_database_error(migrated_database: rest_calls = (await writer.execute(select(SyncRun.rest_calls))).scalar_one() assert isinstance(raised.value.__cause__, IntegrityError) # The row has no started_at, whose NOT NULL SQLite tests before the status CHECK. - assert _sqlite_errorname(raised.value) == "SQLITE_CONSTRAINT_NOTNULL" + assert sqlite_error_name(raised.value.__cause__) == "SQLITE_CONSTRAINT_NOTNULL" assert rest_calls == 0 diff --git a/tests/sync/test_engine.py b/tests/sync/test_engine.py index 054f451..f31442c 100644 --- a/tests/sync/test_engine.py +++ b/tests/sync/test_engine.py @@ -19,8 +19,6 @@ from typing import Any, Final import pytest -from sqlalchemy import event -from sqlalchemy.engine import Engine from github_activity_db.db import engine as db_engine from github_activity_db.db.schema import HEAD_REVISION, MIGRATION_REVISIONS @@ -33,6 +31,7 @@ from tests.db.alembic_support import upgrade from tests.source.replay import synthetic from tests.source.scenario import Scenario +from tests.support import listening from tests.sync.engine_support import World, ended, outcomes, planned, project from tests.sync.records import fetch @@ -87,11 +86,8 @@ def record(context: Any) -> None: names.append(error.sqlite_errorname) after() - event.listen(Engine, "handle_error", record) - try: + with listening("handle_error", record): yield names - finally: - event.remove(Engine, "handle_error", record) def batches(scenario: Scenario) -> list[list[int]]: @@ -1651,9 +1647,11 @@ def at_the_prune( raise KeyboardInterrupt reader.hold() - event.listen(Engine, "before_cursor_execute", at_the_prune) try: - with database_errors(reader.release) as errors: + with ( + listening("before_cursor_execute", at_the_prune), + database_errors(reader.release) as errors, + ): if ending == "fatal": report = await world.sync(prune_untracked=True) assert ended(world, report) == ("failed", 1, {"prebid/Prebid.js": "failed"}) @@ -1665,7 +1663,6 @@ def at_the_prune( == "aborted" ) finally: - event.remove(Engine, "before_cursor_execute", at_the_prune) reader.release() assert len(deletes) == 1 @@ -2430,12 +2427,10 @@ def at_the_close( if statement.startswith("UPDATE sync_runs SET finished_at") and reader.connection is None: reader.hold() - event.listen(Engine, "before_cursor_execute", at_the_close) try: - with database_errors() as errors: + with listening("before_cursor_execute", at_the_close), database_errors() as errors: report = await world.sync() finally: - event.remove(Engine, "before_cursor_execute", at_the_close) reader.release() assert errors == ["SQLITE_BUSY"] @@ -2773,8 +2768,7 @@ def interrupt_the_close( assert run is not None run.cancel() - event.listen(Engine, "before_cursor_execute", interrupt_the_close) - try: + with listening("before_cursor_execute", interrupt_the_close): if interrupt == "keyboard": with pytest.raises(KeyboardInterrupt): await world.sync() @@ -2782,8 +2776,6 @@ def interrupt_the_close( run = asyncio.create_task(world.sync()) with pytest.raises(asyncio.CancelledError): await run - finally: - event.remove(Engine, "before_cursor_execute", interrupt_the_close) assert len(interrupted) == 1 (row,) = world.rows("SELECT id, status, exit_code, finished_at FROM sync_runs") @@ -2872,12 +2864,12 @@ def interrupt( interrupted.append(status) raise KeyboardInterrupt(status) - event.listen(Engine, "before_cursor_execute", interrupt) - try: - with database_errors() as errors, pytest.raises(KeyboardInterrupt) as raised: - await world.sync() - finally: - event.remove(Engine, "before_cursor_execute", interrupt) + with ( + listening("before_cursor_execute", interrupt), + database_errors() as errors, + pytest.raises(KeyboardInterrupt) as raised, + ): + await world.sync() expected = ["complete", "aborted"] if cleanup == "second interrupt" else ["complete"] assert interrupted == expected @@ -2916,12 +2908,14 @@ def interrupt( reader.hold() raise KeyboardInterrupt("complete") - event.listen(Engine, "before_cursor_execute", interrupt) try: - with database_errors() as errors, pytest.raises(KeyboardInterrupt) as raised: + with ( + listening("before_cursor_execute", interrupt), + database_errors() as errors, + pytest.raises(KeyboardInterrupt) as raised, + ): await world.sync() finally: - event.remove(Engine, "before_cursor_execute", interrupt) reader.release() assert interrupted == ["complete"] diff --git a/tests/sync/test_runs.py b/tests/sync/test_runs.py index 17db5e8..d0efbab 100644 --- a/tests/sync/test_runs.py +++ b/tests/sync/test_runs.py @@ -40,8 +40,9 @@ ) from github_activity_db.tracking import read_tracking_config from tests.data_dictionary import SCHEMA_LOCK -from tests.db.alembic_support import alembic_config, upgrade +from tests.db.alembic_support import STEP3A, alembic_config, upgrade from tests.db.seed import Seed, connect +from tests.support import git_environment from tests.sync.records import T0, fetch, ts PACKAGED = read_tracking_config() @@ -111,11 +112,11 @@ async def test_an_empty_file_is_refused(tmp_path: Path) -> None: async def test_an_older_revision_is_refused(tmp_path: Path) -> None: older = tmp_path / "older.db" - upgrade(older, "33f8570f6540") + upgrade(older, STEP3A) with pytest.raises(HeadCheckError) as refused: await check_head(_url(older)) assert str(refused.value) == ( - f"the database is at revision 33f8570f6540, not the code's head {HEAD_REVISION}; " + f"the database is at revision {STEP3A}, not the code's head {HEAD_REVISION}; " "run `alembic upgrade head`" ) @@ -904,23 +905,14 @@ def git_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: fixed date. ``code_version`` runs git with this process's environment, so the variables are set here too, and every other ``GIT_*`` variable is removed; the test's own git calls get the returned environment, built from these values and ``PATH`` alone.""" - isolated = { - "HOME": str(tmp_path), - "GIT_CONFIG_GLOBAL": os.devnull, - "GIT_CONFIG_NOSYSTEM": "1", - "GIT_CEILING_DIRECTORIES": str(tmp_path), - } + environment = git_environment(tmp_path, GIT_CEILING_DIRECTORIES=str(tmp_path)) for name in [name for name in os.environ if name.startswith("GIT_")]: monkeypatch.delenv(name) - for name, value in isolated.items(): - monkeypatch.setenv(name, value) + for name, value in environment.items(): + if name != "PATH": + monkeypatch.setenv(name, value) fixed = "2026-01-01T00:00:00+00:00" - return { - "PATH": os.environ["PATH"], - **isolated, - "GIT_AUTHOR_DATE": fixed, - "GIT_COMMITTER_DATE": fixed, - } + return {**environment, "GIT_AUTHOR_DATE": fixed, "GIT_COMMITTER_DATE": fixed} def _git(directory: Path, env: dict[str, str], *args: str) -> str: diff --git a/tests/sync/test_writer.py b/tests/sync/test_writer.py index 36aa06f..7b6371f 100644 --- a/tests/sync/test_writer.py +++ b/tests/sync/test_writer.py @@ -9,8 +9,7 @@ from typing import Any import pytest -from sqlalchemy import event, text -from sqlalchemy.engine import Engine +from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncConnection from github_activity_db.source.records import ( @@ -61,6 +60,7 @@ ) from github_activity_db.tracking import TrackedRepository from tests.db.seed import Seed, connect +from tests.support import listening from tests.sync.records import ( HYDRATED, T0, @@ -326,11 +326,8 @@ async def _sent_during[T](write: Awaitable[T]) -> tuple[T, list[str]]: def record(connection: Any, cursor: Any, statement: str, *args: Any) -> None: sent.append(statement) - event.listen(Engine, "before_cursor_execute", record) - try: + with listening("before_cursor_execute", record): result = await write - finally: - event.remove(Engine, "before_cursor_execute", record) return result, sent diff --git a/tests/test_check_any_aliases.py b/tests/test_check_any_aliases.py index 9097df0..a72b217 100644 --- a/tests/test_check_any_aliases.py +++ b/tests/test_check_any_aliases.py @@ -8,6 +8,8 @@ import pytest +from tests.support import files_under + ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts" / "check_any_aliases.py" SRC = ROOT / "src" @@ -162,20 +164,13 @@ def _run_on_source( return _run(root) -def _python_file_count(root: Path) -> int: - """The .py and .pyi files under ``root``, counted with os.walk rather than the script's glob.""" - return sum( - Path(name).suffix in {".py", ".pyi"} for _, _, names in os.walk(root) for name in names - ) - - def test_src_has_no_alias_to_any() -> None: result = _run(SRC) assert result.returncode == 0, result.stdout scanned = re.search(r"^scanned (\d+) files", result.stdout, re.MULTILINE) found = re.search(r"^aliases to Any: (\d+)$", result.stdout, re.MULTILINE) assert scanned is not None and found is not None, result.stdout - assert int(scanned.group(1)) == _python_file_count(SRC) + assert int(scanned.group(1)) == len(files_under(SRC, {".py", ".pyi"})) assert int(found.group(1)) == 0 diff --git a/tests/test_check_log_placeholders.py b/tests/test_check_log_placeholders.py index 7dcc834..f371c36 100644 --- a/tests/test_check_log_placeholders.py +++ b/tests/test_check_log_placeholders.py @@ -10,6 +10,8 @@ import pytest +from tests.support import files_under + ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts" / "check_log_placeholders.py" SRC = ROOT / "src" @@ -63,13 +65,6 @@ def _run_on_source(tmp_path: Path, source: str) -> subprocess.CompletedProcess[s return _run(root) -def _python_file_count(root: Path) -> int: - """The .py and .pyi files under ``root``, counted with os.walk rather than the script's glob.""" - return sum( - Path(name).suffix in {".py", ".pyi"} for _, _, names in os.walk(root) for name in names - ) - - def test_no_python_files_is_an_error(tmp_path: Path) -> None: (tmp_path / "empty").mkdir() result = _run(tmp_path / "empty") @@ -86,7 +81,7 @@ def test_src_has_no_percent_style_messages() -> None: scanned = re.search(r"^scanned (\d+) files", result.stdout, re.MULTILINE) found = re.search(r"^%-style loguru messages: (\d+)$", result.stdout, re.MULTILINE) assert scanned is not None and found is not None, result.stdout - assert int(scanned.group(1)) == _python_file_count(SRC) + assert int(scanned.group(1)) == len(files_under(SRC, {".py", ".pyi"})) assert int(found.group(1)) == 0 diff --git a/tests/test_check_query_lock.py b/tests/test_check_query_lock.py index 381af49..3424c03 100644 --- a/tests/test_check_query_lock.py +++ b/tests/test_check_query_lock.py @@ -8,7 +8,6 @@ from __future__ import annotations import json -import os import shutil import subprocess import sys @@ -18,6 +17,8 @@ import pytest +from tests.support import git_environment + SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "check_query_lock.py" HISTORY = Path(__file__).resolve().parent / "fixtures" / "history" GIT = shutil.which("git") @@ -37,14 +38,8 @@ def edit_entry(work: Path) -> None: def environment(directory: Path) -> dict[str, str]: """A git, checker, or step process's whole environment: the directory it runs in, inside the test's own, as its home; no global or system git configuration; fixed commit dates.""" - return { - "PATH": os.environ["PATH"], - "HOME": str(directory), - "GIT_CONFIG_GLOBAL": os.devnull, - "GIT_CONFIG_NOSYSTEM": "1", - "GIT_AUTHOR_DATE": "2026-01-01T00:00:00Z", - "GIT_COMMITTER_DATE": "2026-01-01T00:00:00Z", - } + fixed = "2026-01-01T00:00:00Z" + return git_environment(directory, GIT_AUTHOR_DATE=fixed, GIT_COMMITTER_DATE=fixed) def git(repo: Path, *args: str) -> None: diff --git a/tests/test_count_suppressions.py b/tests/test_count_suppressions.py index 3a64242..378db45 100644 --- a/tests/test_count_suppressions.py +++ b/tests/test_count_suppressions.py @@ -12,6 +12,8 @@ import pytest +from tests.support import files_under + ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts" / "count_suppressions.py" SRC = ROOT / "src" @@ -173,15 +175,10 @@ def _load() -> ModuleType: def _walk(root: Path) -> list[Path]: - """Every regular file at or under ``root``, listed with os.walk rather than a glob.""" + """Every regular file at or under ``root``.""" if root.is_file(): return [root] - return sorted( - Path(directory, name) - for directory, _, names in os.walk(root) - for name in names - if Path(directory, name).is_file() - ) + return [path for path in files_under(root, None) if path.is_file()] def _python_files(root: Path) -> list[Path]: diff --git a/tests/test_data_dictionary.py b/tests/test_data_dictionary.py index 2b5b952..001bdd8 100644 --- a/tests/test_data_dictionary.py +++ b/tests/test_data_dictionary.py @@ -5,6 +5,7 @@ import hashlib import json import sqlite3 +from collections.abc import Callable from contextlib import closing from pathlib import Path @@ -29,14 +30,11 @@ read_lock, view_digest, ) -from tests.db.alembic_support import alembic_config, upgrade +from tests.db.alembic_support import LAST_RUN_ID, STEP3A, STEP3B1, alembic_config from tests.db.schema_support import frozen_views, views # Earlier versions of repository files, copied unchanged (tests/fixtures/history/README.md). HISTORY = Path(__file__).resolve().parent / "fixtures" / "history" -STEP3A = "33f8570f6540" -STEP3B1 = "94a52e4f1d57" -LAST_RUN_ID = "2d2ba76b177c" # The commits that added 94a52e4f1d57 and 2d2ba76b177c, with the dictionary and lock of each. AT_STEP3B1, AT_LAST_RUN_ID = "21a4267", "77663a7" @@ -94,15 +92,13 @@ def entries() -> dict[str, dict[str, str]]: @pytest.fixture(scope="module") -def revisions(tmp_path_factory: pytest.TempPathFactory) -> dict[str, dict[str, list[str]]]: +def revisions(revision_template: Callable[[str], Path]) -> dict[str, dict[str, list[str]]]: """The tables and views, with their columns, of databases the migrations built at Step 3a, at Step 3b-1, and at the revision after it.""" - folder = tmp_path_factory.mktemp("revisions") - columns = {} - for revision in (STEP3A, STEP3B1, LAST_RUN_ID): - upgrade(folder / f"{revision}.db", revision) - columns[revision] = database_columns(folder / f"{revision}.db") - return columns + return { + revision: database_columns(revision_template(revision)) + for revision in (STEP3A, STEP3B1, LAST_RUN_ID) + } def dictionary_at(commit: str) -> dict[str, dict[str, str]]: diff --git a/tests/test_declared_dependencies.py b/tests/test_declared_dependencies.py index 44beee0..19ed45b 100644 --- a/tests/test_declared_dependencies.py +++ b/tests/test_declared_dependencies.py @@ -11,7 +11,6 @@ from __future__ import annotations import ast -import os import re import sys import tomllib @@ -24,6 +23,8 @@ from packaging.requirements import Requirement from packaging.utils import canonicalize_name +from tests.support import files_under + ROOT = Path(__file__).resolve().parents[1] SRC = ROOT / "src" # pyproject.toml as it was before httpx was declared (tests/fixtures/history/README.md). @@ -100,11 +101,6 @@ def _explain(found: list[Undeclared]) -> str: ) -def _python_file_count(root: Path) -> int: - """The .py files under ``root``, counted with os.walk rather than the glob.""" - return sum(Path(name).suffix == ".py" for _, _, names in os.walk(root) for name in names) - - def _importers(name: str) -> list[str]: """The files under src/ with a statement importing ``name``, found in their text.""" statement = re.compile(rf"^[ \t]*(?:from|import)[ \t]+{re.escape(name)}\b", re.MULTILINE) @@ -125,7 +121,7 @@ def _requirement_names(pyproject: Path) -> set[str]: def test_every_third_party_import_in_src_is_declared() -> None: scanned, found = undeclared_imports(SRC, ROOT / "pyproject.toml") - assert scanned == _python_file_count(SRC) + assert scanned == len(files_under(SRC, {".py"})) assert found == [], _explain(found) @@ -140,7 +136,7 @@ def test_reports_what_an_earlier_pyproject_did_not_declare() -> None: for path in _importers(name.replace("-", "_")) ] scanned, found = undeclared_imports(SRC, BEFORE_HTTPX) - assert scanned == _python_file_count(SRC) + assert scanned == len(files_under(SRC, {".py"})) assert sorted(found) == sorted(expected) @@ -167,7 +163,7 @@ def test_src_s_imports_of_a_name_several_distributions_install_need_all_of_them_ else: owners["sqlalchemy"] = installers scanned, found = undeclared_imports(SRC, ROOT / "pyproject.toml", owners) - assert scanned == _python_file_count(SRC) + assert scanned == len(files_under(SRC, {".py"})) importers = _importers("sqlalchemy") assert importers assert sorted(found) == ( diff --git a/tests/test_history_fixtures.py b/tests/test_history_fixtures.py index 8783983..1203bc3 100644 --- a/tests/test_history_fixtures.py +++ b/tests/test_history_fixtures.py @@ -8,7 +8,6 @@ from __future__ import annotations import hashlib -import os import re import shutil import subprocess @@ -16,6 +15,8 @@ import pytest +from tests.support import git_environment + HISTORY = Path(__file__).resolve().parent / "fixtures" / "history" README = "README.md" GIT = shutil.which("git") @@ -90,12 +91,7 @@ def tracked_files(directory: Path, home: Path) -> list[str]: cwd=directory, capture_output=True, check=True, - env={ - "PATH": os.environ["PATH"], - "HOME": str(home), - "GIT_CONFIG_GLOBAL": os.devnull, - "GIT_CONFIG_NOSYSTEM": "1", - }, + env=git_environment(home), timeout=60, ) names = result.stdout.decode().split("\0") @@ -213,12 +209,7 @@ def test_only_the_files_git_tracks_are_compared(tmp_path: Path) -> None: (copies / README).write_text("rows\n") (copies / "abc1234" / "kept.toml").write_text("kept = 1\n") (copies / ".gitignore").write_text("ignored.txt\n") - environment = { - "PATH": os.environ["PATH"], - "HOME": str(home), - "GIT_CONFIG_GLOBAL": os.devnull, - "GIT_CONFIG_NOSYSTEM": "1", - } + environment = git_environment(home) for args in ( ["init", "-q"], ["add", "history/README.md", "history/abc1234/kept.toml"], diff --git a/tests/test_import_boundaries.py b/tests/test_import_boundaries.py index d2dd6e8..de00320 100644 --- a/tests/test_import_boundaries.py +++ b/tests/test_import_boundaries.py @@ -17,7 +17,6 @@ import ast import io -import os import re import tokenize from collections.abc import Iterator @@ -25,6 +24,8 @@ import pytest +from tests.support import files_under + ROOT = Path(__file__).resolve().parents[1] PACKAGE = ROOT / "src" / "github_activity_db" GITHUBKIT_STATEMENT = re.compile(r"^[ \t]*(?:from|import)[ \t]+githubkit\b", re.MULTILINE) @@ -120,10 +121,7 @@ def database_imports(files: list[Path], src: Path) -> list[str]: def _file_count(root: Path) -> int: - """The .py and .pyi files under ``root``, counted with os.walk rather than a glob.""" - return sum( - Path(name).suffix in {".py", ".pyi"} for _, _, names in os.walk(root) for name in names - ) + return len(files_under(root, {".py", ".pyi"})) def _import_lines(path: Path, *, relative: bool) -> set[int]: diff --git a/tests/test_server_side_cursors.py b/tests/test_server_side_cursors.py index e38240b..a48a5fc 100644 --- a/tests/test_server_side_cursors.py +++ b/tests/test_server_side_cursors.py @@ -14,11 +14,12 @@ from __future__ import annotations import ast -import os from pathlib import Path import pytest +from tests.support import files_under + ROOT = Path(__file__).resolve().parents[1] SRC = ROOT / "src" CALLS = {"stream", "stream_scalars", "yield_per"} @@ -49,10 +50,7 @@ def server_side_cursors(files: list[Path]) -> list[str]: def test_no_server_side_cursor_in_src() -> None: files = sorted(SRC.rglob("*.py")) - # Counted with os.walk rather than the glob, so the scan cannot pass on fewer files. - assert len(files) == sum( - Path(name).suffix == ".py" for _, _, names in os.walk(SRC) for name in names - ) + assert len(files) == len(files_under(SRC, {".py"})) assert SRC / "github_activity_db" / "sync" / "snapshot.py" in files assert server_side_cursors(files) == [] diff --git a/tests/test_suite_rules.py b/tests/test_suite_rules.py index 1a7f6d4..ef183b7 100644 --- a/tests/test_suite_rules.py +++ b/tests/test_suite_rules.py @@ -42,7 +42,6 @@ import ast import asyncio -import os import socket from datetime import UTC, datetime, tzinfo from pathlib import Path @@ -57,6 +56,7 @@ from tests.determinism import NotDeterministic from tests.pytest_runs import load_check_junit, run_pytest from tests.source.conftest import FakeClock +from tests.support import files_under TESTS = Path(__file__).resolve().parent SRC = TESTS.parent / "src" / "github_activity_db" @@ -290,10 +290,7 @@ def uncollected_tests(*paths: Path) -> list[str]: def _python_files(root: Path) -> list[Path]: files = sorted(root.rglob("*.py")) - # Counted with os.walk rather than the glob, so a scan cannot pass on fewer files, or on none. - assert len(files) == sum( - Path(name).suffix == ".py" for _, _, names in os.walk(root) for name in names - ) + assert len(files) == len(files_under(root, {".py"})) assert files, f"no Python file under {root}" return files From c4e793f8906573970a35c7f1946c3aafb6a9f276 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 14:31:50 -0600 Subject: [PATCH 28/29] Step 3b-2a-2d: the env.py fix moves to its own change alembic/env.py is main's again. Its fix (an upgrade's exception ends it as itself, and an upgrade runs on one driver connection) moves, with its six tests and its paragraph in docs/database.md, to a change of its own; this step changes nothing under alembic/. --- alembic/env.py | 42 +------- docs/database.md | 5 - docs/redesign-plan.md | 5 +- tests/db/test_migrations.py | 190 ------------------------------------ 4 files changed, 2 insertions(+), 240 deletions(-) diff --git a/alembic/env.py b/alembic/env.py index 307bea2..bdcfadd 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -36,11 +36,6 @@ SQLite ending one by itself (``INSERT OR ROLLBACK``, ``RAISE(ROLLBACK)``). SQLite authorizes a statement when it prepares it, so the connection keeps no prepared statements: each execute is prepared, and checked, again. -- An upgrade runs on one driver connection, which holds its transaction and the state the - authorizer reads. Once it is open, a connect listener closes and refuses any other before - anything runs on it, so a statement after SQLAlchemy invalidated the connection cannot - reconnect outside both. SQLAlchemy raises that statement's refusal wrapped in - ``StatementError``. - Nothing runs on the connection before ``context.configure()``: alembic would treat the transaction as the caller's and never commit it. - ``legacy_alter_table`` stays off, so batch-altering a table a view reads fails unless the @@ -236,29 +231,6 @@ def configure_connection(dbapi_connection: sqlite3.Connection, record: Connectio raise -ONE_CONNECTION = ( - "the upgrade runs on one driver connection: no other is opened, after SQLAlchemy " - "invalidated it or for a second connection to the same engine" -) - - -def refuse_another_connection( - refused: list[str], -) -> Callable[[sqlite3.Connection, ConnectionPoolEntry], None]: - """The connect listener for once the upgrade's connection is open: any other is closed and - refused before anything runs on it, and the refusal is recorded in ``refused``, so a revision - that catches it still fails the upgrade before the commit. Without it, a statement after - SQLAlchemy invalidated the upgrade's connection and a ``rollback()`` reconnects, outside the - transaction and the authorizer's state.""" - - def refuse(dbapi_connection: sqlite3.Connection, _: ConnectionPoolEntry) -> None: - dbapi_connection.close() - refused.append(ONE_CONNECTION) - raise MigrationRefused(ONE_CONNECTION) - - return refuse - - def begin_immediate(connection: Connection) -> None: """Open the upgrade's transaction, taking the write lock at once.""" connection.exec_driver_sql("BEGIN IMMEDIATE") @@ -293,30 +265,18 @@ def run_migrations_online() -> None: event.listen(engine, "connect", configure_connection, insert=True) event.listen(engine, "begin", begin_immediate) with engine.connect() as connection: - refused: list[str] = [] - event.listen(engine, "connect", refuse_another_connection(refused), insert=True) context.configure( connection=connection, target_metadata=Base.metadata, transactional_ddl=True, render_as_batch=True, # autogenerate: SQLite alters tables through batch mode ) - # The mapping the authorizer reads, held before the upgrade runs: SQLAlchemy - # invalidates the connection when an interrupt, or an error it reads as a disconnect, - # ends a statement, and Connection.info then raises in place of that exception. No - # other connection is opened, so this is the live connection's mapping. - state = connection.info with context.begin_transaction(): try: context.run_migrations() - if refused: - raise MigrationRefused( - f"{refused[0]}; a revision went on after the refusal, so nothing was " - "committed" - ) check_foreign_keys(connection) except BaseException: - state[TRANSACTION_STATE] = "failed" + connection.info[TRANSACTION_STATE] = "failed" raise finally: engine.dispose() diff --git a/docs/database.md b/docs/database.md index 50395e9..d54cf29 100644 --- a/docs/database.md +++ b/docs/database.md @@ -67,11 +67,6 @@ converted to UTC, and returns aware UTC values. Defaults are `utc_now()`, never `user_version` and the pragmas reflection reads with an argument (`table_info`, `table_xinfo`, `index_list`, `index_info`, `foreign_key_list`), so `journal_mode`, `legacy_alter_table`, and `ignore_check_constraints` keep the values the connection opened with. -- An upgrade runs on one driver connection. Once it is open, env.py refuses and closes any - other the engine would open: a second connection a revision asks for, or the one SQLAlchemy - would open after it invalidated the first for an interrupt or an error it reads as a - disconnect. The upgrade then fails, even when a revision catches the refusal and goes on, and - nothing is committed. - Refused: offline `--sql` mode, databases other than SQLite, SQLite URI filenames (`?uri=true`), a caller-supplied connection or `sqlalchemy.url`, and the tracked `github_activity.db`. The connect listener asks SQLite which file it opened, so the tracked diff --git a/docs/redesign-plan.md b/docs/redesign-plan.md index 7ca4621..ec16870 100644 --- a/docs/redesign-plan.md +++ b/docs/redesign-plan.md @@ -1942,10 +1942,7 @@ Scope: tests; `docs/testing.md`, `docs/database.md`, `docs/development.md`, `CLA `.github/workflows/check.yml`, which runs the suite a second time in reverse order under another `PYTHONHASHSEED` and compares the two runs; `pyproject.toml`, where ruff skips `tests/fixtures/history/`; two changes under `src/`: `ghactivity sync` times its report on the run's clock, not `time.monotonic` (a wall-clock step during a run shows in the printed seconds), and `db.types.utc_now` reads the time through `_wall_clock`, which a -test replaces or refuses (D40); and one under `alembic/`: `env.py` runs an upgrade on one driver -connection and records a failed upgrade's state on that connection's mapping without asking -SQLAlchemy for the connection again, so an interrupt, a `SystemExit`, or an error SQLAlchemy reads -as a disconnect propagates itself instead of a `PendingRollbackError`. No rule loses its test. Every +test replaces or refuses (D40). Nothing under `alembic/` changes. No rule loses its test. Every test meets D40. `tests/determinism.py` refuses, in the pytest process from collection on, a connection other than to diff --git a/tests/db/test_migrations.py b/tests/db/test_migrations.py index c0ecf5a..c3587ea 100644 --- a/tests/db/test_migrations.py +++ b/tests/db/test_migrations.py @@ -28,8 +28,6 @@ from sqlalchemy.exc import ( DatabaseError, OperationalError, - ProgrammingError, - StatementError, ) from sqlalchemy.pool import Pool @@ -103,8 +101,6 @@ ) # A revision's first lines: the driver connection under SQLAlchemy closed, so the next statement -# fails with an error the pysqlite dialect reads as a disconnect. -CLOSED_CONNECTION = "bind = op.get_bind()\nbind.connection.dbapi_connection.close()\n" def snapshot(database: Path) -> tuple[Any, ...]: @@ -132,15 +128,6 @@ def refusal(error: pytest.ExceptionInfo[BaseException]) -> str: return str(error.value) -def is_closed(connection: sqlite3.Connection) -> bool: - """Whether the driver connection is closed: a closed one runs nothing.""" - try: - connection.execute("SELECT 1") - except sqlite3.ProgrammingError: - return True - return False - - def foreign_key_rows(database: Path) -> dict[str, int]: """The rows of each table that has a foreign key: what ``PRAGMA foreign_key_check`` reads.""" counted = {} @@ -638,183 +625,6 @@ def table_definitions(database: Path) -> dict[str, str]: class TestOneTransaction: - @pytest.mark.parametrize( - "raised", - [ - pytest.param(KeyboardInterrupt(), id="KeyboardInterrupt"), - pytest.param(SystemExit(3), id="SystemExit"), - ], - ) - def test_an_interrupt_after_the_second_revision_applies_none( - self, first_schema: Path, raised: BaseException - ) -> None: - """An interrupt, or a SystemExit, at a set point inside a statement, once the second of - the revisions after the first schema has run: that exception itself ends the upgrade, - and the file stays at the revision the upgrade started from.""" - second = AFTER_THE_FIRST_SCHEMA[1] - invalidated: list[BaseException | None] = [] - - def interrupt(_connection: Any, _cursor: Any, statement: str, *_rest: Any) -> None: - if statement.startswith(f"UPDATE alembic_version SET version_num='{second}'"): - raise raised - - def record(_dbapi_connection: Any, _record: Any, exception: BaseException | None) -> None: - invalidated.append(exception) - - before = snapshot(first_schema) - # Caught here, so a test that fails another way still does not end the run. - with ( - listening("after_cursor_execute", interrupt), - listening("invalidate", record, Pool), - pytest.raises(type(raised)) as error, - ): - upgrade(first_schema) - assert error.value is raised - # Inside a statement, so SQLAlchemy invalidated the connection for it, once. - assert invalidated == [raised] - assert snapshot(first_schema) == before - assert revision_of(first_schema) == FIRST_SCHEMA_HEAD - - def test_an_error_read_as_a_disconnect_applies_none( - self, migrated_database: Path, tmp_path: Path - ) -> None: - """A test-only revision closes its driver connection and runs a statement: SQLAlchemy - reads the driver's error as a disconnect and invalidates the connection, once, and that - error ends the upgrade.""" - versions = tmp_path / "versions" - write_revision( - versions, TEST_REVISION, HEAD_REVISION, f"{CLOSED_CONNECTION}op.execute('SELECT 1')" - ) - invalidated: list[BaseException | None] = [] - - def record(_dbapi_connection: Any, _record: Any, exception: BaseException | None) -> None: - invalidated.append(exception) - - before = snapshot(migrated_database) - with listening("invalidate", record, Pool), pytest.raises(ProgrammingError) as error: - upgrade(migrated_database, test_versions=versions) - assert type(error.value) is ProgrammingError - assert type(error.value.orig) is sqlite3.ProgrammingError - assert error.value.statement == "SELECT 1" - assert error.value.connection_invalidated is True - assert invalidated == [error.value.orig] - assert snapshot(migrated_database) == before - assert revision_of(migrated_database) == HEAD_REVISION - - @pytest.mark.parametrize( - "after", - [ - pytest.param("", id="the revision goes on"), - pytest.param("raise RuntimeError('after the reconnect')", id="the revision raises"), - ], - ) - def test_no_statement_reconnects_after_the_connection_is_invalidated( - self, migrated_database: Path, tmp_path: Path, after: str - ) -> None: - """After SQLAlchemy invalidated the upgrade's connection, a rollback() would let the next - statement reconnect on a second connection, outside the upgrade's transaction and the - state its authorizer reads. env.py refuses that connection and closes it: the statement - fails with the refusal, which SQLAlchemy wraps as it wraps any error raised while it - reconnects, and nothing is applied.""" - versions = tmp_path / "versions" - write_revision( - versions, - TEST_REVISION, - HEAD_REVISION, - f"{CLOSED_CONNECTION}" - "try:\n op.execute('SELECT 1')\nexcept Exception:\n pass\n" - "bind.rollback()\n" - f"op.execute('CREATE TABLE late (x INTEGER)')\n{after}", - ) - connections: list[sqlite3.Connection] = [] - invalidated: list[BaseException | None] = [] - - def connected(dbapi_connection: sqlite3.Connection, _record: Any) -> None: - connections.append(dbapi_connection) - - def record(_dbapi_connection: Any, _record: Any, exception: BaseException | None) -> None: - invalidated.append(exception) - - before = snapshot(migrated_database) - with ( - listening("connect", connected, Pool), - listening("invalidate", record, Pool), - pytest.raises(StatementError) as error, - ): - upgrade(migrated_database, test_versions=versions) - assert type(error.value) is StatementError - assert error.value.statement == "CREATE TABLE late (x INTEGER)" - assert error.value.orig is error.value.__cause__ - assert type(error.value.orig).__name__ == "MigrationRefused" - assert "runs on one driver connection" in str(error.value.orig) - assert [type(exception) for exception in invalidated] == [sqlite3.ProgrammingError] - # The upgrade's connection, closed by the revision, and the refused one, closed by env.py. - assert [is_closed(connection) for connection in connections] == [True, True] - assert snapshot(migrated_database) == before - assert revision_of(migrated_database) == HEAD_REVISION - - def test_an_upgrade_runs_on_one_driver_connection(self, first_schema: Path) -> None: - """The chain from the first schema to head opens one driver connection, and closes it.""" - connections: list[sqlite3.Connection] = [] - - def connected(dbapi_connection: sqlite3.Connection, _record: Any) -> None: - connections.append(dbapi_connection) - - with listening("connect", connected, Pool): - upgrade(first_schema) - assert [is_closed(connection) for connection in connections] == [True] - assert revision_of(first_schema) == HEAD_REVISION - - def test_a_revision_gets_no_second_connection( - self, migrated_database: Path, tmp_path: Path - ) -> None: - """A second connection a revision asks the engine for would hold no part of the - upgrade's transaction: env.py refuses it before anything runs on it, and closes it.""" - versions = tmp_path / "versions" - write_revision(versions, TEST_REVISION, HEAD_REVISION, "op.get_bind().engine.connect()") - connections: list[sqlite3.Connection] = [] - - def connected(dbapi_connection: sqlite3.Connection, _record: Any) -> None: - connections.append(dbapi_connection) - - before = snapshot(migrated_database) - with listening("connect", connected, Pool), pytest.raises(RuntimeError) as error: - upgrade(migrated_database, test_versions=versions) - assert "runs on one driver connection" in refusal(error) - assert [is_closed(connection) for connection in connections] == [True, True] - assert snapshot(migrated_database) == before - assert revision_of(migrated_database) == HEAD_REVISION - - def test_a_revision_that_catches_the_refusal_still_fails_the_upgrade( - self, migrated_database: Path, tmp_path: Path - ) -> None: - """A revision that catches the refusal of a second connection and goes on: the upgrade - fails before the commit, and the file keeps its revision and contents.""" - versions = tmp_path / "versions" - write_revision( - versions, - TEST_REVISION, - HEAD_REVISION, - """ - try: - op.get_bind().engine.connect() - except RuntimeError: - pass - op.execute("CREATE TABLE after_the_refusal (x INTEGER)") - """, - ) - before = snapshot(migrated_database) - with pytest.raises(RuntimeError) as error: - upgrade(migrated_database, test_versions=versions) - assert type(error.value).__name__ == "MigrationRefused" - assert str(error.value) == ( - "the upgrade runs on one driver connection: no other is opened, after SQLAlchemy " - "invalidated it or for a second connection to the same engine; a revision went on " - "after the refusal, so nothing was committed" - ) - assert snapshot(migrated_database) == before - assert revision_of(migrated_database) == HEAD_REVISION - def test_the_chain_commits_a_file_of_seeded_rows(self, step3b1_database: Path) -> None: """The foreign-key check before the upgrade's commit passes a file of valid rows, and each table it reads holds the rows the seed wrote.""" From 0177681b84bfbf341ac92a304553cae38e7f5de2 Mon Sep 17 00:00:00 2001 From: chrishuie Date: Thu, 22 Oct 2026 19:48:18 -0600 Subject: [PATCH 29/29] Check refusals recorded after the last test's check when the guard is undone The final check was a session-scoped fixture, so a session fixture set up before it (a plugin's, a root conftest's) was torn down after it, and a refusal caught there left pytest exiting 0. The check now runs in tests/conftest.py's pytest_unconfigure, after the guard is undone: each refusal is written to stderr and an exit status of 0 becomes 1. run_pytest gives its sample processes a fixed hash seed and makes the directories of the files it writes. --- docs/testing.md | 16 +-- tests/conftest.py | 25 +++-- tests/determinism.py | 9 +- tests/pytest_runs.py | 13 ++- tests/test_suite_rules.py | 216 +++++++++++++++++++++++++++++++++++--- 5 files changed, 240 insertions(+), 39 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index d1fbcd4..f56f4e4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -54,12 +54,16 @@ How the test suite is built and what it counts as evidence. The plan of record i wall clock), and every draw from `SystemClock`'s unseeded random source, by raising `NotDeterministic`. It sets `no_proxy` to `*`, so a client that reads proxies from the environment or the system settings is refused at GitHub's own host on every machine. Each refusal is also - recorded and checked after every test and at the end of the session, so a test fails even where - the code under test caught the exception (a refusal in a wider fixture's teardown fails the next - check); each refusal and that check's failing side have tests. Not refused, and so not enforced at - run time: a socket's other ways to reach a host (`sendto`, `bind`); processes a test starts, which - run without these patches; the event loop's and the standard library's own clocks behind a - timeout; and the time loguru gives a log record, which no test asserts. + recorded and checked after every test, so a test fails even where the code under test caught the + exception (a refusal in a wider fixture's teardown fails the next check). One recorded after the + last test's check (a session fixture's teardown, a later hook) is checked when `tests/conftest.py` + undoes the guard in `pytest_unconfigure`: it fails the run, written to stderr with pytest's exit + status turned from 0 to 1, though neither the JUnit report nor `--same-as` shows it, so pytest's + exit status is part of the verdict. Each refusal and each check's failing side have tests. Not + refused, and so not enforced at run time: a socket's other ways to reach a host (`sendto`, + `bind`); processes a test starts, which run without these patches; the event loop's and the + standard library's own clocks behind a timeout; and the time loguru gives a log record, which no + test asserts. `tests/test_suite_rules.py` reads source: no test file reads the wall clock, sleeps on it, or draws unseeded random numbers (`random.Random` takes a seed, `asyncio.sleep` takes `0`); the package reads them only in its two real clocks, and `alembic/` and `scripts/` read none; and no diff --git a/tests/conftest.py b/tests/conftest.py index b458cf6..a1483cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ import os import shutil +import sys from collections.abc import AsyncIterator, Callable, Iterator from pathlib import Path from typing import TYPE_CHECKING @@ -35,6 +36,8 @@ _GUARD = pytest.MonkeyPatch() +# Starts each line the final check writes to stderr, one for each refusal it finds. +REFUSED_AFTER_THE_LAST_CHECK = "refused after the last check: " def pytest_configure(config: pytest.Config) -> None: @@ -45,7 +48,20 @@ def pytest_configure(config: pytest.Config) -> None: def pytest_unconfigure(config: pytest.Config) -> None: + """The final check, once the guard is off: a refusal recorded after the last test's check (a + session fixture's teardown, a later hook) fails the run. pytest has written its JUnit report by + now, so each one is written to stderr and the exit status goes from 0 to 1.""" _GUARD.undo() + refused = determinism.drain() + if not refused: + return + sys.stdout.flush() + for refusal in refused: + sys.stderr.write(f"{REFUSED_AFTER_THE_LAST_CHECK}{refusal!r}\n") + plugins = config.pluginmanager.get_plugins() + session = next((plugin for plugin in plugins if isinstance(plugin, pytest.Session)), None) + if session is not None and session.exitstatus == pytest.ExitCode.OK: + session.exitstatus = pytest.ExitCode.TESTS_FAILED def pytest_addoption(parser: pytest.Parser) -> None: @@ -64,15 +80,8 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item items.reverse() -@pytest.fixture(autouse=True, scope="session") -def nothing_refused_after_the_last_test() -> Iterator[None]: - """The refusals recorded after the last test's check, in fixtures torn down after it.""" - yield - assert determinism.drain() == [] - - @pytest.fixture(autouse=True) -def nothing_refused(nothing_refused_after_the_last_test: None) -> Iterator[None]: +def nothing_refused() -> Iterator[None]: """A test fails when anything since the previous test's check (collection, a wider fixture's setup or teardown, or the test itself) reached something ``tests/determinism.py`` refuses, even where the code under test caught the refusal.""" diff --git a/tests/determinism.py b/tests/determinism.py index 58fa4eb..17f94eb 100644 --- a/tests/determinism.py +++ b/tests/determinism.py @@ -11,10 +11,11 @@ it was given and is refused there, whatever the machine's proxy settings. A refusal raises ``NotDeterministic``, a ``BaseException`` so that no ``except Exception`` in the code under test can turn it into a handled failure, and is recorded in ``REFUSED``, which the conftest checks after -every test and at the end of the session. Out of scope: processes a test starts (they run without -these patches), the socket's other ways to reach a host (``sendto``, ``sendmsg``, ``bind``, -``listen``, and sockets made in C), and the standard library's clocks and random modules called -directly (``tests/test_suite_rules.py`` reads test files, the package, and ``alembic/`` for those). +every test and once more when it undoes these patches. Out of scope: processes a test starts +(they run without these patches), the socket's other ways to reach a host (``sendto``, +``sendmsg``, ``bind``, ``listen``, and sockets made in C), and the standard library's clocks and +random modules called directly (``tests/test_suite_rules.py`` reads test files, the package, and +``alembic/`` for those). """ from __future__ import annotations diff --git a/tests/pytest_runs.py b/tests/pytest_runs.py index 0fdce8a..1a7ba6d 100644 --- a/tests/pytest_runs.py +++ b/tests/pytest_runs.py @@ -29,12 +29,17 @@ def load_check_junit() -> ModuleType: def run_pytest( root: Path, files: dict[str, str], *options: str, importable: bool = False ) -> subprocess.CompletedProcess[str]: - """pytest over ``files``, written to ``root`` beside a ``pytest.ini`` of their own; the report - is ``root/junit.xml``. With ``importable``, this repository's package and tests import.""" - root.mkdir(exist_ok=True) + """pytest over ``files``, written to ``root`` beside a ``pytest.ini`` of their own, each name a + path under ``root`` whose directories are made; the report is ``root/junit.xml``. With + ``importable``, this repository's package and tests import.""" for name, text in {"pytest.ini": "[pytest]\n", **files}.items(): + (root / name).parent.mkdir(parents=True, exist_ok=True) (root / name).write_text(text) - environment = {"PATH": os.environ.get("PATH", ""), "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"} + environment = { + "PATH": os.environ.get("PATH", ""), + "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", + "PYTHONHASHSEED": "0", + } if importable: environment["PYTHONPATH"] = os.pathsep.join([str(ROOT / "src"), str(ROOT)]) return subprocess.run( diff --git a/tests/test_suite_rules.py b/tests/test_suite_rules.py index ef183b7..fe37f2e 100644 --- a/tests/test_suite_rules.py +++ b/tests/test_suite_rules.py @@ -42,6 +42,7 @@ import ast import asyncio +import json import socket from datetime import UTC, datetime, tzinfo from pathlib import Path @@ -524,16 +525,25 @@ async def sleep(seconds: float) -> None: assert slept == [2.5] +# What the conftest's final check writes to stderr before each refusal it finds; spelled out here, +# not imported, so that this module imports beside a conftest that has no final check. +REFUSED_AFTER_THE_LAST_CHECK = "refused after the last check: " + + def _pytest_over( root: Path, files: dict[str, str], *options: str -) -> tuple[int, list[tuple[str, str]]]: +) -> tuple[int, list[tuple[str, str]], list[tuple[str, str]]]: """pytest run over ``files``, written to ``root`` beside a configuration of their own, with this - repository importable: its exit code, and each test's outcome in the order its JUnit report - lists the tests, which is the order they ran in.""" + repository importable: its exit code, each test's outcome in the order its JUnit report lists + the tests, which is the order they ran in, and each refusal the conftest's final check wrote to + stderr, as (what, detail). Every line on stderr must be such a refusal.""" run = run_pytest(root, files, *options, importable=True) report = root / "junit.xml" assert report.is_file(), run.stdout + run.stderr - return run.returncode, list(load_check_junit().read_outcomes(report).items()) + lines = run.stderr.splitlines() + assert [line for line in lines if not line.startswith(REFUSED_AFTER_THE_LAST_CHECK)] == [] + refused = [ast.literal_eval(line.removeprefix(REFUSED_AFTER_THE_LAST_CHECK)) for line in lines] + return run.returncode, list(load_check_junit().read_outcomes(report).items()), refused # Catches the refusal of the real clock, as code under test may. @@ -548,18 +558,11 @@ def _pytest_over( def test_a_refusal_the_code_caught_fails_the_next_check(tmp_path: Path) -> None: - """Sample tests under this conftest's guard and checks: each refusal caught where it was - raised fails the first check after it, the one after each test or the one at the end of the - session, and the next check passes again.""" + """Sample tests under this conftest, loaded as a plugin: each refusal caught where it was + raised fails the first check after it, the one after each test, and the next check passes + again; one a session fixture's teardown catches after the last test's check is left to the + final check, which writes it to stderr.""" files = { - "conftest.py": ( - "from tests.conftest import (\n" - " nothing_refused,\n" - " nothing_refused_after_the_last_test,\n" - " pytest_configure,\n" - " pytest_unconfigure,\n" - ")\n" - ), "swallow.py": SWALLOW, "test_a_collection.py": "from swallow import swallow\n\nswallow()\n\n\n" "def test_first():\n pass\n", @@ -576,7 +579,7 @@ def test_a_refusal_the_code_caught_fails_the_next_check(tmp_path: Path) -> None: "@pytest.fixture(scope='session')\ndef at_the_end():\n yield\n swallow()\n\n\n" "def test_last(at_the_end):\n pass\n", } - assert _pytest_over(tmp_path / "run", files) == ( + assert _pytest_over(tmp_path / "run", files, "-p", "tests.conftest") == ( 1, [ ("test_a_collection::test_first", "error"), @@ -585,11 +588,188 @@ def test_a_refusal_the_code_caught_fails_the_next_check(tmp_path: Path) -> None: ("test_c_module::test_with_a_setup_that_swallows", "error"), ("test_c_module::test_with_a_teardown_that_swallows", "passed"), ("test_d_next::test_after_the_module_teardown", "error"), - ("test_e_last::test_last", "error"), + ("test_e_last::test_last", "passed"), ], + [("wall clock", "source.clock: datetime.now(UTC)")], ) +# A session-scoped autouse fixture whose cleanup reads CLOCK's time and, as code under test may, +# catches the refusal. +CLEANUP_FIXTURE = """import pytest + +from github_activity_db.source.clock import SystemClock +from tests.determinism import NotDeterministic +from tests.source.conftest import FakeClock + + +@pytest.fixture(autouse=True, scope="session") +def cleanup(): + yield + try: + CLOCK().now() + except NotDeterministic: + pass +""" +# The same cleanup in a plugin's pytest_unconfigure: registered after the guard, it runs before the +# guard's, so after every test's teardown and every session-finish hook, with the guard still on. +CLEANUP_HOOK = """from github_activity_db.source.clock import SystemClock +from tests.determinism import NotDeterministic +from tests.source.conftest import FakeClock + + +def pytest_unconfigure(config): + try: + CLOCK().now() + except NotDeterministic: + pass +""" +# A plugin loaded before the guard is installed: each name tests/determinism.py patches, as it was +# then, and the session, kept from its start. After every other pytest_unconfigure, it writes the +# session's exit status, how many of the names are still patched (and how many were as the session +# finished), and how many refusals are left recorded. +RECORDER = """import json +import socket +from pathlib import Path + +import pytest + +from github_activity_db.db import types +from github_activity_db.source import clock +from github_activity_db.source.clock import SystemClock +from tests import determinism + +GUARDED = [ + (socket.socket, "connect"), + (socket.socket, "connect_ex"), + *((socket, name) for name in determinism.LOOKUPS), + (clock, "datetime"), + (clock, "asyncio"), + (SystemClock, "_random"), + (types, "_wall_clock"), +] +ORIGINALS = [getattr(owner, name) for owner, name in GUARDED] +SEEN = {} + + +def patched(): + return sum( + getattr(owner, name) is not original + for (owner, name), original in zip(GUARDED, ORIGINALS, strict=True) + ) + + +def pytest_sessionstart(session): + SEEN["session"] = session + + +@pytest.hookimpl(trylast=True) +def pytest_sessionfinish(session): + SEEN["patched at sessionfinish"] = patched() + + +@pytest.hookimpl(trylast=True) +def pytest_unconfigure(config): + state = { + "exitstatus": int(SEEN["session"].exitstatus), + "patched at sessionfinish": SEEN["patched at sessionfinish"], + "patched at unconfigure": patched(), + "recorded": len(determinism.REFUSED), + } + Path("state.json").write_text(json.dumps(state)) +""" +TWO_TESTS = "def test_first():\n pass\n\n\ndef test_second():\n pass\n" +BEFORE_THE_GUARD = ("-p", "cleanup", "-p", "tests.conftest") +AFTER_THE_GUARD = ("-p", "tests.conftest", "-p", "cleanup") + + +@pytest.mark.parametrize( + ("files", "options", "module"), + [ + ( + {"cleanup.py": CLEANUP_FIXTURE, "test_sample.py": TWO_TESTS}, + BEFORE_THE_GUARD, + "test_sample", + ), + ( + {"cleanup.py": CLEANUP_FIXTURE, "test_sample.py": TWO_TESTS}, + (*BEFORE_THE_GUARD, "--reverse-order"), + "test_sample", + ), + ( + {"cleanup.py": CLEANUP_FIXTURE, "test_sample.py": TWO_TESTS}, + AFTER_THE_GUARD, + "test_sample", + ), + ( + {"cleanup.py": CLEANUP_FIXTURE, "test_sample.py": TWO_TESTS}, + (*AFTER_THE_GUARD, "--reverse-order"), + "test_sample", + ), + ( + {"cleanup.py": CLEANUP_HOOK, "test_sample.py": TWO_TESTS}, + AFTER_THE_GUARD, + "test_sample", + ), + ( + { + "conftest.py": CLEANUP_FIXTURE, + "guarded/conftest.py": "from tests.conftest import *\n", + "guarded/test_sample.py": TWO_TESTS, + }, + ("guarded",), + "guarded.test_sample", + ), + ], + ids=[ + "a session fixture in a plugin registered before the guard", + "a session fixture in a plugin registered before the guard, last to first", + "a session fixture in a plugin registered after the guard", + "a session fixture in a plugin registered after the guard, last to first", + "a plugin's pytest_unconfigure, run before the guard's", + "a session fixture in the root conftest, the guard in a directory's conftest", + ], +) +def test_a_refusal_caught_after_the_last_tests_check_fails_the_run( + files: dict[str, str], options: tuple[str, ...], module: str, tmp_path: Path +) -> None: + """A cleanup that runs after the last test's check and catches the real clock's refusal fails + the run wherever it sits relative to the guard: pytest exits 1, every test passed, and the + refusal is on stderr. Its twin, reading a fake clock in the same place, exits 0 with nothing on + stderr. After either, the eleven names the guard patched, every one still patched as the + session finished, are restored, and no refusal is left recorded.""" + tests = [f"{module}::test_first", f"{module}::test_second"] + if "--reverse-order" in options: + tests.reverse() + for twin, clock_class, exit_code, refused in [ + ("refusing", "SystemClock", 1, [("wall clock", "source.clock: datetime.now(UTC)")]), + ("clean", "FakeClock", 0, []), + ]: + sample = {"recorder.py": RECORDER} + sample.update({name: text.replace("CLOCK", clock_class) for name, text in files.items()}) + root = tmp_path / twin + assert _pytest_over(root, sample, "-p", "recorder", *options) == ( + exit_code, + [(test, "passed") for test in tests], + refused, + ), twin + assert json.loads((root / "state.json").read_text()) == { + "exitstatus": exit_code, + "patched at sessionfinish": 11, + "patched at unconfigure": 0, + "recorded": 0, + }, twin + + +def test_a_sample_run_hashes_with_a_fixed_seed(tmp_path: Path) -> None: + """The pytest runs these tests start draw no random hash seed (redesign plan D40).""" + files = { + "test_seed.py": "import sys\n\n\n" + "def test_seed():\n assert sys.flags.hash_randomization == 0\n", + } + assert _pytest_over(tmp_path / "run", files) == (0, [("test_seed::test_seed", "passed")], []) + + def test_reverse_order_runs_the_tests_last_to_first( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: @@ -610,6 +790,7 @@ def test_reverse_order_runs_the_tests_last_to_first( ("test_one::test_b", "passed"), ("test_two::test_c", "passed"), ], + [], ) assert _pytest_over(tmp_path / "backward", files, "--reverse-order") == ( 1, @@ -618,6 +799,7 @@ def test_reverse_order_runs_the_tests_last_to_first( ("test_one::test_b", "failed"), ("test_one::test_a", "passed"), ], + [], ) forward, backward = (str(tmp_path / run / "junit.xml") for run in ("forward", "backward")) assert load_check_junit().main([forward, "--max-skipped", "0", "--same-as", backward]) == 1