From 46b7458d9100763056a18e4394f14315506aee7b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 16 Oct 2026 14:41:03 -0700 Subject: [PATCH 01/13] feat(scripts): add --check to generate_schema.py Exits 1 without writing when schema.json is out of date, so the check doesn't depend on the working tree being clean. --- scripts/generate_schema.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/generate_schema.py b/scripts/generate_schema.py index 8e9b8b0..ae75725 100644 --- a/scripts/generate_schema.py +++ b/scripts/generate_schema.py @@ -7,13 +7,15 @@ python scripts/generate_schema.py The script writes the JSON Schema for :class:`~dcd_mapping.schemas.ScoresetMapping` -to ``schema.json`` at the repository root, overwriting the previous file. +to ``schema.json`` at the repository root, overwriting the previous file. With +``--check`` it writes nothing and exits 1 if ``schema.json`` is out of date. If you change schemas.py, re-run this script and commit the result. """ import json import pathlib +import sys from dcd_mapping.schemas import ScoresetMapping @@ -22,9 +24,17 @@ def main() -> None: - """Generate JSON Schema for ScoresetMapping and write to file.""" - schema = ScoresetMapping.model_json_schema() - SCHEMA_PATH.write_text(json.dumps(schema, indent=2, sort_keys=True) + "\n") + """Generate JSON Schema for ScoresetMapping and write it, or check it with --check.""" + schema = ( + json.dumps(ScoresetMapping.model_json_schema(), indent=2, sort_keys=True) + "\n" + ) + if "--check" in sys.argv[1:]: + if SCHEMA_PATH.read_text() != schema: + msg = f"{SCHEMA_PATH} is out of date; run python scripts/generate_schema.py" + print(msg) # noqa: T201 + sys.exit(1) + return + SCHEMA_PATH.write_text(schema) print(f"Written: {SCHEMA_PATH}") # noqa: T201 From d7dbe730ebbabfd3dd5c7e788c7856123cd92970 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 16 Oct 2026 14:41:29 -0700 Subject: [PATCH 02/13] build: add bin/check and bin/t, and run them in CI bin/check runs ruff check, the format check and the schema.json check in about 3 s and reports every failure in one pass. bin/t wraps pytest with CI's MAVEDB_BASE_URL. Both use .venv when present and python3 otherwise, so CI runs the same commands. --- .github/workflows/checks.yaml | 11 +++-------- bin/check | 13 +++++++++++++ bin/t | 9 +++++++++ 3 files changed, 25 insertions(+), 8 deletions(-) create mode 100755 bin/check create mode 100755 bin/t diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index bacb99e..23d7600 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -27,7 +27,7 @@ jobs: # only run tests known to work in CI run: | mkdir -p coverage - python3 -m pytest tests/test_mavedb_data.py --cov-report=lcov:coverage/lcov.info + bin/t tests/test_mavedb_data.py --cov-report=lcov:coverage/lcov.info - name: Upload coverage to Coveralls uses: coverallsapp/github-action@v2 @@ -46,10 +46,5 @@ jobs: - name: Install dependencies run: python3 -m pip install ".[dev]" - - name: Check style - run: python3 -m ruff check . && ruff format --check . - - - name: Verify schema.json is up to date - run: | - python scripts/generate_schema.py - git diff --exit-code schema.json + - name: Lint, format and schema.json checks + run: bin/check diff --git a/bin/check b/bin/check new file mode 100755 index 0000000..1f07999 --- /dev/null +++ b/bin/check @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Fast static checks: lint, format and the schema.json contract. Tests run through bin/t. +# Runs every check before exiting so one invocation reports all failures. +# Uses .venv when present (local development), otherwise the python3 on PATH (CI). +set -uo pipefail +cd "$(dirname "$0")/.." +PY="${PYTHON:-$([ -x .venv/bin/python ] && echo .venv/bin/python || echo python3)}" + +status=0 +"$PY" -m ruff check . || status=1 +"$PY" -m ruff format --check . || status=1 +"$PY" scripts/generate_schema.py --check || status=1 +exit $status diff --git a/bin/t b/bin/t new file mode 100755 index 0000000..0f83da8 --- /dev/null +++ b/bin/t @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Tests: bin/t [test path or node id] [pytest args]. Only tests/test_mavedb_data.py runs in CI; +# most other modules need BLAT, SeqRepo or UTA. MAVEDB_BASE_URL matches CI; requests are mocked. +set -euo pipefail +cd "$(dirname "$0")/.." +PY="${PYTHON:-$([ -x .venv/bin/python ] && echo .venv/bin/python || echo python3)}" + +export MAVEDB_BASE_URL="${MAVEDB_BASE_URL:-https://api.mavedb.org}" +exec "$PY" -m pytest "$@" From 7b569ee38a1d9b7cbbc364de886fd89a66cbd1a4 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 16 Oct 2026 14:42:14 -0700 Subject: [PATCH 03/13] docs: add AGENTS.md and point the README at bin/check and bin/t AGENTS.md (CLAUDE.md imports it) covers what the code doesn't show: the fork's branch flow, which tests run where, the schema.json contract, the GA4GH dependency pin, and the SeqRepo refget rule. --- AGENTS.md | 30 ++++++++++++++++++++++++++++++ CLAUDE.md | 1 + README.md | 7 +++++-- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 AGENTS.md create mode 100644 CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..4df7852 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,30 @@ +# AGENTS.md — dcd_mapping + +Facts about this repo that you can't work out from the code. Keep this file short. Rules shared with the other MaveDB repos are in the workspace `AGENTS.md` one folder up. Update both in the same change that makes any of it untrue. + +The mapper turns a MaveDB score set's variants into GA4GH VRS objects. MaveDB's worker calls it over HTTP (`POST /api/v1/map/{urn}`, served from `src/api/`); the `dcd-map` CLI does the same job locally. This repo is MaveDB's fork of `ave-dcd/dcd_mapping`: work goes to `VariantEffect/dcd_mapping2`, branched from and merged into `mavedb-dev`, which is released to `mavedb-main`. + +## Setup + +- `python3 -m venv .venv && .venv/bin/pip install -e '.[dev,tests]'`, on Python 3.11. The `bin/` scripts use `.venv` when it exists. +- A real mapping run needs UTA, SeqRepo, the Gene Normalizer database and BLAT. The README's Prerequisites section covers each. + +## Commands + +| Task | Command | +|---|---| +| Lint, format and `schema.json` checks | `bin/check` | +| What CI tests | `bin/t tests/test_mavedb_data.py` | +| Other tests | `bin/t tests/test_variant_projection.py` | +| Regenerate `schema.json` | `.venv/bin/python scripts/generate_schema.py` | + +- Run `bin/check` after every edit, and the tests for what you touched before handing work back. +- Only `tests/test_mavedb_data.py` runs in CI. The align and `vrs_map` tests need BLAT, and others need SeqRepo or UTA, so some fail on a machine without them. Check a failure against `mavedb-dev` before blaming your change, and report it rather than skipping or loosening the test. + +## Rules + +- Open pull requests with `gh pr create -R VariantEffect/dcd_mapping2 --base mavedb-dev`. Without `-R`, `gh` may target the upstream `ave-dcd` repo. +- `schema.json` is the output contract the MaveDB api consumes. A change to `schemas.py` that alters it ships with the regenerated file in the same commit; `bin/check` fails otherwise. +- `ga4gh.vrs` is pinned to 2.0.0a6 and `cool-seq-tool` to 0.4.0.dev3, because upgrading either means moving the whole GA4GH stack (gene-normalizer, seqrepo, cdot) at once. Don't bump any of them as part of other work. +- Never store a RefSeq sequence in SeqRepo from another source, such as cdot's genome-assembled transcript. It changes the refget, and with it every allele digest MaveDB has published. +- Docstrings use the reST `:param:` style the code already has, not Google style. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/README.md b/README.md index 84c81f0..69adf0b 100644 --- a/README.md +++ b/README.md @@ -165,8 +165,11 @@ Add pre-commit hooks pre-commit install ``` -Run tests with `pytest` +Run the lint, format and `schema.json` checks with `bin/check`, and tests with `bin/t`, which wraps `pytest`: ``` -pytest +bin/check +bin/t tests/test_mavedb_data.py ``` + +Only `tests/test_mavedb_data.py` runs in CI; most other test modules need BLAT, SeqRepo or UTA. Conventions for contributors and coding agents are in [AGENTS.md](AGENTS.md). From f846ebf01498c8d884f4bcbe25d798a8d612e706 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 16 Oct 2026 15:11:29 -0700 Subject: [PATCH 04/13] build(claude): add shared permission rules Deny --no-verify and reads of ~/.aws and ~/.pgpass. Ask before pushes, AWS and Elastic Beanstalk commands, the prod-API dev server and dependency changes. The same file lives in the workspace and in each repo, because Claude Code reads settings only from the folder a session starts in. These are guardrails, not a security boundary: a rewritten command can get past them. --- .claude/settings.json | 31 +++++++++++++++++++++++++++++++ .gitignore | 5 +++-- 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 .claude/settings.json diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..24b6905 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,31 @@ +{ + "permissions": { + "deny": [ + "Bash(* --no-verify*)", + "Read(~/.aws/**)", + "Read(~/.pgpass)" + ], + "ask": [ + "Bash(git push*)", + "Bash(git -C * push*)", + "Bash(aws *)", + "Bash(eb *)", + "Bash(*MODE=prodapi*)", + "Bash(poetry add *)", + "Bash(poetry remove *)", + "Bash(poetry update*)", + "Bash(poetry lock*)", + "Bash(poetry -C * add *)", + "Bash(poetry -C * remove *)", + "Bash(poetry -C * update*)", + "Bash(poetry -C * lock*)", + "Bash(npm install*)", + "Bash(npm uninstall *)", + "Bash(npm update*)", + "Bash(npm --prefix * install*)", + "Bash(npm --prefix * uninstall *)", + "Bash(npm --prefix * update*)", + "Bash(*pip install *)" + ] + } +} diff --git a/.gitignore b/.gitignore index 439b17c..1aec9a5 100644 --- a/.gitignore +++ b/.gitignore @@ -173,5 +173,6 @@ urn:*.json tmp:*.json *_mapping_*.json -# Agent settings -.claude/ +# Agent settings: personal files stay local; the shared permission rules are committed +.claude/* +!.claude/settings.json From 7df2ec093705c9513df7bb23a14610a0df22975c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 16 Oct 2026 15:40:47 -0700 Subject: [PATCH 05/13] build(claude): run AWS as the read-only agent role and block ~/.ssh Sessions get AWS_PROFILE=mavedb-agent-readonly, so aws commands use the read-only role (see deployment/environments/aws/agent-access.md) and fail rather than fall back to full credentials where the profile is missing. Choosing another profile asks first, replacing the blanket prompt on aws. Reading ~/.ssh is now blocked, alongside ~/.aws and ~/.pgpass. --- .claude/settings.json | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.claude/settings.json b/.claude/settings.json index 24b6905..cb0cb47 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,14 +1,19 @@ { + "env": { + "AWS_PROFILE": "mavedb-agent-readonly" + }, "permissions": { "deny": [ "Bash(* --no-verify*)", "Read(~/.aws/**)", - "Read(~/.pgpass)" + "Read(~/.pgpass)", + "Read(~/.ssh/**)" ], "ask": [ "Bash(git push*)", "Bash(git -C * push*)", - "Bash(aws *)", + "Bash(*--profile*)", + "Bash(*AWS_PROFILE=*)", "Bash(eb *)", "Bash(*MODE=prodapi*)", "Bash(poetry add *)", From 39f12f97ed5830edb3ef9d234a68d952f8f1a23b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 16 Oct 2026 15:59:56 -0700 Subject: [PATCH 06/13] build(claude): ask before edits to instruction and settings files Edits to AGENTS.md, CLAUDE.md and anything under .claude/ now ask first. Those files shape every later session, and settings.json holds the guardrails. The rules cover Edit and Write; a shell redirect into one of these files is not caught. --- .claude/settings.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.claude/settings.json b/.claude/settings.json index cb0cb47..5b1bde9 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -10,6 +10,9 @@ "Read(~/.ssh/**)" ], "ask": [ + "Edit(**/AGENTS.md)", + "Edit(**/CLAUDE.md)", + "Edit(**/.claude/**)", "Bash(git push*)", "Bash(git -C * push*)", "Bash(*--profile*)", From 2df116b70e07fcd8384c138e76982e41db946942 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 16 Oct 2026 16:18:06 -0700 Subject: [PATCH 07/13] build(claude): ask only before hard-to-undo actions Prompts now cover force pushes, pushes to main, mavedb-main, mavedb-dev or release-*, AWS with an explicit profile, eb, adding a dependency and edits to settings.json itself. Ordinary feature-branch pushes, dependency updates and edits to AGENTS.md, CLAUDE.md and .claude/rules no longer ask; the never rule in the workspace AGENTS.md still covers those edits, and PR review sees the diffs. --- .claude/settings.json | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/.claude/settings.json b/.claude/settings.json index 5b1bde9..509725c 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -10,29 +10,27 @@ "Read(~/.ssh/**)" ], "ask": [ - "Edit(**/AGENTS.md)", - "Edit(**/CLAUDE.md)", - "Edit(**/.claude/**)", - "Bash(git push*)", - "Bash(git -C * push*)", - "Bash(*--profile*)", + "Edit(**/.claude/settings*.json)", + "Bash(git push *--force*)", + "Bash(git push * -f*)", + "Bash(git push * main*)", + "Bash(git push * mavedb-main*)", + "Bash(git push * mavedb-dev*)", + "Bash(git push * release-*)", + "Bash(git -C * push *--force*)", + "Bash(git -C * push * -f*)", + "Bash(git -C * push * main*)", + "Bash(git -C * push * mavedb-main*)", + "Bash(git -C * push * mavedb-dev*)", + "Bash(git -C * push * release-*)", + "Bash(aws *--profile*)", "Bash(*AWS_PROFILE=*)", "Bash(eb *)", "Bash(*MODE=prodapi*)", "Bash(poetry add *)", - "Bash(poetry remove *)", - "Bash(poetry update*)", - "Bash(poetry lock*)", "Bash(poetry -C * add *)", - "Bash(poetry -C * remove *)", - "Bash(poetry -C * update*)", - "Bash(poetry -C * lock*)", - "Bash(npm install*)", - "Bash(npm uninstall *)", - "Bash(npm update*)", - "Bash(npm --prefix * install*)", - "Bash(npm --prefix * uninstall *)", - "Bash(npm --prefix * update*)", + "Bash(npm install *)", + "Bash(npm --prefix * install *)", "Bash(*pip install *)" ] } From 8702dca4c3fbcdb751340b0058054bfc176e205e Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 16 Oct 2026 16:23:52 -0700 Subject: [PATCH 08/13] style(claude): format settings.json with Prettier The ui checks it with Prettier, so every copy uses that layout. --- .claude/settings.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.claude/settings.json b/.claude/settings.json index 509725c..3dac982 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -3,12 +3,7 @@ "AWS_PROFILE": "mavedb-agent-readonly" }, "permissions": { - "deny": [ - "Bash(* --no-verify*)", - "Read(~/.aws/**)", - "Read(~/.pgpass)", - "Read(~/.ssh/**)" - ], + "deny": ["Bash(* --no-verify*)", "Read(~/.aws/**)", "Read(~/.pgpass)", "Read(~/.ssh/**)"], "ask": [ "Edit(**/.claude/settings*.json)", "Bash(git push *--force*)", From 37ff409683207f7eddbbbddc3701b6146a6280d9 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 19 Oct 2026 11:15:00 -0700 Subject: [PATCH 09/13] ci: move linked issues across the backlog board from PR events Opening or pushing to a PR moves its linked issues to In review, and merging it into a release branch moves them to Pending release. A release PR into the default branch gets a Closes line for every issue its branch picked up, so GitHub closes them when the release merges. Issues only ever move forward. Adds a pull request template whose Closes line feeds the automation. --- .github/pull_request_template.md | 20 +++ .github/scripts/issue-status.js | 243 ++++++++++++++++++++++++++++ .github/workflows/issue-status.yaml | 41 +++++ 3 files changed, 304 insertions(+) create mode 100644 .github/pull_request_template.md create mode 100644 .github/scripts/issue-status.js create mode 100644 .github/workflows/issue-status.yaml diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..63e6969 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,20 @@ +## Summary + + + +Closes # + + + +## Tests + + + +## Checklist + +- [ ] Targets `mavedb-dev`, not `mavedb-main` +- [ ] Docs and instruction files touched by this change are updated diff --git a/.github/scripts/issue-status.js b/.github/scripts/issue-status.js new file mode 100644 index 0000000..cd06829 --- /dev/null +++ b/.github/scripts/issue-status.js @@ -0,0 +1,243 @@ +/** + * Moves issues across the MaveDB Backlog project as the pull requests linked to them progress. + * + * - A PR is opened, pushed to or marked ready: its linked issues go to In review (In progress while it's a draft). + * - A PR merges into a release branch: its linked issues go to Pending release, once no other open PR still links them. + * - A release branch's PR into the default branch is opened or pushed to: its description gets a "Closes" line for every + * open issue linked from a PR merged into that branch since its last release, so GitHub closes them when the release + * merges. The board's "Item closed" workflow then sets Done. + * + * Release branches are release-* in mavedb-api and mavedb-ui, and the long-lived mavedb-dev in dcd_mapping2. The + * pattern covers every repo because issues are linked across repos, and each repo carries an identical copy of this + * file. + * + * Issues only move forward, so the release PR linking an issue that is already Pending release leaves it there. Issues + * that aren't on the board are skipped, not added: adding one fires the board's "Item added to project" workflow, which + * would race this script's status change. + * + * Called from .github/workflows/issue-status.yml through actions/github-script, with a GitHub App token that can write + * to the org's projects. + */ + +const PROJECT_OWNER = 'VariantEffect'; +const PROJECT_NUMBER = 4; + +// Board order, left to right. Matched by suffix so the option emoji can change without breaking this. +const ORDER = ['New', 'Backlog', 'Ready', 'In progress', 'Waiting/Blocked', 'In review', 'Pending release', 'Done']; + +const RELEASE_SECTION_START = ''; +const RELEASE_SECTION_END = ''; + +const rank = (statusName) => (statusName ? ORDER.findIndex((label) => statusName.endsWith(label)) : -1); +const isReleaseBranch = (ref) => /^(release-.+|mavedb-dev)$/.test(ref); + +async function loadProject(github) { + const { organization } = await github.graphql( + `query($owner: String!, $number: Int!) { + organization(login: $owner) { + projectV2(number: $number) { + id + field(name: "Status") { ... on ProjectV2SingleSelectField { id options { id name } } } + } + } + }`, + { owner: PROJECT_OWNER, number: PROJECT_NUMBER }, + ); + const project = organization.projectV2; + const options = Object.fromEntries( + ORDER.map((label) => { + const option = project.field.options.find((o) => o.name.endsWith(label)); + if (!option) throw new Error(`The project's Status field has no "${label}" option.`); + return [label, option.id]; + }), + ); + return { id: project.id, fieldId: project.field.id, options }; +} + +async function linkedIssues(github, owner, repo, number) { + const { repository } = await github.graphql( + `query($owner: String!, $repo: String!, $number: Int!) { + repository(owner: $owner, name: $repo) { + pullRequest(number: $number) { + closingIssuesReferences(first: 50) { + nodes { + id number state + repository { nameWithOwner } + projectItems(first: 20) { + nodes { + id + project { id } + status: fieldValueByName(name: "Status") { ... on ProjectV2ItemFieldSingleSelectValue { name } } + } + } + closedByPullRequestsReferences(first: 20, includeClosedPrs: false) { + nodes { number headRefName repository { nameWithOwner } } + } + } + } + } + } + }`, + { owner, repo, number }, + ); + return repository.pullRequest.closingIssuesReferences.nodes.filter((issue) => issue.state === 'OPEN'); +} + +async function moveIssue(github, core, project, issue, target) { + const label = `${issue.repository.nameWithOwner}#${issue.number}`; + const item = issue.projectItems.nodes.find((i) => i.project.id === project.id); + if (!item) { + core.notice(`${label} isn't on the project; skipping.`); + return; + } + const current = item.status?.name; + if (rank(current) >= ORDER.indexOf(target)) { + core.info(`${label} is already at "${current}"; leaving it.`); + return; + } + await github.graphql( + `mutation($project: ID!, $item: ID!, $field: ID!, $option: String!) { + updateProjectV2ItemFieldValue(input: { + projectId: $project, itemId: $item, fieldId: $field, value: { singleSelectOptionId: $option } + }) { projectV2Item { id } } + }`, + { project: project.id, item: item.id, field: project.fieldId, option: project.options[target] }, + ); + core.info(`${label}: "${current ?? 'no status'}" -> "${target}".`); +} + +async function markInReview(github, core, project, owner, repo, pr) { + const target = pr.draft ? 'In progress' : 'In review'; + for (const issue of await linkedIssues(github, owner, repo, pr.number)) { + await moveIssue(github, core, project, issue, target); + } +} + +// An issue split across several PRs, possibly in different repos, waits for the last of them. Release PRs link every +// issue in the release, so they don't count. +function stillOpenElsewhere(core, issue) { + const open = issue.closedByPullRequestsReferences.nodes.filter((p) => !isReleaseBranch(p.headRefName)); + if (open.length) { + const refs = open.map((p) => `${p.repository.nameWithOwner}#${p.number}`).join(', '); + core.info(`${issue.repository.nameWithOwner}#${issue.number} is still linked from open ${refs}; leaving it.`); + } + return open.length > 0; +} + +async function markPendingRelease(github, core, project, owner, repo, pr) { + // By the time "closed" fires, the merged PR no longer counts as open. + for (const issue of await linkedIssues(github, owner, repo, pr.number)) { + if (!stillOpenElsewhere(core, issue)) await moveIssue(github, core, project, issue, 'Pending release'); + } +} + +// Long-lived release branches (mavedb-dev) keep their history, so only PRs merged since the branch last shipped count. +async function lastReleasedAt(github, owner, repo, pr) { + const { search } = await github.graphql( + `query($query: String!) { + search(type: ISSUE, query: $query, first: 1) { nodes { ... on PullRequest { mergedAt } } } + }`, + { query: `repo:${owner}/${repo} is:pr is:merged head:${pr.head.ref} base:${pr.base.ref} sort:created-desc` }, + ); + return search.nodes[0]?.mergedAt ?? null; +} + +async function updateReleaseDescription(github, core, owner, repo, pr) { + const since = await lastReleasedAt(github, owner, repo, pr); + const query = `repo:${owner}/${repo} is:pr is:merged base:${pr.head.ref}` + (since ? ` merged:>${since}` : ''); + const issues = new Map(); + let cursor = null; + do { + const { search } = await github.graphql( + `query($query: String!, $cursor: String) { + search(type: ISSUE, query: $query, first: 50, after: $cursor) { + pageInfo { hasNextPage endCursor } + nodes { + ... on PullRequest { + closingIssuesReferences(first: 50) { + nodes { + number state + repository { nameWithOwner } + closedByPullRequestsReferences(first: 20, includeClosedPrs: false) { + nodes { number headRefName repository { nameWithOwner } } + } + } + } + } + } + } + }`, + { query, cursor }, + ); + for (const merged of search.nodes) { + for (const issue of merged.closingIssuesReferences.nodes) { + if (issue.state !== 'OPEN' || stillOpenElsewhere(core, issue)) continue; + const sameRepo = issue.repository.nameWithOwner === `${owner}/${repo}`; + const ref = sameRepo ? `#${issue.number}` : `${issue.repository.nameWithOwner}#${issue.number}`; + issues.set(ref, issue); + } + } + cursor = search.pageInfo.hasNextPage ? search.pageInfo.endCursor : null; + } while (cursor); + + const refs = [...issues.keys()].sort((a, b) => a.localeCompare(b, 'en', { numeric: true })); + const section = [ + RELEASE_SECTION_START, + '### Issues closed by this release', + '', + `_Generated from the issues linked to PRs merged into \`${pr.head.ref}\` for this release, and rewritten on every push. Issues still linked from an open PR are left out. To change the list, change those PRs' linked issues._`, + '', + ...(refs.length ? refs.map((ref) => `- Closes ${ref}`) : ['None.']), + RELEASE_SECTION_END, + ].join('\n'); + + const body = pr.body ?? ''; + const start = body.indexOf(RELEASE_SECTION_START); + const end = body.indexOf(RELEASE_SECTION_END); + const updated = + start !== -1 && end > start + ? body.slice(0, start) + section + body.slice(end + RELEASE_SECTION_END.length) + : `${body.trimEnd()}\n\n${section}\n`; + if (updated === body) { + core.info('Release description is up to date.'); + return; + } + await github.rest.pulls.update({ owner, repo, pull_number: pr.number, body: updated }); + core.info(`Release description lists ${refs.length} issue(s).`); +} + +module.exports = async ({ github, context, core }) => { + const { owner, repo } = context.repo; + const project = await loadProject(github); + + // Linking an issue from the PR's Development sidebar fires no event, so a periodic sweep catches those. + if (context.eventName === 'schedule' || context.eventName === 'workflow_dispatch') { + const prs = await github.paginate(github.rest.pulls.list, { owner, repo, state: 'open', per_page: 100 }); + for (const pr of prs.filter((p) => !isReleaseBranch(p.head.ref))) { + await markInReview(github, core, project, owner, repo, pr); + } + return; + } + + const pr = context.payload.pull_request; + const action = context.payload.action; + + if (isReleaseBranch(pr.head.ref) && pr.base.ref === context.payload.repository.default_branch) { + if (['opened', 'reopened', 'synchronize'].includes(action)) { + await updateReleaseDescription(github, core, owner, repo, pr); + } + return; + } + + if (action === 'closed') { + if (pr.merged && isReleaseBranch(pr.base.ref)) { + await markPendingRelease(github, core, project, owner, repo, pr); + } + return; + } + + // Hotfixes merged straight into the default branch close their issues natively. + if (pr.base.ref !== context.payload.repository.default_branch) { + await markInReview(github, core, project, owner, repo, pr); + } +}; diff --git a/.github/workflows/issue-status.yaml b/.github/workflows/issue-status.yaml new file mode 100644 index 0000000..8d5d305 --- /dev/null +++ b/.github/workflows/issue-status.yaml @@ -0,0 +1,41 @@ +name: Issue status + +# Moves linked issues across the MaveDB Backlog project; the logic and its rules are in .github/scripts/issue-status.js. +# pull_request_target runs this file from the base branch with the org secrets, so it works for fork PRs too. It never +# checks out or runs the PR's code. +on: + pull_request_target: + types: [opened, reopened, edited, synchronize, ready_for_review, converted_to_draft, closed] + schedule: + - cron: '17 * * * *' + workflow_dispatch: + +concurrency: + group: issue-status-${{ github.event.pull_request.number || github.run_id }} + +permissions: + contents: read + +jobs: + update: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: .github/scripts + + # The default GITHUB_TOKEN can't write to org projects. The app needs org Projects read/write and repo Issues + # read and Pull requests read/write, installed on mavedb-api, mavedb-ui and dcd_mapping2. + - uses: actions/create-github-app-token@v2 + id: app-token + with: + app-id: ${{ vars.PROJECT_AUTOMATION_APP_ID }} + private-key: ${{ secrets.PROJECT_AUTOMATION_PRIVATE_KEY }} + owner: VariantEffect + + - uses: actions/github-script@v7 + with: + github-token: ${{ steps.app-token.outputs.token }} + script: | + const run = require('./.github/scripts/issue-status.js'); + await run({ github, context, core }); From d752eee1bea9a8568464ad624b4d7f429661beb0 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 19 Oct 2026 11:27:25 -0700 Subject: [PATCH 10/13] ci: read Closes lines from PR descriptions GitHub turns a Closes line into an issue link only on PRs into the default branch. On a PR into a release branch it stays a plain mention, so the workflow missed issues named that way. The script now reads closing lines from PR descriptions as well as the sidebar links. --- .github/pull_request_template.md | 2 +- .github/scripts/issue-status.js | 123 ++++++++++++++++++++----------- 2 files changed, 82 insertions(+), 43 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 63e6969..69b04ed 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -5,7 +5,7 @@ Closes # diff --git a/.github/scripts/issue-status.js b/.github/scripts/issue-status.js index cd06829..ec24876 100644 --- a/.github/scripts/issue-status.js +++ b/.github/scripts/issue-status.js @@ -11,6 +11,10 @@ * pattern covers every repo because issues are linked across repos, and each repo carries an identical copy of this * file. * + * A PR's issues are the ones linked from its Development sidebar plus those named on a "Closes #N" line in its + * description. GitHub turns that line into a link only on PRs into the default branch; on a PR into a release branch it + * is a plain mention, so this script reads the description itself. + * * Issues only move forward, so the release PR linking an issue that is already Pending release leaves it there. Issues * that aren't on the board are skipped, not added: adding one fires the board's "Item added to project" workflow, which * would race this script's status change. @@ -54,33 +58,73 @@ async function loadProject(github) { return { id: project.id, fieldId: project.field.id, options }; } -async function linkedIssues(github, owner, repo, number) { +const ISSUE_FIELDS = ` + id number state + repository { nameWithOwner } + projectItems(first: 20) { + nodes { + id + project { id } + status: fieldValueByName(name: "Status") { ... on ProjectV2ItemFieldSingleSelectValue { name } } + } + } + closedByPullRequestsReferences(first: 20, includeClosedPrs: false) { + nodes { number headRefName repository { nameWithOwner } } + } +`; + +const CLOSING_REF = /\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?):?\s+(?:([\w.-]+)\/([\w.-]+))?#(\d+)\b/gi; + +// HTML comments are dropped first so the template's own "Closes #123" example doesn't count. +function closingRefs(body, owner, repo) { + const text = (body ?? '').replace(//g, ''); + return [...text.matchAll(CLOSING_REF)].map(([, o, r, n]) => ({ owner: o ?? owner, repo: r ?? repo, number: Number(n) })); +} + +const linkRefs = (links) => + links.nodes.map((i) => { + const [owner, repo] = i.repository.nameWithOwner.split('/'); + return { owner, repo, number: i.number }; + }); + +// Fetches the issues behind a set of refs in one query. Refs that turn out to be PRs, or that don't resolve, are dropped. +async function fetchIssues(github, refs) { + const unique = [...new Map(refs.map((r) => [`${r.owner}/${r.repo}#${r.number}`.toLowerCase(), r])).values()]; + if (!unique.length) return []; + const query = `query { + ${unique + .map( + (r, i) => `i${i}: repository(owner: ${JSON.stringify(r.owner)}, name: ${JSON.stringify(r.repo)}) { + issueOrPullRequest(number: ${r.number}) { ... on Issue { ${ISSUE_FIELDS} } } + }`, + ) + .join('\n')} + }`; + let data; + try { + data = await github.graphql(query); + } catch (error) { + if (!error.data) throw error; + data = error.data; // A ref to a missing repo fails only its own alias. + } + return Object.values(data) + .map((r) => r?.issueOrPullRequest) + .filter((issue) => issue?.id && issue.state === 'OPEN'); +} + +async function linkedIssues(github, owner, repo, pr) { const { repository } = await github.graphql( `query($owner: String!, $repo: String!, $number: Int!) { repository(owner: $owner, name: $repo) { pullRequest(number: $number) { - closingIssuesReferences(first: 50) { - nodes { - id number state - repository { nameWithOwner } - projectItems(first: 20) { - nodes { - id - project { id } - status: fieldValueByName(name: "Status") { ... on ProjectV2ItemFieldSingleSelectValue { name } } - } - } - closedByPullRequestsReferences(first: 20, includeClosedPrs: false) { - nodes { number headRefName repository { nameWithOwner } } - } - } - } + closingIssuesReferences(first: 50) { nodes { number repository { nameWithOwner } } } } } }`, - { owner, repo, number }, + { owner, repo, number: pr.number }, ); - return repository.pullRequest.closingIssuesReferences.nodes.filter((issue) => issue.state === 'OPEN'); + const links = repository.pullRequest.closingIssuesReferences; + return fetchIssues(github, [...linkRefs(links), ...closingRefs(pr.body, owner, repo)]); } async function moveIssue(github, core, project, issue, target) { @@ -108,13 +152,13 @@ async function moveIssue(github, core, project, issue, target) { async function markInReview(github, core, project, owner, repo, pr) { const target = pr.draft ? 'In progress' : 'In review'; - for (const issue of await linkedIssues(github, owner, repo, pr.number)) { + for (const issue of await linkedIssues(github, owner, repo, pr)) { await moveIssue(github, core, project, issue, target); } } // An issue split across several PRs, possibly in different repos, waits for the last of them. Release PRs link every -// issue in the release, so they don't count. +// issue in the release, so they don't count. Only sidebar links are visible here, not "Closes" lines in other PRs. function stillOpenElsewhere(core, issue) { const open = issue.closedByPullRequestsReferences.nodes.filter((p) => !isReleaseBranch(p.headRefName)); if (open.length) { @@ -126,7 +170,7 @@ function stillOpenElsewhere(core, issue) { async function markPendingRelease(github, core, project, owner, repo, pr) { // By the time "closed" fires, the merged PR no longer counts as open. - for (const issue of await linkedIssues(github, owner, repo, pr.number)) { + for (const issue of await linkedIssues(github, owner, repo, pr)) { if (!stillOpenElsewhere(core, issue)) await moveIssue(github, core, project, issue, 'Pending release'); } } @@ -145,7 +189,7 @@ async function lastReleasedAt(github, owner, repo, pr) { async function updateReleaseDescription(github, core, owner, repo, pr) { const since = await lastReleasedAt(github, owner, repo, pr); const query = `repo:${owner}/${repo} is:pr is:merged base:${pr.head.ref}` + (since ? ` merged:>${since}` : ''); - const issues = new Map(); + const refs = []; let cursor = null; do { const { search } = await github.graphql( @@ -154,15 +198,8 @@ async function updateReleaseDescription(github, core, owner, repo, pr) { pageInfo { hasNextPage endCursor } nodes { ... on PullRequest { - closingIssuesReferences(first: 50) { - nodes { - number state - repository { nameWithOwner } - closedByPullRequestsReferences(first: 20, includeClosedPrs: false) { - nodes { number headRefName repository { nameWithOwner } } - } - } - } + body + closingIssuesReferences(first: 50) { nodes { number repository { nameWithOwner } } } } } } @@ -170,24 +207,26 @@ async function updateReleaseDescription(github, core, owner, repo, pr) { { query, cursor }, ); for (const merged of search.nodes) { - for (const issue of merged.closingIssuesReferences.nodes) { - if (issue.state !== 'OPEN' || stillOpenElsewhere(core, issue)) continue; - const sameRepo = issue.repository.nameWithOwner === `${owner}/${repo}`; - const ref = sameRepo ? `#${issue.number}` : `${issue.repository.nameWithOwner}#${issue.number}`; - issues.set(ref, issue); - } + refs.push(...linkRefs(merged.closingIssuesReferences), ...closingRefs(merged.body, owner, repo)); } cursor = search.pageInfo.hasNextPage ? search.pageInfo.endCursor : null; } while (cursor); - const refs = [...issues.keys()].sort((a, b) => a.localeCompare(b, 'en', { numeric: true })); + const issues = new Map(); + for (const issue of await fetchIssues(github, refs)) { + if (stillOpenElsewhere(core, issue)) continue; + const sameRepo = issue.repository.nameWithOwner === `${owner}/${repo}`; + issues.set(sameRepo ? `#${issue.number}` : `${issue.repository.nameWithOwner}#${issue.number}`, issue); + } + + const listed = [...issues.keys()].sort((a, b) => a.localeCompare(b, 'en', { numeric: true })); const section = [ RELEASE_SECTION_START, '### Issues closed by this release', '', - `_Generated from the issues linked to PRs merged into \`${pr.head.ref}\` for this release, and rewritten on every push. Issues still linked from an open PR are left out. To change the list, change those PRs' linked issues._`, + `_Generated from the issues linked to PRs merged into \`${pr.head.ref}\` for this release, and rewritten on every push. Issues still linked from an open PR are left out. To change the list, change those PRs' linked issues or Closes lines._`, '', - ...(refs.length ? refs.map((ref) => `- Closes ${ref}`) : ['None.']), + ...(listed.length ? listed.map((ref) => `- Closes ${ref}`) : ['None.']), RELEASE_SECTION_END, ].join('\n'); @@ -203,7 +242,7 @@ async function updateReleaseDescription(github, core, owner, repo, pr) { return; } await github.rest.pulls.update({ owner, repo, pull_number: pr.number, body: updated }); - core.info(`Release description lists ${refs.length} issue(s).`); + core.info(`Release description lists ${listed.length} issue(s).`); } module.exports = async ({ github, context, core }) => { From a19b6c4b502f809bb35e8a7079ca083e9a1d291a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 19 Oct 2026 11:49:57 -0700 Subject: [PATCH 11/13] style(ci): format the issue-status script with Prettier --- .github/scripts/issue-status.js | 196 ++++++++++++++++---------------- 1 file changed, 98 insertions(+), 98 deletions(-) diff --git a/.github/scripts/issue-status.js b/.github/scripts/issue-status.js index ec24876..1c8afdc 100644 --- a/.github/scripts/issue-status.js +++ b/.github/scripts/issue-status.js @@ -23,20 +23,20 @@ * to the org's projects. */ -const PROJECT_OWNER = 'VariantEffect'; -const PROJECT_NUMBER = 4; +const PROJECT_OWNER = 'VariantEffect' +const PROJECT_NUMBER = 4 // Board order, left to right. Matched by suffix so the option emoji can change without breaking this. -const ORDER = ['New', 'Backlog', 'Ready', 'In progress', 'Waiting/Blocked', 'In review', 'Pending release', 'Done']; +const ORDER = ['New', 'Backlog', 'Ready', 'In progress', 'Waiting/Blocked', 'In review', 'Pending release', 'Done'] -const RELEASE_SECTION_START = ''; -const RELEASE_SECTION_END = ''; +const RELEASE_SECTION_START = '' +const RELEASE_SECTION_END = '' -const rank = (statusName) => (statusName ? ORDER.findIndex((label) => statusName.endsWith(label)) : -1); -const isReleaseBranch = (ref) => /^(release-.+|mavedb-dev)$/.test(ref); +const rank = (statusName) => (statusName ? ORDER.findIndex((label) => statusName.endsWith(label)) : -1) +const isReleaseBranch = (ref) => /^(release-.+|mavedb-dev)$/.test(ref) async function loadProject(github) { - const { organization } = await github.graphql( + const {organization} = await github.graphql( `query($owner: String!, $number: Int!) { organization(login: $owner) { projectV2(number: $number) { @@ -45,17 +45,17 @@ async function loadProject(github) { } } }`, - { owner: PROJECT_OWNER, number: PROJECT_NUMBER }, - ); - const project = organization.projectV2; + {owner: PROJECT_OWNER, number: PROJECT_NUMBER} + ) + const project = organization.projectV2 const options = Object.fromEntries( ORDER.map((label) => { - const option = project.field.options.find((o) => o.name.endsWith(label)); - if (!option) throw new Error(`The project's Status field has no "${label}" option.`); - return [label, option.id]; - }), - ); - return { id: project.id, fieldId: project.field.id, options }; + const option = project.field.options.find((o) => o.name.endsWith(label)) + if (!option) throw new Error(`The project's Status field has no "${label}" option.`) + return [label, option.id] + }) + ) + return {id: project.id, fieldId: project.field.id, options} } const ISSUE_FIELDS = ` @@ -71,49 +71,49 @@ const ISSUE_FIELDS = ` closedByPullRequestsReferences(first: 20, includeClosedPrs: false) { nodes { number headRefName repository { nameWithOwner } } } -`; +` -const CLOSING_REF = /\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?):?\s+(?:([\w.-]+)\/([\w.-]+))?#(\d+)\b/gi; +const CLOSING_REF = /\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?):?\s+(?:([\w.-]+)\/([\w.-]+))?#(\d+)\b/gi // HTML comments are dropped first so the template's own "Closes #123" example doesn't count. function closingRefs(body, owner, repo) { - const text = (body ?? '').replace(//g, ''); - return [...text.matchAll(CLOSING_REF)].map(([, o, r, n]) => ({ owner: o ?? owner, repo: r ?? repo, number: Number(n) })); + const text = (body ?? '').replace(//g, '') + return [...text.matchAll(CLOSING_REF)].map(([, o, r, n]) => ({owner: o ?? owner, repo: r ?? repo, number: Number(n)})) } const linkRefs = (links) => links.nodes.map((i) => { - const [owner, repo] = i.repository.nameWithOwner.split('/'); - return { owner, repo, number: i.number }; - }); + const [owner, repo] = i.repository.nameWithOwner.split('/') + return {owner, repo, number: i.number} + }) // Fetches the issues behind a set of refs in one query. Refs that turn out to be PRs, or that don't resolve, are dropped. async function fetchIssues(github, refs) { - const unique = [...new Map(refs.map((r) => [`${r.owner}/${r.repo}#${r.number}`.toLowerCase(), r])).values()]; - if (!unique.length) return []; + const unique = [...new Map(refs.map((r) => [`${r.owner}/${r.repo}#${r.number}`.toLowerCase(), r])).values()] + if (!unique.length) return [] const query = `query { ${unique .map( (r, i) => `i${i}: repository(owner: ${JSON.stringify(r.owner)}, name: ${JSON.stringify(r.repo)}) { issueOrPullRequest(number: ${r.number}) { ... on Issue { ${ISSUE_FIELDS} } } - }`, + }` ) .join('\n')} - }`; - let data; + }` + let data try { - data = await github.graphql(query); + data = await github.graphql(query) } catch (error) { - if (!error.data) throw error; - data = error.data; // A ref to a missing repo fails only its own alias. + if (!error.data) throw error + data = error.data // A ref to a missing repo fails only its own alias. } return Object.values(data) .map((r) => r?.issueOrPullRequest) - .filter((issue) => issue?.id && issue.state === 'OPEN'); + .filter((issue) => issue?.id && issue.state === 'OPEN') } async function linkedIssues(github, owner, repo, pr) { - const { repository } = await github.graphql( + const {repository} = await github.graphql( `query($owner: String!, $repo: String!, $number: Int!) { repository(owner: $owner, name: $repo) { pullRequest(number: $number) { @@ -121,23 +121,23 @@ async function linkedIssues(github, owner, repo, pr) { } } }`, - { owner, repo, number: pr.number }, - ); - const links = repository.pullRequest.closingIssuesReferences; - return fetchIssues(github, [...linkRefs(links), ...closingRefs(pr.body, owner, repo)]); + {owner, repo, number: pr.number} + ) + const links = repository.pullRequest.closingIssuesReferences + return fetchIssues(github, [...linkRefs(links), ...closingRefs(pr.body, owner, repo)]) } async function moveIssue(github, core, project, issue, target) { - const label = `${issue.repository.nameWithOwner}#${issue.number}`; - const item = issue.projectItems.nodes.find((i) => i.project.id === project.id); + const label = `${issue.repository.nameWithOwner}#${issue.number}` + const item = issue.projectItems.nodes.find((i) => i.project.id === project.id) if (!item) { - core.notice(`${label} isn't on the project; skipping.`); - return; + core.notice(`${label} isn't on the project; skipping.`) + return } - const current = item.status?.name; + const current = item.status?.name if (rank(current) >= ORDER.indexOf(target)) { - core.info(`${label} is already at "${current}"; leaving it.`); - return; + core.info(`${label} is already at "${current}"; leaving it.`) + return } await github.graphql( `mutation($project: ID!, $item: ID!, $field: ID!, $option: String!) { @@ -145,54 +145,54 @@ async function moveIssue(github, core, project, issue, target) { projectId: $project, itemId: $item, fieldId: $field, value: { singleSelectOptionId: $option } }) { projectV2Item { id } } }`, - { project: project.id, item: item.id, field: project.fieldId, option: project.options[target] }, - ); - core.info(`${label}: "${current ?? 'no status'}" -> "${target}".`); + {project: project.id, item: item.id, field: project.fieldId, option: project.options[target]} + ) + core.info(`${label}: "${current ?? 'no status'}" -> "${target}".`) } async function markInReview(github, core, project, owner, repo, pr) { - const target = pr.draft ? 'In progress' : 'In review'; + const target = pr.draft ? 'In progress' : 'In review' for (const issue of await linkedIssues(github, owner, repo, pr)) { - await moveIssue(github, core, project, issue, target); + await moveIssue(github, core, project, issue, target) } } // An issue split across several PRs, possibly in different repos, waits for the last of them. Release PRs link every // issue in the release, so they don't count. Only sidebar links are visible here, not "Closes" lines in other PRs. function stillOpenElsewhere(core, issue) { - const open = issue.closedByPullRequestsReferences.nodes.filter((p) => !isReleaseBranch(p.headRefName)); + const open = issue.closedByPullRequestsReferences.nodes.filter((p) => !isReleaseBranch(p.headRefName)) if (open.length) { - const refs = open.map((p) => `${p.repository.nameWithOwner}#${p.number}`).join(', '); - core.info(`${issue.repository.nameWithOwner}#${issue.number} is still linked from open ${refs}; leaving it.`); + const refs = open.map((p) => `${p.repository.nameWithOwner}#${p.number}`).join(', ') + core.info(`${issue.repository.nameWithOwner}#${issue.number} is still linked from open ${refs}; leaving it.`) } - return open.length > 0; + return open.length > 0 } async function markPendingRelease(github, core, project, owner, repo, pr) { // By the time "closed" fires, the merged PR no longer counts as open. for (const issue of await linkedIssues(github, owner, repo, pr)) { - if (!stillOpenElsewhere(core, issue)) await moveIssue(github, core, project, issue, 'Pending release'); + if (!stillOpenElsewhere(core, issue)) await moveIssue(github, core, project, issue, 'Pending release') } } // Long-lived release branches (mavedb-dev) keep their history, so only PRs merged since the branch last shipped count. async function lastReleasedAt(github, owner, repo, pr) { - const { search } = await github.graphql( + const {search} = await github.graphql( `query($query: String!) { search(type: ISSUE, query: $query, first: 1) { nodes { ... on PullRequest { mergedAt } } } }`, - { query: `repo:${owner}/${repo} is:pr is:merged head:${pr.head.ref} base:${pr.base.ref} sort:created-desc` }, - ); - return search.nodes[0]?.mergedAt ?? null; + {query: `repo:${owner}/${repo} is:pr is:merged head:${pr.head.ref} base:${pr.base.ref} sort:created-desc`} + ) + return search.nodes[0]?.mergedAt ?? null } async function updateReleaseDescription(github, core, owner, repo, pr) { - const since = await lastReleasedAt(github, owner, repo, pr); - const query = `repo:${owner}/${repo} is:pr is:merged base:${pr.head.ref}` + (since ? ` merged:>${since}` : ''); - const refs = []; - let cursor = null; + const since = await lastReleasedAt(github, owner, repo, pr) + const query = `repo:${owner}/${repo} is:pr is:merged base:${pr.head.ref}` + (since ? ` merged:>${since}` : '') + const refs = [] + let cursor = null do { - const { search } = await github.graphql( + const {search} = await github.graphql( `query($query: String!, $cursor: String) { search(type: ISSUE, query: $query, first: 50, after: $cursor) { pageInfo { hasNextPage endCursor } @@ -204,22 +204,22 @@ async function updateReleaseDescription(github, core, owner, repo, pr) { } } }`, - { query, cursor }, - ); + {query, cursor} + ) for (const merged of search.nodes) { - refs.push(...linkRefs(merged.closingIssuesReferences), ...closingRefs(merged.body, owner, repo)); + refs.push(...linkRefs(merged.closingIssuesReferences), ...closingRefs(merged.body, owner, repo)) } - cursor = search.pageInfo.hasNextPage ? search.pageInfo.endCursor : null; - } while (cursor); + cursor = search.pageInfo.hasNextPage ? search.pageInfo.endCursor : null + } while (cursor) - const issues = new Map(); + const issues = new Map() for (const issue of await fetchIssues(github, refs)) { - if (stillOpenElsewhere(core, issue)) continue; - const sameRepo = issue.repository.nameWithOwner === `${owner}/${repo}`; - issues.set(sameRepo ? `#${issue.number}` : `${issue.repository.nameWithOwner}#${issue.number}`, issue); + if (stillOpenElsewhere(core, issue)) continue + const sameRepo = issue.repository.nameWithOwner === `${owner}/${repo}` + issues.set(sameRepo ? `#${issue.number}` : `${issue.repository.nameWithOwner}#${issue.number}`, issue) } - const listed = [...issues.keys()].sort((a, b) => a.localeCompare(b, 'en', { numeric: true })); + const listed = [...issues.keys()].sort((a, b) => a.localeCompare(b, 'en', {numeric: true})) const section = [ RELEASE_SECTION_START, '### Issues closed by this release', @@ -227,56 +227,56 @@ async function updateReleaseDescription(github, core, owner, repo, pr) { `_Generated from the issues linked to PRs merged into \`${pr.head.ref}\` for this release, and rewritten on every push. Issues still linked from an open PR are left out. To change the list, change those PRs' linked issues or Closes lines._`, '', ...(listed.length ? listed.map((ref) => `- Closes ${ref}`) : ['None.']), - RELEASE_SECTION_END, - ].join('\n'); + RELEASE_SECTION_END + ].join('\n') - const body = pr.body ?? ''; - const start = body.indexOf(RELEASE_SECTION_START); - const end = body.indexOf(RELEASE_SECTION_END); + const body = pr.body ?? '' + const start = body.indexOf(RELEASE_SECTION_START) + const end = body.indexOf(RELEASE_SECTION_END) const updated = start !== -1 && end > start ? body.slice(0, start) + section + body.slice(end + RELEASE_SECTION_END.length) - : `${body.trimEnd()}\n\n${section}\n`; + : `${body.trimEnd()}\n\n${section}\n` if (updated === body) { - core.info('Release description is up to date.'); - return; + core.info('Release description is up to date.') + return } - await github.rest.pulls.update({ owner, repo, pull_number: pr.number, body: updated }); - core.info(`Release description lists ${listed.length} issue(s).`); + await github.rest.pulls.update({owner, repo, pull_number: pr.number, body: updated}) + core.info(`Release description lists ${listed.length} issue(s).`) } -module.exports = async ({ github, context, core }) => { - const { owner, repo } = context.repo; - const project = await loadProject(github); +module.exports = async ({github, context, core}) => { + const {owner, repo} = context.repo + const project = await loadProject(github) // Linking an issue from the PR's Development sidebar fires no event, so a periodic sweep catches those. if (context.eventName === 'schedule' || context.eventName === 'workflow_dispatch') { - const prs = await github.paginate(github.rest.pulls.list, { owner, repo, state: 'open', per_page: 100 }); + const prs = await github.paginate(github.rest.pulls.list, {owner, repo, state: 'open', per_page: 100}) for (const pr of prs.filter((p) => !isReleaseBranch(p.head.ref))) { - await markInReview(github, core, project, owner, repo, pr); + await markInReview(github, core, project, owner, repo, pr) } - return; + return } - const pr = context.payload.pull_request; - const action = context.payload.action; + const pr = context.payload.pull_request + const action = context.payload.action if (isReleaseBranch(pr.head.ref) && pr.base.ref === context.payload.repository.default_branch) { if (['opened', 'reopened', 'synchronize'].includes(action)) { - await updateReleaseDescription(github, core, owner, repo, pr); + await updateReleaseDescription(github, core, owner, repo, pr) } - return; + return } if (action === 'closed') { if (pr.merged && isReleaseBranch(pr.base.ref)) { - await markPendingRelease(github, core, project, owner, repo, pr); + await markPendingRelease(github, core, project, owner, repo, pr) } - return; + return } // Hotfixes merged straight into the default branch close their issues natively. if (pr.base.ref !== context.payload.repository.default_branch) { - await markInReview(github, core, project, owner, repo, pr); + await markInReview(github, core, project, owner, repo, pr) } -}; +} From a3b7d48dda6008d138ad9a2a87f42aca1695abb3 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 20 Oct 2026 10:24:54 -0700 Subject: [PATCH 12/13] fix(docker): bake the UCSC liftover chain files into the image agct downloads the hg19<->hg38 chain files on first use, through certifi. certifi 2026.10.18 dropped the emSign roots that UCSC's certificate chain ends at, so every mapping job in the rebuilt 2026.3.1 image failed on liftover with CERTIFICATE_VERIFY_FAILED. Download both files at build time with curl, which uses the system trust store, verify them against UCSC's md5s, and write them under WAGS_TAILS_DIR with the names agct's wags-tails lookup expects. The build runs a test that loads both converters with agct's downloader patched to raise, so a change in that naming fails the build rather than production. Pinning certifi was rejected: it only defers the next root change. --- Dockerfile | 17 +++++++++++++++++ tests/test_liftover_chainfiles.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 tests/test_liftover_chainfiles.py diff --git a/Dockerfile b/Dockerfile index 5f92714..f7c9634 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,6 +46,22 @@ RUN wget -O - https://ftp.ncbi.nlm.nih.gov/genomes/refseq/vertebrate_mammalian/H RUN samtools faidx GCF_000001405.25_GRCh37.p13_genomic.fna.gz RUN samtools faidx GCF_000001405.39_GRCh38.p13_genomic.fna.gz +# Bake in the liftover chain files agct would otherwise download on first use. +# That download goes through certifi, which can drop roots UCSC's TLS chain +# relies on (VariantEffect/dcd_mapping2#117); curl uses the system trust store. +# The paths are what agct's wags-tails lookup expects for unversioned files. +# Checksums from each directory's md5sum.txt. +ENV WAGS_TAILS_DIR=/home/.local/share/wags_tails +RUN mkdir -p $WAGS_TAILS_DIR/ucsc-chainfile && cd $WAGS_TAILS_DIR/ucsc-chainfile && \ + curl -fLO https://hgdownload.soe.ucsc.edu/goldenPath/hg19/liftOver/hg19ToHg38.over.chain.gz && \ + curl -fLO https://hgdownload.soe.ucsc.edu/goldenPath/hg38/liftOver/hg38ToHg19.over.chain.gz && \ + printf '%s\n' \ + "35887f73fe5e2231656504d1f6430900 hg19ToHg38.over.chain.gz" \ + "ff3031d93792f4cbb86af44055efd903 hg38ToHg19.over.chain.gz" | md5sum -c - && \ + gunzip -c hg19ToHg38.over.chain.gz > chainfile_hg19_to_hg38_.chain && \ + gunzip -c hg38ToHg19.over.chain.gz > chainfile_hg38_to_hg19_.chain && \ + rm hg19ToHg38.over.chain.gz hg38ToHg19.over.chain.gz + RUN mkdir /usr/src/app WORKDIR /usr/src/app COPY . . @@ -53,6 +69,7 @@ COPY . . RUN pip install -e '.[dev,tests]' # use polars-lts-cpu to avoid issues with x86 emulation on arm machine RUN pip install -U polars-lts-cpu +RUN python -m pytest -p no:cacheprovider --no-cov tests/test_liftover_chainfiles.py # install gene normalizer with pg dependencies. TODO: can the pg dependencies be specified in pyproject.toml? #RUN pip install 'gene-normalizer[pg]' diff --git a/tests/test_liftover_chainfiles.py b/tests/test_liftover_chainfiles.py new file mode 100644 index 0000000..2d1a65c --- /dev/null +++ b/tests/test_liftover_chainfiles.py @@ -0,0 +1,30 @@ +"""Check that the mapper Docker image ships the UCSC liftover chain files. + +This is an image check, not a unit test: the Dockerfile runs it during +``docker build``, and it skips everywhere else. Outside the image agct is free +to download the chain files on demand. Inside it, that download goes through +certifi, which fails whenever certifi drops a root UCSC's TLS chain relies on +(VariantEffect/dcd_mapping2#117), so the Dockerfile bakes the files in and this +check fails the build if agct can't find them. +""" + +import os +from unittest import mock + +import pytest +from agct import Converter + +# Only the Dockerfile sets DCD_MAPPING_RESOURCES_DIR, so it marks the image. +pytestmark = pytest.mark.skipif( + "DCD_MAPPING_RESOURCES_DIR" not in os.environ, + reason="image check; runs during docker build", +) + + +@pytest.mark.parametrize(("from_db", "to_db"), [("hg19", "hg38"), ("hg38", "hg19")]) +def test_converter_loads_baked_chainfile(from_db: str, to_db: str): + with mock.patch( + "agct.converter.download_http", + side_effect=AssertionError("agct tried to download a chain file"), + ): + Converter(from_db, to_db) From 35d0449e5181e1fc8d106c7890bbde872cddafff Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 20 Oct 2026 11:46:35 -0700 Subject: [PATCH 13/13] chore: bump version to 2026.3.2 --- schema.json | 2 +- src/dcd_mapping/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/schema.json b/schema.json index ee11565..33b5bfb 100644 --- a/schema.json +++ b/schema.json @@ -752,7 +752,7 @@ "title": "Tool Parameters" }, "tool_version": { - "default": "2026.3.1", + "default": "2026.3.2", "title": "Tool Version", "type": "string" }, diff --git a/src/dcd_mapping/version.py b/src/dcd_mapping/version.py index f94f38e..f96c878 100644 --- a/src/dcd_mapping/version.py +++ b/src/dcd_mapping/version.py @@ -1,3 +1,3 @@ """Provide dcd mapping version""" -dcd_mapping_version = "2026.3.1" +dcd_mapping_version = "2026.3.2"