diff --git a/.github/workflows/seidroid-review.yml b/.github/workflows/seidroid-review.yml new file mode 100644 index 0000000..0921b2f --- /dev/null +++ b/.github/workflows/seidroid-review.yml @@ -0,0 +1,3596 @@ +name: seidroid review +run-name: UCI / seidroid review / ${{ (github.event.issue.number || github.event.pull_request.number) && format('#{0}', github.event.issue.number || github.event.pull_request.number) || github.ref_name }} +# Reusable, comment-triggered agentic PR review, driven by the sei-agent-driver +# binary from sei-protocol/sei-internal-skills. That module is where the reviewer's +# logic and its prompt live; this file is the GitHub wiring around it, and the two +# are versioned separately -- `uses:` pins this file, and this file pins the binary +# through the `driver-version` default. A thin caller in the reviewed repo wires the +# triggers and calls this with `uses:`. Flow: comment `@seidroid review` on a pull +# request -> guard gate -> install and run the driver over one managed omnigent +# session -> post the verdict as a new comment, the findings it can place, and its +# check runs. +# +# TWO PATHS, one review. An AUTOMATIC review runs on `pull_request` when the caller +# wires that trigger and passes `mode: review`. A MANUAL one runs when a person writes +# `@seidroid review` on the pull request: on the conversation, in a diff thread, or in +# a review body. Both spend model quota and hold a sandbox, so both are gated -- see +# the guard below. Both paths refuse a fork-originated pull request. The automatic path +# reviews a pull request once. It reviews a later push only where the caller sets +# `re-review-on-push`, or where a block of its own stands. It also refuses a draft and +# honours the skip-review label. The manual path additionally checks who is asking. +# +# FOUR EVENTS reach this file, and the guard refuses any other by name: `pull_request` +# for the automatic path, and `issue_comment`, `pull_request_review_comment` and +# `pull_request_review` for the manual one. A caller wires the ones it wants. +# +# This file is the automation of record. It REPLACES `ai-review.yml` rather than +# running beside it; a repository that wires the automatic path here should retire +# that caller in the same change, or every pull request is reviewed twice. +# +# The driver is installed with `go install` at `driver-version`, which defaults to the +# release this file drives, so every caller that omits the input runs one driver and one +# conclusion vocabulary. The install step checks the binary it got against that contract +# and fails there, before the review spends quota or takes a sandbox. The session is +# keyed on the pull request and OUTLIVES the run, which is why `mode: close` exists: +# it is the only thing that reclaims a sandbox. +# +# What changed versus the Python driver's workflow, and why: +# - No runtime dependency install of the reviewer itself. The binary comes from +# `go install` against the module's committed go.sum, so no package-manager +# step runs unpinned code on a runner that is about to hold a live credential. +# - No curl mint step, and no bearer token ever appears in a step output. +# The driver mints its own session bearer in-process from +# OMNIGENT_MACHINE_CLIENT_ID + OMNIGENT_MACHINE_CLIENT_SECRET +# (client_credentials). The secret is wired as JOB-level env on the +# `review` job below, never as step-level env on a `uses:` step -- +# step-level env on a `uses:` step does not reach a composite action, and +# that exact defect broke every real run of the original Python workflow. +# - Every GitHub-supplied value (repo, PR number, comment id) is routed +# through env vars and read as "$VAR" in shell -- never interpolated as +# ${{ }} directly into a shell or jq command line, so a hostile PR title, +# branch name or comment body cannot inject shell/jq syntax. +# - The verdict file's ABSENCE means "no verdict to post", and no step writes a +# placeholder into it. A review that reaches none still publishes, because a +# pull request with no artifact on it reads as a review that never ran: the +# driver writes a failing check run to the same path it writes the deciding +# one, under the title `no verdict`, and a step of its own reports the run in +# a comment. Exactly one comment +# per review run either way. Neither takes a position on the pull request, and +# neither clears an earlier one -- a review that produced nothing has no +# position to state. +# - The outcome is surfaced even when the driver exits non-zero. The +# surfacing steps gate on whether a verdict was produced rather than on +# the exit code, so a review that reached one still posts it whatever else +# went wrong; `!cancelled()` rather than `success()` keeps that true while +# still skipping a run superseded by a newer trigger. Every surfacing step +# skips such a run. One step does not, and it is the last in the job: it +# withdraws the reactions a cancelled run left on the trigger comment, and +# it reads no verdict, so it can state no outcome. +# - allow-tools is a workflow input, defaulting to Bash,Read because the +# prompt's first step is a shell command and a declined prompt therefore +# produces a turn that reports it could not read the diff. Its access +# control is the trigger gate above, not the allowlist -- see +# https://github.com/sei-protocol/sei-internal-skills/blob/main/sei-agent-driver/cmd/sei-agent-driver/README.md +on: + workflow_call: + inputs: + mode: + description: >- + "review" (default) drives a review turn on the pull request's session, + keeping the conversation so the next invocation can say what changed. + "close" deletes that session instead — the end of the unit of work, and + the only thing that reclaims its sandbox. + required: false + type: string + default: 'review' + driver-version: + description: >- + sei-agent-driver version to install. It names a revision of + sei-protocol/sei-internal-skills, NOT of this repository: the driver is a + nested module there and moving this file did not move it. Passing a uci sha + here fails the install with an unknown revision. + + LEAVE IT UNSET. The default is the release this file drives, and it is what + makes one conclusion mean one thing across callers: a gate written against + `success` in one repository then reads the same in the next. Set it only to + run AHEAD of the default, on a driver change that has shipped and this file + has not yet taken. + + Forward only. The install step below refuses a driver older than the default, + because the conclusion a review reaches for a given set of findings is + specific to the driver that reached it. Pin backward in this repository, + under review, rather than in a caller. + + The module is nested, so the repository carries path-prefixed tags + (sei-agent-driver/vX.Y.Z) while `go install` takes the bare version. Pass + `v0.18.0`; `sei-agent-driver/v0.18.0` is refused as a disallowed version + string. A commit sha resolves to a pseudo-version. + + Verify a pin from an EMPTY module cache: a warm one is a false green, because + it resolves a pin the proxy may never have served. + required: false + type: string + default: 'v0.18.0' + trigger-phrase: + description: >- + The mention a person types to ask for a review. The command is that phrase + followed by `review`, alone on a line, optionally followed by `close`. + + The `@` is optional. `@seidroid review` and `seidroid review` both ask for + the same thing, and a person who types the phrase without the mention still + means it. Two things make the wider form safe. The pattern is anchored to a + whole line, so a comment that discusses the command has other words on the + line and does not match. And ai-assistant.yml claims a body only when it + carries the `@` form, so the bare form reaches this workflow alone and no + second tool answers it. + + It reaches a `grep -E` pattern, so its SHAPE is constrained rather than + escaped: a leading `@` and then letters, digits, `_` and `-` only. None of + those is an ERE metacharacter, so the pattern needs no escaping and a phrase + carrying `.` or `|` cannot widen the match. A phrase outside that shape is + refused with a warning and the default is used, which is what + guidelines-file below does with a name it cannot trust. + + Matched case-sensitively. `@Seidroid review` starts no review. + required: false + type: string + default: '@seidroid' + allowed-team: + description: >- + org/team-slug whose active members may ask for a review by comment. Empty + denies every commenter, which is what ai-review.yml does with the same + input under the same default. + + It gates the comment path alone. An automatic pull_request review has no + commanding actor, so the guard skips this check there and reviews a pull + request opened by anyone. + + The membership read takes "Members" organization permissions, which a + GITHUB_TOKEN cannot carry. Only the App identity can answer it, so a caller + that configures no App is refused on the comment path. Pass SEIDROID_APP_ID + and SEIDROID_APP_PRIVATE_KEY to use that path. Setting this input empty is + not the way out: empty denies. + + Two things still work without the App. An automatic pull_request review + never reaches this check, and neither does `@seidroid review close`. + required: false + type: string + default: 'sei-protocol/sei-core' + allowed-bots: + description: >- + JSON array of exact bot logins that may ask for a review by comment. `[]`, + the default, denies every bot. + + A login is the discriminating control here, and the actor's type is not: a + bot with write access to the calling repository carries MEMBER or + COLLABORATOR like anyone else, so refusing every bot is the only safe reading + of a type. This input names the ones a repository trusts. dependabot and + renovate are the cases it exists for. + + Matched exactly and case-insensitively, twice: once in the guard's job + condition, so an unlisted bot starts no runner, and again in `Admit the + request`. A lookalike login does not pass either. ai-review.yml checks the + same input the same way. + + A listed bot skips the allowed-team check, because a bot is not a team + member. It is held to every other rule: the fork check, the skip label and + the command grammar all apply. + + Malformed JSON is a caller error: `fromJSON` refuses it rather than denying + quietly. `Admit the request` denies on it, so neither reading admits a bot + off a value nobody could parse. + required: false + type: string + default: '[]' + approve-on-success: + description: >- + Approve the pull request when the review concludes clean. Off by + default: clearing a human review requirement is the repository's policy + decision, not this file's, so a clean run stays silent until the + repository has decided it may approve. A blocking conclusion requests + changes either way. + required: false + type: boolean + default: false + publish-ai-review-check: + description: >- + Publish the review's check run a second time, under the name `AI Review`. + On by default. A branch-protection rule matches a check by its name, the + rules on a calling repository are org-level, and no token here reads them. + A repository that retires `ai-review.yml` under a rule that names + `AI Review` waits forever on a check nothing publishes. That rule reads as + pending, not failed. + + Set this false where the repository still runs `ai-review.yml` AND a rule + names `AI Review`. Both tools then publish that name, GitHub lists only the + check run that completed last, and this copy can hide the incumbent's + verdict. This switch does not remove that race. It gives the caller the + choice to accept it or to stand the copy down. + required: false + type: boolean + default: true + skip-review-label: + description: >- + A label on the reviewed pull request that stops the review. Empty + disables the check. + required: false + type: string + default: 'ai: skip-review' + nitpick-label: + description: >- + A label on the reviewed pull request that puts every nit-grade finding on + the line it is about, as a comment thread of its own. Empty disables the + check. + + Without the label a nit is still reported. The driver sends it to the + non-blocking bucket instead, where the verdict comment and the check run + summary both carry it as prose and no thread opens on the code. So this + chooses where a nit lands, not whether the review makes one. + + It also decides what a nit may close. A thread this reviewer left before + is superseded by the comment that restates it, and a nit that opens no + comment supersedes nothing. + + The name matches ai-review.yml's, so a repository running both tools adds + one label rather than two. + + Adding it starts no review. This workflow reviews a pull request once, and + a relabel earns no second one, so label the pull request and then comment + `@seidroid review`. + required: false + type: string + default: 'ai: nitpick' + re-review-on-push: + description: >- + Review again on every push to a pull request this workflow has already + reviewed. Off by default: one automatic review per pull request, and a + person asks for the next one by name. + + Off does not mean exactly one. A review that finds something blocking + leaves a CHANGES_REQUESTED, only a later review withdraws it, so this + workflow reviews every push after that one until a review comes back + clean. Size the saving on that: it covers the pull requests that pass + first time, and a pull request pushed to ten times under a block costs + ten reviews with this off. + + Each review holds a managed sandbox for up to `timeout-minutes` and + spends model quota, so a pull request pushed to ten times costs ten of + each. It also posts ten verdicts. + + A push is the only event either rule covers. Reopening a pull request + or relabelling it changes no code, so neither earns a second review, + whatever this says and whether or not a block stands. + + It governs the automatic path alone. An explicit `@seidroid review` + comment is answered however many reviews have already run. + + Two consequences of leaving it off. The refusal is a notice on the + workflow run: an author who pushes and sees no new review reads nothing + on the pull request that says why, or that a comment asks for one. And + the `review` check run is published against the commit a review read, + so every head after the first carries none -- a repository that + REQUIRES that check would sit pending rather than fail. Turn this on + there, or require the workflow job instead. + required: false + type: boolean + default: false + guidelines-file: + description: >- + The repository's own review standards, read from the base branch and + outranking the driver's checklist. Empty reads REVIEW.md, which is what + ai-review reads and what these repositories keep. Checked before it + reaches a command, so a name carrying a shell metacharacter, an absolute + path or a parent reference falls back to the default. + required: false + type: string + default: '' + extra-instructions: + description: >- + Guidance this repository adds to every review, alongside the standards + above. The one input the review does not treat as data: it comes from + this workflow rather than from the pull request. + required: false + type: string + default: '' + timeout-minutes: + description: >- + Cap on the review job. Keep it comfortably above the driver's own + SEIDROID_RUN_DEADLINE_S: the driver reports a timeout and the reason it + hit one, where the runner killing the job leaves an annotation saying + only that the job was cancelled. + required: false + type: number + default: 45 + runs-on: + description: >- + Runner label for the review job. Defaults to a GitHub-hosted runner, + which pairs with the https base URL below: the two have to describe one + topology. An in-cluster group instead would reach an internet-facing + NLB from inside its own VPC, which does not hairpin, and would also + queue forever on any repository not in that runner group. + required: false + type: string + default: 'ubuntu-latest' + omnigent-base-url: + description: >- + omnigent base URL. The https ingress by default. It cannot be the + in-cluster ClusterIP Service, which is plain http on port 80: a + credentialed client refuses to be built against it, and the token mint + refuses to send the client secret over it. + + This default is a development deployment. Platform changes land there + first, so a change on it can make a review succeed and post nothing. + Brandon Chatham accepts that exposure: no production deployment carries + the seidroid bundle and a provisioned machine client today. Pass a + production URL from a merge-path caller as soon as one exists. + required: false + type: string + default: 'https://seigent.dev.platform.sei.io' + agent-id: + description: >- + The agent NAME to resolve on the server, as SEIDROID_AGENT_ID. Deployment + specific: the driver compiles a default, and a server that calls its agent + something else makes that default unresolvable -- a live run against the dev + deployment failed with `no agent named "seidroid" on this server`, which + nothing in the caller could correct because this input did not exist. + + Empty leaves the driver's own default in place, so a deployment that matches + it needs nothing here. There is no lookup by alias: the name must match what + the server's agent bundle is called, exactly. + + A mismatch does not present as a configuration error. The name is a join key + with the omnigent.ai/agent label the server stamps on the runner Pod and the + admission policy that mounts the git credential from it, so a runner that + does not match attracts no credential and the review reports that it could + not read the repository. + required: false + type: string + default: '' + + machine-client-id: + description: "OMNIGENT_MACHINE_CLIENT_ID for the in-process client_credentials mint. Mirrors the server's own OMNIGENT_MACHINE_CLIENT_ID. Not secret on its own -- the id alone cannot mint a token." + required: false + type: string + default: 'seidroid' + scouts: + description: >- + Independent readings to gather before the review, as `name=agent`, + comma-separated. Each scout reads the same pull request in its own + session on its own agent bundle, seeing neither the review nor another + scout; the review then verifies their claims against the diff and merges + what holds. A scout naming the review's own agent is refused, as are two + scouts sharing one — neither would be a second opinion. Empty runs the + review alone. Passed to the driver as SEIDROID_SCOUTS. + + Two models read every pull request by default. `xreview-scout-codex` is + the one scout bundle sei-internal-skills carries today, and PLT-1168 + tracks a Cursor bundle. A scout that fails costs the review that reading + and nothing else. The driver turns each failure into a note, hands it to + the review, and the review reports with fewer readers. That note is the + only signal, so a bundle absent from a deployment fails quietly on every + pull request. + + Pass the SAME value on the CLOSE job. Scouts hold sessions of their own, + and close derives which to delete from this value, keyed on the scout + NAME. A caller that omits it on both jobs is safe, because close reads + this same default. A caller that sets it on the review job and omits it + on the close job is not. Close then deletes the default name, and the + configured scout keeps its sandbox running. + + Set `scouts: ''` to review on one model. Do that on a deployment that + does not carry the bundle above, and on a caller that wires no + `mode: close` job. A scout holds a sandbox of its own, and close is the + only thing that reclaims one. A caller with no close job leaks one + sandbox per pull request. + required: false + type: string + default: 'codex=xreview-scout-codex' + claude-model: + description: >- + Model to answer the review on, substituting for the one the agent's spec + names. The default pins Opus 5 with the 1M context window. Set it empty + to leave the spec's own. Passed to the driver as SEIDROID_MODEL. + + The pin exists because the runner otherwise launches on the Claude CLI's + own default, which moves on a base image rebuild with no change here. + + The [1m] suffix is load-bearing, not decoration: it selects the 1M + context build. The unsuffixed id runs the standard window, which a + review of a long diff compacts into sooner and reads less of. + + The server forwards the value as-is and enumerates nothing, so an + unrecognised name is not rejected here or at configuration time -- it + fails at turn start, and the review is the thing that does not happen. + + It applies to the review's own agent only. A scout runs on another agent, + so another harness and another provider, and it keeps its spec's model. + required: false + type: string + default: 'claude-opus-5[1m]' + allow-tools: + description: >- + Comma-separated tool_name values to accept. This deployment does stamp + tool_name (measured), so this allowlist discriminates per tool. Passed + to the driver as SEIDROID_ALLOW_TOOLS. + + Defaults to `Bash,Read` because the review cannot happen without them. + The prompt's first step is a `gh pr diff` command, so an empty allowlist + does not yield a shallower review -- it yields a turn that reports it + could not read the diff. The shell runs inside the agent's own sandbox + against its own gh credentials, and the access control on it is the + trigger: only a sei-protocol developer can write the line that starts a + review. + + `Read` is measured, not assumed. A read inside the agent's working + directory raises no prompt, so the staged diff needs no grant; a read + outside one does, and a recorded run had exactly that refused and spent + three extra tool calls recovering. The diff now stages into the working + directory, so this grant is the belt to that braces. + + Brandon Chatham accepts the unrestricted shell. The review builds and + tests the tree where that is straightforward. A reviewer that cannot + compile can only guess at a finding that needs one. ai-review scopes its + own model to `Read,Bash(gh pr diff:*),Bash(gh pr view:*)` and gives up + that capability. That acceptance covers code from inside the + organisation. + + Fork code sits outside that acceptance, and the guard refuses it. An + explicit `@seidroid review` arrives as a comment or review event in the + base repository, which carries the secrets. Those paths reach a fork's + code unless something stops them. The guard's fork check is what stops + it, on every one of them and on the automatic path. This shell therefore + runs only over code from inside the organisation. Weigh that before you + widen or narrow this list. + required: false + type: string + default: 'Bash,Read' + secrets: + OMNIGENT_MACHINE_CLIENT_SECRET: + description: "omnigent machine-client secret, exchanged in-process for a session bearer. Mirrors the server's OMNIGENT_MACHINE_CLIENT_SECRET_HASH -- the server stores only a digest of this value. The one secret an operator must configure to use this workflow." + required: true + SEIDROID_APP_ID: + description: >- + seidroid GitHub App id. Optional for an automatic review, REQUIRED to ask + for one by comment. + + Without it the review posts as the workflow's own identity, which is correct + but reads as github-actions rather than the bot. And the guard's team check + cannot read organisation membership, so it refuses every `@seidroid review` + comment and says why in the run log. An automatic pull_request review and + `@seidroid review close` still run. + + Changing it changes who withdraws a block. A protected branch that restricts + who may dismiss a review takes the dismissal only from a repository admin or + an actor on that list, whoever recorded the review. So put whichever identity + this workflow runs under on the list, and dismiss any standing block by hand + when you change the credential -- otherwise a later clean run fails on a + dismissal the API refuses. + required: false + SEIDROID_APP_PRIVATE_KEY: + description: "seidroid GitHub App private key, exchanged for an installation token scoped to the calling repository. Never written to an output; the action masks it." + required: false + +permissions: {} + +# The marker that opens every verdict this workflow leaves, in one place. Three +# steps use it: the guard finds a review that already ran, the position step opens +# the review it records with it and matches on it to withdraw an earlier block, and +# the verdict comment opens with it. A workflow-level env reaches every step, so +# those uses cannot drift apart. A step-level key of this name shadows this one; +# there is none. +# +# It scopes all of that to this tool's own work. ai-review posts under the same bot +# identity and marks its reviews differently; its position is not this one's to +# change. +# +# Changing the value strands every blocking review posted under the old one: the +# withdrawal matches on startswith, so a later clean run approves and retracts +# nothing, the pull request stays red for a finding that is gone, and only a human +# can clear it. It also makes every reviewed pull request read as unreviewed to the +# guard. Before changing it, confirm no open pull request carries a +# CHANGES_REQUESTED review or a verdict comment whose body starts with the old +# value. +# +# The no-verdict notice and the inline findings carry their own markers, and +# neither starts with this one. A run that reached no verdict does not read as +# reviewed. +env: + VERDICT_MARKER: "" + # What stamps an inline finding as this tool's, read by every step that has to + # recognise one: the step that writes it, the step that reads the history back, and + # the step that closes a thread. One value, because a drift between them fails in the + # worst direction and with no error -- placement keeps stamping the old marker while + # the other two stop matching it, and every finding duplicates again on a green run. + # + # Beside VERDICT_MARKER rather than in the review job, though only that job reads it. + # Two markers in two scopes is how a third one ends up defined somewhere else again. + # + # Not named MARKER. One step below still carries a step-level MARKER of its own, for + # the no-verdict notice, and a step-level key shadows a workflow-level one silently -- + # so a bare MARKER here would work today and become a trap for whichever step later + # forgot to set its own. + FINDING_MARKER: "" + +jobs: + guard: + # Cheap allowlist + command parse on a hosted runner, no secrets, before any + # in-cluster work spins up and before any credential is minted. The + # author_association gate is the trust boundary: only OWNER/MEMBER/ + # COLLABORATOR can fire it -- an untrusted PR author cannot. + # + # Bots are excluded separately, because association does not exclude them: a + # bot with write access to the calling repository carries MEMBER or + # COLLABORATOR like anyone else. Without this, a bot that quotes the command + # -- one of ours echoing an earlier comment, say -- starts a real review, and + # a bot that echoes its own trigger does so repeatedly. + name: Guard + runs-on: ubuntu-latest + # A step condition cannot read `secrets`, so its presence is tested here and + # read back as `env` below — the same shape the review job uses. + # + # Both halves of the credential, because the mint below needs both and carries no + # continue-on-error. An id with no private key would run that step, fail it, and + # take the job with it -- where the App is optional and the answer is to fall back. + env: + HAS_REVIEWER_IDENTITY: >- + ${{ secrets.SEIDROID_APP_ID != '' && secrets.SEIDROID_APP_PRIVATE_KEY != '' }} + # Set at all because the account default is six hours. The guard only reads + # API state, so a minute is generous. + timeout-minutes: 5 + # Read-only, and only what the guard's checks read. Three reads share it. The + # fork check reads the pull request. The once-per-PR gate reads a review, to find + # a standing block, and a comment, to find a verdict. + # + # That second gate read goes to the ISSUE comments endpoint. GitHub documents it + # as taking either permission, so pull-requests alone serves it. issues: read is + # granted beside it so the read does not rest on that alias. A reaction is the + # stricter case and takes issues alone; the review job says so where it needs it. + # + # Every reader prefers the App identity and falls back to this, so a caller that + # configures no App still gets all four. A refused read costs them differently, + # and both costs are deliberate. The once-per-PR gate fails open: a review runs on + # every push, and only the run log says why. The fork check and the skip-review + # label fail closed: the guard refuses the review rather than run it over code it + # cannot place, or against an author who asked for none. + # + # A caller must grant this workflow at least these two, because a reusable + # workflow may only downgrade what its caller granted. pull-requests: read is + # load-bearing rather than nice to have: the label check reads the pull request + # on every review, including an automatic one on a caller that configures no + # App, and it refuses the review when that read fails. GitHub's default token + # grants contents, packages and metadata only, so a caller that takes the + # default reviews nothing on a private repository. + permissions: + pull-requests: read # the pull request the fork and label checks read, and the gate's reviews + issues: read # the comments the gate reads to find a verdict + # Runs for an automatic pull_request review, for any comment-triggered dispatch, + # review or close, and for an event this workflow does not handle. For a comment + # it decides whether the requester may command this workflow at all; for an + # automatic review it decides whether the pull request is in a state worth + # spending a sandbox on. Routing is the caller's: it reads the body and passes the + # mode. A close arriving as a pull_request event skips the guard, since GitHub's + # own event is the authority. + # + # THREE COMMENT EVENTS carry the command. A comment on the conversation arrives as + # issue_comment, a comment in a diff thread as pull_request_review_comment, and a + # review body as pull_request_review. Each names the requester under a different + # payload key, so every read below takes the comment key or the review key, + # whichever the event populated. On pull_request both are empty and this branch + # does not apply. + # + # The two diff-side events are held to their creating action. A review may also + # arrive `edited` or `dismissed`, and a dismissal replays the body of the review it + # dismisses -- so a caller that wires those types would spend a sandbox re-running + # a review every time somebody dismissed one. issue_comment is left as it stands. + # + # A REQUESTER passes on one of two grounds. A person passes on + # author_association, which admits a collaborator on the repository the request + # was made in and no untrusted pull request author. A bot passes only by exact + # login in allowed-bots, because association does not discriminate a bot: one with + # write access to the calling repository carries MEMBER or COLLABORATOR like + # anyone else. `Admit the request` reads both grounds again. + # + # `allowed-bots || '[]'` because a workflow_call default applies only to an input + # the caller OMITS. A caller passing `allowed-bots: ${{ vars.SOMETHING }}` with + # that variable unset passes the empty string, and fromJSON('') is not `[]` -- it + # fails the expression and takes the run with it. Empty therefore reads as the + # documented default here, which denies every bot. A non-empty value that is not + # JSON still fails, and loudly: that is a caller wiring error, not an omission. + # + # The last clause admits an event this workflow does not handle, so the refusal + # step below can name it. Without it the job is skipped, every job after it is + # skipped, and the run reports success having done nothing. pull_request is + # excluded from that clause: a pull_request close skips this guard deliberately, + # and the review job reads that skip as its own trigger. + # + # The pull_request branch carries no author-association check, matching the path + # this file replaces: the event is the push itself rather than a person's + # command. The fork check in `Admit the request` refuses code from outside the + # organisation, on this path as well as the comment path. + if: >- + ${{ (github.event_name == 'pull_request' && inputs.mode == 'review') || + (((github.event_name == 'issue_comment' && github.event.issue.pull_request != null) || + (github.event_name == 'pull_request_review_comment' && github.event.action == 'created') || + (github.event_name == 'pull_request_review' && github.event.action == 'submitted')) && + (((github.event.comment.user.type || github.event.review.user.type) != 'Bot' && + contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), + github.event.comment.author_association || github.event.review.author_association)) || + ((github.event.comment.user.type || github.event.review.user.type) == 'Bot' && + contains(fromJSON(inputs.allowed-bots || '[]'), + github.event.comment.user.login || github.event.review.user.login)))) || + (github.event_name != 'pull_request' && + !contains(fromJSON('["issue_comment","pull_request_review_comment","pull_request_review"]'), + github.event_name)) }} + outputs: + should_run: ${{ steps.parse.outputs.should_run == 'true' && steps.admit.outputs.admit == 'true' }} + pr_number: ${{ steps.parse.outputs.pr_number }} + comment_id: ${{ steps.parse.outputs.comment_id }} + # Which REST collection carries the reactions on the object that asked. The + # acknowledgement and the verdict reaction both take this path, and it differs + # per event: an issue comment lives under issues/comments, a diff-thread comment + # under pulls/comments. Empty where the request is not a reactable object, which + # is also when comment_id is empty, so a step gated on either skips. + # + # The two collections take different token scopes. GitHub grants + # issues/comments/{id}/reactions to Issues alone and pulls/comments/{id}/reactions + # to Pull requests; the review job holds both. + comment_api: ${{ steps.parse.outputs.comment_api }} + # The id of the object that asked, for the log. Populated on every comment + # event, including the one that carries no reactions endpoint, because this + # answers "which request started this run" rather than "what may this run react + # on". comment_id above answers the second and is held back where nothing can + # reach it; the driver's --trigger-id takes this one. + trigger_id: ${{ steps.parse.outputs.trigger_id }} + steps: + # First, and before the identity mint below, so a caller that mis-wired its + # triggers learns the reason without spending a credential. A refusal here fails + # the job, and a failed step skips every step after it. + # + # The job condition above admits an unsupported event for exactly this step. + # Without the refusal the condition simply would not match, the guard and the + # review job would both be skipped, and the run would report success having + # reviewed nothing -- which reads as "the workflow ran" to anyone looking at the + # checks list. + # + # pull_request_target is named apart because it is the one a caller reaches for + # to make a fork review work. It runs against the BASE repository's secrets with + # a writable token while the pull request's head is what a reviewer wants to + # read. This workflow checks nothing out, so no step here would run fork code + # today; the refusal is the control that does not depend on that staying true. + # ai-review.yml refuses the same event for the same reason. + - name: Refuse an event this workflow does not handle + env: + EVENT_NAME: ${{ github.event_name }} + run: | + set -uo pipefail + case "$EVENT_NAME" in + pull_request|issue_comment|pull_request_review_comment|pull_request_review) ;; + pull_request_target) + echo "::error::seidroid review refuses pull_request_target: it runs with the base repository's secrets and a writable token over a head this workflow did not check out. Call it from pull_request, issue_comment, pull_request_review_comment or pull_request_review instead. A fork pull request is refused on every one of those, by design" >&2 + exit 1 + ;; + *) + echo "::error::seidroid review cannot be called from '$EVENT_NAME'; it handles pull_request, issue_comment, pull_request_review_comment and pull_request_review" >&2 + exit 1 + ;; + esac + - id: parse + # Every GitHub-supplied value (the request body, the PR number, the + # comment id) comes in through env and is read back as "$VAR" below -- + # never interpolated as ${{ }} directly into the shell script, even for + # the two fields (issue number, comment id) that GitHub happens to + # always populate with integers. Routing all three the same way means + # there is one pattern to audit, not one safe-looking exception. + # + # Each read takes the comment key or the review key, whichever the arriving + # event populated. issue_comment and pull_request_review_comment both carry + # `comment`; pull_request_review carries `review`, and its own id under + # `review.id`. A read that took only one key would leave the body empty on the + # other events, and an empty body parses as no command -- a request that + # vanishes with no reaction and no run to point at. + env: + BODY: ${{ github.event.comment.body || github.event.review.body }} + PR_NUMBER: ${{ github.event.issue.number || github.event.pull_request.number }} + COMMENT_ID: ${{ github.event.comment.id || github.event.review.id }} + EVENT_NAME: ${{ github.event_name }} + TRIGGER_PHRASE: ${{ inputs.trigger-phrase }} + run: | + set -euo pipefail + # An automatic review has no comment to parse: the event IS the request, and + # the job condition above has already established which event this is. It + # reports the same outputs the command grammar below produces, so every + # later step reads one shape and no step needs a second code path. + if [ "$EVENT_NAME" = "pull_request" ]; then + { + echo "should_run=true" + echo "pr_number=$PR_NUMBER" + echo "comment_id=" + echo "comment_api=" + echo "trigger_id=" + } >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Which REST collection carries the reactions on the object that asked, and + # so whether an acknowledgement can land on it at all. GitHub publishes a + # reactions endpoint for an issue comment and for a pull request review + # comment, and none for a pull request REVIEW -- only the GraphQL schema + # makes a review reactable. So a command in a review body starts a review and + # earns no reaction, and the notice below is the signal that it started. + # + # The id goes out empty with it. `Acknowledge the trigger` and `Answer the + # request` both gate on comment_id, and the eyes one posts are the eyes the + # other withdraws: an id with no endpoint to reach would leave a request + # wearing eyes nothing clears. + comment_api="" + case "$EVENT_NAME" in + issue_comment) comment_api=issues/comments ;; + pull_request_review_comment) comment_api=pulls/comments ;; + esac + + # The phrase is caller-settable and reaches the grep pattern below, so its + # SHAPE is constrained here rather than escaped: an optional leading @, then + # letters, digits, `_` and `-`. None of those is an ERE metacharacter, so the + # pattern carries the phrase verbatim and a phrase holding `.` or `|` cannot + # widen the match. Escaping instead would have to cover every ERE + # metacharacter correctly forever; a character class is one thing to read. + # + # A phrase outside that shape falls back to the default with a warning, + # which is what guidelines-file does with a name it cannot trust. The + # alternative is failing the job on every comment the repository receives, + # for a caller mistake that leaves the documented phrase working. + phrase="${TRIGGER_PHRASE#@}" + case "$phrase" in + ""|*[!A-Za-z0-9_-]*) + echo "::warning::trigger-phrase '$TRIGGER_PHRASE' is not an optional @ followed by letters, digits, _ or -; using @seidroid instead" + phrase=seidroid + ;; + esac + + cmd="$(printf '%s' "$BODY" | tr -d '\r')" + # Require a LINE reading `@seidroid review`, optionally `close`, and nothing + # else on it. Anchoring to a whole line is what keeps a comment that merely + # quotes or discusses the command from triggering a review. + # + # The @ is optional, and that is a widening this workflow keeps. A person who + # types the phrase without the mention still means it. Whole-line anchoring + # is what makes it safe: a comment discussing the command has other words on + # the line and does not match. And ai-assistant.yml reserves a body only when + # it carries the @ form, so the bare form reaches this workflow alone. + cmdline="$(printf '%s\n' "$cmd" \ + | grep -m1 -E "^[[:space:]]*@?${phrase}[[:space:]]+review([[:space:]]+close)?[[:space:]]*$" || true)" + if [ -z "$cmdline" ]; then + # A review runs only on the repository the pull request is on, so a + # request that names a repository is named in this run's log. That is all + # this does: an issue_comment run attaches to no pull request, so the + # annotation reaches whoever opens the run and nobody else, and + # ai-assistant.yml still answers the body conversationally, because its + # reservation is the bare command alone. Both match ai-review, where the + # target form was never a review command either. + # + # It earns its line by naming the reason in the one place a person + # debugging "why did my request do nothing" will look. + # + # notice, not error, because a denied request is a notice throughout + # ai-review: an unauthorised actor, an unlisted bot, a draft and an empty + # team all deny at that level. error there is reserved for a caller that + # wired the workflow wrongly. + # + # Matched on the same whole-line anchor the command grammar above uses, + # so the refusal covers exactly the shapes that fall through it. + # + # Captured, and tested for emptiness, rather than read from a `grep -q` + # exit status. -q stops at the first match, which leaves printf writing + # into a closed pipe on a body larger than the pipe buffer; pipefail then + # reads the SIGPIPE as a failed pipeline and the refusal is not written. + # Measured: at 232 kB the -q form reports status 141 and stays silent. + named_repo="$(printf '%s\n' "$cmd" \ + | grep -m1 -E "^[[:space:]]*@?${phrase}[[:space:]]+review([[:space:]]+close)?[[:space:]]+[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*#[0-9]+[[:space:]]*$" || true)" + if [ -n "$named_repo" ]; then + echo "::notice::seidroid review takes no repository target; a review runs only on the repository the pull request is on" + fi + echo "should_run=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + if [ -z "$comment_api" ]; then + echo "::notice::$EVENT_NAME carries no reactions endpoint, so this request earns no acknowledgement on the object it was written on; the review runs and posts its verdict as a comment" + fi + { + echo "should_run=true" + echo "pr_number=$PR_NUMBER" + # The comment id is passed as --trigger-id, which only labels this + # dispatch in the logs. The pull request, not the comment, is the + # session key — so any dispatch adopts that PR's session and drives a + # fresh review turn on the current tree. + # + # Emitted with the endpoint that reaches it, and held back when there is + # none: the two are read together by every step that reacts. trigger_id + # beside it carries the id whatever the event, so the driver still labels + # a dispatch that can be reacted on nowhere. + if [ -n "$comment_api" ]; then + echo "comment_id=$COMMENT_ID" + else + echo "comment_id=" + fi + echo "comment_api=$comment_api" + echo "trigger_id=$COMMENT_ID" + } >> "$GITHUB_OUTPUT" + + # Only reached once the command itself parsed, so a comment that says + # nothing does not mint a token or call the API. + - name: Mint an identity to ask about the team and the labels + id: identity + if: ${{ steps.parse.outputs.should_run == 'true' && env.HAS_REVIEWER_IDENTITY == 'true' }} + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.SEIDROID_APP_ID }} + private-key: ${{ secrets.SEIDROID_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + # Both keys, together. `owner` on its own mints a token that reaches every + # repository the installation is on; `repositories` holds it to this one. + repositories: ${{ github.event.repository.name }} + + - name: Admit the request + id: admit + env: + GH_TOKEN: ${{ steps.identity.outputs.token }} + ALLOWED_TEAM: ${{ inputs.allowed-team }} + ALLOWED_BOTS: ${{ inputs.allowed-bots }} + SKIP_LABEL: ${{ inputs.skip-review-label }} + # Who asked, under whichever key the arriving event populated. A review body + # names its author under `review.user`; the two comment events name theirs + # under `comment.user`. + ACTOR: ${{ github.event.comment.user.login || github.event.review.user.login }} + ACTOR_TYPE: ${{ github.event.comment.user.type || github.event.review.user.type }} + REPO: ${{ github.repository }} + PR: ${{ steps.parse.outputs.pr_number }} + PARSED: ${{ steps.parse.outputs.should_run }} + EVENT_NAME: ${{ github.event_name }} + IS_DRAFT: ${{ github.event.pull_request.draft }} + ACTION: ${{ github.event.action }} + RE_REVIEW_ON_PUSH: ${{ inputs.re-review-on-push }} + # What the CALLER routed this dispatch as. The fork check and the label + # check below both gate on it, and the fork check states why. + MODE: ${{ inputs.mode }} + # The pull_request payload's own repository ids, which spare that path an + # API call. Empty on every other event, where the API answers instead. + HEAD_REPO_ID: ${{ github.event.pull_request.head.repo.id }} + BASE_REPO_ID: ${{ github.event.pull_request.base.repo.id }} + # The App identity where a caller configured one, this workflow's own + # token where it did not. Named apart from GH_TOKEN above, because the + # team check has no such fallback. Reading an organisation's teams takes + # "Members" organization permissions, which a GITHUB_TOKEN cannot carry. + # The reads below need no more than pull-requests: read. + GATE_TOKEN: ${{ steps.identity.outputs.token || github.token }} + run: | + set -uo pipefail + deny() { echo "::notice::$1"; echo "admit=false" >> "$GITHUB_OUTPUT"; exit 0; } + if [ "$PARSED" != "true" ]; then + echo "admit=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # A draft is not ready to be read, and on the automatic path every push to + # one would otherwise spend a sandbox for a review nobody asked for. Read + # from the event payload rather than the API, so it needs no identity and + # cannot fail open when none is configured. + # + # The manual path skips this check on purpose: commenting `@seidroid review` + # on a draft is a direct request, and refusing it would be surprising. + if [ "$EVENT_NAME" = "pull_request" ] && [ "$IS_DRAFT" = "true" ]; then + deny "$REPO#$PR is a draft; not reviewing" + fi + + # WHO may command a review, checked again here. The job condition has + # already applied the same two grounds, and this is the second reading of + # them: a condition is one expression on one line, and a control worth having + # is worth stating where a person debugging a refusal can read the reason. + # + # A bot is held to allowed-bots and a person to allowed-team, because neither + # test answers for the other. A bot is not a team member, so the membership + # read would refuse every bot however trusted. And a bot's author_association + # says nothing: one with write access to the calling repository carries MEMBER + # or COLLABORATOR like anyone else. + # + # Both fail closed: empty, malformed and unanswerable all deny. This step + # runs without -e, so a read that fails leaves the variable empty, and an + # empty value matches neither "true" nor "active". + # + # Both gate the comment path only. An automatic run has no commander: + # applying either check there would silently stop reviewing every pull + # request opened by anyone outside the team, which is the opposite of what a + # caller sets these inputs for. + # + # Both stop a REVIEW, not a teardown, for the reason the label check below + # states. Any requester the job condition admits may reclaim a sandbox, + # because the alternative is a sandbox nothing reclaims. + if [ "$EVENT_NAME" != "pull_request" ] && [ "$MODE" != "close" ]; then + # Lowercased before the comparison, so this reader and the job condition + # read one type the same way: GitHub's expression `==` ignores case and + # the shell's does not. + actor_type="$(printf '%s' "$ACTOR_TYPE" | tr '[:upper:]' '[:lower:]')" + if [ "$actor_type" = "bot" ]; then + # An exact, case-insensitive login match against a JSON array, which is + # how ai-review.yml reads the same input. Exact, because a substring test + # admits a lookalike in either direction: `bot` passes against a listed + # `dependabot[bot]`, and so does `dependabot[bot]x`. Case-insensitive, + # because GitHub treats one login as one account whatever case it is + # written in, and the job condition above compares it that way too. + # + # A value that is not a JSON array of strings yields no match and denies. + listed="$(printf '%s' "$ALLOWED_BOTS" \ + | jq -r --arg actor "$ACTOR" \ + 'if type == "array" + then any(.[]; type == "string" and ascii_downcase == ($actor | ascii_downcase)) + else false end' 2>/dev/null || true)" + [ "$listed" = "true" ] || deny "$ACTOR is not in allowed-bots; denying" + else + # Reading an organisation's teams needs the App identity, so a caller + # that configures no App is refused here. The notice says so, and names + # the one thing that fixes it. + case "$ALLOWED_TEAM" in + */*) ;; + *) deny "allowed-team is empty or is not org/team-slug; denying" ;; + esac + [ -n "${GH_TOKEN:-}" ] || deny "this run holds no App identity, so it cannot read membership of $ALLOWED_TEAM; denying. Pass SEIDROID_APP_ID and SEIDROID_APP_PRIVATE_KEY to this workflow. An automatic pull_request review and @seidroid review close do not reach this check" + state="$(gh api "orgs/${ALLOWED_TEAM%%/*}/teams/${ALLOWED_TEAM##*/}/memberships/${ACTOR}" --jq .state 2>/dev/null || true)" + [ "$state" = "active" ] || deny "$ACTOR is not an active member of $ALLOWED_TEAM; denying" + fi + fi + + # A fork pull request carries code from outside the organisation. A review + # clones that code into a sandbox. That sandbox holds a live App credential + # and a shell. Both paths refuse it. ai-review.yml refuses the comment path in + # the same words. + # + # The automatic path needs this as much as the comment path. GitHub withholds + # this workflow's secrets from a fork pull_request run by default. The + # machine-client check below then fails such a run. A private or internal + # repository can turn that withholding off, per repository or by organisation + # policy. This check does not rest on a setting nobody here controls. + # + # Keyed on the EVENT, not on whether the payload carried the ids. A + # pull_request payload carries both, so that path spends no API call. + # issue_comment carries no pull request object at all, so the API answers + # there. pull_request_review_comment and pull_request_review DO carry + # `pull_request.head.repo.id` and `.base.repo.id`, and this branch ignores + # them: the label check below reads the same endpoint unconditionally on + # every one of these paths, so reading the payload here would remove one of + # two identical round trips and neither the failure mode nor the dependency. + # A branch keyed on payload shape rather than on event would also have to + # keep this check's fail-closed rule, and that rule is the highest-consequence + # one in the guard. Repository ids, not names, so a rename does not read as a + # fork. A null head repository reads as a fork, which is the safe reading. + # + # This check fails closed, unlike the label check below. Only a definite "same" + # admits, so an unreadable origin refuses. A failed read costs one refused + # review a person can retry. Admitting on a signal nobody could read costs the + # sandbox above. The step runs without -e, so a failed read leaves the variable + # empty. The API's own error goes to the log, where it says why. + # + # It stops a REVIEW, not a teardown, for the reason the label check states. A + # fork pull request must still be able to reclaim its sandbox. + # + # Keyed on the caller's mode, because the caller is the reader of the comment + # body that routes the dispatch. Two readers of one grammar can disagree: this + # guard's own parse accepts a bare `seidroid review close`, where a caller + # matching the documented `@seidroid` form reads that same comment as a + # review. Mode decides what the review job does, so mode is what the checks + # here gate on. + if [ "$MODE" != "close" ]; then + if [ "$EVENT_NAME" = "pull_request" ]; then + # An empty BASE id means the payload did not carry the signal, which is + # the same standing as a read that failed. It must not compare equal to an + # empty HEAD id and admit. + if [ -z "$BASE_REPO_ID" ]; then + origin=unreadable + elif [ "$HEAD_REPO_ID" = "$BASE_REPO_ID" ]; then + origin=same + else + origin=fork + fi + refusal="$REPO#$PR is fork-originated; not reviewing it" + else + origin="$(GH_TOKEN="$GATE_TOKEN" gh api "repos/$REPO/pulls/$PR" \ + --jq 'if .head.repo.id != null and .head.repo.id == .base.repo.id then "same" else "fork" end' \ + || true)" + refusal="explicit re-reviews are disabled for fork-originated pull requests; not reviewing $REPO#$PR" + fi + case "$origin" in + same) ;; + fork) deny "$refusal" ;; + *) deny "could not read where $REPO#$PR comes from, so a fork cannot be ruled out; not reviewing it" ;; + esac + fi + + # The label stops a REVIEW, not a teardown. A pull request that gains the + # label after a session exists must still be able to reclaim its sandbox, and + # nothing else will: no lifetime cap, no sweep. Keyed on mode for the reason + # the fork check states. + # + # It reads under GATE_TOKEN, so the label bites for a caller that configures + # no App. GitHub documents GET /repos/{owner}/{repo}/pulls/{n} as taking + # "Pull requests" read or "Contents" read, and this job grants the first. + # + # It fails CLOSED, like the fork check above and unlike the once-per-PR gate + # below. A refusal costs one review, and the notice names the two fixes. + # Admitting on a read that did not answer costs the label its meaning, on the + # one pull request whose author asked for no review. Anything but a plain + # `false` denies. + if [ "$MODE" != "close" ] && [ -n "$SKIP_LABEL" ]; then + # shellcheck disable=SC2016 # $ENV is jq's own, and jq reads it + carries="$(GH_TOKEN="$GATE_TOKEN" \ + gh api "repos/$REPO/pulls/$PR" \ + --jq 'any(.labels[]?.name; . == $ENV.SKIP_LABEL)')" \ + || deny "could not read the labels on $REPO#$PR, so $SKIP_LABEL cannot be ruled out; not reviewing. Grant pull-requests: read on the calling job, or pass SEIDROID_APP_ID and SEIDROID_APP_PRIVATE_KEY" + if [ "$carries" != "false" ]; then + deny "$REPO#$PR carries $SKIP_LABEL; not reviewing" + fi + fi + + # One automatic review per pull request. The verdict comment is the + # record that one already ran: this workflow posts one for every review + # that reaches a verdict, it hangs on the pull request rather than on a + # commit, and a push does not remove it. The review check run is keyed + # on the head commit, so a push leaves none behind and every push would + # read as a first review. + # + # The comment path never reaches this. A person who asks for a review by + # name is answered, however many have already run. + # + # Last of the checks, and the only one that pages the API. It makes at + # most two reads, and a block found on the first skips the second. + if [ "$EVENT_NAME" = "pull_request" ]; then + re_review=false + if [ "$ACTION" = "synchronize" ] && [ "$RE_REVIEW_ON_PUSH" = "true" ]; then + re_review=true + fi + # A standing block exempts a PUSH to the pull request. The withdrawal lives + # inside a later review, so a gate that refuses every automatic run also + # refuses the only run that can retract a CHANGES_REQUESTED this workflow + # left. The author pushes the fix and the block stands, clearable by a + # comment they were never told to write. + # + # Scoped to `synchronize` for the reason the input above is: reopening a + # pull request or relabelling it changes no code, so a review of it reaches + # the finding the block already names. The block is still accurate, and the + # next push is what clears it. + # + # A dismissed review reads as DISMISSED, so what this finds is a block that + # is still standing. The same shape the withdrawal itself looks for. + if [ "$re_review" = "false" ] && [ "$ACTION" = "synchronize" ]; then + # A read that fails cannot rule a block out, so the gate admits and lets + # the review look. That costs one review. A refusal costs the withdrawal + # of a merge gate this workflow itself left standing, which then needs a + # human. The API's own error goes to the log, where it says why. + read_ok=true + # shellcheck disable=SC2016 # $ENV is jq's own, and jq reads it + blocked="$(GH_TOKEN="$GATE_TOKEN" \ + gh api "repos/$REPO/pulls/$PR/reviews?per_page=100" --paginate \ + --jq '.[] | select(.state == "CHANGES_REQUESTED" and ((.body // "") | startswith($ENV.VERDICT_MARKER))) | .id')" || read_ok=false + if [ -n "$blocked" ]; then + echo "::notice::$REPO#$PR carries a block from this workflow, so this review runs to withdraw it" + re_review=true + elif [ "$read_ok" = "false" ]; then + echo "::warning::could not read the reviews on $REPO#$PR, so a block this workflow left cannot be ruled out; this review proceeds to withdraw one" + re_review=true + fi + fi + + if [ "$re_review" = "false" ]; then + # The author is tested by type rather than by login: the verdict + # posts under the App identity where a caller configured one and + # under this workflow's own where it did not. The marker at the + # top of this file must open the body. A person quoting it is not + # a Bot, and a bot that mentions it does not open with it, so + # neither reads as a verdict. + # + # It fails open, where the label check above refuses. A read that + # fails costs one extra review, and the next push corrects it. A + # refusal on a signal this step could not read costs the review + # itself, on a pull request whose author never learns it was + # refused. + # + # A full page at a time: the pull request this gate matters most + # on is the one pushed to ten times, which is also the one carrying + # the most comments to page through. The page size rides in the + # path because `-F` on a `gh api` that names no `-X` makes the + # request a POST, and this one only ever reads. + # + # A verdict found is a verdict found, whatever the read did afterwards. + # --paginate streams a page at a time, so a read that finds one on page 1 + # and then meets a 5xx on page 3 exits non-zero holding it -- and the pull + # request that pages is the one this gate is worth most on. + read_ok=true + # shellcheck disable=SC2016 # $ENV is jq's own, and jq reads it + prior="$(GH_TOKEN="$GATE_TOKEN" \ + gh api "repos/$REPO/issues/$PR/comments?per_page=100" --paginate \ + --jq '.[] | select(.user.type == "Bot" and ((.body // "") | startswith($ENV.VERDICT_MARKER))) | .id')" || read_ok=false + if [ -n "$prior" ]; then + deny "$REPO#$PR already carries a verdict from this workflow, and this $ACTION event does not earn another; comment @seidroid review to ask for one" + elif [ "$read_ok" = "false" ]; then + echo "::warning::could not read the comments on $REPO#$PR, so a review that already ran cannot be found; this one proceeds" + fi + fi + fi + + echo "admit=true" >> "$GITHUB_OUTPUT" + + # Fail here rather than after the in-cluster job is scheduled. A caller that + # forgot the secret is the most likely misconfiguration, and `required: true` + # does not catch it: a caller passing an unset secret satisfies "provided" + # with an empty string. Without this the driver would spin a runner, mint + # nothing, and report a credential error several minutes later. + # + # Only ever tests emptiness -- the value is never echoed, compared against a + # literal, or written to an output. + # + # Reads the verdict as well as the parse. `deny` exits 0, so every step after it + # still runs. A refused request needs no credential, and a fork pull_request run + # holds none: without the verdict here, refusing one would paint a deliberate + # refusal red and name a caller misconfiguration that does not exist. + - name: Require the machine-client secret + if: steps.parse.outputs.should_run == 'true' && steps.admit.outputs.admit == 'true' + env: + SECRET_PRESENT: ${{ secrets.OMNIGENT_MACHINE_CLIENT_SECRET != '' }} + run: | + if [ "$SECRET_PRESENT" != "true" ]; then + echo "::error::OMNIGENT_MACHINE_CLIENT_SECRET is not set on the calling repository; configure it as a repository or organization secret and pass it to this workflow" >&2 + exit 1 + fi + + # Half a credential mints nothing, and the review runs under the workflow's own + # identity instead of failing. That is the better outcome and a silent one, so + # this says it out loud. Not fatal: the App is optional, and a review under + # github-actions is a review. + # + # Either half alone, not just a missing key. A diagnostic with a blind spot + # sends the reader looking at the half that is already set. + # + # Step-level env, like the check above. A step `if` cannot read `secrets`; a + # step `env` can. + - name: Report a half-configured reviewer identity + if: steps.parse.outputs.should_run == 'true' + env: + APP_ID_PRESENT: ${{ secrets.SEIDROID_APP_ID != '' }} + APP_KEY_PRESENT: ${{ secrets.SEIDROID_APP_PRIVATE_KEY != '' }} + run: | + missing="" + if [ "$APP_ID_PRESENT" = "true" ] && [ "$APP_KEY_PRESENT" != "true" ]; then + missing=SEIDROID_APP_PRIVATE_KEY + elif [ "$APP_KEY_PRESENT" = "true" ] && [ "$APP_ID_PRESENT" != "true" ]; then + missing=SEIDROID_APP_ID + fi + if [ -n "$missing" ]; then + echo "::warning::$missing is not set beside the other half of the App credential, so this review posts as github-actions[bot] rather than as the App. Pass both secrets for the App identity, or unset the half that is set" + fi + + review: + name: Review + needs: guard + # A close arriving as a pull_request event needs no guard verdict — GitHub's own + # event is the trigger — so it runs even though the guard was skipped. always() + # is required: a skipped dependency would otherwise skip this too. + # + # A close asked for in a COMMENT is a different thing and does need one. The + # guard admits it on two things only: the comment grammar it parses, and the + # OWNER/MEMBER/COLLABORATOR filter on the job condition. The team, fork and + # label checks all exempt a teardown, because a close a check refuses leaves a + # sandbox that nothing else reclaims. + # !cancelled() rather than always(), and the guard's RESULT rather than only its + # output. always() started this job when the guard had failed -- the secret check + # is the last thing the guard does, so should_run is already set by then and the + # fast-fail bought nothing -- and started it on a cancelled run, which spends + # model quota and holds a sandbox for a review nobody is waiting for. + # + # !cancelled() is still needed because the close path SKIPS the guard, and a + # skipped dependency would otherwise skip this too. Each path names the guard + # result it expects, so neither admits the other's: a pull_request REVIEW is + # guarded, because that is where draft and skip-label are decided, while a + # pull_request CLOSE is not guarded at all. + # + # The command arrives on three comment events, and all three name the guard's + # success. An event this workflow does not handle fails the guard rather than + # skipping it, so no clause here admits one. + if: >- + ${{ !cancelled() && ( + ((github.event_name == 'issue_comment' || + github.event_name == 'pull_request_review_comment' || + github.event_name == 'pull_request_review' || + (github.event_name == 'pull_request' && inputs.mode == 'review')) && + needs.guard.result == 'success' && + needs.guard.outputs.should_run == 'true') || + (github.event_name == 'pull_request' && + inputs.mode == 'close' && + needs.guard.result == 'skipped')) }} + # Exactly one review per PR: a newer `seidroid review` cancels an in-flight one + # (latest wins, never two posters); the driver traps cancellation and STOPS its + # session, keeping the conversation. Only the close event deletes it, so a rapid + # re-trigger does not reclaim the sandbox. Job-level so the group is entered only + # when a real command runs. + concurrency: + # The mode is in the group: without it a review dispatch and a close dispatch + # for one pull request share a group under cancel-in-progress, so commenting on + # a just-closed pull request cancels the in-flight close -- and close is the + # only thing that reclaims a sandbox. + # + # The event's own number is the third term's fallback, and it is load-bearing: + # a close arriving as a pull_request event skips the guard, so the guard's + # number is empty there. Without it every reclaim run shares one group, and + # two closes for different pull requests cancel each other -- while close is + # the only thing that reclaims a sandbox. + group: >- + seidroid-review-${{ inputs.mode }}-${{ github.repository }}-${{ needs.guard.outputs.pr_number || github.event.pull_request.number }} + cancel-in-progress: true + # The reviewed repo's runner label, GitHub-hosted by default. The default pairs + # with the https base URL: the mint refuses to send the client secret over plain + # http, so the in-cluster ClusterIP Service is not a usable target however the + # runner is hosted. + runs-on: ${{ inputs.runs-on }} + # The outer bound on a wedged review. Without it the account default is six + # hours, and the managed sandbox stays up for the whole of it. Comfortably + # above the driver's own SEIDROID_RUN_DEADLINE_S so the driver reports the + # timeout, with its reason, before the runner kills the job. + timeout-minutes: ${{ inputs.timeout-minutes }} + permissions: + pull-requests: write # post the verdict comment and the review position + contents: read # read PR metadata + checks: write # publish the review's check runs + # React to the triggering comment. BOTH scopes are load-bearing, one per + # collection: a conversation comment's reactions live under issues/comments, + # whose POST and DELETE GitHub's permission table grants to Issues alone, and a + # diff-thread comment's under pulls/comments, granted to Pull requests. The + # guard picks the collection per event, so a review that reacts on a diff thread + # needs pull-requests: write and one on the conversation needs issues: write. + # Prune either and the reaction fails on the path it serves, silently: all three + # reacting steps treat a lost reaction as a courtesy and warn. + # + # The comment itself is grantable by either scope, so the alias stops at the + # reaction. GraphQL addReaction is the other route, and GitHub documents no + # permission for any mutation, so only a live call settles what that route needs. + issues: write # acknowledge the trigger with a reaction + # The credential lives ONLY here, at job level. It must never be re-declared + # as step-level env on a `uses:` step (composite/action steps do not receive + # step-level env at all) -- that is the exact defect that broke every real + # run of the Python driver's workflow. Every step in this job inherits it. + env: + # Empty is not the same as unset for this one: an empty SEIDROID_AGENT_ID would + # override the driver's default with nothing. envOr treats empty as absent, so + # passing it through empty is safe and keeps one place deciding the default. + SEIDROID_AGENT_ID: ${{ inputs.agent-id }} + OMNIGENT_MACHINE_CLIENT_ID: ${{ inputs.machine-client-id }} + OMNIGENT_MACHINE_CLIENT_SECRET: ${{ secrets.OMNIGENT_MACHINE_CLIENT_SECRET }} + # Whether the bot's identity is available, as a value a step condition can + # read. The secrets context is not one of those -- a step `if` that touches + # it is a workflow-file error, not a false condition -- so the presence test + # happens here, the same way the machine-client check below does it. + # + # Both halves, for the reason the guard's copy states. + HAS_REVIEWER_IDENTITY: >- + ${{ secrets.SEIDROID_APP_ID != '' && secrets.SEIDROID_APP_PRIVATE_KEY != '' }} + steps: + # First, deliberately: the reaction is the only signal the trigger was + # seen, and everything after it -- toolchain, driver install, session + # start -- runs for minutes before anything else appears on the pull + # request. An acknowledgement that arrives after the verdict is not one. + # + # Comment path only. An automatic pull_request review has no comment to + # react to, so the guard leaves comment_id empty and this is skipped. + # + # continue-on-error: an acknowledgement is a courtesy. Failing the review + # because a reaction did not post would trade the whole job for the + # signal that the job started. + - name: Acknowledge the trigger + # A review turn, and a comment to react to. `Answer the request` withdraws this + # reaction and it runs on the same two facts, so a mode it skips must not react + # here: a close would leave eyes that nothing clears. + if: ${{ inputs.mode == 'review' && needs.guard.outputs.comment_id != '' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + TRIGGER_ID: ${{ needs.guard.outputs.comment_id }} + # The collection the request's own reactions live under, from the guard. A + # conversation comment and a diff-thread comment take different endpoints, + # and the guard emits an id only where one of them reaches it. + COMMENT_API: ${{ needs.guard.outputs.comment_api }} + run: | + set -euo pipefail + # Reactions are idempotent per (user, content): re-running a review on + # the same comment returns the existing reaction rather than adding a + # second one, so a retry needs no cleanup. + # + # `Answer the request` withdraws this reaction on every path it takes, and the + # last step of the job withdraws it on the one path that step skips. A review + # that keeps it reads as a review that is still running. All three read + # COMMENT_API, so all three reach the collection this one posted to. + if gh api -X POST "repos/$REPO/$COMMENT_API/$TRIGGER_ID/reactions" \ + -f content=eyes >/dev/null 2>&1; then + echo "acknowledged comment $TRIGGER_ID" + else + echo "::warning::could not react to comment $TRIGGER_ID; the review continues" + fi + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + # At or above what the driver's own go.mod requires. A live run proved why + # this cannot lag: pinned at 1.24 while the module declared go 1.25.0, the + # install died with + # requires go >= 1.25.0 (running go 1.24.13; GOTOOLCHAIN=local) + # because setup-go exports GOTOOLCHAIN=local, so the go command may not + # fetch the toolchain the module asks for. The step that installs the driver + # sets GOTOOLCHAIN=auto for that reason and would have recovered this on its + # own -- but a version resolved here rather than downloaded there is one + # fewer network dependency on the path, so both are set. + # + # go-version-file is not usable: this workflow checks nothing out, so there + # is no go.mod on disk to read. Bump this when the driver's does. + go-version: '1.25' + + - name: Install the review driver + id: build + shell: bash + env: + DRIVER_VERSION: ${{ inputs.driver-version }} + # Read by the floor below, which a close is exempt from. + MODE: ${{ inputs.mode }} + # Overrides the GOTOOLCHAIN=local that setup-go exports, for the reason that + # step gives above. + GOTOOLCHAIN: auto + # The oldest driver this file drives, and deliberately the same release as + # the driver-version default: a caller may run ahead of that default, never + # behind it. The conclusion a review reaches for a given set of findings is + # specific to the driver that reached it -- v0.12.0 concludes `success` where + # v0.11.0 concludes `neutral`; v0.14.0 writes a `failure` check for a run that + # reaches no verdict where v0.13.0 writes none; v0.15.0 carries the threads a + # re-review closes where v0.14.0 carries none; v0.16.0 bounds the prior-thread + # history by bytes where v0.15.0 bounds it by a count; and v0.17.0 names which + # thread each finding replaces, which is what lets the resolve step below close + # a thread only once its own replacement reached the code. A merge gate keyed on + # one of those is wrong for the others, so this file serves one and refuses the + # rest. + # + # Move this with the driver-version default above: one value in two places, + # and nothing enforces it. The two mistakes are not symmetric. Raising this + # alone fails every caller that omits the input, at once and in the open. + # Raising the default alone leaves a floor that goes on admitting a driver + # this file no longer drives -- the drift the whole check exists to catch, and + # the direction that says nothing while it happens. + MIN_DRIVER_VERSION: 'v0.18.0' + run: | + set -euo pipefail + # An input default applies only when the caller omits the key. A caller that + # passes the key through from its own optional input sends an empty string, + # which reaches go install as a bare trailing @. The floor is the default, so + # falling back to it here keeps the two the same value by construction rather + # than by the comment below asking. + DRIVER_VERSION="${DRIVER_VERSION:-$MIN_DRIVER_VERSION}" + + # sei-internal-skills is public, so there is no credential and no + # GOPRIVATE. The module path below is absolute and did not change when this + # file moved: the driver stays there. + # + # The driver is a NESTED module. The repository carries path-prefixed tags + # (sei-agent-driver/vX.Y.Z) and `go install` refuses one as a disallowed + # version string; what it takes is the bare version, `v0.18.0`. A sha becomes + # a pseudo-version. + out="$RUNNER_TEMP/bin" + GOBIN="$out" go install \ + "github.com/sei-protocol/sei-internal-skills/sei-agent-driver/cmd/sei-agent-driver@${DRIVER_VERSION}" + bin="$out/sei-agent-driver" + echo "bin=$bin" >> "$GITHUB_OUTPUT" + + # What go install resolved, read from the build info the toolchain stamped + # rather than from the binary's own --version, so a driver that changed how + # it reports itself still answers this. A sha pin reads here as the + # pseudo-version it became, so the log names exactly what reviewed. + version="$(go version -m "$bin" | awk '$1 == "mod" { print $3; exit }')" + echo "installed sei-agent-driver $version" + + # THE CONTRACT CHECK. It fails here, before the driver holds a credential, + # opens a session or spends model quota -- a driver this file cannot drive + # should cost an install, not a review. + # + # Too old is the version floor above, and it is the review's alone. A close + # reaches no conclusion: it deletes the session, and it is the only thing that + # reclaims a sandbox, so holding it to the vocabulary would strand a live + # sandbox behind a caller edit. A close runs on whatever the caller pinned. + # The surface check below still covers it, and `--close` is the part of that + # surface this path needs. + # + # A pseudo-version carries the tag it follows, so a sha sorts against the + # floor the same way a release does, and semver puts a pre-release below its + # own core version. + at_least() { + printf '%s\n%s\n' "$1" "$2" | awk ' + NR == 1 { split(substr($0, 2), got, /[-.+]/); pre = index($0, "-") > 0; next } + { split(substr($0, 2), want, /[.]/) + for (i = 1; i <= 3; i++) { + if (got[i] + 0 > want[i] + 0) { exit 0 } + if (got[i] + 0 < want[i] + 0) { exit 1 } + } + exit pre }' + } + if [ "$MODE" != "close" ] && ! at_least "$version" "$MIN_DRIVER_VERSION"; then + echo "::error::driver contract: sei-agent-driver $version is older than $MIN_DRIVER_VERSION, and reaches a different conclusion for the same findings; drop driver-version from the caller to take this workflow's own default" + exit 1 + fi + + # Too new, or simply not this driver: a renamed subcommand or a renamed flag + # is a contract this file no longer speaks. `review --help` is the whole + # surface this file drives and the only part of it a check can read without a + # credential and without a session. What --help cannot see -- the SEIDROID_* + # variable names and the check.json fields -- still fails at run time. + if ! usage="$("$bin" review --help 2>&1)"; then + echo "::error::driver contract: sei-agent-driver $version has no \`review\` subcommand, which this workflow invokes" + exit 1 + fi + # Every long flag the help names, wherever it sits on the line. Anchoring to + # the start of the line reads only ` --out string`, and cobra prints a flag + # that has a shorthand as ` -o, --out string` -- so adding a shorthand, which + # takes nothing away, would report the flag as missing and fail every review. + supported="$(printf '%s\n' "$usage" | grep -oE -- '--[a-z0-9-]+' \ + | sed 's/^/ /; s/$/ /' | tr -d '\n')" + missing="" + for flag in --out --findings-out --check-out --close --conversation-context \ + --guidelines-file --extra-instructions --include-nits \ + --trigger-id; do + case "$supported" in (*" $flag "*) ;; (*) missing="$missing $flag" ;; esac + done + if [ -n "$missing" ]; then + echo "::error::driver contract: sei-agent-driver $version does not accept \`review\`$missing, which this workflow passes" + exit 1 + fi + + - name: Mint the reviewing identity + # A review is the bot's work, and the identity on it is what a reader + # trusts, so the posting steps below prefer this token to the workflow's own. + # + # Optional on purpose. Without the app credentials every step below falls + # back to the workflow's own token -- correct, just attributed to + # github-actions rather than to the bot. + id: identity + # Minted before the review rather than after it: the step below reads the + # threads this reviewer left last time, and that read needs the same identity + # that wrote them. Gating on a verdict is no longer possible here and no + # longer needed -- an unused token costs one API call, and every step that + # consumes it still carries its own condition. + if: ${{ inputs.mode == 'review' && !cancelled() + && env.HAS_REVIEWER_IDENTITY == 'true' }} + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.SEIDROID_APP_ID }} + private-key: ${{ secrets.SEIDROID_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + # Both keys, together, for the reason the guard's mint states. + repositories: ${{ github.event.repository.name }} + + - name: Name the identity this review posts under + # The logins every step below compares a comment's author against, so a thread + # this tool wrote is told apart from one that merely quotes its marker. One + # place, for the reason FINDING_MARKER is one place: two steps computing them + # from expressions of their own drift, and a drift here reads as a pull request + # this reviewer has never touched. + # + # Two of them, because reading and mutating do not carry the same risk. + # REVIEWER_LOGIN is the identity this run uses and is the only one allowed to + # have written a thread this run closes. WORKFLOW_LOGIN is what a run without + # App credentials posts as, and the history read admits it too: a repository + # that has moved between the two would otherwise lose its whole history, and a + # review that re-reports every finding it already made and can close none of + # them is worse than one that simply closes none. + # + # It cannot ride in the job's env block. That is evaluated before any step runs + # and the app slug is a step output, so this exports to $GITHUB_ENV instead and + # the steps below read a plain variable. + # + # Without app credentials the identity step is skipped and every step here falls + # back to the workflow's own token, which posts as github-actions. + # + # continue-on-error, like the other steps that improve publishing without being + # allowed to prevent reviewing. Every reader treats an unset value as matching no + # author, so a failure here costs the history and the thread closing, and says so + # rather than closing the wrong thread. + if: ${{ inputs.mode == 'review' && !cancelled() }} + continue-on-error: true + shell: bash + env: + APP_SLUG: ${{ steps.identity.outputs.app-slug }} + run: | + set -euo pipefail + # The identity a run without App credentials posts under. Written here rather + # than spelled again in each reader, so the two logins come from one place. + workflow_login="github-actions[bot]" + login="$workflow_login" + if [ -n "${APP_SLUG:-}" ]; then login="${APP_SLUG}[bot]"; fi + echo "REVIEWER_LOGIN=$login" >> "$GITHUB_ENV" + echo "WORKFLOW_LOGIN=$workflow_login" >> "$GITHUB_ENV" + echo "this review reads and posts as $login" + + - name: Read the threads this review left before + id: threads + # What this reviewer said last time, and what the author said back. Read + # here rather than by the agent for the reason the diff is: a step the agent + # must perform is a step it can skip, and prose the author controls should + # not travel through a shell to get here. + # + # Its own threads only, on two tests, and both have to hold. + # + # The marker has to OPEN the body, not merely appear in it. Placement writes it + # as the first bytes, and every verdict-side reader in this file keys the same + # way. A contains test admits any comment that quotes the marker instead -- and + # ai-review posts under the same bot identity, so one of its inline comments + # quoting this file's FINDING_MARKER line, on the repository that defines it, + # would pass an author test as well and be read back as a finding this tool + # left. + # + # The author has to be one this tool posts as: this run's identity, or the + # workflow token's. Either, here, because reading is inert; the step that + # closes a thread takes this run's identity alone. + # + # The thread id travels with each one. It is minted here, when the comment + # posts, and the driver's session has no way to learn it -- so this read is + # where a review gets the handle it names to close a finding it has addressed. + # + # Every thread on the pull request, not the first page of them. What this step + # hands over is the whole set; which of it a prompt can afford is the driver's + # budget to spend, and it reports what it leaves out. A cap here would be a + # second, silent one underneath that. + # + # continue-on-error, and the driver reads an absent file as a first review: + # a history that cannot be fetched must cost the recall, not the review. + if: ${{ inputs.mode == 'review' && !cancelled() }} + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ steps.identity.outputs.token || github.token }} + REPO: ${{ github.repository }} + PR: ${{ needs.guard.outputs.pr_number }} + THREADS: ${{ runner.temp }}/review-prior-threads.json + run: | + set -euo pipefail + echo "threads_path=$THREADS" >> "$GITHUB_OUTPUT" + owner="${REPO%%/*}"; name="${REPO##*/}" + + # Every page, not the first hundred. A pull request reviewed enough times + # carries more threads than one page holds, and the ones past the cut were + # silently absent -- so a re-review could not see a finding it had already + # made there, and made it again. That is the duplicate thread PLT-1145 + # removed, reached by a different route. + # + # --paginate needs the query to take $endCursor and to return pageInfo, which + # is the whole of what gh asks for. It writes one JSON document per page, and + # `jq -s` is what turns them into one array. --slurp would do the same inside + # gh; jq -s is used instead because it asks nothing of the runner's gh version. + # + # The root comment and the recent ones are fetched as two connections, because + # they answer different questions and one window cannot serve both. Every + # ownership test reads the ROOT -- the marker has to open its body and its + # author has to be this tool -- so that one is `first: 1` and cannot move. The + # replies that matter are the NEWEST, which is what a session has no way to + # know, so that one is `last`. A single `first: 20` gave the oldest twenty of a + # busy thread and the driver then showed the last three of those: three replies + # from the middle of the conversation, presented as the latest word on it. + # + # totalCount rides along so a thread with more replies than this reads can say + # so, rather than being quietly shortened. + # $owner and friends are GraphQL variables, so the query stays literal. + # shellcheck disable=SC2016 + gh api graphql --paginate -f query=' + query($owner: String!, $name: String!, $number: Int!, $endCursor: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + reviewThreads(first: 100, after: $endCursor) { + nodes { + id + isResolved + path + line + originalLine + root: comments(first: 1) { nodes { id body author { login } } } + recent: comments(last: 20) { + totalCount + nodes { id body author { login } } + } + } + pageInfo { hasNextPage endCursor } + } + } + } + }' -F owner="$owner" -F name="$name" -F number="$PR" \ + | jq -s '[ .[].data.repository.pullRequest.reviewThreads.nodes[] ]' > "$THREADS.raw" + + # line goes null once a thread is stale against the head commit, and + # originalLine still says where it was written -- which is what makes a + # thread on since-rewritten code readable rather than a finding at line 0. + # + # thread_id is the handle a review names to close one. The driver admits an id + # only when it matches a thread listed here, so this file is the allowlist as + # well as the history, and a thread left out of it can be named and not closed. + # + # The marker and the login reach jq through $ENV rather than through the shell, + # so the program stays one single-quoted string and no comment body is ever + # spliced into it. + # The root is the whole of the ownership test and the finding's own words; the + # replies are every recent comment that is not it, matched on id rather than on + # position, so a thread short enough to carry its own root in the recent window + # does not report the finding back as a reply to itself. + # + # reply_total is how many replies the thread has, against however many of them + # this carries. The count is written whether or not the two differ, so a reader + # of this file needs no second source to tell a whole conversation from a + # shortened one. Nothing consumes it yet: the driver renders "showing N of M" + # from it in the change that lands beside this one, and an older driver ignores + # a key it does not know. + # shellcheck disable=SC2016 # $ENV is jq's, and single quotes are what keep it jq's + jq '[ .[] + | select((.root.nodes[0].body // "") | startswith($ENV.FINDING_MARKER)) + | select(.root.nodes[0].author.login + | . != null and . != "" + and (. == $ENV.REVIEWER_LOGIN or . == $ENV.WORKFLOW_LOGIN)) + | . as $t + | { thread_id: (.id // ""), + file: (.path // ""), + line: (.line // .originalLine // 0), + body: ((.root.nodes[0].body // "") | sub($ENV.FINDING_MARKER + "\n*"; "")), + replies: [ $t.recent.nodes[] + | select(.id != $t.root.nodes[0].id) + | "\(.author.login // "someone"): \(.body)" ], + reply_total: (($t.recent.totalCount // 1) - 1), + resolved: .isResolved } ]' "$THREADS.raw" > "$THREADS" + fetched="$(jq length "$THREADS.raw")" + carried="$(jq length "$THREADS")" + echo "read $fetched review thread(s) across every page; carrying $carried prior finding(s) written by ${REVIEWER_LOGIN:-nobody this run could name} or ${WORKFLOW_LOGIN:-nobody} into this review" + + # A thread with more replies than one page of them is reported rather than + # quietly shortened. The newest are the ones kept, so what is missing is the + # middle of a long conversation -- worth knowing when a reply seems to answer + # something the history does not show. + # + # Counted over the threads this history actually carries, both tests applied. + # Counting every marked thread instead would report a shortened conversation on + # a run that carried no history at all, which says nothing true about what the + # review is working from. + # + # The window plus one, because totalCount counts the root and the root is + # fetched separately and carried in `body`. At totalCount 21 the recent window + # holds comments 2 to 21, which is every reply there is, and the history is + # whole; the first comment a reader actually loses appears at 22. + # shellcheck disable=SC2016 # $ENV is jq's + shortened="$(jq --argjson window 20 '[ .[] + | select((.root.nodes[0].body // "") | startswith($ENV.FINDING_MARKER)) + | select(.root.nodes[0].author.login + | . != null and . != "" + and (. == $ENV.REVIEWER_LOGIN or . == $ENV.WORKFLOW_LOGIN)) + | select(.recent.totalCount > $window + 1) + ] | length' "$THREADS.raw")" + if [ "$shortened" -gt 0 ]; then + echo "::warning::$shortened of this tool's thread(s) on $REPO#$PR carry more replies than this read holds; the 20 most recent of each are in the history and the older ones are not" + fi + + # A marked thread that the login test drops is worth a line, because the two + # ways to get here look identical from the outside: this reviewer has left + # nothing on the pull request yet, or it has and this run does not recognise + # its own identity. Only the second is a defect, and it costs the review its + # history and its ability to close a single thread. + # shellcheck disable=SC2016 # $ENV is jq's + marked="$(jq '[ .[] + | select((.root.nodes[0].body // "") | startswith($ENV.FINDING_MARKER)) + ] | length' "$THREADS.raw")" + if [ "$marked" -gt 0 ] && [ "$carried" -eq 0 ]; then + echo "::warning::$marked thread(s) on $REPO#$PR open with this tool's marker and none was written by ${REVIEWER_LOGIN:-nobody this run could name} or ${WORKFLOW_LOGIN:-nobody}, so this review carries no history and can close no thread" + fi + + - name: Record the commit under review + id: head + if: ${{ inputs.mode == 'review' && !cancelled() }} + # Tolerated, because this is a step that improves publishing and must not be + # able to prevent reviewing. Failing it hard would let a transient api error + # abort a review that would otherwise have run, since the drive step that + # follows carries no condition of its own. No publisher below reads the head + # for itself. Each one states what it cannot do when this step produces + # nothing. + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ steps.identity.outputs.token || github.token }} + REPO: ${{ github.repository }} + PR: ${{ needs.guard.outputs.pr_number }} + run: | + set -euo pipefail + # Resolved once, here, and read from this output by every publisher below. + # Everything published later is published AGAINST a commit, and the review + # takes minutes. A publisher that reads the head for itself reads it after + # the review, so a push mid-review attaches this verdict -- and a green + # check -- to code the driver never saw. Several such reads also disagree + # with each other inside one run. + # + # This is the commit the driver is about to review, and the commit every + # publisher names. A push during the review moves the pull request's head. + # It does not move this. + sha="$(gh api "repos/$REPO/pulls/$PR" --jq .head.sha)" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "reviewing $REPO#$PR at $sha" + + - name: Read the nit setting from the pull request + id: nits + # The author of one pull request is who asks for a polish pass, so the ask + # is a label on that pull request rather than a setting in the caller. + # + # Its own step, and not part of the drive step below, because the read needs + # a GitHub token and that step's env reaches the driver process. Keeping the + # token here is what keeps the driver's credentials to omnigent alone. + # + # It costs one read of the pull request. The guard reads the same object for + # the skip label, under the App identity alone; this one has to answer a + # caller that configured no App as well, so it takes the identity above and + # its own call rather than a share of that one. + if: ${{ inputs.mode == 'review' && !cancelled() && inputs.nitpick-label != '' }} + # An opt-in for advice must not be able to stop a review. A step that fails + # leaves the output unset, the drive step reads that as off, and the review + # runs with nits off. + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ steps.identity.outputs.token || github.token }} + REPO: ${{ github.repository }} + PR: ${{ needs.guard.outputs.pr_number }} + NITPICK_LABEL: ${{ inputs.nitpick-label }} + run: | + set -euo pipefail + # A label that is absent and a read that failed both leave nits off, so the + # warning below is the only thing that tells them apart in the log. It is + # where a polish pass that was asked for and did not happen is explained. + # + # Off, and not a refusal, because this label asks for advice rather than + # withholding work. The guard refuses on a signal it could not read, because + # admitting there runs a review it should not have run. Admitting here runs + # the review that was asked for, carrying one fewer kind of finding. + read_ok=true + # shellcheck disable=SC2016 # $ENV is jq's own, and jq reads it + carries="$(gh api "repos/$REPO/pulls/$PR" \ + --jq 'any(.labels[]?.name; . == $ENV.NITPICK_LABEL)')" || read_ok=false + if [ "$read_ok" = "false" ]; then + echo "::warning::could not read the labels on $REPO#$PR, so $NITPICK_LABEL cannot be found; this review runs without nits" + fi + if [ "$carries" != "true" ]; then carries=false; fi + echo "include_nits=$carries" >> "$GITHUB_OUTPUT" + echo "include nits on $REPO#$PR: $carries" + + - name: Drive session + collect verdict + id: drive + shell: bash + env: + BIN: ${{ steps.build.outputs.bin }} + OMNIGENT_BASE_URL: ${{ inputs.omnigent-base-url }} + SEIDROID_SCOUTS: ${{ inputs.scouts }} + SEIDROID_MODEL: ${{ inputs.claude-model }} + SEIDROID_ALLOW_TOOLS: ${{ inputs.allow-tools }} + MODE: ${{ inputs.mode }} + # The reviewed pull request. The driver keys the session on this pair. + REPO: ${{ github.repository }} + # The guard supplies this for a review; a close event carries its own + # number and skips the guard entirely. + PR: ${{ needs.guard.outputs.pr_number || github.event.pull_request.number }} + # trigger_id, not comment_id: this labels the dispatch in the driver's log + # and nothing else, so it wants the id whatever the event. comment_id is the + # reactable-object id and goes out empty where nothing can react on it, + # which would leave a review-body dispatch with no label at all. + TRIGGER_ID: ${{ needs.guard.outputs.trigger_id }} + # The findings this reviewer left before, so it drops what the author has + # addressed and keeps what the diff still shows. No token rides with it: + # this step reaches GitHub through nothing, which is the boundary that + # keeps the driver's credentials to omnigent alone. + THREADS: ${{ steps.threads.outputs.threads_path }} + GUIDELINES_FILE: ${{ inputs.guidelines-file }} + EXTRA_INSTRUCTIONS: ${{ inputs.extra-instructions }} + INCLUDE_NITS: ${{ steps.nits.outputs.include_nits }} + run: | + set -euo pipefail + # OMNIGENT_MACHINE_CLIENT_ID/SECRET are already in the job env (see + # above) -- the driver mints its own bearer in-process from them and + # that token never transits a workflow step output. + # + # The verdict is written to a directory this run owns and clears first, + # never to the workspace. actions/checkout cleans .driver and nothing + # else, so a workspace-relative verdict.md left by an earlier run -- + # attempt 2 of this one, or any run on a non-ephemeral self-hosted + # runner, which is the steady state here -- satisfies the non-empty + # check below and gets posted as this review. + out_dir="$RUNNER_TEMP/review-out/$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + rm -rf "$out_dir" + mkdir -p "$out_dir" + verdict="$out_dir/verdict.md" + echo "verdict_path=$verdict" >> "$GITHUB_OUTPUT" + # Written only when the review has observations that name a file and a + # line, so its absence means "summary only" rather than a failure. + findings="$out_dir/findings.json" + echo "findings_path=$findings" >> "$GITHUB_OUTPUT" + # Written whenever the review reached a verdict, including a clean one: + # a checks list with no review entry reads as a review that did not run. + check="$out_dir/check.json" + echo "check_path=$check" >> "$GITHUB_OUTPUT" + + if [ "$MODE" = "close" ]; then + args=(review "$REPO" "$PR" --close) + else + args=(review "$REPO" "$PR" --out "$verdict" --findings-out "$findings" + --check-out "$check") + # Absent when the read above failed, and the driver reads that as a + # first review rather than an error. + if [ -s "${THREADS:-}" ]; then args+=(--conversation-context "$THREADS"); fi + # Passed only when set, so the driver's own defaults stay the one + # place either is decided. + if [ -n "${GUIDELINES_FILE:-}" ]; then + args+=(--guidelines-file "$GUIDELINES_FILE") + fi + if [ -n "${EXTRA_INSTRUCTIONS:-}" ]; then + args+=(--extra-instructions "$EXTRA_INSTRUCTIONS") + fi + # Off means redirected rather than dropped: without this flag the driver + # sends a nit-grade finding to the non-blocking bucket, so it reaches the + # reader in prose and opens no thread on the line. + if [ "${INCLUDE_NITS:-}" = "true" ]; then args+=(--include-nits); fi + if [ -n "${TRIGGER_ID:-}" ]; then args+=(--trigger-id "$TRIGGER_ID"); fi + fi + set +e + "$BIN" "${args[@]}" + rc=$? + set -e + # Downstream gates on whether a verdict was PRODUCED (verdict.md + # non-empty), not on the exit code, so a review that reached one still + # publishes whatever else failed, and a no-verdict run never posts a + # placeholder. + if [ "$MODE" = "close" ]; then + # A close deletes a session; it has no verdict to produce, so the + # check below does not apply to it. Its own failure is the exit code, + # and it matters: this is the only thing that reclaims a sandbox, and + # nothing downstream notices when it does not. + echo "verdict_produced=false" >> "$GITHUB_OUTPUT" + if [ "$rc" -ne 0 ]; then + echo "::error::close failed (exit $rc) — the session and its sandbox are still running" + fi + elif [ -s "$verdict" ]; then + echo "verdict_produced=true" >> "$GITHUB_OUTPUT" + if [ "$rc" -ne 0 ]; then + echo "::warning::review exited $rc but produced a verdict (e.g. a teardown leak); see logs" + fi + else + echo "verdict_produced=false" >> "$GITHUB_OUTPUT" + echo "::error::review produced no verdict (exit $rc)" + # A review that produced nothing fails the job, whatever the driver + # exited. An annotation on a green job is a review nobody knows did not + # happen, and every step that reports one runs on !cancelled() rather + # than on success. 5 is the driver's own code for this. + if [ "$rc" -eq 0 ]; then rc=5; fi + fi + exit "$rc" + + - name: Place findings on the code + id: place + # Placement degrades in two steps rather than dropping a finding. The API + # accepts a line only where the diff covers it, and a review that reads the + # files around the diff -- which the prompt asks for, because a change can + # be locally correct and globally wrong -- cites both lines outside the + # hunks and files the pull request never touches. An uncovered line goes on + # its file; an untouched file has nowhere to go and is named in the summary. + # So the cost of a review that sees past the hunks is paid in placement, + # not in lost findings. + # + # Every finding the diff can hold rides in ONE review, which is what + # ai-review posts and what the author reads top to bottom on a single + # notification. Only the middle rung costs a call of its own: the reviews + # API carries a line comment and has no field for a file-level one. + # + # That review is COMMENTED and its body opens with no verdict marker, so the + # withdrawal below -- which selects a CHANGES_REQUESTED review whose body + # opens with VERDICT_MARKER -- does not select it, and the position this run + # takes stays the position step's alone. + # + # Each comment's superseded thread ids are recorded as that comment posts, and + # the resolve step closes a thread only on finding its id in that record. So a + # review superseding three threads closes the ones whose own replacement reached + # the code and leaves the rest standing. A comment that degraded to the file rung + # counts as a replacement, for the reason that rung gives; the summary does not. + if: ${{ inputs.mode == 'review' && !cancelled() + && steps.drive.outputs.verdict_produced == 'true' }} + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ steps.identity.outputs.token || github.token }} + REPO: ${{ github.repository }} + PR: ${{ needs.guard.outputs.pr_number }} + REVIEWED_SHA: ${{ steps.head.outputs.sha }} + FINDINGS: ${{ steps.drive.outputs.findings_path }} + # Findings that reached neither a line nor a file, collected for the + # summary. Declared here so the step below can read it by output. + NOTE: ${{ runner.temp }}/review-unplaced.md + # One superseded thread id per line, appended as the comment replacing it + # posts. Declared here so the resolve step can read it by output. + LINKAGE: ${{ runner.temp }}/review-superseded-placed.txt + run: | + set -euo pipefail + echo "note_path=$NOTE" >> "$GITHUB_OUTPUT" + : > "$NOTE" + # Emptied before anything is appended. RUNNER_TEMP survives a re-run on a + # non-ephemeral self-hosted runner, which is the steady state here, and a + # record left by attempt 1 would close a thread on attempt 2's behalf. + echo "linkage_path=$LINKAGE" >> "$GITHUB_OUTPUT" + : > "$LINKAGE" + if [ ! -s "$FINDINGS" ]; then + echo "no findings to place; the summary carries the review" + # Zero, and not silence. Nothing was placed because there was nothing to + # place, which is a number the summary can state. A step that was skipped + # writes nothing at all, and the summary tells the two apart by that. + # + # superseded_linked says whether the driver named which comment replaces + # which thread, and the resolve step picks its gate on it. Every path that + # returns before reading a finding says false, so the answer to a question + # this run never asked is the one that keeps a thread open. + { echo "on_line=0"; echo "on_file=0"; echo "unplaced=0" + echo "superseded_linked=false"; } >> "$GITHUB_OUTPUT" + exit 0 + fi + # The commit the review read, recorded before it started and used as + # recorded. A comment on any other commit points at code the review never + # saw, so the head is not read again here. + # + # Empty when that record failed. Every call below needs a commit id and the + # API rejects an empty one, so no comment can reach the diff and every + # finding takes the third rung of the ladder above: the summary. The reader + # loses the placement, not the finding. + head_sha="${REVIEWED_SHA:-}" + if [ -z "$head_sha" ]; then + echo "::warning::the reviewed commit was not recorded on $REPO#$PR; every finding goes to the summary instead of the diff" + fi + on_line=0 on_file=0 unplaced=0 + + # The third rung, in two groups. A finding reaches the summary because the + # diff has nowhere to attach it, or because the diff does carry its line and + # the call that would have posted it failed. The summary tells a reader which, + # so the second group is collected apart and headed for what it is. Both count + # as unplaced: neither reached the code. + # + # Each line opens with a dash and a backtick, which is what the summary counts + # when it cuts the list to fit GitHub's limit. + ondiff="$RUNNER_TEMP/review-unplaced-on-diff.md" + : > "$ondiff" + # shellcheck disable=SC2016 # the backticks are markdown, not a substitution + to_summary() { + printf -- '- `%s:%s` (%s) — %s\n' "$1" "$2" "$4" "$5" >> "$NOTE" + unplaced=$((unplaced+1)) + return 0 + } + # shellcheck disable=SC2016 # the backticks are markdown, not a substitution + to_summary_ondiff() { + printf -- '- `%s:%s` (%s) — %s\n' "$1" "$2" "$4" "$5" >> "$ondiff" + unplaced=$((unplaced+1)) + return 0 + } + + # The threads one comment replaced, recorded as that comment posts. The resolve + # step closes a superseded thread on finding its id here and on nothing else, + # so an id written for a comment that did not post takes a live finding off the + # pull request. Neither summary collector records: the summary is not the code. + # + # A dot is a finding that replaces nothing, and no node id can be one -- the + # alphabet GitHub mints from has no dot in it. The ids are space-separated and + # split back onto their own lines, because the record is read line by line. + record_superseded() { + [ "$1" = "." ] && return 0 + printf '%s\n' "$1" | tr ' ' '\n' >> "$LINKAGE" + return 0 + } + + # The second rung, one call per finding. A file-level comment is a field + # the reviews API does not carry, so this one cannot ride in the batch + # below. The file can still be in the pull request, and a comment on it + # reaches the reviewer in the file they are already reading, so the cited + # line rides in the body instead. + # + # Guarded on the commit, not left to the API. Without one the call returns + # 422, and asking for an answer known before the call spends the rate limit + # on it. + # + # $6 is which summary group takes the finding if this rung refuses it too. + # The caller knows whether the diff carries the line; this does not. + on_file_or_summary() { + if [ -n "$head_sha" ] && gh api -X POST "repos/$REPO/pulls/$PR/comments" \ + -f body="$FINDING_MARKER"$'\n'"**$4** — $5"$'\n\n'"_Cited at \`$1:$2\`, outside this diff's changed lines._" \ + -f commit_id="$head_sha" -f path="$1" \ + -f subject_type=file >/dev/null 2>&1; then + on_file=$((on_file+1)) + # A file comment replaces the thread. It is on the diff, in the file the + # reader is already in, and its body carries the cited line. The thread it + # replaces is one whose line this diff no longer covers -- which is why + # this rung was reached -- so not closing it leaves a duplicate for good. + record_superseded "$6" + return 0 + fi + "$7" "$1" "$2" "$3" "$4" "$5" "$6" + return 0 + } + + # All three rungs for one finding, with the API deciding which one it + # takes. This is the path for a finding the batch could not carry. + place_one() { + if [ -n "$head_sha" ] && gh api -X POST "repos/$REPO/pulls/$PR/comments" \ + -f body="$FINDING_MARKER"$'\n'"**$4** — $5" \ + -f commit_id="$head_sha" -f path="$1" \ + -F line="$2" -f side="$3" >/dev/null 2>&1; then + on_line=$((on_line+1)) + record_superseded "$6" + return 0 + fi + on_file_or_summary "$@" + return 0 + } + + # The detail is base64 per record, not @tsv. Finding.Detail is raw model + # prose with no line constraint on it, and @tsv escapes a newline, a tab or + # a backslash into a literal \n, \t or \\ -- so a multi-line detail reached + # the pull request showing its escape sequences instead of its text. The + # five fields that cannot contain a tab stay plain. + # + # The supersedes list rides ahead of the detail rather than after it, and the + # order the fields are READ in is not the order they are passed in. Only the + # last field may be empty; see the reader below. A finding replacing nothing + # writes a dot, so the list is never the empty field either. + tsv='[.file, .line, .side, .severity, + (.supersedes | if length == 0 then "." else join(" ") end), + (.detail | @base64)] | @tsv' + rows() { jq -r "$1 | $tsv" "$2"; } + # Redirected into, never piped into. The counters the summary reads are + # this shell's, and a pipe would count them in a subshell that exits. + place_each() { + local placer="$1" summary="$2" + local path line side severity supersedes detail_b64 detail + while IFS=$'\t' read -r path line side severity supersedes detail_b64; do + [ -z "$path" ] && continue + detail="$(printf '%s' "$detail_b64" | base64 --decode)" + "$placer" "$path" "$line" "$side" "$severity" "$detail" \ + "$supersedes" "$summary" + done + return 0 + } + + # One shape for every path below: a string file, an integer line, a side + # the API names. A line the model wrote as prose reads as 0, which no diff + # covers, so it takes the file rung rather than reaching gh as a field. + # + # Every field but the last is non-empty here, and that is what the reader + # below rests on. @tsv writes an empty field as nothing between two tabs, + # bash reads a tab as IFS whitespace and folds the pair into one delimiter, + # and the record then reads one field short: the severity arrives in the + # side, the base64 detail arrives in the severity, and a finding reaches the + # pull request with its own encoding printed as its severity. + # + # supersedes is the threads this finding's comment replaces. The driver admits + # every id against the threads it was handed, and this admits the SHAPE again: + # the record is read line by line and each id is matched whole, so an id + # carrying a space or a newline would split into two the resolve step then + # cannot find. Dropping one leaves its thread open, which is the safe end. + # A driver that publishes no linkage yields the empty list on every finding. + normalise='[ .[] | { file: (.file | tostring), + line: (.line | if type == "number" then floor + elif type == "string" then ((. | tonumber?) // 0 | floor) + else 0 end), + side: ((.side // "RIGHT") | tostring | ascii_upcase + | if . == "LEFT" then "LEFT" else "RIGHT" end), + severity: (.severity | tostring + | if . == "" then "note" else . end), + supersedes: [ (.supersedes // [])[] + | select(type == "string") + | select(test("^[A-Za-z0-9_=+/-]{1,200}$")) ], + detail: (.detail | tostring) } + | select(.file != "") ]' + normal="$RUNNER_TEMP/review-findings-normalised.json" + if ! jq "$normalise" "$FINDINGS" > "$normal"; then + echo "::warning::the findings file for $REPO#$PR could not be read, so this review places nothing on the diff" + { echo "on_line=0"; echo "on_file=0"; echo "unplaced=0" + echo "superseded_linked=false"; } >> "$GITHUB_OUTPUT" + exit 0 + fi + # One finding naming a thread is the whole test. An older driver publishes the + # key on none of them, and a review that supersedes nothing publishes it on + # none either -- and in that case there is no superseded thread for either gate + # to decide, so the two answers are the same one. + if jq -e 'any(.[]; (.supersedes | length) > 0)' "$normal" >/dev/null; then + echo "superseded_linked=true" >> "$GITHUB_OUTPUT" + else + echo "superseded_linked=false" >> "$GITHUB_OUTPUT" + fi + + # Which lines a comment can name, walked out of the diff the fetch below + # reads, before anything is posted. RIGHT takes an added or a context line, LEFT + # takes a removed or a context one, and ai-review walks the same patches the + # same way. The walk covers only the files the findings name, so its size is + # the finding count and not the diff's. + # + # Three groups, because a file the API sends without a patch -- a binary one, + # or one whose diff was too large to send -- has unknown lines rather than no + # lines. Counting it as no lines would tell an author their cited line is + # outside the diff when it is in it. Those findings stay out of the batch and + # go to the API one at a time, which is the only thing that can still say. + # + # Read ahead of the call because the call is all-or-nothing: one line the API + # refuses loses every comment with it, so a refusal has to be predicted here + # rather than met there. + # shellcheck disable=SC2016 # $findings and friends are jq's, and single quotes are what keep them jq's + partition='def hunks: + reduce ((.patch // "") | split("\n"))[] as $l + ({o: 0, n: 0, R: {}, L: {}}; + if ($l | test("^@@ -[0-9]+(,[0-9]+)? [+][0-9]+(,[0-9]+)? @@")) then + ($l | capture("^@@ -(?[0-9]+)(,[0-9]+)? [+](?[0-9]+)(,[0-9]+)? @@")) as $m + | .o = ($m.o | tonumber) | .n = ($m.n | tonumber) + elif ($l | startswith("+")) then .R[.n | tostring] = true | .n += 1 + elif ($l | startswith("-")) then .L[.o | tostring] = true | .o += 1 + elif ($l | startswith(" ")) then + .R[.n | tostring] = true | .L[.o | tostring] = true | .n += 1 | .o += 1 + else . end) + | {RIGHT: .R, LEFT: .L}; + (reduce $findings[0][] as $f ({}; .[$f.file] = true)) as $wanted + | ([ .[] | select($wanted[.filename // ""] // false) + | {key: .filename, + value: (if has("patch") then hunks else null end)} ] | from_entries) as $index + | ($findings[0] + | map(. as $g | ($index[$g.file]) as $h + | . + { ok: (($h[$g.side][$g.line | tostring]) // false), + unknown: (($index | has($g.file)) and ($h == null)) })) + | { anchored: [ .[] | select(.ok) | del(.ok, .unknown) ], + unknown: [ .[] | select(.ok | not) | select(.unknown) | del(.ok, .unknown) ], + loose: [ .[] | select(.ok | not) | select(.unknown | not) | del(.ok, .unknown) ] }' + files="$RUNNER_TEMP/review-pr-files.json" + placement="$RUNNER_TEMP/review-placement.json" + # The diff at the commit this review read, which is the commit every comment + # below names. GET /pulls/{n}/files answers for the pull request's current + # head and takes no commit: it accepts a sha parameter and ignores it, + # measured. So a push mid-review would index one commit and comment on + # another -- and a line marked from the newer diff that the older commit + # cannot carry costs the whole batch a 422. compare takes a commit. + # + # base.sha with three dots, so the API resolves the merge base itself and + # answers with the diff the pull request shows. A base branch that moves + # during the review does not move that merge base, because the reviewed + # commit is fixed and the fork point with it. + # + # The pull request itself, read once for two fields: the commit its diff + # starts from, and how many files that diff has. + # + # Both tested for shape, and not merely for emptiness. A read that fails + # leaves the API's error object in the file, and a field that object does + # not carry reads as null -- which would otherwise reach the API as a + # commit id. + is_count() { case "${1:-}" in (''|*[!0-9]*) return 1 ;; esac; } + pull="$RUNNER_TEMP/review-pull.json" + base_sha="" changed_files="" + if [ -n "$head_sha" ]; then + if gh api "repos/$REPO/pulls/$PR" > "$pull" 2>/dev/null; then + base_sha="$(jq -r '.base.sha // "" | tostring' "$pull" 2>/dev/null || true)" + changed_files="$(jq -r '.changed_files // "" | tostring' "$pull" 2>/dev/null || true)" + fi + case "$base_sha" in (''|*[!0-9a-f]*) base_sha="" ;; esac + is_count "$changed_files" || changed_files="" + if [ -z "$base_sha" ]; then + echo "::warning::the base commit of $REPO#$PR could not be read, so the diff at $head_sha cannot be; each finding is posted on its own" + fi + fi + # compare sends at most 300 files and drops the rest without saying so: no + # total, no Link header for them, no flag. Its pages are pages of commits, + # and a second page carries no files key at all, so there is nothing to page + # for. Measured: a 398-file pull request answers with 300. + # + # A short list cannot be told apart from a whole one, and a file missing + # from it looks exactly like a file the pull request never touched. Reading + # it as the diff would put a finding on its file under a body telling the + # author their cited line is outside a diff that holds it. So the list is + # counted against the pull request's own total, and a short one indexes + # nothing: every finding then goes to the API one at a time, which is the + # only thing that can still say where it belongs. + # + # 300 is that cap, so a list of 300 is at it. That test only has to carry + # the case where the total could not be read, because the total is exact. + build_index() { + local n + if gh api "repos/$REPO/compare/$base_sha...$head_sha?per_page=100" > "$files.raw" \ + && jq '[ (.files // [])[] ]' "$files.raw" > "$files" \ + && n="$(jq 'length' "$files")" && is_count "$n"; then + if [ -n "$changed_files" ] && [ "$n" -ne "$changed_files" ]; then + echo "::warning::the diff of $REPO#$PR at $head_sha came back with $n of its $changed_files file(s), so this run does not read it as the diff; each finding is posted on its own" + return 1 + fi + if [ -z "$changed_files" ] && [ "$n" -ge 300 ]; then + echo "::warning::the diff of $REPO#$PR at $head_sha came back with $n file(s), which is all this endpoint sends, and the pull request's own total could not be read; each finding is posted on its own" + return 1 + fi + jq --slurpfile findings "$normal" "$partition" "$files" > "$placement" && return 0 + fi + echo "::warning::the diff of $REPO#$PR at $head_sha could not be read, so each finding is posted on its own; the review still reaches the code" + return 1 + } + batched=false + if [ -n "$head_sha" ] && [ -n "$base_sha" ] && build_index; then + batched=true + fi + + if [ "$batched" = true ]; then + anchored="$(jq '.anchored | length' "$placement")" + if [ "$anchored" -gt 0 ]; then + # The marker opens every comment body, as the first bytes. The history + # read and the resolve step both recognise this tool's own findings by + # that, and a body that merely contains it matches neither. + request="$RUNNER_TEMP/review-placement-request.json" + response="$RUNNER_TEMP/review-placement-response.json" + review_body="Findings on the changed lines. The verdict and the summary are in this tool's comment on this pull request." + # shellcheck disable=SC2016 # $sha and friends are jq's, bound by --arg + jq --arg sha "$head_sha" --arg marker "$FINDING_MARKER" --arg body "$review_body" \ + '{commit_id: $sha, event: "COMMENT", body: $body, + comments: [ .anchored[] + | {path: .file, line: .line, side: .side, + body: "\($marker)\n**\(.severity)** — \(.detail)"} ]}' \ + "$placement" > "$request" + # Two failures, two answers. A 4xx is the API refusing the request and + # it creates nothing, so each finding is posted on its own and the one + # comment that was refused costs only itself. This request carries every + # finding's whole detail, which is model prose under no length bound, so + # the size of it is refused as readily as the content: 413 and 422 are + # the same answer here, and so is a 403 the retry can still get past one + # comment at a time. + # + # Any other failure may be a write that landed and then lost its + # connection, and repeating it posts the review twice -- so those + # findings take the summary, where the reader still gets every one of + # them. + # + # The code comes from the response's own status line, which -i puts + # first, and not from a status field in the error body. GitHub's + # validation-error schema declares no such field, and a refusal that + # carries none would read as no refusal at all and send every anchorable + # finding to the summary -- worse than placing each one by hand, which + # is what this rung is for. + # + # A call that reached no response leaves no status line, and an empty + # code takes the summary. That is the case where the write may have + # landed, so it is the case that must not be repeated. + if gh api -i -X POST "repos/$REPO/pulls/$PR/reviews" --input "$request" > "$response"; then + on_line=$((on_line + anchored)) + # The call carries every anchored comment and creates all of them or + # none, so one write records the whole batch's linkage. + jq -r '.anchored[] | .supersedes[]' "$placement" >> "$LINKAGE" + echo "posted one review carrying $anchored comment(s) on $REPO#$PR" + else + status="$(sed -n '1s|^HTTP/[0-9.]* \([0-9][0-9][0-9]\).*|\1|p' "$response" || true)" + case "$status" in + 4??) + echo "::warning::$REPO#$PR refused the review carrying $anchored comment(s) with $status; each finding is posted on its own instead" + place_each place_one to_summary_ondiff < <(rows '.anchored[]' "$placement") ;; + *) + echo "::warning::the review carrying $anchored comment(s) could not be posted on $REPO#$PR; those findings are in the summary instead of on the diff" + place_each to_summary_ondiff '' < <(rows '.anchored[]' "$placement") ;; + esac + fi + fi + place_each place_one to_summary < <(rows '.unknown[]' "$placement") + place_each on_file_or_summary to_summary < <(rows '.loose[]' "$placement") + else + place_each place_one to_summary < <(rows '.[]' "$normal") + fi + # A heading per reason, because the summary collects findings for three and + # only one of them is about code the reader did not change. With no commit to + # attach to, a finding on a changed line lands here as well; so does one whose + # review the API would not take. Filing either under "off the changed lines" + # tells the author the wrong thing about their own diff, and the second is the + # one that hides a real issue behind a heading that denies it. + if [ -s "$ondiff" ] || [ -s "$NOTE" ]; then + if [ -n "$head_sha" ]; then + header='**Observations off the changed lines.** These are about code this pull request does not touch, so there is nowhere in the diff to attach them:' + else + header='**Every finding is here.** The commit under review was not recorded, so none of these could be attached to a line of the diff:' + fi + { printf -- '---\n' + if [ -s "$ondiff" ]; then + printf -- '\n%s\n\n' '**On the changed lines, and not posted.** GitHub would not take the review carrying these, so they are here rather than on the lines they name:' + cat "$ondiff" + fi + if [ -s "$NOTE" ]; then + printf -- '\n%s\n\n' "$header" + cat "$NOTE" + fi + } > "$NOTE.tmp" + mv "$NOTE.tmp" "$NOTE" + fi + # Here and at the two early exits above, and nowhere between: the points + # where these are final. A run that dies in between leaves them unwritten, + # which is the right answer there -- placement neither finished nor was + # skipped, so no number it could publish would be true. + # + # One append for all three, so the summary never reads a half-written set. It + # requires all three for that reason, including the one no term renders. + { echo "on_line=$on_line"; echo "on_file=$on_file"; echo "unplaced=$unplaced"; } >> "$GITHUB_OUTPUT" + echo "findings: $on_line on a line, $on_file on a file, $unplaced in the summary" + + - name: Publish the review check run + # The half of a review a reader sees without opening it. `review` is the check + # of record. The same verdict goes out under `AI Review` too, for the reason + # the second publish below states. + # + # Whichever check the driver wrote, decided or not. A run that reached no + # verdict writes one under the title `no verdict`, naming why, and publishing + # that is the point of dropping the verdict gate here: with nothing in the + # checks list, a review that ran and could not be read is indistinguishable + # from one that never ran, and only one of the two is a reason to look. The + # file's presence is the gate, so a run that never reached the driver still + # publishes nothing. + # + # That check concludes `failure`, so a repository requiring it holds the merge + # until a run reads the change. Read the title, not the conclusion, to tell the + # two apart: a decided review carrying blockers concludes `failure` too. + # + # Tolerated while there is a commit to publish against, like the other + # publish steps: a check run that fails to post must not bury a review that + # was produced, and the steps below run on !cancelled() and still publish it. + # + # Not tolerated when the reviewed commit is missing. head_sha is required and + # has no default, so that is not a post that failed and may work next time -- + # it is a merge gate that cannot exist, and a green job hides it. + if: ${{ inputs.mode == 'review' && !cancelled() }} + continue-on-error: ${{ steps.head.outputs.sha != '' }} + shell: bash + env: + GH_TOKEN: ${{ steps.identity.outputs.token || github.token }} + REPO: ${{ github.repository }} + PR: ${{ needs.guard.outputs.pr_number }} + REVIEWED_SHA: ${{ steps.head.outputs.sha }} + CHECK: ${{ steps.drive.outputs.check_path }} + AI_REVIEW_COPY: ${{ inputs.publish-ai-review-check }} + # The one field this step will not read out of the artifact. See below. + VERDICT_PRODUCED: ${{ steps.drive.outputs.verdict_produced }} + run: | + set -euo pipefail + if [ ! -s "$CHECK" ]; then + echo "no check run to publish" + exit 0 + fi + # A run that produced no verdict publishes failure, whatever the file says. + # The file is written by a separate process across a repository boundary, so + # reading a passing conclusion out of it on a run this workflow already knows + # produced nothing would let one crash between two writes satisfy a required + # check. This step decides the one thing that gates a merge; it reads the + # artifact for everything else. + conclusion="$(jq -r '.conclusion // empty' "$CHECK" || true)" + if [ "${VERDICT_PRODUCED:-}" != "true" ]; then + conclusion="failure" + fi + if [ -z "$conclusion" ]; then + echo "::error::the check file on $REPO#$PR carries no conclusion, so there is nothing to publish as the merge gate" + exit 1 + fi + # The same default again. A jq that fails leaves the title empty, and the `//` + # above only covers a field that is absent. The summary needs no such line, + # because its default is the empty string either way. + title="$(jq -r '.title // "review"' "$CHECK" || true)" + title="${title:-review}" + summary="$(jq -r '.summary // ""' "$CHECK" || true)" + # Against the commit the review read, recorded before it started and used + # as recorded. A check on any other commit attaches this verdict to code the + # review never saw, so the head is not read again here. + # + # Empty when that record failed. There is no weaker check run to publish in + # its place, so this says what is missing and fails; the step comment above + # says why that reaches the job. + head_sha="${REVIEWED_SHA:-}" + if [ -z "$head_sha" ]; then + echo "::error::the reviewed commit was not recorded on $REPO#$PR, so the review check run cannot be published; the verdict comment is the only record of this review" + exit 1 + fi + publish() { + gh api -X POST "repos/$REPO/check-runs" \ + -f name="$1" \ + -f head_sha="$head_sha" \ + -f status=completed \ + -f conclusion="$conclusion" \ + -f output[title]="$title" \ + -f output[summary]="$summary" >/dev/null + } + # The check of record, and the only publish here that can fail the step. A + # failure exits it under set -e, and continue-on-error above decides what + # that costs the job. + publish review + names="check run review" + # The same verdict under the name ai-review.yml publishes. A branch-protection + # rule matches a check by its name, so only a check called `AI Review` + # satisfies a rule that requires `AI Review`. The rules on the calling + # repositories are org-level, and no token here can read them. This step + # therefore publishes both names, and a rule that requires either one passes. + # Delete this call once someone who can read those rules confirms that none + # names `AI Review`. + # + # Best-effort, and that is the whole difference from the publish above. A + # failure here warns and the step still succeeds, so the copy never decides + # whether the review reached the pull request. + # + # A repository that also runs ai-review.yml under this App gets two checks + # called `AI Review` on the commit. GitHub keeps both and lists one: the one + # that completed last. The two publishes therefore race for the name, and + # `publish-ai-review-check` is how such a repository stands this copy down. + if [ "${AI_REVIEW_COPY:-}" = "true" ]; then + if publish "AI Review"; then + names="check runs review and AI Review" + else + echo "::warning::the AI Review copy of the review check could not be published on $REPO#$PR; review carries the verdict" + fi + fi + echo "published $names: $conclusion — $title" + + - name: State the review's position on the pull request + # The check run is the gate a merge reads; this is the one a person reads, + # and the one that shows in the reviewers list. Both come from the same + # conclusion the driver derived from the findings. + # + # Only when there is a position to take. A clean run with approval off, or + # a run that concluded neutral, adds nothing a reader does not already have + # from the comment above — and a review cannot be edited later the way that + # comment is upserted, so an opinionless one is permanent clutter. + # + # A review that reached no verdict has no position at all, and verdict_produced + # is what holds that. It is load-bearing twice over: the withdrawal at the end + # of this step reads the driver's finding counts, the no-verdict check carries + # none, and the no-counts branch there proceeds by design. So an ungated + # no-verdict run would clear a standing block on the strength of a review that + # produced nothing. The check run and the comment publish without this gate; + # anything that clears a merge gate keeps it. + # + # No continue-on-error, for the withdrawal at the end. That is the only work + # in this job that clears a merge gate, and a gate left standing on a green + # run is a block nobody knows to look for. Every other failure here is caught + # at its own site and stated there, so an unreadable check file, a position + # the API refuses and a courtesy log line all stay quiet. What reaches the job + # is the withdrawal. + # + # The App records the position where the caller configured one, and the + # workflow's own identity records it otherwise, so a repository with no App + # still takes a position and can still clear a block it left before. That + # identity reads as github-actions[bot], which costs two things a caller has + # to know: a merge gate that names its reviewers may not count it, and a + # repository whose Actions policy forbids an approval refuses the event + # outright. The COMMENT rung below answers the second. + if: ${{ inputs.mode == 'review' && !cancelled() + && steps.drive.outputs.verdict_produced == 'true' }} + shell: bash + env: + GH_TOKEN: ${{ steps.identity.outputs.token || github.token }} + REPO: ${{ github.repository }} + PR: ${{ needs.guard.outputs.pr_number }} + REVIEWED_SHA: ${{ steps.head.outputs.sha }} + CHECK: ${{ steps.drive.outputs.check_path }} + APPROVE_ON_SUCCESS: ${{ inputs.approve-on-success }} + run: | + set -euo pipefail + if [ ! -s "$CHECK" ]; then + echo "no conclusion to take a position from" + exit 0 + fi + # jq's error goes to the log and the value comes back on stdout, so a check + # file this step cannot read states itself rather than arriving as data. + # + # `// empty` rather than a bare field. A missing conclusion renders as the + # string "null", and every comparison below takes that for a conclusion. + conclusion="$(jq -r '.conclusion // empty' "$CHECK" || true)" + if [ -z "$conclusion" ]; then + echo "::warning::the check file names no conclusion; no position to take on $REPO#$PR" + exit 0 + fi + + # The commit the review read, recorded before it started and used as + # recorded. The head is not read again here, or in any other publisher: one + # read is the state this whole run publishes against. + # + # Empty only when that record failed. commit_id is optional on the reviews + # API and defaults to the pull request's latest commit, so the position + # still lands -- the weaker guarantee, stated rather than assumed. This is + # the one publisher that has a weaker version to fall back to. + head_sha="${REVIEWED_SHA:-}" + if [ -z "$head_sha" ]; then + echo "::warning::the reviewed commit was not recorded; the position goes on the current head of $REPO#$PR, which the review may not have read" + fi + + event="" + note="" + if [ "$conclusion" = "failure" ]; then + event=REQUEST_CHANGES + note="review found something blocking. The findings are on the lines they are about, and the summary is in this tool's comment on this pull request." + elif [ "$conclusion" = "success" ] && [ "$APPROVE_ON_SUCCESS" = "true" ]; then + event=APPROVE + note="review found nothing blocking." + fi + + # commit_id is omitted rather than sent empty when the commit is unknown: + # the API rejects an empty one, and its own default is the pull request's + # latest commit, which is the fallback announced above. + # + # 0 posted, 2 refused, 1 anything else. Only a refusal earns the rung below: + # a 422 is the API declining the opinion itself, and every other failure is + # one a second call would meet again. That keeps a write that landed and then + # lost its connection from posting a second review. + # + # 422, and not 403. GitHub answers a self-review with 422 -- measured, as + # "Review Can not approve your own pull request" -- and answers a secondary + # rate limit with 403 or 429. This endpoint creates content and notifies, so + # it is one the API names as tripping that limit. A rung that fired on 403 + # would turn a rate-limited REQUEST_CHANGES into a comment that blocks + # nothing, where one wait would have kept the block. The narrow rule costs + # the opposite case: a refusal that arrives as 403, such as an Actions policy + # that forbids an approval, earns no comment review and only the warning + # below. Losing the vote is the worse half, so the rule protects it. + # + # A 422 also covers a stale commit_id, which the rung retries with the same + # commit_id and meets identically. That costs one call and no review. + # + # The code comes from GitHub's error object, not from gh's message. gh writes + # the response to stdout and its own line to stderr, so a failed call leaves + # the error object in `body` -- and `status` on it is the code. gh's line is + # a human string outside its compatibility surface, and a reformat there + # would turn this rung into a silent no-op. + # + # `status` is documented on GitHub's Basic Error schema and measured on this + # endpoint's 422, but the 422's own schema does not declare it. So it can be + # absent, and an absent code returns 1: no rung, no downgrade. The vote + # survives a shape this cannot read, which is the same direction the 422 rule + # chooses above. + # + # gh's own line still reaches the log, where it says why. + post_position() { + local args body status + args=(-X POST "repos/$REPO/pulls/$PR/reviews" + -f event="$1" -f body="$VERDICT_MARKER"$'\n'"$note") + if [ -n "$head_sha" ]; then + args+=(-f commit_id="$head_sha") + fi + body="$(gh api "${args[@]}")" && return 0 + status="$(printf '%s' "$body" | jq -r '.status // empty' 2>/dev/null || true)" + if [ "$status" = "422" ]; then + return 2 + fi + return 1 + } + + if [ -n "$event" ]; then + # Guarded, because set -e would abort the step here and take the + # withdrawal below with it. A 422 on self-approval, a stale commit_id or + # a transient 5xx must not also cost the pull request its retraction. The + # API's own error stays on stderr, where it says why. + # + # COMMENT is the rung under both events, and the same one ai-review takes. + # The API refuses an opinion the identity may not hold -- github-actions + # approving where the repository's Actions policy forbids it, or either + # identity taking a side on a pull request it opened itself -- and refuses + # nothing about a review that states its finding without voting. So the + # reviewers list still carries this run, and the check run still carries + # the gate. What the rung costs is the vote: an APPROVE that lands here + # clears no approval requirement, and a REQUEST_CHANGES blocks nothing. + rc=0 + post_position "$event" || rc=$? + if [ "$rc" -eq 0 ]; then + echo "recorded $event on $REPO#$PR" + elif [ "$rc" -eq 2 ] && post_position COMMENT; then + echo "::warning::$REPO#$PR would not take $event, so this review is recorded as a comment; the check run carries the $conclusion conclusion" + else + echo "::warning::could not record $event on $REPO#$PR; the verdict comment stands" + fi + else + echo "no position to record for a $conclusion conclusion" + fi + + # An earlier block this run no longer stands behind. Left standing it + # keeps a pull request red for a finding that is gone, and only a human + # can clear it. + # + # Withdrawn on the two things this run has to be able to say: it read the + # change, and it found nothing blocking in it. A failure says the second is + # false and stands behind its own block. + if [ "$conclusion" = "failure" ]; then + exit 0 + fi + # success says both, because the driver gates it on the line count the review + # reported. neutral says one word over three states -- a review that could not + # show it read the diff, a review whose only blocker is already on the base + # branch, and a review whose findings are all non-blocking. Only the first has + # no ground to clear another review's finding, and the counts are what separate + # it: something written down is a review of the change, and nothing written down + # beside a soft conclusion is a review that did not happen. + # + # Read under `counts`, where the driver writes them, with a sentinel for the + # field it does not: a bare null is a value the arithmetic below would take for + # a zero, and a driver that reports nothing would then read as a review that + # found nothing. + is_count() { case "${1:-}" in (''|*[!0-9]*) return 1 ;; esac; } + blocking="" non_blocking="" pre_existing="" counted=false + read -r blocking non_blocking pre_existing < <(jq -r '[(.counts.blocking // "?"), + (.counts.non_blocking // "?"), (.counts.pre_existing // "?")] | @tsv' \ + "$CHECK" 2>/dev/null) || true + if is_count "$blocking" && is_count "$non_blocking" && is_count "$pre_existing"; then + counted=true + fi + + # A blocking finding beside a conclusion that does not fail is the driver + # disagreeing with itself, and the comment step states it as that. Here it + # stops the withdrawal: whichever half is wrong, clearing a merge gate while + # the review names a blocker is the one outcome that cannot be walked back. + if [ "$counted" = true ] && [ "$blocking" -gt 0 ]; then + echo "::warning::the review reports $blocking blocking finding(s) beside a $conclusion conclusion, so it does not clear an earlier block on $REPO#$PR" + exit 0 + fi + if [ "$conclusion" != "success" ] && [ "$counted" = true ] \ + && [ $(( blocking + non_blocking + pre_existing )) -eq 0 ]; then + echo "::notice::a $conclusion review that wrote nothing down does not clear an earlier block on $REPO#$PR" + exit 0 + fi + # A driver that writes no counts leaves the three states indistinguishable, and + # the withdrawal proceeds rather than stopping. Stopping strands the block + # behind every re-review whose findings are all non-blocking, which that driver + # also concludes neutral, and which is what most re-reviews find. + if [ "$conclusion" != "success" ] && [ "$counted" = false ]; then + echo "::warning::no finding counts beside a $conclusion conclusion on $REPO#$PR, so a review that found only notes reads the same as one that read nothing; the withdrawal proceeds on the conclusion alone. A driver from v0.11.0 writes the counts" + fi + # Listed into a variable, and read from it rather than through a pipe. A + # report written into a pipe the loop reads is not an annotation Actions + # sees: it is a line the loop takes for a review id, and the dismiss fails + # on it. Reporting and data have to travel separately. + # + # A list this step cannot read is a block it cannot find, which reads the + # same on the pull request as a block it failed to clear. + # + # The marker reaches jq through the environment, the way the guard passes + # it. A value carrying a quote or a backslash is data to jq here, where + # substituting it into the program would make it syntax. + # shellcheck disable=SC2016 # $ENV is jq's own, and jq reads it + if ! ids="$(gh api "repos/$REPO/pulls/$PR/reviews" --paginate \ + --jq '[.[] | select(.state == "CHANGES_REQUESTED" and ((.body // "") | startswith($ENV.VERDICT_MARKER)))] | .[].id')"; then + echo "::error::could not list the reviews to withdraw on $REPO#$PR; an earlier block may still stand and only a human can clear it" + exit 1 + fi + # Every id is tried before the step reports. Exiting inside the loop would + # leave a block standing that the next call would have cleared, and would + # name one stuck gate where there are two. + withdrawn=0 + stuck=0 + while read -r id; do + [ -n "$id" ] || continue + if gh api -X PUT "repos/$REPO/pulls/$PR/reviews/$id/dismissals" \ + -f message="Superseded: the latest review found nothing blocking in this change." \ + -f event=DISMISS >/dev/null; then + withdrawn=$((withdrawn+1)) + echo "withdrew review $id" + else + stuck=$((stuck+1)) + echo "::error::could not withdraw review $id on $REPO#$PR; it still blocks the merge on a finding this run did not reproduce. On a protected branch the API takes a dismissal only from a repository admin or an actor named in the branch's dismissal restriction, so check that this run's identity is one of them. A token without pull-requests: write is refused as well. Dismiss review $id by hand to clear it" + fi + done <<< "$ids" + echo "superseded blocks: $withdrawn withdrawn, $stuck still standing" + if [ "$stuck" -gt 0 ]; then + exit 1 + fi + + - name: Answer the request + # id: the withdrawal step at the end of this job reads this step's outcome to + # tell a run that already touched the trigger comment from one that never did. + id: answer + # The verdict, on the comment that asked for it, so the person who asked reads + # the outcome where they asked. The eyes at the top of this job say it started; + # this step withdraws them and says how it ended. + # + # Its own step, and not part of the one that records the position: that step + # is skipped for a run that reached no verdict, and the person who asked is + # owed an answer even then. Everything here needs only GITHUB_TOKEN and the + # job's issues: write. + # + # The condition names three facts: a review turn, not cancelled, and a comment + # to answer. A close produces no verdict, so a request to tear a session down + # earns no answer. + # + # !cancelled() is what keeps a thumb honest, and it is not interchangeable with + # always() here. A cancellation arriving after `drive` finishes leaves + # check_path and verdict_produced populated, so this step would read a real + # conclusion and thumb the request -- while `Post the verdict` below skips and + # the verdict never reaches the pull request. A thumb reads as an answer, so + # that is worse than no reaction at all. The last step of this job withdraws + # the eyes a cancelled run leaves, and it cannot reach a conclusion to post. + # + # No verdict_produced gate: the acknowledgement waits on this step, and a re-run + # replays the trigger comment id, so a run reaching no verdict still has to clear + # the eyes, and a thumb an earlier attempt left there. + if: ${{ inputs.mode == 'review' && !cancelled() + && needs.guard.outputs.comment_id != '' }} + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # The repository the request was written on, which is the one the review + # ran on. + TRIGGER_REPO: ${{ github.repository }} + TRIGGER_ID: ${{ needs.guard.outputs.comment_id }} + # The same collection `Acknowledge the trigger` posted the eyes to. Reading + # a different one leaves them standing. + COMMENT_API: ${{ needs.guard.outputs.comment_api }} + CHECK: ${{ steps.drive.outputs.check_path }} + # Whether there is a verdict to react to. See the conclusion below. + VERDICT_PRODUCED: ${{ steps.drive.outputs.verdict_produced }} + run: | + set -euo pipefail + # An absent check file reads as an absent conclusion, which the case below + # answers by clearing. Returning here leaves the eyes standing, and a thumb an + # earlier attempt left, for a run that reached no verdict. + # + # A run that produced no verdict is read as no conclusion, whatever the file + # holds. It holds `failure` there, and that is the same word a review carrying + # blockers concludes -- so reading it here would thumb down a run that reviewed + # nothing, which says the change is bad when nobody could read it. The title + # separates the two in the checks list; a reaction has no title, so this reads + # the gate the workflow already computed instead. + conclusion="" + if [ -s "$CHECK" ] && [ "${VERDICT_PRODUCED:-}" = "true" ]; then + # jq's error goes to the log and the value comes back on stdout, so a check + # file this step cannot read states itself rather than arriving as data. + conclusion="$(jq -r '.conclusion // empty' "$CHECK" || true)" + fi + + # On the conclusion, not on the position the step above records: a repository + # that has not opted into approve-on-success records none for a clean review, + # and that reader still asked a question. + # + # neutral earns nothing, and neither does a conclusion this step cannot read. + # neutral is what a review concludes when it cannot show it read the diff, and + # a thumb up there says the change is fine when nobody looked at it. + # + # The reaction and the ones it replaces are chosen in one statement, so the + # two cannot drift apart. + # + # Every arm withdraws the eyes. The acknowledgement is a promise of an answer, + # and this step is where the answer arrives, so the arm that reaches no verdict + # withdraws them too -- and with them both thumbs. The comment carries what the + # last run left there, so a run that reaches neither verdict would otherwise + # leave the request wearing a verdict this run did not reach. + case "$conclusion" in + failure) reaction="-1"; stale="+1 eyes" ;; + success) reaction="+1"; stale="-1 eyes" ;; + *) + echo "a ${conclusion:-missing} conclusion earns no reaction; the verdict comment carries what it found" + reaction=""; stale="+1 -1 eyes" ;; + esac + + # Withdraw this bot's stale reactions first: the eyes that acknowledged the + # trigger, and a thumb an earlier attempt left. A reaction is not a toggle and + # a re-run replays the same comment id, so a comment would otherwise wear a + # verdict this run did not reach. Scoped to the reacting identity: a human who + # thumbed the request down is voicing an opinion, and it is not this job's to + # delete. The same scope leaves a human's eyes alone. + # + # GITHUB_TOKEN reacts as github-actions[bot], and an installation token cannot + # ask the API which login it carries, so the login is named here. + me="github-actions[bot]" + # Listed into a variable, and read from it rather than through a pipe, so + # nothing this block reports can be read back as a reaction id. Paginated, + # because a busy comment carries more reactions than one page holds and a + # miss leaves the request wearing the eyes, or a thumb this run replaces. + if ! mine="$(gh api "repos/$TRIGGER_REPO/$COMMENT_API/$TRIGGER_ID/reactions" \ + --paginate \ + --jq ".[] | select(.user.login == \"$me\") | \"\\(.content) \\(.id)\"")"; then + echo "::warning::could not read the reactions on comment $TRIGGER_ID in $TRIGGER_REPO; the eyes or a stale thumb from this bot may stay on it" + mine="" + fi + while read -r content rid; do + [ -n "$rid" ] || continue + case " $stale " in *" $content "*) ;; *) continue ;; esac + if gh api -X DELETE \ + "repos/$TRIGGER_REPO/$COMMENT_API/$TRIGGER_ID/reactions/$rid" \ + >/dev/null; then + echo "withdrew this bot's $content from comment $TRIGGER_ID" + else + echo "::warning::could not withdraw this bot's $content from comment $TRIGGER_ID in $TRIGGER_REPO" + fi + done <<< "$mine" + + # Nothing to post when the conclusion earned no reaction: the clearing above is + # the whole answer there. + [ -n "$reaction" ] || exit 0 + + # Never fatal: a reaction is a courtesy, and losing one must not fail a review + # that ran and published. Idempotent per identity and content, so a re-run on + # the same comment returns the reaction already there rather than a second one. + if gh api -X POST "repos/$TRIGGER_REPO/$COMMENT_API/$TRIGGER_ID/reactions" \ + -f content="$reaction" >/dev/null; then + echo "reacted $reaction on comment $TRIGGER_ID in $TRIGGER_REPO" + else + echo "::warning::could not react on comment $TRIGGER_ID in $TRIGGER_REPO; the review stands" + fi + + - name: Post the verdict + # id: the withdrawal step at the end of this job reads this step's `posted` + # output to decide whether the trigger comment already carries an honest answer. + id: verdict + # Post only when a real verdict was produced, and even when the drive + # step above exited non-zero -- `!cancelled()` runs on any outcome + # except the job itself being cancelled (e.g. superseded by a newer + # `seidroid review`), which is the one case with nothing to post. + # Keying on verdict_produced, not the exit code, so a teardown-only + # failure still posts a valid verdict and a no-verdict run never + # upserts a placeholder. + # + # continue-on-error. This step is the last thing standing between a finished + # review and the reader, and failing the job here throws that review away + # rather than saving it. What it costs is the signal, so the run block states + # the failure itself; see there. + if: ${{ inputs.mode == 'review' && (!cancelled() && steps.drive.outputs.verdict_produced == 'true') }} + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ steps.identity.outputs.token || github.token }} + # The reviewed pull request, which is where a reader looks for a review. + REPO: ${{ github.repository }} + PR: ${{ needs.guard.outputs.pr_number }} + VERDICT: ${{ steps.drive.outputs.verdict_path }} + # Findings the step above could place nowhere on the diff. Empty when it + # placed them all. + NOTE: ${{ steps.place.outputs.note_path }} + # What the driver counted, for the findings line below. Read rather than + # recomputed here: the driver derived the check run's conclusion from these + # same findings, and a second derivation is a second thing that can disagree. + CHECK: ${{ steps.drive.outputs.check_path }} + # What the step above placed. Empty, not zero, when that step died partway -- + # and the difference is the point. "0 posted inline" over a placement that + # never ran is a lie about the review. + ON_LINE: ${{ steps.place.outputs.on_line }} + ON_FILE: ${{ steps.place.outputs.on_file }} + UNPLACED: ${{ steps.place.outputs.unplaced }} + # The commit the review read, for the failure check below. Recorded before + # the review started; absent only when that read failed, and then the + # annotation is the only record. + REVIEWED_SHA: ${{ steps.head.outputs.sha }} + # Read for the failure check below, which publishes under the same names the + # publish step does. One switch drives both, so the set of names matches. + AI_REVIEW_COPY: ${{ inputs.publish-ai-review-check }} + # The notice an earlier no-verdict run left, which this verdict supersedes. + # Its own name and its own value: a notice that reports no verdict has to be + # addressable apart from the verdict itself. + NO_VERDICT_MARKER: "" + # GitHub rejects an issue comment over 65,536 characters. The driver bounds + # the verdict it writes and clips its own text to fit (review.MaxBodyBytes, + # 60,000). Nothing bounds $NOTE: it carries Finding.Detail, raw model prose + # with no length constraint on it. Unbounded, a long tail of unplaced + # findings pushes the body past the cap and the upsert below is rejected -- + # losing a whole review over its least important part. + # + # Bytes, not characters, for the reason the driver counts bytes: a byte count + # is never lower than a rune count, so a body inside this bound is inside + # GitHub's limit whichever unit that limit turns out to count. + MAX_BODY_BYTES: 65536 + # Held back from the note's budget for the truncation notice, which is + # written after the cut point is chosen and so cannot be measured before it. + NOTICE_BYTES: 256 + run: | + set -euo pipefail + body="$VERDICT_MARKER"$'\n'"$(cat "$VERDICT")" + + # The findings line, in the shape ai-review posts, so a reader moving between + # the two reviewers during the transition reads one format. Assembled here + # rather than in the driver because half of it is the placement above, which + # the driver never sees. + # + # It is appended before the note block below, and that ordering is load-bearing + # twice. It puts the line under the review's prose, where the format wants it. + # And the note's byte budget measures $body to decide what room is left, so a + # line added after that measurement is a line nothing accounted for -- and what + # it pushes past GitHub's cap is the whole review. + # + # Every term is dropped rather than guessed, and nothing here can fail the + # publish. A count this run cannot read costs the term; a line with no count + # left in it is not written at all. + is_count() { case "${1:-}" in (''|*[!0-9]*) return 1 ;; esac; } + line="" counted=false + add_term() { if [ -n "$line" ]; then line="$line | "; fi; line="$line$1"; } + + # The driver's own totals, over every finding the review reported rather than + # over the ones that could be placed: a blocker naming no line is still a + # blocker, and a count that omitted it would read as a cleaner review than the + # one that ran. + # + # A driver older than these fields writes none of them and jq answers null. The + # sentinel makes that a value is_count rejects, so an old driver publishes the + # same comment with a shorter line that is still true. + # Under `counts`, which is where the driver writes them: check.json carries + # `counts: {blocking, non_blocking, placeable, pre_existing}` beside a + # top-level `conclusion`. Read at the ROOT they answered null on every run, so + # the line never printed a number, the notice below claimed the driver predates + # fields it has, and neither disagreement warning could fire -- the one automated + # cross-check between this comment and the check run was dead code. + blocking="" non_blocking="" pre_existing="" conclusion="" + if [ -s "${CHECK:-}" ]; then + read -r blocking non_blocking pre_existing conclusion < <(jq -r '[(.counts.blocking // "?"), + (.counts.non_blocking // "?"), (.counts.pre_existing // "?"), (.conclusion // "?")] | @tsv' \ + "$CHECK" 2>/dev/null) || true + fi + if is_count "$blocking" && is_count "$non_blocking"; then + add_term "$blocking blocking" + add_term "$non_blocking non-blocking" + counted=true + else + echo "::notice::this driver reports no finding counts, so the findings line \ + omits them; it predates check.json's blocking and non_blocking fields" + fi + + # Both placements the step above makes: on a line where the diff carries it, on + # the file where it does not. Both land in the pull request's file view, which + # is what "inline" means to the person reading it. What reached neither is not + # counted here -- the note below names those one by one under a heading of their + # own, which is more than a number would say. + if is_count "${ON_LINE:-}" && is_count "${ON_FILE:-}" && is_count "${UNPLACED:-}"; then + add_term "$(( ON_LINE + ON_FILE )) posted inline" + counted=true + else + add_term "inline placement did not run" + echo "::notice::no placement counts, so the findings line says so; that step \ + writes none of them if it dies partway" + fi + + # Counted apart from both, and named. CheckConclusion excludes a pre-existing + # blocker from the gate on purpose -- it is already on the base branch, so + # failing on it would fail every pull request that touches the file, and the + # author who has to clear the check is the one person who did not cause it. + # Folded into "blocking" it would print a number the check run contradicts; + # folded into "non-blocking" it would tell an author their change has problems + # it does not have. Omitted at zero, which is most reviews. + if is_count "$pre_existing" && [ "$pre_existing" -gt 0 ]; then + add_term "$pre_existing pre-existing" + fi + + # Two numbers about one review must not disagree. Both are the driver's, from + # the same findings, so a disagreement is a defect there rather than a case to + # render around. It is stated in the log and the line still prints what it was + # given: correcting either number here would publish a third answer and hide + # the bug that produced the first two. + if is_count "$blocking" && [ "$conclusion" = "failure" ] && [ "$blocking" -eq 0 ]; then + echo "::warning::the driver reported 0 blocking findings beside a failing \ + check; this comment and the check run disagree" + elif is_count "$blocking" && [ "$conclusion" != "failure" ] && [ "$blocking" -gt 0 ]; then + echo "::warning::the driver reported $blocking blocking finding(s) beside a \ + $conclusion check; this comment and the check run disagree" + fi + + if [ "$counted" = true ]; then + body="$body"$'\n\n'"**Findings:** $line" + fi + + # A reader who sees no inline comment for an observation would otherwise + # have to guess whether it was dropped. + # + # Bounded against what the marker and the verdict already spend, so the note + # can never be the thing that pushes the comment past GitHub's cap. Cut on + # whole lines: a byte cut can land inside a UTF-8 sequence or inside a + # finding's markdown, where a line cut can be counted and named. + if [ -n "$NOTE" ] && [ -s "$NOTE" ]; then + room=$(( MAX_BODY_BYTES - $(printf '%s\n\n' "$body" | wc -c) )) + if [ "$(wc -c < "$NOTE")" -le "$room" ]; then + body="$body"$'\n\n'"$(cat "$NOTE")" + else + # Every line the place step writes for a finding opens with "- `". + total="$(grep -c '^- `' "$NOTE" || true)" + budget=$(( room - NOTICE_BYTES )) + kept="" + shown=0 + if [ "$budget" -gt 0 ]; then + # sed drops the last line, which is the one the byte cut may have left + # half-written. + kept="$(head -c "$budget" "$NOTE" | sed '$d')" + shown="$(printf '%s\n' "$kept" | grep -c '^- `' || true)" + fi + if [ "$shown" -gt 0 ]; then + body="$body"$'\n\n'"$kept" + fi + body="$body"$'\n\n'"_Cut to stay inside GitHub's comment limit ($MAX_BODY_BYTES bytes): $shown of $total observation(s) are shown here. All $total are in this workflow run's log._" + echo "::warning::the summary note was cut to $shown of $total unplaced observation(s) to fit GitHub's comment limit; the full list follows" + cat "$NOTE" + fi + fi + # A NEW comment every iteration, never an edit of the last one. Editing leaves + # the comment at its ORIGINAL position in the thread and notifies nobody, so a + # re-review became close to invisible: it did not move, it did not notify, and + # it destroyed the previous verdict's text. A verdict written four minutes ago + # rendered eighteen hours up the page, indistinguishable from a review that + # never ran, and that cost real diagnosis time. + # + # Earlier verdicts are left standing on purpose. They are the record of what the + # review said before the author's fixes, which is the thing a reader compares + # against. A long pull request accumulates a few, which is the accepted cost. + # + # It also removes a hazard the marker lookup carried: this marker is shared with + # the unplaced-findings note posted above, so selecting `.[0]` by marker could + # overwrite that note rather than the previous verdict. + # + # repo/pr from env, not template-interpolated into the script. The failure is + # caught rather than left to `set -e`, so this step can say what was lost. + posted=false + if gh api -X POST "repos/$REPO/issues/$PR/comments" -f body="$body" >/dev/null; then + posted=true + fi + # Whether the verdict is ON the pull request, for the withdrawal step at the + # end of this job: a thumb on the trigger comment answers a verdict only when + # the verdict is there to read. This step's exit status cannot carry that. The + # failure path below ends on a call whose failure it swallows, so the step + # exits 0 whether the comment landed or not, and its outcome reads success + # either way. + echo "posted=$posted" >> "$GITHUB_OUTPUT" + if [ "$posted" = true ]; then + echo "posted the verdict on $REPO#$PR" + # An earlier run may have left a notice saying this review did not complete. + # It did, so that notice now asserts a state that is not true and links a run + # that is not the latest. The check run supersedes itself on the new commit; + # a comment does not, so this withdraws it. + # + # Matched on the marker at the start of the body, and only on this tool's own + # comments, so a person quoting the marker cannot have their comment deleted. + # The listing emits one id per line: --paginate runs the filter once per page, + # so aggregating inside jq would emit one result per page and concatenate them + # into a string no comment carries. + # + # Every failure here is tolerated. The verdict is posted, and a stale notice + # beside it is a smaller cost than failing a step that already did its work. + # shellcheck disable=SC2016 # $ENV is jq's, and single quotes are what keep it jq's + stale_notice="$(gh api --paginate "repos/$REPO/issues/$PR/comments?per_page=100" \ + --jq '.[] | select(.user.type == "Bot") | select(.body | startswith($ENV.NO_VERDICT_MARKER)) | .id' \ + 2>/dev/null | tail -n 1 || true)" + if [ -n "$stale_notice" ]; then + if gh api -X DELETE "repos/$REPO/issues/comments/$stale_notice" >/dev/null 2>&1; then + echo "withdrew the no-verdict notice an earlier run left on $REPO#$PR" + else + echo "::warning::could not withdraw the no-verdict notice $stale_notice on $REPO#$PR; it now says this review did not complete, and it did" + fi + fi + exit 0 + fi + + # continue-on-error above is what stops a publishing failure from discarding a + # review that ran. It also removes the only signal there was, and a review that + # is silently absent is worse than a red job. So the failure is stated in two + # places, because neither alone is enough: an annotation on the run, which needs + # nothing but the runner; and the check run, which is the only one of the two + # that reaches the pull request, where the reader is waiting for a review that + # is not coming. + # + # The verdict goes to the log unposted, so the run still holds what the review + # cost model spend and a sandbox to produce. + bytes="$(printf '%s' "$body" | wc -c | tr -d '[:space:]')" + echo "::error::the review reached a verdict but it could not be posted on $REPO#$PR ($bytes bytes); it is in this step's log below" + echo "--- verdict, unposted ---" + printf '%s\n' "$body" + echo "--- end verdict ---" + # Under every name the publish step above writes. This conclusion then + # supersedes each of them on this commit rather than sitting beside one, and a + # later run that does post clears them. A copy left carrying the earlier + # conclusion is a rule reading green on a verdict that never arrived. + # + # `review` goes first and is the check of record. The `AI Review` copy follows + # it, under the switch that step reads, so the two publish the same set of + # names on both paths. + # + # Best-effort, both of them: whatever stopped the comment can stop this too, + # and then the annotation stands alone. + if [ -n "${REVIEWED_SHA:-}" ]; then + fail_check() { + gh api -X POST "repos/$REPO/check-runs" \ + -f name="$1" \ + -f head_sha="$REVIEWED_SHA" \ + -f status=completed \ + -f conclusion=failure \ + -f output[title]="review produced but not published" \ + -f output[summary]="The review ran and reached a verdict. Posting it to this pull request failed, so the verdict is not here. Read it in the workflow run: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + >/dev/null + } + fail_check review \ + || echo "::warning::the failure check run could not be posted either; the annotation on this run is the only record" + if [ "${AI_REVIEW_COPY:-}" = "true" ]; then + fail_check "AI Review" \ + || echo "::warning::the AI Review copy of the failure check run could not be posted on $REPO#$PR; it still carries this review's earlier conclusion" + fi + else + echo "::warning::no reviewed commit was recorded, so there is no check run to fail; the annotation on this run is the only record" + fi + + - name: Resolve the threads this review closed + # A finding that still holds gets a new thread on every re-review while the old + # one stays open. On one live pull request 5 of 13 threads were byte-identical + # repeats of two findings, so a third re-review has the author read one finding + # in three places and dismiss it three times. The driver names which of its own + # threads this review closed; this closes them. + # + # Two lists, two gates. `addressed` is a finding the diff no longer shows, so it + # closes whenever the review published. `superseded` is a finding restated as a + # new comment, so it closes only once the comment restating IT reached the code + # -- a thread shut behind a comment that never posted takes a live finding off + # the pull request and puts nothing where it was. + # + # That second gate is per THREAD. Each finding names the threads its comment + # replaces, placement records those ids as the comment posts, and a thread here + # closes on finding its id in that record. So a review superseding three threads + # closes the one whose replacement reached the code and leaves the other two + # standing beside the findings that restate them. + # + # A driver that publishes no linkage falls back to the per-review gate: every + # superseded thread together, and only when placement dropped nothing at all. + # Coarse in the safe direction -- one unplaced comment holds all of them open -- + # and placement's superseded_linked output is what selects between the two. + # + # The record is what a comment DID, and `addressed` is what publication alone + # earns. So an `addressed` id needs no record and takes neither gate. + # + # Last of the publishers, so nothing old closes before the new review is on the + # pull request. + # + # Every id is checked here as well as in the driver. The driver refuses an id + # that matches no thread it was handed; this refuses one that is not, right now, + # an unresolved thread this reviewer wrote. The mutation is on someone else's + # pull request and the ids come from model output, so the check runs beside the + # call rather than only upstream of it. + # + # REVIEWER_LOGIN alone, where the history read admits WORKFLOW_LOGIN beside it. + # Reading a thread this run did not write costs nothing; closing one is a + # mutation, and it stays as narrow as the identity performing it. + # + # continue-on-error, like every publisher in this job. A review that ran and + # published must not be failed over a thread it could not close: the cost of not + # closing one is the duplicate this workflow leaves today. + # + # resolveReviewThread is a pull-request operation. GitHub categorises it under + # pulls and publishes no permission requirement of its own for it, and this job + # grants pull-requests: write. So a refusal names the identity rather than the + # thread, and the warning below reports which identity the API refused. + # + # Every page, as the history read takes them. The two have to see the same pull + # request: a resolver stopping at the first page would call a thread the history + # carries one this tool never left. + if: ${{ inputs.mode == 'review' && !cancelled() + && steps.drive.outputs.verdict_produced == 'true' }} + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ steps.identity.outputs.token || github.token }} + REPO: ${{ github.repository }} + PR: ${{ needs.guard.outputs.pr_number }} + # The driver's plan, in the file this job already reads to publish the check + # run. An older driver writes no `threads` key, and this step then closes + # nothing -- which is what the workflow did before it could. + CHECK: ${{ steps.drive.outputs.check_path }} + # Which threads had their replacement reach the code, one id per line, and + # whether the driver published that linkage at all. Absent when placement did + # not run, and the fallback below refuses on its own terms there. + LINKAGE: ${{ steps.place.outputs.linkage_path }} + PLACED_LINKED: ${{ steps.place.outputs.superseded_linked }} + # What placement managed, and what it could not place. Read by the fallback + # gate alone: they are counts over the whole review and say nothing about + # which comment replaced which thread. + PLACED_ON_LINE: ${{ steps.place.outputs.on_line }} + PLACED_ON_FILE: ${{ steps.place.outputs.on_file }} + PLACED_UNPLACED: ${{ steps.place.outputs.unplaced }} + run: | + set -euo pipefail + # What the API said, on one line and bounded. A warning that reports only + # "could not be resolved" tells an operator nothing to act on, and the message + # is the difference between a thread somebody else already touched and a + # credential that cannot do this at all. + saidIt() { printf '%s' "${1:-}" | tr '\n\r' ' ' | cut -c1-300; } + apiSaid() { saidIt "$(cat "$1" 2>/dev/null)"; } + + if [ ! -s "${CHECK:-}" ]; then + echo "no check file, so nothing names a thread to close" + exit 0 + fi + + # The driver already refused these against the history it was handed. Repeated + # here because this is where a person reading the run sees it, and an invented + # id means the review is naming threads that do not exist. + while IFS= read -r id; do + [ -z "$id" ] && continue + echo "::warning::the review named review thread '$id', which is not one this tool left on $REPO#$PR; it was not resolved" + done < <(jq -r '.threads.refused // [] | .[]' "$CHECK") + + wanted="$(jq -r '.threads.addressed // [] | .[]' "$CHECK")" + held="$(jq -r '.threads.superseded // [] | length' "$CHECK")" + + # The plan is the warrant and the record is which comment spent it, so a + # thread has to be in both. The plan alone would close one the record cannot + # account for; the record alone would close one the driver refused. + # + # An id nowhere in the record is a replacement that did not reach the code, + # whatever happened to the rest of the review. That is the whole of the gate: + # no count over the review enters it, because no count can say which comment + # replaced which thread. + if [ "${PLACED_LINKED:-}" = "true" ]; then + landed="${LINKAGE:-}" + [ -s "$landed" ] || landed=/dev/null + closing=0 + while IFS= read -r id; do + [ -z "$id" ] && continue + if grep -qxF -- "$id" "$landed"; then + wanted="$wanted"$'\n'"$id" + closing=$((closing+1)) + continue + fi + echo "review thread $id stays open: nothing replacing it reached the code" + done < <(jq -r '.threads.superseded // [] | .[]' "$CHECK") + if [ "$held" -gt 0 ]; then + echo "superseded: $closing of $held thread(s) had their own replacement reach the code" + fi + else + # The fallback, for a driver that publishes no linkage. Both halves are + # required. Counting only what landed lets one unrelated new finding on a + # line stand in for three superseded replacements that landed nowhere -- and + # those three threads would close with the findings that replaced them + # sitting in the summary instead of on the diff. + # + # The default in ${PLACED_UNPLACED:-1} is deliberate and is not a count. An + # absent output means placement did not report, which has to read as + # "something may be unplaced" rather than as zero: this gate decides whether + # a live finding comes off the pull request, so the unknown falls on the side + # that leaves the thread open. + placed=$(( ${PLACED_ON_LINE:-0} + ${PLACED_ON_FILE:-0} )) + if [ "$placed" -gt 0 ] && [ "${PLACED_UNPLACED:-1}" -eq 0 ]; then + wanted="$wanted"$'\n'"$(jq -r '.threads.superseded // [] | .[]' "$CHECK")" + elif [ "$held" -gt 0 ]; then + echo "$held superseded thread(s) stay open: $placed comment(s) reached the code and ${PLACED_UNPLACED:-an unreported number} could not be placed" + fi + fi + wanted="$(printf '%s\n' "$wanted" | sed '/^$/d' | sort -u)" + if [ -z "$wanted" ]; then + echo "this review closes no thread" + exit 0 + fi + + owner="${REPO%%/*}"; name="${REPO##*/}" + state="$RUNNER_TEMP/review-resolve-threads.json" + # Read again here rather than reused from the history step. Minutes of review + # sit between the two, and a thread an author resolved in that time is one this + # step must not report as its own doing. + # + # Every page, like the history read. The two have to see the same pull request: + # the history carries a thread from page two, the review can name it, and a + # resolver that stopped at the first page would report a thread this tool did + # leave as one it never wrote -- a false statement in the log, and the thread + # left open beside the finding that restates it. + # + # $owner and friends are GraphQL variables, so the query stays literal. + # shellcheck disable=SC2016 + if ! gh api graphql --paginate -f query=' + query($owner: String!, $name: String!, $number: Int!, $endCursor: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + reviewThreads(first: 100, after: $endCursor) { + nodes { + id + isResolved + comments(first: 1) { nodes { body author { login } } } + } + pageInfo { hasNextPage endCursor } + } + } + } + }' -F owner="$owner" -F name="$name" -F number="$PR" 2> "$state.err" \ + | jq -s '[ .[].data.repository.pullRequest.reviewThreads.nodes[] ]' > "$state"; then + echo "::warning::the review threads on $REPO#$PR could not be read as ${REVIEWER_LOGIN:-an unnamed identity}: $(apiSaid "$state.err"); no thread was closed and the author sees each restated finding twice" + exit 0 + fi + + # Three sets, because a thread this run will not close is not one thing. Only + # the last of them is a warning, and saying so takes all three: a message that + # calls a thread this tool wrote one it never left is false, whichever reason + # kept the run from closing it. + # + # open -- this run's own identity, still unresolved. The only set it closes. + # ours -- this run's own identity, any state. Already resolved, nothing to do. + # either -- this tool's, under either identity it posts as. The history read + # admits these, so the review can legitimately name one; this run + # does not close a thread it did not write, and says which identity + # did instead of denying the thread. + # + # The mutation keeps the strict single-login test. Reading a thread another + # identity wrote is inert; closing one is not, and it stays as narrow as the + # identity performing it. + # shellcheck disable=SC2016 # $ENV is jq's, and single quotes are what keep it jq's + marked='.[] | select((.comments.nodes[0].body // "") | startswith($ENV.FINDING_MARKER))' + # shellcheck disable=SC2016 # same + mine=' | select((.comments.nodes[0].author.login // "") == $ENV.REVIEWER_LOGIN)' + # shellcheck disable=SC2016 # same + anyOfOurs=' | select(.comments.nodes[0].author.login + | . != null and . != "" + and (. == $ENV.REVIEWER_LOGIN or . == $ENV.WORKFLOW_LOGIN))' + ours="$(jq -r "$marked$mine | .id" "$state")" + open="$(jq -r "$marked$mine | select(.isResolved | not) | .id" "$state")" + either="$(jq -r "$marked$anyOfOurs | .id" "$state")" + + closed=0 refused=0 failed=0 elsewhere=0 + while IFS= read -r id; do + [ -z "$id" ] && continue + if ! printf '%s\n' "$open" | grep -qxF -- "$id"; then + if printf '%s\n' "$ours" | grep -qxF -- "$id"; then + echo "review thread $id is already resolved; nothing to do" + elif printf '%s\n' "$either" | grep -qxF -- "$id"; then + # This tool left it, under the identity it was posting as then. The + # history read admits both, so naming it is correct; closing it is not + # this run's to do, and calling it a thread this tool never wrote would + # be false. + echo "review thread $id on $REPO#$PR was left under this tool's other identity, not ${REVIEWER_LOGIN:-the one this run uses}; this run does not close it and it stays open" + elsewhere=$((elsewhere+1)) + else + echo "::warning::review thread '$id' is not an unresolved thread this tool left on $REPO#$PR; it was not resolved" + refused=$((refused+1)) + fi + continue + fi + # Both streams are captured, because gh splits a failure across them: the + # API's error body goes to stdout and gh's own line to stderr, and either + # alone can be the half that says why. Discarded on success. + # shellcheck disable=SC2016 # $threadId is GraphQL's + if out="$(gh api graphql -f query=' + mutation($threadId: ID!) { + resolveReviewThread(input: {threadId: $threadId}) { thread { id isResolved } } + }' -F threadId="$id" 2>&1)"; then + closed=$((closed+1)) + else + echo "::warning::review thread $id on $REPO#$PR was not resolved as ${REVIEWER_LOGIN:-an unnamed identity}: $(saidIt "$out"); it stays open and the author reads this finding twice" + # A refusal is about the identity rather than the thread, and which + # identity ran decides what to change. An App installation token carries + # the App's own permissions and this job's permissions block does not + # narrow it; the workflow token carries only what that block granted. + case "$out" in + *"not accessible by integration"*|*Forbidden*|*"HTTP 403"*|*"must have"*) + echo "::warning::that refusal names the identity, not the thread. ${REVIEWER_LOGIN:-the identity this run used} was refused resolveReviewThread on $REPO#$PR: an App identity carries the App installation's permissions, and github-actions carries only what this job's permissions block grants" ;; + esac + failed=$((failed+1)) + fi + done <<< "$wanted" + echo "threads: $closed closed, $refused refused, $elsewhere left under another identity, $failed could not be resolved" + + - name: Report a review that reached no verdict + # The step above answers a review that decided; this one answers a review that + # could not be read. A person who asked would otherwise get a red job, an + # untouched pull request, and one line in a log nobody opens. ai-review.yml + # answers the same case the same way, and this is parity with it. + # + # The complement of the step above, on one output: `== 'true'` there, `!= 'true'` + # here. So a review run that was not cancelled posts one comment or the other, + # never both and never neither -- including a run that died before the driver, + # where verdict_produced is unset and this is the only record left. + # + # It reports and takes no position. No approval, no request for changes, and no + # withdrawal of an earlier block: the step above owns all three, and it keeps + # the verdict gate for that reason. + # + # No byte-cap arithmetic, unlike the step above. What it appends is a bounded + # summary the driver clipped, not the unbounded findings note. + # + # continue-on-error, like its sibling. The drive step already failed the job on + # this path -- it forces a non-zero exit when a review produced no verdict -- so + # failing it again here buys no signal; the annotation in the run block is what + # replaces the one it costs. + if: ${{ inputs.mode == 'review' && !cancelled() + && steps.drive.outputs.verdict_produced != 'true' }} + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ steps.identity.outputs.token || github.token }} + # Its own marker, following the same shape the inline findings use. The + # verdict comment's marker means "this tool's verdict"; this comment reports + # that there is none, so a later step or a reader's query can address one + # without matching the other. This step reads it back to find its own notice. + MARKER: "" + # Where the verdict comment would have gone. + REPO: ${{ github.repository }} + PR: ${{ needs.guard.outputs.pr_number }} + # Why there is no verdict, in the driver's own words. Absent when the run + # never reached the driver, and the body says which of the two happened. + CHECK: ${{ steps.drive.outputs.check_path }} + run: | + set -euo pipefail + # Removes the standing no-verdict notice, if there is one, so exactly one is + # ever on the pull request. + # + # Matched on the marker at the start of the body, and only on this tool's own + # comments, so a person quoting the marker cannot have their comment deleted. + # The listing emits one id per line: --paginate runs the filter once per page, + # so aggregating inside jq would emit one result per page and concatenate them + # into a string no comment carries. + # + # Every failure here is tolerated. Whichever caller runs this has something + # else to say afterwards, and a stale notice is a smaller cost than losing it. + delete_prior_notice() { + local id + # shellcheck disable=SC2016 # $ENV is jq's, and single quotes are what keep it jq's + id="$(gh api --paginate "repos/$REPO/issues/$PR/comments?per_page=100" \ + --jq '.[] | select(.user.type == "Bot") | select(.body | startswith($ENV.MARKER)) | .id' \ + 2>/dev/null | tail -n 1 || true)" + if [ -n "$id" ]; then + if gh api -X DELETE "repos/$REPO/issues/comments/$id" >/dev/null 2>&1; then + echo "withdrew the standing no-verdict notice on $REPO#$PR" + else + echo "::warning::could not withdraw the no-verdict notice $id on $REPO#$PR" + fi + fi + } + + # The check run's own summary, not a second account of the same run. That + # check is published above and a reader compares the two; one string is what + # keeps them from disagreeing. jq's error goes to the log and the value comes + # back empty, so a check file this step cannot read falls to the case below. + reason="" + if [ -s "${CHECK:-}" ]; then + reason="$(jq -r '.summary // empty' "$CHECK" 2>/dev/null || true)" + fi + # Two different runs reach this. One reviewed and could not be read, and the + # driver said why. The other never got as far as the driver -- a failed + # install, a toolchain that would not resolve -- and has no reason to give, so + # this says that rather than implying the review ran. + started=true + if [ -z "$reason" ]; then + reason="This run recorded no reason, which happens when it stopped before the review started." + started=false + fi + body="$(printf '%s\n%s\n\n%s\n\n%s\n' \ + "$MARKER" \ + '**This review did not complete.** It reached no verdict, so there is no review of this change to act on.' \ + "$reason" \ + "Read this run for the rest: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID")" + # One notice per pull request: the previous one is deleted and a new one is + # posted. The verdict comment accumulates on purpose -- an author compares + # this review against the last one -- but a notice that the review did not + # complete carries no such history, and an outage on a caller wired to every + # push would leave one identical copy per push. + # + # Deleted and reposted rather than edited, for the reason `Post the verdict` + # states about itself: an edit stays where it was in the thread and notifies + # nobody, so a second consecutive failure would reach the person who just + # asked as nothing at all. Posting lands it at the bottom, carrying this run's + # url. + # + # A deletion that fails costs a duplicate notice; the post below runs either + # way, so the person who asked always gets an answer. + delete_prior_notice + if gh api -X POST "repos/$REPO/issues/$PR/comments" -f body="$body" >/dev/null; then + echo "reported a no-verdict review on $REPO#$PR" + exit 0 + fi + # The last record standing. continue-on-error above keeps a failed comment + # from adding a second failure to a job the drive step already failed, and it + # also removes the only signal there was -- so the report goes to the log + # where the run still holds it. + if [ "$started" = true ]; then + echo "::error::the review reached no verdict, and that could not be reported on $REPO#$PR; this run's log is the only record" + else + echo "::error::this run stopped before the review started, and that could not be reported on $REPO#$PR; this run's log is the only record" + fi + echo "--- report, unposted ---" + printf '%s\n' "$body" + echo "--- end report ---" + + - name: Withdraw the reactions on a cancelled run + # The one step in this job that runs on a cancelled run, and the whole of what + # a cancelled run is allowed to do to the trigger comment: take this bot's own + # reactions off it and say nothing. + # + # A newer `@seidroid review` cancels the run in flight. The eyes are already on + # the older comment by then, and the newer run answers its OWN comment id, so + # nothing reads the older one again -- it would wear the eyes for good. + # + # THIS STEP HAS NO POST. It withdraws, and there is no code path in it that adds + # a reaction, so nothing it is told can make it state an outcome. That is what + # makes the withdrawal a step of its own rather than always() on `Answer the + # request`: that step chooses a reaction from a conclusion, and a cancellation + # arriving after the driver finishes leaves check_path and verdict_produced + # populated -- so it would read a real conclusion and thumb a review that + # published nothing. + # + # `Post the verdict` decides whether there is anything to withdraw, and the + # driver does not. A cancellation can arrive once the verdict is already on the + # pull request -- during thread resolution, say -- and the thumb `Answer the + # request` posted answers it correctly. Withdrawing it there leaves a published + # review with no reaction on the request that asked for it, which reads as never + # answered. That is this step's own defect, one window later. + # + # Its `posted` OUTPUT separates the two, and its outcome does not. That step + # tolerates a failed comment POST and ends on a call whose failure it swallows, + # so it exits 0 and reads success whether the verdict landed or not. A thumb kept + # on that reading would stand for a review nobody can see. The output is written + # from the POST's own result. + # + # One boolean about another step, with no conclusion in it. It says whether an + # answer already stands, never which answer it would be, so reading it cannot + # give this step an outcome to state. The same holds for the answer step's + # outcome, which the withdrawal set below reads. + # + # Anything but a posted verdict withdraws. A value this step cannot read + # therefore clears, rather than leaving a thumb that stands for a verdict which + # may not be on the pull request. + # + # WHAT IT PROTECTS, AND WHERE THAT STOPS. A thumb survives when the verdict it + # answers is on the pull request AND this run is the run that posted it, or when + # this run never reached the answer step at all. One case is left over: this run + # answers, which withdraws an earlier run's thumb and posts its own, and is then + # cancelled before publishing. The comment ends bare while the earlier run's + # verdict still stands. The earlier thumb is already gone by then -- the answer + # step took it -- so nothing here can put it back, and knowing it happened would + # take a read of the pull request this step deliberately does not make. A + # cancellation lands during the driver far more often than in that window. + # + # LAST in the job, deliberately. The runner evaluates a step's condition when it + # reaches the step, so a clear placed earlier is already skipped once a + # cancellation lands on a later one -- and it would leave both the eyes and the + # thumb the step above had just posted. + # + # cancelled() is only readable in a step or job `if`. It is not available in + # `env` or `run`, and a workflow carrying it there is rejected, so the job status + # cannot be handed to a shell and this decision cannot move inside one script. + if: ${{ inputs.mode == 'review' && cancelled() + && needs.guard.outputs.comment_id != '' + && steps.verdict.outputs.posted != 'true' }} + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ github.token }} + TRIGGER_REPO: ${{ github.repository }} + TRIGGER_ID: ${{ needs.guard.outputs.comment_id }} + # The collection the two steps above reacted on. It differs per event, so + # reading a fixed one would leave a diff-thread request wearing the eyes + # nothing here could reach. + COMMENT_API: ${{ needs.guard.outputs.comment_api }} + # Whether this run ever reached the step that answers. See the withdrawal set + # below. Four words about another step; none of them a conclusion. + ANSWERED: ${{ steps.answer.outcome }} + run: | + set -euo pipefail + # Scoped to the reacting identity, for the reason `Answer the request` states: + # a human who thumbed the request is voicing an opinion, and a cancellation is + # no licence to delete it. GITHUB_TOKEN reacts as github-actions[bot], and an + # installation token cannot ask the API which login it carries, so the login is + # named here. + me="github-actions[bot]" + # WHAT THIS RUN MAY TAKE, and it is not always all three. + # + # A re-run replays the trigger comment id, so the comment can already carry a + # thumb from an earlier run whose verdict IS on the pull request. `Answer the + # request` withdraws that thumb and posts this run's own, so once it has run, + # every reaction on the comment belongs to this run and this run published + # nothing -- take all three. + # + # A run cancelled before it reached that step has posted only the eyes. A thumb + # there answers an earlier run, and taking it leaves that run's published + # verdict with no reaction on the request, which reads as never answered. + # + # skipped is the only value that means "never touched the comment". failure and + # cancelled both mean the step ran partway and most likely cleared the earlier + # thumb already, so they clear, and so does a value this step cannot read: a + # thumb this run posted for a verdict nobody published is the worse of the two + # wrongs. + case "${ANSWERED:-}" in + skipped) takeable=" eyes " ;; + *) takeable=" +1 -1 eyes " ;; + esac + # Listed into a variable and read from it rather than through a pipe, so + # nothing this block reports can be read back as a reaction id. Paginated, + # because a busy comment carries more reactions than one page holds. + if ! mine="$(gh api "repos/$TRIGGER_REPO/$COMMENT_API/$TRIGGER_ID/reactions" \ + --paginate \ + --jq ".[] | select(.user.login == \"$me\") | \"\\(.content) \\(.id)\"")"; then + echo "::warning::could not read the reactions on comment $TRIGGER_ID in $TRIGGER_REPO; the eyes from this cancelled run may stay on it" + mine="" + fi + # Only what this run may take. A reaction of this bot's outside that set + # belongs to whatever put it there. + while read -r content rid; do + [ -n "$rid" ] || continue + case "$takeable" in *" $content "*) ;; *) continue ;; esac + if gh api -X DELETE \ + "repos/$TRIGGER_REPO/$COMMENT_API/$TRIGGER_ID/reactions/$rid" \ + >/dev/null; then + echo "withdrew this bot's $content from comment $TRIGGER_ID" + else + echo "::warning::could not withdraw this bot's $content from comment $TRIGGER_ID in $TRIGGER_REPO" + fi + done <<< "$mine" diff --git a/.github/workflows/workflow-test-self.yml b/.github/workflows/workflow-test-self.yml new file mode 100644 index 0000000..07e7da3 --- /dev/null +++ b/.github/workflows/workflow-test-self.yml @@ -0,0 +1,88 @@ +name: Workflow tests +# The shell and jq inside seidroid-review.yml, run against a gh stub, and its job +# conditions evaluated against synthetic payloads. Nothing here reaches the GitHub +# API, so this needs no token and no permissions. +# +# Three jobs, so each check in the list names the step it covers. A job that ran two +# harnesses would report one name for two things, and one check would go red for the +# other's failure. +# +# `Place findings and resolve threads` covers a pair of steps rather than either half: +# placement records which thread each posted comment replaced, and the resolve step +# closes on that record. `The reaction steps` covers which of the three runs in which +# job state, and what each leaves on the trigger comment. `Guard the request` covers +# the admission path, from the event the workflow accepts to the collection the +# acknowledgement reaches. +on: + pull_request: + paths: + - '.github/workflows/seidroid-review.yml' + # run-guard.sh asserts what ai-assistant.yml claims of the same comment, so an + # edit there can break an invariant stated here. Without this the break lands + # on the next unrelated pull request that touches the file above, pointing at + # the wrong change. + - '.github/workflows/ai-assistant.yml' + - '.github/workflows/workflow-test-self.yml' + - 'test/seidroid-review/**' + push: + branches: [ main ] + paths: + - '.github/workflows/seidroid-review.yml' + # run-guard.sh asserts what ai-assistant.yml claims of the same comment, so an + # edit there can break an invariant stated here. Without this the break lands + # on the next unrelated pull request that touches the file above, pointing at + # the wrong change. + - '.github/workflows/ai-assistant.yml' + - '.github/workflows/workflow-test-self.yml' + - 'test/seidroid-review/**' +permissions: + contents: read +jobs: + place-findings: + name: Place findings and resolve threads + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: '3.x' + - name: Install the YAML reader + run: python3 -m pip install --quiet pyyaml + - name: Run the placement and resolve harness + run: test/seidroid-review/run.sh + + reactions: + name: The reaction steps + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: '3.x' + - name: Install the YAML reader + run: python3 -m pip install --quiet pyyaml + - name: Run the reaction harness + run: test/seidroid-review/reactions.sh + # Which step runs in which job state, which is where the cancellation behaviour + # lives and which the shell harness above cannot see. + - name: Check the step conditions + run: python3 test/seidroid-review/conditions.py .github/workflows/seidroid-review.yml + + guard-admission: + name: Guard the request + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: '3.x' + - name: Install the YAML reader + run: python3 -m pip install --quiet pyyaml + - name: Run the guard harness + run: test/seidroid-review/run-guard.sh diff --git a/test/seidroid-review/.gitignore b/test/seidroid-review/.gitignore new file mode 100644 index 0000000..123dcfa --- /dev/null +++ b/test/seidroid-review/.gitignore @@ -0,0 +1,19 @@ +# Written by run.sh: the two steps extracted from the workflow, the fixtures it +# generates, and one directory of output per case. +place.sh +resolve.sh +fx/gen/ +out/ +# Written by reactions.sh: the three reaction steps, and one directory per case. +ack.sh +answer.sh +withdraw.sh +out-reactions/ +# Written by run-guard.sh: the five steps it extracts from the workflow. Two of +# them are steps reactions.sh also extracts, under names of their own so neither +# harness can overwrite the other's extraction. +refuse.sh +parse.sh +admit.sh +guard-ack.sh +guard-answer.sh diff --git a/test/seidroid-review/README.md b/test/seidroid-review/README.md new file mode 100644 index 0000000..17b1c6b --- /dev/null +++ b/test/seidroid-review/README.md @@ -0,0 +1,190 @@ +# Workflow tests + +Three harnesses over `.github/workflows/seidroid-review.yml`. All three read the +steps out of the YAML on every run, so none can pass against a stale copy. + +```sh +test/seidroid-review/run.sh # placement and thread resolution +test/seidroid-review/reactions.sh # the three reaction steps +test/seidroid-review/run-guard.sh # the guard, and the reaction collection +python3 test/seidroid-review/conditions.py .github/workflows/seidroid-review.yml +``` + +Each needs `bash`, `jq`, and `python3` with PyYAML, and exits non-zero on the +first failed assertion count. + +`extract.py` is shared. It reads a step's `run:` block and a workflow-level env +key out of the YAML, by step name or step id. + +`reactions.sh` and `run-guard.sh` both extract `Acknowledge the trigger` and +`Answer the request`, and they ask different things of them: `reactions.sh` asks +which step runs in which job state and what it leaves on the comment, +`run-guard.sh` asks which REST collection the URL reaches. Each writes its own +extraction — `ack.sh` and `answer.sh` against `guard-ack.sh` and +`guard-answer.sh` — so running both cannot have one overwrite the other. + +## `Place findings on the code` and `Resolve the threads this review closed` + +Runs both steps under `bash`, against a `gh` stub, and checks what they posted, +counted and closed. It prints a table of one row per case. + +Both steps are in one harness because they are one behaviour. Placement records +which thread each posted comment replaced; the resolve step closes a thread on +finding its id in that record. A harness that ran only one of them could not +tell whether the record it wrote is the record the other reads. + +The extractor runs twice, once per step, and the two `FINDING_MARKER` readings +are asserted equal: placement stamps a comment with it and the resolve step +recognises a thread by it. + +`bin/gh` goes on `PATH` ahead of the real `gh`. It logs every call, serves +fixture JSON through the step's own `jq`, keeps the request body the step sent, +and decides per case whether a call succeeds. `STUB_*` variables in `run_case` +and `run_resolve` select the fixtures and the answers. + +### The fixtures + +`fx/files*.json` are `GET /compare` responses. One JSON object each: compare +paginates its commits, and a second page carries no `files` key, so the step +reads one page. + +- `files.json` — four files, two with a patch, two without +- `files-short.json` — the same diff with `pkg/b.go` missing, which is what a + truncated response looks like +- `files-stale.json` — a different commit's diff, for the pushed-head case + +`fx/line-ok.tsv` and `fx/file-ok.txt` list the `path`/`side`/`line` and the +paths the stub accepts. A finding outside them is refused, which is how the +per-finding ladder is exercised. + +`fx/superseded*.json` are findings files carrying the driver's `supersedes` +linkage. `fx/all-placeable.json` carries it on no finding, which is what a +driver older than the linkage writes. + +`fx/check-*.json` are the driver's `check.json`, one per thread plan the resolve +step has to act on. The review threads themselves are generated in `run.sh`, +because every body has to open with the marker the workflow defines now: two +pages, and four threads that fail this step's own tests — the other identity, a +foreign account, no marker, and a marker quoted mid-body. + +## The reaction steps + +`reactions.sh` runs `Acknowledge the trigger`, `Answer the request` and +`Withdraw the reactions on a cancelled run` against `bin-reactions/gh`, which keeps +the reaction list a comment carries and serves it through the step's own `jq`. Each +case reports the exact set left on the trigger comment. + +No case names the step it runs. `conditions.py --select` names it, from the job state +and from `Post the verdict`'s outcome, so the two harnesses cannot drift and a case +cannot quietly stop exercising the step it claims to. + +Each case declares **two** job states, because a cancellation has a moment: the state +the runner reached `Answer the request` in, and the state it reached the withdrawal step +in. `success>cancelled` is a cancellation that arrived after the answer. The answer step +then posts its own thumb, so a late-cancellation case proves the outcome through the +steps rather than placing a reaction by hand. + +A case's `posted` column is `Post the verdict`'s own output: `true` when its comment +landed, `false` when the POST was refused, unreported when that step never ran. Not its +outcome. That step tolerates a refused POST and exits 0 either way, so its outcome reads +`success` on a verdict that never landed, and a thumb kept on that reading would stand +for a review nobody can see. + +Four properties every case holds to. A human's reaction is never withdrawn. Neither is +a reaction of this bot's that no step here chooses, so a `rocket` some other workflow +left survives. A thumb that answers a verdict already on the pull request survives a +later cancellation. And a run cancelled before it reached `Answer the request` takes +only the eyes: a thumb on the comment then belongs to an EARLIER run, whose verdict may +still stand. + +## The step conditions + +`conditions.py` covers what a shell harness cannot see. A step condition decides which +reaction step runs in which job state, and that is where the cancellation behaviour +lives. `Answer the request` reads a conclusion and may thumb the request, so it must +skip a cancelled run. The withdrawal step must take a cancelled run, unless +`Post the verdict` landed its comment. + +It models the runner's own rule that a condition naming none of +`always`/`cancelled`/`failure`/`success` is stored as `success() && (...)`, and treats +any term it does not decide as unknown rather than as false. + +Three checks are stated over the file rather than over a table, so they cover a step +added later. The two that walk steps walk **every job's raw steps list**, so an unnamed +step is not invisible to them, and both search the **whole step** rather than one key: + +- No step that can run on a cancelled job may reach `check_path` or + `verdict_produced`. An inline `${{ steps.drive.outputs.check_path }}` in `run:`, + `with:` or `if:` reaches the same value an `env:` key would. +- Every `steps.` a step reads must be a real id on an earlier step. Delete the id, + or move the reader in front of it, and the read is empty forever with no error + anywhere — and a harness that takes the value as an argument cannot notice. +- The withdrawal is the **last** step of the review job. The runner evaluates a + condition when it reaches the step, so any step after the withdrawal is a step during + which a cancellation leaves the eyes standing. Checking the position covers a step + appended later; mutating one ordering would not. + +No check needs telling where to look. A check that has to be pointed at a step is not +stated over the file. + +## The guard, and the two steps that react + +Runs the request-admission path under `bash` against a `gh` stub of its own, and +evaluates the shipped job conditions against synthetic event payloads. Five +steps are read: `Refuse an event this workflow does not handle`, `parse`, `Admit +the request`, `Acknowledge the trigger` and `Answer the request`. + +`gha.py` covers what a script harness cannot see. A job condition and a step's +`env:` mapping are GitHub expressions, and both decide which payload field a +request is read from, so both are evaluated here rather than restated. It models +four GitHub semantics the conditions rest on — case-insensitive string +comparison, `||` and `&&` yielding one operand each, **both short-circuiting**, +and `contains` over an array testing membership — and `--selftest` checks each +one. That model is read from GitHub's published expression semantics: nothing in +this directory calls a runner. + +Short-circuiting is the one to be careful with. The runner's Or and And nodes +return on the first truthy or falsy operand and never evaluate the rest, so a +`fromJSON` an operand nothing reaches would refuse never runs. A model that +evaluated eagerly reports a failure the runner does not have, and one assertion +here stated the opposite of what a real event does before this was modelled. + +Four modes: + +```sh +gha.py # the job's if:, as true or false +gha.py --env # what a step's env key resolves to +gha.py --input # a declared workflow_call input field +gha.py --selftest # the expression model itself +``` + +One group reaches outside this file. `ai-assistant.yml` answers the same comments +and reserves the exact `@seidroid review` body for the reviewer, so the last group +evaluates that workflow's own reply condition beside the parse and records which +tool answers each body. Every body is checked on all three comment events, because +the assistant has a branch each and this workflow now answers all three: a helper +naming one event would measure the division on the path that already had it and +infer the two this workflow adds. Two bodies both tools answer; the group says +which and why. + +`ai-assistant.yml` is in `workflow-test-self.yml`'s `paths:` filter for that +reason. Without it an edit there breaks an invariant stated here, and the break +lands on the next unrelated pull request that touches `seidroid-review.yml`. + +`bin-guard/gh` logs every call, serves the answer the case chose through the +step's own `--jq` filter, and tells the fork check from the label check by the +filter each sends. A failed read prints nothing and exits non-zero, which is the +shape `Admit the request` is written against: it captures stdout, so an empty +capture is what tells the fork check and the once-per-PR gate that nobody +answered. `bin/gh` beside it files an error body instead, because the placement +step reads one. + +### The fixtures + +There are none. A guard case turns on six payload fields and five API answers, +so each is built in the run from `STUB_*` and context arguments, where the case +that chose it can be read beside the assertion it drives. + +`STUB_TEAM`, `STUB_ORIGIN`, `STUB_LABELS`, `STUB_REVIEWS`, `STUB_COMMENTS` and +`STUB_REACTIONS` choose what the stub answers; `FAIL` on any of them is a read +that nobody answered. \ No newline at end of file diff --git a/test/seidroid-review/bin-guard/gh b/test/seidroid-review/bin-guard/gh new file mode 100755 index 0000000..66c4020 --- /dev/null +++ b/test/seidroid-review/bin-guard/gh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# gh stub for the guard steps: logs every call, serves the answer the case chose, +# and runs the step's own --jq filter over it. +# +# A failed read prints nothing and exits non-zero, which is the shape `Admit the +# request` is written against: it captures stdout, so an empty capture is what +# tells the fork check and the once-per-PR gate that nobody answered. The +# placement stub beside this one files an error body instead, because that step +# reads one. +# +# Five reads reach here. The team membership read and the once-per-PR reads are +# told apart by their path; the fork check and the label check share a path and +# are told apart by the filter each sends. +log() { printf '%s\n' "$*" >> "$STUB_LOG"; } + +argv=("$@") +joined="$*" + +filter="" +n=${#argv[@]} +for ((i = 0; i < n; i++)); do + [ "${argv[i]}" = "--jq" ] && filter="${argv[i + 1]}" +done + +serve() { # fixture-json + printf '%s' "$1" | jq -r "$filter" + exit 0 +} + +refuse() { # what-failed + log "CALL $1 FAILED" + echo "gh: the stub was told this read fails" >&2 + exit 1 +} + +# Ahead of the comment reads below: an issue comment's reactions path carries +# /issues/ and comments too, and the first matching case wins. +case "$joined" in + *"/reactions"*) + method=GET + for ((i = 0; i < n; i++)); do + [ "${argv[i]}" = "-X" ] && method="${argv[i + 1]}" + done + for a in "${argv[@]}"; do case "$a" in repos/*/reactions*) url="$a" ;; esac; done + log "CALL reaction $method ${url:-none}" + [ "${STUB_REACTIONS:-none}" = "FAIL" ] && refuse reaction + case "$method" in + GET) + case "${STUB_REACTIONS:-none}" in + eyes) serve '[{"id":1,"content":"eyes","user":{"login":"github-actions[bot]"}}]' ;; + *) serve '[]' ;; + esac + ;; + *) exit 0 ;; + esac + ;; +esac + +case "$joined" in + *"/teams/"*"/memberships/"*) + for a in "${argv[@]}"; do case "$a" in */memberships/*) who="${a##*/}" ;; esac; done + log "CALL membership $who" + [ "${STUB_TEAM:-active}" = "FAIL" ] && refuse membership + serve "{\"state\":\"${STUB_TEAM:-active}\"}" + ;; +esac + +case "$joined" in + *"/pulls/"*"/reviews"*) + log "CALL reviews" + [ "${STUB_REVIEWS:-none}" = "FAIL" ] && refuse reviews + case "${STUB_REVIEWS:-none}" in + blocked) serve "[{\"id\":11,\"state\":\"CHANGES_REQUESTED\",\"body\":\"${VERDICT_MARKER}a block\"}]" ;; + *) serve '[]' ;; + esac + ;; +esac + +case "$joined" in + *"/issues/"*"/comments"*) + log "CALL comments" + [ "${STUB_COMMENTS:-none}" = "FAIL" ] && refuse comments + case "${STUB_COMMENTS:-none}" in + verdict) serve "[{\"id\":22,\"user\":{\"type\":\"Bot\"},\"body\":\"${VERDICT_MARKER}a verdict\"}]" ;; + *) serve '[]' ;; + esac + ;; +esac + +case "$joined" in + *"/pulls/"*) + # The fork check names head.repo.id in its filter; the label check names labels. + case "$filter" in + *head.repo.id*) + log "CALL origin" + [ "${STUB_ORIGIN:-same}" = "FAIL" ] && refuse origin + case "${STUB_ORIGIN:-same}" in + same) serve '{"head":{"repo":{"id":1}},"base":{"repo":{"id":1}}}' ;; + fork) serve '{"head":{"repo":{"id":2}},"base":{"repo":{"id":1}}}' ;; + null) serve '{"head":{"repo":null},"base":{"repo":{"id":1}}}' ;; + esac + ;; + *labels*) + log "CALL labels" + [ "${STUB_LABELS:-}" = "FAIL" ] && refuse labels + names="$(jq -nc --arg list "${STUB_LABELS:-}" \ + 'if $list == "" then [] else ($list | split(",") | map({name: .})) end')" + serve "{\"labels\":$names}" + ;; + esac + ;; +esac + +log "CALL unhandled $joined" +exit 1 diff --git a/test/seidroid-review/bin-reactions/gh b/test/seidroid-review/bin-reactions/gh new file mode 100755 index 0000000..001122a --- /dev/null +++ b/test/seidroid-review/bin-reactions/gh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# gh stub for the reaction steps: keeps a reaction list, serves it through the +# step's own jq, and decides per case which calls succeed. +# +# STUB_STATE the reaction array this comment carries +# STUB_LOG one line per call +# STUB_ACTOR the login a POST reacts as +# STUB_LIST ok | FAIL +# STUB_DELETE ok | FAIL +# STUB_POST ok | FAIL +set -uo pipefail +log() { printf '%s\n' "$*" >> "$STUB_LOG"; } +[ -s "$STUB_STATE" ] || echo '[]' > "$STUB_STATE" + +verb=GET path="" filter="" ; declare -a fields=() +[ "${1:-}" = api ] && shift +while [ "$#" -gt 0 ]; do + case "$1" in + -X|--method) verb="$2"; shift 2 ;; + --jq) filter="$2"; shift 2 ;; + --paginate|--slurp) shift ;; + -f|--raw-field|--field) fields+=("$2"); shift 2 ;; + -H) shift 2 ;; + *) path="$1"; shift ;; + esac +done + +case "$verb:$path" in + GET:*/reactions) + log "CALL list $path" + [ "${STUB_LIST:-ok}" = FAIL ] && exit 1 + # Through the step's own filter, so the filter is under test and not the harness's + # idea of it. + if [ -n "$filter" ]; then jq -r "$filter" < "$STUB_STATE"; else cat "$STUB_STATE"; fi + ;; + DELETE:*/reactions/*) + rid="${path##*/}" + log "CALL delete $rid $path" + [ "${STUB_DELETE:-ok}" = FAIL ] && exit 1 + jq -e --argjson rid "$rid" 'any(.[]; .id == $rid)' < "$STUB_STATE" > /dev/null || { + log " no such reaction $rid"; exit 1; } + jq --argjson rid "$rid" '[.[] | select(.id != $rid)]' < "$STUB_STATE" > "$STUB_STATE.n" + mv "$STUB_STATE.n" "$STUB_STATE" + ;; + POST:*/reactions) + content="" + for f in "${fields[@]:-}"; do case "$f" in content=*) content="${f#content=}" ;; esac; done + log "CALL post $content $path" + [ "${STUB_POST:-ok}" = FAIL ] && exit 1 + # Idempotent per (user, content): the API returns the reaction already there + # rather than adding a second one. + if jq -e --arg c "$content" --arg u "${STUB_ACTOR:-github-actions[bot]}" \ + 'any(.[]; .content == $c and .user.login == $u)' < "$STUB_STATE" > /dev/null; then + log " already there" + else + next="$(jq '[.[].id] | max // 0 | . + 1' < "$STUB_STATE")" + jq --arg c "$content" --arg u "${STUB_ACTOR:-github-actions[bot]}" --argjson id "$next" \ + '. + [{id: $id, content: $c, user: {login: $u}}]' < "$STUB_STATE" > "$STUB_STATE.n" + mv "$STUB_STATE.n" "$STUB_STATE" + fi + jq -c --arg c "$content" '[.[] | select(.content == $c)] | last' < "$STUB_STATE" + ;; + *) + log "CALL UNSTUBBED $verb $path" + echo "gh stub: unstubbed $verb $path" >&2 + exit 64 + ;; +esac diff --git a/test/seidroid-review/bin/gh b/test/seidroid-review/bin/gh new file mode 100755 index 0000000..1adfd7d --- /dev/null +++ b/test/seidroid-review/bin/gh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# gh stub: logs every call, serves fixture JSON, and decides success per scenario. +log() { printf '%s\n' "$*" >> "$STUB_LOG"; } + +argv=("$@") +joined="$*" + +# --- resolve one review thread (the mutation) -------------------------------- +# Matched before the thread read, because only this one names the mutation. +case "$joined" in + *resolveReviewThread*) + for ((i=0;i<${#argv[@]};i++)); do + case "${argv[i]}" in threadId=*) id="${argv[i]#threadId=}" ;; esac + done + log "CALL resolve ${id:-}" + case "${STUB_RESOLVE:-ok}" in + ok) printf '%s\n' '{"data":{"resolveReviewThread":{"thread":{"isResolved":true}}}}' + exit 0 ;; + # gh splits a failure across both streams, which is why the step captures both. + 403) printf '%s\n' '{"message":"Resource not accessible by integration"}' + echo 'gh: HTTP 403' >&2; exit 1 ;; + *) printf '%s\n' '{"message":"Could not resolve to a node"}'; exit 1 ;; + esac + ;; +esac + +# --- read every page of the pull request's review threads -------------------- +case "$joined" in + *reviewThreads*) + log "CALL threads" + if [ "${STUB_THREADS:-}" = "FAIL" ]; then + echo 'gh: Bad credentials (HTTP 401)' >&2 + exit 1 + fi + # One JSON document per page, which is what --paginate writes and what the + # step's `jq -s` folds into one array. + cat "$STUB_THREADS" + exit 0 + ;; +esac + +# --- read the pull request object (base.sha and changed_files) ---------------- +case "$joined" in + *"/pulls/$STUB_PR"|*"/pulls/$STUB_PR --jq"*) + log "CALL pull" + if [ "$STUB_BASE" = "FAIL" ]; then + # A failed read answers with the error object, which the step files and reads. + printf '%s\n' '{"message":"Not Found","status":"404"}' + exit 1 + fi + # NONE serves a response carrying no changed_files, which is the only way the + # step reaches its file-count cap test. + if [ "${STUB_CHANGED_FILES:-}" = "NONE" ]; then + printf '{"number":%s,"base":{"sha":"%s"}}\n' "$STUB_PR" "$STUB_BASE" + else + printf '{"number":%s,"base":{"sha":"%s"},"changed_files":%s}\n' \ + "$STUB_PR" "$STUB_BASE" "${STUB_CHANGED_FILES:-4}" + fi + exit 0 + ;; +esac + +# --- the diff between two commits --------------------------------------------- +case "$joined" in + *"/compare/"*) + # The commit range the step asked for, so a test can assert which one it read. + for a in "${argv[@]}"; do case "$a" in *"/compare/"*) range="${a#*/compare/}" ;; esac; done + log "CALL compare ${range%%\?*}" + if [ "$STUB_FILES" = "FAIL" ]; then + printf '%s\n' '{"message":"Not Found","status":"404"}' + exit 1 + fi + # One object. compare paginates its commits, and a second page carries no files + # key at all, so the step asks for one page and reads its files. + fx="$STUB_FILES" + case "${range%%\?*}" in + *"...$STUB_STALE_SHA") fx="${STUB_FILES_STALE:-$STUB_FILES}" ;; + esac + cat "$fx" + exit 0 + ;; +esac + +# --- create a review (the batch) --------------------------------------------- +case "$joined" in + *"/reviews"*) + n=${#argv[@]} + for ((i=0;i> "$STUB_BODIES" + if [ "$subject" = "file" ]; then + log "CALL file-comment $path" + grep -qxF -- "$path" "$STUB_FILE_OK" && exit 0 + printf '%s\n' '{"message":"Validation Failed","status":"422"}' + exit 1 + fi + log "CALL line-comment $path $side $line" + grep -qxF -- "$path $side $line" "$STUB_LINE_OK" && exit 0 + printf '%s\n' '{"message":"Validation Failed","status":"422"}' + exit 1 + ;; +esac + +log "CALL unhandled $joined" +exit 1 diff --git a/test/seidroid-review/conditions.py b/test/seidroid-review/conditions.py new file mode 100644 index 0000000..9688670 --- /dev/null +++ b/test/seidroid-review/conditions.py @@ -0,0 +1,412 @@ +#!/usr/bin/env python3 +"""Checks which reaction step runs in which job state. + +A shell harness cannot see a step condition, and the reaction steps put their +whole cancellation behaviour there: `Answer the request` reads a conclusion and +may post a thumb, so it must not run on a cancelled run, and the withdrawal step +must. + +This models the GitHub expression engine over the subset the three conditions +use: &&, ||, !, parentheses, == and !=, single-quoted strings, the status +functions, and dotted context reads. It applies the runner's own wrapping rule -- +actions/runner, PipelineTemplateConverter.ConvertToIfCondition: + + return hasStatusFunction ? condition : $"{Success}() && ({condition})"; + +so a condition that names none of always/cancelled/failure/success skips a +cancelled run without saying so. + + conditions.py + conditions.py --select + +--select names the reaction steps GitHub runs in that state, in job order. +reactions.sh uses it, so no case there hardcodes which step a state selects and the +two harnesses cannot drift. +""" +import re +import sys + +import yaml + +# One name for the step, so the case table, the selector and the position check below +# cannot disagree about which step they mean. +WITHDRAW = "Withdraw the reactions on a cancelled run" + +# step -> (job state, mode, comment id, `Post the verdict` posted?) -> does it run? +# +# Two invariants live here. `Answer the request` reads check_path and verdict_produced, +# and a cancellation arriving after the driver finishes leaves both populated -- so a +# run of it on a cancelled job posts a thumb for a verdict no step published. And the +# withdrawal must not strip a thumb that answers a verdict which DID land: a +# cancellation can arrive after `Post the verdict` posted the comment, and the trigger +# comment then carries an honest answer. That step's `posted` output is the fact, not +# its outcome, which reads success even when the POST was refused. +EXPECTED = { + "Acknowledge the trigger": { + ("success", "review", "7", ""): True, + ("failure", "review", "7", ""): False, + ("cancelled", "review", "7", ""): False, + ("success", "review", "", ""): False, + ("success", "close", "7", ""): False, + }, + "Answer the request": { + ("success", "review", "7", ""): True, + ("failure", "review", "7", ""): True, + ("cancelled", "review", "7", ""): False, + ("cancelled", "review", "7", "true"): False, + ("success", "review", "", ""): False, + ("success", "close", "7", ""): False, + ("cancelled", "close", "7", ""): False, + }, + WITHDRAW: { + # Nothing to withdraw on a run that was not cancelled. + ("success", "review", "7", "true"): False, + ("failure", "review", "7", "false"): False, + # Cancelled before the verdict landed: withdraw. + ("cancelled", "review", "7", "false"): True, + # A value this step cannot read clears rather than leaving a thumb. + ("cancelled", "review", "7", ""): True, + # THE CASE FOR THE LATE WINDOW. The verdict is on the pull request and the + # thumb answers it, so the thumb stays. + ("cancelled", "review", "7", "true"): False, + ("cancelled", "review", "", "false"): False, + ("cancelled", "close", "7", "false"): False, + }, +} + +# What a step must not be able to reach when it runs on a cancelled job, or a cancelled +# run can state an outcome. Matched against the WHOLE step rather than its `env` block: +# an inline ${{ steps.drive.outputs.check_path }} in `run:`, `with:` or `if:` reaches +# the same value and would otherwise pass unseen. +CONCLUSION_INPUTS = ("check_path", "verdict_produced") + +STATUS_FUNCS = ("always", "cancelled", "failure", "success") + +TOKEN = re.compile( + r"\s*(?:(?P'(?:[^']|'')*')" + r"|(?P&&|\|\||==|!=|!|\(|\))" + r"|(?P[A-Za-z_][A-Za-z0-9_.\-]*))" +) + + +def lex(text): + pos, out = 0, [] + while pos < len(text): + if text[pos].isspace(): + pos += 1 + continue + m = TOKEN.match(text, pos) + if not m: + raise SyntaxError(f"cannot lex at {text[pos:pos + 20]!r}") + pos = m.end() + if m.group("str") is not None: + out.append(("str", m.group("str")[1:-1].replace("''", "'"))) + elif m.group("op") is not None: + out.append(("op", m.group("op"))) + else: + out.append(("word", m.group("word"))) + return out + + +class Unknown: + """A term this model does not decide.""" + + def __repr__(self): + return "UNKNOWN" + + +UNKNOWN = Unknown() + + +class Ctx: + def __init__(self, state, mode, comment_id, posted="", lenient=False): + self.state = state + self.lenient = lenient + self.values = { + "inputs.mode": mode, + "needs.guard.outputs.comment_id": comment_id, + # `true` when the verdict comment landed on the pull request, `false` when + # the POST was refused, empty when that step never reported. One boolean + # about another step, with no conclusion in it. + "steps.verdict.outputs.posted": posted, + # Modelled BESIDE it, and derived rather than passed, so a condition that + # goes back to reading the outcome fails an assertion instead of crashing + # this model. `Post the verdict` tolerates a refused POST and ends on a + # call whose failure it swallows, so it exits 0 and reads success whenever + # it reported at all -- including on a verdict that never landed. That is + # why the gate cannot use it. + "steps.verdict.outcome": ( + "success" if posted in ("true", "false") else "skipped" + ), + } + + def func(self, name): + if name == "always": + return True + if name in ("cancelled", "success", "failure"): + return self.state == ("cancelled" if name == "cancelled" else name) + raise KeyError(f"unmodelled function {name}()") + + def read(self, path): + if path not in self.values: + # The sweep at the end asks only whether a step can run on a cancelled + # job. Every other term is UNKNOWN, and the three-valued operators keep + # the answer sound without modelling contexts this file does not decide. + if self.lenient: + return UNKNOWN + raise KeyError(f"unmodelled context read {path}") + return self.values[path] + + +def truthy(value): + if value is UNKNOWN: + return UNKNOWN + if isinstance(value, bool): + return value + if isinstance(value, str): + return value != "" + return bool(value) + + +def and3(left, right): + if left is False or right is False: + return False + if left is UNKNOWN or right is UNKNOWN: + return UNKNOWN + return True + + +def or3(left, right): + if left is True or right is True: + return True + if left is UNKNOWN or right is UNKNOWN: + return UNKNOWN + return True if (left or right) else False + + +def not3(value): + return UNKNOWN if value is UNKNOWN else (not value) + + +class Parser: + def __init__(self, tokens, ctx): + self.t, self.i, self.ctx = tokens, 0, ctx + + def peek(self): + return self.t[self.i] if self.i < len(self.t) else (None, None) + + def take(self, kind=None, value=None): + k, v = self.peek() + if kind and (k != kind or (value is not None and v != value)): + raise SyntaxError(f"expected {value or kind}, found {v!r}") + self.i += 1 + return v + + def parse(self): + value = self.or_() + if self.i != len(self.t): + raise SyntaxError(f"trailing tokens at {self.t[self.i:]}") + return value + + def or_(self): + left = self.and_() + while self.peek() == ("op", "||"): + self.take() + right = self.and_() # parsed first: Python's or short-circuits + left = or3(truthy(left), truthy(right)) + return left + + def and_(self): + left = self.cmp_() + while self.peek() == ("op", "&&"): + self.take() + right = self.cmp_() + left = and3(truthy(left), truthy(right)) + return left + + def cmp_(self): + left = self.unary() + k, v = self.peek() + if k == "op" and v in ("==", "!="): + self.take() + right = self.unary() + if left is UNKNOWN or right is UNKNOWN: + return UNKNOWN + return (left == right) if v == "==" else (left != right) + return left + + def unary(self): + if self.peek() == ("op", "!"): + self.take() + return not3(truthy(self.unary())) + return self.primary() + + def primary(self): + k, v = self.peek() + if k == "op" and v == "(": + self.take() + inner = self.or_() + self.take("op", ")") + return inner + if k == "str": + return self.take() + if k == "word": + name = self.take() + if self.peek() == ("op", "("): + self.take() + self.take("op", ")") + return self.ctx.func(name) + if name in ("true", "false"): + return name == "true" + return self.ctx.read(name) + raise SyntaxError(f"unexpected {v!r}") + + +def stored_condition(raw): + """What the runner keeps as the step condition.""" + expr = " ".join(str(raw).split()) + if expr.startswith("${{") and expr.endswith("}}"): + expr = expr[3:-2].strip() + if {v for k, v in lex(expr) if k == "word"} & set(STATUS_FUNCS): + return expr + return f"success() && ({expr})" + + +def dump(step): + """The whole step as text. An inline ${{ }} in `run:`, `with:` or `if:` reaches the + same value an `env:` key would, so every check searches all of it.""" + return yaml.safe_dump(step, default_flow_style=False) + + +def label(step, index): + """What to call a step in a finding. A step need not have a name.""" + return step.get("name") or step.get("uses") or f"step {index + 1}" + + +REACTION_STEPS = ( + "Acknowledge the trigger", + "Answer the request", + WITHDRAW, +) + + +def load(path): + with open(path, encoding="utf-8") as handle: + return yaml.safe_load(handle) + + +def select(argv): + """Print the reaction steps that run, one per line, in job order.""" + workflow, state, mode, cid, posted = argv + steps = {s["name"]: s for s in load(workflow)["jobs"]["review"]["steps"] if "name" in s} + for name in REACTION_STEPS: + if name not in steps: + continue + expr = stored_condition(steps[name].get("if", "success()")) + ctx = Ctx(state, mode, cid, "" if posted == "-" else posted) + if truthy(Parser(lex(expr), ctx).parse()) is True: + print(name) + return 0 + + +def main(): + if sys.argv[1] == "--select": + return select(sys.argv[2:]) + with open(sys.argv[1], encoding="utf-8") as handle: + doc = yaml.safe_load(handle) + steps = {s["name"]: s for s in doc["jobs"]["review"]["steps"] if "name" in s} + + passed = failed = 0 + + def check(label, want, got): + nonlocal passed, failed + if want == got: + passed += 1 + else: + failed += 1 + print(f" FAIL {label}: want [{want}] got [{got}]") + + for name, cases in EXPECTED.items(): + if name not in steps: + print(f" FAIL no step named {name!r} in the review job") + failed += 1 + continue + expr = stored_condition(steps[name].get("if", "success()")) + print(f"== {name}\n {expr}") + for (state, mode, cid, posted), want in cases.items(): + got = truthy(Parser(lex(expr), Ctx(state, mode, cid, posted)).parse()) + check( + f"{name} / {state} / {mode} / id={cid or 'empty'}" + f" / posted={posted or 'unreported'}", + want, + got, + ) + + # The withdrawal has to be the LAST step of the review job, and that is not a + # tidiness preference. The runner evaluates a condition when it reaches the step, so + # any step placed after this one is a step during which a cancellation leaves the + # eyes standing: the withdrawal was already evaluated and skipped by then. Checking + # the position rather than mutating one ordering covers a step appended later. + print("== the withdrawal is the last step of the review job") + review_steps = doc["jobs"]["review"]["steps"] + last = label(review_steps[-1], len(review_steps) - 1) + check(f"last step is the withdrawal, not {last!r}", WITHDRAW, last) + + # Both checks below walk the RAW steps list. Keying them off a name drops an + # unnamed step, and `- uses: actions/checkout@v7` with no `name:` is the usual + # shape -- so the very step most likely to arrive later would be the one the + # invariant could not see. The guard job already carries one unnamed step. + # + # Every job, not only `review`: a reaction step could be added anywhere, and a + # check that has to be told where to look is not stated over the file. + for job_name, job in doc["jobs"].items(): + raw = job.get("steps") or [] + ids = {st["id"]: i for i, st in enumerate(raw) if st.get("id")} + + # A steps. read only works when that id exists and belongs to an EARLIER + # step. Delete the id, or move the reader in front of it, and the read is + # silently empty forever. The model takes an outcome as an argument, so nothing + # above would notice. + # + # The whole step, not its `if`: the withdrawal set reads an outcome through + # `env`, and `run:` and `with:` reach the same values. + print(f"== every steps. read in job '{job_name}' exists, and runs before it") + for i, st in enumerate(raw): + where = label(st, i) + for ref in sorted(set(re.findall(r"steps\.([A-Za-z0-9_-]+)\.", dump(st)))): + if ref not in ids: + check(f"{where} reads steps.{ref}, which is no step's id", True, False) + elif ids[ref] > i: + check(f"{where} reads steps.{ref}, which runs later", True, False) + else: + check(f"{where} reads steps.{ref}", True, True) + + # No step that can run on a cancelled job may reach a conclusion, or a cancelled + # run can state an outcome. Stated over the file rather than over the table + # above, so a step added later is covered too. + print(f"== nothing in job '{job_name}' that reads a conclusion runs on a cancelled run") + for i, st in enumerate(raw): + where = label(st, i) + expr = stored_condition(st.get("if", "success()")) + outcome = truthy( + Parser( + lex(expr), Ctx("cancelled", "review", "7", lenient=True) + ).parse() + ) + # UNKNOWN counts as "can run": the check must not pass because a term went + # unmodelled. + runs = outcome is not False + reads = [k for k in CONCLUSION_INPUTS if k in dump(st)] + check( + f"{where}" + + (f" runs on a cancelled run and reads {', '.join(reads)}" if reads else ""), + True, + not (runs and reads), + ) + + print(f"\nassertions: {passed} passed, {failed} failed") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/seidroid-review/extract.py b/test/seidroid-review/extract.py new file mode 100644 index 0000000..ad804ff --- /dev/null +++ b/test/seidroid-review/extract.py @@ -0,0 +1,11 @@ +import sys, yaml +path, step, out = sys.argv[1], sys.argv[2], sys.argv[3] +key = sys.argv[4] if len(sys.argv) > 4 else "FINDING_MARKER" +d = yaml.safe_load(open(path, encoding="utf-8")) +for job in d["jobs"].values(): + for s in job.get("steps", []): + if s.get("name") == step or s.get("id") == step: + open(out, "w", encoding="utf-8").write(s["run"]) + print(d["env"][key]) + sys.exit(0) +sys.exit("step not found: " + step) diff --git a/test/seidroid-review/fx/all-placeable.json b/test/seidroid-review/fx/all-placeable.json new file mode 100644 index 0000000..ad8516f --- /dev/null +++ b/test/seidroid-review/fx/all-placeable.json @@ -0,0 +1 @@ +[{"file": "pkg/a.go", "severity": "blocker", "detail": "First finding.\n\nA second paragraph with a\ttab.", "line": 11, "side": "RIGHT"}, {"file": "pkg/a.go", "severity": "suggestion", "detail": "Backtick `code`, a \"quote\" and a $dollar.", "line": 12, "side": "RIGHT"}, {"file": "pkg/b.go", "severity": "nit", "detail": "Third.", "line": 2, "side": "RIGHT"}, {"file": "pkg/a.go", "severity": "blocker", "detail": "On the removed line.", "line": 11, "side": "LEFT"}] \ No newline at end of file diff --git a/test/seidroid-review/fx/broken.json b/test/seidroid-review/fx/broken.json new file mode 100644 index 0000000..f2602dc --- /dev/null +++ b/test/seidroid-review/fx/broken.json @@ -0,0 +1 @@ +{ not json \ No newline at end of file diff --git a/test/seidroid-review/fx/check-abc.json b/test/seidroid-review/fx/check-abc.json new file mode 100644 index 0000000..6b91937 --- /dev/null +++ b/test/seidroid-review/fx/check-abc.json @@ -0,0 +1,12 @@ +{ + "conclusion": "failure", + "title": "3 findings", + "summary": "Three findings restate three open threads.", + "counts": {"blocking": 3, "non_blocking": 0, "placeable": 3, "pre_existing": 0}, + "threads": { + "addressed": [], + "superseded": ["PRRT_kwDOABCDEF4Ax1y2", "PRRT_kwDOABCDEF4Bz3w4", "PRRT_kwDOABCDEF4Cq9r8"], + "refused": [], + "refused_total": 0 + } +} diff --git a/test/seidroid-review/fx/check-addressed.json b/test/seidroid-review/fx/check-addressed.json new file mode 100644 index 0000000..d2d7c81 --- /dev/null +++ b/test/seidroid-review/fx/check-addressed.json @@ -0,0 +1,11 @@ +{ + "conclusion": "success", + "title": "0 findings", + "summary": "The diff addressed one earlier finding.", + "threads": { + "addressed": ["PRRT_kwDOABCDEF4Ax1y2"], + "superseded": [], + "refused": ["PRRT_neverOurs"], + "refused_total": 1 + } +} diff --git a/test/seidroid-review/fx/check-identity.json b/test/seidroid-review/fx/check-identity.json new file mode 100644 index 0000000..f39320d --- /dev/null +++ b/test/seidroid-review/fx/check-identity.json @@ -0,0 +1,11 @@ +{ + "conclusion": "failure", + "title": "4 findings", + "summary": "Four findings restate threads under four identities.", + "threads": { + "addressed": [], + "superseded": ["PRRT_kwDOABCDEF4Dm5n6", "PRRT_kwDOABCDEF4Es7t8", "PRRT_kwDOABCDEF4Gw3x4", "PRRT_kwDOABCDEF4Hy5z6"], + "refused": [], + "refused_total": 0 + } +} diff --git a/test/seidroid-review/fx/check-noplan.json b/test/seidroid-review/fx/check-noplan.json new file mode 100644 index 0000000..c25666e --- /dev/null +++ b/test/seidroid-review/fx/check-noplan.json @@ -0,0 +1,5 @@ +{ + "conclusion": "success", + "title": "0 findings", + "summary": "A driver that writes no thread plan at all." +} diff --git a/test/seidroid-review/fx/check-one.json b/test/seidroid-review/fx/check-one.json new file mode 100644 index 0000000..46acb54 --- /dev/null +++ b/test/seidroid-review/fx/check-one.json @@ -0,0 +1,11 @@ +{ + "conclusion": "failure", + "title": "1 finding", + "summary": "One finding restates one open thread.", + "threads": { + "addressed": [], + "superseded": ["PRRT_kwDOABCDEF4Ax1y2"], + "refused": [], + "refused_total": 0 + } +} diff --git a/test/seidroid-review/fx/check-resolved.json b/test/seidroid-review/fx/check-resolved.json new file mode 100644 index 0000000..5d50ad9 --- /dev/null +++ b/test/seidroid-review/fx/check-resolved.json @@ -0,0 +1,11 @@ +{ + "conclusion": "failure", + "title": "1 finding", + "summary": "One finding restates a thread already marked resolved.", + "threads": { + "addressed": [], + "superseded": ["PRRT_kwDOABCDEF4Fu1v2"], + "refused": [], + "refused_total": 0 + } +} diff --git a/test/seidroid-review/fx/empty.json b/test/seidroid-review/fx/empty.json new file mode 100644 index 0000000..e69de29 diff --git a/test/seidroid-review/fx/emptyarray.json b/test/seidroid-review/fx/emptyarray.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/test/seidroid-review/fx/emptyarray.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/test/seidroid-review/fx/file-ok.txt b/test/seidroid-review/fx/file-ok.txt new file mode 100644 index 0000000..6badff2 --- /dev/null +++ b/test/seidroid-review/fx/file-ok.txt @@ -0,0 +1,4 @@ +pkg/a.go +pkg/b.go +assets/logo.png +pkg/huge.go diff --git a/test/seidroid-review/fx/files-short.json b/test/seidroid-review/fx/files-short.json new file mode 100644 index 0000000..ebd21df --- /dev/null +++ b/test/seidroid-review/fx/files-short.json @@ -0,0 +1 @@ +{"files":[{"filename":"pkg/a.go","patch":"@@ -10,4 +10,6 @@\n ctx10\n-gone11\n+added11\n+added12\n ctx13"},{"filename":"assets/logo.png"},{"filename":"pkg/huge.go"}]} diff --git a/test/seidroid-review/fx/files-stale.json b/test/seidroid-review/fx/files-stale.json new file mode 100644 index 0000000..0e6c73b --- /dev/null +++ b/test/seidroid-review/fx/files-stale.json @@ -0,0 +1 @@ +{"files": [{"filename": "pkg/a.go", "patch": "@@ -40,3 +40,4 @@\n ctx40\n+added41\n+added42\n ctx43"}, {"filename": "pkg/b.go", "patch": "@@ -80,2 +80,3 @@\n eighty\n+eightyone\n eightytwo"}]} diff --git a/test/seidroid-review/fx/files.json b/test/seidroid-review/fx/files.json new file mode 100644 index 0000000..d8b4f07 --- /dev/null +++ b/test/seidroid-review/fx/files.json @@ -0,0 +1 @@ +{"files":[{"filename":"pkg/a.go","patch":"@@ -10,4 +10,6 @@\n ctx10\n-gone11\n+added11\n+added12\n ctx13"},{"filename":"pkg/b.go","patch":"@@ -1,2 +1,3 @@\n one\n+two\n three"},{"filename":"assets/logo.png"},{"filename":"pkg/huge.go"}]} diff --git a/test/seidroid-review/fx/line-ok-minus-one.tsv b/test/seidroid-review/fx/line-ok-minus-one.tsv new file mode 100644 index 0000000..3635ccb --- /dev/null +++ b/test/seidroid-review/fx/line-ok-minus-one.tsv @@ -0,0 +1,4 @@ +pkg/a.go RIGHT 11 +pkg/b.go RIGHT 2 +pkg/a.go LEFT 11 +pkg/huge.go RIGHT 120 diff --git a/test/seidroid-review/fx/line-ok.tsv b/test/seidroid-review/fx/line-ok.tsv new file mode 100644 index 0000000..4030781 --- /dev/null +++ b/test/seidroid-review/fx/line-ok.tsv @@ -0,0 +1,5 @@ +pkg/a.go RIGHT 11 +pkg/a.go RIGHT 12 +pkg/b.go RIGHT 2 +pkg/a.go LEFT 11 +pkg/huge.go RIGHT 120 diff --git a/test/seidroid-review/fx/mixed.json b/test/seidroid-review/fx/mixed.json new file mode 100644 index 0000000..1b96cfb --- /dev/null +++ b/test/seidroid-review/fx/mixed.json @@ -0,0 +1 @@ +[{"file": "pkg/a.go", "severity": "blocker", "detail": "First finding.\n\nA second paragraph with a\ttab.", "line": 11, "side": "RIGHT"}, {"file": "pkg/a.go", "severity": "suggestion", "detail": "Backtick `code`, a \"quote\" and a $dollar.", "line": 12, "side": "RIGHT"}, {"file": "pkg/b.go", "severity": "nit", "detail": "Third.", "line": 2, "side": "RIGHT"}, {"file": "pkg/a.go", "severity": "blocker", "detail": "On the removed line.", "line": 11, "side": "LEFT"}, {"file": "pkg/a.go", "severity": "suggestion", "detail": "Outside the hunks.", "line": 999, "side": "RIGHT"}, {"file": "pkg/untouched.go", "severity": "blocker", "detail": "A file the pull request never touches.", "line": 5, "side": "RIGHT"}, {"file": "pkg/b.go", "severity": "suggestion", "detail": "No line at all."}] \ No newline at end of file diff --git a/test/seidroid-review/fx/none.tsv b/test/seidroid-review/fx/none.tsv new file mode 100644 index 0000000..e69de29 diff --git a/test/seidroid-review/fx/none.txt b/test/seidroid-review/fx/none.txt new file mode 100644 index 0000000..e69de29 diff --git a/test/seidroid-review/fx/nopatch.json b/test/seidroid-review/fx/nopatch.json new file mode 100644 index 0000000..b130f0a --- /dev/null +++ b/test/seidroid-review/fx/nopatch.json @@ -0,0 +1 @@ +[{"file": "pkg/huge.go", "line": 120, "side": "RIGHT", "severity": "blocker", "detail": "A line the index cannot see and the API can."}, {"file": "assets/logo.png", "line": 1, "side": "RIGHT", "severity": "nit", "detail": "A binary file has no line to take."}, {"file": "pkg/a.go", "line": 11, "side": "RIGHT", "severity": "suggestion", "detail": "An ordinary anchored finding beside them."}] \ No newline at end of file diff --git a/test/seidroid-review/fx/odd.json b/test/seidroid-review/fx/odd.json new file mode 100644 index 0000000..a65b899 --- /dev/null +++ b/test/seidroid-review/fx/odd.json @@ -0,0 +1 @@ +[{"file": "pkg/a.go", "line": "12", "side": "right", "severity": "nit", "detail": "A line the model wrote as a string."}, {"file": "pkg/a.go", "line": "@/etc/passwd", "side": "RIGHT", "severity": "blocker", "detail": "A line that is not a number."}, {"file": "pkg/b.go", "line": 2, "severity": "suggestion", "detail": "No side named."}, {"file": "", "line": 1, "side": "RIGHT", "severity": "nit", "detail": "No file named."}] \ No newline at end of file diff --git a/test/seidroid-review/fx/shift.json b/test/seidroid-review/fx/shift.json new file mode 100644 index 0000000..fc7f8df --- /dev/null +++ b/test/seidroid-review/fx/shift.json @@ -0,0 +1 @@ +[{"file": "pkg/a.go", "line": 999, "side": "RIGHT", "severity": "", "detail": "An unrated finding off the hunks."}, {"file": "pkg/untouched.go", "line": 3, "side": "", "severity": "", "detail": "An unrated finding nowhere."}] \ No newline at end of file diff --git a/test/seidroid-review/fx/superseded-b.json b/test/seidroid-review/fx/superseded-b.json new file mode 100644 index 0000000..d3246ec --- /dev/null +++ b/test/seidroid-review/fx/superseded-b.json @@ -0,0 +1 @@ +[{"file": "pkg/a.go", "line": 11, "side": "RIGHT", "severity": "blocker", "detail": "B's replacement, on a changed line.", "supersedes": ["PRRT_kwDOABCDEF4Bz3w4"]}] diff --git a/test/seidroid-review/fx/superseded-batch.json b/test/seidroid-review/fx/superseded-batch.json new file mode 100644 index 0000000..670f7ed --- /dev/null +++ b/test/seidroid-review/fx/superseded-batch.json @@ -0,0 +1 @@ +[{"file": "pkg/a.go", "line": 11, "side": "RIGHT", "severity": "blocker", "detail": "A's replacement.", "supersedes": ["PRRT_kwDOABCDEF4Ax1y2"]}, {"file": "pkg/a.go", "line": 12, "side": "RIGHT", "severity": "blocker", "detail": "B's replacement.", "supersedes": ["PRRT_kwDOABCDEF4Bz3w4"]}] diff --git a/test/seidroid-review/fx/superseded-file.json b/test/seidroid-review/fx/superseded-file.json new file mode 100644 index 0000000..38049fb --- /dev/null +++ b/test/seidroid-review/fx/superseded-file.json @@ -0,0 +1 @@ +[{"file": "pkg/a.go", "line": 999, "side": "RIGHT", "severity": "blocker", "detail": "A's replacement, off the hunks of a file the diff carries.", "supersedes": ["PRRT_kwDOABCDEF4Ax1y2"]}, {"file": "pkg/untouched.go", "line": 3, "side": "RIGHT", "severity": "suggestion", "detail": "B's replacement, on no file of this diff.", "supersedes": ["PRRT_kwDOABCDEF4Bz3w4"]}] diff --git a/test/seidroid-review/fx/superseded-short.json b/test/seidroid-review/fx/superseded-short.json new file mode 100644 index 0000000..819ee29 --- /dev/null +++ b/test/seidroid-review/fx/superseded-short.json @@ -0,0 +1 @@ +[{"file": "pkg/b.go", "line": 2, "side": "RIGHT", "severity": "blocker", "detail": "A's replacement, on a file the short list drops.", "supersedes": ["PRRT_kwDOABCDEF4Ax1y2"]}, {"file": "pkg/untouched.go", "line": 5, "side": "RIGHT", "severity": "blocker", "detail": "B's replacement, on no file of this diff.", "supersedes": ["PRRT_kwDOABCDEF4Bz3w4"]}] diff --git a/test/seidroid-review/fx/superseded-unknown.json b/test/seidroid-review/fx/superseded-unknown.json new file mode 100644 index 0000000..e65c23e --- /dev/null +++ b/test/seidroid-review/fx/superseded-unknown.json @@ -0,0 +1 @@ +[{"file": "pkg/huge.go", "line": 120, "side": "RIGHT", "severity": "blocker", "detail": "A's replacement, on a line the index cannot see.", "supersedes": ["PRRT_kwDOABCDEF4Ax1y2"]}] diff --git a/test/seidroid-review/fx/superseded.json b/test/seidroid-review/fx/superseded.json new file mode 100644 index 0000000..767d414 --- /dev/null +++ b/test/seidroid-review/fx/superseded.json @@ -0,0 +1 @@ +[{"file": "pkg/a.go", "line": 11, "side": "RIGHT", "severity": "blocker", "detail": "A's replacement, on a changed line.", "supersedes": ["PRRT_kwDOABCDEF4Ax1y2"]}, {"file": "pkg/untouched.go", "line": 5, "side": "RIGHT", "severity": "blocker", "detail": "B's replacement, on a file this diff does not carry.", "supersedes": ["PRRT_kwDOABCDEF4Bz3w4"]}, {"file": "pkg/nowhere.go", "line": 7, "side": "RIGHT", "severity": "suggestion", "detail": "C's replacement, likewise nowhere.", "supersedes": ["PRRT_kwDOABCDEF4Cq9r8"]}] diff --git a/test/seidroid-review/gha.py b/test/seidroid-review/gha.py new file mode 100644 index 0000000..f9f85d5 --- /dev/null +++ b/test/seidroid-review/gha.py @@ -0,0 +1,402 @@ +"""Evaluates a job's `if:` expression out of a workflow file. + +The subset covered is the one seidroid-review.yml's own conditions use: the +operators `!`, `==`, `!=`, `<`, `<=`, `>`, `>=`, `&&` and `||`; the functions +`contains`, `fromJSON`, `startsWith`, `endsWith`, `format`, `join`, `toJSON`, +`always`, `cancelled`, `success` and `failure`; and dotted lookups into the +`github`, `inputs`, `needs`, `steps`, `env` and `secrets` contexts. + +Four GitHub semantics carry the conditions under test, so they are modelled +exactly and `--selftest` checks each one: + + * `==` on two strings ignores case, and on mixed types casts both to number, + where null and the empty string are 0 and any other non-numeric string is + NaN. NaN equals nothing. + * `a || b` yields `a` when `a` is truthy and `b` otherwise. False, 0, the + empty string and null are the falsy values. `a && b` yields `a` when `a` is + falsy and `b` otherwise. + * BOTH SHORT-CIRCUIT. The runner's Or and And nodes return on the first + truthy or falsy operand and never evaluate the rest, so an error in an + operand nothing reaches never surfaces. A model that evaluated eagerly would + report a failure the runner does not have -- and would make an assertion here + state the opposite of what a real event does. + * `contains(array, item)` tests membership under that same loose equality, + so a listed login matches whatever case it is written in. `contains` over a + STRING tests substring instead, which is the reading this file's conditions + must not have. + +The fidelity of that model rests on GitHub's published expression semantics, +read rather than measured: nothing here calls a runner. + +Usage: + gha.py -> prints true or false + gha.py --expr '' -> prints true or false + gha.py --env -> prints the value + gha.py --input -> prints a declared field + gha.py --selftest -> checks the model above +""" + +import json +import math +import re +import sys + +import yaml + +NAN = float("nan") + +TOKEN = re.compile( + r"""\s+ + |(?P'(?:[^']|'')*') + |(?P\d+(?:\.\d+)?(?:[eE][-+]?\d+)?) + |(?P==|!=|>=|<=|&&|\|\||[!<>(),.\[\]]) + |(?P[A-Za-z_][A-Za-z0-9_-]*)""", + re.X, +) + + +class Bad(Exception): + """An expression GitHub would refuse, such as fromJSON over a non-JSON input.""" + + +def lex(text): + tokens, i = [], 0 + while i < len(text): + m = TOKEN.match(text, i) + if not m: + raise Bad("cannot read expression at: " + text[i:i + 20]) + i = m.end() + for kind in ("str", "num", "op", "name"): + if m.group(kind) is not None: + tokens.append((kind, m.group(kind))) + break + tokens.append(("end", "")) + return tokens + + +class Parser: + def __init__(self, tokens, ctx): + self.t, self.i, self.ctx = tokens, 0, ctx + # False while walking an operand the runner would not reach. The tokens still + # have to be consumed -- this evaluates as it parses -- so the walk continues + # and only the function calls are held back, which is where an error lives. + self.live = True + + def skip(self, parse): + was, self.live = self.live, False + try: + parse() + finally: + self.live = was + + def peek(self): + return self.t[self.i] + + def take(self, value=None): + kind, text = self.t[self.i] + if value is not None and text != value: + raise Bad("expected %r, found %r" % (value, text)) + self.i += 1 + return text + + def parse(self): + value = self.or_() + if self.peek()[0] != "end": + raise Bad("trailing input at %r" % (self.peek()[1],)) + return value + + def or_(self): + left = self.and_() + while self.peek()[1] == "||": + self.take() + if truthy(left): + self.skip(self.and_) + else: + left = self.and_() + return left + + def and_(self): + left = self.compare() + while self.peek()[1] == "&&": + self.take() + if truthy(left): + left = self.compare() + else: + self.skip(self.compare) + return left + + def compare(self): + left = self.unary() + while self.peek()[1] in ("==", "!=", "<", "<=", ">", ">="): + op = self.take() + right = self.unary() + if op == "==": + left = loose_eq(left, right) + elif op == "!=": + left = not loose_eq(left, right) + else: + a, b = to_number(left), to_number(right) + if math.isnan(a) or math.isnan(b): + left = False + else: + left = {"<": a < b, "<=": a <= b, ">": a > b, ">=": a >= b}[op] + return left + + def unary(self): + if self.peek()[1] == "!": + self.take() + return not truthy(self.unary()) + return self.primary() + + def primary(self): + kind, text = self.peek() + if text == "(": + self.take() + value = self.or_() + self.take(")") + return value + if kind == "str": + self.take() + return text[1:-1].replace("''", "'") + if kind == "num": + self.take() + return float(text) + if kind != "name": + raise Bad("unexpected %r" % (text,)) + self.take() + if text == "true": + return True + if text == "false": + return False + if text == "null": + return None + if self.peek()[1] == "(": + return self.call(text) + return self.path(self.ctx.get(text, {})) + + def path(self, value): + while self.peek()[1] == ".": + self.take() + key = self.take() + value = value.get(key) if isinstance(value, dict) else None + return value + + def call(self, name): + self.take("(") + args = [] + if self.peek()[1] != ")": + args.append(self.or_()) + while self.peek()[1] == ",": + self.take() + args.append(self.or_()) + self.take(")") + # An unreached call is not made, which is the whole of short-circuiting: this + # is where fromJSON would refuse a value the runner never looks at. + if not self.live: + return None + return apply_function(name, args) + + +def apply_function(name, args): + if name == "always": + return True + if name == "success": + return True + if name in ("cancelled", "failure"): + return False + if name == "contains": + return gha_contains(args[0], args[1]) + if name == "startsWith": + return as_string(args[0]).lower().startswith(as_string(args[1]).lower()) + if name == "endsWith": + return as_string(args[0]).lower().endswith(as_string(args[1]).lower()) + if name == "fromJSON": + try: + return json.loads(as_string(args[0])) + except ValueError as err: + raise Bad("fromJSON: %s" % (err,)) + if name == "toJSON": + return json.dumps(args[0]) + if name == "format": + out = as_string(args[0]) + for index, value in enumerate(args[1:]): + out = out.replace("{%d}" % index, as_string(value)) + return out + if name == "join": + sep = as_string(args[1]) if len(args) > 1 else "," + items = args[0] if isinstance(args[0], list) else [args[0]] + return sep.join(as_string(item) for item in items) + raise Bad("unsupported function: " + name) + + +def truthy(value): + if value is None or value is False: + return False + if value is True: + return True + if isinstance(value, (int, float)): + return not (value == 0 or math.isnan(value)) + if isinstance(value, str): + return value != "" + return True + + +def to_number(value): + if value is None: + return 0.0 + if isinstance(value, bool): + return 1.0 if value else 0.0 + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + text = value.strip() + if text == "": + return 0.0 + try: + return float(int(text, 16)) if text[:2].lower() == "0x" else float(text) + except ValueError: + return NAN + return NAN + + +def as_string(value): + if value is None: + return "" + if value is True: + return "true" + if value is False: + return "false" + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + if isinstance(value, (list, dict)): + return json.dumps(value) + return str(value) + + +def loose_eq(a, b): + if isinstance(a, str) and isinstance(b, str): + return a.lower() == b.lower() + if isinstance(a, bool) and isinstance(b, bool): + return a is b + if a is None and b is None: + return True + if isinstance(a, (list, dict)) or isinstance(b, (list, dict)): + return a is b + x, y = to_number(a), to_number(b) + if math.isnan(x) or math.isnan(y): + return False + return x == y + + +def gha_contains(search, item): + if isinstance(search, list): + return any(loose_eq(element, item) for element in search) + if isinstance(search, dict): + return False + return as_string(item).lower() in as_string(search).lower() + + +def evaluate(text, ctx): + text = text.strip() + if text.startswith("${{") and text.endswith("}}"): + text = text[3:-2] + return Parser(lex(text), ctx).parse() + + +def job_condition(workflow, job_id): + doc = yaml.safe_load(open(workflow, encoding="utf-8")) + job = doc["jobs"][job_id] + if "if" not in job: + sys.exit("job has no if: " + job_id) + return job["if"] + + +def input_field(workflow, name, field): + """One field of one workflow_call input, so a declared default is testable.""" + doc = yaml.safe_load(open(workflow, encoding="utf-8")) + # PyYAML reads the `on:` key as the boolean True. + triggers = doc.get("on", doc.get(True)) + spec = triggers["workflow_call"]["inputs"] + if name not in spec: + sys.exit("no such input: " + name) + return spec[name].get(field) + + +def step_env(workflow, step, key): + """The expression a step's env key holds, found by step name or step id. + + The step scripts read these as shell variables, so a script harness cannot + see them. They are where a per-event payload field is chosen. + """ + doc = yaml.safe_load(open(workflow, encoding="utf-8")) + for job in doc["jobs"].values(): + for candidate in job.get("steps", []): + if candidate.get("name") == step or candidate.get("id") == step: + env = candidate.get("env") or {} + if key not in env: + sys.exit("step %s has no env %s" % (step, key)) + return env[key] + sys.exit("step not found: " + step) + + +SELFTEST = [ + ("'Bot' == 'bot'", True), + ("'MEMBER' != 'member'", False), + ("null == ''", True), + ("null == 'Bot'", False), + ("'' != 'Bot'", True), + ("true == 1", True), + ("'' || 'second'", "second"), + ("'first' || 'second'", "first"), + ("null || ''", ""), + ("contains(fromJSON('[\"dependabot[bot]\"]'), 'DependaBot[BOT]')", True), + ("contains(fromJSON('[\"dependabot[bot]\"]'), 'bot')", False), + ("contains('[\"dependabot[bot]\"]', 'bot')", True), + ("contains(fromJSON('[]'), 'anyone')", False), + ("!contains(fromJSON('[\"a\",\"b\"]'), 'c')", True), + # Short-circuiting, stated as the three shapes the conditions rely on. + ("true || fromJSON('not json')", True), + ("false && fromJSON('not json')", False), + ("'' || '[]'", "[]"), + ("'[\"x\"]' || '[]'", '["x"]'), +] + + +def selftest(): + failed = 0 + for expression, want in SELFTEST: + got = evaluate(expression, {}) + if got != want: + failed += 1 + print(" FAIL %s: want %r got %r" % (expression, want, got)) + print("gha.py selftest: %d of %d checks passed" % (len(SELFTEST) - failed, len(SELFTEST))) + return 1 if failed else 0 + + +def main(argv): + if argv[1:2] == ["--selftest"]: + return selftest() + raw = False + if argv[1:2] == ["--expr"]: + text, ctx_path = argv[2], argv[3] + elif argv[1:2] == ["--input"]: + print(as_string(input_field(argv[2], argv[3], argv[4]))) + return 0 + elif argv[1:2] == ["--env"]: + # The value a step's env key resolves to, printed as the shell would see + # it rather than as a truth value. + text, ctx_path, raw = step_env(argv[2], argv[3], argv[4]), argv[5], True + else: + text, ctx_path = job_condition(argv[1], argv[2]), argv[3] + ctx = json.load(open(ctx_path, encoding="utf-8")) + try: + value = evaluate(text, ctx) + except Bad as err: + print("error: %s" % (err,), file=sys.stderr) + return 2 + print(as_string(value) if raw else ("true" if truthy(value) else "false")) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/test/seidroid-review/reactions.sh b/test/seidroid-review/reactions.sh new file mode 100755 index 0000000..fabc90d --- /dev/null +++ b/test/seidroid-review/reactions.sh @@ -0,0 +1,343 @@ +#!/usr/bin/env bash +# Runs the reaction steps of seidroid-review.yml under bash with a gh stub, and checks +# what each case leaves on the trigger comment. +# +# No case names the step it runs. `conditions.py --select` names it, from the job state +# and from whether `Post the verdict` landed its comment, so the shell layer and the +# condition layer cannot drift, and a case cannot stop exercising the step it claims to. +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPOROOT="$(cd "$HERE/../.." && pwd)" +WORKFLOW="$REPOROOT/.github/workflows/seidroid-review.yml" +SELECT="python3 $HERE/conditions.py --select $WORKFLOW" + +extract() { # step name, output file + python3 "$HERE/extract.py" "$WORKFLOW" "$1" "$2" > /dev/null || { + echo "could not read '$1' out of $WORKFLOW"; exit 1; } +} +extract "Acknowledge the trigger" "$HERE/ack.sh" +extract "Answer the request" "$HERE/answer.sh" +extract "Withdraw the reactions on a cancelled run" "$HERE/withdraw.sh" + +# Step name -> the file it was extracted to. +script_for() { + case "$1" in + "Acknowledge the trigger") echo "$HERE/ack.sh" ;; + "Answer the request") echo "$HERE/answer.sh" ;; + "Withdraw the reactions on a cancelled run") echo "$HERE/withdraw.sh" ;; + *) echo "" ;; + esac +} + +pass=0 fail=0 +rows=() + +BOT='github-actions[bot]' +NONE='[]' +HUMAN_ALL='[{"id":21,"content":"+1","user":{"login":"brandon"}}, + {"id":22,"content":"eyes","user":{"login":"brandon"}}, + {"id":23,"content":"-1","user":{"login":"brandon"}}]' +STALE_DOWN='[{"id":31,"content":"-1","user":{"login":"github-actions[bot]"}}, + {"id":21,"content":"+1","user":{"login":"brandon"}}]' +STALE_UP='[{"id":32,"content":"+1","user":{"login":"github-actions[bot]"}}]' +# An EARLIER run's thumb, on a comment a re-run replays. Its verdict is on the pull +# request, so it is not this run's to take. +EARLIER_THUMB='[{"id":51,"content":"+1","user":{"login":"github-actions[bot]"}}]' +# A reaction of this bot's that no step here ever chooses. Whatever put it there owns it. +FOREIGN='[{"id":41,"content":"rocket","user":{"login":"github-actions[bot]"}}, + {"id":21,"content":"+1","user":{"login":"brandon"}}]' + +check() { # label expected actual + if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo " FAIL $1: want [$2] got [$3]"; fi +} + +left() { jq -r '[.[] | "\(.user.login):\(.content)"] | sort | join(" ")' < "$STUB_STATE"; } +calls() { grep -c "^CALL $1" "$CASE/calls.log" || true; } +ran() { grep -c "^$1\$" "$CASE/ran.txt" || true; } + +# run_case \ +# [K=V...] +# +# Two states, because a cancellation has a moment. `answer state` is the job state when +# the runner reached `Answer the request`; `final state` is the state when it reached the +# withdrawal step at the end. A cancellation arriving after the verdict landed gives +# success then cancelled, so the answer step posts its thumb and the fixture does not +# have to place one. +# +# `posted` is `Post the verdict`'s own output: true when its comment landed, false when +# the POST was refused, `-` when that step never reported. Not its outcome, which reads +# success even on a refused POST. +run_case() { + local name="$1" seed="$2" answer_state="$3" state="$4" posted="$5" + local conclusion="$6" produced="$7" have_check="$8" + shift 8 + CASE="$HERE/out-reactions/$name" + rm -rf "$CASE"; mkdir -p "$CASE" + # Every per-case knob is cleared here, not at the end of the case that set it. An + # export leaks to every later case otherwise, and a case that stops exercising what + # it claims fails nothing. + unset ANSWERED_AS ACK_POST SKIP_ACK + export STUB_STATE="$CASE/reactions.json"; printf '%s\n' "$seed" > "$STUB_STATE" + export STUB_LIST=ok STUB_DELETE=ok STUB_POST=ok STUB_ACTOR="$BOT" + export PATH="$HERE/bin-reactions:$PATH" + export GH_TOKEN=x REPO=owner/repo TRIGGER_REPO=owner/repo TRIGGER_ID=7 + # The collection the guard resolved for the object that asked. All three steps read + # it, and the last group below is the one that varies it; every case before that is + # about what a step leaves on the comment rather than where it reached. + export COMMENT_API=issues/comments + local check_path="" + if [ "$have_check" = yes ]; then + printf '{"conclusion":"%s","title":"t"}\n' "$conclusion" > "$CASE/check.json" + check_path="$CASE/check.json" + fi + export CHECK="$check_path" VERDICT_PRODUCED="$produced" + for kv in "$@"; do export "${kv?}"; done + : > "$CASE/ran.txt" + + # The acknowledgement runs before any cancellation could land, so it is selected in + # the state the job starts in, not in the state it ends in. + local ack_step + ack_step="$($SELECT success review 7 - | grep '^Acknowledge the trigger$' || true)" + if [ -n "$ack_step" ] && [ "${SKIP_ACK:-no}" = no ]; then + STUB_LOG="$CASE/ack-calls.log"; export STUB_LOG; : > "$STUB_LOG" + # ACK_POST refuses the acknowledgement alone, so a case can start from a comment + # that never got the eyes without also refusing the answer. + STUB_POST="${ACK_POST:-${STUB_POST:-ok}}" bash "$HERE/ack.sh" > "$CASE/ack.out" 2>&1 + echo "$ack_step" >> "$CASE/ran.txt" + fi + + STUB_LOG="$CASE/calls.log"; export STUB_LOG; : > "$STUB_LOG" + : > "$CASE/step.out" + local rc=0 step script + run_selected() { # state, verdict outcome, step to keep + while IFS= read -r step; do + [ "$step" = "$3" ] || continue + script="$(script_for "$step")" + if [ -z "$script" ]; then echo "no script for step '$step'"; exit 1; fi + bash "$script" >> "$CASE/step.out" 2>&1 || rc=$? + echo "$step" >> "$CASE/ran.txt" + done < <($SELECT "$1" review 7 "$2") + } + run_selected "$answer_state" "$posted" "Answer the request" + # The withdrawal reads the answer step's outcome to decide what it may take. Derived + # from whether the harness just ran that step, not passed in, so a case cannot claim + # an outcome the timeline it declared would not produce. ANSWERED_AS overrides it, for + # the arms a two-state timeline cannot reach. + if [ "$(ran 'Answer the request')" = 1 ]; then + ANSWERED="${ANSWERED_AS-success}" + else + ANSWERED="${ANSWERED_AS-skipped}" + fi + export ANSWERED + run_selected "$state" "$posted" "Withdraw the reactions on a cancelled run" + echo "$rc" > "$CASE/rc" + + rows+=("$(printf '%-29s %-19s posted=%-10s ran=%-9s list=%s del=%s post=%s left=%s' \ + "$name" "$answer_state>$state" "${posted/-/unreported}" \ + "$(sed -n 's/^Answer the request$/answer/p;s/^Withdraw.*/withdraw/p' \ + "$CASE/ran.txt" | paste -sd+ - || true)" \ + "$(calls list)" "$(calls delete)" "$(calls post)" "$(left)")") +} + +echo "== a green review thumbs the request up ==" +run_case success "$NONE" success success true success true yes +check "answer ran" 1 "$(ran 'Answer the request')" +check "left" "$BOT:+1" "$(left)" + +echo "== a blocking review thumbs it down ==" +run_case failure "$NONE" success success true failure true yes +check "left" "$BOT:-1" "$(left)" + +echo "== a run that reached no verdict clears and says nothing ==" +run_case no-verdict "$NONE" success success - failure false yes +check "left" "" "$(left)" +check "no post" 0 "$(calls post)" + +echo "== neutral earns no reaction ==" +run_case neutral "$NONE" success success - neutral true yes +check "left" "" "$(left)" + +echo "== a cancelled run with the verdict outputs POPULATED still posts nothing." +echo " A cancellation after the driver finishes leaves a real conclusion on disk. ==" +run_case cancelled-after-drive "$NONE" cancelled cancelled - success true yes +check "withdraw ran" 1 "$(ran 'Withdraw the reactions on a cancelled run')" +check "answer skipped" 0 "$(ran 'Answer the request')" +check "left" "" "$(left)" +check "no post" 0 "$(calls post)" + +echo "== a cancelled run before the driver finishes ==" +run_case cancelled-early "$NONE" cancelled cancelled - - '' no +check "left" "" "$(left)" +check "no post" 0 "$(calls post)" + +echo "== cancelled while the verdict was posting: the thumb goes, it may not have landed ==" +run_case cancelled-mid-publish "$NONE" success cancelled - success true yes +check "withdraw ran" 1 "$(ran 'Withdraw the reactions on a cancelled run')" +check "left" "" "$(left)" + +echo "== THE VERDICT COMMENT WAS REFUSED. `Post the verdict` tolerates that and exits 0," +echo " so its OUTCOME reads success while nothing landed. The thumb has to go: reading" +echo " the outcome here would leave it standing for a review nobody can see. ==" +run_case cancelled-publish-failed "$NONE" success cancelled false success true yes +check "answer posted a thumb" 1 "$(calls post)" +check "withdraw ran" 1 "$(ran 'Withdraw the reactions on a cancelled run')" +check "THUMB GOES" "" "$(left)" + +echo "== an outcome this step cannot read clears rather than leaving a thumb ==" +run_case cancelled-unreported "$NONE" success cancelled - success true yes +check "withdraw ran" 1 "$(ran 'Withdraw the reactions on a cancelled run')" +check "left" "" "$(left)" + +echo "== CANCELLED AFTER THE VERDICT PUBLISHED. The thumb answers a review that is on" +echo " the pull request, so it survives: withdrawing it would read as never answered. ==" +run_case cancelled-after-publish "$NONE" success cancelled true success true yes +check "answer ran" 1 "$(ran 'Answer the request')" +check "withdraw skipped" 0 "$(ran 'Withdraw the reactions on a cancelled run')" +check "one post, no later delete" "1 1" "$(calls post) $(calls delete)" +check "THUMB SURVIVES" "$BOT:+1" "$(left)" + +echo "== A RE-RUN REPLAYS THE TRIGGER COMMENT ID, so an earlier run's thumb can already" +echo " be on it. A run cancelled before it answers has posted only the eyes, and that" +echo " thumb answers a verdict still on the pull request. ==" +run_case rerun-cancelled-before-answer "$EARLIER_THUMB" cancelled cancelled - - '' no +check "answer never ran" 0 "$(ran 'Answer the request')" +check "withdraw ran" 1 "$(ran 'Withdraw the reactions on a cancelled run')" +check "took the eyes only" 1 "$(calls delete)" +check "EARLIER THUMB SURVIVES" "$BOT:+1" "$(left)" + +echo "== the same, beside a human's ==" +run_case rerun-cancelled-human \ + '[{"id":51,"content":"+1","user":{"login":"github-actions[bot]"}}, + {"id":23,"content":"-1","user":{"login":"brandon"}}]' \ + cancelled cancelled - - '' no +check "left" "brandon:-1 $BOT:+1" "$(left)" + +echo "== but once this run has answered, every reaction on the comment is its own ==" +run_case rerun-answered-then-cancelled "$EARLIER_THUMB" success cancelled - success true yes +check "answer ran" 1 "$(ran 'Answer the request')" +check "withdraw ran" 1 "$(ran 'Withdraw the reactions on a cancelled run')" +check "left" "" "$(left)" + +echo "== a partial answer clears: it most likely took the earlier thumb already ==" +run_case rerun-answer-failed "$EARLIER_THUMB" cancelled cancelled - - '' no ANSWERED_AS=failure +check "left" "" "$(left)" +run_case rerun-answer-cancelled "$EARLIER_THUMB" cancelled cancelled - - '' no ANSWERED_AS=cancelled +check "left" "" "$(left)" + +echo "== an outcome the step cannot read clears: a thumb standing for nothing is worse ==" +run_case rerun-answer-unreported "$EARLIER_THUMB" cancelled cancelled - - '' no ANSWERED_AS= +check "left" "" "$(left)" + +echo "== and it survives beside a human's reactions ==" +run_case cancelled-after-publish-human \ + '[{"id":21,"content":"-1","user":{"login":"brandon"}}]' \ + success cancelled true success true yes +check "left" "brandon:-1 $BOT:+1" "$(left)" + +echo "== a human's reaction survives every path ==" +run_case success-human "$HUMAN_ALL" success success true success true yes +check "left" "brandon:+1 brandon:-1 brandon:eyes $BOT:+1" "$(left)" +run_case failure-human "$HUMAN_ALL" success success true failure true yes +check "left" "brandon:+1 brandon:-1 brandon:eyes $BOT:-1" "$(left)" +run_case noverdict-human "$HUMAN_ALL" success success - failure false yes +check "left" "brandon:+1 brandon:-1 brandon:eyes" "$(left)" +run_case cancelled-human "$HUMAN_ALL" cancelled cancelled - success true yes +check "left" "brandon:+1 brandon:-1 brandon:eyes" "$(left)" + +echo "== a run that answers replaces the stale thumb; a human's stays ==" +run_case stale-thumb "$STALE_DOWN" success success true success true yes +check "left" "brandon:+1 $BOT:+1" "$(left)" + +echo "== a cancelled run that never answered leaves it: the earlier verdict may stand ==" +run_case stale-thumb-cancelled "$STALE_DOWN" cancelled cancelled - success true yes +check "left" "brandon:+1 $BOT:-1" "$(left)" +run_case stale-up-noverdict "$STALE_UP" success success - failure false yes +check "left" "" "$(left)" + +echo "== a reaction no step here chooses is not this job's to withdraw ==" +run_case foreign "$FOREIGN" success success true success true yes +check "left" "brandon:+1 $BOT:+1 $BOT:rocket" "$(left)" +run_case foreign-cancelled "$FOREIGN" cancelled cancelled - success true yes +check "left" "brandon:+1 $BOT:rocket" "$(left)" + +echo "== a refused call warns and never fails the step ==" +run_case list-refused '[{"id":21,"content":"+1","user":{"login":"brandon"}}]' \ + success success true success true yes STUB_LIST=FAIL +check "rc" 0 "$(cat "$CASE/rc")" +check "warned" 1 "$(grep -c '::warning::could not read the reactions' "$CASE/step.out")" +check "thumb still" 1 "$(calls post)" +run_case delete-refused "$NONE" success success true success true yes STUB_DELETE=FAIL +check "rc" 0 "$(cat "$CASE/rc")" +check "eyes stay" "$BOT:+1 $BOT:eyes" "$(left)" +run_case post-refused "$NONE" success success true success true yes STUB_POST=FAIL +check "rc" 0 "$(cat "$CASE/rc")" +check "left" "" "$(left)" +run_case list-refused-cancelled "$NONE" cancelled cancelled - - '' no STUB_LIST=FAIL +check "rc" 0 "$(cat "$CASE/rc")" +check "warned" 1 "$(grep -c '::warning::could not read the reactions' "$CASE/step.out")" +run_case delete-refused-cancelled "$NONE" cancelled cancelled - - '' no STUB_DELETE=FAIL +check "rc" 0 "$(cat "$CASE/rc")" +check "eyes stay" "$BOT:eyes" "$(left)" + +echo "== the acknowledgement itself refused: no eyes to clear, the answer still lands ==" +run_case ack-refused "$NONE" success success true success true yes ACK_POST=FAIL +check "rc" 0 "$(cat "$CASE/rc")" +check "ack warned" 1 "$(grep -c '::warning::could not react to comment' "$CASE/ack.out")" +check "nothing to clear" 0 "$(calls delete)" +check "left" "$BOT:+1" "$(left)" + +echo "== a close reacts nowhere, so nothing is left to clear ==" +CASE="$HERE/out-reactions/close-mode"; rm -rf "$CASE"; mkdir -p "$CASE" +check "close selects no step" "" "$($SELECT success close 7 - | paste -sd, -)" +check "cancelled close selects no step" "" "$($SELECT cancelled close 7 false | paste -sd, -)" + +# The trigger's reactions live under a different collection per event, so every step +# that touches them builds its URL from the guard's comment_api rather than naming one. +# A step that named one would post the eyes where the answer cannot reach them, or +# withdraw from a comment that never carried them. +echo "== every reaction reaches the collection the guard resolved ==" +run_case api-issue "$NONE" success success true success true yes +check "the answer read the issue collection" 1 \ + "$(grep -c '^CALL list repos/owner/repo/issues/comments/7/reactions$' "$CASE/calls.log")" +check "and thumbed there" 1 \ + "$(grep -c '^CALL post +1 repos/owner/repo/issues/comments/7/reactions$' "$CASE/calls.log")" +check "the acknowledgement too" 1 \ + "$(grep -c '^CALL post eyes repos/owner/repo/issues/comments/7/reactions$' "$CASE/ack-calls.log")" +run_case api-thread "$NONE" success success true success true yes COMMENT_API=pulls/comments +check "the answer read the pull collection" 1 \ + "$(grep -c '^CALL list repos/owner/repo/pulls/comments/7/reactions$' "$CASE/calls.log")" +check "and thumbed there" 1 \ + "$(grep -c '^CALL post +1 repos/owner/repo/pulls/comments/7/reactions$' "$CASE/calls.log")" +check "the acknowledgement too" 1 \ + "$(grep -c '^CALL post eyes repos/owner/repo/pulls/comments/7/reactions$' "$CASE/ack-calls.log")" +check "nothing reached the other one" 0 \ + "$(cat "$CASE/calls.log" "$CASE/ack-calls.log" | grep -c 'issues/comments')" +# The withdrawal step, which runs on no path the two cases above take. It only ever +# reads and deletes, so a wrong collection there is the case where the eyes stay. The +# acknowledgement seeds the eyes, and the id it lands is the stub's to choose, so the +# delete is matched on its collection rather than on that id. +run_case api-withdraw-issue "$NONE" cancelled cancelled false - '' no +check "the withdrawal read the issue collection" 1 \ + "$(grep -c '^CALL list repos/owner/repo/issues/comments/7/reactions$' "$CASE/calls.log")" +check "and deleted the eyes there" 1 \ + "$(grep -cE '^CALL delete [0-9]+ repos/owner/repo/issues/comments/7/reactions/[0-9]+$' "$CASE/calls.log")" +check "the comment is clear" "" "$(left)" +run_case api-withdraw-thread "$NONE" cancelled cancelled false - '' no COMMENT_API=pulls/comments +check "the withdrawal read the pull collection" 1 \ + "$(grep -c '^CALL list repos/owner/repo/pulls/comments/7/reactions$' "$CASE/calls.log")" +check "and deleted the eyes there" 1 \ + "$(grep -cE '^CALL delete [0-9]+ repos/owner/repo/pulls/comments/7/reactions/[0-9]+$' "$CASE/calls.log")" +check "the comment is clear" "" "$(left)" +check "nothing reached the other one" 0 \ + "$(cat "$CASE/calls.log" "$CASE/ack-calls.log" | grep -c 'issues/comments')" +check "and it still posts nothing" 0 "$(calls post)" + +echo "== no step made a call the stub does not serve ==" +check "unstubbed calls" 0 \ + "$(grep -rh 'CALL UNSTUBBED' "$HERE/out-reactions" 2>/dev/null | wc -l | tr -d ' ')" + +echo +printf '%s\n' "${rows[@]}" +echo +echo "assertions: $pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/test/seidroid-review/run-guard.sh b/test/seidroid-review/run-guard.sh new file mode 100755 index 0000000..12b0193 --- /dev/null +++ b/test/seidroid-review/run-guard.sh @@ -0,0 +1,625 @@ +#!/usr/bin/env bash +# Runs the guard's shipped step scripts under bash with a gh stub on PATH, and +# evaluates the shipped job conditions with gha.py. +# +# Every script and every condition is read out of the workflow on each run, so a +# run tests what the file says now and cannot pass against a stale copy. +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPOROOT="$(cd "$HERE/../.." && pwd)" +WORKFLOW="$REPOROOT/.github/workflows/seidroid-review.yml" +ASSISTANT="$REPOROOT/.github/workflows/ai-assistant.yml" +REFUSE="$HERE/refuse.sh" +PARSE="$HERE/parse.sh" +ADMIT="$HERE/admit.sh" +# Named apart from the ack.sh and answer.sh reactions.sh writes. Both harnesses +# extract the same two steps, and a shared path lets one overwrite the other's +# extraction mid-run. +ACK="$HERE/guard-ack.sh" +ANSWER="$HERE/guard-answer.sh" +MARKER="$(python3 "$HERE/extract.py" "$WORKFLOW" "Refuse an event this workflow does not handle" "$REFUSE" VERDICT_MARKER)" || { + echo "could not read the refusal step out of $WORKFLOW"; exit 1; } +python3 "$HERE/extract.py" "$WORKFLOW" parse "$PARSE" VERDICT_MARKER >/dev/null || { + echo "could not read the parse step out of $WORKFLOW"; exit 1; } +python3 "$HERE/extract.py" "$WORKFLOW" "Admit the request" "$ADMIT" VERDICT_MARKER >/dev/null || { + echo "could not read the admission step out of $WORKFLOW"; exit 1; } +python3 "$HERE/extract.py" "$WORKFLOW" "Acknowledge the trigger" "$ACK" VERDICT_MARKER >/dev/null || { + echo "could not read the acknowledgement step out of $WORKFLOW"; exit 1; } +python3 "$HERE/extract.py" "$WORKFLOW" "Answer the request" "$ANSWER" VERDICT_MARKER >/dev/null || { + echo "could not read the answering step out of $WORKFLOW"; exit 1; } + +mkdir -p "$HERE/out" +CTX="$HERE/out/ctx.json" +PATH="$HERE/bin-guard:$PATH" +export PATH + +pass=0 fail=0 +check() { # label expected actual + if [ "$2" = "$3" ]; then pass=$((pass + 1)); else + fail=$((fail + 1)); echo " FAIL $1: want [$2] got [$3]"; fi +} + +# --- the job conditions ------------------------------------------------------- +# One context per case, built here rather than committed: a condition reads six +# payload fields and a context file per case would be six lines of JSON each. +ctx_comment() { # event action login type association allowed-bots + jq -nc --arg ev "$1" --arg action "$2" --arg login "$3" --arg type "$4" \ + --arg assoc "$5" --arg bots "$6" ' + {github: {event_name: $ev, + event: {action: $action, pull_request: {number: 7}}}, + inputs: {mode: "review", "allowed-bots": $bots}} + | .github.event |= (if $ev == "pull_request_review" + then . + {review: {id: 99, user: {login: $login, type: $type}, + author_association: $assoc, body: "@seidroid review"}} + else . + {comment: {id: 42, user: {login: $login, type: $type}, + author_association: $assoc, body: "@seidroid review"}} + end) + | if $ev == "issue_comment" + then .github.event.issue = {number: 7, pull_request: {url: "https://api/pulls/7"}} + else . end' > "$CTX" +} + +ctx_issue_only() { # an issue_comment on an ISSUE, which carries no pull_request + ctx_comment issue_comment created alice User MEMBER '[]' + jq -c '.github.event.issue.pull_request = null' "$CTX" > "$CTX.tmp" && mv "$CTX.tmp" "$CTX" +} + +ctx_plain() { # event mode [allowed-bots] + jq -nc --arg ev "$1" --arg mode "$2" --arg bots "${3-[]}" ' + {github: {event_name: $ev, event: {action: "opened", pull_request: {number: 7}}}, + inputs: {mode: $mode, "allowed-bots": $bots}}' > "$CTX" +} + +ctx_needs() { # event mode guard-result should_run + # The guard's four request outputs, with distinct values, so an assertion on a + # consumer names which output it read rather than only that it read something. + jq -nc --arg ev "$1" --arg mode "$2" --arg result "$3" --arg run "$4" ' + {github: {event_name: $ev, event: {pull_request: {number: 7}}}, + inputs: {mode: $mode}, + needs: {guard: {result: $result, + outputs: {should_run: $run, pr_number: "7", + comment_id: "111", comment_api: "pulls/comments", + trigger_id: "222"}}}}' > "$CTX" +} + +input_of_file() { # workflow input field + python3 "$HERE/gha.py" --input "$1" "$2" "$3" 2>/dev/null || printf 'error' +} + +input_of() { # input field + input_of_file "$WORKFLOW" "$1" "$2" +} + +env_of() { # step key -- against the context $CTX holds + python3 "$HERE/gha.py" --env "$WORKFLOW" "$1" "$2" "$CTX" 2>/dev/null || printf 'error' +} + +cond() { # job-id + local out + if out="$(python3 "$HERE/gha.py" "$WORKFLOW" "$1" "$CTX" 2>/dev/null)"; then + printf '%s' "$out" + else + printf 'error' + fi +} + +# --- the step scripts --------------------------------------------------------- +run_case() { # name script KEY=VALUE... + local name="$1" script="$2" + shift 2 + CASE="$HERE/out/$name" + rm -rf "$CASE" + mkdir -p "$CASE" + export STUB_LOG="$CASE/calls.log"; : > "$STUB_LOG" + export GITHUB_OUTPUT="$CASE/output.txt"; : > "$GITHUB_OUTPUT" + export VERDICT_MARKER="$MARKER" + # What `parse` reads. + export EVENT_NAME=issue_comment + export BODY='@seidroid review' + export PR_NUMBER=7 + export COMMENT_ID=42 + export TRIGGER_PHRASE='@seidroid' + # What `Admit the request` reads. + export GH_TOKEN=app-token GATE_TOKEN=app-token + export ALLOWED_TEAM=sei-protocol/sei-core + export ALLOWED_BOTS='[]' + export SKIP_LABEL='ai: skip-review' + export ACTOR=alice ACTOR_TYPE=User + export REPO=owner/repo PR=7 PARSED=true + export IS_DRAFT=false ACTION=created RE_REVIEW_ON_PUSH=false MODE=review + export HEAD_REPO_ID='' BASE_REPO_ID='' + # What the two reacting steps read. + export TRIGGER_REPO=owner/repo TRIGGER_ID=42 COMMENT_API=issues/comments + export CHECK="$CASE/check.json" VERDICT_PRODUCED=true + printf '%s\n' '{"conclusion":"success"}' > "$CASE/check.json" + # What the gh stub answers. + export STUB_TEAM=active STUB_ORIGIN=same STUB_LABELS='' STUB_REVIEWS=none STUB_COMMENTS=none + export STUB_REACTIONS=none + for kv in "$@"; do export "${kv?}"; done + bash "$script" > "$CASE/stdout.txt" 2> "$CASE/stderr.txt" + echo "$?" > "$CASE/rc" +} + +rc() { cat "$CASE/rc"; } +out() { grep -E "^$1=" "$CASE/output.txt" | tail -1 | cut -d= -f2-; } +written() { grep -cE "^$1=" "$CASE/output.txt"; } +calls() { grep -c "^CALL $1" "$CASE/calls.log" || true; } +said() { grep -c -- "$1" "$CASE/stdout.txt" || true; } +cried() { grep -c -- "$1" "$CASE/stderr.txt" || true; } + +echo "== 0. the expression model gha.py holds ==" +python3 "$HERE/gha.py" --selftest +check "expression model clean" 0 "$?" + +echo +echo "== 1. the guard's condition: which requests reach a runner ==" +ctx_comment issue_comment created alice User MEMBER '[]' +check "issue_comment, member" true "$(cond guard)" +ctx_comment pull_request_review_comment created alice User MEMBER '[]' +check "diff-thread comment, member" true "$(cond guard)" +ctx_comment pull_request_review submitted alice User MEMBER '[]' +check "review body, member" true "$(cond guard)" +ctx_comment pull_request_review_comment edited alice User MEMBER '[]' +check "an edited diff-thread comment" false "$(cond guard)" +ctx_comment pull_request_review dismissed alice User MEMBER '[]' +check "a dismissed review" false "$(cond guard)" +ctx_comment pull_request_review edited alice User MEMBER '[]' +check "an edited review" false "$(cond guard)" +ctx_comment pull_request_review submitted mallory User NONE '[]' +check "review body, outsider" false "$(cond guard)" +ctx_comment pull_request_review_comment created mallory User NONE '[]' +check "diff-thread comment, outsider" false "$(cond guard)" +ctx_comment pull_request_review submitted alice User COLLABORATOR '[]' +check "review body, collaborator" true "$(cond guard)" +ctx_comment pull_request_review_comment created alice User OWNER '[]' +check "diff-thread comment, owner" true "$(cond guard)" +ctx_issue_only +check "a comment on an issue" false "$(cond guard)" + +echo +echo "== 2. the guard's condition: allowed-bots ==" +ctx_comment issue_comment created 'dependabot[bot]' Bot MEMBER '[]' +check "empty list denies a bot" false "$(cond guard)" +ctx_comment issue_comment created 'dependabot[bot]' Bot NONE '["dependabot[bot]"]' +check "a listed bot" true "$(cond guard)" +ctx_comment issue_comment created 'dependabot[bot]' Bot NONE '["renovate[bot]"]' +check "an unlisted bot" false "$(cond guard)" +ctx_comment issue_comment created 'DependaBot[BOT]' Bot NONE '["dependabot[bot]"]' +check "one login, either case" true "$(cond guard)" +# `bot` is a substring of every listed login here. A condition that tested the +# input as a STRING would admit it; membership of the parsed array does not. +ctx_comment issue_comment created 'bot' Bot NONE '["dependabot[bot]","renovate[bot]"]' +check "a login inside a listed one" false "$(cond guard)" +ctx_comment pull_request_review submitted 'dependabot[bot]' Bot NONE '["dependabot[bot]"]' +check "a listed bot in a review body" true "$(cond guard)" +ctx_comment pull_request_review_comment created 'dependabot[bot]' Bot NONE '["dependabot[bot]"]' +check "a listed bot in a diff thread" true "$(cond guard)" +# A value fromJSON cannot read, on the two requesters that reach it differently. +# The runner short-circuits, so a person is admitted on association before the list +# is parsed at all; a bot is the requester whose admission depends on parsing it. A +# caller wiring error therefore fails the requests it governs and no others. +ctx_comment issue_comment created alice User MEMBER 'not json' +check "a person never reads the list" true "$(cond guard)" +ctx_comment issue_comment created 'dependabot[bot]' Bot NONE 'not json' +check "a bot reads it, and it fails loudly" error "$(cond guard)" +ctx_plain pull_request review 'not json' +check "an automatic review never reads it" true "$(cond guard)" +# An UNSET input, which is the likelier accident: a workflow_call default applies +# only to an input the caller omits, so `allowed-bots: ${{ vars.UNSET }}` arrives as +# the empty string and fromJSON('') is not []. Empty has to read as the documented +# default -- deny every bot -- rather than take the run down. +ctx_comment issue_comment created 'dependabot[bot]' Bot NONE '' +check "an unset list denies every bot" false "$(cond guard)" +ctx_comment issue_comment created 'dependabot[bot]' Bot MEMBER '' +check "and association is no way round it" false "$(cond guard)" +ctx_comment issue_comment created alice User MEMBER '' +check "and a person is still admitted" true "$(cond guard)" +ctx_plain pull_request review '' +check "and an automatic review still runs" true "$(cond guard)" + +echo +echo "== 3. the guard's condition: the events it does not handle ==" +ctx_plain pull_request_target review +check "pull_request_target reaches the guard" true "$(cond guard)" +ctx_plain push review +check "push reaches the guard" true "$(cond guard)" +ctx_plain workflow_dispatch review +check "workflow_dispatch reaches the guard" true "$(cond guard)" +ctx_plain pull_request review +check "an automatic review" true "$(cond guard)" +# A pull_request close skips this guard, and the review job reads that skip as +# its own trigger. Admitting it here would break the only path that reclaims a +# sandbox. +ctx_plain pull_request close +check "a pull_request close skips the guard" false "$(cond guard)" + +echo +echo "== 4. the review job's condition ==" +ctx_needs issue_comment review success true +check "issue_comment, admitted" true "$(cond review)" +ctx_needs pull_request_review_comment review success true +check "diff-thread comment, admitted" true "$(cond review)" +ctx_needs pull_request_review review success true +check "review body, admitted" true "$(cond review)" +ctx_needs pull_request_review review success false +check "review body, refused" false "$(cond review)" +ctx_needs pull_request_review_comment review failure '' +check "diff-thread comment, guard failed" false "$(cond review)" +ctx_needs pull_request_target review failure '' +check "pull_request_target never reviews" false "$(cond review)" +ctx_needs pull_request review success true +check "an automatic review, admitted" true "$(cond review)" +ctx_needs pull_request close skipped '' +check "a pull_request close still runs" true "$(cond review)" + +echo +echo "== 5. the refusal: an event this workflow does not handle ==" +run_case refuse-target "$REFUSE" EVENT_NAME=pull_request_target +check "rc" 1 "$(rc)" +check "names the event" 1 "$(cried pull_request_target)" +check "says why" 1 "$(cried "base repository's secrets")" +check "names the four it handles" 1 "$(cried "pull_request, issue_comment, pull_request_review_comment or pull_request_review")" +run_case refuse-push "$REFUSE" EVENT_NAME=push +check "rc" 1 "$(rc)" +check "names the event" 1 "$(cried "cannot be called from 'push'")" +check "names the four it handles" 1 "$(cried "pull_request, issue_comment, pull_request_review_comment and pull_request_review")" +run_case refuse-dispatch "$REFUSE" EVENT_NAME=workflow_dispatch +check "rc" 1 "$(rc)" +check "names the event" 1 "$(cried workflow_dispatch)" +for ev in pull_request issue_comment pull_request_review_comment pull_request_review; do + run_case "refuse-ok-$ev" "$REFUSE" "EVENT_NAME=$ev" + check "$ev passes" 0 "$(rc)" + check "$ev says nothing" 0 "$(cried '::')" +done + +echo +echo "== 6. the parse: which body is a command, and what it resolves to ==" +run_case parse-issue "$PARSE" EVENT_NAME=issue_comment +check "should_run" true "$(out should_run)" +check "comment_id" 42 "$(out comment_id)" +check "comment_api" issues/comments "$(out comment_api)" +check "trigger_id" 42 "$(out trigger_id)" +run_case parse-thread "$PARSE" EVENT_NAME=pull_request_review_comment +check "should_run" true "$(out should_run)" +check "comment_id" 42 "$(out comment_id)" +check "comment_api" pulls/comments "$(out comment_api)" +check "trigger_id" 42 "$(out trigger_id)" +run_case parse-review "$PARSE" EVENT_NAME=pull_request_review COMMENT_ID=99 +check "should_run" true "$(out should_run)" +check "comment_id held back" '' "$(out comment_id)" +check "comment_id written once" 1 "$(written comment_id)" +check "comment_api held back" '' "$(out comment_api)" +# The log id is a different fact from the reactable id, and this is the event where +# they differ: nothing can react on a review, but the run still has a request to name. +check "trigger_id still carries the id" 99 "$(out trigger_id)" +check "says the request earns no reaction" 1 "$(said 'carries no reactions endpoint')" +run_case parse-auto "$PARSE" EVENT_NAME=pull_request BODY='' +check "should_run" true "$(out should_run)" +check "comment_id" '' "$(out comment_id)" +check "comment_api written" 1 "$(written comment_api)" +check "trigger_id" '' "$(out trigger_id)" +run_case parse-bare "$PARSE" BODY='seidroid review' +check "the bare phrase still asks" true "$(out should_run)" +run_case parse-prose "$PARSE" BODY='Do we need @seidroid review on this one?' +check "prose about the command" false "$(out should_run)" +run_case parse-close "$PARSE" BODY='@seidroid review close' +check "a close" true "$(out should_run)" +run_case parse-multiline "$PARSE" BODY='Looks good otherwise. +@seidroid review +Thanks!' +check "a command on its own line" true "$(out should_run)" +run_case parse-target "$PARSE" BODY='@seidroid review owner/repo#5' +check "a named repository" false "$(out should_run)" +check "and it says so" 1 "$(said 'takes no repository target')" +run_case parse-padded "$PARSE" BODY=' @seidroid review ' +check "padding around the command" true "$(out should_run)" +run_case parse-suffix "$PARSE" BODY='@seidroidX review' +check "a longer login" false "$(out should_run)" +run_case parse-target-own "$PARSE" TRIGGER_PHRASE='@mybot' BODY='@mybot review owner/repo#5' +check "a named repository, own phrase" false "$(out should_run)" +check "and it says so" 1 "$(said 'takes no repository target')" + +echo +echo "== 7. the parse: a caller's own trigger phrase ==" +run_case phrase-own "$PARSE" TRIGGER_PHRASE='@mybot' BODY='@mybot review' +check "the phrase the caller set" true "$(out should_run)" +check "no warning" 0 "$(said '::warning')" +run_case phrase-not-default "$PARSE" TRIGGER_PHRASE='@mybot' BODY='@seidroid review' +check "and not the default" false "$(out should_run)" +run_case phrase-hyphen "$PARSE" TRIGGER_PHRASE='@sei-droid' BODY='@sei-droid review' +check "a hyphen is part of the phrase" true "$(out should_run)" +check "no warning" 0 "$(said '::warning')" +run_case phrase-bare-ok "$PARSE" TRIGGER_PHRASE='mybot' BODY='mybot review' +check "a phrase written without the @" true "$(out should_run)" +run_case phrase-case "$PARSE" TRIGGER_PHRASE='@Seidroid' BODY='@Seidroid review' +check "the phrase is matched as written" true "$(out should_run)" +run_case phrase-case-other "$PARSE" TRIGGER_PHRASE='@Seidroid' BODY='@seidroid review' +check "and not in another case" false "$(out should_run)" +# A `.` reaching the pattern unconstrained matches any character, so `@myXbot` +# would ask for a review under a phrase nobody configured. +run_case phrase-dot "$PARSE" TRIGGER_PHRASE='@my.bot' BODY='@myXbot review' +check "a dot cannot stand for a character" false "$(out should_run)" +check "and the phrase is refused" 1 "$(said "trigger-phrase '@my.bot' is not")" +run_case phrase-dot-fallback "$PARSE" TRIGGER_PHRASE='@my.bot' BODY='@seidroid review' +check "the default takes over" true "$(out should_run)" +# A `|` reaching the pattern unconstrained splits it into two alternatives, and +# the first is `^[[:space:]]*@?a` -- which any line starting with `a` matches. +run_case phrase-pipe "$PARSE" TRIGGER_PHRASE='@a|b' BODY='a note about the diff' +check "a pipe cannot widen the match" false "$(out should_run)" +check "and the phrase is refused" 1 "$(said "trigger-phrase '@a|b' is not")" +run_case phrase-pipe-fallback "$PARSE" TRIGGER_PHRASE='@a|b' BODY='@seidroid review' +check "the default takes over" true "$(out should_run)" +run_case phrase-empty "$PARSE" TRIGGER_PHRASE='' BODY='@seidroid review' +check "an empty phrase falls back" true "$(out should_run)" +check "and says so" 1 "$(said "trigger-phrase '' is not")" +run_case phrase-space "$PARSE" TRIGGER_PHRASE='@my bot' BODY='@seidroid review' +check "a phrase with a space falls back" true "$(out should_run)" +run_case phrase-at-only "$PARSE" TRIGGER_PHRASE='@' BODY='@seidroid review' +check "a bare @ falls back" true "$(out should_run)" + +echo +echo "== 8. the admission: who may ask, on every comment path ==" +run_case admit-member "$ADMIT" EVENT_NAME=issue_comment +check "admit" true "$(out admit)" +check "the team was read" 1 "$(calls membership)" +run_case admit-thread "$ADMIT" EVENT_NAME=pull_request_review_comment +check "admit" true "$(out admit)" +check "the team was read" 1 "$(calls membership)" +check "the origin was read" 1 "$(calls origin)" +check "the labels were read" 1 "$(calls labels)" +run_case admit-reviewbody "$ADMIT" EVENT_NAME=pull_request_review +check "admit" true "$(out admit)" +check "the team was read" 1 "$(calls membership)" +check "the origin was read" 1 "$(calls origin)" +check "the labels were read" 1 "$(calls labels)" +run_case admit-pending "$ADMIT" EVENT_NAME=issue_comment STUB_TEAM=pending +check "admit" false "$(out admit)" +check "and names the team" 1 "$(said 'not an active member of sei-protocol/sei-core')" +run_case admit-thread-pending "$ADMIT" EVENT_NAME=pull_request_review_comment STUB_TEAM=pending +check "a diff thread is no way round it" false "$(out admit)" +run_case admit-review-pending "$ADMIT" EVENT_NAME=pull_request_review STUB_TEAM=pending +check "a review body is no way round it" false "$(out admit)" +run_case admit-team-fail "$ADMIT" EVENT_NAME=pull_request_review STUB_TEAM=FAIL +check "a membership read that fails" false "$(out admit)" +run_case admit-no-team "$ADMIT" EVENT_NAME=pull_request_review ALLOWED_TEAM='' +check "an empty team denies" false "$(out admit)" +check "and reads nothing" 0 "$(calls membership)" +run_case admit-no-app "$ADMIT" EVENT_NAME=pull_request_review_comment GH_TOKEN='' +check "no App identity denies" false "$(out admit)" +check "and names the two secrets" 1 "$(said 'SEIDROID_APP_ID and SEIDROID_APP_PRIVATE_KEY')" +check "and reads nothing" 0 "$(calls membership)" + +echo +echo "== 9. the admission: a bot is held to allowed-bots ==" +run_case bot-listed "$ADMIT" ACTOR='dependabot[bot]' ACTOR_TYPE=Bot ALLOWED_BOTS='["dependabot[bot]"]' +check "admit" true "$(out admit)" +check "the team is not read for a bot" 0 "$(calls membership)" +run_case bot-case "$ADMIT" ACTOR='DependaBot[BOT]' ACTOR_TYPE=Bot ALLOWED_BOTS='["dependabot[bot]"]' +check "one login, either case" true "$(out admit)" +run_case bot-type-case "$ADMIT" ACTOR='dependabot[bot]' ACTOR_TYPE=bot ALLOWED_BOTS='["dependabot[bot]"]' +check "the type is read either case" true "$(out admit)" +check "the team is not read" 0 "$(calls membership)" +run_case bot-unlisted "$ADMIT" ACTOR='renovate[bot]' ACTOR_TYPE=Bot ALLOWED_BOTS='["dependabot[bot]"]' +check "an unlisted bot" false "$(out admit)" +check "and names the input" 1 "$(said 'is not in allowed-bots')" +check "and reads no team" 0 "$(calls membership)" +run_case bot-empty-list "$ADMIT" ACTOR='dependabot[bot]' ACTOR_TYPE=Bot ALLOWED_BOTS='[]' +check "the default list denies" false "$(out admit)" +run_case bot-substring "$ADMIT" ACTOR='bot' ACTOR_TYPE=Bot ALLOWED_BOTS='["dependabot[bot]"]' +check "a login inside a listed one" false "$(out admit)" +run_case bot-prefix "$ADMIT" ACTOR='dependabot[bot]x' ACTOR_TYPE=Bot ALLOWED_BOTS='["dependabot[bot]"]' +check "a login that extends a listed one" false "$(out admit)" +run_case bot-not-json "$ADMIT" ACTOR='dependabot[bot]' ACTOR_TYPE=Bot ALLOWED_BOTS='not json' +check "a list that is not JSON" false "$(out admit)" +run_case bot-not-array "$ADMIT" ACTOR='dependabot[bot]' ACTOR_TYPE=Bot ALLOWED_BOTS='"dependabot[bot]"' +check "a string where an array belongs" false "$(out admit)" +run_case bot-numbers "$ADMIT" ACTOR='123' ACTOR_TYPE=Bot ALLOWED_BOTS='[123]' +check "a list of numbers matches nothing" false "$(out admit)" +run_case bot-no-actor "$ADMIT" ACTOR='' ACTOR_TYPE=Bot ALLOWED_BOTS='["dependabot[bot]"]' +check "an empty login matches nothing" false "$(out admit)" +run_case bot-null-list "$ADMIT" ACTOR='dependabot[bot]' ACTOR_TYPE=Bot ALLOWED_BOTS='[null]' +check "a list of nulls matches nothing" false "$(out admit)" +run_case bot-review "$ADMIT" EVENT_NAME=pull_request_review ACTOR='dependabot[bot]' ACTOR_TYPE=Bot \ + ALLOWED_BOTS='["dependabot[bot]"]' +check "a listed bot in a review body" true "$(out admit)" +run_case bot-thread "$ADMIT" EVENT_NAME=pull_request_review_comment ACTOR='dependabot[bot]' ACTOR_TYPE=Bot \ + ALLOWED_BOTS='["dependabot[bot]"]' +check "a listed bot in a diff thread" true "$(out admit)" +run_case bot-fork "$ADMIT" ACTOR='dependabot[bot]' ACTOR_TYPE=Bot ALLOWED_BOTS='["dependabot[bot]"]' \ + STUB_ORIGIN=fork +check "a listed bot still meets the fork check" false "$(out admit)" +run_case bot-label "$ADMIT" ACTOR='dependabot[bot]' ACTOR_TYPE=Bot ALLOWED_BOTS='["dependabot[bot]"]' \ + STUB_LABELS='ai: skip-review' +check "a listed bot still meets the label" false "$(out admit)" + +echo +echo "== 10. the admission: the rules that were already there, on the new paths ==" +run_case fork-thread "$ADMIT" EVENT_NAME=pull_request_review_comment STUB_ORIGIN=fork +check "a fork in a diff thread" false "$(out admit)" +check "and names the refusal" 1 "$(said 'explicit re-reviews are disabled for fork-originated')" +run_case fork-review "$ADMIT" EVENT_NAME=pull_request_review STUB_ORIGIN=fork +check "a fork in a review body" false "$(out admit)" +run_case fork-unreadable "$ADMIT" EVENT_NAME=pull_request_review STUB_ORIGIN=FAIL +check "an origin nobody could read" false "$(out admit)" +check "and says a fork is not ruled out" 1 "$(said 'could not read where owner/repo#7 comes from')" +run_case fork-null-head "$ADMIT" EVENT_NAME=pull_request_review_comment STUB_ORIGIN=null +check "a null head repository reads as a fork" false "$(out admit)" +run_case label-thread "$ADMIT" EVENT_NAME=pull_request_review_comment STUB_LABELS='ai: skip-review' +check "the skip label in a diff thread" false "$(out admit)" +check "and names the label" 1 "$(said 'carries ai: skip-review')" +run_case label-review "$ADMIT" EVENT_NAME=pull_request_review STUB_LABELS='ai: skip-review' +check "the skip label in a review body" false "$(out admit)" +run_case label-other "$ADMIT" EVENT_NAME=pull_request_review STUB_LABELS='needs-rebase,ai: nitpick' +check "another label admits" true "$(out admit)" +run_case label-fail "$ADMIT" EVENT_NAME=pull_request_review STUB_LABELS=FAIL +check "a label read that fails" false "$(out admit)" +check "and names both fixes" 1 "$(said 'Grant pull-requests: read on the calling job')" +run_case gate-review "$ADMIT" EVENT_NAME=pull_request_review STUB_COMMENTS=verdict +check "a review already ran, asked by name" true "$(out admit)" +check "and the gate reads nothing" 0 "$(calls comments)" +run_case gate-thread "$ADMIT" EVENT_NAME=pull_request_review_comment STUB_COMMENTS=verdict +check "the same from a diff thread" true "$(out admit)" +run_case not-parsed "$ADMIT" EVENT_NAME=pull_request_review PARSED=false +check "a body that is no command" false "$(out admit)" +check "and reads nothing at all" 0 "$(( $(calls membership) + $(calls origin) + $(calls labels) ))" + +echo +echo "== 11. the admission: the paths that were already there ==" +run_case close-bot "$ADMIT" MODE=close ACTOR='renovate[bot]' ACTOR_TYPE=Bot ALLOWED_BOTS='[]' +check "a teardown is not refused" true "$(out admit)" +check "and checks nothing" 0 "$(( $(calls membership) + $(calls origin) + $(calls labels) ))" +run_case close-outsider "$ADMIT" MODE=close STUB_TEAM=pending STUB_ORIGIN=fork STUB_LABELS='ai: skip-review' +check "a teardown from outside the team" true "$(out admit)" +run_case auto-draft "$ADMIT" EVENT_NAME=pull_request IS_DRAFT=true ACTION=opened BASE_REPO_ID=1 HEAD_REPO_ID=1 +check "a draft" false "$(out admit)" +check "and names it" 1 "$(said 'is a draft; not reviewing')" +run_case auto-first "$ADMIT" EVENT_NAME=pull_request ACTION=opened BASE_REPO_ID=1 HEAD_REPO_ID=1 +check "a first automatic review" true "$(out admit)" +check "and reads no team" 0 "$(calls membership)" +run_case auto-again "$ADMIT" EVENT_NAME=pull_request ACTION=synchronize BASE_REPO_ID=1 HEAD_REPO_ID=1 \ + STUB_COMMENTS=verdict +check "a push after a verdict" false "$(out admit)" +check "and points at the comment" 1 "$(said 'comment @seidroid review to ask for one')" +run_case auto-fork "$ADMIT" EVENT_NAME=pull_request ACTION=opened BASE_REPO_ID=1 HEAD_REPO_ID=2 +check "a fork pull request" false "$(out admit)" +check "and spends no API call" 0 "$(calls origin)" + +echo +echo "== 12. the payload field each step reads, per event ==" +ctx_comment issue_comment created alice User MEMBER '[]' +check "parse reads the comment body" '@seidroid review' "$(env_of parse BODY)" +check "parse reads the comment id" 42 "$(env_of parse COMMENT_ID)" +check "parse reads the pull request number" 7 "$(env_of parse PR_NUMBER)" +check "admit reads the commenter" alice "$(env_of "Admit the request" ACTOR)" +check "admit reads the commenter's type" User "$(env_of "Admit the request" ACTOR_TYPE)" +ctx_comment pull_request_review_comment created alice User MEMBER '[]' +check "parse reads the thread comment body" '@seidroid review' "$(env_of parse BODY)" +check "parse reads the thread comment id" 42 "$(env_of parse COMMENT_ID)" +check "parse reads the pull request number" 7 "$(env_of parse PR_NUMBER)" +check "admit reads the commenter" alice "$(env_of "Admit the request" ACTOR)" +ctx_comment pull_request_review submitted alice User MEMBER '[]' +# The review event names none of these under `comment`, so a step reading that +# key alone would see an empty body, parse no command, and refuse in silence. +check "parse reads the review body" '@seidroid review' "$(env_of parse BODY)" +check "parse reads the review id" 99 "$(env_of parse COMMENT_ID)" +check "parse reads the pull request number" 7 "$(env_of parse PR_NUMBER)" +check "admit reads the reviewer" alice "$(env_of "Admit the request" ACTOR)" +check "admit reads the reviewer's type" User "$(env_of "Admit the request" ACTOR_TYPE)" +ctx_comment pull_request_review submitted 'dependabot[bot]' Bot NONE '["dependabot[bot]"]' +check "admit reads a review bot's login" 'dependabot[bot]' "$(env_of "Admit the request" ACTOR)" +check "admit reads a review bot's type" Bot "$(env_of "Admit the request" ACTOR_TYPE)" + +echo +echo "== 12b. each consumer reads the output meant for it ==" +ctx_needs issue_comment review success true +check "the driver labels with trigger_id" 222 \ + "$(env_of "Drive session + collect verdict" TRIGGER_ID)" +check "the acknowledgement reacts on comment_id" 111 \ + "$(env_of "Acknowledge the trigger" TRIGGER_ID)" +check "and on the collection beside it" pulls/comments \ + "$(env_of "Acknowledge the trigger" COMMENT_API)" +check "the answer reads the same two" "111 pulls/comments" \ + "$(env_of "Answer the request" TRIGGER_ID) $(env_of "Answer the request" COMMENT_API)" +check "so does the withdrawal" "111 pulls/comments" \ + "$(env_of "Withdraw the reactions on a cancelled run" TRIGGER_ID) $(env_of "Withdraw the reactions on a cancelled run" COMMENT_API)" + +echo +echo "== 13. the defaults a caller inherits ==" +check "trigger-phrase default" '@seidroid' "$(input_of trigger-phrase default)" +check "trigger-phrase optional" false "$(input_of trigger-phrase required)" +check "allowed-bots default" '[]' "$(input_of allowed-bots default)" +check "allowed-bots optional" false "$(input_of allowed-bots required)" +check "allowed-team default" 'sei-protocol/sei-core' "$(input_of allowed-team default)" + +echo +echo "== 14. the acknowledgement lands on the object that asked ==" +run_case ack-issue "$ACK" COMMENT_API=issues/comments TRIGGER_ID=42 +check "the issue comments collection" \ + "CALL reaction POST repos/owner/repo/issues/comments/42/reactions" "$(cat "$CASE/calls.log")" +check "and reports it" 1 "$(said 'acknowledged comment 42')" +run_case ack-thread "$ACK" COMMENT_API=pulls/comments TRIGGER_ID=77 +check "the pull comments collection" \ + "CALL reaction POST repos/owner/repo/pulls/comments/77/reactions" "$(cat "$CASE/calls.log")" +run_case ack-fails "$ACK" COMMENT_API=pulls/comments STUB_REACTIONS=FAIL +check "a reaction that does not post warns" 1 "$(said '::warning::could not react to comment 42')" +check "and the review goes on" 0 "$(rc)" + +echo +echo "== 15. the answer withdraws from the same collection ==" +run_case answer-issue "$ANSWER" COMMENT_API=issues/comments STUB_REACTIONS=eyes +check "read from the issue collection" 1 \ + "$(grep -c '^CALL reaction GET repos/owner/repo/issues/comments/42/reactions' "$CASE/calls.log")" +check "withdrew the eyes there" 1 \ + "$(grep -c '^CALL reaction DELETE repos/owner/repo/issues/comments/42/reactions/1' "$CASE/calls.log")" +check "and thumbed up there" 1 \ + "$(grep -c '^CALL reaction POST repos/owner/repo/issues/comments/42/reactions' "$CASE/calls.log")" +run_case answer-thread "$ANSWER" COMMENT_API=pulls/comments STUB_REACTIONS=eyes +check "read from the pull collection" 1 \ + "$(grep -c '^CALL reaction GET repos/owner/repo/pulls/comments/42/reactions' "$CASE/calls.log")" +check "withdrew the eyes there" 1 \ + "$(grep -c '^CALL reaction DELETE repos/owner/repo/pulls/comments/42/reactions/1' "$CASE/calls.log")" +check "and thumbed up there" 1 \ + "$(grep -c '^CALL reaction POST repos/owner/repo/pulls/comments/42/reactions' "$CASE/calls.log")" +check "nothing reached the other collection" 0 "$(grep -c 'issues/comments' "$CASE/calls.log")" + +echo +echo "== 16. what ai-assistant.yml claims of the same body, on every event ==" +# The two tools read one comment, on the same three events. This group evaluates the +# assistant's own reply condition beside the parse above, so which of them answers a +# given body is measured rather than reasoned about. Both conditions read one phrase, +# and the reasoning holds only while the two defaults agree. +check "the assistant takes the same phrase" '@seidroid' "$(input_of_file "$ASSISTANT" trigger-phrase default)" +claims() { # event body -- true when the assistant's reply job would run + # Per event, because the assistant has a branch each and this branch keys on the + # payload key the event populates. A helper that named one event would measure the + # overlap on the path that already had it, and infer the two this workflow adds. + jq -nc --arg ev "$1" --arg b "$2" ' + {github: {event_name: $ev, event: {}}, + inputs: {"trigger-phrase": "@seidroid"}} + | .github.event |= (if $ev == "pull_request_review" + then {review: {body: $b, user: {type: "User"}}} + else {comment: {body: $b, user: {type: "User"}}} end) + | if $ev == "issue_comment" + then .github.event.issue = {number: 7, pull_request: {url: "u"}} + else . end' > "$CTX" + python3 "$HERE/gha.py" "$ASSISTANT" reply "$CTX" 2>/dev/null || printf 'error' +} +# Every body is checked on all three events, because this workflow now answers all +# three and the division of labour has to hold on each. +for ev in issue_comment pull_request_review_comment pull_request_review; do + # The bare form is the widening this workflow keeps. The assistant needs the @ to + # claim a body at all, so nothing else answers it, on any event. + run_case "claim-bare-$ev" "$PARSE" "EVENT_NAME=$ev" BODY='seidroid review' + check "$ev bare: this workflow" true "$(out should_run)" + check "$ev bare: the assistant" false "$(claims "$ev" 'seidroid review')" + run_case "claim-exact-$ev" "$PARSE" "EVENT_NAME=$ev" BODY='@seidroid review' + check "$ev exact: this workflow" true "$(out should_run)" + check "$ev exact: the assistant" false "$(claims "$ev" '@seidroid review')" + run_case "claim-prose-$ev" "$PARSE" "EVENT_NAME=$ev" BODY='Do we need @seidroid review here?' + check "$ev prose: this workflow" false "$(out should_run)" + check "$ev prose: the assistant" true "$(claims "$ev" 'Do we need @seidroid review here?')" + # Two bodies both tools answer. Whole-line anchoring is what admits the second, and + # it is also what keeps the prose above from starting a review, so the overlap is + # the price of that. The assistant reserves the exact body, and neither of these is + # it. This branch widens both onto the two events it adds, which is why each is + # asserted per event rather than once. + run_case "claim-close-$ev" "$PARSE" "EVENT_NAME=$ev" BODY='@seidroid review close' + check "$ev close: this workflow" true "$(out should_run)" + check "$ev close: the assistant too" true "$(claims "$ev" '@seidroid review close')" + run_case "claim-multiline-$ev" "$PARSE" "EVENT_NAME=$ev" BODY='Looks good. +@seidroid review +Thanks!' + check "$ev amid prose: this workflow" true "$(out should_run)" + check "$ev amid prose: the assistant too" true "$(claims "$ev" 'Looks good. +@seidroid review +Thanks!')" +done +# A review with no body at all. The assistant names that case; this workflow reads an +# empty body as no command. +run_case claim-empty-review "$PARSE" EVENT_NAME=pull_request_review BODY='' +check "an empty review body: this workflow" false "$(out should_run)" +check "an empty review body: the assistant" false "$(claims pull_request_review '')" + +echo +echo "assertions: $pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/test/seidroid-review/run.sh b/test/seidroid-review/run.sh new file mode 100755 index 0000000..acce567 --- /dev/null +++ b/test/seidroid-review/run.sh @@ -0,0 +1,787 @@ +#!/usr/bin/env bash +# Runs the shipped step script under bash with a gh stub on PATH. +# +# The step and the marker it writes are both read out of the workflow on every +# run, so a run tests what the file says now and cannot pass against a stale copy. +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPOROOT="$(cd "$HERE/../.." && pwd)" +WORKFLOW="$REPOROOT/.github/workflows/seidroid-review.yml" +STEP="Place findings on the code" +SCRIPT="$HERE/place.sh" +MARKER="$(python3 "$HERE/extract.py" "$WORKFLOW" "$STEP" "$SCRIPT")" || { + echo "could not read '$STEP' out of $WORKFLOW"; exit 1; } +pass=0 fail=0 +rows=() + +# A file list at the endpoint's cap, generated rather than committed. compare sends +# 300 files however large the diff is, so 300 is the length that tests the cap; the +# findings and the accepted line go with it so the case stays one thing to read. +GEN="$HERE/fx/gen" +rm -rf "$GEN"; mkdir -p "$GEN" +jq -nc '{files: [ range(300) | {filename: "pkg/gen\(.).go", + patch: "@@ -1,2 +1,3 @@\n one\n+two\n three"} ]}' \ + > "$GEN/files-at-cap.json" +jq -nc '[{file: "pkg/gen7.go", line: 2, side: "RIGHT", severity: "blocker", + detail: "A finding in a diff as long as the endpoint will send."}]' \ + > "$GEN/findings-at-cap.json" +printf 'pkg/gen7.go\tRIGHT\t2\n' > "$GEN/line-ok-gen.tsv" + +# Every shape of thread id one comment can name, generated because one of them is +# 201 characters and a fixture carrying it does not read. The step admits the shape +# GitHub mints and drops the rest, which is what keeps the record one id per line. +jq -nc --arg long "$(printf 'A%.0s' {1..201})" \ + '[{file: "pkg/a.go", line: 11, side: "RIGHT", severity: "blocker", + detail: "Several shapes of id on one comment.", + supersedes: ["PRRT_kwDOABCDEF4Ax1y2", "PRRT_has a space", + "carries\na line break", 42, $long, + "PRRT_kwDOABCDEF4Bz3w4"]}]' > "$GEN/findings-odd-ids.json" + +run_case() { + # $1 name, then KEY=VALUE overrides + local name="$1"; shift + CASE="$HERE/out/$name" + rm -rf "$CASE"; mkdir -p "$CASE" + export STUB_LOG="$CASE/calls.log"; : > "$STUB_LOG" + export STUB_REQUEST="$CASE/request.json"; : > "$STUB_REQUEST" + export STUB_BODIES="$CASE/bodies.txt"; : > "$STUB_BODIES" + export STUB_FILES="$HERE/fx/files.json" + export STUB_FILES_STALE="$HERE/fx/files-stale.json" + # What the pull request says its diff holds. fx/files.json serves exactly this + # many, so the step reads that list as whole; a case that serves a different + # number sets this to match, or to the number that exposes the shortfall. + export STUB_CHANGED_FILES=4 + export STUB_PR=7 + export STUB_BASE=0ba5e0ba5e0ba5e0ba5e0ba5e0ba5e0ba5e0ba5e0 + export STUB_STALE_SHA=facefeed1234facefeed1234facefeed1234face + export STUB_REVIEW=ok + export STUB_LINE_OK="$HERE/fx/line-ok.tsv" + export STUB_FILE_OK="$HERE/fx/file-ok.txt" + export FINDINGS="$HERE/fx/all-placeable.json" + export REVIEWED_SHA=deadbeefcafedeadbeefcafedeadbeefcafedead + for kv in "$@"; do export "${kv?}"; done + + export PATH="$HERE/bin:$PATH" + export RUNNER_TEMP="$CASE/tmp"; mkdir -p "$RUNNER_TEMP" + export GITHUB_OUTPUT="$CASE/output.txt"; : > "$GITHUB_OUTPUT" + export NOTE="$CASE/note.md" + export LINKAGE="$CASE/superseded-placed.txt" + export REPO=owner/repo PR=7 GH_TOKEN=x + export FINDING_MARKER="$MARKER" + bash "$SCRIPT" > "$CASE/stdout.txt" 2> "$CASE/stderr.txt" + echo "$?" > "$CASE/rc" +} + +# Runs the step again over the case directory and the RUNNER_TEMP the last run +# left, which is what a second attempt of a job on a non-ephemeral self-hosted +# runner sees. Only the overrides given here change. +rerun_case() { # KEY=VALUE overrides + for kv in "$@"; do export "${kv?}"; done + : > "$STUB_LOG"; : > "$GITHUB_OUTPUT"; : > "$STUB_BODIES" + bash "$SCRIPT" > "$CASE/stdout.txt" 2> "$CASE/stderr.txt" + echo "$?" > "$CASE/rc" +} + +out() { grep -E "^$1=" "$CASE/output.txt" | tail -1 | cut -d= -f2- ; } +calls() { grep -c "^CALL $1" "$CASE/calls.log" || true; } +# The thread ids the step recorded, sorted and space-joined, so a case names the set +# it expects on one line. Empty when no comment carrying a linkage posted. +linked() { sort -u "$CASE/superseded-placed.txt" 2>/dev/null | sed '/^$/d' | tr '\n' ' ' \ + | sed 's/ $//'; } + +check() { # name expected actual + if [ "$2" = "$3" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo " FAIL $1: want [$2] got [$3]"; fi +} + +report() { # label + rows+=("$(printf '%-34s rc=%s pull=%s cmp=%s reviews=%s line=%s file=%s on_line=%s on_file=%s unplaced=%s note=%s' \ + "$1" "$(cat "$CASE/rc")" "$(calls pull)" "$(calls compare)" "$(calls reviews)" "$(calls line-comment)" "$(calls file-comment)" \ + "$(out on_line)" "$(out on_file)" "$(out unplaced)" "$(grep -c '^- `' "$CASE/note.md" 2>/dev/null; true)")") +} + +echo "== 1. several findings, all placeable ==" +run_case all-placeable +report "1 all placeable" +check "one review call" 1 "$(calls reviews)" +check "no comment calls" 0 "$(( $(calls line-comment) + $(calls file-comment) ))" +check "on_line" 4 "$(out on_line)" +check "on_file" 0 "$(out on_file)" +check "unplaced" 0 "$(out unplaced)" +check "comments in request" 4 "$(jq '.comments | length' "$CASE/request.json")" +check "every body marked" 4 "$(jq --arg m "$FINDING_MARKER" '[.comments[] | select(.body | startswith($m + "\n"))] | length' "$CASE/request.json")" +check "event" COMMENT "$(jq -r .event "$CASE/request.json")" +check "commit_id" deadbeefcafedeadbeefcafedeadbeefcafedead "$(jq -r .commit_id "$CASE/request.json")" +check "body non-empty" true "$(jq -r '(.body | length) > 0' "$CASE/request.json")" +check "paths/lines/sides" 'pkg/a.go:11:RIGHT pkg/a.go:12:RIGHT pkg/b.go:2:RIGHT pkg/a.go:11:LEFT' \ + "$(jq -r '[.comments[] | "\(.path):\(.line):\(.side)"] | join(" ")' "$CASE/request.json")" +check "marker is first bytes" 4 "$(jq --arg m "$FINDING_MARKER" '[.comments[] | select((.body | .[0:($m|length)]) == $m)] | length' "$CASE/request.json")" +check "review body unmarked" false "$(jq -r --arg v "" '.body | startswith($v)' "$CASE/request.json")" +check "review body has no marker" false "$(jq -r --arg m "$FINDING_MARKER" '.body | startswith($m)' "$CASE/request.json")" +check "multi-line detail intact" true \ + "$(jq -r '.comments[0].body | contains("A second paragraph with a\ttab.")' "$CASE/request.json")" +check "quoting intact" true \ + "$(jq -r '.comments[1].body | contains("Backtick `code`, a \"quote\" and a $dollar.")' "$CASE/request.json")" + +echo "== 2. mixed: off-hunk line, untouched file, no line at all ==" +run_case mixed FINDINGS="$HERE/fx/mixed.json" +report "2 mixed" +check "one review call" 1 "$(calls reviews)" +check "no line-comment call" 0 "$(calls line-comment)" +check "three file attempts" 3 "$(calls file-comment)" +check "on_line" 4 "$(out on_line)" +check "on_file" 2 "$(out on_file)" +check "unplaced" 1 "$(out unplaced)" +check "note names the file" 1 "$(grep -c 'pkg/untouched.go:5' "$CASE/note.md")" +check "note header" 1 "$(grep -c 'Observations off the changed lines' "$CASE/note.md")" + +echo "== 3. the batch is rejected with 422 ==" +run_case batch-422 FINDINGS="$HERE/fx/mixed.json" STUB_REVIEW=422 +report "3 batch 422" +check "one review call" 1 "$(calls reviews)" +check "four line retries" 4 "$(calls line-comment)" +check "on_line" 4 "$(out on_line)" +check "on_file" 2 "$(out on_file)" +check "unplaced" 1 "$(out unplaced)" +check "every fallback body marked" 7 "$(grep -cxF -- "$FINDING_MARKER" "$CASE/bodies.txt")" +check "warning emitted" 1 "$(grep -c '::warning::owner/repo#7 refused the review' "$CASE/stdout.txt")" + +echo "== 4. batch 422, and one line the API also refuses on its own ==" +run_case batch-422-partial FINDINGS="$HERE/fx/mixed.json" STUB_REVIEW=422 \ + STUB_LINE_OK="$HERE/fx/line-ok-minus-one.tsv" +report "4 batch 422 + one bad line" +check "on_line" 3 "$(out on_line)" +check "on_file" 3 "$(out on_file)" +check "unplaced" 1 "$(out unplaced)" + +echo "== 5. the batch fails with something other than 422 ==" +run_case batch-500 FINDINGS="$HERE/fx/mixed.json" STUB_REVIEW=500 +report "5 batch 500" +check "one review call" 1 "$(calls reviews)" +check "no line retries" 0 "$(calls line-comment)" +check "on_line" 0 "$(out on_line)" +check "on_file" 2 "$(out on_file)" +check "unplaced" 5 "$(out unplaced)" +check "warning emitted" 1 "$(grep -c '::warning::the review carrying 4 comment(s) could not be posted' "$CASE/stdout.txt")" + +echo "== 6. the batch fails and the error body has no readable code ==" +run_case batch-noshape FINDINGS="$HERE/fx/mixed.json" STUB_REVIEW=noshape +report "6 batch, unreadable error" +check "no line retries" 0 "$(calls line-comment)" +check "on_line" 0 "$(out on_line)" +check "unplaced" 5 "$(out unplaced)" + +echo "== 7. zero findings (empty file) ==" +run_case zero FINDINGS="$HERE/fx/empty.json" +report "7 zero findings" +check "no calls at all" 0 "$(( $(calls reviews) + $(calls line-comment) + $(calls file-comment) + $(calls compare) + $(calls pull) ))" +check "on_line" 0 "$(out on_line)" +check "on_file" 0 "$(out on_file)" +check "unplaced" 0 "$(out unplaced)" + +echo "== 8. an empty findings array ==" +run_case emptyarray FINDINGS="$HERE/fx/emptyarray.json" +report "8 empty array" +check "no review call" 0 "$(calls reviews)" +check "on_line" 0 "$(out on_line)" +check "unplaced" 0 "$(out unplaced)" + +echo "== 9. the findings file cannot be read ==" +run_case broken FINDINGS="$HERE/fx/broken.json" +report "9 unreadable findings" +check "no posting calls" 0 "$(( $(calls reviews) + $(calls line-comment) + $(calls file-comment) ))" +check "on_line" 0 "$(out on_line)" +check "on_file" 0 "$(out on_file)" +check "unplaced" 0 "$(out unplaced)" +check "counts still written" 3 "$(grep -cE '^(on_line|on_file|unplaced)=' "$CASE/output.txt")" + +echo "== 10. the changed-file list cannot be read ==" +run_case nofiles FINDINGS="$HERE/fx/mixed.json" STUB_FILES=FAIL +report "10 file list fails" +check "no review call" 0 "$(calls reviews)" +check "seven line attempts" 7 "$(calls line-comment)" +check "on_line" 4 "$(out on_line)" +check "on_file" 2 "$(out on_file)" +check "unplaced" 1 "$(out unplaced)" + +echo "== 11. the reviewed commit was never recorded ==" +run_case nosha FINDINGS="$HERE/fx/mixed.json" REVIEWED_SHA="" +report "11 no reviewed commit" +check "no calls at all" 0 "$(( $(calls reviews) + $(calls line-comment) + $(calls file-comment) + $(calls compare) + $(calls pull) ))" +check "on_line" 0 "$(out on_line)" +check "on_file" 0 "$(out on_file)" +check "unplaced" 7 "$(out unplaced)" +check "note header" 1 "$(grep -c 'Every finding is here' "$CASE/note.md")" + +echo "== 12. a string line, a junk line, a missing side, a missing file ==" +run_case odd FINDINGS="$HERE/fx/odd.json" +report "12 odd field shapes" +check "one review call" 1 "$(calls reviews)" +check "two anchored" 2 "$(jq '.comments | length' "$CASE/request.json")" +check "string line coerced" 'pkg/a.go:12:RIGHT pkg/b.go:2:RIGHT' \ + "$(jq -r '[.comments[] | "\(.path):\(.line):\(.side)"] | join(" ")' "$CASE/request.json")" +check "junk line to file" 1 "$(calls file-comment)" +check "on_line" 2 "$(out on_line)" +check "on_file" 1 "$(out on_file)" +check "unplaced" 0 "$(out unplaced)" +check "no file named is dropped" 0 "$(grep -c 'No file named' "$CASE/note.md")" + +echo +echo "== 13. a finding with an empty severity ==" +run_case shift FINDINGS="$HERE/fx/shift.json" STUB_REVIEW=ok +report "13 empty severity" +check "two file attempts" 2 "$(calls file-comment)" +check "on_file" 1 "$(out on_file)" +check "unplaced" 1 "$(out unplaced)" +# shellcheck disable=SC2016 # the backticks are the summary's markdown, not a substitution +check "no field shift" 1 "$(grep -c '^- `pkg/untouched.go:3` (note) — An unrated finding nowhere.$' "$CASE/note.md")" +check "both bodies marked" 2 "$(grep -cxF -- "$FINDING_MARKER" "$CASE/bodies.txt")" + +echo +echo "== 14. the head moved during the review: the index reads the reviewed commit ==" +run_case at-reviewed-commit +report "14 index at reviewed commit" +check "one pull read" 1 "$(calls pull)" +check "one compare" 1 "$(calls compare)" +check "compared at REVIEWED_SHA" "CALL compare 0ba5e0ba5e0ba5e0ba5e0ba5e0ba5e0ba5e0ba5e0...deadbeefcafedeadbeefcafedeadbeefcafedead" \ + "$(grep '^CALL compare' "$CASE/calls.log")" +check "one review call" 1 "$(calls reviews)" +check "on_line" 4 "$(out on_line)" + +echo "== 14b. the same run, had the index read the pushed head instead ==" +run_case at-pushed-head REVIEWED_SHA=facefeed1234facefeed1234facefeed1234face STUB_CHANGED_FILES=2 +report "14b index at pushed head" +check "compared at that sha" "CALL compare 0ba5e0ba5e0ba5e0ba5e0ba5e0ba5e0ba5e0ba5e0...facefeed1234facefeed1234facefeed1234face" \ + "$(grep '^CALL compare' "$CASE/calls.log")" +check "no request was built" "" "$(cat "$CASE/request.json")" +check "no review call" 0 "$(calls reviews)" +check "all four to the file rung" 4 "$(calls file-comment)" + +echo "== 14c. base.sha cannot be read ==" +run_case nobase FINDINGS="$HERE/fx/mixed.json" STUB_BASE=FAIL +report "14c no base sha" +check "no compare" 0 "$(calls compare)" +check "no review call" 0 "$(calls reviews)" +check "ladder ran" 7 "$(calls line-comment)" +check "on_line" 4 "$(out on_line)" +check "on_file" 2 "$(out on_file)" +check "unplaced" 1 "$(out unplaced)" + +echo "== 15. a non-422 failure heads its findings for what they are ==" +run_case headings FINDINGS="$HERE/fx/mixed.json" STUB_REVIEW=500 +report "15 headings on a 500" +check "on-diff heading" 1 "$(grep -c '^\*\*On the changed lines, and not posted\.\*\*' "$CASE/note.md")" +check "off-diff heading" 1 "$(grep -c '^\*\*Observations off the changed lines\.\*\*' "$CASE/note.md")" +check "on-diff group first" true \ + "$([ "$(grep -n 'On the changed lines, and not posted' "$CASE/note.md" | cut -d: -f1)" -lt \ + "$(grep -n 'Observations off the changed lines' "$CASE/note.md" | cut -d: -f1)" ] && echo true || echo false)" +check "4 under on-diff" 4 "$(sed -n '/On the changed lines, and not posted/,/Observations off/p' "$CASE/note.md" | grep -c '^- `')" +check "1 under off-diff" 1 "$(sed -n '/Observations off/,$p' "$CASE/note.md" | grep -c '^- `')" +check "one rule, not two" 1 "$(grep -c '^---$' "$CASE/note.md")" +check "unplaced counts both" 5 "$(out unplaced)" +check "summary total" 5 "$(grep -c '^- `' "$CASE/note.md")" + +echo "== 15b. a 422 ladder puts what it believed on-diff in that group ==" +run_case headings-422 FINDINGS="$HERE/fx/mixed.json" STUB_REVIEW=422 \ + STUB_LINE_OK="$HERE/fx/none.tsv" STUB_FILE_OK="$HERE/fx/none.txt" +report "15b headings on a 422" +check "on-diff heading" 1 "$(grep -c '^\*\*On the changed lines, and not posted\.\*\*' "$CASE/note.md")" +check "4 under on-diff" 4 "$(sed -n '/On the changed lines, and not posted/,/Observations off/p' "$CASE/note.md" | grep -c '^- `')" +check "3 under off-diff" 3 "$(sed -n '/Observations off/,$p' "$CASE/note.md" | grep -c '^- `')" +check "unplaced" 7 "$(out unplaced)" + +echo "== 15c. no on-diff group means no on-diff heading ==" +run_case headings-clean FINDINGS="$HERE/fx/mixed.json" +report "15c only off-diff" +check "no on-diff heading" 0 "$(grep -c 'On the changed lines, and not posted' "$CASE/note.md")" +check "off-diff heading" 1 "$(grep -c '^\*\*Observations off the changed lines\.\*\*' "$CASE/note.md")" +check "one rule" 1 "$(grep -c '^---$' "$CASE/note.md")" + +echo +echo "== 16. a 422 whose body carries no status field ==" +run_case no-status-field FINDINGS="$HERE/fx/mixed.json" STUB_REVIEW=422nostatus +report "16 422 with no status field" +check "one review call" 1 "$(calls reviews)" +check "ladder ran" 4 "$(calls line-comment)" +check "on_line" 4 "$(out on_line)" +check "on_file" 2 "$(out on_file)" +check "unplaced" 1 "$(out unplaced)" +check "refusal warning" 1 "$(grep -c 'refused the review carrying' "$CASE/stdout.txt")" + +echo "== 17. a file the diff carries whose patch the API did not send ==" +run_case no-patch FINDINGS="$HERE/fx/nopatch.json" +report "17 file with no patch" +check "one review call" 1 "$(calls reviews)" +check "only the known line batched" 1 "$(jq '.comments | length' "$CASE/request.json")" +check "batched the right one" 'pkg/a.go:11:RIGHT' \ + "$(jq -r '[.comments[] | "\(.path):\(.line):\(.side)"] | join(" ")' "$CASE/request.json")" +check "the API was asked" 2 "$(calls line-comment)" +check "line accepted where the index could not see" 1 \ + "$(grep -c '^CALL line-comment pkg/huge.go RIGHT 120' "$CASE/calls.log")" +check "on_line" 2 "$(out on_line)" +check "on_file" 1 "$(out on_file)" +check "unplaced" 0 "$(out unplaced)" +check "no false 'outside this diff' on the accepted line" 0 \ + "$(grep -c '^CALL file-comment pkg/huge.go' "$CASE/calls.log")" + +echo +echo "== 18. the diff came back short of the pull request's own file count ==" +# pkg/b.go is in the pull request and missing from this list, which is what a +# truncated compare looks like. Read as the diff, it would put the finding on +# pkg/b.go under a body saying line 2 is outside a diff that adds line 2. +run_case short-list STUB_FILES="$HERE/fx/files-short.json" STUB_CHANGED_FILES=4 +report "18 short file list" +check "no review call" 0 "$(calls reviews)" +check "the ladder ran" 4 "$(calls line-comment)" +check "on_line" 4 "$(out on_line)" +check "on_file" 0 "$(out on_file)" +check "unplaced" 0 "$(out unplaced)" +check "nothing sent to a file" 0 "$(calls file-comment)" +check "no false off-diff body" 0 "$(grep -c "outside this diff's changed lines" "$CASE/bodies.txt")" +check "warning names the shortfall" 1 \ + "$(grep -c 'came back with 3 of its 4 file(s)' "$CASE/stdout.txt")" + +echo "== 19. a diff at the cap whose length the pull request confirms ==" +run_case at-cap STUB_FILES="$GEN/files-at-cap.json" STUB_CHANGED_FILES=300 \ + FINDINGS="$GEN/findings-at-cap.json" +report "19 at the cap, count agrees" +check "one review call" 1 "$(calls reviews)" +check "one comment batched" 1 "$(jq '.comments | length' "$CASE/request.json")" +check "batched the right one" 'pkg/gen7.go:2:RIGHT' \ + "$(jq -r '[.comments[] | "\(.path):\(.line):\(.side)"] | join(" ")' "$CASE/request.json")" +check "on_line" 1 "$(out on_line)" +check "unplaced" 0 "$(out unplaced)" +check "no shortfall warning" 0 "$(grep -c 'came back with' "$CASE/stdout.txt")" + +echo "== 20. a diff at the cap whose true length could not be read ==" +run_case at-cap-unknown STUB_FILES="$GEN/files-at-cap.json" STUB_CHANGED_FILES=NONE \ + FINDINGS="$GEN/findings-at-cap.json" STUB_LINE_OK="$GEN/line-ok-gen.tsv" +report "20 at the cap, count unknown" +check "no review call" 0 "$(calls reviews)" +check "the ladder ran" 1 "$(calls line-comment)" +check "on_line" 1 "$(out on_line)" +check "unplaced" 0 "$(out unplaced)" +check "warning names the cap" 1 \ + "$(grep -c 'which is all this endpoint sends' "$CASE/stdout.txt")" + +echo "== 21. a diff under the cap whose true length could not be read ==" +run_case under-cap-unknown STUB_CHANGED_FILES=NONE +report "21 under the cap, count unknown" +check "one review call" 1 "$(calls reviews)" +check "four comments batched" 4 "$(jq '.comments | length' "$CASE/request.json")" +check "on_line" 4 "$(out on_line)" +check "unplaced" 0 "$(out unplaced)" +check "no cap warning" 0 "$(grep -c 'all this endpoint sends' "$CASE/stdout.txt")" + +echo +echo "== 22. the batch is refused with 413, which an unbounded detail invites ==" +run_case batch-413 FINDINGS="$HERE/fx/mixed.json" STUB_REVIEW=413 +report "22 batch 413" +check "one review call" 1 "$(calls reviews)" +check "four line retries" 4 "$(calls line-comment)" +check "on_line" 4 "$(out on_line)" +check "on_file" 2 "$(out on_file)" +check "unplaced" 1 "$(out unplaced)" +check "warning names the code" 1 \ + "$(grep -c 'refused the review carrying 4 comment(s) with 413' "$CASE/stdout.txt")" + +echo "== 23. the batch is refused with 403 ==" +run_case batch-403 FINDINGS="$HERE/fx/mixed.json" STUB_REVIEW=403 +report "23 batch 403" +check "four line retries" 4 "$(calls line-comment)" +check "on_line" 4 "$(out on_line)" +check "unplaced" 1 "$(out unplaced)" + +echo "== 24. the batch is refused with 400 ==" +run_case batch-400 FINDINGS="$HERE/fx/mixed.json" STUB_REVIEW=400 +report "24 batch 400" +check "four line retries" 4 "$(calls line-comment)" +check "on_line" 4 "$(out on_line)" +check "unplaced" 1 "$(out unplaced)" + +echo "== 25. a 502 keeps its findings off the retry: the write may have landed ==" +run_case batch-502 FINDINGS="$HERE/fx/mixed.json" STUB_REVIEW=502 +report "25 batch 502" +check "one review call" 1 "$(calls reviews)" +check "no line retries" 0 "$(calls line-comment)" +check "on_line" 0 "$(out on_line)" +check "on_file" 2 "$(out on_file)" +check "unplaced" 5 "$(out unplaced)" +check "on-diff heading" 1 \ + "$(grep -c '^\*\*On the changed lines, and not posted\.\*\*' "$CASE/note.md")" + +echo +echo "== 26. three threads superseded, and only the first replacement places ==" +# The ticket's own case. The batch carries A's replacement; B's and C's name files +# this diff does not touch, so both reach the summary. The record has to name A and +# nothing else -- a record naming all three closes two threads with their +# replacements sitting in the summary instead of on the diff. +run_case superseded FINDINGS="$HERE/fx/superseded.json" +report "26 A places, B and C do not" +check "one review call" 1 "$(calls reviews)" +check "two file attempts" 2 "$(calls file-comment)" +check "on_line" 1 "$(out on_line)" +check "on_file" 0 "$(out on_file)" +check "unplaced" 2 "$(out unplaced)" +check "the linkage is published" true "$(out superseded_linked)" +check "only A's thread recorded" PRRT_kwDOABCDEF4Ax1y2 "$(linked)" + +echo "== 27. a replacement that degrades to a file comment still replaces ==" +# The recorded decision. A file comment is on the diff, in the file the reader is +# in, and its body carries the cited line, so it closes the thread it replaces. The +# sibling that reached neither a line nor a file closes nothing. +run_case superseded-file FINDINGS="$HERE/fx/superseded-file.json" +report "27 file comment replaces" +check "no review call" 0 "$(calls reviews)" +check "two file attempts" 2 "$(calls file-comment)" +check "on_line" 0 "$(out on_line)" +check "on_file" 1 "$(out on_file)" +check "unplaced" 1 "$(out unplaced)" +check "the file comment recorded its thread" PRRT_kwDOABCDEF4Ax1y2 "$(linked)" + +echo "== 27b. a replacement on a file whose patch the API did not send ==" +# The third bucket. A file with no patch has unknown lines rather than no lines, so +# its findings leave the batch and go to the API one at a time -- and the record has +# to follow them there, because that rung is the only thing that can still say +# whether the line exists. +run_case superseded-unknown FINDINGS="$HERE/fx/superseded-unknown.json" +report "27b unknown-lines bucket" +check "no review call" 0 "$(calls reviews)" +check "one line attempt" 1 "$(calls line-comment)" +check "on_line" 1 "$(out on_line)" +check "unplaced" 0 "$(out unplaced)" +check "its thread recorded" PRRT_kwDOABCDEF4Ax1y2 "$(linked)" + +echo "== 28. the batch is refused and the ladder splits the two threads ==" +# One review carried both replacements and the API refused it, so each is posted on +# its own. A's lands on its line; B's is refused there and refused on its file. The +# gate is per thread, so the refusal costs B's thread and not A's. +run_case superseded-batch FINDINGS="$HERE/fx/superseded-batch.json" STUB_REVIEW=422 \ + STUB_LINE_OK="$HERE/fx/line-ok-minus-one.tsv" STUB_FILE_OK="$HERE/fx/none.txt" +report "28 batch refused, ladder run" +check "one review call" 1 "$(calls reviews)" +check "two line retries" 2 "$(calls line-comment)" +check "one file attempt" 1 "$(calls file-comment)" +check "on_line" 1 "$(out on_line)" +check "unplaced" 1 "$(out unplaced)" +check "only A's thread recorded" PRRT_kwDOABCDEF4Ax1y2 "$(linked)" + +echo "== 29. the compare list is short, so the ladder runs and the record holds ==" +run_case superseded-short FINDINGS="$HERE/fx/superseded-short.json" \ + STUB_FILES="$HERE/fx/files-short.json" STUB_CHANGED_FILES=4 +report "29 short list, linkage kept" +check "no review call" 0 "$(calls reviews)" +check "two line attempts" 2 "$(calls line-comment)" +check "on_line" 1 "$(out on_line)" +check "unplaced" 1 "$(out unplaced)" +check "the linkage is published" true "$(out superseded_linked)" +check "only A's thread recorded" PRRT_kwDOABCDEF4Ax1y2 "$(linked)" +check "warning names the shortfall" 1 \ + "$(grep -c 'came back with 3 of its 4 file(s)' "$CASE/stdout.txt")" + +echo "== 30. a driver that publishes no linkage ==" +# fx/all-placeable.json carries the key on no finding, which is every driver before +# this change. The step has to say so, because that answer is what sends the resolve +# step to its per-review gate rather than to an empty record. +run_case superseded-older +report "30 older driver, no linkage" +check "one review call" 1 "$(calls reviews)" +check "on_line" 4 "$(out on_line)" +check "unplaced" 0 "$(out unplaced)" +check "no linkage published" false "$(out superseded_linked)" +check "nothing recorded" "" "$(linked)" + +echo "== 30b. an older driver whose batch is refused, so the ladder runs ==" +# The per-finding rungs are handed a dot where the file names no thread, and a dot +# is not an id and does not reach the record. A record that is non-empty when +# nothing was replaced says something untrue about what this review did. +run_case superseded-older-ladder FINDINGS="$HERE/fx/mixed.json" STUB_REVIEW=422 +report "30b older driver, ladder" +check "four line retries" 4 "$(calls line-comment)" +check "no linkage published" false "$(out superseded_linked)" +check "nothing recorded" "" "$(linked)" +check "the record is empty" 0 "$(wc -c < "$CASE/superseded-placed.txt" | tr -d ' ')" + +echo "== 31. a 502 records nothing: the write may have landed ==" +# The one place the record must under-report. A 502 may be a write that landed and +# lost its connection, so those findings are not retried -- and a thread closed on a +# comment this run cannot confirm is the defect the record exists to remove. +run_case superseded-502 FINDINGS="$HERE/fx/superseded.json" STUB_REVIEW=502 +report "31 502 records nothing" +check "one review call" 1 "$(calls reviews)" +check "no line retries" 0 "$(calls line-comment)" +check "on_line" 0 "$(out on_line)" +check "unplaced" 3 "$(out unplaced)" +check "the linkage is published" true "$(out superseded_linked)" +check "nothing recorded" "" "$(linked)" + +echo "== 32. one comment, six ids, and only two of them are ids ==" +# The record is read line by line and each id is matched whole, so an id carrying a +# space or a line break would split into something the resolve step cannot find. The +# two well-formed ids land on their own lines; the rest are dropped, which leaves +# their threads open. +run_case superseded-odd-ids FINDINGS="$GEN/findings-odd-ids.json" +report "32 odd ids dropped" +check "one review call" 1 "$(calls reviews)" +check "on_line" 1 "$(out on_line)" +check "the linkage is published" true "$(out superseded_linked)" +check "both ids recorded" "PRRT_kwDOABCDEF4Ax1y2 PRRT_kwDOABCDEF4Bz3w4" "$(linked)" +check "two lines, not six" 2 "$(wc -l < "$CASE/superseded-placed.txt" | tr -d ' ')" + +echo "== 32c. one comment naming two threads, posted on its own ==" +# The batch writes the record straight out of the placement file, one id per line. +# This is the other writer: the per-finding rung is handed the ids as one string and +# has to split them, because the record is read line by line and matched whole. +run_case superseded-two-ids FINDINGS="$GEN/findings-odd-ids.json" STUB_REVIEW=422 +report "32c two ids, one comment" +check "one review call" 1 "$(calls reviews)" +check "one line retry" 1 "$(calls line-comment)" +check "on_line" 1 "$(out on_line)" +check "both ids recorded" "PRRT_kwDOABCDEF4Ax1y2 PRRT_kwDOABCDEF4Bz3w4" "$(linked)" +check "on their own lines" 2 "$(wc -l < "$CASE/superseded-placed.txt" | tr -d ' ')" + +echo "== 32b. a second attempt does not inherit the first one's record ==" +# RUNNER_TEMP survives a re-run of a job on a non-ephemeral self-hosted runner, +# which is the steady state here. Attempt 1 replaces thread A; attempt 2 replaces +# thread B and nothing else, so A must not still be in the record to close. +run_case superseded-rerun FINDINGS="$HERE/fx/superseded.json" +check "attempt 1 recorded A" PRRT_kwDOABCDEF4Ax1y2 "$(linked)" +rerun_case FINDINGS="$HERE/fx/superseded-b.json" +report "32b a second attempt" +check "one review call" 1 "$(calls reviews)" +check "on_line" 1 "$(out on_line)" +check "attempt 2 records its own alone" PRRT_kwDOABCDEF4Bz3w4 "$(linked)" + +# ============================================================================ +# Resolve the threads this review closed +# ============================================================================ +# Extracted from the same workflow, so the two halves are one file's behaviour: +# placement writes the record and this reads it. extract.py prints the marker +# again, and the two have to be one value -- placement stamps a comment with it +# and this recognises a thread by it, so a drift between them closes nothing and +# says nothing. +RESOLVE_STEP="Resolve the threads this review closed" +RESOLVE="$HERE/resolve.sh" +RESOLVE_MARKER="$(python3 "$HERE/extract.py" "$WORKFLOW" "$RESOLVE_STEP" "$RESOLVE")" || { + echo "could not read '$RESOLVE_STEP' out of $WORKFLOW"; exit 1; } + +echo +echo "== the two steps read one marker ==" +check "one FINDING_MARKER" "$MARKER" "$RESOLVE_MARKER" + +# The identities this run posts as. REVIEWER is the one it holds now and the only +# one allowed to have written a thread it closes; WORKFLOW_ID is what a run without +# app credentials falls back to, which the history read admits and this does not. +REVIEWER='seidroid[bot]' +WORKFLOW_ID='github-actions[bot]' + +# The pull request's review threads, in two pages, because both readers page. Eight +# threads over the four ways one can fail this step's tests: the wrong identity, no +# marker, a marker quoted mid-body, and already resolved. +jq -nc --arg m "$MARKER" --arg rev "$REVIEWER" --arg wf "$WORKFLOW_ID" ' + def t($id; $resolved; $login; $body): + {id: $id, isResolved: $resolved, + comments: {nodes: [{body: $body, author: {login: $login}}]}}; + def page($nodes; $more): + {data: {repository: {pullRequest: {reviewThreads: + {nodes: $nodes, pageInfo: {hasNextPage: $more, endCursor: "c1"}}}}}}; + page([ t("PRRT_kwDOABCDEF4Ax1y2"; false; $rev; $m + "\n**blocker** — A still holds"), + t("PRRT_kwDOABCDEF4Bz3w4"; false; $rev; $m + "\n**blocker** — B still holds") ]; + true), + page([ t("PRRT_kwDOABCDEF4Cq9r8"; false; $rev; $m + "\n**suggestion** — C still holds"), + t("PRRT_kwDOABCDEF4Dm5n6"; false; $wf; $m + "\n**blocker** — under the other identity"), + t("PRRT_kwDOABCDEF4Es7t8"; false; "someone-else"; $m + "\n**blocker** — not ours"), + t("PRRT_kwDOABCDEF4Fu1v2"; true; $rev; $m + "\n**blocker** — ours, already resolved"), + t("PRRT_kwDOABCDEF4Gw3x4"; false; $rev; "A comment of ours carrying no marker."), + t("PRRT_kwDOABCDEF4Hy5z6"; false; $rev; "Quoting " + $m + " in prose, not opening with it.") ]; + false)' > "$GEN/threads.json" + +tA=PRRT_kwDOABCDEF4Ax1y2 +tB=PRRT_kwDOABCDEF4Bz3w4 +tC=PRRT_kwDOABCDEF4Cq9r8 +tD=PRRT_kwDOABCDEF4Dm5n6 +tE=PRRT_kwDOABCDEF4Es7t8 +tF=PRRT_kwDOABCDEF4Fu1v2 +tG=PRRT_kwDOABCDEF4Gw3x4 +tH=PRRT_kwDOABCDEF4Hy5z6 + +run_resolve() { + # $1 name, then KEY=VALUE overrides. RECORD is the record placement left, as a + # space-separated list of ids. KEY=UNSET removes the variable, which is how the + # case for an output placement never wrote is set up. + local name="$1"; shift + CASE="$HERE/out/$name" + rm -rf "$CASE"; mkdir -p "$CASE" + export STUB_LOG="$CASE/calls.log"; : > "$STUB_LOG" + export STUB_THREADS="$GEN/threads.json" + export STUB_RESOLVE=ok + export CHECK="$HERE/fx/check-abc.json" + export PLACED_LINKED=true + export PLACED_ON_LINE=1 PLACED_ON_FILE=0 PLACED_UNPLACED=0 + export REVIEWER_LOGIN="$REVIEWER" WORKFLOW_LOGIN="$WORKFLOW_ID" + RECORD="" + for kv in "$@"; do + case "$kv" in + *=UNSET) unset "${kv%%=*}" ;; + *) export "${kv?}" ;; + esac + done + + export PATH="$HERE/bin:$PATH" + export RUNNER_TEMP="$CASE/tmp"; mkdir -p "$RUNNER_TEMP" + export GITHUB_OUTPUT="$CASE/output.txt"; : > "$GITHUB_OUTPUT" + export REPO=owner/repo PR=7 GH_TOKEN=x + export FINDING_MARKER="$MARKER" + # What placement would have left behind, one id per line. + export LINKAGE="$CASE/superseded-placed.txt" + printf '%s' "$RECORD" | tr ' ' '\n' | sed '/^$/d' > "$LINKAGE" + bash "$RESOLVE" > "$CASE/stdout.txt" 2> "$CASE/stderr.txt" + echo "$?" > "$CASE/rc" +} + +# The threads the step actually closed, sorted and space-joined. +closed() { grep '^CALL resolve ' "$CASE/calls.log" 2>/dev/null | awk '{print $3}' \ + | sort -u | tr '\n' ' ' | sed 's/ $//'; } + +report_resolve() { # label + rows+=("$(printf '%-34s rc=%s threads=%s resolve=%s closed=[%s]' \ + "$1" "$(cat "$CASE/rc")" "$(calls threads)" "$(calls resolve)" "$(closed)")") +} + +echo +echo "== 33. A's replacement placed and B's and C's did not ==" +# The ticket's invariant, on the closing side. A closes because the comment that +# replaced it is on the code. B and C stay open because theirs are in the summary, +# and the per-review gate closed all three on A's placing. +run_resolve per-thread-a RECORD="$tA" +report_resolve "33 A closes, B and C hold" +check "one resolve call" 1 "$(calls resolve)" +check "closed" "$tA" "$(closed)" +check "two held open" 2 "$(grep -c 'stays open: nothing replacing it reached the code' "$CASE/stdout.txt")" +check "the tally" 1 "$(grep -c 'superseded: 1 of 3 thread(s) had their own replacement reach the code' "$CASE/stdout.txt")" +check "no refusal warning" 0 "$(grep -c '::warning::' "$CASE/stdout.txt")" + +echo "== 34. two replacements placed, and one of them is on page two ==" +run_resolve per-thread-ac RECORD="$tA $tC" +report_resolve "34 A and C close" +check "two resolve calls" 2 "$(calls resolve)" +check "closed" "$tA $tC" "$(closed)" +check "one held open" 1 "$(grep -c 'stays open: nothing' "$CASE/stdout.txt")" + +echo "== 35. no replacement placed, so no superseded thread closes ==" +run_resolve per-thread-none RECORD="" +report_resolve "35 nothing closes" +check "no thread read" 0 "$(calls threads)" +check "no resolve call" 0 "$(calls resolve)" +check "says so" 1 "$(grep -c 'this review closes no thread' "$CASE/stdout.txt")" + +echo "== 36. an addressed thread needs no record ==" +# addressed is a finding the diff no longer shows. Nothing replaces it, so nothing +# can be recorded for it, and it closes on the review publishing. +run_resolve addressed CHECK="$HERE/fx/check-addressed.json" RECORD="" +report_resolve "36 addressed closes" +check "one resolve call" 1 "$(calls resolve)" +check "closed" "$tA" "$(closed)" +check "the refused id is echoed" 1 \ + "$(grep -c "::warning::the review named review thread 'PRRT_neverOurs'" "$CASE/stdout.txt")" + +echo "== 37. an older driver publishes no linkage, so the per-review gate runs ==" +# The same empty record as case 35 and the same plan. What changes is one bit, and +# a driver that cannot say which comment replaced which thread closes all three on +# placement having dropped nothing. +run_resolve older-clean PLACED_LINKED=false PLACED_ON_LINE=3 PLACED_UNPLACED=0 RECORD="" +report_resolve "37 older driver, clean" +check "three resolve calls" 3 "$(calls resolve)" +check "closed" "$tA $tB $tC" "$(closed)" + +echo "== 38. an older driver with one finding unplaced closes none of them ==" +run_resolve older-unplaced PLACED_LINKED=false PLACED_ON_LINE=3 PLACED_UNPLACED=1 RECORD="" +report_resolve "38 older driver, one unplaced" +check "no resolve call" 0 "$(calls resolve)" +check "says why" 1 \ + "$(grep -c '3 superseded thread(s) stay open: 3 comment(s) reached the code and 1 could not be placed' "$CASE/stdout.txt")" + +echo "== 39. an older driver that reported its placings and not its unplaced count ==" +# An absent output is not a zero. Placement reported three comments on the code and +# said nothing about what it dropped, so something may be unplaced -- and this gate +# decides whether a live finding comes off a pull request, so the unknown falls on +# the side that leaves the thread open. +run_resolve older-half PLACED_LINKED=false PLACED_ON_LINE=3 PLACED_ON_FILE=0 \ + PLACED_UNPLACED=UNSET RECORD="" +report_resolve "39 older driver, half-reported" +check "no resolve call" 0 "$(calls resolve)" +check "names the unknown" 1 "$(grep -c 'an unreported number could not be placed' "$CASE/stdout.txt")" + +echo "== 39b. an older driver whose placement reported nothing at all ==" +run_resolve older-silent PLACED_LINKED=false PLACED_ON_LINE=UNSET PLACED_ON_FILE=UNSET \ + PLACED_UNPLACED=UNSET RECORD="" +report_resolve "39b older driver, silent" +check "no resolve call" 0 "$(calls resolve)" +check "names the unknown" 1 "$(grep -c 'an unreported number could not be placed' "$CASE/stdout.txt")" + +echo "== 40. the strict single-login test, and the marker test with it ==" +# Four threads the record names and this step closes none of. One this tool wrote +# under its other identity, one another account wrote, one of ours carrying no +# marker, and one quoting the marker mid-body. Only the first is not a warning. +run_resolve identity CHECK="$HERE/fx/check-identity.json" \ + RECORD="$tD $tE $tG $tH" +report_resolve "40 identity and marker" +check "no resolve call" 0 "$(calls resolve)" +check "the other identity is named" 1 \ + "$(grep -c "review thread $tD on owner/repo#7 was left under this tool's other identity" "$CASE/stdout.txt")" +check "three refusals" 3 \ + "$(grep -c "::warning::review thread '.*' is not an unresolved thread this tool left" "$CASE/stdout.txt")" +check "a foreign thread is refused" 1 "$(grep -c "review thread '$tE'" "$CASE/stdout.txt")" +check "an unmarked thread is refused" 1 "$(grep -c "review thread '$tG'" "$CASE/stdout.txt")" +check "a quoted marker does not open a body" 1 "$(grep -c "review thread '$tH'" "$CASE/stdout.txt")" +check "the tally" 1 \ + "$(grep -c 'threads: 0 closed, 3 refused, 1 left under another identity, 0 could not be resolved' "$CASE/stdout.txt")" + +echo "== 41. a thread of ours that is already resolved ==" +run_resolve already CHECK="$HERE/fx/check-resolved.json" RECORD="$tF" +report_resolve "41 already resolved" +check "no resolve call" 0 "$(calls resolve)" +check "says so" 1 "$(grep -c "review thread $tF is already resolved" "$CASE/stdout.txt")" +check "no warning" 0 "$(grep -c '::warning::' "$CASE/stdout.txt")" + +echo "== 42. the thread read fails, so nothing is closed ==" +run_resolve unreadable STUB_THREADS=FAIL RECORD="$tA" +report_resolve "42 thread read fails" +check "one thread read" 1 "$(calls threads)" +check "no resolve call" 0 "$(calls resolve)" +# -F, because the identity carries a [bot] suffix a regex reads as a character class. +check "warning names the identity" 1 \ + "$(grep -cF "::warning::the review threads on owner/repo#7 could not be read as $REVIEWER" "$CASE/stdout.txt")" + +echo "== 43. the mutation is refused, and the warning names the identity ==" +run_resolve refused CHECK="$HERE/fx/check-one.json" RECORD="$tA" STUB_RESOLVE=403 +report_resolve "43 mutation refused" +check "one resolve call" 1 "$(calls resolve)" +check "the thread warning" 1 "$(grep -c "review thread $tA on owner/repo#7 was not resolved" "$CASE/stdout.txt")" +check "the identity warning" 1 "$(grep -c 'that refusal names the identity, not the thread' "$CASE/stdout.txt")" +check "the tally" 1 "$(grep -c 'threads: 0 closed, 0 refused, 0 left under another identity, 1 could not be resolved' "$CASE/stdout.txt")" + +echo "== 44. the record cannot close a thread the plan does not name ==" +# The plan is the warrant. It is what the driver admitted against the history it +# was handed, and a record naming more than it closes a thread the driver refused. +run_resolve record-wider CHECK="$HERE/fx/check-one.json" RECORD="$tA $tB $tC" +report_resolve "44 record cannot widen" +check "one resolve call" 1 "$(calls resolve)" +check "closed" "$tA" "$(closed)" + +echo "== 45. no check file, so nothing names a thread ==" +run_resolve nocheck CHECK="$CASE/never-written.json" RECORD="$tA" +report_resolve "45 no check file" +check "no calls at all" 0 "$(( $(calls threads) + $(calls resolve) ))" +check "says so" 1 "$(grep -c 'no check file, so nothing names a thread to close' "$CASE/stdout.txt")" + +echo "== 46. a driver that writes no threads key at all ==" +run_resolve noplan CHECK="$HERE/fx/check-noplan.json" RECORD="$tA" +report_resolve "46 no thread plan" +check "no calls at all" 0 "$(( $(calls threads) + $(calls resolve) ))" +check "says so" 1 "$(grep -c 'this review closes no thread' "$CASE/stdout.txt")" + +echo +printf '%s\n' "${rows[@]}" +echo +echo "assertions: $pass passed, $fail failed" +[ "$fail" -eq 0 ]