diff --git a/config.example.yaml b/config.example.yaml index 4cb59b7f6..3cf792d4b 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -297,6 +297,13 @@ sync: github: enabled: true schedule: "*/15 * * * *" + # mode: poll # poll = gh CLI on this machine; hosted = the + # channel gateway polls GitHub and streams the + # records (needs channels.hosted). Restart-only. + # pr_watch: true # hosted only: a record when one of your own + # open PRs changes (state, commit, review, CI, + # comment). Its reason is always "author", and + # only allow_repos/deny_repos apply to it. # Guardrails (case-insensitive globs; deny wins; non-empty allow is # fail-closed). allow_* = allowlist (empty = all pass). # repo_name — the notification's repo full_name ("ClickHouse/nerve"). diff --git a/docs/config.md b/docs/config.md index 4326c1666..ce0864686 100644 --- a/docs/config.md +++ b/docs/config.md @@ -242,7 +242,8 @@ A reload is always explicit. Two things cause one: | Cron jobs (`config/cron/*.yaml`) | ✅ | | Custom cron gate plugins (new, edited *and* deleted `.py` files) | ✅ the registry is rebuilt from the directory | | Cron file locations (`cron.jobs_file`, `system_file`, `gate_plugins_dir`) | ✅ re-read from the new config | -| Cron sources: `sync.telegram`, `.gmail`, `.github`, `.github_events`, `.github_repos`, `.message_ttl_days`, and each source's `schedule` | ✅ runners are rebuilt and rescheduled, all or nothing: a `schedule` the scheduler will not take refuses the source reload with the running sources on their old triggers, and reports the error rather than `ok`. **`sync.codex` is not one of these**; see the restart table | +| Cron sources: `sync.telegram`, `.gmail`, `.github`, `.github_events`, `.github_repos`, `.message_ttl_days`, and each source's `schedule` | ✅ runners are rebuilt and rescheduled, all or nothing: a `schedule` the scheduler will not take refuses the source reload with the running sources on their old triggers, and reports the error rather than `ok`. In hosted mode the GitHub filters and `pr_watch` are published to the gateway again. **`sync.codex` and `sync.github.mode` are not among these**; see the restart table | +| `cron.delivery_debounce_seconds` | ✅ read for each delivery | | MCP servers (`mcp_servers`) | ✅ new sessions get the new set | | Skills (`skills/`) | ✅ re-scanned | | `lockdown` | ✅ the write guards and the layer stack both follow | @@ -254,7 +255,7 @@ A reload is always explicit. Two things cause one: | `external_agents.targets` (including each target's `enabled`), `.sync_interval_minutes`, `.conflict_policy` | ✅ from the next sweep, provided at least one target existed at startup (see the restart table) | | `sessions.sticky_period_minutes` | ✅ | | `telegram.dm_policy`, `.stream_mode` | ✅ read per update. Tightening `open` to `pairing` takes effect on the next message; `allowed_users` does not follow it (see the restart table) | -| `slack.*` | ✅ `enabled` starts or stops the channel; same-workspace token changes reconnect and roll back on failure; other changes apply to the next event. In hosted mode `enabled: false` pauses intake and notifications until it is `true` again, `source.*` applies to the next event, and `mode` needs a restart (see the restart table) | +| `slack.*` | ✅ `enabled` starts or stops the channel; same-workspace token changes reconnect and roll back on failure; other changes apply to the next event. In hosted mode `enabled: false` pauses intake, the records of a hosted GitHub source included, and notifications until it is `true` again, `source.*` applies to the next event, and `mode` needs a restart (see the restart table) | | `workflows.*` and `workflows.review_loop.*` — budget caps, concurrency, the warning fraction, iteration and criteria caps, leg engines/models, the verifier sandbox | ✅ read per use, by loops and runs already in flight as well as new ones. The two `enabled` flags and the two loop cadences are the exceptions; see the restart table | | `provider.*` and the API keys it selects (`aws_region`, `aws_profile`, `aws_access_key_id`, and the effective Anthropic key) | ✅ for sessions started **after** the reload. Each client's environment is built from the live reference when the session is created, by the same seam as `agent.*` below | | **`agent.*` and `codex.*`**: backend choice and models (`agent.backend`, `agent.cron_model`, `agent.model`, `codex.model`, `codex.cron_model`, `codex.models`), `max_turns`, `agent.effort`/`cron_effort` and `codex.effort_map`, `agent.thinking`, `agent.context_1m*`, `agent.background_agent_permissions`, `agent.agent_teams`, `agent.subagents.*`, idle timeouts, cache TTL, `codex.sandbox`, `.approval_policy`, `.web_search`, `.extra_config`, `.tool_timeout_sec`, `.bin_path`, `.auth`/`.api_key`/`.api_key_env`, `.pricing`, `.min_version`/`.max_version`, `.ultracode.*` | ✅ for sessions and turns **started after** the reload. The engine and both backends resolve these through one live reference, so a key cannot be hot in one and frozen in the other | @@ -294,6 +295,7 @@ reload cannot inspect, and are documented here only. | `mcp_endpoint.*` | fixed when the app was created | | `mcp_gateway_url` | the catalog reader is built at startup. See [MCP servers in external mode](#mcp-servers-in-external-mode) | | `slack.mode`, `channels.hosted.*` | the hosted channel, its stream endpoint, and its token verifier are built at startup. A reload reads the file that `gateway_jwks_file` names again, but not a changed path. Changing `mode` to `socket` reports a Slack reload error until a restart, because the hosted channel still holds the `slack` name. Changing it to `hosted` stops the Socket Mode connection at once, and the stream endpoint opens after a restart | +| `sync.github.mode` | the GitHub source, and in hosted mode the stream endpoint and the consumer of the gateway's records, are chosen at startup. Until a restart the source keeps the mode it started with, and so does the `github_pr_activity` gate. The filters and `pr_watch` do follow a reload: in hosted mode the reload publishes them to the gateway again. To go back to `poll`, set `enabled: false` and reload first (see [sources.md](sources.md#hosted-github-source)) | | `auth.jwt_secret` | pinned at startup for every consumer, web gateway and MCP endpoint alike. A reload that changes or removes it is reported and changes nothing live: removing the key must not reopen the instance, and rotating it must not swap the key under live sessions half-way. The next restart applies it (with the key removed, the secret generated into `nerve.db` takes over) | | `workflows.enabled`, `workflows.review_loop.enabled` | each service is created at startup and only when its flag is on. Turning one **off** does not stop the service already running, and turning it **on** creates nothing for a reload to reach | | `workflows.poll_interval_seconds`, `workflows.review_loop.reconcile_interval_seconds` | both loops were handed their interval when they started. Everything else under `workflows.*` is read per use (see the table above) | @@ -1319,7 +1321,9 @@ latest accepted event. reload. The stream endpoint still accepts streams, and events stay in the gateway's inbox until a reload sets `enabled` to `true`. While it is paused, Nerve sends no notifications and refuses `send_channel_message`. A turn that -is already running still sends its reply. +is already running still sends its reply. Nerve reads nothing from the +gateway's inbox while Slack is paused, so the records of a hosted GitHub +source wait there too (see below). Replies, reactions, and `send_channel_message` go to Slack as operations on a gateway stream that advertises them for the connection. A reply goes on the @@ -1360,6 +1364,21 @@ reports this and the local settings that still apply. hosted channels reads the key file again. Changing `mode` from `socket` to `hosted` stops the Socket Mode connection at once. +The same streams carry hosted sources. With `sync.github.mode: hosted` the +gateway polls GitHub for the agent and streams the records; the +`channels.hosted` settings above apply unchanged, and they are enough on their +own: Slack need not be hosted too. See +[Hosted GitHub source](sources.md#hosted-github-source). + +When Slack is hosted as well, its switch also holds GitHub back. Nerve reads +the events of every hosted provider from one ordered inbox, and it reads +nothing while hosted Slack is switched off (`slack.enabled: false`): the +GitHub records wait at the gateway until Slack is switched on again, and past +the gateway's limit of stored records the oldest are dropped. Keep a hosted +Slack switched on while GitHub is hosted; `nerve doctor` warns when it is off. +This is a known limitation: a read for each provider on its own, which would +keep GitHub records arriving while hosted Slack is off, is a follow-up. + ### Setting up the Slack app Socket Mode means the bot dials out to Slack, so Nerve needs no public URL. @@ -1713,7 +1732,14 @@ the channel transport, matching messages are written to a local buffer. | Key | Type | Default | Description | |-----|------|---------|-------------| -| `sync.github.schedule` | cron | `*/15 * * * *` | Fetch frequency | +| `sync.github.schedule` | cron | `*/15 * * * *` | Fetch frequency; in hosted mode the fallback cadence of the drain | +| `sync.github.mode` | string | `poll` | `poll`: fetch notifications here with the gh CLI. `hosted`: the channel gateway polls GitHub, applies the guardrails, and streams the records (needs the `channels.hosted` settings, see [Hosted mode](#hosted-mode)). An unknown value falls back to `poll` with a warning. Restart-only. See [sources.md](sources.md#hosted-github-source) | +| `sync.github.pr_watch` | bool | `true` | Hosted mode: also deliver a `github_pr_activity` record whenever one of the agent's own open pull requests changes. Its `reason` is always `author`, and only `allow_repos` and `deny_repos` apply to it (see [sources.md](sources.md#github-pull-request-activity-records)) | + +`cron.delivery_debounce_seconds` (int, default `60`) spaces the job runs that a +hosted delivery starts; a negative value reads as `0`, and a value above `600` +waits `600` seconds (`nerve config validate` warns). See +[cron.md](cron.md#runs-on-delivery). ## Memory (memU) diff --git a/docs/cron.md b/docs/cron.md index 16caf4683..23184c2c8 100644 --- a/docs/cron.md +++ b/docs/cron.md @@ -140,7 +140,7 @@ prompt definition. | `reminder_mode` | bool | no | Persistent only: send short reminder instead of full prompt on subsequent runs (default: false) | | `catchup` | bool | no | Fire once if the job missed a run while the server was down (on startup) or while the machine was paused (on resume — see [After a pause](#after-a-pause-sleep-suspend-snapshot-restore)) (default: true) | | `enabled` | bool | no | Whether the job is active (default: true) | -| `lock` | bool | no | Serialize this job's runs inside the daemon: a web-triggered run waits for the previous one to finish. An overlapping scheduled fire or catch-up run is skipped either way, and `nerve cron ` runs outside the lock (default: false) | +| `lock` | bool | no | Serialize this job's runs inside the daemon: a web-triggered run waits for the previous one to finish, and so does a scheduled fire that comes while a run for delivered source records goes on (see [Runs on delivery](#runs-on-delivery)). Any other overlapping scheduled fire or catch-up run is skipped either way, and `nerve cron ` runs outside the lock (default: false) | | `run_if` | list | no | Run gates — preconditions that must all hold for the job to fire. See [Run Gates](#run-gates) | | `workflow` | map | yes* | Launch a budget-capped [workflow run](workflow-runs.md) instead of a prompt: `{engine, prompt, budget_usd[, title, model, effort, cwd]}`. Takes precedence over `prompt`/`prompt_file`; the cron job only launches the run (fire-and-forget) — the run notifies on its own | @@ -253,6 +253,13 @@ run_if: consumer: inbox ``` +Checking the gate keeps the consumer's cursors from expiring: each check renews +the TTL (`sync.consumer_cursor_ttl_days`) of the live cursors it reads. A cursor +expires when nothing reads it for that long, and an expired cursor starts again at +the source's newest message. So without the renewal, a source that stayed quiet +for a few days would lose its first new message to the restart. A cursor that has +already expired still starts again at the newest message, as a new one does. + > **Legacy shorthand.** The older `skip_when_idle: []` / > `idle_consumer: ` fields still work — they are translated into an > equivalent `messages` gate at load time. Prefer `run_if` for new jobs. @@ -292,6 +299,15 @@ rather than bot autodetection, because bots are not reliably identifiable here: direction: an unlisted bot costs one extra wake, whereas a wrong autodetect would silently drop human feedback. +With `sync.github.mode: hosted` the gate always passes and runs no `gh`: the +channel gateway watches the agent's own open pull requests (`sync.github.pr_watch`) +and delivers a `github_pr_activity` record for each change. A record does not +start a job by itself. It starts a job gated on `messages` with +`sources: [github]` at once (see [Runs on delivery](#runs-on-delivery)), so gate +pull request monitors that way. A job gated on `github_pr_activity` alone is +never started by a delivery, and since the gate always passes, it runs, in full, +on every scheduled tick; `nerve doctor` names the jobs that still use this gate. + ### Forecasts A gate can also say ahead of time what it will decide. `forecast(ctx, at)` @@ -673,6 +689,35 @@ Source runners: See [sources.md](sources.md) for full documentation. +### Runs on delivery + +A hosted source — GitHub with `sync.github.mode: hosted`, see +[Hosted GitHub source](sources.md#hosted-github-source) — does not wait for its +schedule. When the channel gateway delivers records, the source's runner runs +at once, after any run of it already in progress (which may have fetched +before the records arrived); deliveries during that drain make it run once +more. Then every enabled job on the schedule (a job whose schedule did not +parse is not) whose gates include a `messages` gate naming that source, or +naming no source, runs once `cron.delivery_debounce_seconds` after the first +delivery of a burst. Later deliveries in that window add nothing: one burst, +one run. + +```yaml +cron: + delivery_debounce_seconds: 60 # default; 0 runs the jobs as soon as the records are in, at most 600 applies +``` + +Such a run goes through the job's gates, `lock` and run log like a scheduled +run, and the schedule listeners hear of it. It waits for a run of the job that +is already in flight. While it runs, a scheduled fire of the job is skipped, as +an overlapping scheduled fire is, or waits for it when the job has `lock`; a +catch-up is skipped, as it is while any run of the job goes on; and a run +started by hand starts as it always does. A catch-up queued before the run does +not run the job again, since the job has run since. The drain counts as a run +of the source's runner in the same way. Jobs without such a gate keep to their +schedule, and the schedules stay the fallback for everything. The setting is +read for each delivery, so a reload applies it. + ## Logging Every cron and source run is logged in the `cron_logs` SQLite table: diff --git a/docs/sources.md b/docs/sources.md index 5c6af1488..ded25c5ff 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -55,7 +55,7 @@ Consumers read from the inbox using the `poll_source` agent tool with a named co **Key properties:** - Consumer cursors are independent — multiple consumers can read the same source at different positions - New consumer cursors initialize to the latest message (no backlog flooding) -- Consumer cursors expire after N days of inactivity (configurable, default: 2 days) +- Consumer cursors expire after N days of inactivity (configurable, default: 2 days); a `messages` gate that checks the consumer renews the cursors it reads (see [cron.md](cron.md#messages)) - Cross-source deduplication happens naturally because one agent session sees messages from all sources ### Default Consumer: `inbox-processor` @@ -151,6 +151,101 @@ visible rather than failing the fetch. - **Enrichment:** Each notification is enriched with actual content from the subject (PR/issue body, state, assignees, labels) and the latest comment, fetched in parallel (up to 5 concurrent `gh api` calls) - **Metadata:** `reason`, `repo_name`, `actors` (every login involved) and `ci_branch` (branch of a CheckSuite run, `""` otherwise) — all four are guardrail-filterable, see [Guardrails](#guardrails-inbox-filtering) - **Default schedule:** `*/15 * * * *` (every 15 min) +- **Hosted mode:** with `sync.github.mode: hosted` the channel gateway polls GitHub instead; see below + +#### Hosted GitHub source + +With `sync.github.mode: hosted`, this process does not poll GitHub and needs no +`gh` login. The shared channel gateway (the one that serves +[hosted Slack](config.md#hosted-mode), set up with the same `channels.hosted` +settings) polls GitHub for the agent with its own credential, builds each record +the way the polling source does, enrichment included, and keeps the records +until Nerve reads them, also while Nerve is not running. Nerve reads them over +the hosted channel streams. + +- **What arrives:** `github_notification` records, one for each notification + thread whose `updated_at` moved, with the summary, content and metadata of the + polling source. With `pr_watch: true` (the default) also `github_pr_activity` + records: one whenever one of the agent's own open pull requests changes — + its state, head commit, review decision, CI checks, or a comment or review by + someone else. A pull request seen for the first time gives no record. Such a + record has the notification metadata plus `number`, `state`, `head`, + `review_decision` and `changes`, a list of `state`, `head`, `ci`, `review` + and `comment`. The agent is always its author, so its `reason` is always + `author`, and its `actors` are the reviewers and commenters. Only + `allow_repos` and `deny_repos` apply to it, at the gateway and here; the + actor, reason and CI branch lists are not checked for it (see + [Pull request activity records](#github-pull-request-activity-records)). +- **The filters are published from this config.** Nerve sends the gateway the + guardrail patterns of `sync.github` (`allow_repos`, `deny_repos`, + `allow_actors`, `deny_actors`, `allow_reasons`, `deny_reasons`, + `deny_ci_branches`) and `pr_watch` whenever a stream opens and again on every + reload. The gateway keeps the newest set and applies it before it stores a + record, so a record that the filters drop never reaches this agent and is + never a reason to start it. The guardrail still runs here too. A list that the + gateway cannot take — more than 256 patterns, or a pattern longer than 256 + bytes or with a control character — goes to the gateway wider, never + narrower: an allow list is left out and a deny list keeps the patterns that + fit, and the local guardrail drops the rest. Blank patterns are not sent; here + a blank allow pattern matches only an empty value, which GitHub never gives a + repo, an actor or a reason. If the whole subscription is still larger than + the gateway's frame limit, it goes with no patterns at all, and the guardrail + here does all the filtering. With `enabled: false` Nerve tells the gateway + that it subscribes to nothing. +- **The gateway must know the subscriptions.** A gateway that predates hosted + sources closes a stream that publishes them, hosted Slack traffic included, + so update the gateway before an agent switches to `mode: hosted`. +- **Same source, same name.** The records go to a local buffer, and the + `source:github` runner drains it. The drop hook, `condense`, health checks, + consumer cursors, `messages` gates and the Sources page work as they do for + the polling source. The drain also runs on `sync.github.schedule`, as a + fallback. +- **Runs on delivery.** As soon as records arrive, the runner drains them, and + the jobs gated on `messages` for `github`, or for any source, run once + `cron.delivery_debounce_seconds` (default 60) after the first record of a + burst; see [Runs on delivery](cron.md#runs-on-delivery). Gate a pull request + monitor on `messages` with `sources: [github]` rather than on + `github_pr_activity`, which always passes in hosted mode. +- **The events source stays off.** `github_events` reads your own activity + with `gh` on this machine, which hosted GitHub does not set up. While the + mode is `hosted`, the GitHub switch of the setup page's Sources step and + `nerve init` write `sync.github_events.enabled: false`, where in poll mode + they switch it on and off together with GitHub. On a machine with a `gh` + login it can still be switched on in the settings, until the GitHub switch + is saved again. +- **A switched-off hosted Slack holds GitHub back.** Nerve reads the events of + every hosted provider from one ordered inbox at the gateway, and it stops + reading while a hosted channel cannot take events. With `slack.mode: hosted` + and `slack.enabled: false`, Nerve therefore reads nothing from the gateway: + the GitHub records wait there too, no delivery starts a job, and past the + gateway's limit of stored records the oldest are dropped. They arrive once + Slack is switched on again. While GitHub is hosted, keep a hosted Slack + switched on, or switch both off; `nerve doctor` warns about this + combination. This is a known limitation: reading each provider's events on + its own, so that GitHub records arrive while hosted Slack is off, is a + follow-up. +- **Restart-only.** A change of `mode` takes effect at the next restart. To go + back from `hosted` to `poll`, switch the source off first: set + `enabled: false` and reload while the mode is still `hosted`. Nerve then + tells the gateway that it subscribes to nothing (a subscriptions frame of + `{"version":"1"}`), and the gateway stops polling GitHub for the agent at + once. Then set `mode: poll`, switch `enabled` back on, and restart. In poll + mode Nerve publishes no subscriptions at all. The gateway also counts an agent + as subscribed to nothing when a stream opens and publishes no subscriptions + within a short grace period, or when Nerve no longer serves the stream + endpoint. Until it notices, though, it keeps polling and delivering records + that Nerve no longer takes, which is why the reload comes first. + +```yaml +sync: + github: + mode: hosted + pr_watch: true + deny_reasons: ["subscribed"] # applied by the gateway and again here +``` + +`nerve doctor` checks the hosted settings and names the jobs that still rely +on the `github_pr_activity` gate. ### GitHub Events - **Adapter:** `nerve/sources/github_events.py` — uses `gh api /users/{username}/events` CLI @@ -346,6 +441,8 @@ sync: github: enabled: true + mode: poll # poll = gh CLI here; hosted = the channel gateway polls (restart-only) + pr_watch: true # Hosted mode: also a record when one of your open PRs changes schedule: "*/15 * * * *" batch_size: 30 condense: true # Haiku extraction for long notifications @@ -515,6 +612,18 @@ would flood the inbox with default-branch runs. |-------|------|---------|-------------| | `github.deny_ci_branches` | list | `[]` | Denylist of branch globs for CI runs. Empty = all CI notifications pass | +### GitHub pull request activity records + +With `sync.github.mode: hosted` and `pr_watch: true`, the channel gateway also +delivers `github_pr_activity` records, one for each change on the agent's own open +pull requests (see [Hosted GitHub source](#hosted-github-source)). The agent is always +the author of those pull requests: the `reason` of such a record is always `author`, and +its `actors` are the reviewers and commenters. So only the repo guardrail applies to it. +`allow_actors`, `deny_actors`, `allow_reasons`, `deny_reasons` and `deny_ci_branches` are +not checked for these records, neither by the gateway nor here. An `allow_actors` list +that names none of the reviewers, or an `allow_reasons` list without `author`, does not +drop them; `deny_repos` and an `allow_repos` list that the repo does not match do. + ### Debugging drops Dropped records are never persisted, so the run summary count is backed by per-record log @@ -693,7 +802,7 @@ The Sources page (`/sources`) has three tabs: - `consumer_cursors` — Per (consumer, source) read position with TTL and session linking - `source_messages` — Inbox messages with `raw_content` (original HTML), `processed_content` (LLM-condensed), TTL-based expiry - `source_run_log` — Per-run diagnostics (records ingested, errors, timestamps) -- `channel_observations` — Push buffer for chat messages a channel collected for the inbox, drained by `ChannelSource`. Trimmed toward a per-transport row target and TTL-swept by daily cleanup +- `channel_observations` — Push buffer for chat messages a channel collected for the inbox, drained by `ChannelSource`, and for the records of the hosted GitHub source, drained by the `github` runner. Trimmed toward a per-transport row target and TTL-swept by daily cleanup - `cron_logs` — Job execution history (source jobs use `source:` as job ID) ### API Endpoints @@ -860,7 +969,8 @@ export function MyRenderer({ content, metadata, summary }: Props) { | `nerve/sources/registry.py` | Config → `list[SourceRunner]` factory | | `nerve/sources/telegram.py` | Telegram adapter (Telethon) | | `nerve/sources/gmail.py` | Gmail adapter (gog CLI) | -| `nerve/sources/github.py` | GitHub notifications adapter (gh CLI) | +| `nerve/sources/github.py` | GitHub notifications adapter (gh CLI), record builder, GitHub guardrail | +| `nerve/sources/github_hosted.py` | GitHub source in hosted mode: drains the records the channel gateway delivered | | `nerve/sources/github_events.py` | GitHub user events adapter (gh CLI) | | `nerve/agent/tools.py` | Consumer tools: `list_sources`, `poll_source`, `read_source` | | `web/src/components/Sources/` | Source message renderers (email, GitHub, default markdown) | diff --git a/nerve/channels/hosted/__init__.py b/nerve/channels/hosted/__init__.py index b19864789..fdbaadbbd 100644 --- a/nerve/channels/hosted/__init__.py +++ b/nerve/channels/hosted/__init__.py @@ -4,4 +4,8 @@ WebSocket streams to ``/_internal/channel/v1/stream``, authenticated with a token that the gateway signs, and Nerve pulls admitted events from the gateway's inbox over them. + +Hosted sources use the same streams: the gateway polls a source (GitHub, with +``sync.github.mode: hosted``) under the filters that Nerve publishes, and +Nerve reads the records like any other event (see :mod:`.sources`). """ diff --git a/nerve/channels/hosted/contract/__init__.py b/nerve/channels/hosted/contract/__init__.py index 01309def9..0fa93e061 100644 --- a/nerve/channels/hosted/contract/__init__.py +++ b/nerve/channels/hosted/contract/__init__.py @@ -7,9 +7,11 @@ from nerve.channels.hosted.contract.model import ( DELIVERY_MODE_PULL, + EVENT_KIND_SOURCE_RECORD, MAX_FRAME_BYTES, MAX_TRANSFER_BYTES, PROTOCOL_VERSION, + SUBSCRIPTIONS_VERSION, AuthorReference, Capabilities, ConnectionStatus, @@ -23,12 +25,17 @@ InboxAckResult, InboxRead, InboxReadResult, + MetadataEntry, Negotiation, Nudge, Operation, OperationResult, Payload, + SourceFilters, + SourceRecordPayload, + SourceSubscription, StreamLimits, + Subscriptions, decode_envelope, encode_envelope, validate_inbox_read_result, @@ -38,9 +45,11 @@ __all__ = [ "DELIVERY_MODE_PULL", + "EVENT_KIND_SOURCE_RECORD", "MAX_FRAME_BYTES", "MAX_TRANSFER_BYTES", "PROTOCOL_VERSION", + "SUBSCRIPTIONS_VERSION", "AuthorReference", "Capabilities", "ConnectionStatus", @@ -54,6 +63,7 @@ "InboxAckResult", "InboxRead", "InboxReadResult", + "MetadataEntry", "Negotiation", "Nudge", "Operation", @@ -61,7 +71,11 @@ "Payload", "Rejected", "RejectionReason", + "SourceFilters", + "SourceRecordPayload", + "SourceSubscription", "StreamLimits", + "Subscriptions", "decode_envelope", "encode_envelope", "validate_inbox_read_result", diff --git a/nerve/channels/hosted/contract/model.py b/nerve/channels/hosted/contract/model.py index c2abbb3a1..34b5112bd 100644 --- a/nerve/channels/hosted/contract/model.py +++ b/nerve/channels/hosted/contract/model.py @@ -10,6 +10,7 @@ from __future__ import annotations import dataclasses +import unicodedata import uuid from datetime import datetime, timezone from typing import Any @@ -55,6 +56,17 @@ ) # Operations that the gateway may run again without a second provider effect. RETRY_SAFE_OPERATIONS = frozenset({"file_read", "typing"}) +# The event kind of a hosted source's record. It carries a ``record`` member +# and no message. +EVENT_KIND_SOURCE_RECORD = "source_record" +# The subscriptions payload, and the bounds that the gateway enforces on it. +SUBSCRIPTIONS_VERSION = "1" +MAX_SUBSCRIPTION_SOURCES = 16 +MAX_SUBSCRIPTION_PATTERNS = 256 +MAX_SUBSCRIPTION_PATTERN_BYTES = 256 +# A provider name is a lowercase token of at most this many bytes. +MAX_PROVIDER_BYTES = 64 +_PROVIDER_CHARACTERS = frozenset("abcdefghijklmnopqrstuvwxyz0123456789_-.") def byte_length(value: str) -> int: @@ -70,6 +82,32 @@ def _one_of(name: str, value: str, allowed: tuple[str, ...]) -> None: reject(RejectionReason.KIND_UNSUPPORTED, f"unknown {name} {value!r}") +def valid_pattern(pattern: str) -> bool: + """Whether the gateway takes *pattern* in a subscription filter. + + It must be non-empty, within the pattern byte limit, and free of control + characters. + """ + return ( + bool(pattern) + and byte_length(pattern) <= MAX_SUBSCRIPTION_PATTERN_BYTES + and not any(unicodedata.category(character) == "Cc" for character in pattern) + ) + + +def valid_provider(provider: str) -> bool: + """Whether the gateway takes *provider* as a provider name. + + It must be a non-empty token of lowercase ASCII letters, digits, ``_``, + ``-`` and ``.``, within the provider byte limit. + """ + return ( + bool(provider) + and all(character in _PROVIDER_CHARACTERS for character in provider) + and len(provider) <= MAX_PROVIDER_BYTES + ) + + # ---------------------------------------------------------------------- # # References and content # # ---------------------------------------------------------------------- # @@ -230,6 +268,39 @@ class EventDelivery: inbox_id: str = "" +@_record +class MetadataEntry: + """One metadata member of a source record: a string, or a list of strings.""" + + key: str = "" + value: str = omit_empty() + values: tuple[str, ...] = omit_empty(()) + + +@_record +class SourceRecordPayload: + """One record of a hosted source, normalized the way the local source builds it. + + ``id`` is the source's own key for the record. A changed record keeps its + ID, and Nerve stores it in place of the earlier copy. + """ + + id: str = "" + record_type: str = "" + summary: str = "" + content: str = omit_empty() + timestamp: datetime = ZERO_TIME + metadata: tuple[MetadataEntry, ...] = omit_empty(()) + + def validate(self) -> None: + if not self.id or not self.record_type or not self.summary: + malformed("source record has no ID, record type, or summary") + if max(byte_length(self.id), byte_length(self.record_type)) > MAX_IDENTIFIER_BYTES: + reject(RejectionReason.LIMIT_EXCEEDED, "source record ID or type exceeds the identifier limit") + if max(byte_length(self.summary), byte_length(self.content)) > MAX_TEXT_BYTES: + reject(RejectionReason.LIMIT_EXCEEDED, "source record text exceeds the text limit") + + @_record class Event: """One normalized provider event. @@ -253,9 +324,10 @@ class Event: attachments: tuple[AttachmentReference, ...] = omit_empty(()) reaction: Reaction | None = omit_empty(None) interaction: InteractionReference | None = omit_empty(None) + record: SourceRecordPayload | None = omit_empty(None) def validate(self) -> None: - """Check the members that intake and the hosted channel read.""" + """Check the members that intake, the hosted channel, and the hosted sources read.""" if not self.event_id or self.connection_id == NIL_UUID or not self.conversation.id: malformed("event has no event ID, connection, or conversation") if self.delivery is None or not self.delivery.inbox_id: @@ -271,6 +343,10 @@ def validate(self) -> None: malformed("reaction event has no reaction") if self.kind == "interaction" and (self.message is None or self.interaction is None): malformed("interaction event has no message or interaction") + if self.kind == EVENT_KIND_SOURCE_RECORD: + if self.record is None: + malformed("source record event has no record") + self.record.validate() # ---------------------------------------------------------------------- # @@ -556,6 +632,99 @@ def validate_operation_result(operation: Operation, result: OperationResult) -> malformed("successful file read has no transfer") +# ---------------------------------------------------------------------- # +# Source subscriptions # +# ---------------------------------------------------------------------- # + + +@_record +class SourceFilters: + """The guardrail patterns of one source. + + They match the way Nerve's inbox filter matches: deny wins, a non-empty + allow list must match, globs ignore case, and a list-valued field matches + when any of its elements does. A pull request activity record is matched + against the repository lists only. + """ + + allow_repos: tuple[str, ...] = omit_empty(()) + deny_repos: tuple[str, ...] = omit_empty(()) + allow_actors: tuple[str, ...] = omit_empty(()) + deny_actors: tuple[str, ...] = omit_empty(()) + allow_reasons: tuple[str, ...] = omit_empty(()) + deny_reasons: tuple[str, ...] = omit_empty(()) + deny_ci_branches: tuple[str, ...] = omit_empty(()) + + def pattern_lists(self) -> dict[str, tuple[str, ...]]: + """Every pattern list, by member name.""" + return {field.name: getattr(self, field.name) for field in dataclasses.fields(self)} + + +@_record +class SourceSubscription: + """What the agent takes from one provider's hosted source. + + ``pr_watch`` asks for a record whenever one of the agent's own open pull + requests changes. + """ + + provider: str = "" + filters: SourceFilters = struct(SourceFilters) + pr_watch: bool = omit_empty(False) + + +@_record +class Subscriptions: + """The agent's hosted source subscriptions, sent one way to the gateway. + + The gateway keeps the newest one it received for the agent, also while + no stream is open. No sources means that the agent subscribes to nothing. + """ + + version: str = "" + sources: tuple[SourceSubscription, ...] = omit_empty(()) + + def validate(self) -> None: + """Check what the gateway checks, before Nerve sends it. + + The version, the number of sources, each provider name and that no + provider appears twice, and the bounds of every filter list. + """ + if self.version != SUBSCRIPTIONS_VERSION: + reject(RejectionReason.VERSION_UNSUPPORTED, f"unknown subscriptions version {self.version!r}") + if len(self.sources) > MAX_SUBSCRIPTION_SOURCES: + reject(RejectionReason.LIMIT_EXCEEDED, "subscriptions name too many sources") + providers: set[str] = set() + for source in self.sources: + if not source.provider: + malformed("source subscription has no provider") + if byte_length(source.provider) > MAX_PROVIDER_BYTES: + reject(RejectionReason.LIMIT_EXCEEDED, "source subscription provider exceeds its byte limit") + if not valid_provider(source.provider): + malformed("source subscription provider is not a lowercase token") + if source.provider in providers: + malformed("subscriptions repeat a provider") + providers.add(source.provider) + for patterns in source.filters.pattern_lists().values(): + if len(patterns) > MAX_SUBSCRIPTION_PATTERNS: + reject(RejectionReason.LIMIT_EXCEEDED, "source filter has too many patterns") + if not all(valid_pattern(pattern) for pattern in patterns): + reject( + RejectionReason.LIMIT_EXCEEDED, + "source filter pattern is empty, too long, or has a control character", + ) + + def without_filters(self) -> Subscriptions: + """The same sources with no filter patterns. + + The gateway then keeps every record of those sources, and Nerve's own + inbox filter is the only one that drops records. + """ + return dataclasses.replace(self, sources=tuple( + dataclasses.replace(source, filters=SourceFilters()) for source in self.sources + )) + + # ---------------------------------------------------------------------- # # Envelope # # ---------------------------------------------------------------------- # @@ -578,6 +747,7 @@ class Payload: drain: Drain | None = omit_empty(None) negotiation: Negotiation | None = omit_empty(None) transfer: TransferChunk | None = omit_empty(None) + subscriptions: Subscriptions | None = omit_empty(None) @_record diff --git a/nerve/channels/hosted/manager.py b/nerve/channels/hosted/manager.py index 2f94c25f4..9f59d5812 100644 --- a/nerve/channels/hosted/manager.py +++ b/nerve/channels/hosted/manager.py @@ -25,7 +25,7 @@ from starlette.websockets import WebSocket from nerve.channels.hosted.auth import GatewayTokenVerifier, TokenRejected -from nerve.channels.hosted.contract import Capabilities, StreamLimits +from nerve.channels.hosted.contract import Capabilities, StreamLimits, Subscriptions from nerve.channels.hosted.stream import ( CLOSE_GOING_AWAY, ChannelStream, @@ -65,6 +65,8 @@ class StreamManager: """Accept authenticated gateway streams and choose among them. ``on_ready`` and ``on_nudge`` report stream events to the inbox reader. + ``subscriptions``, when given, is what every stream publishes after it + negotiates (see :meth:`ChannelStream.publish_subscriptions`). """ def __init__( @@ -76,6 +78,7 @@ def __init__( timing: StreamTiming = StreamTiming(), on_ready: Callable[[ChannelStream], None] = lambda stream: None, on_nudge: Callable[[ChannelStream, str], None] = lambda stream, purpose: None, + subscriptions: Callable[[], Subscriptions | None] | None = None, ) -> None: self._verifier = verifier self._receive_limits = receive_limits @@ -83,6 +86,7 @@ def __init__( self._timing = timing self._on_ready = on_ready self._on_nudge = on_nudge + self._subscriptions = subscriptions self._streams: dict[int, ChannelStream] = {} self._ids = itertools.count(1) self._changed = asyncio.Event() @@ -134,6 +138,7 @@ async def serve(self, websocket: WebSocket) -> None: receive_limits=self._receive_limits, listener=self, timing=self._timing, + subscriptions=self._subscriptions, ) # Reserved before the first await, so concurrent upgrades cannot # both pass the stream limit. @@ -235,6 +240,26 @@ async def wait_for_change(self, timeout: float) -> None: except TimeoutError: pass + async def publish_subscriptions(self) -> int: + """Send the current source subscriptions on every ready stream. + + Returns the number of streams that sent them. Each stream bounds its + own send, so a stalled one costs at most that bound; a stream that + fails is logged and the others still send. A stream that negotiates + later sends the settings current at that time. + """ + streams = [stream for stream in self.streams if stream.ready] + outcomes = await asyncio.gather( + *(stream.publish_subscriptions() for stream in streams), return_exceptions=True, + ) + for stream, outcome in zip(streams, outcomes): + if isinstance(outcome, Exception): + logger.error( + "Channel stream %d: publishing the source subscriptions failed: %r", + stream.id, outcome, exc_info=outcome, + ) + return sum(1 for outcome in outcomes if outcome is True) + def capabilities_for(self, connection_id: uuid.UUID) -> Capabilities | None: """The capabilities of *connection_id* on the preferred stream that has them.""" for stream in self.streams: diff --git a/nerve/channels/hosted/runtime.py b/nerve/channels/hosted/runtime.py index 9a88d7d5e..b0e905cdf 100644 --- a/nerve/channels/hosted/runtime.py +++ b/nerve/channels/hosted/runtime.py @@ -1,10 +1,12 @@ -"""Build and run hosted channels from the process config. +"""Build and run hosted channels and hosted sources from the process config. The runtime owns the token verifier, the stream manager, the operation -runner, the inbox reader, and one :class:`HostedChannel` for each provider in -hosted mode. The hosted settings and the provider modes are read once at -startup; a change needs a restart. A reload reads the gateway key file again. -The source grant of each provider is read per event, so it follows a reload. +runner, the inbox reader, one :class:`HostedChannel` for each chat provider in +hosted mode, and one :class:`HostedSourceConsumer` for each hosted source. The +hosted settings and the provider modes are read once at startup; a change +needs a restart. A reload reads the gateway key file again and publishes the +source subscriptions again. The source grant of each chat provider is read +per event, so it follows a reload. """ from __future__ import annotations @@ -19,10 +21,16 @@ MAX_FRAME_BYTES, MAX_TRANSFER_BYTES, StreamLimits, + Subscriptions, ) -from nerve.channels.hosted.intake import InboxReader, ReaderSettings +from nerve.channels.hosted.intake import EventConsumer, InboxReader, ReaderSettings from nerve.channels.hosted.manager import StreamManager from nerve.channels.hosted.operations import OperationRunner, OperationTiming +from nerve.channels.hosted.sources import ( + HostedSourceConsumer, + hosted_sources, + subscriptions_for, +) from nerve.channels.hosted.stream import ChannelStream, StreamTiming if TYPE_CHECKING: @@ -30,6 +38,7 @@ from nerve.channels.router import ChannelRouter from nerve.config import NerveConfig + from nerve.db import Database logger = logging.getLogger(__name__) @@ -44,8 +53,8 @@ ) -def hosted_providers(config: NerveConfig) -> list[str]: - """The providers whose traffic comes from the gateway. +def hosted_channel_providers(config: NerveConfig) -> list[str]: + """The chat providers whose traffic comes from the gateway. A provider that is switched off is included. Its channel reads ``enabled`` for each event, so intake stays paused until a reload @@ -54,8 +63,22 @@ def hosted_providers(config: NerveConfig) -> list[str]: return ["slack"] if config.slack.mode == "hosted" else [] +def hosted_providers(config: NerveConfig) -> list[str]: + """Every provider whose traffic comes from the gateway: chat, then sources. + + A hosted source that is switched off is included too: the gateway is + told that the agent subscribes to nothing, and the records already on + their way are kept for when it is switched on. + """ + return hosted_channel_providers(config) + hosted_sources(config) + + class HostedChannelRuntime: - """Everything hosted mode needs, started and stopped as one unit.""" + """Everything hosted mode needs, started and stopped as one unit. + + ``db`` is where hosted sources buffer their records; without it, the + router's engine database is used. + """ def __init__( self, @@ -67,15 +90,22 @@ def __init__( reader_settings: ReaderSettings = ReaderSettings(), operation_timing: OperationTiming = OperationTiming(), notification_service: Any | None = None, + db: Database | None = None, ) -> None: settings = config.channels.hosted problems = settings.problems() if problems: raise ValueError("; ".join(problems)) - providers = hosted_providers(config) - if not providers: + providers = hosted_channel_providers(config) + source_providers = hosted_sources(config) + if not providers and not source_providers: raise ValueError("no provider is in hosted mode") + if db is None: + db = getattr(getattr(router, "engine", None), "db", None) + if source_providers and db is None: + raise ValueError("hosted sources need a database to buffer their records") self.router = router + self._config = config_getter tenant_id = uuid.UUID(settings.tenant_id) agent_id = uuid.UUID(settings.agent_id) self.verifier = GatewayTokenVerifier( @@ -92,6 +122,7 @@ def __init__( timing=stream_timing, on_ready=self._stream_ready, on_nudge=self._stream_nudged, + subscriptions=self.subscriptions if source_providers else None, ) self.operations = OperationRunner(self.streams, operation_timing) self.channels: dict[str, HostedChannel] = { @@ -105,7 +136,14 @@ def __init__( ) for provider in providers } - self.reader = InboxReader(self.streams, dict(self.channels), settings=reader_settings) + # Source consumers read events but are not channels: they stay out of + # the router, so nothing routes a reply or a notification to them. + self.sources: dict[str, HostedSourceConsumer] = { + provider: HostedSourceConsumer(provider, db, config_getter) + for provider in source_providers + } + consumers: dict[str, EventConsumer] = {**self.channels, **self.sources} + self.reader = InboxReader(self.streams, consumers, settings=reader_settings) self._registered: list[HostedChannel] = [] async def start(self) -> None: @@ -119,14 +157,17 @@ async def start(self) -> None: await channel.start() self.reader.start() logger.info( - "Hosted channels ready for gateway streams: %s", ", ".join(self.channels), + "Hosted providers ready for gateway streams: %s", + ", ".join([*self.channels, *(f"{name} (source)" for name in self.sources)]), ) async def stop(self) -> None: - """Drain and close the streams, then stop the channels.""" + """Drain and close the streams, then stop the channels and sources.""" await self.streams.drain() await self.reader.stop() await self.streams.close() + for source in self.sources.values(): + await source.stop() for channel in self._registered: try: await channel.stop() @@ -142,6 +183,31 @@ def reload_keys(self) -> str: """ return f"{self.verifier.reload_keys()} gateway key(s)" + def subscriptions(self) -> Subscriptions | None: + """The ``subscriptions`` payload from the current config. + + ``None`` without hosted sources: such a runtime sends no frame, so a + gateway that predates the frame never receives one. + """ + if not self.sources: + return None + return subscriptions_for(self._config(), self.sources) + + async def republish_subscriptions(self) -> str | None: + """Send the subscriptions again on every open stream, after a reload. + + ``None`` without hosted sources, otherwise a summary for the reload. + """ + if not self.sources: + return None + sent = await self.streams.publish_subscriptions() + return f"published on {sent} stream(s)" + + def set_delivery_listener(self, listener: Callable[[str], Any] | None) -> None: + """Call *listener* with the provider name whenever a source record is stored.""" + for source in self.sources.values(): + source.set_delivery_listener(listener) + async def serve(self, websocket: WebSocket) -> None: """Handle one upgrade at the stream endpoint.""" await self.streams.serve(websocket) @@ -181,4 +247,9 @@ def _reaction_names(provider: str) -> dict[str, str]: return {} -__all__ = ["RECEIVE_LIMITS", "HostedChannelRuntime", "hosted_providers"] +__all__ = [ + "RECEIVE_LIMITS", + "HostedChannelRuntime", + "hosted_channel_providers", + "hosted_providers", +] diff --git a/nerve/channels/hosted/sources.py b/nerve/channels/hosted/sources.py new file mode 100644 index 000000000..078740d68 --- /dev/null +++ b/nerve/channels/hosted/sources.py @@ -0,0 +1,267 @@ +"""Hosted sources: records that the shared gateway polls for this agent. + +In hosted mode the channel gateway polls a source instead of this process. +It applies the filters that Nerve publishes before it stores a record, keeps +the records in the agent's inbox until Nerve reads them, also while Nerve is +not running, and Nerve reads them over the same streams that carry hosted +channel traffic. GitHub is the only hosted source. + +:class:`HostedSourceConsumer` takes a provider's records from the inbox reader +and writes each one to the local ``channel_observations`` buffer under the +provider's name. The provider's source runner drains that buffer into the +inbox with the guardrail, drop hook, and condensing of the polling source +(:mod:`nerve.sources.github_hosted`). A record is acknowledged only after its +buffer row is written. + +:func:`subscriptions_for` builds the ``subscriptions`` frame from the same +settings that build the inbox filter, so the gateway and Nerve drop the same +records. +""" + +from __future__ import annotations + +import asyncio +import inspect +import logging +from collections.abc import Callable, Iterable +from typing import TYPE_CHECKING, Any + +from nerve.channels.hosted.contract import ( + EVENT_KIND_SOURCE_RECORD, + SUBSCRIPTIONS_VERSION, + Event, + SourceFilters, + SourceSubscription, + Subscriptions, +) +from nerve.channels.hosted.contract.model import ( + MAX_SUBSCRIPTION_PATTERN_BYTES, + MAX_SUBSCRIPTION_PATTERNS, + ZERO_TIME, + valid_pattern, +) +from nerve.channels.hosted.contract.wire import format_time +from nerve.channels.hosted.intake import Disposition + +if TYPE_CHECKING: + from nerve.config import GitHubSyncConfig, NerveConfig + from nerve.db import Database + +logger = logging.getLogger(__name__) + +GITHUB = "github" + +# Row target of a provider's buffer. The gateway keeps at most this many +# source records for an agent, and the runner drains the buffer after every +# delivery, so it is reached only while the drain keeps failing. +BUFFER_MAX_ROWS = 2_000 + + +def hosted_sources(config: NerveConfig) -> list[str]: + """The source providers whose records come from the gateway.""" + return [GITHUB] if config.sync.github.mode == "hosted" else [] + + +# ---------------------------------------------------------------------- # +# Subscriptions # +# ---------------------------------------------------------------------- # + + +def subscriptions_for(config: NerveConfig, providers: Iterable[str]) -> Subscriptions: + """What the agent subscribes to, among the hosted *providers*. + + A provider that is switched off is left out, so the gateway stops polling + it for this agent. + """ + sources: list[SourceSubscription] = [] + if GITHUB in providers and config.sync.github.enabled: + sources.append(github_subscription(config.sync.github)) + return Subscriptions(version=SUBSCRIPTIONS_VERSION, sources=tuple(sources)) + + +def github_subscription(settings: GitHubSyncConfig) -> SourceSubscription: + """The GitHub subscription, with the inbox guardrail's patterns as filters. + + A list the gateway cannot take is published wider, never narrower: an + allow list goes empty and a deny list keeps the patterns that fit. The + gateway then stores records that Nerve's inbox filter still drops, which + costs a wake but loses nothing. + """ + return SourceSubscription( + provider=GITHUB, + filters=SourceFilters( + allow_repos=_allow_list("allow_repos", settings.allow_repos), + deny_repos=_deny_list("deny_repos", settings.deny_repos), + allow_actors=_allow_list("allow_actors", settings.allow_actors), + deny_actors=_deny_list("deny_actors", settings.deny_actors), + allow_reasons=_allow_list("allow_reasons", settings.allow_reasons), + deny_reasons=_deny_list("deny_reasons", settings.deny_reasons), + deny_ci_branches=_deny_list("deny_ci_branches", settings.deny_ci_branches), + ), + pr_watch=bool(settings.pr_watch), + ) + + +def _patterns(values: object) -> list[str]: + """The non-blank patterns of a setting, stripped as the inbox filter strips them.""" + if not isinstance(values, (list, tuple)): + return [] + return [text for text in (str(value).strip() for value in values if value is not None) if text] + + +def _allow_list(key: str, values: object) -> tuple[str, ...]: + patterns = _patterns(values) + if len(patterns) <= MAX_SUBSCRIPTION_PATTERNS and all(valid_pattern(p) for p in patterns): + return tuple(patterns) + logger.warning( + "sync.github.%s has more than %d patterns, or one longer than %d bytes or with a " + "control character; the gateway is told to keep every record and the inbox " + "filter applies the list alone", + key, MAX_SUBSCRIPTION_PATTERNS, MAX_SUBSCRIPTION_PATTERN_BYTES, + ) + return () + + +def _deny_list(key: str, values: object) -> tuple[str, ...]: + patterns = _patterns(values) + kept = [p for p in patterns if valid_pattern(p)][:MAX_SUBSCRIPTION_PATTERNS] + if len(kept) < len(patterns): + logger.warning( + "sync.github.%s has more than %d patterns, or one longer than %d bytes or with " + "a control character; the gateway gets the first %d usable ones and the inbox " + "filter applies the rest", + key, MAX_SUBSCRIPTION_PATTERNS, MAX_SUBSCRIPTION_PATTERN_BYTES, len(kept), + ) + return tuple(kept) + + +# ---------------------------------------------------------------------- # +# Consumer # +# ---------------------------------------------------------------------- # + + +def buffered_record(event: Event) -> dict[str, Any]: + """The buffer payload of one source record event. + + ``record.metadata`` keeps the wire entries (``key`` with ``value`` or + ``values``); the source that drains the buffer knows which keys hold + lists and which hold flags. + """ + record = event.record + timestamp = record.timestamp if record.timestamp != ZERO_TIME else event.occurred_at + return { + "event_id": event.event_id, + "connection_id": str(event.connection_id), + "conversation_id": event.conversation.id, + "record": { + "id": record.id, + "record_type": record.record_type, + "summary": record.summary, + "content": record.content, + "timestamp": format_time(timestamp) if timestamp != ZERO_TIME else "", + "metadata": [ + {"key": entry.key, "value": entry.value, "values": list(entry.values)} + for entry in record.metadata + ], + }, + } + + +class HostedSourceConsumer: + """The inbox reader's consumer for one provider's hosted source records. + + It holds no stream state and never refuses capacity: the reader pauses + every provider while any consumer refuses, and a write to the local + buffer needs no room. A switched-off source still buffers the records + already on their way; its runner drains them once it is switched on. + """ + + def __init__( + self, + provider: str, + db: Database, + config_getter: Callable[[], NerveConfig], + ) -> None: + self._provider = provider + self._db = db + self._config = config_getter + self._listener: Callable[[str], Any] | None = None + self._tasks: set[asyncio.Future] = set() + + @property + def name(self) -> str: + return self._provider + + def can_accept(self) -> bool: + return True + + def set_delivery_listener(self, listener: Callable[[str], Any] | None) -> None: + """Call *listener* with the provider name after each stored record. + + A listener that returns an awaitable runs as its own task, so the + reader never waits for it. + """ + self._listener = listener + + async def deliver(self, event: Event) -> Disposition: + purpose = event.admission.purpose + if event.kind != EVENT_KIND_SOURCE_RECORD or purpose != "observe" or event.record is None: + return Disposition.rejected( + "admission_rejected", f"{event.kind} ({purpose}) is not a {self._provider} source record", + ) + config = self._config() + try: + await self._db.insert_channel_observation( + channel=self._provider, + channel_key=f"{self._provider}:{event.conversation.id}", + payload=buffered_record(event), + ttl_days=config.sync.message_ttl_days, + max_rows=BUFFER_MAX_ROWS, + ) + except Exception as error: # noqa: BLE001 - the row comes back on a later read + logger.warning( + "Could not buffer the %s record %s: %s", self._provider, event.record.id, error, + ) + return Disposition.deferred("the source record could not be stored") + self._notify() + return Disposition.accepted() + + def _notify(self) -> None: + listener = self._listener + if listener is None: + return + try: + outcome = listener(self._provider) + except Exception: # the record is stored; the runner's schedule still drains it + logger.exception("The %s delivery listener failed", self._provider) + return + if inspect.isawaitable(outcome): + task = asyncio.ensure_future(outcome) + self._tasks.add(task) + task.add_done_callback(self._finished) + + def _finished(self, task: asyncio.Future) -> None: + self._tasks.discard(task) + if not task.cancelled() and task.exception() is not None: + logger.error( + "The %s delivery listener failed: %s", self._provider, task.exception(), + exc_info=task.exception(), + ) + + async def stop(self) -> None: + """Cancel the listener calls still running.""" + tasks, self._tasks = list(self._tasks), set() + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +__all__ = [ + "BUFFER_MAX_ROWS", + "GITHUB", + "HostedSourceConsumer", + "buffered_record", + "github_subscription", + "hosted_sources", + "subscriptions_for", +] diff --git a/nerve/channels/hosted/stream.py b/nerve/channels/hosted/stream.py index 5b9e3a3ae..4f99375c4 100644 --- a/nerve/channels/hosted/stream.py +++ b/nerve/channels/hosted/stream.py @@ -22,6 +22,14 @@ stops when the result arrives first. Nerve starts an upload only while its size, together with the uploads already in flight, fits the gateway's ``memory_bytes``. + +When the agent has hosted sources, the stream sends its ``subscriptions`` +frame as soon as negotiation completes, and again whenever the runtime +publishes changed settings. The first frame waits for the stream for as long +as it is open; a republish gives up after a short bound, so a stalled stream +does not hold up a reload. A subscription that the gateway's bounds or frame +limit would refuse goes without its filter patterns, so it never closes the +stream; Nerve's own inbox filter still applies to every record. """ from __future__ import annotations @@ -49,6 +57,7 @@ Rejected, RejectionReason, StreamLimits, + Subscriptions, decode_envelope, encode_envelope, validate_inbox_read_result, @@ -71,6 +80,10 @@ _BEFORE_NEGOTIATION = frozenset({"negotiation", "heartbeat", "drain"}) # A drain is a courtesy; a peer that does not take it quickly is closed anyway. _DRAIN_SEND_SECONDS = 2.0 +# How long a republished subscriptions frame may take to go out, bounded as +# the drain is: a stream that is stalled must not hold up a reload. The frame +# sent after negotiation has no such bound. +_SUBSCRIPTIONS_SEND_SECONDS = 2.0 # Room left in an upload chunk frame for a longer offset or final marker. _CHUNK_FRAME_MARGIN = 32 # How long a cancelled sender still lets its frame finish. @@ -169,7 +182,9 @@ class ChannelStream: ``receive_limits`` is what Nerve accepts on this stream. Nerve advertises the same limits on every stream, so a row that one stream cannot carry - cannot be carried by another either. + cannot be carried by another either. ``subscriptions`` returns the + agent's current source subscriptions, or ``None`` when it has no hosted + sources and sends no ``subscriptions`` frame. """ def __init__( @@ -181,6 +196,7 @@ def __init__( listener: StreamListener, timing: StreamTiming = StreamTiming(), clock: Callable[[], float] = time.monotonic, + subscriptions: Callable[[], Subscriptions | None] | None = None, ) -> None: self.id = stream_id self._websocket = websocket @@ -188,6 +204,8 @@ def __init__( self._listener = listener self._timing = timing self._clock = clock + self._subscriptions = subscriptions + self._publish_task: asyncio.Task | None = None self.opened_at = clock() self.state = "negotiating" # The gateway sent drain: new requests go to another stream. @@ -304,11 +322,14 @@ async def run(self) -> None: except StreamClosed: pass finally: - watchdog.cancel() - try: - await watchdog - except (asyncio.CancelledError, Exception): - pass + for task in (watchdog, self._publish_task): + if task is None: + continue + task.cancel() + try: + await task + except (asyncio.CancelledError, Exception): + pass self._mark_closed() async def close(self, code: int = CLOSE_NORMAL, reason: str = "") -> None: @@ -407,6 +428,17 @@ def _handle(self, envelope: Envelope, frame_bytes: int) -> None: self.peer_limits = body.receive_limits self.state = "ready" logger.info("Channel stream %d negotiated version %s", self.id, PROTOCOL_VERSION) + if self._subscriptions is not None: + # Started before the reader hears of the stream, so the + # subscriptions are the first frame after negotiation. Nothing + # waits for this publish and nothing repeats it before a + # reload, so it is not bounded: a stall delays the frame and + # never drops it. The stream's end cancels it. + self._publish_task = asyncio.get_running_loop().create_task( + self.publish_subscriptions(bounded=False), + name=f"channel-stream-{self.id}-subscriptions", + ) + self._publish_task.add_done_callback(self._publish_done) self._listener.stream_ready(self) elif kind == "heartbeat": pass @@ -557,13 +589,115 @@ async def _send_locked(self, text: str) -> None: self._mark_closed() raise StreamClosed() from error + def _one_way_id(self) -> str: + """The request ID of the next one-way frame, taken in call order.""" + self._next_one_way += 1 + return f"o-{self._next_one_way}" + async def _send_one_way( self, kind: str, body: Any, connection_id: uuid.UUID | None = None, ) -> None: - self._next_one_way += 1 - text = self._encode(kind, body, connection_id, f"o-{self._next_one_way}") + text = self._encode(kind, body, connection_id, self._one_way_id()) await self._send_text(text) + async def publish_subscriptions(self, *, bounded: bool = True) -> bool: + """Send the agent's current source subscriptions; ``True`` when sent. + + Frames leave in call order, and each one is built from the settings + current when the stream is free to send it, so the last frame to leave + carries the newest settings even after a wait. Subscriptions that the + gateway would refuse, by its bounds or its frame limit, go without + their filter patterns: the gateway then keeps every record, and Nerve's + inbox filter still drops what the patterns would have. Subscriptions + that it would refuse even so are not sent, since the gateway would + close the stream over them. A stream that is not ready, or that Nerve + is draining, sends nothing. + + A *bounded* call that cannot send the frame within + ``_SUBSCRIPTIONS_SEND_SECONDS`` gives up, as the drain does, so that a + stalled stream does not hold up the reload that waits for it. An + unbounded call waits for as long as the stream is open. + """ + if self._subscriptions is None or not self.ready or self.drain_sent: + return False + request_id = self._one_way_id() + bound = _SUBSCRIPTIONS_SEND_SECONDS if bounded else None + try: + async with asyncio.timeout(bound): + async with self._send_lock: + if not self.ready or self.drain_sent: + return False + text = self._subscriptions_frame(request_id) + if text is None: + return False + await self._send_locked(text) + except StreamClosed: + return False + except TimeoutError: + logger.warning( + "Channel stream %d: source subscriptions not sent: the stream could not send " + "them within %gs", self.id, bound, + ) + return False + return True + + def _subscriptions_frame(self, request_id: str) -> str | None: + """The ``subscriptions`` frame from the current settings, encoded. + + ``None`` when there is nothing to send: no hosted sources, settings + that cannot be read, or subscriptions that the gateway would refuse + even without their filter patterns. + """ + try: + body = self._subscriptions() + except Exception: # a bad setting must not close the stream + logger.exception("Channel stream %d: could not build the source subscriptions", self.id) + return None + if body is None: + return None + try: + body.validate() + except Rejected as error: + wide = body.without_filters() + try: + wide.validate() + except Rejected as still: + logger.warning( + "Channel stream %d: source subscriptions not sent: the gateway would " + "refuse them (%s)", self.id, still.detail, + ) + return None + logger.warning( + "Channel stream %d: source subscriptions exceed the gateway's bounds (%s); " + "sending them without filter patterns", self.id, error.detail, + ) + body = wide + try: + try: + return self._encode("subscriptions", body, None, request_id) + except Rejected as error: + if error.reason != RejectionReason.FRAME_TOO_LARGE: + raise + logger.warning( + "Channel stream %d: source subscriptions exceed the gateway's frame " + "limit; sending them without filter patterns", self.id, + ) + return self._encode("subscriptions", body.without_filters(), None, request_id) + except Rejected as error: + logger.warning("Channel stream %d: source subscriptions not sent: %s", self.id, error) + return None + + def _publish_done(self, task: asyncio.Task) -> None: + """Log a failure of the publish that negotiation started, when it happens.""" + if task.cancelled(): + return + error = task.exception() + if error is not None: + logger.error( + "Channel stream %d: publishing the source subscriptions failed: %r", + self.id, error, exc_info=error, + ) + def _check_room( self, kind: str, body: Any, connection_id: uuid.UUID | None, upload: bytes | None, ) -> None: diff --git a/nerve/cli.py b/nerve/cli.py index de65652c2..1d7ef2685 100644 --- a/nerve/cli.py +++ b/nerve/cli.py @@ -1535,6 +1535,49 @@ def _hosted_slack_lines(config, warnings: list[str]) -> list[str]: return lines +def _hosted_github_lines(config, errors: list[str], warnings: list[str]) -> list[str]: + """Doctor lines for ``sync.github.mode: hosted``. + + The gateway polls GitHub, so there is no gh login to check; the hosted + stream settings are, as for hosted Slack, also when the source is off. + Nerve reads the gateway's events for every hosted provider in one + ordered inbox and stops reading while hosted Slack is switched off, so + a switched-off hosted Slack holds the GitHub records back too. + """ + github = config.sync.github + if github.mode != "hosted": + return [] + from nerve.channels.hosted.auth import KeyFileError, read_key_file + + lines: list[str] = [] + if not github.enabled: + lines.append( + "[--] GitHub is hosted but switched off (sync.github.enabled is false); " + "the gateway is told that this agent subscribes to nothing" + ) + hosted = config.channels.hosted + problems = hosted.problems() + if not problems: + try: + read_key_file(hosted.gateway_jwks_file) + except KeyFileError as error: + problems = [f"channels.hosted.gateway_jwks_file: {error}"] + if problems: + errors.extend(f"[ERR] GitHub is hosted but {problem}" for problem in problems) + elif github.enabled: + lines.append( + "[OK] GitHub notifications come from the channel gateway: filters " + f"published from sync.github, PR watch {'on' if github.pr_watch else 'off'}" + ) + if github.enabled and config.slack.mode == "hosted" and not config.slack.enabled: + warnings.append( + "[WARN] Slack is hosted but switched off (slack.enabled is false): Nerve " + "reads nothing from the channel gateway until it is switched on, so the " + "hosted GitHub records wait there too" + ) + return lines + + def _source_suffix(config, path: str) -> str: """`` (environment)`` / `` (config-repo)`` after a configured connection line when the value did not come from a file on this machine; ``""`` for @@ -1997,6 +2040,7 @@ def doctor_report(config, config_source: str = "", check_api: bool = False) -> s ) else: lines.append("[--] Slack disabled") + lines.extend(_hosted_github_lines(config, errors, warnings)) # Check SSL if config.gateway.ssl.enabled: @@ -2193,6 +2237,19 @@ def doctor_report(config, config_source: str = "", check_api: bool = False) -> s all_jobs = list(merged.values()) enabled = sum(1 for j in all_jobs if j.enabled) + if config.sync.github.mode == "hosted": + pr_gated = sorted( + j.id for j in all_jobs + if j.enabled and any(g.type == "github_pr_activity" for g in j.gates) + ) + if pr_gated: + warnings.append( + f"[WARN] Cron job(s) {', '.join(pr_gated)} use the github_pr_activity " + "gate, which always passes while GitHub is hosted — gate them on " + "messages with sources: [github]; the gateway delivers PR changes as " + "records (sync.github.pr_watch)" + ) + if all_jobs: detail = f"{enabled}/{len(all_jobs)} enabled" if overridden: @@ -2254,9 +2311,13 @@ def doctor_report(config, config_source: str = "", check_api: bool = False) -> s # Check external tools import shutil + + from nerve.setup_checks import gh_needed for tool_name in ["gog", "gh"]: if shutil.which(tool_name): lines.append(f"[OK] {tool_name} CLI found") + elif tool_name == "gh" and not gh_needed(config): + lines.append("[--] gh CLI not found (not needed: GitHub comes from the channel gateway)") else: warnings.append(f"[WARN] {tool_name} CLI not found (needed for sync)") diff --git a/nerve/config.py b/nerve/config.py index 204227f4e..c9d1564b3 100644 --- a/nerve/config.py +++ b/nerve/config.py @@ -1909,6 +1909,11 @@ def from_dict(cls, d: dict) -> ImapSyncConfig: ) +# How GitHub notifications reach the inbox: this process polls GitHub with the +# gh CLI, or a shared channel gateway polls it and streams the records here. +GITHUB_SYNC_MODES: tuple[str, ...] = ("poll", "hosted") + + @dataclass class GitHubSyncConfig: enabled: bool = True @@ -1958,11 +1963,29 @@ class GitHubSyncConfig: # message was dropped. Best effort; empty = off. See # SourceRunner._run_drop_hook. drop_hook: list[str] = field(default_factory=list) + # "poll": this process fetches notifications with the gh CLI. + # "hosted": a shared channel gateway polls GitHub, applies the guardrails + # above before it stores anything, and streams the records over the + # hosted channel streams (see HostedChannelsConfig). Restart-only. + mode: str = "poll" + # Hosted mode only: ask the gateway to also watch the agent's own open + # pull requests and deliver a record when one changes (state, head + # commit, review, CI, or a comment). + pr_watch: bool = True @classmethod @_coerced def from_dict(cls, d: dict) -> GitHubSyncConfig: + mode = str(d.get("mode") or "poll").strip().lower() + if mode not in GITHUB_SYNC_MODES: + logger.warning( + "sync.github.mode %r is not one of %s; falling back to 'poll'", + mode, GITHUB_SYNC_MODES, + ) + mode = "poll" return cls( + mode=mode, + pr_watch=d.get("pr_watch", True), enabled=d.get("enabled", True), schedule=d.get("schedule", "*/15 * * * *"), processor=d.get("processor", "agent"), @@ -2282,6 +2305,23 @@ def _locked_cron_path(configured: Path, default: Path, root: Path, key: str) -> ) +DEFAULT_DELIVERY_DEBOUNCE_SECONDS = 60 +# The longest wait the cron service applies; a larger value is used as this, +# and `nerve config validate` says so. +MAX_DELIVERY_DEBOUNCE_SECONDS = 600 + + +def _delivery_debounce(value: object) -> int: + """Coerce ``cron.delivery_debounce_seconds``, holding it at or above 0.""" + seconds = _lenient_int( + value, DEFAULT_DELIVERY_DEBOUNCE_SECONDS, label="cron.delivery_debounce_seconds", + ) + if seconds >= 0: + return seconds + logger.warning("cron.delivery_debounce_seconds is %r, below 0; using 0", value) + return 0 + + @dataclass class CronConfig: # Bare defaults (no workspace context) point at the legacy machine-local @@ -2291,6 +2331,11 @@ class CronConfig: # Directory scanned at startup for drop-in custom gate plugins (.py files # defining CronGate subclasses). See nerve/cron/gate_plugins.py. gate_plugins_dir: Path = field(default_factory=lambda: paths.cron_dir() / "gates") + # When a hosted source delivers records, the jobs gated on that source run + # this many seconds after the first delivery of a burst, so a burst is one + # run. 0 runs them as soon as the records are in the inbox; at most + # MAX_DELIVERY_DEBOUNCE_SECONDS applies. Read per delivery. + delivery_debounce_seconds: int = DEFAULT_DELIVERY_DEBOUNCE_SECONDS @classmethod @_coerced @@ -2321,6 +2366,7 @@ def from_dict(cls, d: dict, workspace: Path | None = None, locked: bool = False) jobs_file=jobs_file, system_file=system_file, gate_plugins_dir=gate_plugins_dir, + delivery_debounce_seconds=_delivery_debounce(d.get("delivery_debounce_seconds")), ) @@ -4436,6 +4482,11 @@ def get_config() -> NerveConfig: return _config +def loaded_config() -> NerveConfig | None: + """The global config if one is loaded, without loading it.""" + return _config + + def set_config(config: NerveConfig) -> None: """Override the global config (for testing or CLI-driven loading).""" global _config diff --git a/nerve/config_reload.py b/nerve/config_reload.py index 238460d4d..0cf58a0a0 100644 --- a/nerve/config_reload.py +++ b/nerve/config_reload.py @@ -65,6 +65,7 @@ "proxy", "slack.mode", "sync.codex", + "sync.github.mode", "telegram.allowed_users", "telegram.bot_token", "telegram.enabled", @@ -354,6 +355,25 @@ def _reload_hosted_keys(engine) -> str | None: return f"{_ERROR_PREFIX}{detail}" +async def _republish_hosted_subscriptions(engine) -> str | None: + """Send the hosted source subscriptions to the gateway again. + + The filters and ``pr_watch`` follow a reload this way; the gateway keeps + the newest subscription it received. ``None`` without hosted sources. + """ + if engine is None: + return None + try: + runtime = engine.get_channel_runtime("hosted") + if runtime is None: + return None + return await runtime.republish_subscriptions() + except Exception as e: # noqa: BLE001 - report the subsystem and continue + detail = str(e) or type(e).__name__ + logger.warning("Hosted source subscriptions were not published: %s", detail) + return f"{_ERROR_PREFIX}{detail}" + + async def reload_all(engine, cron_service, config_dir: Path) -> dict: """Re-read config and hot-reload all reloadable subsystems. @@ -414,6 +434,9 @@ async def reload_all(engine, cron_service, config_dir: Path) -> dict: summary["sources"] = await cron_service.reload_sources() except Exception as e: # noqa: BLE001 summary["sources"] = f"{_ERROR_PREFIX}{e}" + subscriptions = await _republish_hosted_subscriptions(engine) + if subscriptions is not None: + summary["subscriptions"] = subscriptions # 3. MCP servers. if engine is not None: diff --git a/nerve/config_validate.py b/nerve/config_validate.py index 5799cf534..c799711d0 100644 --- a/nerve/config_validate.py +++ b/nerve/config_validate.py @@ -328,6 +328,7 @@ def validate_config_bundle( # Source runners are scheduled from the same parser as cron jobs, so a # typo'd sync schedule fails identically — and needs the same gate. _validate_source_schedules(config, result) + _validate_delivery_debounce(config, result) # Files of the bundle that were actually opened, so "it validated" can be # told from "it found nothing to validate". read = [ @@ -871,6 +872,17 @@ def _validate_source_schedules(config, result: ValidationResult) -> None: result.errors.append(f"sync.{f.name}.schedule: {problem}") +def _validate_delivery_debounce(config, result: ValidationResult) -> None: + """``cron.delivery_debounce_seconds`` above its maximum waits the maximum.""" + seconds = config.cron.delivery_debounce_seconds + if isinstance(seconds, int) and seconds > cfg.MAX_DELIVERY_DEBOUNCE_SECONDS: + result.warnings.append( + f"cron.delivery_debounce_seconds: {seconds} is above the maximum of " + f"{cfg.MAX_DELIVERY_DEBOUNCE_SECONDS}, so the jobs a hosted delivery " + f"starts wait {cfg.MAX_DELIVERY_DEBOUNCE_SECONDS}s" + ) + + def _job_gate_specs(job, where: str, result: ValidationResult) -> list: """Every gate spec *job* will build, mirroring ``CronJob._build_gates``. diff --git a/nerve/cron/gates.py b/nerve/cron/gates.py index 6b9524455..327fae5c7 100644 --- a/nerve/cron/gates.py +++ b/nerve/cron/gates.py @@ -153,7 +153,14 @@ class MessagesGate(CronGate): empty), the gate fires when *any* source the consumer already tracks has unread messages — so a shared/default inbox job can be gated without hard-coding the particular set of connected sources. Either way the check - only reads cursor state; it never advances a cursor. + never advances a cursor. + + The check does keep the cursors it reads from expiring: it renews a live + cursor's ``sync.consumer_cursor_ttl_days``. A cursor expires when nothing + reads it for that long, and an expired one starts again at the source's + newest message, so without this a source that stayed quiet for a few days + would lose its first new message to the restart, here and in the job's + own poll. The job reads the cursors on every check, so they stay live. """ type = "messages" @@ -169,6 +176,11 @@ def __init__(self, sources: list[str] | None = None, consumer: str = "inbox") -> self.consumer = consumer async def is_satisfied(self, ctx: GateContext) -> bool: + satisfied = await self._has_unread(ctx) + await self._keep_cursors_alive(ctx) + return satisfied + + async def _has_unread(self, ctx: GateContext) -> bool: if self.sources: for source in self.sources: cursor_seq = await ctx.db.get_consumer_cursor(self.consumer, source) @@ -177,20 +189,35 @@ async def is_satisfied(self, ctx: GateContext) -> bool: return True return False # No explicit sources: fire if any source the consumer tracks has - # unread messages. Read-only and expiry-agnostic — it never - # initializes or advances a cursor (unlike get_consumer_cursor) and - # still sees a backlog after a quiet inbox's cursors pass their TTL. + # unread messages. Expiry-agnostic — it never initializes or advances + # a cursor (unlike get_consumer_cursor) and still sees a backlog after + # a quiet inbox's cursors pass their TTL. return await ctx.db.consumer_has_unread(self.consumer) + async def _keep_cursors_alive(self, ctx: GateContext) -> None: + """Renew the TTL of the live cursors this gate reads (see the class). + + Best effort: a failure is logged and never changes the answer. + """ + try: + await ctx.db.keep_consumer_cursors_alive( + self.consumer, self.sources or None, ttl_days=_consumer_cursor_ttl_days(), + ) + except Exception as e: # noqa: BLE001 — the gate's answer stands + logger.warning( + "messages gate: could not renew the cursors of consumer %s: %s", self.consumer, e, + ) + async def forecast(self, ctx: GateContext, at: datetime) -> bool: - """:meth:`is_satisfied` at *at*, without its one write. + """:meth:`is_satisfied` at *at*, without its writes. With explicit ``sources``, the check starts a missing or expired cursor at the source's newest message, so that source has nothing unread. The forecast reads the cursor instead and gives the same answer for it — as the cursor will be at *at*: one that is live now but expires before - then (``sync.consumer_cursor_ttl_days`` after it last moved) counts as - expired. + then (``sync.consumer_cursor_ttl_days`` after it last moved or was + renewed) counts as expired, since the check reads a cursor before it + renews it. The forecast renews no cursor. """ if not self.sources: return await ctx.db.consumer_has_unread(self.consumer) @@ -362,6 +389,12 @@ class GitHubPrActivityGate(CronGate): force_run_after_hours: 8 ignore_actors: [codecov, sonarqubecloud] + With ``sync.github.mode: hosted`` the gate is satisfied without running + ``gh``: the gateway watches the agent's open PRs (``sync.github.pr_watch``) + and delivers a ``github_pr_activity`` record for every change, so a job + that should wake on PR activity is gated on ``messages`` with + ``sources: [github]`` instead. + Config: * ``author`` (required) — GitHub login whose open PRs to watch. Also @@ -415,6 +448,10 @@ def _norm_actor(login: str) -> str: return s[:-5] if s.endswith("[bot]") else s async def is_satisfied(self, ctx: GateContext) -> bool: + if github_is_hosted(): + # No gh here: the gateway's PR watch delivers the changes as + # records, which a messages gate sees. + return True fp = await self._fingerprint() if fp is None: # gh failed entirely — fail open (run) rather than risk stranding a PR. @@ -613,6 +650,36 @@ def _save_state(self, job_id: str, fingerprint: str, now: datetime) -> None: logger.warning("github_pr_activity: could not persist state: %s", e) +def _consumer_cursor_ttl_days() -> int: + """``sync.consumer_cursor_ttl_days`` of the loaded config, never loading + one; its default without a config or with an unusable value.""" + from nerve.config import SyncConfig, loaded_config + + config = loaded_config() + days = getattr(getattr(config, "sync", None), "consumer_cursor_ttl_days", None) + if isinstance(days, int) and not isinstance(days, bool) and days > 0: + return days + return SyncConfig.consumer_cursor_ttl_days + + +def github_is_hosted() -> bool: + """Whether GitHub records come from the channel gateway (``sync.github.mode: hosted``). + + The mode is restart-only, so this reads the configuration the process + started with, as the GitHub source runner and the hosted streams do: + after a reload that changes the mode, the gate keeps to the mode in + effect until the restart. A process that did not start the gateway reads + the loaded config and never loads one, so one without a config (a test, + a one-off command) counts as polling. + """ + from nerve.config import loaded_config + from nerve.config_reload import startup_config + + config = startup_config() or loaded_config() + github = getattr(getattr(config, "sync", None), "github", None) + return getattr(github, "mode", None) == "hosted" + + #: Maps a gate's ``type`` key to its implementing class. Register new gate #: types here to make them usable from config. GATE_REGISTRY: dict[str, type[CronGate]] = { diff --git a/nerve/cron/service.py b/nerve/cron/service.py index efaf75842..0abfbbf66 100644 --- a/nerve/cron/service.py +++ b/nerve/cron/service.py @@ -1,12 +1,15 @@ """Cron scheduler — APScheduler integration. -Runs cron jobs and source runners on schedule. +Runs cron jobs and source runners on schedule, and, when a hosted source +delivers records, its runner and the jobs gated on it right away. """ from __future__ import annotations import asyncio import contextlib +import copy +import functools import inspect import json import logging @@ -80,8 +83,9 @@ # How a run of a cron job or source runner was started: by its schedule, as a # catch-up for fires missed while the daemon was down or the machine paused, -# or by hand (the Cron page's "Run now", the trigger API). -RunOrigin = Literal["schedule", "catchup", "manual"] +# by hand (the Cron page's "Run now", the trigger API), or for source records +# that a hosted source delivered (see CronService.on_source_delivered). +RunOrigin = Literal["schedule", "catchup", "manual", "delivery"] class _RunsInFlight: @@ -89,13 +93,14 @@ class _RunsInFlight: The scheduler starts a job at most once at a time, but it only counts the runs it started itself. The run wrappers record every run here, so that - whatever starts a run can first see what is already going, and whether a - run has started since it last looked. + whatever starts a run can first see what is already going, whether a run + has started since it last looked, or wait until no run of a job is left. """ def __init__(self) -> None: self._origins: dict[str, list[RunOrigin]] = {} self._started: dict[str, int] = {} + self._idle: dict[str, asyncio.Event] = {} def running(self, job_id: str, origin: RunOrigin | None = None) -> bool: """Whether a run of *job_id* is in progress — one started as *origin*, @@ -121,6 +126,17 @@ def track(self, job_id: str, origin: RunOrigin) -> Iterator[None]: origins.remove(origin) if not origins: del self._origins[job_id] + idle = self._idle.pop(job_id, None) + if idle is not None: + idle.set() + + async def wait_until_idle(self, job_id: str) -> None: + """Return once no run of *job_id* is in progress; at once when none is.""" + while self.running(job_id): + idle = self._idle.get(job_id) + if idle is None: + idle = self._idle[job_id] = asyncio.Event() + await idle.wait() # How far ahead CronService.next_scheduled() looks by default. @@ -452,7 +468,20 @@ def __init__(self, config: NerveConfig, engine: AgentEngine, db: Database): # add_schedule_listener), and the async ones still running. self._schedule_listeners: list[Callable[[], object]] = [] self._listener_tasks: set[asyncio.Task] = set() - # Set when stop() begins; from then on nothing new is started. + # sync.github.mode is restart-only: every source reload builds the + # GitHub source for the mode the service started with. + github_mode = getattr(getattr(getattr(config, "sync", None), "github", None), "mode", None) + self._github_mode = github_mode if isinstance(github_mode, str) else None + # Hosted source deliveries (see on_source_delivered): the sources whose + # drain is running, those delivered to again meanwhile, the pending + # delivery run of each job, and the delivery runs that have started. + # The runs themselves count in _runs like any other. + self._draining: set[str] = set() + self._drain_again: set[str] = set() + self._delivery_tasks: dict[str, asyncio.Task] = {} + self._delivery_started: set[asyncio.Task] = set() + # Set when stop() begins; from then on nothing new is started, by the + # scheduler, a catch-up or a source delivery. self._stopping = False def _own_attribution(self) -> Attribution: @@ -899,13 +928,33 @@ def _schedule_source_runners(self, runners: list) -> None: planned.append((runner, trigger, schedule_str)) self._apply_source_runners(planned) + def _source_config(self) -> NerveConfig: + """The config to build source runners from. + + The live config, except that ``sync.github.mode`` keeps the value the + service started with: the mode is restart-only, so a reload must not + swap the GitHub source under a running hosted runtime (or bring one up + that does not exist). Shallow copies, so nothing else is frozen. + """ + config = self.config + github = getattr(getattr(config, "sync", None), "github", None) + if self._github_mode is None or github is None or github.mode == self._github_mode: + return config + pinned_github = copy.copy(github) + pinned_github.mode = self._github_mode + pinned_sync = copy.copy(config.sync) + pinned_sync.github = pinned_github + pinned = copy.copy(config) + pinned.sync = pinned_sync + return pinned + def _register_source_runners(self) -> None: """Build + schedule source runners at startup. Swallows errors so a bad source config never crashes the daemon boot.""" try: from nerve.sources.registry import build_source_runners - self._source_runners = build_source_runners(self.config, self.db) + self._source_runners = build_source_runners(self._source_config(), self.db) self._schedule_source_runners(self._source_runners) except Exception as e: # noqa: BLE001 — boot must not fail on a bad source logger.warning("Failed to register source runners: %s", e, exc_info=True) @@ -939,7 +988,7 @@ async def reload_sources(self) -> dict: async with self._reload_lock: # -- Plan: everything that can fail, before anything is applied ---- - new_runners = build_source_runners(self.config, self.db) # may raise → caller reports + new_runners = build_source_runners(self._source_config(), self.db) # may raise → caller reports planned = self._plan_source_runners(new_runners) # -- Apply: scheduler bookkeeping only ----------------------------- @@ -958,14 +1007,16 @@ async def reload_sources(self) -> dict: async def stop(self) -> None: """Stop the clock watcher, the catch-ups it started, the schedule - listeners still running, and the scheduler. + listeners still running, the runs for delivered source records, and + the scheduler. Nothing new starts once this begins: the scheduler is paused first, and - the clock check, the catch-ups and the schedule listeners do nothing - more. The watcher, the catch-ups and the listener tasks are cancelled - and given ``_STOP_WAIT_SECONDS`` to wind down; then the scheduler shuts - down, which cancels the runs it has in flight. Stopping again does - nothing. + the clock check, the catch-ups, the schedule listeners and the source + deliveries do nothing more. The watcher, the catch-ups, the listener + tasks and the runs for delivered source records, pending or already + going, are cancelled and given ``_STOP_WAIT_SECONDS`` to wind down; + then the scheduler shuts down, which cancels the runs it has in + flight. Stopping again does nothing. """ if self._stopping: return @@ -974,7 +1025,10 @@ async def stop(self) -> None: self.scheduler.pause() watcher, self._clock_watch_task = self._clock_watch_task, None pending = { - t for t in (watcher, *self._catchup_tasks, *self._listener_tasks) + t for t in ( + watcher, *self._catchup_tasks, *self._listener_tasks, + *self._delivery_tasks.values(), *self._delivery_started, + ) if t is not None } for task in pending: @@ -1597,7 +1651,7 @@ async def _run_job_wrapper( change what runs next. """ try: - if not self._may_start(job.id, origin): + if not self._may_start(job.id, origin, lock=job.lock): return with self._runs.track(job.id, origin): if job.lock: @@ -1609,15 +1663,20 @@ async def _run_job_wrapper( finally: self._schedule_changed() - def _may_start(self, job_id: str, origin: RunOrigin) -> bool: + def _may_start(self, job_id: str, origin: RunOrigin, *, lock: bool = False) -> bool: """Whether a run of *job_id* started as *origin* may start now. The scheduler never starts a job while its previous scheduled run is still going; it skips that fire. A catch-up stands in for a scheduled fire, so the same rule holds between the two: a catch-up is skipped while any run of the job is going, and a scheduled fire while a - catch-up of the job is going. Each skip is logged. A manual run always - starts (with ``lock`` it waits for the run before it). + catch-up of the job is going. A run for delivered source records goes + on in place of the job's scheduled run too, so a scheduled fire is + skipped while one is going, except for a job with *lock*, whose fire + waits for it on the lock instead. Each skip is logged. A manual run + always starts (with ``lock`` it waits for the run before it), and so + does a run for delivered records, which waits for the job's runs in + flight before it starts. """ if origin == "catchup" and self._runs.running(job_id): logger.info("Not catching up %s: a run of it is still going", job_id) @@ -1628,6 +1687,13 @@ def _may_start(self, job_id: str, origin: RunOrigin) -> bool: job_id, ) return False + if origin == "schedule" and not lock and self._runs.running(job_id, "delivery"): + logger.info( + "Skipping the scheduled run of %s: a run for newly delivered source " + "records is still going", + job_id, + ) + return False return True async def _run_job_inner( @@ -2013,6 +2079,121 @@ async def run_job( await self._run_job_wrapper(job, attribution, origin="manual") + # -- Hosted source deliveries ------------------------------------------------ + + async def on_source_delivered(self, source: str) -> None: + """Take a hosted source's newly delivered records into the inbox now, + then run the jobs gated on that source. + + Runs the ``source:`` runner at once, after any run of it that + is in progress (that run may have fetched before the records arrived). + A run that takes the runner first (a scheduled fire, a manual sync) + leaves this one nothing to do; it started after the records arrived, + so it is waited for instead. Deliveries during the drain make it run + once more rather than start a second drain. Then every enabled, + scheduled job whose gates include a ``messages`` gate naming + *source*, or naming no source, gets one run + ``cron.delivery_debounce_seconds`` after the first delivery of a + burst: later deliveries in that window add nothing. That run waits + for a run of the job already in flight, and goes through the gates, + ``lock`` and the run log like any other; while it goes on, the job's + runs are skipped or wait as :meth:`_may_start` says. The drain counts + as a run of the source runner in the same way. Jobs without such a + gate are left to their schedule. A source without a runner (switched + off) runs nothing, and once :meth:`stop` has begun nothing starts. + + Never takes the reload lock: it reads the job and runner lists, which a + reload replaces in one step. + """ + if self._stopping or not self.scheduler.running: + return + if source in self._draining: + self._drain_again.add(source) + return + self._draining.add(source) + try: + while not self._stopping: + self._drain_again.discard(source) + runner = next( + (r for r in self._source_runners if r.source.source_name == source), None, + ) + if runner is None: + return + await runner.wait_until_idle() + await self._run_source_wrapper(runner, origin="delivery") + # The wrapper awaits before the run starts, so another run can + # take the runner in between; this run then returns at once, + # and the records are in only when that other run ends. + await runner.wait_until_idle() + if source not in self._drain_again: + break + finally: + self._draining.discard(source) + self._schedule_delivery_runs(source) + + def _delivery_debounce(self) -> float: + """``cron.delivery_debounce_seconds``, within 0 and its maximum.""" + from nerve.config import ( + DEFAULT_DELIVERY_DEBOUNCE_SECONDS, + MAX_DELIVERY_DEBOUNCE_SECONDS, + ) + + seconds = getattr(getattr(self.config, "cron", None), "delivery_debounce_seconds", None) + if not isinstance(seconds, int) or isinstance(seconds, bool): + seconds = DEFAULT_DELIVERY_DEBOUNCE_SECONDS + return float(min(max(seconds, 0), MAX_DELIVERY_DEBOUNCE_SECONDS)) + + def _schedule_delivery_runs(self, source: str) -> None: + """Give each job gated on *source* one pending delivery run.""" + from nerve.cron.gates import MessagesGate + + if self._stopping: + return + delay = self._delivery_debounce() + for job in self._jobs: + if not job.enabled or job.id in self._delivery_tasks: + continue + if self.scheduler.get_job(job.id) is None: + continue + if not any( + isinstance(gate, MessagesGate) and (not gate.sources or source in gate.sources) + for gate in job.gates + ): + continue + task = asyncio.get_running_loop().create_task( + self._run_after_delivery(job.id, delay), name=f"cron-delivery-{job.id}", + ) + self._delivery_tasks[job.id] = task + task.add_done_callback(functools.partial(self._delivery_run_done, job.id)) + + async def _run_after_delivery(self, job_id: str, delay: float) -> None: + """The pending delivery run of *job_id*: wait out the window, then run.""" + if delay > 0: + await asyncio.sleep(delay) + await self._runs.wait_until_idle(job_id) + # From here on a new delivery schedules a run of its own: this one may + # already have looked at the inbox by the time those records land. + # It moves to the started runs in the same step, so stop() finds it in + # one place or the other. + task = asyncio.current_task() + if self._delivery_tasks.get(job_id) is task: + del self._delivery_tasks[job_id] + self._delivery_started.add(task) + job = next((j for j in self._jobs if j.id == job_id), None) + if job is None or not job.enabled or self.scheduler.get_job(job_id) is None: + return + if self._stopping: + return + logger.info("Running cron job %s for newly delivered source records", job_id) + await self._run_job_wrapper(job, origin="delivery") + + def _delivery_run_done(self, job_id: str, task: asyncio.Task) -> None: + if self._delivery_tasks.get(job_id) is task: + del self._delivery_tasks[job_id] + self._delivery_started.discard(task) + if not task.cancelled() and (error := task.exception()) is not None: + logger.error("Delivery run of cron job %s failed: %r", job_id, error) + async def rotate_session(self, job_id: str) -> dict: """Force-rotate a persistent cron to a fresh chat session. diff --git a/nerve/db/sources.py b/nerve/db/sources.py index 038decc30..66cdaab95 100644 --- a/nerve/db/sources.py +++ b/nerve/db/sources.py @@ -399,6 +399,38 @@ async def set_consumer_cursor( (consumer, source, cursor_seq, session_id, now.isoformat(), expires), ) + async def keep_consumer_cursors_alive( + self, consumer: str, sources: list[str] | None = None, ttl_days: int = 2, + ) -> int: + """Renew the TTL of a consumer's live cursors, leaving their positions. + + For a reader that still uses the cursors without moving them, such as + a ``messages`` gate checked on every run of its job: a quiet source + then keeps its place instead of starting again at its newest message. + Only a live cursor with less than half of *ttl_days* left is renewed, + to *ttl_days* from now, so a frequent caller seldom writes; an expired + cursor stays expired. *sources* limits it to those sources, ``None`` + means all of the consumer's. Returns the number of cursors renewed. + """ + if sources is not None and not sources: + return 0 + now = datetime.now(UTC) + renew_before = (now + timedelta(days=ttl_days / 2)).isoformat() + where = "consumer = ? AND expires_at IS NOT NULL AND expires_at > ? AND expires_at < ?" + params: list = [consumer, now.isoformat(), renew_before] + if sources is not None: + where += f" AND source IN ({', '.join('?' for _ in sources)})" + params.extend(sources) + async with self.db.execute(f"SELECT COUNT(*) FROM consumer_cursors WHERE {where}", params) as cursor: + row = await cursor.fetchone() + if not row or not row[0]: + return 0 # nothing due, so no write + expires = (now + timedelta(days=ttl_days)).isoformat() + result = await self._write( + f"UPDATE consumer_cursors SET expires_at = ? WHERE {where}", [expires, *params], + ) + return result.rowcount or 0 + async def list_consumer_cursors(self, consumer: str | None = None) -> list[dict]: """List active (non-expired) consumer cursors with unread counts.""" now = datetime.now(timezone.utc).isoformat() diff --git a/nerve/gateway/routes/setup.py b/nerve/gateway/routes/setup.py index 5326f76ae..ac8e3f52e 100644 --- a/nerve/gateway/routes/setup.py +++ b/nerve/gateway/routes/setup.py @@ -587,8 +587,11 @@ async def _context(actor: Actor) -> _Context: def _hosted_gateway_status(config: NerveConfig) -> dict[str, Any] | None: """What the hosted channel runtime reports about the gateway's streams - (``slack.mode: hosted`` only); a quiet default when it has not started.""" - if config.slack.mode != "hosted": + (``slack.mode: hosted`` or ``sync.github.mode: hosted`` only); a quiet + default when it has not started.""" + from nerve.channels.hosted.runtime import hosted_providers + + if not hosted_providers(config): return None try: from nerve.gateway import server @@ -1912,9 +1915,9 @@ def _refuse_closed_sources(ctx: _Context, req: SourcesRequest) -> list[str]: any other. Without local secrets every source keeps a credential on this machine — a GitHub login or token, the Gmail keyring, the Telegram API credentials — so nothing goes through (409, the policy's two sentences), - except GitHub when the operator provides its token from the environment: - that switch stores nothing, and the Gmail and Telegram fields alone are - refused, by name, with the step's reason.""" + except GitHub when the channel gateway polls it or its token comes from + the environment: that switch stores nothing, and the Gmail and Telegram + fields alone are refused, by name, with the step's reason.""" available, unavailable_reason = sources_available(ctx.facts) if not available: _refuse_without_local_secrets(ctx) @@ -1995,7 +1998,11 @@ async def set_sources(req: SourcesRequest, actor: Actor = Depends(require_person ) portable_paths: list[str] = [] if req.github is not None: - portable_paths += ["sync.github.enabled", "sync.github_events.enabled"] + # The events source goes with the switch. It reads with the gh + # CLI on this machine, which a GitHub that the channel gateway + # polls does not set up, so with hosted GitHub the writer keeps it + # off in both directions. + portable_paths.extend(("sync.github.enabled", "sync.github_events.enabled")) if req.gmail is not None: portable_paths.append("sync.gmail.enabled") if req.telegram is not None: diff --git a/nerve/gateway/server.py b/nerve/gateway/server.py index a94b4a2af..2cf3392db 100644 --- a/nerve/gateway/server.py +++ b/nerve/gateway/server.py @@ -608,9 +608,10 @@ async def _unwind_startup() -> None: # retryable while the web UI and other channels continue starting. logger.error("Slack bot failed to start: %s", e, exc_info=True) - # Hosted channels: a shared gateway holds the provider connections and - # streams admitted events to /_internal/channel/v1/stream. Optional like - # Slack: a failure leaves the endpoint refusing upgrades. + # Hosted channels and sources: a shared gateway holds the provider + # connections, polls the hosted sources, and streams admitted events to + # /_internal/channel/v1/stream. Optional like Slack: a failure leaves + # the endpoint refusing upgrades. from nerve.channels.hosted.runtime import HostedChannelRuntime, hosted_providers if hosted_providers(config): @@ -618,6 +619,7 @@ async def _unwind_startup() -> None: runtime = HostedChannelRuntime( config, _engine.router, get_config, notification_service=notification_service, + db=db, ) await runtime.start() startup_cleanups.append(("hosted channels", runtime.stop)) @@ -642,6 +644,10 @@ async def _unwind_startup() -> None: startup_cleanups.append(("cron service", cron.stop)) _cron_service = cron logger.info("Cron service started") + if _hosted_channels is not None: + # A hosted source's records reach the inbox as they arrive, and + # the jobs gated on that source run soon after. + _hosted_channels.set_delivery_listener(cron.on_source_delivered) # Register cron jobs that suppress the session label in notifications for job in cron._jobs: diff --git a/nerve/setup_checklist.py b/nerve/setup_checklist.py index 28e7fea2e..1db19a819 100644 --- a/nerve/setup_checklist.py +++ b/nerve/setup_checklist.py @@ -131,6 +131,12 @@ "GitHub is provided by the operator of this deployment; " "Gmail and Telegram are not available on this instance." ) +# The same, when the channel gateway polls GitHub (``sync.github.mode: +# hosted``): no GitHub credential is used on this machine at all. +SOURCES_GITHUB_HOSTED_TEXT = ( + "GitHub comes from the channel gateway of this deployment; " + "Gmail and Telegram are not available on this instance." +) MODE_LOCK_REASON = ( "Mode was chosen when the workspace was created. Changing it means starting a " "new workspace: stop Nerve, point workspace: in config.yaml at an empty " @@ -231,9 +237,10 @@ class Facts: #: ``None`` (:func:`nerve.setup_provenance.credential_sources`). A path #: missing here reads as ``file`` when the value is configured. credential_sources: dict[str, str | None] = field(default_factory=dict) - #: ``slack.mode: hosted`` only: the stream manager's ``status()`` — - #: ``{"streams", "connected", "last_event_at"}``; ``None`` otherwise or - #: when no process can ask (``nerve doctor``). + #: ``slack.mode: hosted`` or ``sync.github.mode: hosted`` only: the + #: stream manager's ``status()`` — ``{"streams", "connected", + #: "last_event_at"}``; ``None`` otherwise or when no process can ask + #: (``nerve doctor``). hosted_gateway: dict[str, Any] | None = None #: The Telethon session file for the Telegram message sync exists. telegram_session_present: bool = False @@ -684,18 +691,27 @@ def github_from_environment(facts: Facts) -> bool: return facts.credential_sources.get("github_token") == SOURCE_ENVIRONMENT +def github_hosted(facts: Facts) -> bool: + """Whether the channel gateway polls GitHub (``sync.github.mode: + hosted``). No GitHub credential is used on this machine then, so + switching that source on stores nothing here.""" + return facts.config.sync.github.mode == "hosted" + + def sources_available(facts: Facts) -> tuple[list[str], str | None]: """``(the sources the page may switch here, why the others are not)``. All three where the page may store a credential. Under - ``setup.local_secrets: false`` none of them — except GitHub when its - token comes from the environment: the operator holds the token, the + ``setup.local_secrets: false`` none of them — except GitHub when the + channel gateway polls it or its token comes from the environment: the toggle writes an ``enabled`` flag and nothing else, so it is offered alone while Gmail (a keyring, addresses) and Telegram (API credentials, a signed-in session) stay closed. """ if facts.config.setup.local_secrets: return ["github", "gmail", "telegram"], None + if github_hosted(facts): + return ["github"], SOURCES_GITHUB_HOSTED_TEXT if github_from_environment(facts): return ["github"], SOURCES_GITHUB_ONLY_TEXT return [], NO_LOCAL_SECRETS_TEXT @@ -707,11 +723,19 @@ def _sources_detail(facts: Facts) -> tuple[dict[str, Any], str]: session = telegram_session(facts) accounts = list(sync.gmail.accounts or []) available, unavailable_reason = sources_available(facts) + # ``source`` for GitHub is where a token Nerve forwards to the GitHub + # CLI comes from (``github_token`` / GH_TOKEN); ``None`` when the CLI's + # own login is all there is (the preflight reports that one). + github: dict[str, Any] = {"enabled": bool(sync.github.enabled), "source": sources.get("github_token")} + gateway = None + if github_hosted(facts): + # What the stream manager reports about the gateway's streams, as + # the Channels step shows for a hosted Slack. Volatile: outside the + # step's etag. + gateway = facts.hosted_gateway or {"streams": 0, "connected": False, "last_event_at": None} + github.update(mode="hosted", gateway=gateway) detail = { - # ``source`` for GitHub is where a token Nerve forwards to the GitHub - # CLI comes from (``github_token`` / GH_TOKEN); ``None`` when the CLI's - # own login is all there is (the preflight reports that one). - "github": {"enabled": bool(sync.github.enabled), "source": sources.get("github_token")}, + "github": github, "gmail": { "enabled": bool(sync.gmail.enabled), "accounts": accounts, @@ -730,7 +754,12 @@ def _sources_detail(facts: Facts) -> tuple[dict[str, Any], str]: "unavailable_reason": unavailable_reason, } on = [] - if sync.github.enabled: + if sync.github.enabled and gateway is not None: + on.append( + "GitHub from the channel gateway · " + + ("connected" if gateway.get("connected") else "waiting for the gateway") + ) + elif sync.github.enabled: on.append("GitHub") if sync.gmail.enabled: on.append("Gmail") @@ -854,9 +883,10 @@ def derive(facts: Facts) -> list[Step]: # done — a key from the environment, a Slack the channel gateway holds, an # answer that stores nothing. keyless = not config.setup.local_secrets - # A GitHub token from the environment: the one source whose switch - # stores nothing, so the policy leaves that toggle open. - github_env = github_from_environment(facts) + # GitHub polled by the channel gateway, or with its token from the + # environment: the one source whose switch stores nothing, so the policy + # leaves that toggle open. + github_open = github_hosted(facts) or github_from_environment(facts) steps: list[Step] = [] def _lockdown(reason: str) -> tuple[str, str, str]: @@ -982,11 +1012,12 @@ def _until_mode(reason: str) -> tuple[str, str, str]: status = STATUS_DONE elif locked: status, blocked_reason, blocked_kind = _lockdown(_LOCKDOWN_CONFIG_STEP) - elif keyless and not github_env: + elif keyless and not github_open: # A GitHub login or token, the Gmail keyring, the Telegram - # API credentials and the signed-in session. With the GitHub - # token in the environment the step stays open for that one - # toggle (``detail["available"]``), to answer or to skip. + # API credentials and the signed-in session. With GitHub + # hosted, or its token in the environment, the step stays + # open for that one toggle (``detail["available"]``), to + # answer or to skip. status, blocked_reason, blocked_kind = _policy() elif mode is None: status, blocked_reason, blocked_kind = _until_mode(_CHOOSE_MODE_FIRST) @@ -1035,10 +1066,11 @@ def _until_mode(reason: str) -> tuple[str, str, str]: # too: whatever made it done (a key from the environment, a token # already on disk, an answer) is not replaced from here. A Slack the # channel gateway holds keeps its one control, the on/off switch — - # and so does GitHub while its token comes from the environment. + # and so does GitHub while the gateway polls it or its token comes + # from the environment. if keyless and status == STATUS_DONE and step_id in _POLICY_STEPS and not ( (step_id == STEP_CHANNELS and config.slack.mode == "hosted") - or (step_id == STEP_SOURCES and github_env) + or (step_id == STEP_SOURCES and github_open) ): if not read_only: read_only, read_only_reason = True, NO_LOCAL_SECRETS_TEXT diff --git a/nerve/setup_checks.py b/nerve/setup_checks.py index 0278d844b..4a88d2668 100644 --- a/nerve/setup_checks.py +++ b/nerve/setup_checks.py @@ -501,7 +501,22 @@ def reset_caches() -> None: # Sources preflight # # --------------------------------------------------------------------------- # -_sources_cache: tuple[float, dict[str, Any]] | None = None +# (taken at, GitHub hosted, result) +_sources_cache: tuple[float, bool, dict[str, Any]] | None = None + +GH_HOSTED_HINT = "The channel gateway polls GitHub for this agent; gh is not needed on the server." + + +def gh_needed(config: NerveConfig) -> bool: + """Whether something in *config* runs the gh CLI. + + Hosted GitHub (``sync.github.mode: hosted``) does not; the GitHub events + and repos sources still do. + """ + sync = config.sync + if sync.github.mode != "hosted": + return True + return bool(sync.github_events.enabled or sync.github_repos.enabled) def _run(argv: list[str], timeout: float) -> subprocess.CompletedProcess: @@ -529,10 +544,18 @@ def _gh_status() -> dict[str, Any]: } -def _sources_probe() -> dict[str, Any]: +def _gh_hosted_status() -> dict[str, Any]: + """The GitHub preflight in hosted mode: nothing to sign in, so gh is not run.""" + return { + "cli": bool(shutil.which("gh")), "authenticated": False, "login": None, + "hint": GH_HOSTED_HINT, "hosted": True, + } + + +def _sources_probe(github_hosted: bool = False) -> dict[str, Any]: gog = bool(shutil.which("gog")) return { - "github": _gh_status(), + "github": _gh_hosted_status() if github_hosted else _gh_status(), "gmail": { "cli": gog, "hint": None if gog else "Install gog, then run gog gmail setup on the server.", @@ -544,18 +567,23 @@ def _sources_probe() -> dict[str, Any]: async def sources_preflight(config: NerveConfig) -> dict[str, Any]: global _sources_cache now = time.monotonic() - if _sources_cache is not None and now - _sources_cache[0] < CACHE_SECONDS: - result = dict(_sources_cache[1]) + hosted = config.sync.github.mode == "hosted" + cached = _sources_cache + if cached is not None and cached[1] == hosted and now - cached[0] < CACHE_SECONDS: + result = dict(cached[2]) else: try: - result = await run_blocking(_sources_probe, HTTP_TEST_TIMEOUT + 2) + result = await run_blocking(lambda: _sources_probe(hosted), HTTP_TEST_TIMEOUT + 2) except asyncio.TimeoutError: result = { - "github": {"cli": bool(shutil.which("gh")), "authenticated": False, "login": None, "hint": "gh did not answer."}, + "github": _gh_hosted_status() if hosted else { + "cli": bool(shutil.which("gh")), "authenticated": False, "login": None, + "hint": "gh did not answer.", + }, "gmail": {"cli": bool(shutil.which("gog")), "hint": None}, "checked_at": _now(), } - _sources_cache = (now, dict(result)) + _sources_cache = (now, hosted, dict(result)) result["telegram"] = { "credentials_configured": bool(config.sync.telegram.api_id and config.sync.telegram.api_hash), } @@ -682,6 +710,10 @@ async def environment(config: NerveConfig) -> list[dict[str, Any]]: except asyncio.TimeoutError: checks = [] _environment_cache = (now, [dict(c) for c in checks]) + if not gh_needed(config): + for check in checks: + if check["id"] == "gh" and check["status"] == "missing": + check.update(status="ok", detail="not needed", hint=GH_HOSTED_HINT) tz_ok = True try: from zoneinfo import ZoneInfo diff --git a/nerve/setup_writer.py b/nerve/setup_writer.py index 3ff27a2d2..ee44dc2b2 100644 --- a/nerve/setup_writer.py +++ b/nerve/setup_writer.py @@ -61,6 +61,7 @@ DEFAULT_MODEL, DEFAULT_RECALL_MODEL, DEFAULT_TITLE_MODEL, + GITHUB_SYNC_MODES, _expand_path, _interpolate_str, workspace_settings_file, @@ -332,6 +333,13 @@ class SetupChoices: enabled_crons: list[str] = field(default_factory=list) # sync sources github_sync: bool = False + # How GitHub reaches the inbox (``sync.github.mode``), which no writer + # sets: with ``hosted`` the channel gateway polls it, and the GitHub + # switch then leaves the events source off, since that source reads with + # gh on this machine. ``None`` takes the mode the tracked settings state + # (the installer asks nothing about it); the web steps carry the running + # config's. + github_mode: str | None = None gmail_sync: bool = False gmail_accounts: list[str] = field(default_factory=list) telegram_sync: bool = False @@ -596,7 +604,12 @@ def build_config_layers( "telegram": {"enabled": choices.telegram_sync}, "gmail": {"enabled": choices.gmail_sync}, "github": {"enabled": choices.github_sync}, - "github_events": {"enabled": choices.github_sync}, + # The events source reads with gh on this machine: it follows the + # GitHub switch while this machine polls GitHub, and stays off + # while the channel gateway does. + "github_events": { + "enabled": choices.github_sync and choices.github_mode != "hosted", + }, # Disabled by default — requires an explicit list of repos to watch. "github_repos": {"enabled": False, "repos": []}, } @@ -802,13 +815,31 @@ def write_workspace_settings(choices: SetupChoices) -> SettingsOutcome: prints a tick, and discards the answer because the key already exists. Overwriting is recoverable -- there is a .bak, and the file is in a repo where the diff is visible before anyone commits it. + + A GitHub mode the choices leave open is the one this file states: it is + the layer of ``sync.github.mode`` that the installer keeps, since it + rewrites config.yaml without one. """ + settings_path = workspace_settings_file(workspace_dir(choices)) + if choices.github_mode is None: + choices = dataclasses.replace(choices, github_mode=settings_github_mode(settings_path)) _machine, portable, shadowed = build_config_layers(choices) - return _merge_settings( - workspace_settings_file(workspace_dir(choices)), - leaf_paths(portable), - shadowed, - ) + return _merge_settings(settings_path, leaf_paths(portable), shadowed) + + +def settings_github_mode(settings_path: Path) -> str: + """The ``sync.github.mode`` that a settings file states, read as the + loader reads it: ``poll`` when the file, the key or a known value is + missing.""" + try: + section: Any = yaml.safe_load(settings_path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + return "poll" + for key in ("sync", "github"): + section = section.get(key) if isinstance(section, dict) else None + mode = section.get("mode") if isinstance(section, dict) else None + mode = str(mode or "poll").strip().lower() + return mode if mode in GITHUB_SYNC_MODES else "poll" def merge_settings_paths( @@ -1984,6 +2015,7 @@ def choices_from_config(config) -> SetupChoices: ], telegram_enabled=bool(config.telegram.enabled), github_sync=bool(config.sync.github.enabled), + github_mode=str(config.sync.github.mode), gmail_sync=bool(config.sync.gmail.enabled), gmail_accounts=list(config.sync.gmail.accounts or []), telegram_sync=bool(config.sync.telegram.enabled), diff --git a/nerve/sources/filters.py b/nerve/sources/filters.py index 82ac622ef..0ec4a8e94 100644 --- a/nerve/sources/filters.py +++ b/nerve/sources/filters.py @@ -20,6 +20,8 @@ one of its patterns or the record is dropped. An absent field cannot satisfy a non-empty allow list (fail-closed). * an empty allow list means "allow anything not denied". +* a rule may exempt record types: a record of such a type passes the rule + whatever its value. A record is kept only if it passes **every** rule (logical AND). @@ -64,11 +66,14 @@ class FieldRule: allow: If non-empty, the field value must match at least one pattern. deny: If the field value matches any pattern, the record is dropped (takes precedence over ``allow``). + exempt_record_types: Record types the rule does not apply to; a + record of one of them passes the rule without being matched. """ field: str allow: list[str] = dataclasses.field(default_factory=list) deny: list[str] = dataclasses.field(default_factory=list) + exempt_record_types: tuple[str, ...] = () @property def active(self) -> bool: @@ -92,6 +97,8 @@ def _values(self, record: SourceRecord) -> list[str]: def passes(self, record: SourceRecord) -> bool: """Return True if *record* satisfies this rule (should be kept).""" + if record.record_type in self.exempt_record_types: + return True values = self._values(record) # deny wins — drop on any deny match. diff --git a/nerve/sources/github.py b/nerve/sources/github.py index 2b16d6891..4dcc260e6 100644 --- a/nerve/sources/github.py +++ b/nerve/sources/github.py @@ -17,11 +17,15 @@ import logging import re from datetime import datetime, timedelta, timezone -from typing import Any +from typing import TYPE_CHECKING, Any from nerve.sources.base import Source +from nerve.sources.filters import FieldRule, InboxFilter from nerve.sources.models import FetchResult, SourceRecord +if TYPE_CHECKING: + from nerve.config import GitHubSyncConfig + logger = logging.getLogger(__name__) # Cap for PR/issue body and comment text to keep records reasonable. @@ -38,6 +42,12 @@ # A run on the default branch is an upstream sync or a schedule, never a PR. _CI_BRANCH_RE = re.compile(r"\bfor (?P\S+) branch$") +# The record of a change on one of the agent's own open pull requests, which +# the channel gateway delivers in hosted mode (``sync.github.pr_watch``). The +# agent is always the author: its ``reason`` is ``author`` and its ``actors`` +# are the reviewers and commenters. +PR_ACTIVITY_RECORD_TYPE = "github_pr_activity" + def _ci_branch(subject_type: str, title: str) -> str: """Branch of a CI run, or ``""`` for anything else. @@ -101,6 +111,178 @@ def _collect_actors( return actors +def github_inbox_filter(settings: GitHubSyncConfig) -> InboxFilter: + """The inbox guardrail of the GitHub source, from ``sync.github``. + + Restricts which repos (the ``repo_name`` metadata key), which GitHub + actors (``actors``: every login involved in a record) and which reasons + (``reason``) reach the inbox, and which CI branches (``ci_branch``) do + not. The rules AND together; within each, deny wins and a non-empty allow + list is fail-closed. ``ci_branch`` is deny-only on purpose: it is empty + for everything except CheckSuite notifications, so an allow list there + would fail closed and drop the whole feed. + + Only the repo rule applies to a ``github_pr_activity`` record. It is about + the agent's own pull request, so its reason is always ``author`` and its + actors are the reviewers and commenters: the actor, reason and CI branch + rules, written for notifications, would drop every such record. + + The polling source and the hosted one share this filter, and hosted mode + publishes the same patterns to the gateway, which applies them the same + way. + """ + own_pull_requests = (PR_ACTIVITY_RECORD_TYPE,) + return InboxFilter(rules=[ + FieldRule(field="repo_name", allow=settings.allow_repos, deny=settings.deny_repos), + FieldRule( + field="actors", allow=settings.allow_actors, deny=settings.deny_actors, + exempt_record_types=own_pull_requests, + ), + FieldRule( + field="reason", allow=settings.allow_reasons, deny=settings.deny_reasons, + exempt_record_types=own_pull_requests, + ), + FieldRule( + field="ci_branch", deny=settings.deny_ci_branches, + exempt_record_types=own_pull_requests, + ), + ]) + + +def notification_record(notif: dict[str, Any], extra: dict[str, Any] | BaseException) -> SourceRecord: + """The inbox record of one notification and its enrichment. + + *extra* is what :meth:`GitHubSource._enrich_notification` returned for + the notification, or the exception it raised (``gather`` hands back a + cancellation too); a failed enrichment gives a record built from the + notification alone. + """ + subject = notif.get("subject", {}) + repo = notif.get("repository", {}) + updated_at = notif.get("updated_at", "") + reason = notif.get("reason", "") + repo_name = repo.get("full_name", "?") + subject_title = subject.get("title", "?") + subject_type = subject.get("type", "") + + # Build enriched content + if isinstance(extra, BaseException): + logger.warning("Enrichment failed for %s: %s", notif.get("id"), extra) + extra = {} + + html_url = extra.get("html_url") or repo.get("html_url", "") + subject_body = extra.get("body", "") + subject_state = extra.get("state", "") + subject_user = extra.get("user", "") + assignees = extra.get("assignees", []) + labels = extra.get("labels", []) + comment = extra.get("latest_comment") + assigner = extra.get("assigner", "") + + content_parts = [ + f"Repository: {repo_name}", + f"Type: {subject_type}", + f"Title: {subject_title}", + f"Reason: {reason}", + f"State: {subject_state}" if subject_state else None, + f"Author: {subject_user}" if subject_user else None, + f"Assignees: {', '.join(assignees)}" if assignees else None, + # An assignment carries no comment text, so without this the + # record never says who asked for the work. + f"Assigned by: {assigner}" if assigner else None, + f"Labels: {', '.join(labels)}" if labels else None, + f"Updated: {updated_at}", + f"URL: {html_url}", + ] + + # Add the PR/issue body + if subject_body: + body_text = subject_body[:_MAX_BODY_CHARS] + if len(subject_body) > _MAX_BODY_CHARS: + body_text += "\n[... truncated]" + content_parts.append(f"\n--- Description ---\n{body_text}") + + # Add the latest comment that triggered the notification + if comment: + comment_body = comment.get("body", "")[:_MAX_COMMENT_CHARS] + comment_user = comment.get("user", "?") + comment_date = comment.get("created_at", "") + content_parts.append( + f"\n--- Latest Comment ({comment_user}, {comment_date}) ---\n" + f"{comment_body}" + ) + + # Add latest review state (APPROVED, CHANGES_REQUESTED, etc.) + latest_review = extra.get("latest_review") + if latest_review: + reviewer = latest_review.get("user", "?") + review_state = latest_review.get("state", "") + review_body = latest_review.get("body", "") + content_parts.append( + f"\n--- Latest Review ({reviewer}) ---\n" + f"State: {review_state}" + ) + if review_body: + content_parts.append( + review_body[:_MAX_COMMENT_CHARS] + ) + + # Add inline review comments (code-level feedback) + inline_comments = extra.get("review_inline_comments", []) + if inline_comments: + content_parts.append("\n--- Inline Review Comments ---") + for ic in inline_comments: + ic_user = ic.get("user", "?") + ic_path = ic.get("path", "") + ic_line = ic.get("line") + ic_body = ic.get("body", "") + loc = f"{ic_path}:{ic_line}" if ic_line else ic_path + content_parts.append( + f"\n{ic_user} on {loc}:\n{ic_body}" + ) + + # Add recent human comments (catches mentions buried by + # bot comments — see _enrich_recent_comments). + recent_comments = extra.get("recent_comments", []) + if recent_comments: + content_parts.append( + "\n--- Recent Comments (human, newest first) ---" + ) + for rc in recent_comments: + rc_user = rc.get("user", "?") + rc_date = rc.get("created_at", "") + rc_body = rc.get("body", "") + content_parts.append( + f"\n{rc_user} ({rc_date}):\n{rc_body}" + ) + + # Surface every involved login so the inbox guardrail can + # allow/deny by actor (the raw notification carries no actor). + actors = _collect_actors( + subject_user, assignees, comment, latest_review, + inline_comments, recent_comments, assigner, + ) + + return SourceRecord( + id=notif.get("id", ""), + source="github", + record_type="github_notification", + summary=f"[{repo_name}] {subject_title} ({reason})", + content="\n".join(p for p in content_parts if p), + timestamp=updated_at or datetime.now(timezone.utc).isoformat(), + metadata={ + "reason": reason, + "unread": notif.get("unread", False), + "subject_type": subject_type, + "subject_url": html_url, + "repo_name": repo_name, + "repo_url": repo.get("html_url", ""), + "actors": actors, + "ci_branch": _ci_branch(subject_type, subject_title), + }, + ) + + class GitHubSource(Source): """GitHub notification source using the gh CLI.""" @@ -118,13 +300,24 @@ async def fetch(self, cursor: str | None, limit: int = 100) -> FetchResult: # cursor's own second. A re-listed unchanged thread is skipped by storage. # IMPORTANT: use Z suffix, not +00:00 — the `+` in a URL query # string is interpreted as a space, silently breaking the filter. + since = None if cursor: try: + if cursor.isdigit(): + # Digits are the buffer position of the hosted source, not + # a time, though eight of them parse as a date. + raise ValueError(cursor) cursor_dt = datetime.fromisoformat(cursor.replace("Z", "+00:00")) since = cursor_dt.strftime("%Y-%m-%dT%H:%M:%SZ") except ValueError: - since = cursor - else: + # Not a timestamp, for example the buffer position that the + # hosted source keeps under the same name. GitHub refuses such a + # `since`, which would fail every run, so look back as on a + # first run instead. + logger.warning( + "GitHub cursor %r is not a timestamp; fetching the last 24 hours", cursor, + ) + if since is None: since = (datetime.now(timezone.utc) - timedelta(hours=24)).strftime("%Y-%m-%dT%H:%M:%SZ") try: @@ -160,133 +353,10 @@ async def fetch(self, cursor: str | None, limit: int = 100) -> FetchResult: ] enriched = await asyncio.gather(*enrich_tasks, return_exceptions=True) - records: list[SourceRecord] = [] - - for notif, extra in zip(notifications, enriched): - subject = notif.get("subject", {}) - repo = notif.get("repository", {}) - updated_at = notif.get("updated_at", "") - reason = notif.get("reason", "") - repo_name = repo.get("full_name", "?") - subject_title = subject.get("title", "?") - subject_type = subject.get("type", "") - - # Build enriched content - if isinstance(extra, Exception): - logger.warning("Enrichment failed for %s: %s", notif.get("id"), extra) - extra = {} - - html_url = extra.get("html_url") or repo.get("html_url", "") - subject_body = extra.get("body", "") - subject_state = extra.get("state", "") - subject_user = extra.get("user", "") - assignees = extra.get("assignees", []) - labels = extra.get("labels", []) - comment = extra.get("latest_comment") - assigner = extra.get("assigner", "") - - content_parts = [ - f"Repository: {repo_name}", - f"Type: {subject_type}", - f"Title: {subject_title}", - f"Reason: {reason}", - f"State: {subject_state}" if subject_state else None, - f"Author: {subject_user}" if subject_user else None, - f"Assignees: {', '.join(assignees)}" if assignees else None, - # An assignment carries no comment text, so without this the - # record never says who asked for the work. - f"Assigned by: {assigner}" if assigner else None, - f"Labels: {', '.join(labels)}" if labels else None, - f"Updated: {updated_at}", - f"URL: {html_url}", - ] - - # Add the PR/issue body - if subject_body: - body_text = subject_body[:_MAX_BODY_CHARS] - if len(subject_body) > _MAX_BODY_CHARS: - body_text += "\n[... truncated]" - content_parts.append(f"\n--- Description ---\n{body_text}") - - # Add the latest comment that triggered the notification - if comment: - comment_body = comment.get("body", "")[:_MAX_COMMENT_CHARS] - comment_user = comment.get("user", "?") - comment_date = comment.get("created_at", "") - content_parts.append( - f"\n--- Latest Comment ({comment_user}, {comment_date}) ---\n" - f"{comment_body}" - ) - - # Add latest review state (APPROVED, CHANGES_REQUESTED, etc.) - latest_review = extra.get("latest_review") - if latest_review: - reviewer = latest_review.get("user", "?") - review_state = latest_review.get("state", "") - review_body = latest_review.get("body", "") - content_parts.append( - f"\n--- Latest Review ({reviewer}) ---\n" - f"State: {review_state}" - ) - if review_body: - content_parts.append( - review_body[:_MAX_COMMENT_CHARS] - ) - - # Add inline review comments (code-level feedback) - inline_comments = extra.get("review_inline_comments", []) - if inline_comments: - content_parts.append("\n--- Inline Review Comments ---") - for ic in inline_comments: - ic_user = ic.get("user", "?") - ic_path = ic.get("path", "") - ic_line = ic.get("line") - ic_body = ic.get("body", "") - loc = f"{ic_path}:{ic_line}" if ic_line else ic_path - content_parts.append( - f"\n{ic_user} on {loc}:\n{ic_body}" - ) - - # Add recent human comments (catches mentions buried by - # bot comments — see _enrich_recent_comments). - recent_comments = extra.get("recent_comments", []) - if recent_comments: - content_parts.append( - "\n--- Recent Comments (human, newest first) ---" - ) - for rc in recent_comments: - rc_user = rc.get("user", "?") - rc_date = rc.get("created_at", "") - rc_body = rc.get("body", "") - content_parts.append( - f"\n{rc_user} ({rc_date}):\n{rc_body}" - ) - - # Surface every involved login so the inbox guardrail can - # allow/deny by actor (the raw notification carries no actor). - actors = _collect_actors( - subject_user, assignees, comment, latest_review, - inline_comments, recent_comments, assigner, - ) - - records.append(SourceRecord( - id=notif.get("id", ""), - source="github", - record_type="github_notification", - summary=f"[{repo_name}] {subject_title} ({reason})", - content="\n".join(p for p in content_parts if p), - timestamp=updated_at or datetime.now(timezone.utc).isoformat(), - metadata={ - "reason": reason, - "unread": notif.get("unread", False), - "subject_type": subject_type, - "subject_url": html_url, - "repo_name": repo_name, - "repo_url": repo.get("html_url", ""), - "actors": actors, - "ci_branch": _ci_branch(subject_type, subject_title), - }, - )) + records = [ + notification_record(notif, extra) + for notif, extra in zip(notifications, enriched) + ] # The first item came from the first page, so no notification that arrives # while later pages are read is older than its `updated_at`. diff --git a/nerve/sources/github_hosted.py b/nerve/sources/github_hosted.py new file mode 100644 index 000000000..5d548e79b --- /dev/null +++ b/nerve/sources/github_hosted.py @@ -0,0 +1,147 @@ +"""GitHub source in hosted mode: drains the records that the gateway delivered. + +With ``sync.github.mode: hosted`` the channel gateway polls GitHub for the +agent, builds each record the way :class:`~nerve.sources.github.GitHubSource` +builds it (enrichment included), and streams it to Nerve. The hosted source +consumer writes every record to the ``channel_observations`` buffer under the +channel ``github`` (see :mod:`nerve.channels.hosted.sources`), and this +source turns the buffered payloads back into :class:`SourceRecord` objects. It +makes no GitHub calls. + +The source keeps the name ``github``, so it runs as ``source:github`` with the +GitHub guardrail, drop hook, condensing, health checks, consumer cursors and +``messages`` gates of the polling source. + +The cursor is the buffer's row ID, as for the chat channel drains. A cursor +left by the polling source is a timestamp; it reads as 0, so the first drain +starts at the beginning of the buffer. + +Buffered payload (one row per record):: + + {"event_id": ..., "connection_id": ..., "conversation_id": "owner/repo", + "record": {"id", "record_type", "summary", "content", + "timestamp": RFC 3339, + "metadata": [{"key", "value", "values"}, ...]}} + +Metadata values arrive as strings. ``actors`` and ``changes`` are lists, and +``unread`` becomes the boolean the polling source stores. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from nerve.sources.base import Source +from nerve.sources.models import FetchResult, SourceRecord + +if TYPE_CHECKING: + from nerve.db import Database + +logger = logging.getLogger(__name__) + +SOURCE_NAME = "github" +# The buffer channel that the hosted source consumer writes for this source. +BUFFER_CHANNEL = "github" +# Metadata keys whose value is a list of strings, and those that are flags. +_LIST_KEYS = frozenset({"actors", "changes"}) +_FLAG_KEYS = frozenset({"unread"}) + + +class HostedGitHubSource(Source): + """The GitHub source over the hosted records buffer.""" + + source_name = SOURCE_NAME + + def __init__(self, db: Database): + self._db = db + + async def fetch(self, cursor: str | None, limit: int = 100) -> FetchResult: + """Read buffered records past *cursor*, oldest first. + + A failed read raises, so the runner reports it and its health checks + count it. + """ + rows = await self._db.read_channel_observations( + BUFFER_CHANNEL, after_id=_buffer_position(cursor), limit=limit, + ) + if not rows: + return FetchResult(records=[], next_cursor=cursor, has_more=False) + + records: list[SourceRecord] = [] + for row_id, payload in rows: + record = record_from_buffer(payload) if isinstance(payload, dict) else None + if record is None: + logger.warning("Hosted GitHub source: skipping unreadable buffer row %s", row_id) + continue + records.append(record) + # The cursor tracks what was scanned, so an unreadable row is passed + # and does not stop the drain. + return FetchResult( + records=records, + next_cursor=str(rows[-1][0]), + has_more=len(rows) >= limit, + ) + + +def _buffer_position(cursor: str | None) -> int: + """The buffer row ID that *cursor* names, or 0.""" + if not cursor: + return 0 + try: + return int(cursor) + except ValueError: + logger.info( + "Hosted GitHub source: cursor %r is not a buffer position (a cursor of " + "the polling source); draining from the start of the buffer", cursor, + ) + return 0 + + +def record_from_buffer(payload: dict[str, Any]) -> SourceRecord | None: + """The inbox record of one buffered payload, or ``None`` when it is unusable.""" + record = payload.get("record") + if not isinstance(record, dict): + return None + record_id = record.get("id") + record_type = record.get("record_type") + summary = record.get("summary") + if not all(isinstance(value, str) and value for value in (record_id, record_type, summary)): + return None + content = record.get("content") + timestamp = record.get("timestamp") + return SourceRecord( + id=record_id, + source=SOURCE_NAME, + record_type=record_type, + summary=summary, + content=content if isinstance(content, str) else "", + timestamp=timestamp if isinstance(timestamp, str) else "", + metadata=_metadata(record.get("metadata")), + ) + + +def _metadata(entries: object) -> dict[str, Any]: + metadata: dict[str, Any] = {} + if not isinstance(entries, list): + return metadata + for entry in entries: + if not isinstance(entry, dict): + continue + key = entry.get("key") + if not isinstance(key, str) or not key: + continue + value = entry.get("value") + value = value if isinstance(value, str) else "" + values = entry.get("values") + values = [str(item) for item in values] if isinstance(values, list) else [] + if values or key in _LIST_KEYS: + metadata[key] = values or ([value] if value else []) + elif key in _FLAG_KEYS: + metadata[key] = value.strip().lower() == "true" + else: + metadata[key] = value + return metadata + + +__all__ = ["BUFFER_CHANNEL", "SOURCE_NAME", "HostedGitHubSource", "record_from_buffer"] diff --git a/nerve/sources/registry.py b/nerve/sources/registry.py index 788945d14..e59cacc1d 100644 --- a/nerve/sources/registry.py +++ b/nerve/sources/registry.py @@ -152,26 +152,23 @@ def build_source_runners( imap.match.only_matched, vision.enabled, ) - # GitHub (notifications) + # GitHub (notifications). In hosted mode the gateway polls GitHub and the + # runner drains the records it delivered; everything past the fetch — + # guardrail, drop hook, condensing, the source name — is the same. gh = config.sync.github if gh.enabled: - from nerve.sources.filters import FieldRule, InboxFilter - from nerve.sources.github import GitHubSource + from nerve.sources.github import github_inbox_filter - source = GitHubSource() - # Guardrails: restrict which repos (matched on the "repo_name" metadata - # key) and which GitHub actors (matched on the "actors" metadata key — - # every login involved in a notification) reach the inbox. The rules - # AND together; within each, deny wins and a non-empty allow is - # fail-closed. ci_branch is deny-only on purpose: it is empty for - # everything except CheckSuite notifications, so an allow list there - # would fail closed and drop the whole feed. - gh_filter = InboxFilter(rules=[ - FieldRule(field="repo_name", allow=gh.allow_repos, deny=gh.deny_repos), - FieldRule(field="actors", allow=gh.allow_actors, deny=gh.deny_actors), - FieldRule(field="reason", allow=gh.allow_reasons, deny=gh.deny_reasons), - FieldRule(field="ci_branch", deny=gh.deny_ci_branches), - ]) + hosted = gh.mode == "hosted" + if hosted: + from nerve.sources.github_hosted import HostedGitHubSource + + source = HostedGitHubSource(db) + else: + from nerve.sources.github import GitHubSource + + source = GitHubSource() + gh_filter = github_inbox_filter(gh) gh_runner = SourceRunner( source=source, db=db, @@ -186,12 +183,13 @@ def build_source_runners( runners.append(gh_runner) # The runner's copy is validated (a wrong-shaped value turns it off). hook = gh_runner.drop_hook + label = "github (hosted)" if hosted else "github" if gh_filter.active: logger.info( - "Registered source: github (batch=%d, guardrail: " + "Registered source: %s (batch=%d, guardrail: " "repos allow=%s deny=%s; actors allow=%s deny=%s; " "reasons allow=%s deny=%s; ci_branches deny=%s; drop_hook=%s)", - gh.batch_size, + label, gh.batch_size, gh.allow_repos or "*", gh.deny_repos or [], gh.allow_actors or "*", gh.deny_actors or [], gh.allow_reasons or "*", gh.deny_reasons or [], @@ -205,7 +203,7 @@ def build_source_runners( hook[0], ) else: - logger.info("Registered source: github (batch=%d)", gh.batch_size) + logger.info("Registered source: %s (batch=%d)", label, gh.batch_size) if hook: logger.warning( "sync.github.drop_hook is set but no guardrail is, so " diff --git a/nerve/sources/runner.py b/nerve/sources/runner.py index ec40fedb5..27e20b0fd 100644 --- a/nerve/sources/runner.py +++ b/nerve/sources/runner.py @@ -233,6 +233,17 @@ def set_notification_service(self, service: Any) -> None: """Set the notification service for degradation alerts.""" self._notification_service = service + async def wait_until_idle(self) -> None: + """Return once no run of this runner is in progress; at once when idle. + + A run in progress may have fetched before newer records arrived, and + :meth:`run` returns without fetching while one is, so a caller that + must see those records waits for it first. + """ + if self._lock.locked(): + async with self._lock: + pass + async def _notify_state_change(self, new_state: str) -> None: """Send a notification when health state transitions.""" svc = self._notification_service diff --git a/tests/fake_channel_gateway.py b/tests/fake_channel_gateway.py index 882e41a7b..1c3ee213c 100644 --- a/tests/fake_channel_gateway.py +++ b/tests/fake_channel_gateway.py @@ -8,6 +8,9 @@ It records every operation and answers it with the next scripted reply for its kind, or with success. It serves ``file_read`` from ``files``, and it answers a ``file_send`` after the final chunk and keeps it in ``uploads``. +It keeps every ``subscriptions`` payload that Nerve sends, in arrival order, +in ``subscriptions``, and ``source_record_event`` builds the records of a +hosted GitHub source. ``NerveServer`` runs an ASGI app on a loopback port with uvicorn, so the tests exercise real HTTP upgrades and real status codes. @@ -37,6 +40,8 @@ TENANT_ID = "6f1d3a2c-0b4e-4f7a-9c1d-2e5b8a3f7c04" AGENT_ID = "b28c5e91-7d4a-4c3b-8f61-0a9e2d4b6c17" CONNECTION_ID = "3d9a4f2e-1c7b-4a85-9e60-2f8b7c1d5a43" +# The connection of the agent's hosted GitHub source records. +GITHUB_CONNECTION_ID = "7b9e8d6c-5a0f-4e24-93a4-6c2f1a5b9e87" AGENT_AUTHOR = "U_FIXTURE_AGENT" ISSUER = "nerve-gateway" AUDIENCE = "nerve-channel" @@ -166,6 +171,81 @@ def message_event( return event +def source_record_event( + *, + record_id: str = "9000000001", + record_type: str = "github_notification", + repository: str = "owner/repo", + title: str = "Fix the flaky retry test", + reason: str = "mention", + actors: tuple[str, ...] = ("alice", "bob"), + updated_at: str = "2026-09-16T10:00:00Z", + version: int = 1789552800, + purpose: str = "observe", + extra_metadata: list[dict[str, Any]] | None = None, + event_id: str | None = None, + connection_id: str = GITHUB_CONNECTION_ID, +) -> dict[str, Any]: + """A hosted GitHub record in the shape of the contract samples, without a delivery. + + The content and summary follow the polling source's layout for a mention + on a pull request. With the defaults, the record is the one of the + contract's notification sample. + """ + summary = f"[{repository}] {title} ({reason})" + content = "\n".join([ + f"Repository: {repository}", + "Type: PullRequest", + f"Title: {title}", + f"Reason: {reason}", + "State: open", + "Author: alice", + f"Updated: {updated_at}", + f"URL: https://github.com/{repository}/pull/12", + "", + "--- Description ---", + "The retry test waits on a timer that a busy CI host can starve.", + "", + "--- Latest Comment (bob, 2026-09-16T09:59:30Z) ---", + "@example-agent could you take a look?", + ]) + metadata: list[dict[str, Any]] = [ + {"key": "reason", "value": reason}, + {"key": "unread", "value": "true"}, + {"key": "subject_type", "value": "PullRequest"}, + {"key": "subject_url", "value": f"https://github.com/{repository}/pull/12"}, + {"key": "repo_name", "value": repository}, + {"key": "repo_url", "value": f"https://github.com/{repository}"}, + {"key": "actors", "values": list(actors)} if actors else {"key": "actors"}, + {"key": "ci_branch"}, + ] + metadata.extend(extra_metadata or []) + admission: dict[str, Any] = {"purpose": purpose} + if purpose == "observe": + admission["observe"] = {"provenance": {"provider": "github", "issuer": "", "installation_id": ""}} + else: + admission["invoke"] = {"resolved_principal_id": "1c4a8f30-95d2-4b7e-a6f1-38c0e7b52a49"} + return { + "event_id": event_id or f"github:{record_type}:{record_id}:{version}", + "message_revision_id": f"{record_id}:{version}", + "connection_id": connection_id, + "provider": "github", + "kind": "source_record", + "conversation": {"id": repository, "kind": "repository"}, + "author": {"id": actors[-1] if actors else "bob", "kind": "human"}, + "admission": admission, + "occurred_at": updated_at, + "record": { + "id": record_id, + "record_type": record_type, + "summary": summary, + "content": content, + "timestamp": updated_at, + "metadata": metadata, + }, + } + + def _selected_trigger( kind: str, conversation_kind: str, mention_agent: bool, thread: str | None, message_id: str, ) -> str: @@ -235,6 +315,9 @@ def __init__( self.uploads: list[dict[str, Any]] = [] # Largest chunk that file_read sends. self.chunk_bytes = 64 * 1024 + # Every subscriptions payload Nerve sent, on any stream, in arrival order. + self.subscriptions: list[dict[str, Any]] = [] + self.subscriptions_changed = asyncio.Event() # ------------------------------------------------------------------ # # Tokens and keys # @@ -320,7 +403,7 @@ async def wait_for(self, predicate, timeout: float = EVENT_TIMEOUT) -> None: pass async def _any_change(self) -> None: - events = (self.read_changed, self.ack_changed, self.operation_changed) + events = (self.read_changed, self.ack_changed, self.operation_changed, self.subscriptions_changed) waits = {asyncio.ensure_future(event.wait()) for event in events} try: await asyncio.wait(waits, return_when=asyncio.FIRST_COMPLETED) @@ -543,6 +626,10 @@ async def _read(self) -> None: await self.gateway.serve_operation(self, frame) elif frame["kind"] == "transfer": await self.gateway.receive_chunk(self, frame) + elif frame["kind"] == "subscriptions": + # One-way, as at the gateway: kept, never answered. + self.gateway.subscriptions.append(frame["payload"]["subscriptions"]) + self.gateway.subscriptions_changed.set() except ConnectionClosed: pass finally: diff --git a/tests/test_channel_contract.py b/tests/test_channel_contract.py index 1b476fe53..a139e1dd8 100644 --- a/tests/test_channel_contract.py +++ b/tests/test_channel_contract.py @@ -22,11 +22,15 @@ InboxAck, InboxAckItem, InboxRead, + MetadataEntry, Negotiation, Operation, OperationResult, Payload, Rejected, + SourceFilters, + SourceSubscription, + Subscriptions, decode_envelope, encode_envelope, validate_inbox_read_result, @@ -58,8 +62,15 @@ TransferChunk, TypingOperation, ) +from nerve.channels.hosted.contract.model import MAX_TEXT_BYTES from nerve.channels.hosted.runtime import RECEIVE_LIMITS -from tests.fake_channel_gateway import CONNECTION_ID, capabilities, gateway_negotiation, message_event +from tests.fake_channel_gateway import ( + CONNECTION_ID, + capabilities, + gateway_negotiation, + message_event, + source_record_event, +) def direct_message(**changes: Any) -> dict[str, Any]: @@ -390,3 +401,170 @@ def test_a_frame_larger_than_the_gateway_limit_is_not_sent(self): with pytest.raises(Rejected) as error: encode_envelope(envelope, 16384) assert error.value.reason == "frame_too_large" + + +def record_page(**changes: Any) -> dict[str, Any]: + return page(source_record_event(**changes)) + + +def record_of(value: dict[str, Any]) -> dict[str, Any]: + return value["payload"]["inbox_read_result"]["events"][0]["record"] + + +class TestSourceRecords: + def test_a_record_decodes_into_its_typed_members(self): + event = decode(record_page()).body.events[0] + + assert (event.provider, event.kind, event.admission.purpose) == ("github", "source_record", "observe") + assert (event.conversation.id, event.conversation.kind) == ("owner/repo", "repository") + assert event.message is None + record = event.record + assert (record.id, record.record_type) == ("9000000001", "github_notification") + assert record.summary == "[owner/repo] Fix the flaky retry test (mention)" + assert record.content.startswith("Repository: owner/repo\nType: PullRequest") + assert record.timestamp.isoformat() == "2026-09-16T10:00:00+00:00" + entries = {entry.key: entry for entry in record.metadata} + assert entries["actors"] == MetadataEntry(key="actors", values=("alice", "bob")) + assert entries["unread"] == MetadataEntry(key="unread", value="true") + assert entries["ci_branch"] == MetadataEntry(key="ci_branch") + + def test_a_record_needs_no_message_author_or_content(self): + value = record_page() + event = value["payload"]["inbox_read_result"]["events"][0] + del event["author"] + del event["record"]["content"] + del event["record"]["metadata"] + + record = decode(value).body.events[0].record + assert (record.content, record.metadata) == ("", ()) + + @pytest.mark.parametrize("member", ["id", "record_type", "summary"]) + def test_a_record_needs_its_id_type_and_summary(self, member): + value = record_page() + del record_of(value)[member] + + assert reason(value) == "malformed_frame" + + def test_a_source_record_event_needs_its_record(self): + value = record_page() + del value["payload"]["inbox_read_result"]["events"][0]["record"] + + assert reason(value) == "malformed_frame" + + def test_the_base_checks_still_apply(self): + value = record_page() + value["payload"]["inbox_read_result"]["events"][0]["conversation"] = {"kind": "repository"} + + assert reason(value) == "malformed_frame" + + def test_a_record_time_needs_a_time_zone(self): + value = record_page() + record_of(value)["timestamp"] = "2026-09-16T10:00:00" + + assert reason(value) == "malformed_frame" + + def test_list_metadata_of_another_type_is_malformed(self): + value = record_page() + record_of(value)["metadata"][6] = {"key": "actors", "values": "alice"} + + assert reason(value) == "malformed_frame" + + @pytest.mark.parametrize(("member", "size"), [ + ("id", 257), ("record_type", 257), ("summary", MAX_TEXT_BYTES + 1), ("content", MAX_TEXT_BYTES + 1), + ]) + def test_record_members_above_nerves_limits_are_refused(self, member, size): + value = record_page() + record_of(value)[member] = "x" * size + + assert reason(value) == "limit_exceeded" + + def test_the_gateways_own_caps_fit_inside_nerves_limits(self): + value = record_page() + record_of(value)["summary"] = "é" * 256 # 512 bytes + record_of(value)["content"] = "x" * 64 * 1024 + + record = decode(value).body.events[0].record + assert len(record.content) == 64 * 1024 + + def test_a_record_member_on_another_kind_is_not_checked(self): + value = page(direct_message()) + value["payload"]["inbox_read_result"]["events"][0]["record"] = {"id": ""} + + assert decode(value).body.events[0].kind == "message" + + +def github_subscription(**filters: tuple[str, ...]) -> Subscriptions: + return Subscriptions(version="1", sources=( + SourceSubscription(provider="github", filters=SourceFilters(**filters), pr_watch=True), + )) + + +class TestSubscriptions: + def test_nerve_sends_exactly_the_contract_members(self): + body = github_subscription( + allow_repos=("owner/*",), deny_repos=("owner/secret",), deny_actors=("codecov", "*-bot"), + deny_reasons=("subscribed",), deny_ci_branches=("main", "release/*"), + ) + envelope = Envelope(version="1", kind="subscriptions", request_id="o-2", payload=Payload(subscriptions=body)) + + value = json.loads(encode_envelope(envelope)) + + assert value == {"version": "1", "kind": "subscriptions", "request_id": "o-2", "payload": { + "subscriptions": {"version": "1", "sources": [{ + "provider": "github", + "filters": { + "allow_repos": ["owner/*"], + "deny_repos": ["owner/secret"], + "deny_actors": ["codecov", "*-bot"], + "deny_reasons": ["subscribed"], + "deny_ci_branches": ["main", "release/*"], + }, + "pr_watch": True, + }]}, + }} + + def test_no_sources_subscribes_to_nothing(self): + envelope = Envelope( + version="1", kind="subscriptions", request_id="o-2", + payload=Payload(subscriptions=Subscriptions(version="1")), + ) + + assert json.loads(encode_envelope(envelope))["payload"] == {"subscriptions": {"version": "1"}} + + def test_without_filters_keeps_the_sources_and_the_pull_request_watch(self): + wide = github_subscription(allow_repos=("owner/*",)).without_filters() + + assert wide.sources == (SourceSubscription(provider="github", filters=SourceFilters(), pr_watch=True),) + wide.validate() + + @pytest.mark.parametrize(("subscriptions", "expected"), [ + (Subscriptions(version="2"), "version_unsupported"), + (Subscriptions(version="1", sources=(SourceSubscription(provider="github"),) * 17), "limit_exceeded"), + (Subscriptions(version="1", sources=(SourceSubscription(),)), "malformed_frame"), + (github_subscription(deny_repos=tuple(f"owner/r{n}" for n in range(257))), "limit_exceeded"), + (github_subscription(deny_repos=("x" * 257,)), "limit_exceeded"), + (github_subscription(allow_actors=("",)), "limit_exceeded"), + (github_subscription(deny_reasons=("sub\tscribed",)), "limit_exceeded"), + (Subscriptions(version="1", sources=(SourceSubscription(provider="github"),) * 2), "malformed_frame"), + (Subscriptions(version="1", sources=(SourceSubscription(provider="GitHub"),)), "malformed_frame"), + (Subscriptions(version="1", sources=(SourceSubscription(provider="git hub"),)), "malformed_frame"), + (Subscriptions(version="1", sources=(SourceSubscription(provider="gitlab/self"),)), "malformed_frame"), + (Subscriptions(version="1", sources=(SourceSubscription(provider="g" * 65),)), "limit_exceeded"), + ]) + def test_the_gateways_bounds_are_checked_before_sending(self, subscriptions, expected): + with pytest.raises(Rejected) as error: + subscriptions.validate() + assert error.value.reason == expected + + def test_a_provider_is_any_lowercase_token_within_its_limit(self): + Subscriptions(version="1", sources=( + SourceSubscription(provider="github"), + SourceSubscription(provider="git_hub-2.x"), + SourceSubscription(provider="g" * 64), + )).validate() + + def test_the_largest_subscription_within_bounds_is_valid(self): + github_subscription(deny_repos=tuple("é" * 128 for _ in range(256))).validate() + + def test_the_gateway_never_sends_subscriptions(self): + assert reason(frame("subscriptions", {"version": "1"})) == "kind_unsupported" diff --git a/tests/test_cron_gates.py b/tests/test_cron_gates.py index b0199e44f..e79343ae1 100644 --- a/tests/test_cron_gates.py +++ b/tests/test_cron_gates.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta, timezone from unittest.mock import AsyncMock import pytest @@ -21,6 +21,7 @@ evaluate_gates, ) from nerve.cron.jobs import CronJob +from nerve.sources.models import SourceRecord # --------------------------------------------------------------------------- @@ -249,6 +250,104 @@ def test_from_config_legacy_keys(self): assert gate.consumer == "inbox" +def _message(record_id: str, source: str = "github") -> SourceRecord: + return SourceRecord( + id=record_id, source=source, record_type="github_notification", summary=f"message {record_id}", + content="body", timestamp="2026-09-16T10:00:00Z", metadata={}, + ) + + +async def _read_up_to_date(db, consumer: str, source: str, expires_in: timedelta) -> None: + """A consumer that read *source* up to its newest message, its cursor + expiring in *expires_in*.""" + await db.set_consumer_cursor(consumer, source, await db.get_source_max_rowid(source)) + await db._write( + "UPDATE consumer_cursors SET expires_at = ? WHERE consumer = ? AND source = ?", + ((datetime.now(UTC) + expires_in).isoformat(), consumer, source), + ) + + +async def _expiry(db, consumer: str, source: str) -> datetime: + async with db.db.execute( + "SELECT expires_at FROM consumer_cursors WHERE consumer = ? AND source = ?", (consumer, source), + ) as cursor: + row = await cursor.fetchone() + return datetime.fromisoformat(row[0]) + + +class _ThreeHoursLater(datetime): + @classmethod + def now(cls, tz=None): + return datetime.now(tz) + timedelta(hours=3) + + +@pytest.mark.asyncio +class TestMessagesGateKeepsItsCursorsAlive: + async def test_a_check_renews_a_live_cursor_about_to_expire(self, db): + await db.insert_source_messages([_message("1")], "github") + await _read_up_to_date(db, "inbox", "github", timedelta(hours=1)) + + assert await MessagesGate(sources=["github"]).is_satisfied(_ctx(db)) is False + + assert await _expiry(db, "inbox", "github") > datetime.now(UTC) + timedelta(days=1) + + async def test_a_quiet_spell_does_not_cost_the_first_new_message(self, db, monkeypatch): + """The job's gate is checked while its cursor has an hour left, and the + next record arrives three hours later. Had the cursor expired, the gate + would start it again at that record and never report it.""" + gate = MessagesGate(sources=["github"]) + await db.insert_source_messages([_message("1")], "github") + await _read_up_to_date(db, "inbox", "github", timedelta(hours=1)) + assert await gate.is_satisfied(_ctx(db)) is False + + monkeypatch.setattr("nerve.db.sources.datetime", _ThreeHoursLater) + await db.insert_source_messages([_message("2")], "github") + + assert await gate.is_satisfied(_ctx(db)) is True + + async def test_a_check_without_sources_renews_the_consumers_cursors_only(self, db): + for source in ("github", "gmail"): + await db.insert_source_messages([_message("1", source)], source) + await _read_up_to_date(db, "inbox", source, timedelta(hours=1)) + await _read_up_to_date(db, "triage", "github", timedelta(hours=1)) + + assert await MessagesGate().is_satisfied(_ctx(db)) is False + + later = datetime.now(UTC) + timedelta(days=1) + assert await _expiry(db, "inbox", "github") > later + assert await _expiry(db, "inbox", "gmail") > later + assert await _expiry(db, "triage", "github") < later + + async def test_only_a_live_cursor_past_half_its_lifetime_is_renewed(self, db): + await db.insert_source_messages([_message("1")], "github") + for consumer, expires_in in (("due", timedelta(hours=1)), ("fresh", timedelta(days=1.5)), + ("expired", timedelta(hours=-1))): + await _read_up_to_date(db, consumer, "github", expires_in) + + renewed = [ + await db.keep_consumer_cursors_alive(consumer, ["github"], ttl_days=2) + for consumer in ("due", "fresh", "expired") + ] + + assert renewed == [1, 0, 0] + assert await db.peek_consumer_cursor("expired", "github") is None + + async def test_the_forecast_renews_nothing(self, db): + await db.insert_source_messages([_message("1")], "github") + await _read_up_to_date(db, "inbox", "github", timedelta(hours=1)) + before = await _expiry(db, "inbox", "github") + + assert await MessagesGate(sources=["github"]).forecast(_ctx(db), datetime.now(UTC)) is False + + assert await _expiry(db, "inbox", "github") == before + + async def test_a_failed_renewal_leaves_the_answer_alone(self): + db = _db(get_consumer_cursor=5, get_source_max_rowid=9) + db.keep_consumer_cursors_alive.side_effect = RuntimeError("database is locked") + + assert await MessagesGate(sources=["gmail"]).is_satisfied(_ctx(db)) is True + + # --------------------------------------------------------------------------- # build_gate / build_gates # --------------------------------------------------------------------------- diff --git a/tests/test_cron_source_delivery.py b/tests/test_cron_source_delivery.py new file mode 100644 index 000000000..1caf5e21b --- /dev/null +++ b/tests/test_cron_source_delivery.py @@ -0,0 +1,477 @@ +"""CronService.on_source_delivered: drain at once, then run the gated jobs. + +The scheduler runs paused, so nothing fires on its own; source runners are +stand-ins with the runner's lock semantics, and each job run is recorded +instead of reaching an agent. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest +import pytest_asyncio +from apscheduler.triggers.cron import CronTrigger + +from nerve.config import NerveConfig +from nerve.cron.jobs import CronJob +from nerve.cron.service import CronService +from nerve.identity import attribution_for +from nerve.sources.github import GitHubSource +from nerve.sources.github_hosted import HostedGitHubSource +from nerve.sources.models import IngestResult +from tests.actor_rows import FAKE_SYSTEM_ACTOR, mock_system_principal + +pytestmark = pytest.mark.asyncio + +NEVER = "0 0 1 1 *" + + +class FakeRunner: + """A source runner: one run at a time, and a run while busy returns at once.""" + + def __init__(self, name: str = "github") -> None: + self.source = SimpleNamespace(source_name=name) + self.job_id = f"source:{name}" + self.runs = 0 + self.hold: asyncio.Event | None = None + self.started = asyncio.Event() + self._lock = asyncio.Lock() + + async def run(self) -> IngestResult: + if self._lock.locked(): + return IngestResult(records_ingested=0) + async with self._lock: + self.runs += 1 + self.started.set() + if self.hold is not None: + await self.hold.wait() + return IngestResult(records_ingested=1) + + async def wait_until_idle(self) -> None: + if self._lock.locked(): + async with self._lock: + pass + + +class Runs: + """Records job runs; a job id in ``hold`` blocks its runs until released.""" + + def __init__(self) -> None: + self.started: list[tuple[str, Any]] = [] + self.finished: list[str] = [] + self.hold: dict[str, asyncio.Event] = {} + self.changed = asyncio.Event() + + async def __call__(self, job: CronJob, attribution: Any = None) -> None: + self.started.append((job.id, attribution)) + self.changed.set() + gate = self.hold.get(job.id) + if gate is not None: + await gate.wait() + self.finished.append(job.id) + self.changed.set() + + def ids(self) -> list[str]: + return [job_id for job_id, _ in self.started] + + +async def eventually(predicate, timeout: float = 5.0) -> None: + deadline = asyncio.get_running_loop().time() + timeout + while not predicate(): + if asyncio.get_running_loop().time() > deadline: + raise AssertionError("timed out waiting for the condition") + await asyncio.sleep(0.005) + + +def job(job_id: str, *, sources: list[str] | None = None, gated: bool = True, **fields: Any) -> CronJob: + run_if = [{"type": "messages", **({"sources": sources} if sources is not None else {})}] if gated else [] + return CronJob(id=job_id, schedule=NEVER, prompt="do it", run_if=run_if, **fields) + + +@pytest_asyncio.fixture +async def svc(monkeypatch): + config = MagicMock() + config.timezone = "UTC" + config.cron.delivery_debounce_seconds = 0 + db = AsyncMock() + mock_system_principal(db) + service = CronService(config, AsyncMock(), db) + runs = Runs() + monkeypatch.setattr(service, "_run_job_inner", runs) + service.scheduler.start(paused=True) + service.runs = runs + try: + yield service + finally: + if service.scheduler.running: + await service.stop() + + +def install(service: CronService, *jobs: CronJob, runners: list[FakeRunner] | None = None) -> None: + service._jobs = list(jobs) + for item in jobs: + if item.enabled: + service.scheduler.add_job( + service._run_job_wrapper, CronTrigger.from_crontab(NEVER), args=[item], id=item.id, + ) + service._source_runners = runners if runners is not None else [FakeRunner()] + + +class TestDrain: + async def test_the_runner_drains_at_once(self, svc): + runner = FakeRunner() + install(svc, runners=[runner]) + + await svc.on_source_delivered("github") + + assert runner.runs == 1 + svc.db.log_cron_start.assert_awaited() + assert svc.db.log_cron_start.await_args.args[0] == "source:github" + + async def test_deliveries_during_a_drain_make_it_run_once_more(self, svc): + runner = FakeRunner() + runner.hold = asyncio.Event() + install(svc, runners=[runner]) + + first = asyncio.create_task(svc.on_source_delivered("github")) + await runner.started.wait() + for _ in range(3): + await svc.on_source_delivered("github") # returns at once + runner.hold.set() + await first + + assert runner.runs == 2 + + async def test_a_run_in_progress_is_waited_for(self, svc): + runner = FakeRunner() + runner.hold = asyncio.Event() + install(svc, runners=[runner]) + scheduled = asyncio.create_task(runner.run()) + await runner.started.wait() + + delivered = asyncio.create_task(svc.on_source_delivered("github")) + await asyncio.sleep(0.05) + assert not delivered.done() + runner.hold.set() + await asyncio.gather(scheduled, delivered) + + assert runner.runs == 2 + + async def test_the_jobs_wait_for_a_run_that_took_the_runner_first(self, svc): + """The delivery's run logs its start before it runs. A scheduled fire + or a manual sync that takes the runner meanwhile leaves the delivery's + run nothing to do, and the records are in only once that run ends.""" + runner = FakeRunner() + runner.hold = asyncio.Event() + install(svc, job("pr-monitor", sources=["github"]), runners=[runner]) + others: list[asyncio.Task] = [] + + async def log_cron_start(job_id, **kwargs): + if job_id == "source:github" and not others: + others.append(asyncio.create_task(runner.run())) + await runner.started.wait() + return 1 + + svc.db.log_cron_start = log_cron_start + delivered = asyncio.create_task(svc.on_source_delivered("github")) + await runner.started.wait() + await asyncio.sleep(0.05) + + assert svc.runs.started == [] + assert not delivered.done() + runner.hold.set() + await asyncio.gather(delivered, *others) + await eventually(lambda: svc.runs.finished == ["pr-monitor"]) + assert runner.runs == 1 + + async def test_a_scheduled_fire_of_the_runner_is_skipped_while_it_drains(self, svc): + """The drain is a run of the source runner for delivered records, so + the runner's own fire meanwhile is skipped like that of a job.""" + runner = FakeRunner() + runner.hold = asyncio.Event() + install(svc, runners=[runner]) + drained = asyncio.create_task(svc.on_source_delivered("github")) + await runner.started.wait() + + await svc._run_source_wrapper(runner) + + assert svc.db.log_cron_start.await_count == 1 + runner.hold.set() + await drained + assert runner.runs == 1 + + async def test_a_source_without_a_runner_runs_nothing(self, svc): + install(svc, job("pr-monitor", sources=["github"]), runners=[FakeRunner("gmail:me@example.net")]) + + await svc.on_source_delivered("github") + await asyncio.sleep(0.05) + + assert svc.runs.started == [] + assert svc._delivery_tasks == {} + + async def test_a_stopped_service_runs_nothing(self, svc): + runner = FakeRunner() + install(svc, job("pr-monitor", sources=["github"]), runners=[runner]) + await svc.stop() + await asyncio.sleep(0) # the scheduler applies its shutdown on the loop + + await svc.on_source_delivered("github") + + assert runner.runs == 0 + + +class TestJobs: + async def test_only_jobs_gated_on_the_source_or_on_any_source_run(self, svc): + unscheduled = job("broken-schedule", sources=["github"]) + install( + svc, + job("pr-monitor", sources=["github", "gmail"]), + job("inbox", sources=None), + job("mail", sources=["gmail"]), + job("briefing", gated=False), + job("paused", sources=["github"], enabled=False), + ) + svc._jobs.append(unscheduled) + + await svc.on_source_delivered("github") + + await eventually(lambda: len(svc.runs.finished) == 2) + await asyncio.sleep(0.05) + assert sorted(svc.runs.ids()) == ["inbox", "pr-monitor"] + assert all(attribution is None for _, attribution in svc.runs.started) + + async def test_a_burst_within_the_debounce_window_is_one_run(self, svc): + svc.config.cron.delivery_debounce_seconds = 1 + install(svc, job("pr-monitor", sources=["github"])) + + for _ in range(3): + await svc.on_source_delivered("github") + await asyncio.sleep(0.1) + assert svc.runs.started == [] + + await eventually(lambda: svc.runs.finished == ["pr-monitor"]) + await asyncio.sleep(0.2) + assert svc.runs.ids() == ["pr-monitor"] + + async def test_a_delivery_after_the_run_started_gets_a_run_of_its_own(self, svc): + install(svc, job("pr-monitor", sources=["github"])) + svc.runs.hold["pr-monitor"] = asyncio.Event() + + await svc.on_source_delivered("github") + await eventually(lambda: svc.runs.ids() == ["pr-monitor"]) + await svc.on_source_delivered("github") + assert "pr-monitor" in svc._delivery_tasks + svc.runs.hold["pr-monitor"].set() + + await eventually(lambda: svc.runs.finished == ["pr-monitor", "pr-monitor"]) + + async def test_the_delivery_run_waits_for_a_run_in_flight(self, svc): + monitor = job("pr-monitor", sources=["github"]) + install(svc, monitor) + svc.runs.hold["pr-monitor"] = asyncio.Event() + scheduled = asyncio.create_task(svc._run_job_wrapper(monitor)) + await eventually(lambda: svc.runs.ids() == ["pr-monitor"]) + + await svc.on_source_delivered("github") + await asyncio.sleep(0.05) + assert len(svc.runs.started) == 1 + svc.runs.hold["pr-monitor"].set() + await scheduled + + await eventually(lambda: svc.runs.finished == ["pr-monitor", "pr-monitor"]) + + async def test_a_fire_of_its_own_is_skipped_while_a_delivery_run_goes_on(self, svc): + monitor = job("pr-monitor", sources=["github"]) + install(svc, monitor) + svc.runs.hold["pr-monitor"] = asyncio.Event() + await svc.on_source_delivered("github") + await eventually(lambda: svc.runs.ids() == ["pr-monitor"]) + + await asyncio.wait_for(svc._run_job_wrapper(monitor), timeout=1) + assert svc.runs.ids() == ["pr-monitor"] + + person = attribution_for(FAKE_SYSTEM_ACTOR) + manual = asyncio.create_task(svc.run_job("pr-monitor", attribution=person)) + await eventually(lambda: len(svc.runs.started) == 2) + svc.runs.hold["pr-monitor"].set() + await manual + await eventually(lambda: len(svc.runs.finished) == 2) + assert svc.runs.started[1] == ("pr-monitor", person) + + async def test_a_run_by_hand_starts_while_a_delivery_run_goes_on(self, svc): + """How a run was started decides, not who started it: a run by hand + always starts, also one without a person's attribution.""" + install(svc, job("pr-monitor", sources=["github"])) + svc.runs.hold["pr-monitor"] = asyncio.Event() + await svc.on_source_delivered("github") + await eventually(lambda: svc.runs.ids() == ["pr-monitor"]) + + by_hand = asyncio.create_task(svc.run_job("pr-monitor")) + await eventually(lambda: len(svc.runs.started) == 2) + svc.runs.hold["pr-monitor"].set() + await by_hand + + assert svc.runs.started[1] == ("pr-monitor", None) + + async def test_a_catch_up_is_skipped_while_a_delivery_run_goes_on(self, svc): + """A catch-up stands in for a fire that was missed, so it does not + start on top of any run of its job, a delivery run included, and + whether or not the job has ``lock``.""" + monitor = job("pr-monitor", sources=["github"], lock=True) + install(svc, monitor) + svc.runs.hold["pr-monitor"] = asyncio.Event() + await svc.on_source_delivered("github") + await eventually(lambda: svc.runs.ids() == ["pr-monitor"]) + + await asyncio.wait_for(svc._run_job_wrapper(monitor, origin="catchup"), timeout=1) + + assert svc.runs.ids() == ["pr-monitor"] + svc.runs.hold["pr-monitor"].set() + await eventually(lambda: svc.runs.finished == ["pr-monitor"]) + + async def test_a_delivery_run_counts_for_a_catch_up_queued_before_it(self, svc): + """A catch-up does not run a job that has run since it was queued.""" + install(svc, job("pr-monitor", sources=["github"])) + queued = svc._runs.started("pr-monitor") + + await svc.on_source_delivered("github") + await eventually(lambda: svc.runs.finished == ["pr-monitor"]) + + assert svc._catch_up_run("pr-monitor", queued) is None + + async def test_a_locked_job_waits_for_the_delivery_run_instead(self, svc): + monitor = job("pr-monitor", sources=["github"], lock=True) + install(svc, monitor) + svc.runs.hold["pr-monitor"] = asyncio.Event() + await svc.on_source_delivered("github") + await eventually(lambda: svc.runs.ids() == ["pr-monitor"]) + + scheduled = asyncio.create_task(svc._run_job_wrapper(monitor)) + await asyncio.sleep(0.05) + assert len(svc.runs.started) == 1 + svc.runs.hold["pr-monitor"].set() + await scheduled + + assert svc.runs.finished == ["pr-monitor", "pr-monitor"] + + async def test_a_job_removed_by_a_reload_before_its_run_is_not_run(self, svc): + svc.config.cron.delivery_debounce_seconds = 1 + install(svc, job("pr-monitor", sources=["github"])) + await svc.on_source_delivered("github") + + svc._jobs = [] + await asyncio.sleep(1.2) + + assert svc.runs.started == [] + assert svc._delivery_tasks == {} + + async def test_stop_cancels_the_pending_runs(self, svc): + svc.config.cron.delivery_debounce_seconds = 600 + install(svc, job("pr-monitor", sources=["github"])) + await svc.on_source_delivered("github") + task = svc._delivery_tasks["pr-monitor"] + + await svc.stop() + await asyncio.sleep(0) + + assert task.cancelled() + assert svc.runs.started == [] + + async def test_stop_cancels_a_run_that_has_started(self, svc): + install(svc, job("pr-monitor", sources=["github"])) + svc.runs.hold["pr-monitor"] = asyncio.Event() + await svc.on_source_delivered("github") + await eventually(lambda: svc.runs.ids() == ["pr-monitor"]) + assert svc._delivery_tasks == {} + + await svc.stop() + svc.runs.hold["pr-monitor"].set() + await asyncio.sleep(0.05) + + assert svc.runs.finished == [] + assert svc._delivery_started == set() + + async def test_a_drain_that_ends_while_stop_winds_down_starts_no_run(self, svc, monkeypatch): + """stop() waits for the runs it cancels, with the scheduler still up. + A drain that ends meanwhile must not start another run.""" + runner = FakeRunner() + install(svc, job("pr-monitor", sources=["github"]), runners=[runner]) + started: list[str] = [] + + async def winding_down(job: CronJob, attribution: Any = None) -> None: + started.append(job.id) + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + runner.hold.set() # the next delivery's drain ends now + await asyncio.sleep(0.05) + raise + + monkeypatch.setattr(svc, "_run_job_inner", winding_down) + await svc.on_source_delivered("github") + await eventually(lambda: started == ["pr-monitor"]) + runner.hold = asyncio.Event() + runner.started.clear() + delivered = asyncio.create_task(svc.on_source_delivered("github")) + await runner.started.wait() + + await svc.stop() + await asyncio.wait_for(delivered, timeout=5) + await asyncio.sleep(0.05) + + assert started == ["pr-monitor"] + assert svc._delivery_tasks == {} + + async def test_schedule_listeners_hear_of_the_drain_and_the_run(self, svc): + install(svc, job("pr-monitor", sources=["github"])) + heard: list[int] = [] + svc.add_schedule_listener(lambda: heard.append(1)) + + await svc.on_source_delivered("github") + await eventually(lambda: svc.runs.finished == ["pr-monitor"]) + + assert len(heard) >= 2 + + @pytest.mark.parametrize(("value", "expected"), [ + ("soon", 60.0), (None, 60.0), (True, 60.0), (900, 600.0), (-1, 0.0), (5, 5.0), + ]) + async def test_the_debounce_is_held_within_its_range(self, svc, value, expected): + svc.config.cron.delivery_debounce_seconds = value + + assert svc._delivery_debounce() == expected + + +async def test_source_reloads_keep_the_github_mode_the_service_started_with(): + config = NerveConfig() + config.sync.github.mode = "hosted" + service = CronService(config, AsyncMock(), MagicMock()) + service.scheduler.start(paused=True) + try: + service._register_source_runners() + config.sync.github.mode = "poll" + + await service.reload_sources() + + (runner,) = [r for r in service._source_runners if r.job_id == "source:github"] + assert isinstance(runner.source, HostedGitHubSource) + finally: + await service.stop() + + +async def test_a_service_started_in_poll_mode_keeps_polling(): + config = NerveConfig() + service = CronService(config, AsyncMock(), MagicMock()) + service.scheduler.start(paused=True) + try: + config.sync.github.mode = "hosted" + + await service.reload_sources() + + (runner,) = [r for r in service._source_runners if r.job_id == "source:github"] + assert isinstance(runner.source, GitHubSource) + finally: + await service.stop() diff --git a/tests/test_gateway_startup.py b/tests/test_gateway_startup.py index 8b1f58306..18a3e10bf 100644 --- a/tests/test_gateway_startup.py +++ b/tests/test_gateway_startup.py @@ -268,6 +268,46 @@ async def test_a_failure_after_the_hosted_channels_start_stops_them(harness, mon assert hosted.starts == 1 and hosted.stops == 1 +@pytest.mark.asyncio +async def test_hosted_source_deliveries_are_handed_to_the_cron_service(harness, monkeypatch): + """With GitHub hosted, the runtime's source consumer calls the cron + service's delivery hook, which drains the records and runs the jobs + gated on them.""" + import nerve.cron.service as cron_module + from nerve.config import GitHubSyncConfig, HostedChannelsConfig + from tests.fake_channel_gateway import FakeChannelGateway + from tests.test_hosted_channel_config import HOSTED + + class Cron: + def __init__(self, config, engine, db): + self._jobs = [] + self.notification_service = None + + async def start(self): + pass + + async def stop(self): + pass + + async def on_source_delivered(self, source): + pass + + gateway = FakeChannelGateway(jwks_file=harness["config"].workspace.parent / "gateway-jwks.json") + harness["config"].sync.github = GitHubSyncConfig(mode="hosted") + harness["config"].channels.hosted = HostedChannelsConfig.from_dict( + {**HOSTED, "gateway_jwks_file": str(gateway.jwks_file)}, + ) + monkeypatch.setattr(cron_module, "CronService", Cron) + monkeypatch.setattr(harness["server"], "_hosted_channels", None) + + async with harness["server"].lifespan(MagicMock()): + runtime = harness["server"]._hosted_channels + cron = harness["server"]._cron_service + assert runtime is not None and set(runtime.sources) == {"github"} + assert isinstance(cron, Cron) + assert runtime.sources["github"]._listener == cron.on_source_delivered + + @pytest.mark.asyncio async def test_hosted_channels_start_while_slack_is_switched_off(harness, monkeypatch): """A reload can switch hosted Slack on only if its runtime exists.""" diff --git a/tests/test_hosted_sources.py b/tests/test_hosted_sources.py new file mode 100644 index 000000000..e125d0630 --- /dev/null +++ b/tests/test_hosted_sources.py @@ -0,0 +1,1144 @@ +"""The hosted GitHub source: settings, subscriptions, the record consumer, the +drain into the inbox, and the jobs that a delivery runs. + +The end-to-end tests run the fake gateway against the real stream endpoint, as +the hosted channel tests do, with a real database, source runner, and cron +service; only the agent turn is a stand-in. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import copy +import logging +from collections.abc import AsyncIterator, Callable +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest +import pytest_asyncio +import yaml + +import nerve.config as config_module +from nerve import config_reload +from nerve.agent.sessions import SessionManager +from nerve.channels.hosted.contract import ( + SourceFilters, + SourceSubscription, + Subscriptions, +) +from nerve.channels.hosted.contract.model import MAX_SUBSCRIPTION_PATTERNS +from nerve.channels.hosted.runtime import ( + HostedChannelRuntime, + hosted_channel_providers, + hosted_providers, +) +from nerve.channels.hosted.sources import ( + BUFFER_MAX_ROWS, + HostedSourceConsumer, + github_subscription, + subscriptions_for, +) +from nerve.channels.router import ChannelRouter +from nerve.cli import doctor_report +from nerve.config import ( + GitHubSyncConfig, + NerveConfig, + SlackConfig, + validate_config_keys, +) +from nerve.cron.gates import GateContext, GitHubPrActivityGate +from nerve.cron.service import CronService +from nerve.gateway import server as gateway_server +from nerve.sources.github import GitHubSource, notification_record +from nerve.sources.github_hosted import HostedGitHubSource, record_from_buffer +from nerve.sources.registry import build_source_runners +from tests.fake_channel_gateway import ( + EVENT_TIMEOUT, + FakeChannelGateway, + FakeStream, + NerveServer, + gateway_negotiation, + message_event, + source_record_event, +) +from tests.test_hosted_channel import FAST_READER, FAST_STREAM +from tests.test_hosted_channel import hosted_config as slack_hosted_config +from tests.test_hosted_channel_config import HOSTED + +GITHUB_DENY = {"deny_repos": ["owner/noisy"], "deny_reasons": ["subscribed"]} + + +def hosted_github(**settings: Any) -> GitHubSyncConfig: + return GitHubSyncConfig(mode="hosted", **settings) + + +async def eventually(predicate, timeout: float = EVENT_TIMEOUT) -> None: + deadline = asyncio.get_running_loop().time() + timeout + while not predicate(): + if asyncio.get_running_loop().time() > deadline: + raise AssertionError("timed out waiting for the condition") + await asyncio.sleep(0.01) + + +# ---------------------------------------------------------------------- # +# Settings # +# ---------------------------------------------------------------------- # + + +class TestSettings: + def test_poll_is_the_default_and_hosted_is_read(self): + assert NerveConfig().sync.github.mode == "poll" + config = NerveConfig.from_dict({"sync": {"github": {"mode": " Hosted ", "pr_watch": "false"}}}) + + assert (config.sync.github.mode, config.sync.github.pr_watch) == ("hosted", False) + assert NerveConfig().sync.github.pr_watch is True + + def test_an_unknown_mode_falls_back_to_poll(self, caplog): + with caplog.at_level(logging.WARNING): + config = NerveConfig.from_dict({"sync": {"github": {"mode": "webhook"}}}) + + assert config.sync.github.mode == "poll" + assert "sync.github.mode" in caplog.text + + @pytest.mark.parametrize(("value", "expected"), [ + (None, 60), (0, 0), (15, 15), ("120", 120), (900, 900), (-5, 0), ("soon", 60), + ]) + def test_the_delivery_debounce_is_read_as_a_whole_number_of_seconds(self, value, expected): + cron = {} if value is None else {"delivery_debounce_seconds": value} + + assert NerveConfig.from_dict({"cron": cron}).cron.delivery_debounce_seconds == expected + + def test_config_validate_warns_about_a_debounce_above_the_maximum(self, tmp_path): + from nerve.config_validate import ValidationResult, _validate_delivery_debounce + + result = ValidationResult() + _validate_delivery_debounce( + NerveConfig.from_dict({"cron": {"delivery_debounce_seconds": 900}}), result, + ) + quiet = ValidationResult() + _validate_delivery_debounce(NerveConfig(), quiet) + + assert result.warnings == [( + "cron.delivery_debounce_seconds: 900 is above the maximum of 600, so the jobs " + "a hosted delivery starts wait 600s" + )] + assert quiet.warnings == [] + + def test_the_new_keys_are_known(self): + merged = { + "sync": {"github": {"mode": "hosted", "pr_watch": True}}, + "cron": {"delivery_debounce_seconds": 30}, + } + + assert validate_config_keys(merged) == [] + + def test_the_mode_needs_a_restart_and_the_filters_do_not(self): + before = NerveConfig() + after = NerveConfig() + after.sync.github = hosted_github(deny_repos=["owner/noisy"], pr_watch=False) + + changed = config_reload.restart_required(before, after) + + assert changed == ["sync.github.mode: 'poll' → 'hosted'"] + + def test_hosted_github_is_a_hosted_provider_without_hosted_slack(self): + config = NerveConfig() + config.sync.github = hosted_github() + + assert hosted_providers(config) == ["github"] + assert hosted_channel_providers(config) == [] + config.slack = SlackConfig(mode="hosted") + assert hosted_providers(config) == ["slack", "github"] + + +# ---------------------------------------------------------------------- # +# Subscriptions # +# ---------------------------------------------------------------------- # + + +class TestSubscriptionsBuilder: + def test_the_guardrail_patterns_and_the_pull_request_watch_are_published(self): + config = NerveConfig() + config.sync.github = hosted_github( + allow_repos=["owner/*", " "], deny_repos=[" owner/secret "], allow_actors=["alice"], + deny_actors=["codecov"], allow_reasons=["mention"], deny_reasons=["subscribed"], + deny_ci_branches=["main"], pr_watch=False, + ) + + assert subscriptions_for(config, ["github"]) == Subscriptions(version="1", sources=( + SourceSubscription( + provider="github", + filters=SourceFilters( + allow_repos=("owner/*",), deny_repos=("owner/secret",), allow_actors=("alice",), + deny_actors=("codecov",), allow_reasons=("mention",), deny_reasons=("subscribed",), + deny_ci_branches=("main",), + ), + pr_watch=False, + ), + )) + + def test_a_switched_off_source_subscribes_to_nothing(self): + config = NerveConfig() + config.sync.github = hosted_github(enabled=False) + + assert subscriptions_for(config, ["github"]) == Subscriptions(version="1") + + def test_an_allow_list_the_gateway_cannot_take_is_published_empty(self, caplog): + patterns = [f"owner/repo-{n}" for n in range(MAX_SUBSCRIPTION_PATTERNS + 1)] + + with caplog.at_level(logging.WARNING): + subscription = github_subscription(hosted_github(allow_repos=patterns)) + + assert subscription.filters.allow_repos == () + assert "sync.github.allow_repos" in caplog.text + + @pytest.mark.parametrize("bad", ["x" * 257, "owner/\x07bell", "line\nbreak"]) + def test_an_unusable_allow_pattern_widens_the_whole_list(self, bad): + subscription = github_subscription(hosted_github(allow_actors=["alice", bad])) + + assert subscription.filters.allow_actors == () + + def test_a_deny_list_keeps_the_patterns_that_fit(self): + patterns = ["x" * 257, "owner/noisy", "tab\there"] + [f"owner/r{n}" for n in range(300)] + + subscription = github_subscription(hosted_github(deny_repos=patterns)) + + assert subscription.filters.deny_repos[0] == "owner/noisy" + assert len(subscription.filters.deny_repos) == MAX_SUBSCRIPTION_PATTERNS + Subscriptions(version="1", sources=(subscription,)).validate() + + +# ---------------------------------------------------------------------- # +# Consumer and buffer # +# ---------------------------------------------------------------------- # + + +def decoded(event: dict[str, Any]): + """*event* as the reader hands it to a consumer: decoded and checked.""" + from nerve.channels.hosted.contract.model import Event + from nerve.channels.hosted.contract.wire import decode_record + + row = {**event, "delivery": {"inbox_id": "1001"}} + value = decode_record(Event, row) + value.validate() + return value + + +@pytest.mark.asyncio +class TestConsumer: + @pytest.fixture + def config(self) -> NerveConfig: + config = NerveConfig() + config.sync.github = hosted_github() + return config + + async def test_a_record_is_accepted_once_its_buffer_row_is_stored(self, db, config): + heard: list[str] = [] + consumer = HostedSourceConsumer("github", db, lambda: config) + consumer.set_delivery_listener(heard.append) + + disposition = await consumer.deliver(decoded(source_record_event())) + + assert disposition.outcome == "accepted" + [(_, payload)] = await db.read_channel_observations("github") + assert payload["conversation_id"] == "owner/repo" + assert payload["record"]["id"] == "9000000001" + assert payload["record"]["timestamp"] == "2026-09-16T10:00:00Z" + assert {"key": "actors", "value": "", "values": ["alice", "bob"]} in payload["record"]["metadata"] + assert heard == ["github"] + + async def test_it_never_refuses_capacity(self, db, config): + config.sync.github.enabled = False + + assert HostedSourceConsumer("github", db, lambda: config).can_accept() + + async def test_an_async_listener_runs_on_its_own(self, db, config): + started = asyncio.Event() + release = asyncio.Event() + + async def listener(provider: str) -> None: + started.set() + await release.wait() + + consumer = HostedSourceConsumer("github", db, lambda: config) + consumer.set_delivery_listener(listener) + + assert (await consumer.deliver(decoded(source_record_event()))).outcome == "accepted" + await asyncio.wait_for(started.wait(), EVENT_TIMEOUT) + await consumer.stop() + + @pytest.mark.parametrize("event", [ + message_event(purpose="observe"), + source_record_event(purpose="invoke"), + ]) + async def test_anything_else_is_rejected_with_a_reason_the_gateway_takes(self, db, config, event): + disposition = await HostedSourceConsumer("github", db, lambda: config).deliver(decoded(event)) + + assert (disposition.outcome, disposition.reason_code) == ("rejected", "admission_rejected") + assert await db.read_channel_observations("github") == [] + + async def test_a_failed_write_defers_the_row(self, db, config, monkeypatch): + async def locked(*args, **kwargs): + raise RuntimeError("database is locked") + + monkeypatch.setattr(db, "insert_channel_observation", locked) + heard: list[str] = [] + consumer = HostedSourceConsumer("github", db, lambda: config) + consumer.set_delivery_listener(heard.append) + + assert (await consumer.deliver(decoded(source_record_event()))).outcome == "deferred" + assert heard == [] + + async def test_the_buffer_has_its_own_row_target(self, db, config, monkeypatch): + seen: dict[str, Any] = {} + original = db.insert_channel_observation + + async def spy(**kwargs): + seen.update(kwargs) + return await original(**kwargs) + + monkeypatch.setattr(db, "insert_channel_observation", spy) + await HostedSourceConsumer("github", db, lambda: config).deliver(decoded(source_record_event())) + + assert (seen["channel"], seen["max_rows"]) == ("github", BUFFER_MAX_ROWS) + + +class TestBufferedRecords: + def payload(self, **record: Any) -> dict[str, Any]: + base = { + "id": "9000000001", "record_type": "github_notification", + "summary": "[owner/repo] Fix it (mention)", "content": "Repository: owner/repo", + "timestamp": "2026-09-16T10:00:00Z", + "metadata": [ + {"key": "reason", "value": "mention", "values": []}, + {"key": "unread", "value": "false", "values": []}, + {"key": "actors", "value": "", "values": ["alice"]}, + {"key": "changes", "value": "", "values": []}, + {"key": "ci_branch", "value": "", "values": []}, + {"key": "", "value": "ignored", "values": []}, + ], + } + base.update(record) + return {"record": base} + + def test_a_payload_maps_to_the_polling_sources_record_shape(self): + record = record_from_buffer(self.payload()) + + assert (record.id, record.source, record.record_type) == ("9000000001", "github", "github_notification") + assert record.timestamp == "2026-09-16T10:00:00Z" + assert record.metadata == { + "reason": "mention", "unread": False, "actors": ["alice"], "changes": [], "ci_branch": "", + } + + @pytest.mark.parametrize("change", [{"id": ""}, {"summary": None}, {"record_type": 7}]) + def test_an_unusable_payload_gives_no_record(self, change): + assert record_from_buffer(self.payload(**change)) is None + assert record_from_buffer({"record": "text"}) is None + + @pytest.mark.asyncio + async def test_the_drain_reads_past_a_polling_cursor_and_skips_unreadable_rows(self, db): + await db.insert_channel_observation("github", "github:owner/repo", self.payload()) + await db.insert_channel_observation("github", "github:owner/repo", {"record": {}}) + await db.insert_channel_observation("slack", "slack:C1", {"text": "not ours"}) + + result = await HostedGitHubSource(db).fetch("2026-09-16T09:00:00Z", limit=10) + + assert [r.id for r in result.records] == ["9000000001"] + assert (result.next_cursor, result.has_more) == ("2", False) + again = await HostedGitHubSource(db).fetch(result.next_cursor, limit=10) + assert (again.records, again.next_cursor) == ([], "2") + + +@pytest.mark.asyncio +class TestRegistry: + def runner(self, config, db): + runners = [r for r in build_source_runners(config, db) if r.job_id == "source:github"] + return runners[0] if runners else None + + async def test_hosted_mode_drains_the_buffer_with_the_github_guardrail(self, db): + config = NerveConfig() + config.sync.github = hosted_github(**GITHUB_DENY, drop_hook=["/bin/true"]) + + runner = self.runner(config, db) + + assert isinstance(runner.source, HostedGitHubSource) + assert runner.source.source_name == "github" + assert [rule.field for rule in runner.inbox_filter.rules if rule.active] == ["repo_name", "reason"] + assert runner.drop_hook == ["/bin/true"] + + async def test_poll_mode_is_unchanged(self, db): + runner = self.runner(NerveConfig(), db) + + assert isinstance(runner.source, GitHubSource) + + async def test_a_switched_off_source_has_no_runner(self, db): + config = NerveConfig() + config.sync.github = hosted_github(enabled=False) + + assert self.runner(config, db) is None + + +@pytest.mark.asyncio +class TestPollingCursor: + @staticmethod + def listings(monkeypatch) -> list[str]: + """The notification listings that the polling source asks gh for.""" + seen: list[str] = [] + + class _Proc: + returncode = 0 + + async def communicate(self): + return b"", b"" + + async def fake_exec(*argv, **kwargs): + seen.append(argv[2]) + return _Proc() + + monkeypatch.setattr("nerve.sources.github.asyncio.create_subprocess_exec", fake_exec) + return seen + + async def test_a_cursor_that_is_no_timestamp_looks_back_a_day(self, monkeypatch): + seen = self.listings(monkeypatch) + + result = await GitHubSource().fetch("57") + + assert result.next_cursor == "57" + assert seen and "since=57" not in seen[0] and "since=20" in seen[0] + + @pytest.mark.parametrize("failure", [RuntimeError("gh failed"), asyncio.CancelledError()]) + async def test_a_failed_or_cancelled_enrichment_gives_the_notification_alone(self, failure): + notification = { + "id": "9000000001", "reason": "mention", "unread": True, "updated_at": "2026-09-16T10:00:00Z", + "subject": {"title": "Fix the flaky retry test", "type": "PullRequest"}, + "repository": {"full_name": "owner/repo"}, + } + + record = notification_record(notification, failure) + + assert (record.id, record.summary) == ("9000000001", "[owner/repo] Fix the flaky retry test (mention)") + assert record.metadata["repo_name"] == "owner/repo" and record.metadata["actors"] == [] + + async def test_digits_are_no_timestamp_even_when_they_read_as_a_date(self, monkeypatch): + seen = self.listings(monkeypatch) + a_day_ago = datetime.now(UTC) - timedelta(hours=24, minutes=5) + + await GitHubSource().fetch("20000101") + + since = seen[0].split("since=", 1)[1].split("&", 1)[0] + assert datetime.strptime(since, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=UTC) > a_day_ago + + +# ---------------------------------------------------------------------- # +# End to end over real streams # +# ---------------------------------------------------------------------- # + + +@dataclass +class Harness: + config: NerveConfig + db: Any + router: ChannelRouter + gateway: FakeChannelGateway + runtime: HostedChannelRuntime + server: NerveServer + + async def stream(self, **kwargs: Any) -> FakeStream: + return await self.gateway.open_stream(self.server.stream_url, **kwargs) + + async def acknowledged(self, *inbox_ids: str) -> dict[str, str]: + await self.gateway.wait_for(lambda: all(i in self.gateway.acknowledged for i in inbox_ids)) + return {i: self.gateway.acknowledged[i] for i in inbox_ids} + + async def subscriptions(self, count: int = 1) -> list[dict[str, Any]]: + await self.gateway.wait_for(lambda: len(self.gateway.subscriptions) >= count) + return self.gateway.subscriptions + + def github_runner(self): + (runner,) = [r for r in build_source_runners(self.config, self.db) if r.job_id == "source:github"] + return runner + + +async def start( + tmp_path, db, monkeypatch, *, slack: bool = False, github: bool = True, + config_getter: Callable[[], NerveConfig] | None = None, **settings: Any, +) -> Harness: + """A hosted runtime on real streams. Its config getter returns this config + object unless *config_getter* is given (``get_config`` follows a reload).""" + config = slack_hosted_config(tmp_path) + if not slack: + config.slack = SlackConfig() + if github: + config.sync.github = hosted_github(**settings) + monkeypatch.setattr(config_module, "_config", config) + engine = MagicMock() + engine.db = db + engine.sessions = SessionManager(db) + router = ChannelRouter(engine) + engine.router = router + gateway = FakeChannelGateway(jwks_file=config.channels.hosted.gateway_jwks_file) + runtime = HostedChannelRuntime( + config, router, config_getter or (lambda: config), + stream_timing=FAST_STREAM, reader_settings=FAST_READER, db=db, + ) + await runtime.start() + monkeypatch.setattr(gateway_server, "_hosted_channels", runtime) + server = NerveServer(gateway_server.create_app()) + await server.__aenter__() + return Harness(config, db, router, gateway, runtime, server) + + +async def stop(harness: Harness) -> None: + await harness.runtime.stop() + await harness.gateway.close_all() + await harness.server.__aexit__(None, None, None) + + +@pytest_asyncio.fixture +async def hosted(tmp_path, db, monkeypatch): + harness = await start(tmp_path, db, monkeypatch, **GITHUB_DENY) + try: + yield harness + finally: + await stop(harness) + + +@contextlib.asynccontextmanager +async def stalled_through_negotiation(harness: Harness) -> AsyncIterator[FakeStream]: + """A stream that negotiates while Nerve's side of it cannot send, as when + another frame is stuck on the socket. The stall clears on leaving, and the + gateway then advertises its capabilities.""" + stream = await harness.stream(negotiate=False) + await asyncio.wait_for(stream.nerve_negotiation.wait(), EVENT_TIMEOUT) + [nerve_side] = harness.runtime.streams.streams + await nerve_side._send_lock.acquire() + try: + await stream.send("negotiation", gateway_negotiation()) + await eventually(lambda: nerve_side.ready) + yield stream + finally: + nerve_side._send_lock.release() + await stream.advertise() + + +@pytest.mark.asyncio +class TestStreams: + async def test_a_stream_publishes_the_subscriptions_right_after_negotiation(self, hosted): + stream = await hosted.stream() + + [subscriptions] = await hosted.subscriptions() + + assert subscriptions == {"version": "1", "sources": [{ + "provider": "github", + "filters": {"deny_repos": ["owner/noisy"], "deny_reasons": ["subscribed"]}, + "pr_watch": True, + }]} + await hosted.gateway.wait_for(lambda: bool(stream.sent("inbox_read"))) + kinds = [frame["kind"] for frame in stream.frames if frame["kind"] != "heartbeat"] + assert kinds[:3] == ["negotiation", "subscriptions", "inbox_read"] + frame = stream.sent("subscriptions")[0] + assert frame["request_id"].startswith("o-") and "connection_id" not in frame + + async def test_a_record_is_buffered_then_drained_into_the_inbox(self, hosted): + stream = await hosted.stream() + inbox_id = hosted.gateway.store(source_record_event()) + + await stream.nudge("observe") + + assert (await hosted.acknowledged(inbox_id))[inbox_id] == "accepted" + assert len(await hosted.db.read_channel_observations("github")) == 1 + assert hosted.router.get_channel("github") is None + result = await hosted.github_runner().run() + assert result.records_ingested == 1 + stored = await hosted.db.get_source_message("github", "9000000001") + assert stored["summary"] == "[owner/repo] Fix the flaky retry test (mention)" + assert stored["content"].endswith("@example-agent could you take a look?") + assert stored["metadata"]["actors"] == ["alice", "bob"] + assert stored["metadata"]["unread"] is True + assert stored["metadata"]["ci_branch"] == "" + + async def test_a_changed_record_resurfaces_in_place_of_the_stored_one(self, hosted): + stream = await hosted.stream() + runner = hosted.github_runner() + first = hosted.gateway.store(source_record_event()) + await stream.nudge("observe") + assert (await hosted.acknowledged(first))[first] == "accepted" + assert (await runner.run()).records_ingested == 1 + before = await hosted.db.get_source_max_rowid("github") + + second = hosted.gateway.store(source_record_event( + title="Fix the flaky retry test, take two", updated_at="2026-09-16T11:00:00Z", version=1789556400, + )) + await stream.nudge("observe") + + assert (await hosted.acknowledged(second))[second] == "accepted" + assert (await runner.run()).records_ingested == 1 + stored = await hosted.db.get_source_message("github", "9000000001") + assert stored["summary"] == "[owner/repo] Fix the flaky retry test, take two (mention)" + assert await hosted.db.get_source_max_rowid("github") > before + + async def test_the_inbox_filter_drops_what_the_published_filters_deny(self, hosted): + stream = await hosted.stream() + noisy = hosted.gateway.store(source_record_event(record_id="9000000002", repository="owner/noisy")) + kept = hosted.gateway.store(source_record_event(record_id="9000000003")) + + await stream.nudge("observe") + + assert await hosted.acknowledged(noisy, kept) == {noisy: "accepted", kept: "accepted"} + result = await hosted.github_runner().run() + assert (result.records_ingested, result.records_dropped) == (1, 1) + assert await hosted.db.get_source_message("github", "9000000002") is None + + async def test_a_pull_request_record_meets_the_repo_rule_only(self, tmp_path, db, monkeypatch): + harness = await start( + tmp_path, db, monkeypatch, allow_actors=["someone-else"], allow_reasons=["mention"], + ) + try: + stream = await harness.stream() + pull_request = harness.gateway.store(source_record_event( + record_id="pr:owner/repo#12", record_type="github_pr_activity", reason="author", + )) + notification = harness.gateway.store(source_record_event(record_id="9000000004")) + + await stream.nudge("observe") + + await harness.acknowledged(pull_request, notification) + result = await harness.github_runner().run() + assert (result.records_ingested, result.records_dropped) == (1, 1) + stored = await db.get_source_message("github", "pr:owner/repo#12") + assert stored["record_type"] == "github_pr_activity" + assert stored["metadata"]["reason"] == "author" + assert await db.get_source_message("github", "9000000004") is None + finally: + await stop(harness) + + async def test_other_github_events_are_rejected_and_settled(self, hosted): + stream = await hosted.stream() + invoke = hosted.gateway.store(source_record_event(purpose="invoke")) + message = message_event(purpose="observe") + message["provider"] = "github" + chat = hosted.gateway.store(message) + + await stream.nudge("observe") + + assert await hosted.acknowledged(invoke, chat) == {invoke: "rejected", chat: "rejected"} + assert {item["reason_code"] for ack in hosted.gateway.acks for item in ack} == {"admission_rejected"} + + async def test_a_failed_buffer_write_leaves_the_row_for_a_later_read(self, hosted, monkeypatch): + calls = 0 + original = hosted.db.insert_channel_observation + + async def failing_once(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError("database is locked") + return await original(*args, **kwargs) + + monkeypatch.setattr(hosted.db, "insert_channel_observation", failing_once) + stream = await hosted.stream() + inbox_id = hosted.gateway.store(source_record_event()) + + await stream.nudge("observe") + + assert (await hosted.acknowledged(inbox_id))[inbox_id] == "accepted" + assert calls == 2 + + async def test_a_reload_publishes_the_changed_settings_on_every_stream(self, hosted): + await hosted.stream() + await hosted.stream() + await hosted.subscriptions(2) + hosted.config.sync.github.deny_reasons = ["subscribed", "comment"] + hosted.config.sync.github.pr_watch = False + engine = SimpleNamespace(get_channel_runtime={"hosted": hosted.runtime}.get) + + outcome = await config_reload._republish_hosted_subscriptions(engine) + + assert outcome == "published on 2 stream(s)" + latest = (await hosted.subscriptions(4))[-1] + assert latest["sources"][0]["filters"]["deny_reasons"] == ["subscribed", "comment"] + assert "pr_watch" not in latest["sources"][0] + + async def test_a_config_reload_publishes_the_reloaded_settings(self, tmp_path, db, monkeypatch): + harness = await start( + tmp_path, db, monkeypatch, config_getter=config_module.get_config, **GITHUB_DENY, + ) + try: + await harness.stream() + await harness.subscriptions(1) + reloaded = copy.deepcopy(harness.config) + reloaded.sync.github.deny_reasons = ["subscribed", "comment"] + reloaded.sync.github.pr_watch = False + monkeypatch.setattr(config_module, "load_config", lambda config_dir=None: reloaded) + engine = SimpleNamespace( + get_channel_runtime={"hosted": harness.runtime}.get, + reload_mcp_config=AsyncMock(return_value=[]), + ) + + summary = await config_reload.reload_all(engine, None, tmp_path) + + assert summary["subscriptions"] == "published on 1 stream(s)" + assert config_reload.reload_failures(summary) == {} + latest = (await harness.subscriptions(2))[-1] + assert latest["sources"][0]["filters"]["deny_reasons"] == ["subscribed", "comment"] + assert "pr_watch" not in latest["sources"][0] + finally: + await stop(harness) + + async def test_a_stalled_stream_does_not_hold_up_a_republish(self, hosted, monkeypatch, caplog): + monkeypatch.setattr("nerve.channels.hosted.stream._SUBSCRIPTIONS_SEND_SECONDS", 0.2) + await hosted.stream() + await hosted.stream() + await hosted.subscriptions(2) + stalled = hosted.runtime.streams.streams[0] + caplog.set_level(logging.WARNING, logger="nerve.channels.hosted.stream") + await stalled._send_lock.acquire() + try: + outcome = await asyncio.wait_for(hosted.runtime.republish_subscriptions(), timeout=3) + finally: + stalled._send_lock.release() + + assert outcome == "published on 1 stream(s)" + assert ( + f"Channel stream {stalled.id}: source subscriptions not sent: the stream could not " + "send them within 0.2s" + ) in caplog.text + + async def test_a_stall_right_after_negotiation_delays_the_subscriptions_but_never_drops_them( + self, hosted, monkeypatch, caplog, + ): + """Nothing waits for the publish that negotiation starts, and nothing + sends the frame again before a reload, so a stall longer than a + republish's bound still ends with the frame sent, once.""" + monkeypatch.setattr("nerve.channels.hosted.stream._SUBSCRIPTIONS_SEND_SECONDS", 0.2) + caplog.set_level(logging.WARNING, logger="nerve.channels.hosted.stream") + + async with stalled_through_negotiation(hosted) as stream: + await asyncio.sleep(0.6) + assert stream.sent("subscriptions") == [] + + await hosted.gateway.wait_for(lambda: bool(stream.sent("inbox_read"))) + await asyncio.sleep(0.3) + kinds = [frame["kind"] for frame in stream.frames if frame["kind"] != "heartbeat"] + assert kinds[:3] == ["negotiation", "subscriptions", "inbox_read"] + assert len(stream.sent("subscriptions")) == 1 + assert hosted.gateway.subscriptions[0]["sources"][0]["filters"] == GITHUB_DENY + assert stream.close_code is None + assert "source subscriptions not sent" not in caplog.text + + async def test_subscriptions_that_waited_through_a_stall_carry_the_settings_of_when_they_left( + self, hosted, monkeypatch, + ): + """A reload during the stall gives up on the stream after its bound, so + the frame that waited is the one the gateway gets: it is built when it + leaves, from the reloaded settings.""" + monkeypatch.setattr("nerve.channels.hosted.stream._SUBSCRIPTIONS_SEND_SECONDS", 0.2) + + async with stalled_through_negotiation(hosted) as stream: + hosted.config.sync.github.deny_reasons = ["subscribed", "comment"] + outcome = await asyncio.wait_for(hosted.runtime.republish_subscriptions(), timeout=3) + assert outcome == "published on 0 stream(s)" + + await hosted.gateway.wait_for(lambda: bool(stream.sent("inbox_read"))) + [frame] = stream.sent("subscriptions") + filters = frame["payload"]["subscriptions"]["sources"][0]["filters"] + assert filters["deny_reasons"] == ["subscribed", "comment"] + + async def test_a_stream_that_fails_to_publish_is_logged_and_the_others_still_send( + self, hosted, monkeypatch, caplog, + ): + await hosted.stream() + await hosted.stream() + await hosted.subscriptions(2) + failing = hosted.runtime.streams.streams[0] + + async def broken() -> bool: + raise RuntimeError("socket gone") + + monkeypatch.setattr(failing, "publish_subscriptions", broken) + caplog.set_level(logging.ERROR, logger="nerve.channels.hosted.manager") + + outcome = await hosted.runtime.republish_subscriptions() + + assert outcome == "published on 1 stream(s)" + assert f"Channel stream {failing.id}: publishing the source subscriptions failed" in caplog.text + + async def test_a_publish_after_negotiation_that_fails_is_logged_at_once(self, hosted, monkeypatch, caplog): + class Unchecked: + def validate(self) -> None: + raise RuntimeError("cannot check") + + monkeypatch.setattr( + "nerve.channels.hosted.runtime.subscriptions_for", lambda config, providers: Unchecked(), + ) + caplog.set_level(logging.ERROR, logger="nerve.channels.hosted.stream") + + stream = await hosted.stream() + + await eventually(lambda: "publishing the source subscriptions failed" in caplog.text) + assert stream.sent("subscriptions") == [] + assert stream.close_code is None + + async def test_subscriptions_above_the_gateways_frame_limit_go_without_patterns(self, hosted): + hosted.config.sync.github.allow_repos = [f"owner/{'r' * 200}-{n}" for n in range(100)] + stream = await hosted.stream(negotiate=False) + await stream.negotiate(frame_bytes=16384) + + [subscriptions] = await hosted.subscriptions() + + assert subscriptions["sources"] == [{"provider": "github", "filters": {}, "pr_watch": True}] + assert stream.close_code is None + + async def test_subscriptions_refused_even_without_patterns_are_not_sent(self, hosted, monkeypatch, caplog): + refused = Subscriptions(version="1", sources=( + SourceSubscription(provider="github", filters=SourceFilters(allow_repos=("",))), + SourceSubscription(provider="github"), + )) + monkeypatch.setattr( + "nerve.channels.hosted.runtime.subscriptions_for", lambda config, providers: refused, + ) + caplog.set_level(logging.WARNING, logger="nerve.channels.hosted.stream") + + stream = await hosted.stream() + await hosted.gateway.wait_for(lambda: bool(stream.sent("inbox_read"))) + + assert stream.sent("subscriptions") == [] + assert stream.close_code is None + assert "source subscriptions not sent: the gateway would refuse them" in caplog.text + + async def test_a_switched_off_source_subscribes_to_nothing_and_keeps_what_arrives(self, hosted): + hosted.config.sync.github.enabled = False + stream = await hosted.stream() + inbox_id = hosted.gateway.store(source_record_event()) + + [subscriptions] = await hosted.subscriptions() + await stream.nudge("observe") + + assert subscriptions == {"version": "1"} + assert (await hosted.acknowledged(inbox_id))[inbox_id] == "accepted" + assert len(await hosted.db.read_channel_observations("github")) == 1 + + +@pytest.mark.asyncio +class TestSharedStreams: + async def test_slack_and_github_share_the_streams_and_the_reader(self, tmp_path, db, monkeypatch): + harness = await start(tmp_path, db, monkeypatch, slack=True) + try: + assert set(harness.runtime.channels) == {"slack"} + assert set(harness.runtime.sources) == {"github"} + assert harness.router.get_channel("github") is None + stream = await harness.stream() + observed = harness.gateway.store(message_event( + purpose="observe", conversation="C_FIXTURE_OBSERVATION", message_id="1700002000.000100", + )) + record = harness.gateway.store(source_record_event()) + + await stream.nudge("observe") + + assert await harness.acknowledged(observed, record) == {observed: "accepted", record: "accepted"} + assert len(await db.read_channel_observations("slack")) == 1 + assert len(await db.read_channel_observations("github")) == 1 + assert len(await harness.subscriptions()) == 1 + finally: + await stop(harness) + + async def test_a_switched_off_hosted_slack_holds_the_github_records_back(self, tmp_path, db, monkeypatch): + """One ordered inbox for every provider, which Nerve stops reading + while a channel cannot take events: GitHub records wait while hosted + Slack is off and arrive once it is on again.""" + harness = await start(tmp_path, db, monkeypatch, slack=True) + try: + harness.config.slack.enabled = False + stream = await harness.stream() + record = harness.gateway.store(source_record_event()) + + await stream.nudge("observe") + await asyncio.sleep(0.5) + + assert record not in harness.gateway.acknowledged + assert await db.read_channel_observations("github") == [] + harness.config.slack.enabled = True + assert (await harness.acknowledged(record))[record] == "accepted" + assert len(await db.read_channel_observations("github")) == 1 + finally: + await stop(harness) + + async def test_without_hosted_sources_no_subscriptions_are_sent(self, tmp_path, db, monkeypatch): + harness = await start(tmp_path, db, monkeypatch, slack=True, github=False) + try: + stream = await harness.stream() + await harness.gateway.wait_for(lambda: bool(stream.sent("inbox_read"))) + + assert harness.runtime.subscriptions() is None + assert stream.sent("subscriptions") == [] + assert await harness.runtime.republish_subscriptions() is None + finally: + await stop(harness) + + async def test_a_runtime_without_any_hosted_provider_is_refused(self, tmp_path, db): + config = slack_hosted_config(tmp_path) + config.slack = SlackConfig() + + with pytest.raises(ValueError, match="no provider is in hosted mode"): + HostedChannelRuntime(config, ChannelRouter(MagicMock()), lambda: config, db=db) + + +# ---------------------------------------------------------------------- # +# Jobs that a delivery runs # +# ---------------------------------------------------------------------- # + + +def write_jobs(path: Path, jobs: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump({"jobs": jobs}), encoding="utf-8") + + +@pytest.mark.asyncio +class TestRunsOnDelivery: + async def test_a_delivery_runs_the_gated_job_once_and_not_the_others(self, tmp_path, db, monkeypatch): + harness = await start(tmp_path, db, monkeypatch) + cron_dir = tmp_path / "cron" + harness.config.cron.jobs_file = cron_dir / "jobs.yaml" + harness.config.cron.system_file = cron_dir / "system.yaml" + harness.config.cron.gate_plugins_dir = cron_dir / "gates" + harness.config.cron.delivery_debounce_seconds = 1 + never = "0 0 1 1 *" + write_jobs(harness.config.cron.jobs_file, [ + {"id": "pr-monitor", "schedule": never, "prompt": "Look at the PRs", + "run_if": [{"type": "messages", "sources": ["github"]}]}, + {"id": "mail-triage", "schedule": never, "prompt": "Look at the mail", + "run_if": [{"type": "messages", "sources": ["gmail"]}]}, + {"id": "daily-briefing", "schedule": never, "prompt": "Brief me"}, + ]) + turns: list[str] = [] + + async def run_cron(**kwargs: Any): + turns.append(kwargs["job_id"]) + return SimpleNamespace(text="done", outcome=SimpleNamespace(ok=True, describe=lambda: "")) + + engine = MagicMock() + engine.run_cron = run_cron + cron = CronService(harness.config, engine, db) + await cron.start() + harness.runtime.set_delivery_listener(cron.on_source_delivered) + try: + # The consumer cursor exists before the records arrive, as it does for a + # job that has polled before. + await db.get_consumer_cursor("inbox", "github") + stream = await harness.stream() + inbox_ids = [ + harness.gateway.store(source_record_event(record_id=str(9000000010 + n))) + for n in range(3) + ] + + await stream.nudge("observe") + + await harness.acknowledged(*inbox_ids) + await eventually(lambda: turns == ["pr-monitor"]) + assert await db.get_source_message("github", "9000000012") is not None + await asyncio.sleep(1.5) + assert turns == ["pr-monitor"] + logs = await db.get_cron_logs(job_id="source:github", limit=10) + assert logs and all(log["status"] == "success" for log in logs) + finally: + await cron.stop() + await stop(harness) + + +# ---------------------------------------------------------------------- # +# The pull request gate, doctor, and the setup page # +# ---------------------------------------------------------------------- # + + +def recorded_gh(monkeypatch) -> list[tuple[str, ...]]: + """The processes the gate starts. Each fails, so the gate fails open, and + only the record tells whether gh ran.""" + calls: list[tuple[str, ...]] = [] + + async def failing(*args, **kwargs): + calls.append(args) + raise FileNotFoundError("gh") + + monkeypatch.setattr("nerve.cron.gates.asyncio.create_subprocess_exec", failing) + return calls + + +def github_config(mode: str) -> NerveConfig: + config = NerveConfig() + config.sync.github = GitHubSyncConfig(mode=mode) + return config + + +@pytest.mark.asyncio +class TestPullRequestGate: + async def test_in_hosted_mode_it_passes_without_gh(self, db, monkeypatch): + monkeypatch.setattr(config_module, "_config", github_config("hosted")) + calls = recorded_gh(monkeypatch) + gate = GitHubPrActivityGate(author="example-agent") + + assert await gate.is_satisfied(GateContext(job_id="pr-monitor", db=db)) is True + assert calls == [] + + async def test_in_poll_mode_it_still_runs_gh(self, db, monkeypatch): + monkeypatch.setattr(config_module, "_config", NerveConfig()) + calls = recorded_gh(monkeypatch) + + assert await GitHubPrActivityGate(author="example-agent").is_satisfied( + GateContext(job_id="pr-monitor", db=db), + ) is True + assert calls and calls[0][0] == "gh" + + @pytest.mark.parametrize(("started", "reloaded", "runs_gh"), [ + ("poll", "hosted", True), + ("hosted", "poll", False), + ]) + async def test_it_keeps_the_mode_the_process_started_with( + self, db, monkeypatch, started, reloaded, runs_gh, + ): + """The mode is restart-only: after a reload that changes it, the + source still runs in the old mode until the restart, and so does the + gate.""" + config_reload.remember_startup_config(github_config(started)) + monkeypatch.setattr(config_module, "_config", github_config(reloaded)) + calls = recorded_gh(monkeypatch) + + assert await GitHubPrActivityGate(author="example-agent").is_satisfied( + GateContext(job_id="pr-monitor", db=db), + ) is True + assert bool(calls) is runs_gh + + +def doctor_config(tmp_path: Path, **github: Any) -> NerveConfig: + gateway = FakeChannelGateway(jwks_file=tmp_path / "gateway-jwks.json") + return NerveConfig.from_dict({ + "sync": {"github": {"mode": "hosted", **github}}, + "channels": {"hosted": {**HOSTED, "gateway_jwks_file": str(gateway.jwks_file)}}, + }) + + +class TestDoctor: + def test_hosted_github_is_reported(self, tmp_path): + report = doctor_report(doctor_config(tmp_path, pr_watch=False)) + + assert ( + "[OK] GitHub notifications come from the channel gateway: filters published " + "from sync.github, PR watch off" + ) in report + + def test_unusable_hosted_settings_are_errors(self, tmp_path): + config = doctor_config(tmp_path) + config.channels.hosted.gateway_jwks_file = tmp_path / "missing.json" + + report = doctor_report(config) + + assert "[ERR] GitHub is hosted but channels.hosted.gateway_jwks_file: cannot read" in report + + def test_a_switched_off_hosted_source_says_so(self, tmp_path): + report = doctor_report(doctor_config(tmp_path, enabled=False)) + + assert "[--] GitHub is hosted but switched off (sync.github.enabled is false)" in report + + @pytest.mark.parametrize(("slack", "github", "warned"), [ + ({"mode": "hosted", "enabled": False}, {}, True), + ({"mode": "hosted"}, {}, False), + ({"mode": "socket", "enabled": False}, {}, False), + ({"mode": "hosted", "enabled": False}, {"enabled": False}, False), + ({"mode": "hosted", "enabled": False}, {"mode": "poll"}, False), + ]) + def test_a_switched_off_hosted_slack_is_said_to_hold_github_back(self, tmp_path, slack, github, warned): + config = doctor_config(tmp_path, **github) + config.slack = SlackConfig.from_dict(slack) + + report = doctor_report(config) + + assert ("[WARN] Slack is hosted but switched off (slack.enabled is false)" in report) is warned + + def test_jobs_on_the_pull_request_gate_are_named(self, tmp_path): + config = doctor_config(tmp_path) + config.cron.jobs_file = tmp_path / "cron" / "jobs.yaml" + config.cron.system_file = tmp_path / "cron" / "system.yaml" + write_jobs(config.cron.jobs_file, [ + {"id": "pr-monitor", "schedule": "30m", "prompt": "PRs", + "run_if": [{"type": "github_pr_activity", "author": "example-agent"}]}, + {"id": "inbox", "schedule": "30m", "prompt": "Inbox", + "run_if": [{"type": "messages", "sources": ["github"]}]}, + ]) + + report = doctor_report(config) + + assert "[WARN] Cron job(s) pr-monitor use the github_pr_activity gate" in report + assert "inbox use the" not in report + + @pytest.mark.parametrize(("mode", "expected"), [ + ("hosted", "[--] gh CLI not found (not needed: GitHub comes from the channel gateway)"), + ("poll", "[WARN] gh CLI not found (needed for sync)"), + ]) + def test_gh_is_asked_for_only_when_something_runs_it(self, tmp_path, monkeypatch, mode, expected): + import shutil + + monkeypatch.setattr(shutil, "which", lambda name: None) + config = doctor_config(tmp_path) + config.sync.github.mode = mode + + assert expected in doctor_report(config) + + +@pytest.mark.asyncio +class TestSetupPage: + async def test_the_gateway_status_is_reported_when_only_github_is_hosted(self, monkeypatch): + from nerve.gateway.routes.setup import _hosted_gateway_status + + config = NerveConfig() + assert _hosted_gateway_status(config) is None + config.sync.github = hosted_github() + monkeypatch.setattr(gateway_server, "_hosted_channels", None) + + assert _hosted_gateway_status(config) == {"streams": 0, "connected": False, "last_event_at": None} + + async def test_the_sources_preflight_does_not_run_gh_in_hosted_mode(self, monkeypatch): + import subprocess + + from nerve import setup_checks + + setup_checks.reset_caches() + + def no_gh(argv, **kwargs): + raise AssertionError(f"ran {argv}") + + monkeypatch.setattr(subprocess, "run", no_gh) + monkeypatch.setattr(setup_checks.shutil, "which", lambda name: None) + config = NerveConfig() + config.sync.github = hosted_github() + try: + result = await setup_checks.sources_preflight(config) + finally: + setup_checks.reset_caches() + + assert result["github"] == { + "cli": False, "authenticated": False, "login": None, + "hint": setup_checks.GH_HOSTED_HINT, "hosted": True, + } + + async def test_the_environment_check_does_not_ask_for_gh_in_hosted_mode(self, monkeypatch): + from nerve import setup_checks + + setup_checks.reset_caches() + monkeypatch.setattr(setup_checks, "_version_of", lambda argv: None) + config = NerveConfig() + config.sync.github = hosted_github() + try: + checks = {check["id"]: check for check in await setup_checks.environment(config)} + config.sync.github.mode = "poll" + setup_checks.reset_caches() + polling = {check["id"]: check for check in await setup_checks.environment(config)} + finally: + setup_checks.reset_caches() + + assert (checks["gh"]["status"], checks["gh"]["detail"]) == ("ok", "not needed") + assert polling["gh"]["status"] == "missing" diff --git a/tests/test_setup_api.py b/tests/test_setup_api.py index 94e25d550..1d177799f 100644 --- a/tests/test_setup_api.py +++ b/tests/test_setup_api.py @@ -3287,6 +3287,7 @@ async def test_hosted_slack_reports_the_gateway_and_counts_as_done(self, claimed _NO_SECRETS = "This instance does not store credentials entered here. Connections are provided by the operator of this deployment." _GITHUB_ONLY = "GitHub is provided by the operator of this deployment; Gmail and Telegram are not available on this instance." +_GITHUB_HOSTED = "GitHub comes from the channel gateway of this deployment; Gmail and Telegram are not available on this instance." def _set_local_secrets(install: Install, allowed: bool, **settings) -> None: @@ -3490,6 +3491,34 @@ async def test_a_github_token_from_the_environment_opens_that_one_toggle(self, c assert (claimed.config_dir / "config.local.yaml").read_bytes() == local_before assert "ghp_REPLACE" not in json.dumps(off.json()) + async def test_a_hosted_github_opens_that_one_toggle_and_keeps_the_events_source_off(self, claimed): + """The channel gateway polls GitHub, so the switch needs no credential + here. The events source reads with gh on this machine, which a hosted + GitHub does not set up, so the switch turns it off whichever way it + goes, also when an earlier write had turned it on.""" + choose(claimed, "personal") + _keep_no_secrets(claimed, **{"sync.github.mode": "hosted", "sync.github_events.enabled": True}) + async with claimed.client() as http: + sources = step((await http.get("/api/setup")).json(), "sources") + assert sources["status"] == "pending" and sources["read_only"] is False + assert sources["detail"]["available"] == ["github"] + assert sources["detail"]["unavailable_reason"] == _GITHUB_HOSTED + refused = await http.put("/api/setup/sources", json={"github": True, "gmail": False}) + assert refused.status_code == 409 and refused.json() == {"detail": _GITHUB_HOSTED, "field": "gmail"} + on = await http.put("/api/setup/sources", json={"github": True}) + assert on.status_code == 200, on.text + sources = step(on.json(), "sources") + assert sources["status"] == "done" and sources["read_only"] is False + assert sources["summary"] == "GitHub from the channel gateway · waiting for the gateway" + settings = claimed.settings_yaml()["sync"] + assert settings["github"] == {"enabled": True, "mode": "hosted"} + assert settings["github_events"]["enabled"] is False + off = await http.put("/api/setup/sources", json={"github": False}) + assert off.status_code == 200, off.text + settings = claimed.settings_yaml()["sync"] + assert settings["github"] == {"enabled": False, "mode": "hosted"} + assert settings["github_events"]["enabled"] is False + async def test_the_github_toggle_waits_for_a_mode_and_can_be_skipped(self, claimed, monkeypatch): """Open, the step behaves like any other: it waits for the Mode step (while the closed fields still answer with the policy) and may be diff --git a/tests/test_setup_checklist.py b/tests/test_setup_checklist.py index cd21abbb6..44d34646b 100644 --- a/tests/test_setup_checklist.py +++ b/tests/test_setup_checklist.py @@ -237,6 +237,24 @@ def test_sources(self, personal): # Where the page may store credentials every source is on offer. assert d["available"] == ["github", "gmail", "telegram"] and d["unavailable_reason"] is None + def test_sources_show_the_gateway_when_only_github_is_hosted(self, personal): + """As the Channels step does for a hosted Slack; volatile, so outside + the etag.""" + hosted = cfg(personal, sync={"github": {"enabled": True, "mode": "hosted"}}) + live = {"streams": 1, "connected": True, "last_event_at": "2026-10-21T09:00:00+00:00"} + + connected = step(facts(personal, hosted, hosted_gateway=live), "sources") + waiting = step(facts(personal, hosted), "sources") + + assert connected["detail"]["github"] == { + "enabled": True, "source": None, "mode": "hosted", "gateway": live, + } + assert connected["summary"] == "GitHub from the channel gateway · connected" + assert waiting["detail"]["github"]["gateway"] == {"streams": 0, "connected": False, "last_event_at": None} + assert waiting["summary"] == "GitHub from the channel gateway · waiting for the gateway" + assert connected["etag"] == waiting["etag"] + assert step(facts(personal, hosted), "channels")["detail"]["slack"]["gateway"] is None + def test_automation(self, personal): s = step(facts(personal), "automation") assert s["status"] == "pending" @@ -584,6 +602,19 @@ def test_a_github_token_from_the_environment_keeps_that_toggle_open(self, person assert step(skipped, "sources")["status"] == "skipped" assert checklist.summarize(skipped)["steps_left"] == 3 + def test_a_hosted_github_keeps_that_toggle_open_without_a_token(self, personal): + """The channel gateway polls GitHub, so the switch needs no credential + on this machine and stores nothing here.""" + f = facts(personal, keyless(personal, sync={"github": {"mode": "hosted"}})) + s = step(f, "sources") + assert s["status"] == "pending" and s["blocked_kind"] is None + assert s["read_only"] is False and s["read_only_reason"] is None + assert s["detail"]["available"] == ["github"] + assert s["detail"]["unavailable_reason"] == checklist.SOURCES_GITHUB_HOSTED_TEXT + on = keyless(personal, sync={"github": {"enabled": True, "mode": "hosted"}}) + done = step(facts(personal, on), "sources") + assert done["status"] == "done" and done["read_only"] is False and done["read_only_reason"] is None + def test_github_on_with_a_token_from_the_environment_is_done_and_keeps_its_switch(self, personal): """Done like any switched-on source, but not read-only: the switch is the one control, as the hosted Slack's is.""" diff --git a/tests/test_setup_writer.py b/tests/test_setup_writer.py index a78a0e499..10e66392e 100644 --- a/tests/test_setup_writer.py +++ b/tests/test_setup_writer.py @@ -175,14 +175,21 @@ def run_writers(root: Path, choices: SetupChoices) -> dict[str, str]: return produced -def run_unattended(root: Path, mode: str, extra: dict[str, str]) -> dict[str, str]: +def run_unattended( + root: Path, mode: str, extra: dict[str, str], *, on_path: tuple[str, ...] = (), +) -> dict[str, str]: """``run_non_interactive`` under a deterministic environment. - No CLIs on PATH (so no source is auto-enabled), no web build, a fixed - credential and timezone. The extra files are the workspace files whose - placeholders the unattended path must leave exactly as the template has - them (only the terminal wizard, given a name, fills them in). + No CLIs on PATH but those *on_path* names (so by default no source is + auto-enabled), no web build, a fixed credential and timezone. The extra + files are the workspace files whose placeholders the unattended path must + leave exactly as the template has them (only the terminal wizard, given a + name, fills them in). """ + + def which(name: str, *args: object, **kwargs: object) -> str | None: + return f"/usr/bin/{name}" if name in on_path else None + config_dir = root / "config" config_dir.mkdir(parents=True, exist_ok=True) env = { @@ -199,7 +206,7 @@ def run_unattended(root: Path, mode: str, extra: dict[str, str]) -> dict[str, st "NERVE_TASK": "Keep the build green." if mode == "worker" else "", }) with mock.patch.dict(os.environ, env, clear=True), \ - mock.patch("nerve.bootstrap.shutil.which", return_value=None), \ + mock.patch("nerve.bootstrap.shutil.which", side_effect=which), \ mock.patch.object(SetupWizard, "_build_web_ui", lambda self: None): run_non_interactive(config_dir) produced = {} @@ -1019,3 +1026,63 @@ def test_worker_default_crons_leave_task_planner_available_but_off(self) -> None "memory-maintenance": True, "task-planner": False, "skill-extractor": True, "skill-reviser": True, } + + +class TestHostedGitHubKeepsTheEventsSourceOff: + """The events source reads with gh on this machine, which a GitHub that + the channel gateway polls does not set up. Wherever the writers derive it + from the GitHub switch, it stays off while GitHub is hosted.""" + + @pytest.mark.parametrize(("mode", "events"), [(None, True), ("poll", True), ("hosted", False)]) + def test_the_layers_follow_the_switch_only_while_github_is_polled_here(self, mode, events) -> None: + _machine, portable, _shadowed = setup_writer.build_config_layers( + SetupChoices(mode="personal", github_sync=True, github_mode=mode), + ) + assert portable["sync"]["github"] == {"enabled": True} + assert portable["sync"]["github_events"] == {"enabled": events} + + @pytest.mark.parametrize(("settings", "events"), [ + ("", True), + ("sync:\n github:\n mode: poll\n", True), + ("sync:\n github:\n mode: hosted\n", False), + ("sync:\n github:\n mode: ' Hosted '\n", False), + ("sync:\n github:\n mode: elsewhere\n", True), + ]) + def test_the_installer_takes_the_mode_from_the_settings_it_writes( + self, tmp_path: Path, settings: str, events: bool, + ) -> None: + workspace = tmp_path / "workspace" + settings_file = workspace / "config" / "settings.yaml" + settings_file.parent.mkdir(parents=True) + settings_file.write_text(settings, encoding="utf-8") + + run_writers(tmp_path, personal_choices(workspace)) + + sync = yaml.safe_load(settings_file.read_text(encoding="utf-8"))["sync"] + assert sync["github"]["enabled"] is True + assert sync["github_events"] == {"enabled": events} + + def test_unattended_init_with_gh_on_the_path_keeps_it_off_over_hosted_settings( + self, tmp_path: Path, + ) -> None: + """The unattended installer switches GitHub on whenever gh is on the + PATH; over settings that say ``hosted`` the events source stays off.""" + settings_file = tmp_path / "workspace" / "config" / "settings.yaml" + settings_file.parent.mkdir(parents=True) + settings_file.write_text("sync:\n github:\n mode: hosted\n", encoding="utf-8") + + produced = run_unattended(tmp_path, "personal", {}, on_path=("gh",)) + + sync = yaml.safe_load(produced["settings.yaml"])["sync"] + assert sync["github"] == {"mode": "hosted", "enabled": True} + assert sync["github_events"] == {"enabled": False} + + def test_the_web_steps_carry_the_running_mode(self, tmp_path: Path) -> None: + hosted = choices_from_config(NerveConfig.from_dict({ + "workspace": str(tmp_path / "ws"), "sync": {"github": {"enabled": True, "mode": "hosted"}}, + })) + assert hosted.github_mode == "hosted" + _machine, portable, _shadowed = setup_writer.build_config_layers(hosted) + assert portable["sync"]["github_events"] == {"enabled": False} + polled = choices_from_config(NerveConfig.from_dict({"workspace": str(tmp_path / "ws")})) + assert polled.github_mode == "poll" diff --git a/tests/test_source_filters.py b/tests/test_source_filters.py index 29d9d04ea..eb4c4763f 100644 --- a/tests/test_source_filters.py +++ b/tests/test_source_filters.py @@ -9,8 +9,10 @@ import pytest +from nerve.config import GitHubSyncConfig from nerve.sources.base import Source from nerve.sources.filters import FieldRule, InboxFilter +from nerve.sources.github import github_inbox_filter from nerve.sources.models import FetchResult, SourceRecord from nerve.sources.runner import SourceRunner @@ -103,6 +105,33 @@ def test_special_source_and_record_type_fields(): # InboxFilter aggregation # --------------------------------------------------------------------------- +# The glob dialect that the hosted GitHub gateway matches with too: Python +# fnmatch on lower-cased values, where "*" also crosses "/", "?" is one +# character, [...] is a class, and there is neither "**" nor escaping. Each +# pattern admits its value through an allow list, and drops it through a deny +# list, exactly when it matches. +@pytest.mark.parametrize(("pattern", "value", "matched"), [ + ("*nerve*", "OWNER/Nerve-Tools", True), + ("owner*", "owner/repo", True), + ("owner/*", "owner/sub/repo", True), + ("*/repo", "a/b/repo", True), + ("owner/rep?", "owner/repo", True), + ("owner/rep?", "owner/repository", False), + ("[ab]*/x", "B/x", True), + ("[!a]*/x", "a/x", False), + ("exact/name", "exact/name2", False), +]) +def test_the_glob_vectors_shared_with_the_gateway(pattern, value, matched): + assert FieldRule(field="repo_name", allow=[pattern]).passes(_rec(value)) is matched + assert FieldRule(field="repo_name", deny=[pattern]).passes(_rec(value)) is not matched + + +def test_the_list_vector_shared_with_the_gateway(): + record = _rec(actors=["alice", "Codecov"]) + assert FieldRule(field="actors", allow=["codecov"]).passes(record) is True + assert FieldRule(field="actors", deny=["codecov"]).passes(record) is False + + def test_inbox_filter_inactive_partition_is_passthrough(): flt = InboxFilter() records = [_rec("a/b"), _rec("c/d")] @@ -271,3 +300,61 @@ def test_rejects_names_the_first_failing_rule(): # Rules are evaluated in order, so the repo rule is reported first. by_repo = flt.rejects(_rec("other/repo", ci_branch="main")) assert by_repo is not None and by_repo.field == "repo_name" + + +def test_a_rule_does_not_apply_to_the_record_types_it_exempts(): + rule = FieldRule(field="actors", allow=["alice"], exempt_record_types=("github_pr_activity",)) + exempt = SourceRecord( + id="pr:owner/repo#12", source="github", record_type="github_pr_activity", + summary="s", content="c", timestamp="t", metadata={"actors": ["stranger"]}, + ) + assert rule.passes(exempt) is True + assert rule.passes(_rec(actors=["stranger"])) is False + assert rule.passes(_rec(actors=["alice"])) is True + + +# --------------------------------------------------------------------------- +# GitHub: a record about the agent's own pull request meets the repo rule only +# --------------------------------------------------------------------------- + +def _github(record_type: str, *, repo: str = "owner/repo") -> SourceRecord: + """A GitHub record as hosted mode delivers it: the agent is the author, + and the actors are a reviewer and a commenter.""" + return SourceRecord( + id=f"{record_type}:{repo}", source="github", record_type=record_type, + summary=f"[{repo}] Fix the flaky retry test (author)", content="body", + timestamp="2026-01-01T00:00:00Z", + metadata={"repo_name": repo, "reason": "author", "actors": ["alice", "bob"], "ci_branch": ""}, + ) + + +def test_an_actor_allow_list_keeps_a_pull_request_record_but_not_a_notification(): + """The vector both ends of hosted GitHub check, the gateway before it + stores a record and Nerve before the inbox.""" + flt = github_inbox_filter(GitHubSyncConfig(allow_actors=["someone-else"])) + + assert flt.passes(_github("github_pr_activity")) is True + assert flt.passes(_github("github_notification")) is False + + +@pytest.mark.parametrize("settings", [ + {"allow_actors": ["someone-else"]}, + {"deny_actors": ["bob"]}, + {"allow_reasons": ["mention", "review_requested"]}, + {"deny_reasons": ["author"]}, + {"deny_ci_branches": ["*"]}, +]) +def test_only_the_repo_rule_applies_to_a_pull_request_record(settings): + flt = github_inbox_filter(GitHubSyncConfig(**settings)) + + assert flt.passes(_github("github_pr_activity")) is True + assert flt.passes(_github("github_notification")) is False + + +@pytest.mark.parametrize("settings", [{"allow_repos": ["other/*"]}, {"deny_repos": ["owner/*"]}]) +def test_the_repo_rule_still_applies_to_a_pull_request_record(settings): + flt = github_inbox_filter(GitHubSyncConfig(**settings)) + + rejected_by = flt.rejects(_github("github_pr_activity")) + + assert rejected_by is not None and rejected_by.field == "repo_name" diff --git a/web/src/api/client.ts b/web/src/api/client.ts index fc6b8979e..4bffab8c7 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -415,7 +415,8 @@ export interface PairingCode { } export interface SourcesPreflight { - github: { cli: boolean; authenticated: boolean; login: string | null; hint: string | null }; + /** `hosted`: the channel gateway polls GitHub (`sync.github.mode: hosted`), so gh was not checked. */ + github: { cli: boolean; authenticated: boolean; login: string | null; hint: string | null; hosted?: boolean }; gmail: { cli: boolean; hint: string | null }; telegram: { credentials_configured: boolean }; checked_at: string; diff --git a/web/src/components/Setup/SetupWizard.test.tsx b/web/src/components/Setup/SetupWizard.test.tsx index 880649fd4..6710ce165 100644 --- a/web/src/components/Setup/SetupWizard.test.tsx +++ b/web/src/components/Setup/SetupWizard.test.tsx @@ -807,6 +807,24 @@ describe('forms open on server values and send the right payload', () => { expect(screen.getByText(/stays idle until the tool is set up/)).toBeInTheDocument(); }); + it('sources: hosted GitHub needs no gh, says so, and does not warn when enabled', async () => { + m(api.setupSourcesPreflight).mockResolvedValue({ + github: { + cli: false, authenticated: false, login: null, hosted: true, + hint: 'The channel gateway polls GitHub for this agent; gh is not needed on the server.', + }, + gmail: { cli: false, hint: 'Install gog.' }, + telegram: { credentials_configured: false }, + checked_at: '2026-10-10T09:12:03Z', + }); + renderAt('/setup/sources'); + await heading('Choose what Nerve reads'); + expect(await screen.findByText(/gh is not needed on the server/)).toBeInTheDocument(); + await userEvent.click(screen.getByRole('checkbox', { name: /GitHub/ })); + expect(screen.queryByText(/stays idle until the tool is set up/)).not.toBeInTheDocument(); + expect(screen.queryByText(/gh is installed and signed in/)).not.toBeInTheDocument(); + }); + it('agents: nothing is pre-checked, the footer line says what Continue writes, and [] is a decision', async () => { renderAt('/setup/agents'); await heading('Connect coding agents'); diff --git a/web/src/components/Setup/steps/SourcesStep.tsx b/web/src/components/Setup/steps/SourcesStep.tsx index 70467c295..7e66c7381 100644 --- a/web/src/components/Setup/steps/SourcesStep.tsx +++ b/web/src/components/Setup/steps/SourcesStep.tsx @@ -177,7 +177,9 @@ export function SourcesStep({ step, state }: { step: SetupStep; state: SetupStat try { await loadPreflight(); } finally { setRechecking(false); } }; - const githubOk = preflight ? preflight.github.cli && preflight.github.authenticated : null; + // Hosted GitHub is polled by the channel gateway: nothing to install or sign in here. + const githubHosted = preflight?.github.hosted === true; + const githubOk = preflight ? githubHosted || (preflight.github.cli && preflight.github.authenticated) : null; const gmailOk = preflight ? preflight.gmail.cli : null; // A first run turns the inbox job off because nothing was connected; a @@ -231,8 +233,9 @@ export function SourcesStep({ step, state }: { step: SetupStep; state: SetupStat />

{githubOk === null ? 'Checking…' - : githubOk ? `gh is installed and signed in${preflight?.github.login ? ` as ${preflight.github.login}` : ''}` - : preflight?.github.hint ?? 'gh is not signed in on the server'} + : githubHosted ? preflight?.github.hint + : githubOk ? `gh is installed and signed in${preflight?.github.login ? ` as ${preflight.github.login}` : ''}` + : preflight?.github.hint ?? 'gh is not signed in on the server'}

{githubManaged &&

{managedLine('Token', githubManaged)}

} {github && githubOk === false && {SOURCES_IDLE_HINT}}