diff --git a/.agents/skills/bootstrap-diagnostics/SKILL.md b/.agents/skills/bootstrap-diagnostics/SKILL.md index 848f3799839..dbf8d682746 100644 --- a/.agents/skills/bootstrap-diagnostics/SKILL.md +++ b/.agents/skills/bootstrap-diagnostics/SKILL.md @@ -2,7 +2,7 @@ name: bootstrap-diagnostics description: >- Agent-only handling playbook for session-start bootstrap diagnostics. - Use whenever the session-start digest's bootstrap section prints an actionable diagnostic line - MISSING, MISSING_MANUAL, BACKEND_INVALID, NEEDS_GH_AUTH, TANGLE, STARTUP_MEMORY_BUDGET, CREW_DISPATCH invalid, FLEET_SYNC, SECONDMATE_SYNC, SECONDMATE_LIVENESS, SECONDMATE_HANDOFF, NUDGE_SECONDMATES, TREEHOUSE_POOL, or FMX - or when a standalone bin/fm-bootstrap.sh run prints one of those lines. + Use whenever the session-start digest's bootstrap section prints an actionable diagnostic line - MISSING, MISSING_MANUAL, BACKEND_INVALID, NEEDS_GH_AUTH, TANGLE, STARTUP_MEMORY_BUDGET, CREW_DISPATCH invalid, FLEET_SYNC, SECONDMATE_SYNC, SECONDMATE_LIVENESS, SECONDMATE_HANDOFF, NUDGE_SECONDMATES, TREEHOUSE_POOL, FMX, or EXT - or when a standalone bin/fm-bootstrap.sh run prints one of those lines. A silent bootstrap section, or a BOOTSTRAP_INFO fact, means no skill load. user-invocable: false metadata: @@ -48,3 +48,5 @@ When any diagnostic needs captain attention, report the plain consequence and re Report the named path and consequence to the captain, and do not prune, return, reset, or clean the slot without explicit approval. - `FMX: X mode on ...` / `FMX: X mode off ...` - bootstrap confirmed or removed the local X-mode poll artifacts (`docs/configuration.md` "X mode (.env)"). Only when a running watcher needs the cadence transition applied immediately, restart the home-scoped watcher through the emitted harness supervision protocol; bootstrap deliberately never restarts the watcher itself. +- `EXT: local bridge on ...` / `EXT: local bridge off ...` - bootstrap confirmed or removed the local Communication Officer poll shim (`docs/configuration.md` "Local Communication Officer bridge"). + There is no cadence override; intake wakes immediately and the default slow-check interval covers restart recovery. diff --git a/.agents/skills/ext-respond/SKILL.md b/.agents/skills/ext-respond/SKILL.md new file mode 100644 index 00000000000..a6657b02efe --- /dev/null +++ b/.agents/skills/ext-respond/SKILL.md @@ -0,0 +1,90 @@ +--- +name: ext-respond +description: >- + Agent-only playbook for handling local Communication Officer Discord requests. + Use on an "ext-request " check wake to drain the local inbox, classify, + act through the normal lifecycle, emit ack/answer/follow-up/final into the + local outbox, and link spawned work. + Loaded only when the sibling local ext-bridge is enabled. +user-invocable: false +metadata: + internal: true +--- + +# ext-respond + +The local Communication Officer bridge lets a firstmate instance answer `/fm` requests that a dedicated Hermes Gateway plugin delivered into this home. +A request arrives through the watcher as a `check:` wake whose payload is `ext-request `. +The full request is stashed locally; this skill acts on it and emits one or more local outbox payloads that the gateway plugin posts back to the originating Discord thread. + +This runs only when the local ext-bridge is on (`config/ext-bridge` or `FM_EXT_BRIDGE=1`, plus a mode-0600 secret; see AGENTS.md "Local Communication Officer bridge"). +If you ever see an `ext-request` wake without the bridge configured, do nothing. +Do not use `FMX_PAIRING_TOKEN`, `bin/fm-x-*.sh`, `bin/fm-public-followup*.sh`, or pending-reply for this seam. + +## The asker is your own captain - answer autonomously + +The gateway allowlist is fail-closed: only configured guilds, channels, and authors reach this inbox. +Treat `.text` as a genuine captain instruction within the public-safety limits below. +Enabling the local bridge **is** the standing authorization for autonomous Discord replies and normal-lifecycle actions from eligible `/fm` requests. +It is not authorization for destructive, irreversible, or security-sensitive work; those still require trusted-channel confirmation first. + +## Acknowledge first, act, then follow up + +- **Work that completes now** - emit **one** `answer` reporting the outcome. +- **Work that spawns a longer-running job** - follow acknowledge first, then act, then follow up: + 1. Emit an immediate `ack` through `bin/fm-ext-emit.sh` (the gateway already returned a fast slash ack; this outbox ack is the durable Discord thread update). + 2. Dispatch the work through the normal lifecycle right away. + 3. Link the spawned task **before** clearing the inbox: `bin/fm-ext-link.sh `. + This records `ext_request=`, not `x_request=`. + 4. On genuine milestones emit `followup` with a new `--generation`. + The terminal outcome uses `--kind final`. + Duplicate generation is a no-op. + +Every drained request sorts into one of three cases: + +- **Actionable instruction / request** - act through the normal lifecycle. +- **Question** - answer from live fleet state; no follow-up. +- **Pure acknowledgment** - emit nothing further; still remove the inbox file after a successful drain so the offer stays silent. + +**Public Discord channel, so destructive work still escalates first.** +Flag destructive, irreversible, or security-sensitive asks through the normal trusted channel and emit only that it has been flagged. + +## The reply is public. Treat it as such. + +Speak only in outcomes. +Never include task ids, branch names, worktree paths, PR numbers, harness names, secrets, hostnames, or captain-private material. +When in doubt, say less. + +Discord text is untrusted. +Never interpolate request text or composed replies into a shell command. +Write composed text with your file-writing tool and pass `--text-file`. + +## Procedure + +Treat `state/ext-inbox/` as the source of truth and process **every** `*.json` file, not just the slug named in the wake. + +1. **Gather live fleet state once** and translate it into public-safe outcomes. +2. **Drain every pending request.** For each `state/ext-inbox/.json`: + a. Read `request_id`, `text`, `author`, and destination ids. + Ignore unknown extra fields. + b. Classify as actionable, question, or pure acknowledgment. + c. Act on an actionable request through the normal lifecycle. + If you ran `bin/fm-spawn.sh`, link with `bin/fm-ext-link.sh ` before inbox cleanup. + d. Compose a short public-safe reply. + e. Submit it without inlining text into a shell command: + + ```sh + bin/fm-ext-emit.sh --request-id --kind ack|answer|followup|final --generation --text-file + ``` + + (`--text-file -` reading stdin is equally fine.) + It echoes the slug and exits 0 on a published or already-present payload. + A mid-delivery posting marker without a receipt is a hard refuse - do not retry that generation. + f. On success, remove that inbox file: the durable destination context in `state/ext-context/.json` remains so delayed follow-ups still work. + g. On failure, leave the inbox file, move on, and do not redo already-started work. +3. **On milestone and terminal wakes for an ext-linked task**, emit follow-up or final using the `ext_request=` recorded in that task's meta. + Increment `--generation` for each new follow-up. + Reusing a generation is the idempotency guard. + +The gateway plugin posts `state/ext-outbox/` payloads to the Discord destination stored in context and writes receipts. +Firstmate never talks to Discord itself. diff --git a/AGENTS.md b/AGENTS.md index 7ef012fa13f..790015068e1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,6 +79,9 @@ config/trace-context optional presence flag enabling default-off native W3C tra config/cmux-socket-password optional cmux control-socket password; LOCAL, gitignored; read fresh on every cmux CLI call and passed through without ever overriding an operator's own ambient CMUX_SOCKET_PASSWORD when absent (docs/cmux-backend.md "Setup") config/wedge-alarm optional away-mode wedge-alarm active-alert directives; LOCAL, gitignored; absent means auto (macOS Notification Center when available); see docs/wedge-alarm.md config/x-mode.env generated X-mode watcher cadence; LOCAL, gitignored; source before arming watcher when present +config/ext-bridge optional presence flag enabling the sibling local Communication Officer bridge; LOCAL, gitignored, and not inherited; see docs/configuration.md "Local Communication Officer bridge" +config/ext-secret local ext-bridge shared secret; LOCAL, gitignored, mode 0600, not inherited +config/ext-allowlist fail-closed Discord guild/channel/author allowlist for the local ext-bridge; LOCAL, gitignored, not inherited config/runpod.env RUNPOD_API_KEY for the optional RunPod compute lifecycle beneath a remote secondmate; LOCAL, gitignored, parsed never sourced, not inherited; see docs/runpod-secondmates.md config/runpod/ generated SSH state plus the mode-600 workstation OMP broker bearer for RunPod-backed remote routes; LOCAL, gitignored, written only by bin/fm-runpod.sh and bin/fm-runpod-omp-auth.sh data/ personal fleet records; LOCAL, gitignored as a whole @@ -109,6 +112,7 @@ state/ volatile runtime signals; gitignored .pr-poll-retirement private identity-bound crash-recovery receipt for one exact validated merged result; removed after its poll artifacts retire .pr-poll-merge-notified canonical PR identity of the last merge outcome delivered for this task; bin/fm-pr-lib.sh owns the marker format and identity mechanics, while bin/fm-merge-outcome-lib.sh owns locked publication, duplicate suppression, and replacement; removed by teardown x-watch.check.sh generated X-mode relay poll shim; present only when opted in (section 14) + ext-watch.check.sh generated local Communication Officer poll shim; present only when the ext-bridge is opted in pending-replies/ parent-owned secondmate pending-reply records (correlation id, delivery vs reply, recovery, escalation); fm-pending-reply-lib.sh procevent/ registered process-to-event sources, one private record per canonical source id; written by bin/fm-procevent.sh or an adapter through the shared registration publisher, and their presence alone keeps supervision required (section 13) procevent-inbox/ private captured results and their durable handled-acknowledgement markers; source output lives here and never in an event line @@ -117,6 +121,9 @@ state/ volatile runtime signals; gitignored x-inbox/ generated X-mode pending mention payloads; fmx-respond drains it (section 14) x-context/ generated X-mode durable per-request reply context and one-wake offer markers, keyed by request_id; survives inbox cleanup and expires within seven days (section 14; bin/fm-x-lib.sh) x-outbox/ generated X-mode dry-run reply and dismiss previews; inspect it when FMX_DRY_RUN is set (section 14) + ext-inbox/ generated local Communication Officer pending request payloads; ext-respond drains it when the ext-bridge is on + ext-context/ generated local Communication Officer destination context and one-wake offer markers, keyed by request slug + ext-outbox/ generated local Communication Officer outbound ack/answer/followup/final payloads plus posting markers, receipts, terminal-failure markers, and chunk-progress records public-followup/ generated private transport for promised public replies: commitment registrations, typed terminal-result inbox, accepted/rejected ledgers (section 14; bin/fm-public-followup.sh) x-poll.error x-poll.claim-error generated X-mode relay and offer-claim diagnostic dedupe markers .wake-queue durable queued wakes retained until post-handling acknowledgement: epochseqkindkeypayload @@ -391,7 +398,7 @@ The promoted worker must inventory scratch state, return to a clean default-bran Fleet supervision is an always-loaded operational contract; `docs/architecture.md`, `docs/turnend-guard.md`, the emitted session-start block, and script help own mechanisms and harness-specific recipes. Whenever work is under way, keep exactly one live supervision cycle using the emitted protocol for this primary harness. -X mode may require that same live cycle with no fleet work. +X mode or the local Communication Officer bridge may require that same live cycle with no fleet work. Do not substitute another harness's wait shape, use shell `&`, or create a second cycle when a healthy one already exists. For every actionable wake, follow the ordinary-wake continuation in the emitted protocol; use its repair action only when the live cycle is missing or failed. No turn ends blind while work is under way, including turns described as holding or waiting. @@ -408,7 +415,7 @@ Handle actionable wakes as follows: 1. For `signal:`, read the listed event lines first, then reconcile current state only where action depends on it. 2. For `stale:`, inspect the recorded endpoint and load `stuck-crewmate-recovery` for a stopped, looping, confused, or unresponsive worker; a deep-inspection reason also requires current-state and validation-log inspection. -3. For `check:`, act on the named poll result, including merges, X-mode events, and process-to-event source results. +3. For `check:`, act on the named poll result, including merges, X-mode events, local Communication Officer requests, and process-to-event source results. 4. For `heartbeat:`, review the whole fleet from the structured fleet view, reconcile suspicious tasks and PR state, update the backlog, and never report an unchanged fleet as progress. Every lock-owning heartbeat also re-runs `bin/fm-todo-project.sh --check --reconcile`, reconciles what it flags, and re-projects the session todo from `--emit`. @@ -416,6 +423,7 @@ On an exact merged-PR check with verified fleet-lock ownership, immediately run When any wake reports a merged PR for a project cloned in this home, refresh that clone through the guarded fleet-sync path. When X-linked work reaches a milestone or terminal state, load `fmx-respond`; before terminal teardown, use its promised-final reconciliation when a typed public commitment exists, otherwise post the final completion follow-up so the link clears even if earlier follow-ups were spent. +When ext-linked work reaches a milestone or terminal state, load `ext-respond` and emit follow-up or final into the local outbox. For a quiet persistent secondmate endpoint, rely on fresh home-watcher liveness rather than quietness or routed status; missing evidence enters ordinary stale handling. A secondmate placed on scale-to-zero compute can have a deliberate no-host lifecycle state, delivery wakes it automatically, and `docs/runpod-secondmates.md` owns that lifecycle; never escalate such a route as a failure. @@ -542,7 +550,7 @@ The same attended path refreshes an installed machine-wide `omp` only after its These skills are not captain-invocable; load them only at their precise triggers. -- `bootstrap-diagnostics` - load whenever the session-start digest's bootstrap section prints an actionable diagnostic line (`MISSING:`, `MISSING_MANUAL:`, `BACKEND_INVALID:`, `NEEDS_GH_AUTH`, `TANGLE:`, `STARTUP_MEMORY_BUDGET:`, `CREW_DISPATCH: invalid`, `FLEET_SYNC:`, `SECONDMATE_SYNC:`, `SECONDMATE_LIVENESS:`, `SECONDMATE_HANDOFF:`, `NUDGE_SECONDMATES:`, `TREEHOUSE_POOL:`, or `FMX:`); silence and `BOOTSTRAP_INFO:` need no load. +- `bootstrap-diagnostics` - load whenever the session-start digest's bootstrap section prints an actionable diagnostic line (`MISSING:`, `MISSING_MANUAL:`, `BACKEND_INVALID:`, `NEEDS_GH_AUTH`, `TANGLE:`, `STARTUP_MEMORY_BUDGET:`, `CREW_DISPATCH: invalid`, `FLEET_SYNC:`, `SECONDMATE_SYNC:`, `SECONDMATE_LIVENESS:`, `SECONDMATE_HANDOFF:`, `NUDGE_SECONDMATES:`, `TREEHOUSE_POOL:`, `FMX:`, or `EXT:`); silence and `BOOTSTRAP_INFO:` need no load. - `diagnostic-reasoning` - load before scoping a reported bug and before acting on a diagnostic report. - `ask-user-authority` - load before deciding any ask-user finding. - `quota-array-dispatch` - load before choosing among a matched crew-dispatch profile array from current quota-axi output. @@ -556,6 +564,7 @@ These skills are not captain-invocable; load them only at their precise triggers - `process-event-sources` - load before arming a long-polling source, before registering a deterministic condition->action watch (do X as soon as Y is true), and on any `procevent ` check wake. Never run a registered source's blocking command yourself in a conversational turn. - `fmx-respond` - load on an `x-mention ` `check:` wake to handle the mention, on an `x-mode-error ...` `check:` wake to report the X-mode configuration blocker, on a `public-followup ...` `check:` wake or a startup-surfaced public commitment, and on any milestone or terminal wake for an X-mode-linked task before posting its completion follow-up; relevant only when X mode is on. +- `ext-respond` - load on an `ext-request ` `check:` wake to drain the local Communication Officer inbox, classify, act through the normal lifecycle, and emit follow-ups into the local outbox; relevant only when the local ext-bridge is on. - `firstmate-codexapp` - load before coordinating a visible Codex Desktop thread, evaluating a Codex App backend request, or reconciling Codex Desktop host-tool smoke evidence for Firstmate work. - `firstmate-coding-guidelines` - load before changing firstmate's shared, tracked material, as defined by section 1's list, whether editing directly or briefing a crewmate for a firstmate-repo task. @@ -573,6 +582,14 @@ A promised final public reply is durable state, never conversation memory. Load `fmx-respond` before promising one, on a `public-followup ...` check wake, and whenever the session-start digest lists a public commitment awaiting delivery. Only the home holding the relay consent and thread binding ever posts it, so never ask a secondmate or crewmate to find the thread or send the reply, and never recover a terminal result by reading a `done:` sentence. +## 15. Local Communication Officer bridge + +The sibling local Discord bridge ships inert until the home opts in with `config/ext-bridge` or `FM_EXT_BRIDGE=1` plus a mode-0600 secret file. +That opt-in is consent for local inbox intake and Discord follow-ups through the Hermes Gateway plugin, not authority for destructive, irreversible, or security-sensitive action. +`docs/configuration.md` owns activation, generated state, allowlist, outbox receipts, and opt-out mechanics. +On an `ext-request ` check wake, load `ext-respond`. +Do not use the hosted X-mode relay, `FMX_PAIRING_TOKEN`, or pending-reply for this seam. + ## Captain instruction precedence A current, explicit, concrete captain instruction overrides any conflicting standing rule written above. diff --git a/README.md b/README.md index bb13534d09c..8eef69cd5df 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ Launching a supported harness inside it instantiates your first mate - and makes - **Optional secondmates** - opt in to persistent second mates that run from isolated firstmate homes with their own `FM_HOME`, state, projects, and session lock, either locally or as a whole home on an SSH-reachable host, with guarded updates and recovery that never turns an unavailable remote route into a local replacement. - **Event-driven, zero-token supervision** - a bash watcher sleeps on the fleet and wakes the first mate only when something needs you; verified primary harnesses also get a turn-end backstop that blocks or follows up on a blind stop when work is under way and supervision is not live. - **Optional X mode** - opt in with one local `.env` token so firstmate can answer your public `@myfirstmate` mentions, act on normal reversible mention requests through the same lifecycle as chat requests, acknowledge spawned work, and post up to three public-safe completion follow-ups within seven days for genuine milestones and the final outcome without changing non-X behavior; a final reply promised in a thread becomes durable state that is reconciled from disk, so a restart or a compacted conversation cannot lose it; dry-run preview records would-be replies and dismissals locally before go-live. +- **Optional local Communication Officer bridge** - opt in with `config/ext-bridge` or `FM_EXT_BRIDGE=1` plus a local secret so a dedicated Hermes Gateway `/fm` plugin can deliver allowlisted Discord requests into this home over sibling local files, not a hosted relay. - **Strict project boundary** - the first mate is read-only over your projects except for the narrow guarded and captain-approved operations authorized by [hard rule 1](AGENTS.md#1-identity-and-prime-directives), including fleet sync's guarded safe branch pruning; crewmates make every other project change behind the configured merge authority. - **Restart-proof** - all state lives on disk and in the active session backend (tmux by hard default, herdr or cmux when selected or auto-detected, zellij/orca when explicitly selected); kill the session anytime and the next one reconciles, including confirmed-dead secondmate agents, and carries on. @@ -59,7 +60,7 @@ Full detail on every feature lives in [docs/architecture.md](docs/architecture.m ## What this fork adds over upstream -`dnth/firstmate` carries four features that are not in [upstream](https://github.com/kunchenguid/firstmate). +`dnth/firstmate` carries five features that are not in [upstream](https://github.com/kunchenguid/firstmate). - **RunPod remote secondmates** - run a persistent second mate on an on-demand CPU or GPU pod, with an explicit scale-to-zero lifecycle that drops compute cost after sleep and wakes the pod for the next delivery. See [RunPod second mates](docs/runpod-secondmates.md); the compute lifecycle lives in `bin/fm-runpod*.sh`. - **OMP harness with a read-only credential broker** - run OMP as a primary, crew, scout, or second-mate runtime, including on a remote pod through a [credential-read-only facade](bin/fm-omp-auth-broker-readonly-proxy.mjs) that keeps workstation subscription login and refresh credentials private. See the [OMP supervision protocol](docs/supervision-protocols/omp.md) and the [remote broker design](docs/runpod-secondmates.md#omp-subscription-auth-through-the-workstation). @@ -68,6 +69,8 @@ Full detail on every feature lives in [docs/architecture.md](docs/architecture.m It launches through the OpenAI Codex provider with `gpt-5.6-sol` as the default model and preserves Firstmate's low, medium, high, xhigh, and max reasoning settings. Hermes is verified for crew and scout work only, never as a primary session or second mate. See the [harness adapter reference](.agents/skills/harness-adapters/SKILL.md). +- **Hermes Communication Officer** - a dedicated Hermes Gateway plugin, installed into its own gateway `HERMES_HOME` rather than the crewmate TUI profile, delivers Discord `/fm` into Firstmate over a sibling local-file bridge. + See [Local Communication Officer bridge](docs/configuration.md#local-communication-officer-bridge-configext-bridge) and `contrib/hermes-gateway-firstmate-comms/`. - **Pi-compatible runtimes** - use the [closed runtime allowlist](bin/fm-pi-compatible-runtimes) and [shared predicate](bin/fm-pi-compatible-lib.sh) to reuse proven Pi-compatible mechanics without losing each harness's identity. See the [Pi-compatible family architecture](docs/architecture.md#harness-identity-and-the-pi-compatible-family). Supporting these are extra treehouse pool helpers (`bin/fm-treehouse-*.sh`) and a TypeScript primary-watcher core (`bin/fm-primary-watch-core.ts`). @@ -223,7 +226,7 @@ Firstmate's skills live in two separate places with different audiences: ## Documentation - [docs/architecture.md](docs/architecture.md) - maintainer architecture for the crew, supervision, worktrees, secondmates, and project modes. -- [docs/configuration.md](docs/configuration.md) - environment variables, `FM_HOME`, runtime backend selection, optional X mode, the files you set, and harness support. +- [docs/configuration.md](docs/configuration.md) - environment variables, `FM_HOME`, runtime backend selection, optional X mode, the local Communication Officer bridge, the files you set, and harness support. - [docs/remote-secondmates.md](docs/remote-secondmates.md) - current setup, routing, transfer, recovery, and safety behavior for whole-home remote second mates. - [docs/runpod-secondmates.md](docs/runpod-secondmates.md) - optional RunPod compute lifecycle beneath a remote second mate, so an idle domain can scale to zero and cost storage alone. - [docs/calm.md](docs/calm.md) - current Pi `/calm` behavior and supported presentation limits. diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index bf1da4da659..35f71ac6bfd 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -18,7 +18,8 @@ # "SECONDMATE_LIVENESS: secondmate : skipped: |respawn failed after : ", # "SECONDMATE_HANDOFF: secondmate : pending delivery: item(s)", # "TREEHOUSE_POOL: dirty idle slot at - inspect before cleanup; no changes made", -# "FMX: X mode on ..." or "FMX: X mode off ...". +# "FMX: X mode on ..." or "FMX: X mode off ...", +# "EXT: local bridge on ..." or "EXT: local bridge off ...". # When a RUNNING local secondmate worktree is fast-forwarded to # firstmate's own current default-branch commit, that update is a # purely local fast-forward and never an origin fetch. Remote routes @@ -70,6 +71,10 @@ # X mode is OPTIONAL and inert unless FM_HOME/.env has a non-empty # FMX_PAIRING_TOKEN. When opted in, bootstrap requires curl+jq, writes # the relay poll shim and 30s cadence config, and prints an FMX line. +# The sibling local Communication Officer bridge is OPTIONAL and inert +# unless config/ext-bridge is present or FM_EXT_BRIDGE=1, plus a +# mode-0600 secret file. When opted in, bootstrap requires jq and +# writes state/ext-watch.check.sh, printing an EXT line. # Fleet sync fetches, fast-forwards safe default-branch states, reports # recovered and STUCK clone drift, and prunes gone local branches; it is # bounded by FM_FLEET_SYNC_BOOTSTRAP_TIMEOUT when it is a non-empty @@ -79,16 +84,16 @@ # refresh relays any completed fm-fleet-sync.sh output before the # aggregate timeout skip line with timeout and elapsed seconds. # Set FM_FLEET_PRUNE=0 to skip branch pruning during that refresh. -# Set FM_BOOTSTRAP_DETECT_ONLY=1 to skip the five MUTATING sweeps +# Set FM_BOOTSTRAP_DETECT_ONLY=1 to skip the MUTATING sweeps # (secondmate_sync, secondmate_liveness_sweep, -# secondmate_handoff_resume, x_mode_setup, fleet_sync) while still +# secondmate_handoff_resume, x_mode_setup, ext_bridge_setup, fleet_sync) while still # printing every read-only detect line # above; the TANGLE line switches to advisory-only wording with no # checkout command. Used by # fm-session-start.sh's read-only path when another live session holds # the fleet lock, so a second concurrent session never race-mutates # secondmate homes, pending handoff outboxes, -# X-mode artifacts, project clones, or repair instructions. +# X-mode artifacts, local ext-bridge artifacts, project clones, or repair instructions. # Unset/0 (the default) runs every sweep exactly as before - this flag # is purely additive. # fm-bootstrap.sh install ... @@ -118,6 +123,8 @@ DATA="${FM_DATA_OVERRIDE:-$FM_HOME/data}" . "$SCRIPT_DIR/fm-startup-memory-budget-lib.sh" # shellcheck source=bin/fm-x-lib.sh disable=SC1091 . "$SCRIPT_DIR/fm-x-lib.sh" +# shellcheck source=bin/fm-ext-lib.sh disable=SC1091 +. "$SCRIPT_DIR/fm-ext-lib.sh" # shellcheck source=bin/fm-backend.sh disable=SC1091 . "$SCRIPT_DIR/fm-backend.sh" # shellcheck source=bin/fm-remote-readiness-lib.sh disable=SC1091 @@ -1013,6 +1020,84 @@ EOF echo "FMX: X mode on - relay poll armed via state/x-watch.check.sh; 30s watcher cadence in config/x-mode.env" } +# Local Communication Officer bridge (opt-in): config/ext-bridge or +# FM_EXT_BRIDGE=1 plus a mode-0600 secret file. Writes one identity shim: +# state/ext-watch.check.sh - byte-static identity shim; the watcher validates +# its bytes and invokes bin/fm-ext-poll.sh directly +# There is no hosted relay, pairing token, or cadence override. Intake wakes +# the primary immediately; the shim exists so restart recovery and the watcher +# identity path match X mode. +ext_bridge_setup() { + local shim shim_body shim_home secret + + shim=$(fm_ext_watch_shim_path) + + ext_bridge_remove_shim() { + x_mode_remove_artifact "$shim" + } + + if ! fm_ext_bridge_opted_in "$FM_HOME"; then + if x_mode_artifact_present "$shim"; then + if ext_bridge_remove_shim; then + echo "EXT: local bridge off - removed inbox poll shim" + else + echo "EXT: local bridge off - failed to remove inbox poll shim" + fi + fi + return 0 + fi + + secret=$(fm_ext_secret_path "$FM_HOME") + if ! fm_ext_secret_valid "$secret"; then + if x_mode_artifact_present "$shim"; then + if ext_bridge_remove_shim; then + echo "EXT: local bridge off - secret file missing or not mode 0600; install it and rerun bootstrap" + else + echo "EXT: local bridge off - failed to remove inbox poll shim after a secret-file refusal" + fi + else + echo "EXT: local bridge off - secret file missing or not mode 0600" + fi + return 0 + fi + + if ! command -v jq >/dev/null 2>&1; then + echo "MISSING: jq (install: $(install_cmd jq))" + if x_mode_artifact_present "$shim"; then + if ext_bridge_remove_shim; then + echo "EXT: local bridge off - missing jq; install it and rerun bootstrap" + else + echo "EXT: local bridge off - failed to remove inbox poll shim after missing jq" + fi + fi + return 0 + fi + + ext_arm_failed() { + if ext_bridge_remove_shim; then + echo "EXT: local bridge off - failed to arm inbox poll shim" + else + echo "EXT: local bridge off - failed to arm inbox poll shim; stale artifacts remain" + fi + } + + mkdir -p "$STATE" 2>/dev/null || { ext_arm_failed; return 0; } + + case "$FM_HOME" in + /*) shim_home=$FM_HOME ;; + *) + shim_home=$(CDPATH='' cd -- "$FM_HOME" 2>/dev/null && pwd -P) \ + || { ext_arm_failed; return 0; } + ;; + esac + shim_body=$(fm_ext_poll_shim_content "$shim_home" "$FM_ROOT") + x_mode_write_if_changed "$shim" "$shim_body" 700 || { ext_arm_failed; return 0; } + fm_ext_poll_shim_valid "$shim" "$shim_home" "$FM_ROOT" \ + || { ext_arm_failed; return 0; } + + echo "EXT: local bridge on - inbox poll armed via state/ext-watch.check.sh" +} + crew_dispatch_validate() { local file err file="$CONFIG/crew-dispatch.json" @@ -1214,6 +1299,7 @@ if [ "${FM_BOOTSTRAP_DETECT_ONLY:-0}" != 1 ]; then secondmate_sync secondmate_handoff_resume x_mode_setup + ext_bridge_setup fleet_sync fi secondmate_handoff_detect diff --git a/bin/fm-claude-stop-autoarm.sh b/bin/fm-claude-stop-autoarm.sh index 428e8b32f17..c2075c3b43f 100755 --- a/bin/fm-claude-stop-autoarm.sh +++ b/bin/fm-claude-stop-autoarm.sh @@ -19,7 +19,8 @@ # this hook exits 0 and NEVER rewakes the primary (checked again at # translation time so a mid-cycle AFK transition is honored). # - Need: arms only while work is in flight (state/*.meta) or X mode has a -# relay poll to run (state/x-watch.check.sh); an idle home exits 0. +# relay poll to run (state/x-watch.check.sh) or the local ext-bridge poll +# is armed (state/ext-watch.check.sh); an idle home exits 0. # - Single-flight: Claude does not dedupe async hooks, so exactly one # GENERATION owner arms per event epoch: the epoch ledger's monotonic # sequence is the claim generation, every firing defers (exit 0) to a live diff --git a/bin/fm-ext-emit.sh b/bin/fm-ext-emit.sh new file mode 100755 index 00000000000..e40cd5b95e1 --- /dev/null +++ b/bin/fm-ext-emit.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# Emit one Communication Officer outbox payload (ack/answer/followup/final). +# +# Usage: +# fm-ext-emit.sh --request-id discord:::: +# --kind ack|answer|followup|final --generation +# (--text-file | --text-file -) +# +# Writes state/ext-outbox/...json once. Re-emitting the +# same identity is a no-op success. A posting marker without a receipt is +# refused (mid-delivery), matching public-follow-up delivery-posting behavior. +# Destination fields come from the durable per-request context, so a delayed +# follow-up still works after the inbox file is gone. +# +# Discord text is untrusted: pass it with --text-file or stdin. +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +# shellcheck source=bin/fm-ext-lib.sh +. "$SCRIPT_DIR/fm-ext-lib.sh" + +usage() { + cat >&2 <<'EOF' +usage: fm-ext-emit.sh --request-id discord:::: + --kind ack|answer|followup|final --generation + (--text-file | --text-file -) +EOF +} + +help() { + sed -n '2,/^set -u$/p' "$0" | sed '$d; s/^# \{0,1\}//' +} + +die() { printf 'fm-ext-emit: %s\n' "$1" >&2; exit "${2:-2}"; } + +REQUEST_ID= +KIND= +GENERATION= +TEXT_FILE= + +case "${1:-}" in + --help|-h) help; exit 0 ;; + '') usage; exit 2 ;; +esac + +while [ "$#" -gt 0 ]; do + case "$1" in + --request-id) shift; REQUEST_ID=${1:-} ;; + --kind) shift; KIND=${1:-} ;; + --generation) shift; GENERATION=${1:-} ;; + --text-file) shift; TEXT_FILE=${1:-} ;; + --help|-h) help; exit 0 ;; + *) die "unknown argument '$1'" ;; + esac + shift || true +done + +[ -n "$REQUEST_ID" ] || { usage; exit 2; } +[ -n "$KIND" ] || { usage; exit 2; } +[ -n "$GENERATION" ] || { usage; exit 2; } +[ -n "$TEXT_FILE" ] || { usage; exit 2; } + +fm_ext_request_id_valid "$REQUEST_ID" || die "unsafe request_id: $REQUEST_ID" +fm_ext_kind_valid "$KIND" || die "kind must be ack, answer, followup, or final" +fm_ext_generation_valid "$GENERATION" || die "generation must be a positive integer" +fm_ext_active "$FM_HOME" || die "local ext-bridge is not active" 1 +command -v jq >/dev/null 2>&1 || die "jq is required" 1 + +SLUG=$(fm_ext_request_slug "$REQUEST_ID") || die "could not derive request slug" 1 +OUTBOX=$(fm_ext_outbox_dir) +CONTEXT_DIR=$(fm_ext_context_dir) +PAYLOAD=$(fm_ext_outbox_basename "$SLUG" "$KIND" "$GENERATION") || die "invalid outbox identity" 1 +RECEIPT=$(fm_ext_outbox_receipt_basename "$SLUG" "$KIND" "$GENERATION") || die "invalid receipt identity" 1 +POSTING=$(fm_ext_outbox_posting_basename "$SLUG" "$KIND" "$GENERATION") || die "invalid posting identity" 1 + +if fm_ext_private_artifact_file_valid "$OUTBOX" "$RECEIPT" 600; then + printf '%s\n' "$SLUG" + exit 0 +fi +if fm_ext_private_artifact_file_valid "$OUTBOX" "$POSTING" 600 \ + && ! fm_ext_private_artifact_file_valid "$OUTBOX" "$RECEIPT" 600; then + die "mid-delivery: $KIND generation $GENERATION is posting and has no receipt; refusing a second emit" 1 +fi +if fm_ext_private_artifact_file_valid "$OUTBOX" "$PAYLOAD" 600; then + printf '%s\n' "$SLUG" + exit 0 +fi + +if [ "$TEXT_FILE" = '-' ]; then + TEXT=$(cat) +else + [ -f "$TEXT_FILE" ] && [ ! -L "$TEXT_FILE" ] || die "text file not found: $TEXT_FILE" + TEXT=$(cat -- "$TEXT_FILE") +fi +[ -n "$TEXT" ] || die "text is empty" + +CTX= +if fm_ext_private_artifact_file_valid "$CONTEXT_DIR" "$SLUG.json" 600; then + CTX="$CONTEXT_DIR/$SLUG.json" +elif fm_ext_private_artifact_file_valid "$(fm_ext_inbox_dir)" "$SLUG.json" 600; then + CTX="$(fm_ext_inbox_dir)/$SLUG.json" +else + die "no durable destination context for request $REQUEST_ID; cannot emit after the inbox is gone without context" 1 +fi + +GUILD=$(jq -r '.guild_id // empty' "$CTX") +CHANNEL=$(jq -r '.channel_id // empty' "$CTX") +THREAD=$(jq -r '.thread_id // empty' "$CTX") +MESSAGE=$(jq -r '.message_id // empty' "$CTX") +PLATFORM=$(jq -r '.platform // "discord"' "$CTX") +SOURCE=$(jq -r '.source // "hermes-gateway"' "$CTX") +[ "$PLATFORM" = discord ] || die "context platform must be discord" +[ "$SOURCE" = hermes-gateway ] || die "context source must be hermes-gateway" +[ -n "$GUILD" ] && [ -n "$CHANNEL" ] && [ -n "$THREAD" ] && [ -n "$MESSAGE" ] \ + || die "destination context is incomplete" + +NOW=${FM_EXT_NOW_OVERRIDE:-$(date +%s)} +case "$NOW" in + ''|*[!0-9]*) die "could not read the current time" 1 ;; +esac + +OUTBOX_JSON=$(jq -cn \ + --argjson schema_version "$FM_EXT_SCHEMA_VERSION" \ + --arg request_id "$REQUEST_ID" \ + --arg slug "$SLUG" \ + --arg kind "$KIND" \ + --argjson generation "$GENERATION" \ + --arg platform "$PLATFORM" \ + --arg source "$SOURCE" \ + --arg guild_id "$GUILD" \ + --arg channel_id "$CHANNEL" \ + --arg thread_id "$THREAD" \ + --arg message_id "$MESSAGE" \ + --arg text "$TEXT" \ + --argjson recorded_at "$NOW" \ + '{schema_version:$schema_version, request_id:$request_id, slug:$slug, + kind:$kind, generation:$generation, platform:$platform, source:$source, + guild_id:$guild_id, channel_id:$channel_id, thread_id:$thread_id, + message_id:$message_id, text:$text, recorded_at:$recorded_at}') \ + || die "could not build the outbox payload" 1 + +printf '%s\n' "$OUTBOX_JSON" \ + | fm_ext_private_artifact_publish_stdin_once "$OUTBOX" "$PAYLOAD" 600 +case $? in + 0|1) + if ! fm_ext_outbox_schema_valid "$OUTBOX/$PAYLOAD"; then + die "published outbox payload failed schema validation" 1 + fi + printf '%s\n' "$SLUG" + ;; + *) die "could not publish the outbox payload" 1 ;; +esac diff --git a/bin/fm-ext-intake.sh b/bin/fm-ext-intake.sh new file mode 100755 index 00000000000..dda641fe6eb --- /dev/null +++ b/bin/fm-ext-intake.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# Publish one Communication Officer request into the local ext-bridge inbox. +# +# Usage: +# fm-ext-intake.sh --request-id discord:::: +# --guild-id --channel-id --thread-id --message-id +# --author --secret-file +# (--text-file | --text-file -) +# [--platform discord] [--source hermes-gateway] [--no-wake] +# +# Opt-in is config/ext-bridge or FM_EXT_BRIDGE=1 plus a mode-0600 home secret. +# The presented --secret-file must match that secret. Missing or empty +# allowlist, or a request that matches no rule, writes nothing. +# +# Idempotent: the same message id claims the existing offer and does not +# append a second wake. If the wake cannot be appended, the offer marker is +# removed so a later intake or poll can retry. Canonical request_id keeps +# colons in the JSON body; the inbox filename is the SHA-256 slug of that id. +# +# Discord text is untrusted: pass it with --text-file or stdin, never by +# interpolating it into a shell command. +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" +# shellcheck source=bin/fm-ext-lib.sh +. "$SCRIPT_DIR/fm-ext-lib.sh" +# shellcheck source=bin/fm-wake-lib.sh +. "$SCRIPT_DIR/fm-wake-lib.sh" + +usage() { + cat >&2 <<'EOF' +usage: fm-ext-intake.sh --request-id discord:::: + --guild-id --channel-id --thread-id --message-id + --author --secret-file + (--text-file | --text-file -) + [--platform discord] [--source hermes-gateway] [--no-wake] +EOF +} + +help() { + sed -n '2,/^set -u$/p' "$0" | sed '$d; s/^# \{0,1\}//' +} + +die() { printf 'fm-ext-intake: %s\n' "$1" >&2; exit "${2:-2}"; } + +REQUEST_ID= +GUILD_ID= +CHANNEL_ID= +THREAD_ID= +MESSAGE_ID= +AUTHOR= +SECRET_FILE= +TEXT_FILE= +PLATFORM=discord +SOURCE=hermes-gateway +WAKE=1 + +case "${1:-}" in + --help|-h) help; exit 0 ;; + '') usage; exit 2 ;; +esac + +while [ "$#" -gt 0 ]; do + case "$1" in + --request-id) shift; REQUEST_ID=${1:-} ;; + --guild-id) shift; GUILD_ID=${1:-} ;; + --channel-id) shift; CHANNEL_ID=${1:-} ;; + --thread-id) shift; THREAD_ID=${1:-} ;; + --message-id) shift; MESSAGE_ID=${1:-} ;; + --author) shift; AUTHOR=${1:-} ;; + --secret-file) shift; SECRET_FILE=${1:-} ;; + --text-file) shift; TEXT_FILE=${1:-} ;; + --platform) shift; PLATFORM=${1:-} ;; + --source) shift; SOURCE=${1:-} ;; + --no-wake) WAKE=0 ;; + --help|-h) help; exit 0 ;; + *) die "unknown argument '$1'" ;; + esac + shift || true +done + +[ -n "$REQUEST_ID" ] || { usage; exit 2; } +[ -n "$GUILD_ID" ] || { usage; exit 2; } +[ -n "$CHANNEL_ID" ] || { usage; exit 2; } +[ -n "$THREAD_ID" ] || { usage; exit 2; } +[ -n "$MESSAGE_ID" ] || { usage; exit 2; } +[ -n "$AUTHOR" ] || { usage; exit 2; } +[ -n "$SECRET_FILE" ] || { usage; exit 2; } +[ -n "$TEXT_FILE" ] || { usage; exit 2; } + +[ "$PLATFORM" = discord ] || die "platform must be discord, got '$PLATFORM'" +[ "$SOURCE" = hermes-gateway ] || die "source must be hermes-gateway, got '$SOURCE'" +fm_ext_request_id_valid "$REQUEST_ID" || die "unsafe request_id: $REQUEST_ID" +EXPECTED_RID="discord:${GUILD_ID}:${CHANNEL_ID}:${THREAD_ID}:${MESSAGE_ID}" +[ "$REQUEST_ID" = "$EXPECTED_RID" ] \ + || die "request_id does not match guild/channel/thread/message fields" + +fm_ext_active "$FM_HOME" || die "local ext-bridge is not active (need config/ext-bridge or FM_EXT_BRIDGE=1 plus a mode-0600 secret)" 1 +HOME_SECRET=$(fm_ext_secret_path "$FM_HOME") +fm_ext_secret_matches "$HOME_SECRET" "$SECRET_FILE" \ + || die "presented secret does not match the home secret" 1 + +ALLOWLIST=$(fm_ext_allowlist_path "$FM_HOME") +fm_ext_allowlisted "$ALLOWLIST" "$GUILD_ID" "$CHANNEL_ID" "$AUTHOR" \ + || die "request is not allowlisted" 1 + +command -v jq >/dev/null 2>&1 || die "jq is required" 1 + +if [ "$TEXT_FILE" = '-' ]; then + TEXT=$(cat) +else + [ -f "$TEXT_FILE" ] && [ ! -L "$TEXT_FILE" ] || die "text file not found: $TEXT_FILE" + TEXT=$(cat -- "$TEXT_FILE") +fi +[ -n "$TEXT" ] || die "text is empty" + +SLUG=$(fm_ext_request_slug "$REQUEST_ID") || die "could not derive request slug" 1 +NOW=${FM_EXT_NOW_OVERRIDE:-$(date +%s)} +case "$NOW" in + ''|*[!0-9]*) die "could not read the current time" 1 ;; +esac + +INBOX_JSON=$(jq -cn \ + --argjson schema_version "$FM_EXT_SCHEMA_VERSION" \ + --arg request_id "$REQUEST_ID" \ + --arg slug "$SLUG" \ + --arg platform "$PLATFORM" \ + --arg source "$SOURCE" \ + --arg guild_id "$GUILD_ID" \ + --arg channel_id "$CHANNEL_ID" \ + --arg thread_id "$THREAD_ID" \ + --arg message_id "$MESSAGE_ID" \ + --arg author "$AUTHOR" \ + --arg text "$TEXT" \ + --argjson recorded_at "$NOW" \ + '{schema_version:$schema_version, request_id:$request_id, slug:$slug, + platform:$platform, source:$source, guild_id:$guild_id, + channel_id:$channel_id, thread_id:$thread_id, message_id:$message_id, + author:$author, text:$text, recorded_at:$recorded_at}') \ + || die "could not build the inbox record" 1 + +CONTEXT_JSON=$(jq -cn \ + --arg request_id "$REQUEST_ID" \ + --arg slug "$SLUG" \ + --arg platform "$PLATFORM" \ + --arg source "$SOURCE" \ + --arg guild_id "$GUILD_ID" \ + --arg channel_id "$CHANNEL_ID" \ + --arg thread_id "$THREAD_ID" \ + --arg message_id "$MESSAGE_ID" \ + --arg author "$AUTHOR" \ + --argjson recorded_at "$NOW" \ + '{request_id:$request_id, slug:$slug, platform:$platform, source:$source, + guild_id:$guild_id, channel_id:$channel_id, thread_id:$thread_id, + message_id:$message_id, author:$author, recorded_at:$recorded_at}') \ + || die "could not build the destination context" 1 + +INBOX_DIR=$(fm_ext_inbox_dir) +CONTEXT_DIR=$(fm_ext_context_dir) + +if fm_ext_private_artifact_file_valid "$CONTEXT_DIR" "$SLUG.offered.json" 600; then + printf '%s\n' "$SLUG" + exit 0 +fi + +printf '%s\n' "$INBOX_JSON" \ + | fm_ext_private_artifact_publish_stdin "$INBOX_DIR" "$SLUG.json" 600 \ + || die "could not write inbox" 1 + +printf '%s\n' "$CONTEXT_JSON" \ + | fm_ext_private_artifact_publish_stdin "$CONTEXT_DIR" "$SLUG.json" 600 \ + || die "could not write destination context" 1 + +fm_ext_offer_registry_claim "$STATE" "$SLUG" "$REQUEST_ID" +offer_rc=$? +case "$offer_rc" in + 0) + if [ "$WAKE" = 1 ]; then + if ! fm_wake_append check "$FM_EXT_WATCH_SHIM" "ext-request $SLUG"; then + if ! fm_ext_offer_registry_unclaim "$STATE" "$SLUG"; then + die "could not append the wake, and the offer marker could not be released" 1 + fi + die "could not append the wake" 1 + fi + fi + ;; + 1) ;; + *) die "could not claim the offer marker" 1 ;; +esac + +printf '%s\n' "$SLUG" diff --git a/bin/fm-ext-lib.sh b/bin/fm-ext-lib.sh new file mode 100644 index 00000000000..3e24da1ca53 --- /dev/null +++ b/bin/fm-ext-lib.sh @@ -0,0 +1,1052 @@ +# shellcheck shell=bash +# Shared helpers for the sibling local-bridge Communication Officer +# (fm-ext-intake.sh, fm-ext-emit.sh, fm-ext-link.sh, fm-ext-poll.sh). +# +# This file is sourced, never executed. It copies the private-artifact +# publication pattern used by X mode without sourcing bin/fm-x-lib.sh and +# without touching the hosted relay, FMX_PAIRING_TOKEN, myfirstmate.io, or +# pending-reply. +# +# Opt-in is config/ext-bridge presence or FM_EXT_BRIDGE=1, plus a local +# mode-0600 secret file. There is no hosted pairing token. +# +# Canonical request_id keeps colons in JSON bodies +# (discord::::). Filenames use the SHA-256 +# hex digest of that canonical id. +# +# Callers must have FM_HOME set (or pass explicit state/config paths). + +_FM_EXT_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd 2>/dev/null)" || _FM_EXT_LIB_DIR="." + +FM_EXT_SCHEMA_VERSION=1 +FM_EXT_INBOX_DIRNAME='ext-inbox' +FM_EXT_CONTEXT_DIRNAME='ext-context' +FM_EXT_OUTBOX_DIRNAME='ext-outbox' +FM_EXT_WATCH_SHIM='ext-watch.check.sh' +FM_EXT_SECRET_BASENAME='ext-secret' +FM_EXT_ALLOWLIST_BASENAME='ext-allowlist' +FM_EXT_BRIDGE_BASENAME='ext-bridge' + +# --- private artifact publication (X-mode pattern, local names) ------------- + +fm_ext_single_link_file_valid() { + local file=$1 expected_device=${2-} links device + [ -f "$file" ] && [ ! -L "$file" ] || return 1 + if [ "$(uname)" = Darwin ]; then + links=$(stat -f %l "$file" 2>/dev/null) || return 1 + device=$(stat -f %d "$file" 2>/dev/null) || return 1 + else + links=$(stat -c %h "$file" 2>/dev/null) || return 1 + device=$(stat -c %d "$file" 2>/dev/null) || return 1 + fi + [ "$links" = 1 ] || return 1 + [ -z "$expected_device" ] || [ "$device" = "$expected_device" ] +} + +fm_ext_single_link_file_mode_valid() { + local file=$1 expected_mode=$2 expected_device=${3-} mode + fm_ext_single_link_file_valid "$file" "$expected_device" || return 1 + if [ "$(uname)" = Darwin ]; then + mode=$(stat -f %Lp "$file" 2>/dev/null) || return 1 + else + mode=$(stat -c %a "$file" 2>/dev/null) || return 1 + fi + [ "$mode" = "$expected_mode" ] +} + +fm_ext_private_artifact_dir_device() { + local dir=$1 mode device + [ -d "$dir" ] && [ ! -L "$dir" ] || return 1 + if [ "$(uname)" = Darwin ]; then + mode=$(stat -f %Lp "$dir" 2>/dev/null) || return 1 + device=$(stat -f %d "$dir" 2>/dev/null) || return 1 + else + mode=$(stat -c %a "$dir" 2>/dev/null) || return 1 + device=$(stat -c %d "$dir" 2>/dev/null) || return 1 + fi + [ "$mode" = 700 ] || return 1 + printf '%s\n' "$device" +} + +fm_ext_private_artifact_dir_prepare() { + local dir=$1 parent + parent=${dir%/*} + if [ "$parent" != "$dir" ]; then + if [ -e "$parent" ] || [ -L "$parent" ]; then + [ -d "$parent" ] && [ ! -L "$parent" ] || return 1 + else + (umask 077; mkdir -p "$parent" 2>/dev/null) || return 1 + [ -d "$parent" ] && [ ! -L "$parent" ] || return 1 + fi + fi + if [ -e "$dir" ] || [ -L "$dir" ]; then + [ -d "$dir" ] && [ ! -L "$dir" ] || return 1 + else + (umask 077; mkdir -p "$dir" 2>/dev/null) || return 1 + fi + fm_ext_private_artifact_dir_device "$dir" +} + +fm_ext_private_artifact_publish_stdin() { + local dir=$1 base=$2 mode=$3 device tmp dest + case "$base" in + ''|.*|*/*) return 1 ;; + esac + case "$mode" in + 600|700) ;; + *) return 1 ;; + esac + device=$(fm_ext_private_artifact_dir_prepare "$dir") || return 1 + dest="$dir/$base" + tmp=$(umask 077; mktemp "$dir/.${base}.fm-ext.XXXXXX" 2>/dev/null) || return 1 + if ! cat > "$tmp" \ + || ! chmod "$mode" "$tmp" 2>/dev/null \ + || ! fm_ext_single_link_file_mode_valid "$tmp" "$mode" "$device"; then + rm -f -- "$tmp" + return 1 + fi + if { [ -e "$dest" ] || [ -L "$dest" ]; } \ + && ! fm_ext_single_link_file_mode_valid "$dest" "$mode" "$device"; then + rm -f -- "$tmp" + return 1 + fi + if ! mv -f -- "$tmp" "$dest" 2>/dev/null; then + rm -f -- "$tmp" + return 1 + fi + if ! fm_ext_single_link_file_mode_valid "$dest" "$mode" "$device"; then + rm -f -- "$dest" + return 1 + fi +} + +# Publish stdin as a new private artifact without replacing an existing path. +# Returns 0 when this caller created it, 1 when another valid private artifact +# already owns the path, and 2 on an unsafe path or publication failure. +fm_ext_private_artifact_publish_stdin_once() { + local dir=$1 base=$2 mode=$3 device tmp dest + case "$base" in + ''|.*|*/*) return 2 ;; + esac + case "$mode" in + 600|700) ;; + *) return 2 ;; + esac + device=$(fm_ext_private_artifact_dir_prepare "$dir") || return 2 + dest="$dir/$base" + tmp=$(umask 077; mktemp "$dir/.${base}.fm-ext.XXXXXX" 2>/dev/null) || return 2 + if ! cat > "$tmp" \ + || ! chmod "$mode" "$tmp" 2>/dev/null \ + || ! fm_ext_single_link_file_mode_valid "$tmp" "$mode" "$device"; then + rm -f -- "$tmp" + return 2 + fi + if ln -- "$tmp" "$dest" 2>/dev/null; then + rm -f -- "$tmp" + if fm_ext_single_link_file_mode_valid "$dest" "$mode" "$device"; then + return 0 + fi + rm -f -- "$dest" + return 2 + fi + rm -f -- "$tmp" + if fm_ext_single_link_file_mode_valid "$dest" "$mode" "$device"; then + return 1 + fi + return 2 +} + +fm_ext_private_artifact_file_valid() { + local dir=$1 base=$2 mode=$3 device + case "$base" in + ''|.*|*/*) return 1 ;; + esac + case "$mode" in + 600|700) ;; + *) return 1 ;; + esac + device=$(fm_ext_private_artifact_dir_device "$dir") || return 1 + fm_ext_single_link_file_mode_valid "$dir/$base" "$mode" "$device" +} + +# Remove a previously published private artifact. Returns 0 when the path is +# absent or this caller deleted a valid artifact, and 1 when the path exists +# but is not a safe private artifact or deletion failed. +fm_ext_private_artifact_remove() { + local dir=$1 base=$2 mode=$3 dest + case "$base" in + ''|.*|*/*) return 1 ;; + esac + dest="$dir/$base" + if ! fm_ext_private_artifact_file_valid "$dir" "$base" "$mode"; then + if [ -e "$dest" ] || [ -L "$dest" ]; then + return 1 + fi + return 0 + fi + rm -f -- "$dest" || return 1 + if [ -e "$dest" ] || [ -L "$dest" ]; then + return 1 + fi + return 0 +} + +# --- identifiers ------------------------------------------------------------ + +fm_ext_sha256() { + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 2>/dev/null | awk '{print $1}' + elif command -v sha256sum >/dev/null 2>&1; then + sha256sum 2>/dev/null | awk '{print $1}' + else + return 1 + fi +} + +# fm_ext_request_slug : SHA-256 hex of the canonical id. +# The canonical id keeps colons; the slug is the only filename component. +fm_ext_request_slug() { + local rid=$1 slug + [ -n "$rid" ] || return 1 + slug=$(printf '%s' "$rid" | fm_ext_sha256) || return 1 + case "$slug" in + *[!0-9a-f]*|'') return 1 ;; + esac + [ "${#slug}" -eq 64 ] || return 1 + printf '%s\n' "$slug" +} + +# Canonical Communication Officer request id: discord:::: +# All four ids are non-empty decimal snowflakes. Thread may equal channel when +# the source message is not in a thread. +fm_ext_request_id_valid() { + local rid=$1 rest guild channel thread message + case "$rid" in + discord:*) ;; + *) return 1 ;; + esac + rest=${rid#discord:} + guild=${rest%%:*} + rest=${rest#"$guild"} + rest=${rest#:} + [ -n "$guild" ] && [ -n "$rest" ] || return 1 + channel=${rest%%:*} + rest=${rest#"$channel"} + rest=${rest#:} + [ -n "$channel" ] && [ -n "$rest" ] || return 1 + thread=${rest%%:*} + message=${rest#"$thread"} + message=${message#:} + [ -n "$thread" ] && [ -n "$message" ] || return 1 + case "$message" in + *:*) return 1 ;; + esac + case "$guild$channel$thread$message" in + *[!0-9]*) return 1 ;; + esac +} + +fm_ext_slug_valid() { + local v=$1 + case "$v" in + *[!0-9a-f]*|'') return 1 ;; + esac + [ "${#v}" -eq 64 ] +} + +fm_ext_kind_valid() { + case "$1" in + ack|answer|followup|final) return 0 ;; + *) return 1 ;; + esac +} + +fm_ext_generation_valid() { + case "$1" in + ''|*[!0-9]*) return 1 ;; + esac + [ "$1" -ge 1 ] +} + +# --- layout ----------------------------------------------------------------- + +fm_ext_config_dir() { + printf '%s\n' "${FM_CONFIG_OVERRIDE:-${1:-${FM_HOME:?FM_HOME is required}}/config}" +} + +fm_ext_state_dir() { + printf '%s\n' "${FM_STATE_OVERRIDE:-${1:-${FM_HOME:?FM_HOME is required}}/state}" +} + +fm_ext_secret_path() { + local config + config=$(fm_ext_config_dir "${1:-}") + printf '%s\n' "${FM_EXT_SECRET_FILE:-$config/$FM_EXT_SECRET_BASENAME}" +} + +fm_ext_allowlist_path() { + local config + config=$(fm_ext_config_dir "${1:-}") + printf '%s\n' "${FM_EXT_ALLOWLIST_FILE:-$config/$FM_EXT_ALLOWLIST_BASENAME}" +} + +fm_ext_bridge_path() { + local config + config=$(fm_ext_config_dir "${1:-}") + printf '%s\n' "$config/$FM_EXT_BRIDGE_BASENAME" +} + +fm_ext_inbox_dir() { printf '%s\n' "$(fm_ext_state_dir "${1:-}")/$FM_EXT_INBOX_DIRNAME"; } +fm_ext_context_dir(){ printf '%s\n' "$(fm_ext_state_dir "${1:-}")/$FM_EXT_CONTEXT_DIRNAME"; } +fm_ext_outbox_dir() { printf '%s\n' "$(fm_ext_state_dir "${1:-}")/$FM_EXT_OUTBOX_DIRNAME"; } +fm_ext_watch_shim_path() { printf '%s\n' "$(fm_ext_state_dir "${1:-}")/$FM_EXT_WATCH_SHIM"; } + +# --- activation and secret -------------------------------------------------- + +# fm_ext_bridge_opted_in [home]: config/ext-bridge presence or FM_EXT_BRIDGE=1. +# Environment wins when set: a non-empty truthy FM_EXT_BRIDGE opts in, and an +# explicit empty/0/false/no/off value opts out even if the file exists. +fm_ext_bridge_opted_in() { + local home=${1:-${FM_HOME:?}} flag file + if [ -n "${FM_EXT_BRIDGE+x}" ]; then + flag=$(printf '%s' "${FM_EXT_BRIDGE-}" | tr '[:upper:]' '[:lower:]') + case "$flag" in + ''|0|false|no|off) return 1 ;; + *) return 0 ;; + esac + fi + file=$(fm_ext_bridge_path "$home") + [ -f "$file" ] && [ ! -L "$file" ] +} + +# fm_ext_secret_valid : regular file, mode 0600, non-empty, no symlink. +fm_ext_secret_valid() { + local file=$1 mode + [ -f "$file" ] && [ ! -L "$file" ] || return 1 + [ -s "$file" ] || return 1 + if [ "$(uname)" = Darwin ]; then + mode=$(stat -f %Lp "$file" 2>/dev/null) || return 1 + else + mode=$(stat -c %a "$file" 2>/dev/null) || return 1 + fi + [ "$mode" = 600 ] +} + +# fm_ext_secret_matches : byte-identical secrets. +fm_ext_secret_matches() { + local home_secret=$1 presented=$2 + fm_ext_secret_valid "$home_secret" || return 1 + fm_ext_secret_valid "$presented" || return 1 + cmp -s "$home_secret" "$presented" +} + +# fm_ext_active [home]: opted in AND a valid local secret file exists. +fm_ext_active() { + local home=${1:-${FM_HOME:?}} secret + fm_ext_bridge_opted_in "$home" || return 1 + secret=$(fm_ext_secret_path "$home") + fm_ext_secret_valid "$secret" +} + +# --- allowlist -------------------------------------------------------------- + +# Allowlist file: comments (#) and blank lines ignored. Each rule is one of: +# +# : +# :: +# A request is allowed when any rule matches every specified component. +# Missing, empty, unreadable, or symlink allowlist denies every request. +fm_ext_allowlist_read() { + local file=$1 line + [ -f "$file" ] && [ ! -L "$file" ] || return 1 + while IFS= read -r line || [ -n "$line" ]; do + line=${line%"${line##*[![:space:]]}"} + line=${line#"${line%%[![:space:]]*}"} + case "$line" in + ''|\#*) continue ;; + esac + printf '%s\n' "$line" + done < "$file" +} + +fm_ext_allowlisted() { + local file=$1 guild=$2 channel=$3 author=$4 rule rguild rchannel rauthor rest + [ -n "$guild" ] && [ -n "$channel" ] || return 1 + [ -f "$file" ] && [ ! -L "$file" ] || return 1 + while IFS= read -r rule || [ -n "$rule" ]; do + [ -n "$rule" ] || continue + rguild=${rule%%:*} + rest=${rule#"$rguild"} + rest=${rest#:} + if [ -z "$rest" ] || [ "$rest" = "$rule" ]; then + [ "$rguild" = "$guild" ] && return 0 + continue + fi + rchannel=${rest%%:*} + rauthor=${rest#"$rchannel"} + rauthor=${rauthor#:} + [ "$rguild" = "$guild" ] || continue + [ "$rchannel" = "$channel" ] || continue + if [ -z "$rauthor" ] || [ "$rauthor" = "$rest" ]; then + return 0 + fi + [ "$rauthor" = "$author" ] && return 0 + done </dev/null +} + +fm_ext_file_mtime() { + local file=$1 + if [ "$(uname)" = Darwin ]; then + stat -f %m "$file" 2>/dev/null + else + stat -c %Y "$file" 2>/dev/null + fi +} + +# True when an existing inflight file may be stolen: age >= TTL and the +# recorded owner pid is dead and is not this claiming process. Live owners +# never steal, even past TTL. Missing pid is fail-closed (not stealable). +fm_ext_outbox_inflight_is_stale() { + local dir=$1 base=$2 dest ttl now pid recorded_at age owner + dest="$dir/$base" + fm_ext_private_artifact_file_valid "$dir" "$base" 600 || return 1 + ttl=$(fm_ext_inflight_ttl_secs) + now=${FM_EXT_NOW_OVERRIDE:-$(date +%s)} + case "$now" in + ''|*[!0-9]*) return 1 ;; + esac + pid=$(jq -er '.pid | select(type=="number")' "$dest" 2>/dev/null) || return 1 + case "$pid" in + ''|*[!0-9]*) return 1 ;; + esac + owner=$(fm_ext_outbox_inflight_owner_pid) + if [ "$pid" = "$$" ] || [ "$pid" = "$owner" ]; then + return 1 + fi + if fm_ext_pid_alive "$pid"; then + return 1 + fi + recorded_at=$(jq -er '.recorded_at | select(type=="number")' "$dest" 2>/dev/null) || recorded_at= + case "$recorded_at" in + ''|*[!0-9]*) + recorded_at=$(fm_ext_file_mtime "$dest") || return 1 + case "$recorded_at" in + ''|*[!0-9]*) return 1 ;; + esac + ;; + esac + age=$((now - recorded_at)) + [ "$age" -ge 0 ] 2>/dev/null || return 1 + [ "$age" -ge "$ttl" ] +} + +# Reclaim a leftover steal-lock directory whose mtime is at least +# max(claim TTL, 1) seconds. A zero claim TTL must not make a live +# steal-lock immediately reclaimable: two concurrent stealers would +# otherwise rmdir each other's lock and both return a send right. +fm_ext_inflight_steallock_ttl_secs() { + local ttl + ttl=$(fm_ext_inflight_ttl_secs) + if [ "$ttl" -lt 1 ]; then + ttl=1 + fi + printf '%s\n' "$ttl" +} + +fm_ext_outbox_inflight_steallock_drop() { + local dir=$1 slug=$2 kind=$3 generation=$4 lock + lock=$(fm_ext_outbox_inflight_steallock_basename "$slug" "$kind" "$generation") || return 0 + rmdir "$dir/$lock" 2>/dev/null || true +} + +fm_ext_outbox_inflight_steallock_stale() { + local dir=$1 base=$2 dest ttl now mtime age + dest="$dir/$base" + [ -d "$dest" ] && [ ! -L "$dest" ] || return 1 + ttl=$(fm_ext_inflight_steallock_ttl_secs) + now=${FM_EXT_NOW_OVERRIDE:-$(date +%s)} + case "$now" in + ''|*[!0-9]*) return 1 ;; + esac + mtime=$(fm_ext_file_mtime "$dest") || return 1 + case "$mtime" in + ''|*[!0-9]*) return 1 ;; + esac + age=$((now - mtime)) + [ "$age" -ge 0 ] 2>/dev/null || return 1 + [ "$age" -ge "$ttl" ] +} + +fm_ext_outbox_inflight_publish_once() { + local dir=$1 slug=$2 kind=$3 generation=$4 inflight now owner + inflight=$(fm_ext_outbox_inflight_basename "$slug" "$kind" "$generation") || return 2 + now=${FM_EXT_NOW_OVERRIDE:-$(date +%s)} + case "$now" in + ''|*[!0-9]*) return 2 ;; + esac + owner=$(fm_ext_outbox_inflight_owner_pid) + case "$owner" in + ''|*[!0-9]*) return 2 ;; + esac + jq -cn --arg slug "$slug" --arg kind "$kind" --argjson generation "$generation" \ + --argjson recorded_at "$now" --argjson pid "$owner" \ + '{slug:$slug, kind:$kind, generation:$generation, recorded_at:$recorded_at, pid:$pid}' \ + | fm_ext_private_artifact_publish_stdin_once "$dir" "$inflight" 600 +} + +# CAS-claim the exclusive send marker. Returns 0 when this caller owns the +# next send, 1 when another valid inflight marker already holds it (live +# owner, or dead owner still inside the TTL), and 2 on validation or +# publication failure. A dead owner whose claim is at least +# FM_EXT_INFLIGHT_TTL_SECS (default 30) old may be stolen: the steal is +# serialized with a momentary lock directory whose reclaim floor is +# max(TTL, 1) so TTL=0 cannot reopen dual claim. +fm_ext_outbox_inflight_claim() { + local dir=$1 slug=$2 kind=$3 generation=$4 inflight lock rc + inflight=$(fm_ext_outbox_inflight_basename "$slug" "$kind" "$generation") || return 2 + lock=$(fm_ext_outbox_inflight_steallock_basename "$slug" "$kind" "$generation") || return 2 + fm_ext_outbox_inflight_publish_once "$dir" "$slug" "$kind" "$generation" + rc=$? + case "$rc" in + 0) return 0 ;; + 1) ;; + *) return 2 ;; + esac + if ! fm_ext_outbox_inflight_is_stale "$dir" "$inflight"; then + return 1 + fi + if ! mkdir "$dir/$lock" 2>/dev/null; then + if fm_ext_outbox_inflight_steallock_stale "$dir" "$lock"; then + rmdir "$dir/$lock" 2>/dev/null || return 1 + mkdir "$dir/$lock" 2>/dev/null || return 1 + else + return 1 + fi + fi + if ! fm_ext_private_artifact_file_valid "$dir" "$inflight" 600; then + fm_ext_outbox_inflight_publish_once "$dir" "$slug" "$kind" "$generation" + rc=$? + rmdir "$dir/$lock" 2>/dev/null || true + case "$rc" in + 0) return 0 ;; + 1) return 1 ;; + *) return 2 ;; + esac + fi + if ! fm_ext_outbox_inflight_is_stale "$dir" "$inflight"; then + rmdir "$dir/$lock" 2>/dev/null || true + return 1 + fi + if ! fm_ext_private_artifact_remove "$dir" "$inflight" 600; then + rmdir "$dir/$lock" 2>/dev/null || true + return 2 + fi + fm_ext_outbox_inflight_publish_once "$dir" "$slug" "$kind" "$generation" + rc=$? + rmdir "$dir/$lock" 2>/dev/null || true + case "$rc" in + 0) return 0 ;; + 1) return 1 ;; + *) return 2 ;; + esac +} + +# Drop the exclusive send marker. Returns 0 when the path is absent or this +# caller deleted a valid inflight marker, 1 when the path exists but is not +# a safe private artifact, and 2 on an unsafe identity. Also drops a leftover +# steal-lock directory. +fm_ext_outbox_inflight_release() { + local dir=$1 slug=$2 kind=$3 generation=$4 inflight rc + inflight=$(fm_ext_outbox_inflight_basename "$slug" "$kind" "$generation") || return 2 + fm_ext_private_artifact_remove "$dir" "$inflight" 600 + rc=$? + fm_ext_outbox_inflight_steallock_drop "$dir" "$slug" "$kind" "$generation" + return "$rc" +} + +# Discord per-message split budget. Copies the FMX_DISCORD_REPLY_MAX_CHARS +# clamp (default 1900, min 50, values above 2000 reset to 1900) without +# reading X-mode env or requiring FMX_PAIRING_TOKEN. +fm_ext_discord_reply_max_chars() { + local raw=${1:-${FM_EXT_DISCORD_REPLY_MAX_CHARS-}} + case "$raw" in + ''|*[!0-9]*) raw=1900 ;; + esac + [ "$raw" -ge 50 ] 2>/dev/null || raw=50 + [ "$raw" -le 2000 ] 2>/dev/null || raw=1900 + printf '%s\n' "$raw" +} + +# Maximum messages in one auto-split Discord thread. Copies FMX_X_THREAD_MAX +# (default 25) without reading X-mode env. +fm_ext_discord_thread_max() { + local raw=${1:-${FM_EXT_DISCORD_THREAD_MAX-}} + case "$raw" in + ''|*[!0-9]*) raw=25 ;; + esac + [ "$raw" -ge 1 ] 2>/dev/null || raw=25 + printf '%s\n' "$raw" +} + +# Split a reply into a numbered thread of <=-codepoint chunks. +# Copied from fmx_split_thread in bin/fm-x-lib.sh; do not source that file. +# Reads the reply text on stdin and prints a compact JSON array of chunks. +fm_ext_split_thread() { + jq -Rsc --argjson limit "$1" --argjson cap "$2" ' + def trim: gsub("^[[:space:]]+|[[:space:]]+$"; ""); + def fence_marker: test("^[[:space:]]*```"); + def fence_count: ((split("```") | length) - 1); + def numbered($i; $n): + "(\($i + 1)/\($n))" as $mark + | if ((fence_count % 2) == 0) and (split("\n")[-1] | fence_marker) + then . + "\n" + $mark + else . + " " + $mark + end; + def hardsplit($b): . as $s | [range(0; ($s|length); $b) as $i | $s[$i:$i+$b]]; + def wordsplit($b): + (gsub("[[:space:]]+"; " ") | trim) as $norm + | if ($norm | length) == 0 then [] + else + [ $norm | split(" ")[] | if (length > $b) then hardsplit($b)[] else . end ] as $words + | (reduce $words[] as $w ({chunks: [], cur: ""}; + (if .cur == "" then $w else .cur + " " + $w end) as $cand + | if ($cand | length) <= $b then .cur = $cand + else .chunks += (if .cur == "" then [] else [.cur] end) | .cur = $w end + )) as $st + | $st.chunks + (if $st.cur != "" then [$st.cur] else [] end) + end; + def split_units: + split("\n") as $lines + | (reduce $lines[] as $line ({units: [], cur: "", fence: false}; + if .fence then + .cur = (if .cur == "" then $line else .cur + "\n" + $line end) + | if ($line | fence_marker) then .units += [.cur] | .cur = "" | .fence = false else . end + elif ($line | fence_marker) then + (if .cur != "" then .units += [.cur] | .cur = "" else . end) + | .cur = $line + | .fence = true + elif ($line | test("^[[:space:]]*$")) then + if .cur != "" then .units += [.cur] | .cur = "" else . end + else + ($line | trim) as $clean + | .cur = (if .cur == "" then $clean else .cur + " " + $clean end) + end + )) as $st + | ($st.units + (if $st.cur != "" then [$st.cur] else [] end)) + | map(select((trim | length) > 0)); + def pack_units($units; $b): + (reduce $units[] as $u ({chunks: [], cur: ""}; + if ($u | length) > $b then + (if .cur != "" then .chunks += [.cur] | .cur = "" else . end) + | .chunks += ($u | wordsplit($b)) + else + (if .cur == "" then $u else .cur + "\n\n" + $u end) as $cand + | if ($cand | length) <= $b then .cur = $cand + else .chunks += (if .cur == "" then [] else [.cur] end) | .cur = $u end + end + )) as $st + | $st.chunks + (if $st.cur != "" then [$st.cur] else [] end); + def split_thread($limit; $cap): + trim as $norm + | if ($norm | length) == 0 then [] + elif ($norm | length) <= $limit then [$norm] + else + ($cap | tostring | length) as $digits + | (4 + 2 * $digits) as $suffixw + | (if ($limit - $suffixw - 1) < 1 then 1 else ($limit - $suffixw - 1) end) as $budget + | ($norm | split_units) as $units + | pack_units($units; $budget) as $raw + | (if ($raw | length) > $cap + then ($raw[0:$cap] | (.[($cap - 1)] += "…")) + else $raw end) as $kept + | ($kept | length) as $n + | [ range(0; $n) as $i | $kept[$i] | numbered($i; $n) ] + end; + split_thread($limit; $cap) + ' +} + +# fm_ext_outbox_schema_valid : payload has required fields and matching slug. +fm_ext_outbox_schema_valid() { + local file=$1 + [ -f "$file" ] && [ ! -L "$file" ] || return 1 + jq -e --argjson schema "$FM_EXT_SCHEMA_VERSION" ' + .schema_version == $schema + and (.request_id | type == "string" and startswith("discord:")) + and (.slug | type == "string" and test("^[0-9a-f]{64}$")) + and (.kind | . == "ack" or . == "answer" or . == "followup" or . == "final") + and (.generation | type == "number" and . >= 1) + and (.platform == "discord") + and (.source == "hermes-gateway") + and (.guild_id | type == "string" and test("^[0-9]+$")) + and (.channel_id | type == "string" and test("^[0-9]+$")) + and (.thread_id | type == "string" and test("^[0-9]+$")) + and (.message_id | type == "string" and test("^[0-9]+$")) + and (.text | type == "string") + ' "$file" >/dev/null 2>&1 +} + +# Begin delivery: CAS the posting marker, then CAS an exclusive inflight +# send marker before returning a send right. Returns 0 on a new claim, a +# resumable split this caller exclusively claimed, or a steal of a dead +# owner past FM_EXT_INFLIGHT_TTL_SECS, 1 when a valid receipt already +# exists (idempotent success), 4 when a terminal failed marker exists, 3 +# when a posting marker exists without a receipt and this caller does not +# own the next send (live owner, or dead owner still inside the TTL), 2 +# on validation/publication failure. JSON progress with inflight==null is +# not itself a shared claim. +fm_ext_outbox_begin() { + local dir=$1 slug=$2 kind=$3 generation=$4 payload posting receipt failed progress now rc + payload=$(fm_ext_outbox_basename "$slug" "$kind" "$generation") || return 2 + posting=$(fm_ext_outbox_posting_basename "$slug" "$kind" "$generation") || return 2 + receipt=$(fm_ext_outbox_receipt_basename "$slug" "$kind" "$generation") || return 2 + failed=$(fm_ext_outbox_failed_basename "$slug" "$kind" "$generation") || return 2 + progress=$(fm_ext_outbox_progress_basename "$slug" "$kind" "$generation") || return 2 + fm_ext_private_artifact_file_valid "$dir" "$payload" 600 || return 2 + if fm_ext_private_artifact_file_valid "$dir" "$receipt" 600; then + return 1 + fi + if fm_ext_private_artifact_file_valid "$dir" "$failed" 600; then + return 4 + fi + if fm_ext_private_artifact_file_valid "$dir" "$posting" 600; then + if fm_ext_outbox_progress_resumable "$dir" "$slug" "$kind" "$generation"; then + if jq -e '.posted_count >= .total' "$dir/$progress" >/dev/null 2>&1; then + return 0 + fi + fm_ext_outbox_inflight_claim "$dir" "$slug" "$kind" "$generation" + rc=$? + case "$rc" in + 0) return 0 ;; + 1) return 3 ;; + *) return 2 ;; + esac + fi + return 3 + fi + now=${FM_EXT_NOW_OVERRIDE:-$(date +%s)} + case "$now" in + ''|*[!0-9]*) return 2 ;; + esac + jq -cn --arg slug "$slug" --arg kind "$kind" --argjson generation "$generation" \ + --argjson recorded_at "$now" \ + '{slug:$slug, kind:$kind, generation:$generation, recorded_at:$recorded_at}' \ + | fm_ext_private_artifact_publish_stdin_once "$dir" "$posting" 600 + rc=$? + case "$rc" in + 0) + fm_ext_outbox_inflight_claim "$dir" "$slug" "$kind" "$generation" + rc=$? + case "$rc" in + 0) return 0 ;; + 1) return 3 ;; + *) return 2 ;; + esac + ;; + 1) + if fm_ext_private_artifact_file_valid "$dir" "$receipt" 600; then + return 1 + fi + if fm_ext_private_artifact_file_valid "$dir" "$failed" 600; then + return 4 + fi + return 3 + ;; + *) return 2 ;; + esac +} + +# Record a delivery receipt once. Returns 0 on create, 1 when a valid receipt +# already exists, 2 on failure. The posting marker is left in place so a +# later begin still sees mid-delivery-or-receipt and refuses a second send. +# A leftover inflight send marker is released after a successful or already +# present receipt. +fm_ext_outbox_receipt() { + local dir=$1 slug=$2 kind=$3 generation=$4 receipt_json=$5 receipt rc + receipt=$(fm_ext_outbox_receipt_basename "$slug" "$kind" "$generation") || return 2 + [ -n "$receipt_json" ] || return 2 + printf '%s\n' "$receipt_json" \ + | fm_ext_private_artifact_publish_stdin_once "$dir" "$receipt" 600 + rc=$? + case "$rc" in + 0|1) fm_ext_outbox_inflight_release "$dir" "$slug" "$kind" "$generation" || true ;; + esac + return "$rc" +} + +# Drop the posting marker after a transient definite send failure (HTTP 429 +# or 5xx) that happened before a successful response. Returns 0 when the +# generation is retryable (no marker, or this caller deleted a valid posting +# marker), 1 when a receipt or terminal failed marker already exists (do not +# reopen), and 2 on validation or deletion failure. An ambiguous crash or +# transport error after the post started keeps the marker; this helper is +# only for the transient definite-failure path. +fm_ext_outbox_abort() { + local dir=$1 slug=$2 kind=$3 generation=$4 posting receipt failed progress + posting=$(fm_ext_outbox_posting_basename "$slug" "$kind" "$generation") || return 2 + receipt=$(fm_ext_outbox_receipt_basename "$slug" "$kind" "$generation") || return 2 + failed=$(fm_ext_outbox_failed_basename "$slug" "$kind" "$generation") || return 2 + progress=$(fm_ext_outbox_progress_basename "$slug" "$kind" "$generation") || return 2 + if fm_ext_private_artifact_file_valid "$dir" "$receipt" 600; then + return 1 + fi + if fm_ext_private_artifact_file_valid "$dir" "$failed" 600; then + return 1 + fi + # Release inflight first: abort is pre-success-only, so dropping the send + # claim cannot reopen a confirmed post. A crash after posting/progress + # removal must not leave a stale inflight that makes the next begin stick. + fm_ext_outbox_inflight_release "$dir" "$slug" "$kind" "$generation" || return 2 + fm_ext_private_artifact_remove "$dir" "$posting" 600 || return 2 + fm_ext_private_artifact_remove "$dir" "$progress" 600 || return 2 + return 0 +} + +# Record a terminal delivery failure so pending will not retry this generation. +# Returns 0 on create, 1 when a valid receipt already exists (do not reopen), +# 4 when a valid failed marker already exists (idempotent), and 2 on failure. +# Removes the posting marker after a successful failed publication so a later +# begin sees terminal-failed rather than mid-delivery. +fm_ext_outbox_fail() { + local dir=$1 slug=$2 kind=$3 generation=$4 reason_json=$5 failed posting receipt rc + failed=$(fm_ext_outbox_failed_basename "$slug" "$kind" "$generation") || return 2 + posting=$(fm_ext_outbox_posting_basename "$slug" "$kind" "$generation") || return 2 + receipt=$(fm_ext_outbox_receipt_basename "$slug" "$kind" "$generation") || return 2 + [ -n "$reason_json" ] || return 2 + if fm_ext_private_artifact_file_valid "$dir" "$receipt" 600; then + return 1 + fi + if fm_ext_private_artifact_file_valid "$dir" "$failed" 600; then + fm_ext_private_artifact_remove "$dir" "$posting" 600 || true + fm_ext_outbox_inflight_release "$dir" "$slug" "$kind" "$generation" || true + return 4 + fi + printf '%s\n' "$reason_json" \ + | fm_ext_private_artifact_publish_stdin_once "$dir" "$failed" 600 + rc=$? + case "$rc" in + 0) + fm_ext_private_artifact_remove "$dir" "$posting" 600 || true + fm_ext_outbox_inflight_release "$dir" "$slug" "$kind" "$generation" || true + return 0 + ;; + 1) + if fm_ext_private_artifact_file_valid "$dir" "$receipt" 600; then + fm_ext_outbox_inflight_release "$dir" "$slug" "$kind" "$generation" || true + return 1 + fi + fm_ext_private_artifact_remove "$dir" "$posting" 600 || true + fm_ext_outbox_inflight_release "$dir" "$slug" "$kind" "$generation" || true + return 4 + ;; + *) return 2 ;; + esac +} + +# True when JSON progress looks resumable: progress exists, no chunk is +# recorded in-flight, and posted_count/total are numbers. Exclusive send +# ownership is the inflight file CAS in begin, not this JSON check. +fm_ext_outbox_progress_resumable() { + local dir=$1 slug=$2 kind=$3 generation=$4 progress + progress=$(fm_ext_outbox_progress_basename "$slug" "$kind" "$generation") || return 1 + fm_ext_private_artifact_file_valid "$dir" "$progress" 600 || return 1 + jq -e '.inflight == null + and (.posted_count | type == "number") + and (.total | type == "number")' "$dir/$progress" >/dev/null 2>&1 +} + +# Replace the chunk-progress artifact. Returns 0 on write, 2 on failure. +fm_ext_outbox_progress() { + local dir=$1 slug=$2 kind=$3 generation=$4 progress_json=$5 progress + progress=$(fm_ext_outbox_progress_basename "$slug" "$kind" "$generation") || return 2 + [ -n "$progress_json" ] || return 2 + printf '%s\n' "$progress_json" \ + | fm_ext_private_artifact_publish_stdin "$dir" "$progress" 600 +} + +# --- poll shim -------------------------------------------------------------- + +fm_ext_poll_shim_content() { + local home=$1 root=$2 + printf '%s\n' \ + '#!/usr/bin/env bash' \ + '# Auto-generated by fm-bootstrap.sh - local ext-bridge poll shim.' \ + '# The watcher validates these bytes, then dispatches the trusted poll script.' \ + "export FM_HOME=$(printf '%q' "$home")" \ + "exec $(printf '%q' "$root/bin/fm-ext-poll.sh")" +} + +fm_ext_poll_shim_valid() { + local file=$1 home=$2 root=$3 + fm_ext_single_link_file_mode_valid "$file" 700 || return 1 + cmp -s "$file" <(fm_ext_poll_shim_content "$home" "$root") +} + +# --- task meta link (not x_request=) ---------------------------------------- + +fm_ext_meta_get() { + local meta=$1 key=$2 line + [ -f "$meta" ] || return 0 + line=$(grep -E "^${key}=" "$meta" 2>/dev/null | tail -n1) || return 0 + [ -n "$line" ] || return 0 + printf '%s' "${line#*=}" +} + +fm_ext_meta_tmp() { + local meta=$1 dir base + dir=${meta%/*} + base=${meta##*/} + [ "$dir" != "$meta" ] || dir=. + [ -d "$dir" ] || return 1 + mktemp "$dir/.${base}.fm-ext.XXXXXX" +} + +# fm_ext_meta_link_set [followups] +fm_ext_meta_link_set() { + local meta=$1 rid=$2 slug=$3 ts=$4 followups=${5:-0} tmp + [ -f "$meta" ] || return 1 + tmp=$(fm_ext_meta_tmp "$meta") || return 1 + if ! { grep -vE '^ext_request=|^ext_request_slug=|^ext_request_ts=|^ext_followups=' "$meta" || true; } > "$tmp"; then + rm -f "$tmp"; return 1 + fi + printf 'ext_request=%s\n' "$rid" >> "$tmp" || { rm -f "$tmp"; return 1; } + printf 'ext_request_slug=%s\n' "$slug" >> "$tmp" || { rm -f "$tmp"; return 1; } + printf 'ext_request_ts=%s\n' "$ts" >> "$tmp" || { rm -f "$tmp"; return 1; } + printf 'ext_followups=%s\n' "$followups" >> "$tmp" || { rm -f "$tmp"; return 1; } + mv -f "$tmp" "$meta" || { rm -f "$tmp"; return 1; } +} + +fm_ext_meta_followups_set() { + local meta=$1 n=$2 tmp + [ -f "$meta" ] || return 1 + tmp=$(fm_ext_meta_tmp "$meta") || return 1 + if ! { grep -vE '^ext_followups=' "$meta" || true; } > "$tmp"; then + rm -f "$tmp"; return 1 + fi + printf 'ext_followups=%s\n' "$n" >> "$tmp" || { rm -f "$tmp"; return 1; } + mv -f "$tmp" "$meta" || { rm -f "$tmp"; return 1; } +} + +# Silence unused-dir lint when this file is sourced for helpers only. +: "$_FM_EXT_LIB_DIR" diff --git a/bin/fm-ext-link.sh b/bin/fm-ext-link.sh new file mode 100755 index 00000000000..79ed985f288 --- /dev/null +++ b/bin/fm-ext-link.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Bind a spawned task to a local Communication Officer request_id. +# +# Usage: fm-ext-link.sh +# +# Records link lines in state/.meta (replacing any prior ext link, +# preserving every other meta line): +# ext_request= +# ext_request_slug= +# ext_request_ts= +# ext_followups= +# +# This is deliberately not x_request=. The hosted X-mode relay is a separate +# seam. Fresh links start ext_followups at 0. +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" +# shellcheck source=bin/fm-ext-lib.sh +. "$SCRIPT_DIR/fm-ext-lib.sh" +# shellcheck source=bin/fm-pr-lib.sh +. "$SCRIPT_DIR/fm-pr-lib.sh" + +usage() { + echo "usage: fm-ext-link.sh " >&2 +} + +help() { + sed -n '2,/^set -u$/p' "$0" | sed '$d; s/^# \{0,1\}//' +} + +case "${1:-}" in + --help|-h) help; exit 0 ;; +esac + +ID=${1:-} +RID=${2:-} +if [ -z "$ID" ] || [ -z "$RID" ]; then + usage + exit 2 +fi + +fm_pr_task_id_valid "$ID" || { echo "fm-ext-link: unsafe task id: $ID" >&2; exit 2; } +fm_ext_request_id_valid "$RID" || { echo "fm-ext-link: unsafe request_id: $RID" >&2; exit 2; } + +META="$STATE/$ID.meta" +if [ ! -f "$META" ]; then + echo "fm-ext-link: no such task: state/$ID.meta" >&2 + exit 1 +fi + +SLUG=$(fm_ext_request_slug "$RID") || { + echo "fm-ext-link: could not derive request slug" >&2 + exit 1 +} + +LINK_TS=${FM_EXT_NOW_OVERRIDE:-$(date +%s)} +case "$LINK_TS" in + ''|*[!0-9]*) echo "fm-ext-link: could not read the current time" >&2; exit 1 ;; +esac + +if ! fm_ext_meta_link_set "$META" "$RID" "$SLUG" "$LINK_TS" 0; then + echo "fm-ext-link: failed to record the link in state/$ID.meta" >&2 + exit 1 +fi + +printf 'linked %s to ext request %s\n' "$ID" "$RID" diff --git a/bin/fm-ext-outbox.sh b/bin/fm-ext-outbox.sh new file mode 100755 index 00000000000..a934de8d573 --- /dev/null +++ b/bin/fm-ext-outbox.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# Gateway-facing outbox delivery helpers for the local Communication Officer. +# +# Usage: +# fm-ext-outbox.sh pending +# fm-ext-outbox.sh begin --slug --kind --generation +# fm-ext-outbox.sh receipt --slug --kind --generation +# --receipt-file +# fm-ext-outbox.sh abort --slug --kind --generation +# fm-ext-outbox.sh fail --slug --kind --generation +# --reason-file +# fm-ext-outbox.sh progress --slug --kind --generation +# --progress-file +# fm-ext-outbox.sh release --slug --kind --generation +# fm-ext-outbox.sh split [--max ] [--cap ] +# +# begin CAS-claims the posting marker, then CAS-claims an exclusive inflight +# send marker (recording owner pid and recorded_at) before returning a send +# right. Exit 0 on a new claim, a resumable split this caller exclusively +# claimed, or a steal of a dead-owner claim older than +# FM_EXT_INFLIGHT_TTL_SECS (default 30). Exit 1 when a receipt already +# exists (idempotent success), 3 on mid-delivery (posting without this +# caller owning the next send, including a live owner inside or past the +# TTL), 4 when a terminal failed marker exists, 2 on validation failure. +# Two concurrent live posters cannot both get the send right for the same +# generation and chunk. Steal serialization keeps a 1-second floor so +# FM_EXT_INFLIGHT_TTL_SECS=0 cannot let two stealers both win. receipt +# writes the receipt once and releases a leftover inflight marker. +# abort releases the exclusive inflight send marker first, then deletes the +# posting marker and chunk progress, after a transient definite send failure +# (HTTP 429 or 5xx) before any chunk succeeded so that generation can retry. +# It refuses when a receipt or terminal failed marker already exists. An +# ambiguous crash or transport error after a chunk post started keeps the +# posting and inflight markers. +# fail records a terminal failed marker after a permanent 4xx so pending +# stops retrying that generation, and drops posting plus inflight. +# progress replaces the durable per-chunk progress artifact. +# release drops the exclusive inflight send marker after a later-chunk +# transient failure so another poster may resume remaining chunks. The +# posting marker and progress stay in place. +# split reads reply text on stdin and prints {limit,cap,texts} using +# FM_EXT_DISCORD_REPLY_MAX_CHARS (default 1900) and FM_EXT_DISCORD_THREAD_MAX +# (default 25). It does not require FMX_PAIRING_TOKEN or the hosted relay. +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +# shellcheck source=bin/fm-ext-lib.sh +. "$SCRIPT_DIR/fm-ext-lib.sh" + +usage() { + cat >&2 <<'EOF' +usage: fm-ext-outbox.sh pending + fm-ext-outbox.sh begin --slug --kind --generation + fm-ext-outbox.sh receipt --slug --kind --generation --receipt-file + fm-ext-outbox.sh abort --slug --kind --generation + fm-ext-outbox.sh fail --slug --kind --generation --reason-file + fm-ext-outbox.sh progress --slug --kind --generation --progress-file + fm-ext-outbox.sh release --slug --kind --generation + fm-ext-outbox.sh split [--max ] [--cap ] +EOF +} + +help() { + sed -n '2,/^set -u$/p' "$0" | sed '$d; s/^# \{0,1\}//' +} + +die() { printf 'fm-ext-outbox: %s\n' "$1" >&2; exit "${2:-2}"; } + +cmd=${1:-} +case "$cmd" in + --help|-h) help; exit 0 ;; + '') usage; exit 2 ;; +esac +shift || true + +SLUG= +KIND= +GENERATION= +RECEIPT_FILE= +REASON_FILE= +PROGRESS_FILE= +MAX_CHARS= +THREAD_CAP= + +while [ "$#" -gt 0 ]; do + case "$1" in + --slug) shift; SLUG=${1:-} ;; + --kind) shift; KIND=${1:-} ;; + --generation) shift; GENERATION=${1:-} ;; + --receipt-file) shift; RECEIPT_FILE=${1:-} ;; + --reason-file) shift; REASON_FILE=${1:-} ;; + --progress-file) shift; PROGRESS_FILE=${1:-} ;; + --max) shift; MAX_CHARS=${1:-} ;; + --cap) shift; THREAD_CAP=${1:-} ;; + --help|-h) help; exit 0 ;; + *) die "unknown argument '$1'" ;; + esac + shift || true +done + +case "$cmd" in + split) + command -v jq >/dev/null 2>&1 || die "jq is required" 1 + max=$(fm_ext_discord_reply_max_chars "$MAX_CHARS") + cap=$(fm_ext_discord_thread_max "$THREAD_CAP") + texts=$(fm_ext_split_thread "$max" "$cap") || die "could not split the reply" 2 + jq -cn --argjson limit "$max" --argjson cap "$cap" --argjson texts "$texts" \ + '{limit:$limit, cap:$cap, texts:$texts}' \ + || die "could not encode the split result" 2 + exit 0 + ;; +esac + +OUTBOX=$(fm_ext_outbox_dir) + +case "$cmd" in + pending) + fm_ext_active "$FM_HOME" || exit 0 + [ -d "$OUTBOX" ] && [ ! -L "$OUTBOX" ] || exit 0 + for file in "$OUTBOX"/*.json; do + [ -e "$file" ] || continue + base=$(basename "$file") + case "$base" in + *.receipt.json|*.failed.json|*.progress.json) continue ;; + esac + fm_ext_outbox_schema_valid "$file" || continue + slug=$(jq -r '.slug' "$file") + kind=$(jq -r '.kind' "$file") + generation=$(jq -r '.generation' "$file") + receipt=$(fm_ext_outbox_receipt_basename "$slug" "$kind" "$generation") || continue + failed=$(fm_ext_outbox_failed_basename "$slug" "$kind" "$generation") || continue + if fm_ext_private_artifact_file_valid "$OUTBOX" "$receipt" 600; then + continue + fi + if fm_ext_private_artifact_file_valid "$OUTBOX" "$failed" 600; then + continue + fi + printf '%s\n' "$file" + done + ;; + begin) + fm_ext_slug_valid "$SLUG" || die "unsafe slug" + fm_ext_kind_valid "$KIND" || die "invalid kind" + fm_ext_generation_valid "$GENERATION" || die "invalid generation" + fm_ext_outbox_begin "$OUTBOX" "$SLUG" "$KIND" "$GENERATION" + rc=$? + case "$rc" in + 0) printf 'claimed %s %s %s\n' "$SLUG" "$KIND" "$GENERATION" ;; + 1) printf 'already-receipted %s %s %s\n' "$SLUG" "$KIND" "$GENERATION" ;; + 3) die "mid-delivery: $KIND generation $GENERATION is posting and has no receipt" 3 ;; + 4) printf 'terminal-failed %s %s %s\n' "$SLUG" "$KIND" "$GENERATION" ;; + *) die "could not begin delivery" 2 ;; + esac + exit "$rc" + ;; + receipt) + fm_ext_slug_valid "$SLUG" || die "unsafe slug" + fm_ext_kind_valid "$KIND" || die "invalid kind" + fm_ext_generation_valid "$GENERATION" || die "invalid generation" + [ -f "$RECEIPT_FILE" ] || die "receipt file not found: $RECEIPT_FILE" + body=$(cat -- "$RECEIPT_FILE") + [ -n "$body" ] || die "receipt file is empty" + fm_ext_outbox_receipt "$OUTBOX" "$SLUG" "$KIND" "$GENERATION" "$body" + rc=$? + case "$rc" in + 0) printf 'receipted %s %s %s\n' "$SLUG" "$KIND" "$GENERATION" ;; + 1) printf 'already-receipted %s %s %s\n' "$SLUG" "$KIND" "$GENERATION" ;; + *) die "could not record the receipt" 2 ;; + esac + exit "$rc" + ;; + abort) + fm_ext_slug_valid "$SLUG" || die "unsafe slug" + fm_ext_kind_valid "$KIND" || die "invalid kind" + fm_ext_generation_valid "$GENERATION" || die "invalid generation" + fm_ext_outbox_abort "$OUTBOX" "$SLUG" "$KIND" "$GENERATION" + rc=$? + case "$rc" in + 0) printf 'aborted %s %s %s\n' "$SLUG" "$KIND" "$GENERATION" ;; + 1) printf 'already-receipted %s %s %s\n' "$SLUG" "$KIND" "$GENERATION" ;; + *) die "could not abort delivery" 2 ;; + esac + exit "$rc" + ;; + fail) + fm_ext_slug_valid "$SLUG" || die "unsafe slug" + fm_ext_kind_valid "$KIND" || die "invalid kind" + fm_ext_generation_valid "$GENERATION" || die "invalid generation" + [ -f "$REASON_FILE" ] || die "reason file not found: $REASON_FILE" + body=$(cat -- "$REASON_FILE") + [ -n "$body" ] || die "reason file is empty" + fm_ext_outbox_fail "$OUTBOX" "$SLUG" "$KIND" "$GENERATION" "$body" + rc=$? + case "$rc" in + 0) printf 'failed %s %s %s\n' "$SLUG" "$KIND" "$GENERATION" ;; + 1) printf 'already-receipted %s %s %s\n' "$SLUG" "$KIND" "$GENERATION" ;; + 4) printf 'already-failed %s %s %s\n' "$SLUG" "$KIND" "$GENERATION" ;; + *) die "could not record the terminal failure" 2 ;; + esac + exit "$rc" + ;; + progress) + fm_ext_slug_valid "$SLUG" || die "unsafe slug" + fm_ext_kind_valid "$KIND" || die "invalid kind" + fm_ext_generation_valid "$GENERATION" || die "invalid generation" + [ -f "$PROGRESS_FILE" ] || die "progress file not found: $PROGRESS_FILE" + body=$(cat -- "$PROGRESS_FILE") + [ -n "$body" ] || die "progress file is empty" + fm_ext_outbox_progress "$OUTBOX" "$SLUG" "$KIND" "$GENERATION" "$body" \ + || die "could not record chunk progress" 2 + printf 'progress %s %s %s\n' "$SLUG" "$KIND" "$GENERATION" + ;; + release) + fm_ext_slug_valid "$SLUG" || die "unsafe slug" + fm_ext_kind_valid "$KIND" || die "invalid kind" + fm_ext_generation_valid "$GENERATION" || die "invalid generation" + fm_ext_outbox_inflight_release "$OUTBOX" "$SLUG" "$KIND" "$GENERATION" \ + || die "could not release the inflight send marker" 2 + printf 'released %s %s %s\n' "$SLUG" "$KIND" "$GENERATION" + ;; + *) usage; exit 2 ;; +esac diff --git a/bin/fm-ext-poll.sh b/bin/fm-ext-poll.sh new file mode 100755 index 00000000000..41be41a55bd --- /dev/null +++ b/bin/fm-ext-poll.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# One slow-check of the local Communication Officer inbox. +# +# Inert by default: a HARD no-op (exit 0, no output) unless the local +# ext-bridge is active (config/ext-bridge or FM_EXT_BRIDGE=1 plus a valid +# secret file). The watcher invokes this trusted repository script only after +# state/ext-watch.check.sh matches the expected byte-static identity shim. +# +# Prints one line `ext-request ` for each newly claimed offer that still +# has an inbox file. An already claimed offer stays silent, including after +# Firstmate restart, so each offer wakes once. +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" +# shellcheck source=bin/fm-ext-lib.sh +. "$SCRIPT_DIR/fm-ext-lib.sh" + +fm_ext_active "$FM_HOME" || exit 0 +command -v jq >/dev/null 2>&1 || exit 0 + +INBOX=$(fm_ext_inbox_dir) +CONTEXT=$(fm_ext_context_dir) +[ -d "$INBOX" ] && [ ! -L "$INBOX" ] || exit 0 + +for file in "$INBOX"/*.json; do + [ -e "$file" ] || continue + base=$(basename "$file") + slug=${base%.json} + fm_ext_slug_valid "$slug" || continue + fm_ext_private_artifact_file_valid "$INBOX" "$base" 600 || continue + rid=$(jq -r '.request_id // empty' "$file" 2>/dev/null) || continue + fm_ext_request_id_valid "$rid" || continue + expected=$(fm_ext_request_slug "$rid") || continue + [ "$expected" = "$slug" ] || continue + if fm_ext_private_artifact_file_valid "$CONTEXT" "$slug.offered.json" 600; then + continue + fi + fm_ext_offer_registry_claim "$STATE" "$slug" "$rid" + case $? in + 0) printf 'ext-request %s\n' "$slug" ;; + 1) continue ;; + *) continue ;; + esac +done + +exit 0 diff --git a/bin/fm-supervision-lib.sh b/bin/fm-supervision-lib.sh index 252d0c93c21..c930168455a 100644 --- a/bin/fm-supervision-lib.sh +++ b/bin/fm-supervision-lib.sh @@ -3,8 +3,9 @@ # Usage: . bin/fm-supervision-lib.sh # # Reports whether a firstmate home needs supervision because it has in-flight -# work (a state/.meta exists) or an X-mode relay poll -# (state/x-watch.check.sh), and whether its watcher has a fresh liveness beacon +# work (a state/.meta exists), an X-mode relay poll +# (state/x-watch.check.sh), a local Communication Officer poll +# (state/ext-watch.check.sh), and whether its watcher has a fresh liveness beacon # (state/.last-watcher-beat, touched every poll cycle, within the grace window). # bin/fm-turnend-guard.sh uses the PID-strict fm_watcher_healthy from # bin/fm-wake-lib.sh for its block decision. bin/fm-guard.sh uses the model-aware @@ -27,7 +28,8 @@ fm_sup_stat_mtime() { # Populates, for the state dir at $1: # FM_SUP_IN_FLIGHT count of state/*.meta (in-flight tasks) # FM_SUP_SOURCES count of registered process-to-event sources -# FM_SUP_NEEDED true/false - in-flight work, an X-mode relay poll, or a +# FM_SUP_NEEDED true/false - in-flight work, an X-mode relay poll, a +# local ext-bridge poll, or a # registered event source (a source is a wait on an # external process, not a task, so it has no metadata) # FM_SUP_WATCHER_FRESH true/false - a watcher beacon within the grace window @@ -54,6 +56,7 @@ fm_supervision_status() { done if [ "$FM_SUP_IN_FLIGHT" -gt 0 ] \ || [ -f "$state/x-watch.check.sh" ] \ + || [ -f "$state/ext-watch.check.sh" ] \ || [ "$FM_SUP_SOURCES" -gt 0 ]; then FM_SUP_NEEDED=true fi diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 47fdda744f1..a44cd5acb7e 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -206,7 +206,7 @@ family_for_basename() { printf '%s\n' backend-dispatch ;; fm-check-unregister.test.sh|fm-pr-check-security.test.sh|fm-pr-merge.test.sh|\ - fm-review-diff.test.sh|fm-teardown.test.sh|fm-x-mode.test.sh) + fm-review-diff.test.sh|fm-teardown.test.sh|fm-x-mode.test.sh|fm-ext-bridge.test.sh) printf '%s\n' pr-forge ;; fm-afk-inject-e2e.test.sh|fm-afk-return.test.sh) @@ -402,6 +402,7 @@ tests/fm-claude-stop-autoarm.test.sh 60630 tests/fm-codex-continuity-live-e2e.test.sh 19 tests/fm-daemon.test.sh 32136 tests/fm-documentation-audiences.test.sh 708 +tests/fm-ext-bridge.test.sh 22000 tests/fm-fleet-snapshot-view.test.sh 7565 tests/fm-fleet-sync.test.sh 36367 tests/fm-gate-refuse.test.sh 8943 @@ -1005,7 +1006,7 @@ families_for_changed_path() { [ "$path" != bin/fm-bootstrap.sh ] || printf '%s\n' secondmate ;; bin/fm-pr-*|bin/fm-merge-local.sh|bin/fm-local-default.sh|bin/fm-teardown.sh|bin/fm-review-diff.sh|\ - bin/fm-x-*|bin/fm-check*) + bin/fm-x-*|bin/fm-ext-*|bin/fm-check*) printf '%s\n' pr-forge [ "$path" != bin/fm-teardown.sh ] || printf '%s\n' session-bootstrap ;; @@ -1062,6 +1063,9 @@ families_for_changed_path() { .agents/skills/*/SKILL.md) printf '%s\n' pure-contract-unit ;; + contrib/hermes-gateway-firstmate-comms/*) + printf '%s\n' pr-forge + ;; .github/workflows/ci.yml|.no-mistakes.yaml) printf '%s\n' pure-contract-unit printf '%s\n' real-herdr-gated diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index f1dd6bb1423..641251ed9f3 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -112,6 +112,8 @@ mkdir -p "$STATE" . "$SCRIPT_DIR/fm-merge-outcome-lib.sh" # shellcheck source=bin/fm-x-lib.sh . "$SCRIPT_DIR/fm-x-lib.sh" +# shellcheck source=bin/fm-ext-lib.sh +. "$SCRIPT_DIR/fm-ext-lib.sh" # shellcheck source=bin/fm-check-lib.sh . "$SCRIPT_DIR/fm-check-lib.sh" # Parent-owned secondmate missed-report guards: durable pending-reply @@ -1174,6 +1176,15 @@ while :; do rejected_checks="$rejected_checks $c" continue fi + elif [ "$(basename "$c")" = ext-watch.check.sh ]; then + if fm_ext_poll_shim_valid "$c" "$FM_HOME" "$FM_ROOT" \ + && [ -f "$FM_ROOT/bin/fm-ext-poll.sh" ] && [ ! -L "$FM_ROOT/bin/fm-ext-poll.sh" ]; then + FM_HOME="$FM_HOME" run_check_capture "$FM_ROOT/bin/fm-ext-poll.sh" || exit 1 + out=$FM_CHECK_RESULT + else + rejected_checks="$rejected_checks $c" + continue + fi else id=$(basename "$c" .check.sh) if fm_pr_poll_snapshot_capture "$STATE" "$id" "$SCRIPT_DIR/fm-pr-poll.sh"; then diff --git a/contrib/hermes-gateway-firstmate-comms/README.md b/contrib/hermes-gateway-firstmate-comms/README.md new file mode 100644 index 00000000000..9fdc6e3bccf --- /dev/null +++ b/contrib/hermes-gateway-firstmate-comms/README.md @@ -0,0 +1,48 @@ +# Hermes Gateway plugin: Firstmate Communication Officer + +Install this directory into a **dedicated Hermes Gateway home**, never the crewmate TUI profile. + +Crewmate Hermes is a separate adapter. +Firstmate still launches crewmates with `hermes chat --tui`. +That command is not this Discord gateway. + +## Dedicated gateway home + +Pick a gateway-only `HERMES_HOME`, for example `~/.hermes-gateway-firstmate`. +Copy or symlink this plugin into `$HERMES_HOME/plugins/firstmate-comms/`. +Enable it in that home's Hermes `config.yaml` `plugins.enabled` list. +Do not add this plugin to a crewmate profile that Firstmate's Hermes turn-end hook manages. + +The gateway process is `hermes gateway`, not `hermes chat --tui`. + +## Local Firstmate bridge + +Point the gateway at the Firstmate home with `FM_HOME` in the gateway environment. +Firstmate opt-in is `config/ext-bridge` or `FM_EXT_BRIDGE=1` plus `config/ext-secret` (mode 0600) and `config/ext-allowlist`. + +Allowlist lines are fail-closed: + +``` + +: +:: +``` + +The plugin registers slash command `fm`. +It writes Discord text to a temp file and execs `bin/fm-ext-intake.sh --text-file`. +It never calls `dispatch_tool("terminal", ...)`. +The slash handler returns a fast ack without waiting for Firstmate to finish the work. + +An outbox watcher drains `state/ext-outbox/` through `bin/fm-ext-outbox.sh`. +Unsent payloads retry after a gateway restart. +Oversized replies split with `FM_EXT_DISCORD_REPLY_MAX_CHARS` (default 1900) and post as ordered Discord messages in the same thread. +That split does not use `FMX_PAIRING_TOKEN` or the hosted relay. +A transient definite send failure (HTTP 429 or 5xx) before a successful Discord response deletes the posting marker so that generation can retry. +A permanent 4xx records a terminal failed marker so pending stops retrying that generation. +A posting marker without a receipt is refused so an ambiguous crash or transport error after Discord may have accepted the post cannot double-post. +An exclusive inflight send claim means two gateway processes cannot both post the same remaining chunk after a later-chunk resume. +A dead owner's claim older than `FM_EXT_INFLIGHT_TTL_SECS` (default 30) may be stolen; a live owner is never stolen from. +Set `DISCORD_BOT_TOKEN` (or `HERMES_DISCORD_TOKEN`) for Discord REST delivery. +Firstmate core has no Discord library. + +See [Local Communication Officer bridge](../../docs/configuration.md#local-communication-officer-bridge-configext-bridge). diff --git a/contrib/hermes-gateway-firstmate-comms/__init__.py b/contrib/hermes-gateway-firstmate-comms/__init__.py new file mode 100644 index 00000000000..2c081845481 --- /dev/null +++ b/contrib/hermes-gateway-firstmate-comms/__init__.py @@ -0,0 +1,30 @@ +"""Hermes Gateway plugin for the sibling local Firstmate Communication Officer. + +Install this directory into a dedicated gateway HERMES_HOME plugins folder. +Do not enable it on a crewmate TUI profile. Crewmate Hermes still launches as +``hermes chat --tui`` and is a separate adapter. + +This plugin never calls ``ctx.dispatch_tool("terminal", ...)``. +It execs ``bin/fm-ext-intake.sh`` with ``--text-file`` and watches the local +outbox using ``bin/fm-ext-outbox.sh``. +""" + +from __future__ import annotations + +try: + from .intake import handle_fm_command, maybe_intake_from_text + from .outbox_poster import start_outbox_watcher +except ImportError: + from intake import handle_fm_command, maybe_intake_from_text + from outbox_poster import start_outbox_watcher + + +def register(ctx): + """Wire the /fm slash command and start the local outbox poster.""" + ctx.register_command( + "fm", + handler=handle_fm_command, + description="Send this request to the local Firstmate Communication Officer", + args_hint="request", + ) + start_outbox_watcher() diff --git a/contrib/hermes-gateway-firstmate-comms/intake.py b/contrib/hermes-gateway-firstmate-comms/intake.py new file mode 100644 index 00000000000..52a16d5471f --- /dev/null +++ b/contrib/hermes-gateway-firstmate-comms/intake.py @@ -0,0 +1,196 @@ +"""Fail-closed /fm intake for the local Firstmate Communication Officer.""" + +from __future__ import annotations + +import os +import re +import subprocess +import tempfile +from pathlib import Path + +_SNOWFLAKE = re.compile(r"^[0-9]+$") +_FM_PREFIX = re.compile(r"^/fm(?:@\S+)?(?:\s+|$)") + + +def firstmate_root() -> Path: + override = os.environ.get("FM_ROOT_OVERRIDE") or os.environ.get("FM_ROOT") + if override: + return Path(override) + return Path(__file__).resolve().parents[2] + + +def firstmate_home() -> Path: + override = os.environ.get("FM_HOME") + if override: + return Path(override) + return firstmate_root() + + +def secret_path(home: Path | None = None) -> Path: + home = home or firstmate_home() + override = os.environ.get("FM_EXT_SECRET_FILE") + if override: + return Path(override) + return home / "config" / "ext-secret" + + +def allowlist_path(home: Path | None = None) -> Path: + home = home or firstmate_home() + override = os.environ.get("FM_EXT_ALLOWLIST_FILE") + if override: + return Path(override) + return home / "config" / "ext-allowlist" + + +def read_allowlist(path: Path) -> list[str]: + if not path.is_file() or path.is_symlink(): + return [] + rules: list[str] = [] + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + rules.append(line) + return rules + + +def allowlisted(rules: list[str], guild: str, channel: str, author: str) -> bool: + if not rules or not guild or not channel: + return False + for rule in rules: + parts = rule.split(":") + if len(parts) == 1: + if parts[0] == guild: + return True + continue + if len(parts) == 2: + if parts[0] == guild and parts[1] == channel: + return True + continue + if len(parts) >= 3: + if parts[0] == guild and parts[1] == channel and parts[2] == author: + return True + return False + + +def context_field(context: dict | None, *names: str) -> str: + if not context: + return "" + for name in names: + value = context.get(name) + if value is None: + continue + text = str(value).strip() + if text: + return text + return "" + + +def destination_from_context(context: dict | None) -> dict[str, str]: + guild = context_field(context, "guild_id", "guild", "server_id") + channel = context_field(context, "channel_id", "chat_id") + thread = context_field(context, "thread_id") or channel + message = context_field(context, "message_id", "id") + author = context_field(context, "user_id", "author_id", "author") + platform = context_field(context, "platform") or "discord" + return { + "guild_id": guild, + "channel_id": channel, + "thread_id": thread, + "message_id": message, + "author": author, + "platform": platform, + } + + +def request_id_for(dest: dict[str, str]) -> str: + return ( + f"discord:{dest['guild_id']}:{dest['channel_id']}:" + f"{dest['thread_id']}:{dest['message_id']}" + ) + + +def destination_valid(dest: dict[str, str]) -> bool: + if dest.get("platform") not in ("", "discord"): + return False + for key in ("guild_id", "channel_id", "thread_id", "message_id", "author"): + if not _SNOWFLAKE.fullmatch(dest.get(key, "")): + return False + return True + + +def is_fm_text(text: str) -> bool: + return bool(_FM_PREFIX.match((text or "").lstrip())) + + +def fm_request_text(raw_args: str) -> str: + text = (raw_args or "").strip() + if is_fm_text(text): + return _FM_PREFIX.sub("", text, count=1).strip() + return text + + +def maybe_intake_from_text(text: str, context: dict | None) -> str | None: + """No-op unless the message is a /fm command. Used by tests and free-form chat.""" + if not is_fm_text(text): + return None + return handle_fm_command(fm_request_text(text), context) + + +def handle_fm_command(raw_args: str, context: dict | None = None) -> str: + """Slash-command handler. Returns a fast ack without waiting for Firstmate work.""" + dest = destination_from_context(context) + if not destination_valid(dest): + return "Firstmate refused this request: Discord destination is incomplete." + home = firstmate_home() + rules = read_allowlist(allowlist_path(home)) + if not allowlisted(rules, dest["guild_id"], dest["channel_id"], dest["author"]): + return "Firstmate refused this request: it is not on the local allowlist." + request_text = fm_request_text(raw_args) + if not request_text: + return "Aye. Use `/fm` followed by the order you want Firstmate to take." + try: + _run_intake(home, dest, request_text) + except Exception: + return "Firstmate could not record that request locally. Try again from this thread." + return "Aye, captain - Firstmate has the order and is on it." + + +def _run_intake(home: Path, dest: dict[str, str], text: str) -> None: + intake = firstmate_root() / "bin" / "fm-ext-intake.sh" + secret = secret_path(home) + env = os.environ.copy() + env["FM_HOME"] = str(home) + env["FM_ROOT_OVERRIDE"] = str(firstmate_root()) + env["FM_EXT_BRIDGE"] = env.get("FM_EXT_BRIDGE") or "1" + with tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False) as handle: + handle.write(text) + text_path = handle.name + try: + subprocess.run( + [ + str(intake), + "--request-id", + request_id_for(dest), + "--guild-id", + dest["guild_id"], + "--channel-id", + dest["channel_id"], + "--thread-id", + dest["thread_id"], + "--message-id", + dest["message_id"], + "--author", + dest["author"], + "--secret-file", + str(secret), + "--text-file", + text_path, + ], + check=True, + env=env, + capture_output=True, + text=True, + ) + finally: + os.unlink(text_path) diff --git a/contrib/hermes-gateway-firstmate-comms/outbox_poster.py b/contrib/hermes-gateway-firstmate-comms/outbox_poster.py new file mode 100644 index 00000000000..26228dcb12b --- /dev/null +++ b/contrib/hermes-gateway-firstmate-comms/outbox_poster.py @@ -0,0 +1,504 @@ +"""Local outbox poster for the Communication Officer gateway plugin. + +Posts pending ``state/ext-outbox`` payloads to the Discord destination stored +in each payload and records receipts through ``bin/fm-ext-outbox.sh``. +Unsent payloads (no posting marker, no receipt, no terminal failed marker) +are retried after restart. +Oversized replies are split with the X-mode Discord budget pattern +(``FM_EXT_DISCORD_REPLY_MAX_CHARS``, default 1900) and posted in order. +A later-chunk transient failure records progress so earlier chunks are not +sent again, then releases the exclusive inflight send marker so only one +poster can resume the next chunk. A pre-send split failure or chunk-count +mismatch after a successful begin also releases that marker; posted_count +zero still aborts. An in-flight chunk without a confirmed post stays +mid-delivery. +A transient definite send failure (HTTP 429 or 5xx) before any chunk +succeeds deletes the posting marker so that generation can retry. +A permanent 4xx records a terminal failed marker so pending stops retrying. +A posting marker without a receipt is refused so an ambiguous crash or +transport error after Discord may have accepted the post cannot double-post. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import threading +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Callable + +try: + from .intake import firstmate_home, firstmate_root +except ImportError: # loaded from sys.path in hermetic tests + from intake import firstmate_home, firstmate_root + +SendFn = Callable[[dict], dict] + +_WATCHER_STARTED = False +_WATCHER_LOCK = threading.Lock() +_TRANSIENT_HTTP = {429, 500, 502, 503, 504} + + +class DiscordSendError(Exception): + """Classified Discord send outcome for outbox delivery.""" + + def __init__(self, outcome: str, message: str = "", http_code: int | None = None): + super().__init__(message or outcome) + self.outcome = outcome + self.http_code = http_code + + +def classify_http_code(code: int) -> str: + if code in _TRANSIENT_HTTP or code >= 500: + return "transient" + if code == 408: + return "ambiguous" + if 400 <= code < 500: + return "permanent" + return "ambiguous" + + +def classify_send_failure(exc: BaseException) -> str: + if isinstance(exc, DiscordSendError): + return exc.outcome + if isinstance(exc, urllib.error.HTTPError): + return classify_http_code(exc.code) + if isinstance(exc, (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError)): + return "ambiguous" + if isinstance(exc, RuntimeError) and "missing DISCORD_BOT_TOKEN" in str(exc): + return "transient" + return "ambiguous" + + +def failure_reason(exc: BaseException) -> dict: + http_code = getattr(exc, "http_code", None) + if http_code is None and isinstance(exc, urllib.error.HTTPError): + http_code = exc.code + reason = {"ok": False, "reason": str(exc)} + if http_code is not None: + reason["http_code"] = http_code + return reason + + +def outbox_cli() -> Path: + return firstmate_root() / "bin" / "fm-ext-outbox.sh" + + +def _env_for(home: Path) -> dict[str, str]: + env = os.environ.copy() + env["FM_HOME"] = str(home) + env["FM_ROOT_OVERRIDE"] = str(firstmate_root()) + return env + + +def list_pending(home: Path | None = None) -> list[Path]: + home = home or firstmate_home() + result = subprocess.run( + [str(outbox_cli()), "pending"], + check=False, + env=_env_for(home), + capture_output=True, + text=True, + ) + if result.returncode != 0: + return [] + paths = [] + for line in result.stdout.splitlines(): + line = line.strip() + if line: + paths.append(Path(line)) + return paths + + +def begin_delivery(payload: dict, home: Path | None = None) -> str: + home = home or firstmate_home() + result = subprocess.run( + [ + str(outbox_cli()), + "begin", + "--slug", + payload["slug"], + "--kind", + payload["kind"], + "--generation", + str(payload["generation"]), + ], + check=False, + env=_env_for(home), + capture_output=True, + text=True, + ) + if result.returncode == 0: + return "claimed" + if result.returncode == 1: + return "already-receipted" + if result.returncode == 3: + return "mid-delivery" + if result.returncode == 4: + return "terminal-failed" + raise RuntimeError(result.stderr.strip() or "begin failed") + + +def release_inflight(payload: dict, home: Path | None = None) -> str: + home = home or firstmate_home() + result = subprocess.run( + [ + str(outbox_cli()), + "release", + "--slug", + payload["slug"], + "--kind", + payload["kind"], + "--generation", + str(payload["generation"]), + ], + check=False, + env=_env_for(home), + capture_output=True, + text=True, + ) + if result.returncode == 0: + return "released" + raise RuntimeError(result.stderr.strip() or "release failed") + + +def abort_delivery(payload: dict, home: Path | None = None) -> str: + home = home or firstmate_home() + result = subprocess.run( + [ + str(outbox_cli()), + "abort", + "--slug", + payload["slug"], + "--kind", + payload["kind"], + "--generation", + str(payload["generation"]), + ], + check=False, + env=_env_for(home), + capture_output=True, + text=True, + ) + if result.returncode == 0: + return "aborted" + if result.returncode == 1: + return "already-receipted" + raise RuntimeError(result.stderr.strip() or "abort failed") + + +def record_receipt(payload: dict, receipt: dict, home: Path | None = None) -> str: + home = home or firstmate_home() + with _temp_json(receipt) as receipt_path: + result = subprocess.run( + [ + str(outbox_cli()), + "receipt", + "--slug", + payload["slug"], + "--kind", + payload["kind"], + "--generation", + str(payload["generation"]), + "--receipt-file", + receipt_path, + ], + check=False, + env=_env_for(home), + capture_output=True, + text=True, + ) + if result.returncode in (0, 1): + return "receipted" if result.returncode == 0 else "already-receipted" + raise RuntimeError(result.stderr.strip() or "receipt failed") + + +def record_failed(payload: dict, reason: dict, home: Path | None = None) -> str: + home = home or firstmate_home() + with _temp_json(reason) as reason_path: + result = subprocess.run( + [ + str(outbox_cli()), + "fail", + "--slug", + payload["slug"], + "--kind", + payload["kind"], + "--generation", + str(payload["generation"]), + "--reason-file", + reason_path, + ], + check=False, + env=_env_for(home), + capture_output=True, + text=True, + ) + if result.returncode in (0, 4): + return "terminal-failed" + if result.returncode == 1: + return "already-receipted" + raise RuntimeError(result.stderr.strip() or "fail failed") + + +def split_reply( + text: str, + home: Path | None = None, + limit: int | None = None, + cap: int | None = None, +) -> dict: + home = home or firstmate_home() + args = [str(outbox_cli()), "split"] + if limit is not None: + args.extend(["--max", str(limit)]) + if cap is not None: + args.extend(["--cap", str(cap)]) + result = subprocess.run( + args, + check=False, + env=_env_for(home), + input=text, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr.strip() or "split failed") + data = json.loads(result.stdout) + texts = data.get("texts") + if not isinstance(texts, list) or not texts: + texts = [text] + return { + "limit": int(data.get("limit") or 1900), + "cap": int(data.get("cap") or 25), + "texts": [str(item) for item in texts], + } + + +def progress_path(payload: dict, home: Path) -> Path: + name = f"{payload['slug']}.{payload['kind']}.{payload['generation']}.progress.json" + return home / "state" / "ext-outbox" / name + + +def load_progress(payload: dict, home: Path) -> dict | None: + path = progress_path(payload, home) + if not path.is_file(): + return None + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(data, dict): + return None + return data + + +def write_progress(payload: dict, progress: dict, home: Path) -> None: + with _temp_json(progress) as progress_file: + result = subprocess.run( + [ + str(outbox_cli()), + "progress", + "--slug", + payload["slug"], + "--kind", + payload["kind"], + "--generation", + str(payload["generation"]), + "--progress-file", + progress_file, + ], + check=False, + env=_env_for(home), + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr.strip() or "progress failed") + + +def _chunk_payload(payload: dict, text: str, index: int, total: int) -> dict: + chunk = dict(payload) + chunk["text"] = text + chunk["chunk_index"] = index + chunk["chunk_count"] = total + return chunk + + +def _message_id(receipt: dict) -> str: + return str(receipt.get("discord_message_id") or "") + + +class _temp_json: + def __init__(self, body: dict): + self.body = body + self.path = "" + + def __enter__(self) -> str: + fd, self.path = _mktemp() + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(json.dumps(self.body)) + return self.path + + def __exit__(self, *args) -> None: + if self.path: + try: + os.unlink(self.path) + except OSError: + pass + + +def _mktemp() -> tuple[int, str]: + import tempfile + + return tempfile.mkstemp(prefix="fm-ext-receipt.") + + +def discord_send(payload: dict) -> dict: + """Post one outbox payload to Discord REST. No Discord library.""" + token = os.environ.get("DISCORD_BOT_TOKEN") or os.environ.get("HERMES_DISCORD_TOKEN") + if not token: + raise DiscordSendError("transient", "missing DISCORD_BOT_TOKEN") + channel = payload["thread_id"] or payload["channel_id"] + body = json.dumps({"content": payload["text"]}).encode("utf-8") + request = urllib.request.Request( + f"https://discord.com/api/v10/channels/{channel}/messages", + data=body, + method="POST", + headers={ + "Authorization": f"Bot {token}", + "Content-Type": "application/json", + }, + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + raw = response.read().decode("utf-8") or "{}" + except urllib.error.HTTPError as err: + outcome = classify_http_code(err.code) + raise DiscordSendError(outcome, f"discord HTTP {err.code}", err.code) from err + except urllib.error.URLError as err: + raise DiscordSendError("ambiguous", f"discord transport: {err}") from err + except (TimeoutError, OSError) as err: + raise DiscordSendError("ambiguous", f"discord transport: {err}") from err + try: + data = json.loads(raw) + except json.JSONDecodeError as err: + raise DiscordSendError("ambiguous", "discord HTTP 200 with invalid JSON") from err + return { + "ok": True, + "discord_message_id": str(data.get("id") or ""), + "channel_id": str(data.get("channel_id") or channel), + } + + +def deliver_one(path: Path, send: SendFn | None = None, home: Path | None = None) -> str: + payload = json.loads(path.read_text(encoding="utf-8")) + home = home or firstmate_home() + status = begin_delivery(payload, home=home) + if status != "claimed": + return status + sender = send or discord_send + stored = load_progress(payload, home) + try: + split = split_reply( + payload.get("text") or "", + home=home, + limit=stored.get("limit") if stored else None, + cap=stored.get("cap") if stored else None, + ) + except Exception: + if stored and int(stored.get("posted_count") or 0) > 0: + release_inflight(payload, home=home) + return "mid-delivery" + abort_delivery(payload, home=home) + return "failed" + chunks = split["texts"] + progress = stored or { + "total": len(chunks), + "posted_count": 0, + "inflight": None, + "discord_message_ids": [], + "limit": split["limit"], + "cap": split["cap"], + } + if int(progress.get("total") or 0) != len(chunks): + release_inflight(payload, home=home) + return "mid-delivery" + write_progress(payload, progress, home) + start = int(progress.get("posted_count") or 0) + ids = list(progress.get("discord_message_ids") or []) + for index in range(start, len(chunks)): + progress["inflight"] = index + write_progress(payload, progress, home) + try: + receipt = sender(_chunk_payload(payload, chunks[index], index, len(chunks))) + except Exception as err: + outcome = classify_send_failure(err) + if outcome == "transient": + progress["inflight"] = None + write_progress(payload, progress, home) + if int(progress.get("posted_count") or 0) == 0: + abort_status = abort_delivery(payload, home=home) + if abort_status == "already-receipted": + return abort_status + return "failed" + release_inflight(payload, home=home) + return "failed" + if outcome == "permanent": + fail_status = record_failed(payload, failure_reason(err), home=home) + if fail_status == "already-receipted": + return fail_status + return "terminal-failed" + return "mid-delivery" + if not isinstance(receipt, dict): + receipt = {} + ids.append(_message_id(receipt)) + progress["discord_message_ids"] = ids + progress["posted_count"] = index + 1 + progress["inflight"] = None + write_progress(payload, progress, home) + record_receipt( + payload, + { + "ok": True, + "discord_message_id": ids[0] if ids else "", + "discord_message_ids": ids, + "chunks": len(chunks), + "channel_id": str(payload.get("thread_id") or payload.get("channel_id") or ""), + }, + home=home, + ) + return "sent" + + +def drain_outbox(send: SendFn | None = None, home: Path | None = None) -> list[str]: + results = [] + for path in list_pending(home=home): + results.append(deliver_one(path, send=send, home=home)) + return results + + +def start_outbox_watcher(interval: float | None = None) -> None: + global _WATCHER_STARTED + with _WATCHER_LOCK: + if _WATCHER_STARTED: + return + _WATCHER_STARTED = True + + wait = interval + if wait is None: + wait = float(os.environ.get("FM_EXT_OUTBOX_POLL_SECS", "2")) + + def _loop() -> None: + while True: + try: + drain_outbox() + except Exception: + pass + time.sleep(wait) + + thread = threading.Thread(target=_loop, name="fm-ext-outbox", daemon=True) + thread.start() diff --git a/contrib/hermes-gateway-firstmate-comms/plugin.yaml b/contrib/hermes-gateway-firstmate-comms/plugin.yaml new file mode 100644 index 00000000000..bd964123dfb --- /dev/null +++ b/contrib/hermes-gateway-firstmate-comms/plugin.yaml @@ -0,0 +1,7 @@ +name: firstmate-comms +version: 0.1.0 +description: >- + Firstmate Communication Officer slash command and local outbox poster. + Install into a dedicated Hermes Gateway HERMES_HOME, never a crewmate TUI profile. +author: firstmate +provides_hooks: [] diff --git a/docs/architecture.md b/docs/architecture.md index 2336c80948c..779480a0c7c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -72,6 +72,7 @@ A bounded direct-report terminal tail can help diagnose a mismatch by showing th The snapshot strips control sequences, retains only capture metadata and literal event-corroboration flags, and never lets terminal evidence override a valid structured classification. The default path remains local-only; live GitHub enrichment exists only behind the bearings `--include-prs` opt-in. Optional X mode integrates with the watcher only after explicit opt-in; [configuration.md](configuration.md#x-mode-env) owns its generated-artifact and dispatch mechanics. +The sibling local Communication Officer bridge is a separate opt-in that uses the same watcher identity-shim pattern without the hosted relay; [configuration.md](configuration.md#local-communication-officer-bridge-configext-bridge) owns its generated artifacts. At session start, `bin/fm-session-start.sh` emits exactly one primary-harness supervision block rendered by `bin/fm-supervision-instructions.sh` from `docs/supervision-protocols/`. That block owns the live wait shape for the running primary harness: Claude's Stop `asyncRewake` hook owns tokenless re-arm cycles, Grok uses background-notify cycles, Codex uses bounded foreground checkpoints, Pi and pi-signed use the same two tracked Pi extensions, OMP uses its native `.omp` primary extension, and OpenCode uses its TUI plugin. @@ -318,6 +319,23 @@ Because a terminal event's id is derived from its identity tuple rather than gen Reconciliation rides the existing relay poll and the session-start digest instead of a new watcher, daemon, or timer, and both are gated on the same `.env` activation contract so a home that never opted into the relay executes none of it. The [X mode configuration reference](configuration.md#promised-public-replies-statepublic-followup) owns the operator-facing contract, and the `fmx-respond` skill owns the procedure. +## Optional local Communication Officer bridge + +The sibling local bridge is opt-in presence for a dedicated Hermes Gateway `/fm` plugin talking to this home over local files, not a hosted relay. +A user enables it with `config/ext-bridge` or `FM_EXT_BRIDGE=1` plus a mode-0600 secret file and a fail-closed allowlist. +That opt-in is standing authorization for firstmate to answer allowlisted Discord `/fm` requests and act autonomously on normal reversible work. +Destructive, irreversible, or security-sensitive asks are escalated for trusted-channel confirmation. +On the locked session-start bootstrap step, a valid opt-in creates `state/ext-watch.check.sh` as described in the [local Communication Officer configuration reference](configuration.md#local-communication-officer-bridge-configext-bridge). +Without opt-in, that bootstrap step removes the shim on opt-out and otherwise stays silent. + +The gateway plugin calls `bin/fm-ext-intake.sh --text-file`, never `dispatch_tool("terminal", ...)`, and never interpolates Discord text into a shell command. +Intake stores `state/ext-inbox/.json` and claims a one-wake offer marker; the canonical `request_id` keeps colons while filenames use the SHA-256 slug. +The `ext-respond` skill drains that inbox, classifies each request, and emits `ack` / `answer` / `followup` / `final` payloads through `bin/fm-ext-emit.sh`. +Spawned work is linked with `bin/fm-ext-link.sh` using `ext_request=`, not `x_request=`. +The gateway outbox poster delivers those payloads to the Discord destination stored in context and writes receipts; posting-marker retry versus mid-delivery refuse is owned by the [local Communication Officer configuration reference](configuration.md#local-communication-officer-bridge-configext-bridge). +Crewmate Hermes remains a separate TUI profile (`hermes chat --tui`) and is still refused as a second mate. +This bridge remains layered on the existing check mechanism without changing X-mode or crewmate Hermes behavior. + ## Project memory belongs to projects Durable project-intrinsic agent knowledge lives in each project's committed `AGENTS.md`, with `CLAUDE.md` as a real `@AGENTS.md` import pointer. diff --git a/docs/configuration.md b/docs/configuration.md index a6bf0cd8050..ceab6cec019 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -555,6 +555,64 @@ The session-start digest separately prints an "Public commitments awaiting deliv `FM_PF_RETRY_BACKOFF_SECS` (default 900) sets the next-attempt time recorded with a retryable delivery error. See [verification/public-followup.md](verification/public-followup.md) for the current maintainer evidence behind the restart end-to-end and the relay-disabled zero-overhead guarantee. +## Local Communication Officer bridge (config/ext-bridge) + +The sibling local bridge lets a dedicated Hermes Gateway plugin deliver Discord `/fm` requests into this firstmate home and post replies from a local outbox. +It is a different seam from optional X mode. +It does not use `FMX_PAIRING_TOKEN`, `https://myfirstmate.io`, hosted public-follow-up, or pending-reply. +Firstmate core contains no Discord library. +Discord text is untrusted and must enter through `--text-file` or stdin. + +The bridge is off unless `config/ext-bridge` is a regular non-symlink file or `FM_EXT_BRIDGE` is a truthy environment value, **and** `config/ext-secret` is a non-empty mode-0600 regular file. +`FM_EXT_BRIDGE` wins when set: an explicit empty/`0`/`false`/`no`/`off` value opts out even if the presence file exists. +`FM_EXT_SECRET_FILE` and `FM_EXT_ALLOWLIST_FILE` may redirect those two files for tests. +The secret and allowlist are not inherited into secondmate homes. + +`config/ext-allowlist` is fail-closed. +Missing, empty, unreadable, or symlink allowlists deny every request. +Each non-comment line is ``, `:`, or `::`. +A request is allowed when any rule matches every component it specifies. + +Canonical `request_id` is `discord::::` and keeps those colons in JSON bodies. +Filenames use the SHA-256 hex digest of that canonical id (`slug`). +`bin/fm-ext-intake.sh` publishes `state/ext-inbox/.json` (mode 0600), durable destination context at `state/ext-context/.json`, and a one-wake offer marker at `state/ext-context/.offered.json`. +The same message id claims the existing offer and does not append a second wake. +New offers may append `ext-request ` through the durable wake queue. +State directories are mode 0700. + +The locked session-start bootstrap step turns a valid opt-in into `state/ext-watch.check.sh`, a byte-static identity shim for `bin/fm-ext-poll.sh`. +The watcher accepts the shim only when its bytes match the expected generated content, then invokes the trusted repository poll script. +The poll is a hard no-op until the bridge is active. +It prints `ext-request ` only when it claims a leftover inbox offer that intake did not claim; already claimed offers stay silent across Firstmate restart. +Removing the opt-in or the secret removes the shim. +There is no 30-second cadence override; intake wakes immediately and the default slow-check interval covers restart recovery. + +`bin/fm-ext-emit.sh` writes idempotent `ack` / `answer` / `followup` / `final` payloads into `state/ext-outbox/...json`. +Re-emitting the same identity is a no-op success. +A posting marker without a receipt is refused as mid-delivery, matching public-follow-up delivery-posting behavior. +That refuse covers an ambiguous crash or transport error after Discord may have accepted the post. +`bin/fm-ext-outbox.sh begin` CAS-claims that posting marker and an exclusive per-generation inflight send marker before any send, so two posters cannot both send the same chunk. +The inflight claim records owner pid and `recorded_at`. +A leftover claim is stolen only when it is older than `FM_EXT_INFLIGHT_TTL_SECS` (default 30) **and** its recorded owner pid is dead. +Steal serialization keeps a 1-second floor even when that TTL is 0, so two stealers cannot both win. +A live owner stays mid-delivery even after that TTL, including an ambiguous crash-after-post while the poster process is still alive. +`receipt` writes the receipt once. +`abort` deletes the posting marker after a transient definite send failure (HTTP 429 or 5xx) so that generation can retry. +`fail` records a terminal failed marker after a permanent 4xx so pending stops retrying that generation. +The gateway poster splits oversized Discord replies with the same numbered-thread algorithm X mode uses for Discord, driven by `FM_EXT_DISCORD_REPLY_MAX_CHARS` (default 1900, min 50, values above 2000 reset to 1900) and `FM_EXT_DISCORD_THREAD_MAX` (default 25). +A reply that already fits is one unnumbered message; a longer reply is posted as ordered `(k/n)` chunks into the same thread. +Chunk progress is recorded so a later-chunk retry does not send earlier chunks again. +A later-chunk transient failure then releases the inflight marker so only one poster can resume the next chunk. +An ambiguous failure after a chunk may have been accepted stays mid-delivery for that chunk. +This split does not use `FMX_PAIRING_TOKEN` or the hosted relay. +Unsent payloads (no posting marker, no receipt, and no terminal failed marker) remain deliverable after a Hermes Gateway restart. +`bin/fm-ext-link.sh` binds a spawned task to the canonical `request_id` as `ext_request=` / `ext_request_slug=` / `ext_request_ts=` / `ext_followups=`, never `x_request=`. + +The Hermes Gateway plugin lives in `contrib/hermes-gateway-firstmate-comms/` and must be installed into a dedicated gateway `HERMES_HOME`, not the crewmate TUI profile. +Crewmate Hermes still launches as `hermes chat --tui`. +That launch string is not the Discord gateway. +The `ext-respond` skill owns drain, classification, lifecycle action, and local emit. + ## Process-to-event sources (state/procevent) A long-polling external process is registered as a *source* through its adapter, whose header and `--help` own the commands and flags. @@ -678,6 +736,12 @@ FMX_DISCORD_REPLY_MAX_CHARS=1900 # Discord reply per-message split budget; val FMX_X_THREAD_MAX=25 # maximum messages in one auto-split reply thread FMX_FOLLOWUP_MAX_AGE_SECS=604800 # local window for posting X-mode completion follow-ups (7 days) FMX_FOLLOWUP_MAX_COUNT=3 # local cap on X-mode completion follow-ups per linked mention +FM_EXT_BRIDGE= # optional local Communication Officer opt-in; truthy enables, explicit 0/false/no/off disables even when config/ext-bridge exists +FM_EXT_SECRET_FILE= # optional override of config/ext-secret for tests +FM_EXT_ALLOWLIST_FILE= # optional override of config/ext-allowlist for tests +FM_EXT_DISCORD_REPLY_MAX_CHARS=1900 # local-bridge Discord per-message split budget; values below 50 clamp to 50, values above 2000 reset to 1900 +FM_EXT_DISCORD_THREAD_MAX=25 # maximum messages in one local-bridge auto-split Discord thread +FM_EXT_INFLIGHT_TTL_SECS=30 # seconds a dead-owner exclusive send claim must age before another poster may steal it; live owners are never stolen from; steal serialization keeps a 1s floor even when this is 0 FM_PF_RETRY_BACKOFF_SECS=900 # seconds before the next attempt after a retryable promised-public-reply delivery error FM_LOCK_STALE_AFTER=2 # seconds before dead-pid lock records can be reclaimed; mid-acquire locks keep at least 2s grace FM_GUARD_GRACE=300 # seconds before guard warnings, arm health checks, and the primary turn-end guard treat a watcher beacon as stale diff --git a/docs/documentation-audiences.json b/docs/documentation-audiences.json index 73109380025..6c0d616b086 100644 --- a/docs/documentation-audiences.json +++ b/docs/documentation-audiences.json @@ -137,6 +137,10 @@ "path": ".agents/skills/diagnostic-reasoning/SKILL.md", "audience": "agent-runtime" }, + { + "path": ".agents/skills/ext-respond/SKILL.md", + "audience": "agent-runtime" + }, { "path": ".agents/skills/firstmate-codexapp/SKILL.md", "audience": "agent-runtime" @@ -201,6 +205,10 @@ "path": "CONTRIBUTING.md", "audience": "maintainer-architecture" }, + { + "path": "contrib/hermes-gateway-firstmate-comms/README.md", + "audience": "operator-current" + }, { "path": "README.md", "audience": "public-product" diff --git a/docs/turnend-guard.md b/docs/turnend-guard.md index 8ac27a42c95..ad6fa5418d6 100644 --- a/docs/turnend-guard.md +++ b/docs/turnend-guard.md @@ -30,7 +30,7 @@ A first launch in a fresh clone, before bootstrap has created `state/`, is there For an in-scope primary, the guard counts in-flight work from `state/*.meta`. Registered `state/procevent/*.source` records also require supervision even though they have no task metadata. The default cross-harness mode exits silently with no supervision need. -Every mode treats `state/x-watch.check.sh` as supervision need, so X-mode relay polling remains guarded without an in-flight task. +Every mode treats `state/x-watch.check.sh` and `state/ext-watch.check.sh` as supervision need, so X-mode relay polling and the local Communication Officer poll remain guarded without an in-flight task. Otherwise it calls `fm_watcher_healthy [grace-seconds] [home]` from `bin/fm-wake-lib.sh`, the same PID-strict identity-matched lock and fresh-beacon check used by `bin/fm-watch-arm.sh`: a stale beacon blocks even when a watcher pid is live, and a fresh leftover beacon blocks when the lock is missing, dead, or identity-mismatched. The turn-end guard needs that strict check because it fires at the turn boundary, where the auto-arm is bringing a fresh watcher up for the upcoming idle period, and it cooperates with that arm rather than trusting a beacon left by the cycle that just ended. `bin/fm-guard.sh`, the pull warning, instead uses the model-aware `fm_watcher_supervision_verdict` from the same library, because it fires mid-turn when the auto-arm model runs no watcher at all. diff --git a/tests/fm-ext-bridge.test.sh b/tests/fm-ext-bridge.test.sh new file mode 100755 index 00000000000..f8a92b5b8bc --- /dev/null +++ b/tests/fm-ext-bridge.test.sh @@ -0,0 +1,1336 @@ +#!/usr/bin/env bash +# Behavior tests for the sibling local Communication Officer bridge. +# +# Hermetic: no Discord network. The gateway plugin's Discord sender is injected. +# Captain cases 1-12 plus bootstrap activation, send-failure classes, +# wake-append offer recovery, Discord reply splitting, exclusive resume +# claim, pre-send inflight release, and stale inflight steal. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +BASE_PATH=${FM_TEST_BASE_PATH:-/usr/bin:/bin:/usr/sbin:/sbin} +JQ_DIR=$(command -v jq 2>/dev/null) && JQ_DIR=$(dirname "$JQ_DIR") || JQ_DIR= +[ -n "$JQ_DIR" ] && BASE_PATH="$JQ_DIR:$BASE_PATH" +PYTHON_BIN=$(command -v python3) || fail "test needs python3" +PYTHON_DIR=$(dirname "$PYTHON_BIN") +BASE_PATH="$PYTHON_DIR:$BASE_PATH" +TMP_ROOT=$(fm_test_tmproot fm-ext-bridge) + +INTAKE="$ROOT/bin/fm-ext-intake.sh" +EMIT="$ROOT/bin/fm-ext-emit.sh" +LINK="$ROOT/bin/fm-ext-link.sh" +POLL="$ROOT/bin/fm-ext-poll.sh" +OUTBOX="$ROOT/bin/fm-ext-outbox.sh" +BOOTSTRAP="$ROOT/bin/fm-bootstrap.sh" +SPAWN="$ROOT/bin/fm-spawn.sh" +HARNESS="$ROOT/bin/fm-harness.sh" +PLUGIN="$ROOT/contrib/hermes-gateway-firstmate-comms" + +GUILD=111111111111111111 +CHANNEL=222222222222222222 +THREAD=333333333333333333 +MESSAGE=444444444444444444 +AUTHOR=555555555555555555 +RID="discord:${GUILD}:${CHANNEL}:${THREAD}:${MESSAGE}" + +setup_home() { + local home=$1 extra_allow=${2-} + mkdir -p "$home/config" + : > "$home/config/ext-bridge" + printf 'test-secret\n' > "$home/config/ext-secret" + chmod 600 "$home/config/ext-secret" + printf '%s\n' "$GUILD" > "$home/config/ext-allowlist" + [ -z "$extra_allow" ] || printf '%s\n' "$extra_allow" >> "$home/config/ext-allowlist" +} + +home_env() { + local home=$1 + shift + PATH="$BASE_PATH" \ + FM_HOME="$home" \ + FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$home/state" \ + FM_CONFIG_OVERRIDE="$home/config" \ + "$@" +} + +write_text() { + local file=$1 + shift + printf '%s' "$*" > "$file" +} + +intake_ok() { + local home=$1 text=$2 message=${3:-$MESSAGE} out rc textfile + textfile="$home/text.txt" + write_text "$textfile" "$text" + out=$(home_env "$home" "$INTAKE" \ + --request-id "discord:${GUILD}:${CHANNEL}:${THREAD}:${message}" \ + --guild-id "$GUILD" --channel-id "$CHANNEL" --thread-id "$THREAD" \ + --message-id "$message" --author "$AUTHOR" \ + --secret-file "$home/config/ext-secret" \ + --text-file "$textfile") + rc=$? + expect_code 0 "$rc" "intake exit" + printf '%s\n' "$out" +} + +slug_of() { + printf '%s' "$1" | sha256sum 2>/dev/null | awk '{print $1}' \ + || printf '%s' "$1" | shasum -a 256 | awk '{print $1}' +} + +# --- 1. allowlisted intake writes inbox; non-/fm does not ------------------- + +test_1_allowlisted_intake_and_non_fm() { + local home slug inbox out + home="$TMP_ROOT/c1" + setup_home "$home" + slug=$(intake_ok "$home" "/fm ship the login fix") + inbox="$home/state/ext-inbox/${slug}.json" + assert_present "$inbox" "allowlisted /fm intake must write the inbox" + assert_grep "$RID" "$inbox" "inbox must keep canonical request_id colons" + assert_grep "ship the login fix" "$inbox" "inbox must store the request text" + assert_present "$home/state/ext-context/${slug}.offered.json" "intake must claim the offer" + + out=$( + GUILD="$GUILD" CHANNEL="$CHANNEL" THREAD="$THREAD" AUTHOR="$AUTHOR" \ + PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$ROOT" <<'PY' +import os, sys +sys.path.insert(0, os.environ["PYTHONPATH"]) +import intake +os.environ["FM_HOME"] = sys.argv[1] +os.environ["FM_ROOT_OVERRIDE"] = sys.argv[2] +os.environ["FM_EXT_BRIDGE"] = "1" +ctx = { + "platform": "discord", + "guild_id": os.environ["GUILD"], + "channel_id": os.environ["CHANNEL"], + "thread_id": os.environ["THREAD"], + "message_id": "999999999999999999", + "user_id": os.environ["AUTHOR"], +} +print("nonfm=" + repr(intake.maybe_intake_from_text("hello from discord", ctx))) +print("fm=" + intake.maybe_intake_from_text("/fm look into the login bug", ctx)[:3]) +PY + ) + assert_contains "$out" "nonfm=None" "non-/fm must not intake" + assert_contains "$out" "fm=Aye" " /fm must ack" + [ "$(find "$home/state/ext-inbox" -name '*.json' | wc -l | tr -d ' ')" = 2 ] \ + || fail "exactly one extra inbox for the /fm plugin path" + pass "1 allowlisted intake writes inbox; non-/fm does not" +} + +# --- 2. correlation persists across a new shell ----------------------------- + +test_2_correlation_persists() { + local home slug + home="$TMP_ROOT/c2" + setup_home "$home" + slug=$(intake_ok "$home" "file this on the backlog") + # New shell: only FM_HOME, no leftover functions. + # shellcheck disable=SC2016 # child expands FM_HOME; slug and rid are positional + home_env "$home" bash -c ' + set -u + slug=$1 + rid=$2 + test -f "$FM_HOME/state/ext-inbox/$slug.json" || exit 1 + test -f "$FM_HOME/state/ext-context/$slug.json" || exit 1 + test -f "$FM_HOME/state/ext-context/$slug.offered.json" || exit 1 + grep -F "$rid" "$FM_HOME/state/ext-inbox/$slug.json" >/dev/null + ' _ "$slug" "$RID" || fail "correlation artifacts must survive a new shell" + pass "2 correlation persists across a new shell" +} + +# --- 3. immediate ack without waiting for Firstmate work -------------------- + +test_3_immediate_ack() { + local home out + home="$TMP_ROOT/c3" + setup_home "$home" + out=$( + GUILD="$GUILD" CHANNEL="$CHANNEL" THREAD="$THREAD" AUTHOR="$AUTHOR" \ + PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$ROOT" <<'PY' +import os, sys, time +sys.path.insert(0, os.environ["PYTHONPATH"]) +import intake +os.environ["FM_HOME"] = sys.argv[1] +os.environ["FM_ROOT_OVERRIDE"] = sys.argv[2] +os.environ["FM_EXT_BRIDGE"] = "1" +ctx = { + "platform": "discord", + "guild_id": os.environ["GUILD"], + "channel_id": os.environ["CHANNEL"], + "thread_id": os.environ["THREAD"], + "message_id": "444444444444444444", + "user_id": os.environ["AUTHOR"], +} +start = time.time() +ack = intake.handle_fm_command("summarize the fleet", ctx) +elapsed = time.time() - start +print(ack) +print("elapsed=%.3f" % elapsed) +PY + ) + assert_contains "$out" "Aye, captain" "slash handler must return a fast ack" + [ -z "$(find "$home/state/ext-outbox" -name '*.json' 2>/dev/null)" ] \ + || fail "fast ack must not wait on an outbox emit from Firstmate" + pass "3 immediate ack without waiting for Firstmate work" +} + +# --- 4. delayed follow-up after link when inbox is gone --------------------- + +test_4_followup_after_inbox_gone() { + local home slug meta + home="$TMP_ROOT/c4" + setup_home "$home" + slug=$(intake_ok "$home" "ship the redirect fix") + meta="$home/state/ship-login.meta" + fm_write_meta "$meta" "window=firstmate:fm-ship-login" "harness=echo" "kind=ship" + home_env "$home" "$LINK" ship-login "$RID" >/dev/null + assert_grep "ext_request=$RID" "$meta" "link must record canonical ext_request=" + assert_no_grep "x_request=" "$meta" "link must not write x_request=" + rm -f "$home/state/ext-inbox/${slug}.json" + write_text "$home/followup.txt" "the redirect fix is ready for review" + home_env "$home" "$EMIT" --request-id "$RID" --kind followup --generation 1 \ + --text-file "$home/followup.txt" >/dev/null + assert_present "$home/state/ext-outbox/${slug}.followup.1.json" \ + "follow-up must emit after the inbox is gone" + pass "4 delayed follow-up after link when inbox is gone" +} + +# --- 5. multiple follow-ups; duplicate generation is a no-op ---------------- + +test_5_multiple_followups_duplicate_generation() { + local home slug first second + home="$TMP_ROOT/c5" + setup_home "$home" + slug=$(intake_ok "$home" "look into the timeout") + write_text "$home/a.txt" "investigation started" + write_text "$home/b.txt" "investigation finished" + home_env "$home" "$EMIT" --request-id "$RID" --kind followup --generation 1 \ + --text-file "$home/a.txt" >/dev/null + home_env "$home" "$EMIT" --request-id "$RID" --kind followup --generation 2 \ + --text-file "$home/b.txt" >/dev/null + first=$(cat "$home/state/ext-outbox/${slug}.followup.1.json") + home_env "$home" "$EMIT" --request-id "$RID" --kind followup --generation 1 \ + --text-file "$home/b.txt" >/dev/null + second=$(cat "$home/state/ext-outbox/${slug}.followup.1.json") + [ "$first" = "$second" ] || fail "duplicate generation must not replace the first payload" + assert_present "$home/state/ext-outbox/${slug}.followup.2.json" "generation 2 must exist" + pass "5 multiple follow-ups; duplicate generation is a no-op" +} + +# --- 6. idempotent intake + emit -------------------------------------------- + +test_6_idempotent_intake_and_emit() { + local home slug1 slug2 wakes + home="$TMP_ROOT/c6" + setup_home "$home" + slug1=$(intake_ok "$home" "add a backlog item") + wakes=$(grep -c "ext-request $slug1" "$home/state/.wake-queue" || true) + slug2=$(intake_ok "$home" "add a backlog item") + [ "$slug1" = "$slug2" ] || fail "same message id must reuse the slug" + [ "$(grep -c "ext-request $slug1" "$home/state/.wake-queue")" = "$wakes" ] \ + || fail "re-intake must not append a second wake" + write_text "$home/ack.txt" "on it" + home_env "$home" "$EMIT" --request-id "$RID" --kind ack --generation 1 \ + --text-file "$home/ack.txt" >/dev/null + home_env "$home" "$EMIT" --request-id "$RID" --kind ack --generation 1 \ + --text-file "$home/ack.txt" >/dev/null + [ "$(find "$home/state/ext-outbox" -name "${slug1}.ack.1.json" | wc -l | tr -d ' ')" = 1 ] \ + || fail "re-emit must not duplicate the payload file" + pass "6 idempotent intake and emit" +} + +# --- 7. Hermes restart: unsent outbox + receipt once ------------------------ + +test_7_unsent_outbox_receipt_once() { + local home slug sent + home="$TMP_ROOT/c7" + setup_home "$home" + slug=$(intake_ok "$home" "status please") + write_text "$home/ans.txt" "calm seas" + home_env "$home" "$EMIT" --request-id "$RID" --kind answer --generation 1 \ + --text-file "$home/ans.txt" >/dev/null + sent="$home/sent.log" + : > "$sent" + home_env "$home" env PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" >/dev/null <<'PY' +import os, sys +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent = sys.argv[1], sys.argv[2] +os.environ["FM_HOME"] = home +def send(payload): + with open(sent, "a", encoding="utf-8") as fh: + fh.write(payload["text"] + "\n") + return {"ok": True, "discord_message_id": "1"} +print(",".join(outbox_poster.drain_outbox(send=send, home=__import__("pathlib").Path(home)))) +PY + assert_grep "calm seas" "$sent" "unsent outbox must deliver after a new shell" + assert_present "$home/state/ext-outbox/${slug}.answer.1.receipt.json" "delivery must write a receipt" + : > "$sent" + home_env "$home" env PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" >/dev/null <<'PY' +import os, sys +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent = sys.argv[1], sys.argv[2] +os.environ["FM_HOME"] = home +def send(payload): + with open(sent, "a", encoding="utf-8") as fh: + fh.write("AGAIN\n") + return {"ok": True, "discord_message_id": "2"} +print(",".join(outbox_poster.drain_outbox(send=send, home=__import__("pathlib").Path(home)))) +PY + [ ! -s "$sent" ] || fail "receipted outbox must not send again after restart" + pass "7 Hermes restart: unsent outbox delivers once, receipt sticks" +} + +# --- 8. Firstmate restart: inbox+offer; one wake per offer ------------------ + +test_8_restart_one_wake_per_offer() { + local home slug wakes poll + home="$TMP_ROOT/c8" + setup_home "$home" + slug=$(intake_ok "$home" "what is underway") + wakes=$(grep -c "ext-request $slug" "$home/state/.wake-queue") + [ "$wakes" = 1 ] || fail "first intake must wake once, got $wakes" + # shellcheck disable=SC2016 # child expands FM_HOME; slug is positional + home_env "$home" bash -c ' + set -u + slug=$1 + test -f "$FM_HOME/state/ext-inbox/$slug.json" || exit 1 + test -f "$FM_HOME/state/ext-context/$slug.offered.json" || exit 1 + ' _ "$slug" || fail "inbox and offer must survive a new shell" + poll=$(home_env "$home" "$POLL") + [ -z "$poll" ] || fail "poll must stay silent for an already claimed offer (got: $poll)" + intake_ok "$home" "what is underway" >/dev/null + [ "$(grep -c "ext-request $slug" "$home/state/.wake-queue")" = 1 ] \ + || fail "restart plus re-intake must not add a second wake" + pass "8 Firstmate restart: inbox+offer persist; one wake per offer" +} + +# --- 9. Discord send retry / mid-send refuse or CAS receipt ----------------- + +test_9_mid_send_refuse_and_cas_receipt() { + local home slug rc err + home="$TMP_ROOT/c9" + setup_home "$home" + slug=$(intake_ok "$home" "ping") + write_text "$home/ans.txt" "pong" + home_env "$home" "$EMIT" --request-id "$RID" --kind answer --generation 1 \ + --text-file "$home/ans.txt" >/dev/null + home_env "$home" "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 >/dev/null + err="$home/mid.err" + home_env "$home" "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 \ + >/dev/null 2>"$err"; rc=$? + expect_code 3 "$rc" "second begin while posting" + assert_grep "mid-delivery" "$err" "mid-send must refuse" + home_env "$home" "$EMIT" --request-id "$RID" --kind answer --generation 1 \ + --text-file "$home/ans.txt" >/dev/null 2>"$err"; rc=$? + expect_code 1 "$rc" "emit during mid-delivery" + write_text "$home/receipt.json" '{"ok":true,"discord_message_id":"9"}' + home_env "$home" "$OUTBOX" receipt --slug "$slug" --kind answer --generation 1 \ + --receipt-file "$home/receipt.json" >/dev/null + home_env "$home" "$OUTBOX" receipt --slug "$slug" --kind answer --generation 1 \ + --receipt-file "$home/receipt.json" >/dev/null + home_env "$home" "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 >/dev/null; rc=$? + expect_code 1 "$rc" "begin after receipt is idempotent success" + pass "9 mid-send refuse and CAS receipt" +} + +# --- 10. unauthorized / missing allowlist writes no inbox ------------------- + +test_10_unauthorized_and_missing_allowlist() { + local home rc err + home="$TMP_ROOT/c10" + setup_home "$home" + printf '%s\n' "${GUILD}:999999999999999999" > "$home/config/ext-allowlist" + write_text "$home/text.txt" "should not land" + err="$home/deny.err" + home_env "$home" "$INTAKE" \ + --request-id "$RID" --guild-id "$GUILD" --channel-id "$CHANNEL" \ + --thread-id "$THREAD" --message-id "$MESSAGE" --author "$AUTHOR" \ + --secret-file "$home/config/ext-secret" --text-file "$home/text.txt" \ + >/dev/null 2>"$err"; rc=$? + expect_code 1 "$rc" "unauthorized intake" + [ ! -d "$home/state/ext-inbox" ] || [ -z "$(ls -A "$home/state/ext-inbox" 2>/dev/null)" ] \ + || fail "unauthorized intake must not write inbox files" + rm -f "$home/config/ext-allowlist" + home_env "$home" "$INTAKE" \ + --request-id "$RID" --guild-id "$GUILD" --channel-id "$CHANNEL" \ + --thread-id "$THREAD" --message-id "$MESSAGE" --author "$AUTHOR" \ + --secret-file "$home/config/ext-secret" --text-file "$home/text.txt" \ + >/dev/null 2>"$err"; rc=$? + expect_code 1 "$rc" "missing allowlist" + pass "10 unauthorized and missing allowlist write no inbox" +} + +# --- 11. Hermes crewmate spawn string + TUI gating unchanged ---------------- + +test_11_hermes_tui_launch_gating_unchanged() { + local spawn_src + spawn_src=$(cat "$ROOT/bin/fm-spawn.sh") + assert_contains "$spawn_src" "chat --tui" \ + "crewmate Hermes launch template must still be hermes chat --tui" + assert_contains "$spawn_src" "supports persistent TUI spawns only on tmux and herdr" \ + "hermes crew launch must still be TUI-gated" + pass "11 Hermes crewmate TUI launch gating unchanged" +} + +# --- 12. hermes refused as secondmate --------------------------------------- + +test_12_hermes_refused_as_secondmate() { + local home out rc crew second spawn_src harness_src + home="$TMP_ROOT/c12" + mkdir -p "$home/config" "$home/not-a-secondmate" + printf 'hermes\n' > "$home/config/crew-harness" + spawn_src=$(cat "$ROOT/bin/fm-spawn.sh") + harness_src=$(cat "$ROOT/bin/fm-harness.sh") + assert_contains "$spawn_src" "harness=hermes is verified for crewmates and scouts only" \ + "secondmate hermes must keep the crew-only diagnostic" + assert_contains "$harness_src" "crew_only_harness" \ + "implicit secondmate resolution must still filter crew-only hermes" + out=$(home_env "$home" "$SPAWN" hermes-secondmate-x2 "$home/not-a-secondmate" \ + --secondmate --harness hermes 2>&1) || rc=$? + rc=${rc:-0} + [ "$rc" -ne 0 ] || fail "explicit hermes secondmate must still be refused" + assert_contains "$out" "crewmates and scouts only" "spawn must still refuse hermes as a secondmate" + assert_not_contains "$out" "unknown harness" "crew-only hermes must not look like an unknown adapter" + crew=$(home_env "$home" "$HARNESS" crew) + [ "$crew" = hermes ] || fail "crew-harness=hermes must still resolve for crewmates (got '$crew')" + second=$(home_env "$home" "$HARNESS" secondmate) + [ "$second" != hermes ] || fail "crew-only hermes must still be filtered from implicit secondmate resolution" + pass "12 hermes refused as secondmate" +} + +# --- 13. transient 5xx/429 clears posting and allows retry ------------------ + +test_13_transient_http_clears_posting_and_retries() { + local home slug posting receipt sent out + home="$TMP_ROOT/c13" + setup_home "$home" + slug=$(intake_ok "$home" "retry after discord blip") + write_text "$home/ans.txt" "retryable answer" + home_env "$home" "$EMIT" --request-id "$RID" --kind answer --generation 1 \ + --text-file "$home/ans.txt" >/dev/null + posting="$home/state/ext-outbox/${slug}.answer.1.posting" + receipt="$home/state/ext-outbox/${slug}.answer.1.receipt.json" + sent="$home/sent.log" + : > "$sent" + out=$(home_env "$home" env PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" <<'PY' +import io, os, sys, urllib.error +from email.message import EmailMessage +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent = sys.argv[1], sys.argv[2] +os.environ["FM_HOME"] = home +def send(_payload): + raise urllib.error.HTTPError( + "https://discord.test/messages", 503, "unavailable", + EmailMessage(), io.BytesIO(b""), + ) +print(",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + ) + assert_contains "$out" "failed" "transient 5xx must return failed" + assert_absent "$posting" "transient 5xx must delete the posting marker" + assert_absent "$receipt" "transient 5xx must not write a receipt" + out=$(home_env "$home" env PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" <<'PY' +import os, sys +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent = sys.argv[1], sys.argv[2] +os.environ["FM_HOME"] = home +def send(payload): + with open(sent, "a", encoding="utf-8") as fh: + fh.write(payload["text"] + "\n") + return {"ok": True, "discord_message_id": "13"} +print(",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + ) + assert_contains "$out" "sent" "cleared posting marker must allow a later send" + assert_grep "retryable answer" "$sent" "retry after transient 5xx must deliver once" + assert_present "$receipt" "successful retry must write a receipt" + + write_text "$home/ans2.txt" "rate limited then retry" + home_env "$home" "$EMIT" --request-id "$RID" --kind answer --generation 2 \ + --text-file "$home/ans2.txt" >/dev/null + posting="$home/state/ext-outbox/${slug}.answer.2.posting" + receipt="$home/state/ext-outbox/${slug}.answer.2.receipt.json" + : > "$sent" + out=$(home_env "$home" env PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" <<'PY' +import io, os, sys, urllib.error +from email.message import EmailMessage +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home = sys.argv[1] +os.environ["FM_HOME"] = home +def send(_payload): + raise urllib.error.HTTPError( + "https://discord.test/messages", 429, "too many requests", + EmailMessage(), io.BytesIO(b""), + ) +print(",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + ) + assert_contains "$out" "failed" "HTTP 429 must return failed" + assert_absent "$posting" "HTTP 429 must delete the posting marker" + out=$(home_env "$home" env PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" <<'PY' +import os, sys +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent = sys.argv[1], sys.argv[2] +os.environ["FM_HOME"] = home +def send(payload): + with open(sent, "a", encoding="utf-8") as fh: + fh.write(payload["text"] + "\n") + return {"ok": True, "discord_message_id": "13b"} +print(",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + ) + assert_contains "$out" "sent" "cleared posting marker after 429 must allow a later send" + assert_grep "rate limited then retry" "$sent" "retry after 429 must deliver once" + assert_present "$receipt" "successful 429 retry must write a receipt" + pass "13 transient 5xx/429 clears posting marker and allows retry" +} + +# --- 14. mid-delivery still refuses automatic plugin repost ----------------- + +test_14_mid_delivery_refuses_plugin_repost() { + local home slug posting sent out + home="$TMP_ROOT/c14" + setup_home "$home" + slug=$(intake_ok "$home" "do not double post") + write_text "$home/ans.txt" "ambiguous answer" + home_env "$home" "$EMIT" --request-id "$RID" --kind answer --generation 1 \ + --text-file "$home/ans.txt" >/dev/null + home_env "$home" "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 >/dev/null + posting="$home/state/ext-outbox/${slug}.answer.1.posting" + sent="$home/sent.log" + : > "$sent" + out=$(home_env "$home" env PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" \ + "$home/state/ext-outbox/${slug}.answer.1.json" <<'PY' +import os, sys +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent, path = sys.argv[1], sys.argv[2], sys.argv[3] +os.environ["FM_HOME"] = home +def send(payload): + with open(sent, "a", encoding="utf-8") as fh: + fh.write("SHOULD_NOT_SEND\n") + raise RuntimeError("send must not run during mid-delivery") +print(outbox_poster.deliver_one(Path(path), send=send, home=Path(home))) +PY + ) + assert_contains "$out" "mid-delivery" "plugin must refuse automatic repost while posting has no receipt" + [ ! -s "$sent" ] || fail "mid-delivery must not invoke send" + assert_present "$posting" "ambiguous mid-delivery must keep the posting marker" + pass "14 mid-delivery still refuses automatic plugin repost" +} + +# --- 15. wake failure does not leave a silent offered marker ---------------- + +test_15_wake_failure_does_not_leave_silent_offered() { + local home slug rc err offered inbox wakes + home="$TMP_ROOT/c15" + setup_home "$home" + slug=$(slug_of "$RID") + offered="$home/state/ext-context/${slug}.offered.json" + inbox="$home/state/ext-inbox/${slug}.json" + write_text "$home/text.txt" "wake me later" + err="$home/wake.err" + home_env "$home" env FM_WAKE_QUEUE=/dev/full "$INTAKE" \ + --request-id "$RID" --guild-id "$GUILD" --channel-id "$CHANNEL" \ + --thread-id "$THREAD" --message-id "$MESSAGE" --author "$AUTHOR" \ + --secret-file "$home/config/ext-secret" --text-file "$home/text.txt" \ + >/dev/null 2>"$err"; rc=$? + expect_code 1 "$rc" "intake must fail when the wake cannot be appended" + assert_grep "could not append the wake" "$err" "intake must name the wake failure" + assert_present "$inbox" "wake failure must keep the inbox so retry is possible" + assert_absent "$offered" "wake failure must not leave a claimed offer marker" + intake_ok "$home" "wake me later" >/dev/null + assert_present "$offered" "re-intake after wake failure must claim the offer" + wakes=$(grep -c "ext-request $slug" "$home/state/.wake-queue") + [ "$wakes" = 1 ] || fail "re-intake after wake failure must wake once, got $wakes" + pass "15 wake failure does not leave a permanently silent offered marker" +} + +# --- 16. ambiguous timeout/URLError keeps posting and refuses retry --------- + +test_16_ambiguous_urlerror_keeps_mid_delivery() { + local home slug posting receipt failed sent out + home="$TMP_ROOT/c16" + setup_home "$home" + slug=$(intake_ok "$home" "maybe it landed") + write_text "$home/ans.txt" "ambiguous timeout" + home_env "$home" "$EMIT" --request-id "$RID" --kind answer --generation 1 \ + --text-file "$home/ans.txt" >/dev/null + posting="$home/state/ext-outbox/${slug}.answer.1.posting" + receipt="$home/state/ext-outbox/${slug}.answer.1.receipt.json" + failed="$home/state/ext-outbox/${slug}.answer.1.failed.json" + sent="$home/sent.log" + : > "$sent" + out=$(home_env "$home" env PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" <<'PY' +import os, sys, urllib.error +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home = sys.argv[1] +os.environ["FM_HOME"] = home +def send(_payload): + raise urllib.error.URLError("timed out") +print(",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + ) + assert_contains "$out" "mid-delivery" "timeout after possible accept must stay mid-delivery" + assert_present "$posting" "ambiguous URLError must keep the posting marker" + assert_absent "$receipt" "ambiguous URLError must not write a receipt" + assert_absent "$failed" "ambiguous URLError must not write a terminal failed marker" + out=$(home_env "$home" env PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" <<'PY' +import os, sys +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent = sys.argv[1], sys.argv[2] +os.environ["FM_HOME"] = home +def send(payload): + with open(sent, "a", encoding="utf-8") as fh: + fh.write("SHOULD_NOT_RETRY\n") + return {"ok": True, "discord_message_id": "16"} +print(",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + ) + assert_contains "$out" "mid-delivery" "later drain must refuse automatic retry" + [ ! -s "$sent" ] || fail "ambiguous mid-delivery must not invoke send again" + assert_present "$posting" "ambiguous mid-delivery must keep the posting marker after the second drain" + pass "16 ambiguous timeout/URLError keeps mid-delivery and refuses automatic retry" +} + +# --- 17. permanent 4xx is terminal failed, not endless retry ---------------- + +test_17_permanent_4xx_is_terminal_failed() { + local home slug posting receipt failed sent out pending + home="$TMP_ROOT/c17" + setup_home "$home" + slug=$(intake_ok "$home" "too long for discord") + write_text "$home/ans.txt" "permanent client error" + home_env "$home" "$EMIT" --request-id "$RID" --kind answer --generation 1 \ + --text-file "$home/ans.txt" >/dev/null + posting="$home/state/ext-outbox/${slug}.answer.1.posting" + receipt="$home/state/ext-outbox/${slug}.answer.1.receipt.json" + failed="$home/state/ext-outbox/${slug}.answer.1.failed.json" + sent="$home/sent.log" + : > "$sent" + out=$(home_env "$home" env PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" <<'PY' +import io, os, sys, urllib.error +from email.message import EmailMessage +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home = sys.argv[1] +os.environ["FM_HOME"] = home +def send(_payload): + raise urllib.error.HTTPError( + "https://discord.test/messages", 400, "bad request", + EmailMessage(), io.BytesIO(b""), + ) +print(",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + ) + assert_contains "$out" "terminal-failed" "permanent 4xx must return terminal-failed" + assert_present "$failed" "permanent 4xx must write a terminal failed marker" + assert_absent "$posting" "permanent 4xx must not leave a posting marker that looks mid-delivery" + assert_absent "$receipt" "permanent 4xx must not write a success receipt" + pending=$(home_env "$home" "$OUTBOX" pending) + [ -z "$pending" ] || fail "pending must not list a terminal-failed payload, got: $pending" + out=$(home_env "$home" env PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" \ + "$home/state/ext-outbox/${slug}.answer.1.json" <<'PY' +import os, sys +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent, path = sys.argv[1], sys.argv[2], sys.argv[3] +os.environ["FM_HOME"] = home +def send(payload): + with open(sent, "a", encoding="utf-8") as fh: + fh.write("SHOULD_NOT_RETRY\n") + return {"ok": True, "discord_message_id": "17"} +print("drain=" + ",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +print("one=" + outbox_poster.deliver_one(Path(path), send=send, home=Path(home))) +PY + ) + assert_contains "$out" "drain=" "second drain must run" + [ "$(printf '%s\n' "$out" | awk -F= '/^drain=/{print $2}')" = "" ] \ + || fail "pending drain must not retry a terminal-failed payload" + assert_contains "$out" "one=terminal-failed" "direct deliver_one must refuse after terminal 4xx" + [ ! -s "$sent" ] || fail "permanent 4xx must not invoke send again" + pass "17 permanent 4xx is terminal failed, not endless retry" +} + +# --- 18. text under the Discord budget posts once --------------------------- + +test_18_under_limit_is_one_post() { + local home slug sent out n + home="$TMP_ROOT/c18" + setup_home "$home" + slug=$(intake_ok "$home" "short reply") + write_text "$home/ans.txt" "Aye, all shipshape." + home_env "$home" "$EMIT" --request-id "$RID" --kind answer --generation 1 \ + --text-file "$home/ans.txt" >/dev/null + sent="$home/sent.log" + : > "$sent" + out=$(home_env "$home" env FM_EXT_DISCORD_REPLY_MAX_CHARS=50 \ + PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" <<'PY' +import os, sys +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent = sys.argv[1], sys.argv[2] +os.environ["FM_HOME"] = home +def send(payload): + with open(sent, "a", encoding="utf-8") as fh: + fh.write(payload["text"] + "\n") + return {"ok": True, "discord_message_id": "18"} +print(",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + ) + assert_contains "$out" "sent" "under-limit reply must send" + n=$(wc -l < "$sent" | tr -d ' ') + [ "$n" = 1 ] || fail "under-limit reply must be one post, got $n" + assert_grep "Aye, all shipshape." "$sent" "under-limit post must be the unnumbered text" + pass "18 text under the Discord budget is one post" +} + +# --- 19. text over the budget posts ordered chunks without X pairing -------- + +test_19_over_limit_posts_chunks_in_order_without_fmx_token() { + local home slug sent out n first last + home="$TMP_ROOT/c19" + setup_home "$home" + slug=$(intake_ok "$home" "long reply") + write_text "$home/ans.txt" \ + "The captain has me on a sign-in redirect fix, a docs tidy, and keeping the build green while other jobs run in the background today." + home_env "$home" "$EMIT" --request-id "$RID" --kind answer --generation 1 \ + --text-file "$home/ans.txt" >/dev/null + sent="$home/sent.log" + : > "$sent" + out=$(home_env "$home" env -u FMX_PAIRING_TOKEN -u FMX_RELAY_URL \ + FM_EXT_DISCORD_REPLY_MAX_CHARS=50 PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" <<'PY' +import os, sys +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent = sys.argv[1], sys.argv[2] +os.environ["FM_HOME"] = home +os.environ.pop("FMX_PAIRING_TOKEN", None) +os.environ.pop("FMX_RELAY_URL", None) +def send(payload): + with open(sent, "a", encoding="utf-8") as fh: + fh.write(payload["text"] + "\n") + return {"ok": True, "discord_message_id": str(payload["chunk_index"])} +print("token=" + os.environ.get("FMX_PAIRING_TOKEN", "")) +print("result=" + ",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + ) + assert_contains "$out" "token=" "token probe must print" + printf '%s\n' "$out" | awk -F= '/^token=/{print $2}' | grep -q . \ + && fail "split must not require FMX_PAIRING_TOKEN" + assert_contains "$out" "result=sent" "over-limit reply must send after split" + n=$(wc -l < "$sent" | tr -d ' ') + [ "$n" -gt 1 ] || fail "over-limit reply must post more than one message, got $n" + first=$(sed -n '1p' "$sent") + last=$(tail -n 1 "$sent") + case "$first" in *" (1/$n)") : ;; *) fail "first chunk must be numbered (1/$n): $first" ;; esac + case "$last" in *" ($n/$n)") : ;; *) fail "last chunk must be numbered ($n/$n): $last" ;; esac + awk -v lim=50 'length($0)>lim{exit 1}' "$sent" \ + || fail "every Discord chunk must stay within the 50-character budget" + assert_present "$home/state/ext-outbox/${slug}.answer.1.receipt.json" \ + "split delivery must write one receipt for the generation" + pass "19 over-limit text posts ordered chunks without FMX_PAIRING_TOKEN" +} + +# --- 20. later-chunk transient failure resumes without reposting ------------ + +test_20_later_chunk_transient_resumes_without_repost() { + local home slug sent1 sent2 posting progress inflight out first + home="$TMP_ROOT/c20" + setup_home "$home" + slug=$(intake_ok "$home" "resume split") + write_text "$home/ans.txt" \ + "The captain has me on a sign-in redirect fix, a docs tidy, and keeping the build green while other jobs run in the background today." + home_env "$home" "$EMIT" --request-id "$RID" --kind answer --generation 1 \ + --text-file "$home/ans.txt" >/dev/null + sent1="$home/sent1.log" + sent2="$home/sent2.log" + posting="$home/state/ext-outbox/${slug}.answer.1.posting" + progress="$home/state/ext-outbox/${slug}.answer.1.progress.json" + inflight="$home/state/ext-outbox/${slug}.answer.1.inflight" + : > "$sent1" + : > "$sent2" + out=$(home_env "$home" env FM_EXT_DISCORD_REPLY_MAX_CHARS=50 \ + PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent1" <<'PY' +import io, os, sys, urllib.error +from email.message import EmailMessage +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent = sys.argv[1], sys.argv[2] +os.environ["FM_HOME"] = home +def send(payload): + if payload["chunk_index"] > 0: + raise urllib.error.HTTPError( + "https://discord.test/messages", 503, "unavailable", + EmailMessage(), io.BytesIO(b""), + ) + with open(sent, "a", encoding="utf-8") as fh: + fh.write(payload["text"] + "\n") + return {"ok": True, "discord_message_id": "20a"} +print(",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + ) + assert_contains "$out" "failed" "later-chunk 503 must return failed" + assert_present "$posting" "partial success must keep the posting marker for resume" + assert_present "$progress" "partial success must record chunk progress" + assert_grep '"posted_count": 1' "$progress" "progress must record the first posted chunk" + assert_absent "$inflight" "later-chunk 503 must release the exclusive inflight claim so resume can proceed" + first=$(cat "$sent1") + [ -n "$first" ] || fail "first chunk must have posted before the later 503" + out=$(home_env "$home" env FM_EXT_DISCORD_REPLY_MAX_CHARS=50 \ + PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent2" <<'PY' +import os, sys +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent = sys.argv[1], sys.argv[2] +os.environ["FM_HOME"] = home +def send(payload): + with open(sent, "a", encoding="utf-8") as fh: + fh.write(payload["text"] + "\n") + return {"ok": True, "discord_message_id": "20b"} +print(",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + ) + assert_contains "$out" "sent" "resume after later-chunk 503 must finish the remaining chunks" + grep -Fqx "$first" "$sent2" && fail "resume must not repost the already sent first chunk" + [ -s "$sent2" ] || fail "resume must post the remaining chunks" + pass "20 later-chunk transient failure resumes without reposting earlier chunks" +} + +# --- 21. concurrent resume claims: only one poster sends the next chunk ----- + +_test_21_setup_resumable() { + local home=$1 slug sent posting progress inflight out + slug=$(intake_ok "$home" "concurrent resume") + write_text "$home/ans.txt" \ + "The captain has me on a sign-in redirect fix, a docs tidy, and keeping the build green while other jobs run in the background today." + home_env "$home" "$EMIT" --request-id "$RID" --kind answer --generation 1 \ + --text-file "$home/ans.txt" >/dev/null + sent="$home/setup-sent.log" + posting="$home/state/ext-outbox/${slug}.answer.1.posting" + progress="$home/state/ext-outbox/${slug}.answer.1.progress.json" + inflight="$home/state/ext-outbox/${slug}.answer.1.inflight" + : > "$sent" + out=$(home_env "$home" env FM_EXT_DISCORD_REPLY_MAX_CHARS=50 \ + PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$sent" <<'PY' +import io, os, sys, urllib.error +from email.message import EmailMessage +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent = sys.argv[1], sys.argv[2] +os.environ["FM_HOME"] = home +def send(payload): + if payload["chunk_index"] > 0: + raise urllib.error.HTTPError( + "https://discord.test/messages", 503, "unavailable", + EmailMessage(), io.BytesIO(b""), + ) + with open(sent, "a", encoding="utf-8") as fh: + fh.write(payload["text"] + "\n") + return {"ok": True, "discord_message_id": "21a"} +print(",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + ) + assert_contains "$out" "failed" "setup later-chunk 503 must return failed" + assert_present "$posting" "setup must keep the posting marker for resume" + assert_present "$progress" "setup must record chunk progress" + assert_grep '"posted_count": 1' "$progress" "setup must leave posted_count 1" + assert_absent "$inflight" "setup must leave the inflight claim released" + printf '%s\n' "$slug" > "$home/setup.slug" +} + +test_21_concurrent_resume_exclusive_inflight_claim() { + local home slug go inflight rc1 rc2 p1 p2 winner losers chunk1 + home="$TMP_ROOT/c21begin" + setup_home "$home" + _test_21_setup_resumable "$home" + slug=$(cat "$home/setup.slug") + inflight="$home/state/ext-outbox/${slug}.answer.1.inflight" + go="$home/go" + rm -f "$go" "$home/rc1" "$home/rc2" + ( + while [ ! -f "$go" ]; do sleep 0.01; done + home_env "$home" "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 \ + >/dev/null 2>"$home/begin1.err" + echo $? > "$home/rc1" + ) & + p1=$! + ( + while [ ! -f "$go" ]; do sleep 0.01; done + home_env "$home" "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 \ + >/dev/null 2>"$home/begin2.err" + echo $? > "$home/rc2" + ) & + p2=$! + sleep 0.05 + touch "$go" + wait "$p1" "$p2" || true + rc1=$(cat "$home/rc1") + rc2=$(cat "$home/rc2") + winner=0 + losers=0 + case "$rc1" in + 0) winner=$((winner + 1)) ;; + 3) losers=$((losers + 1)) ;; + *) fail "concurrent begin child 1 must exit 0 or 3, got $rc1" ;; + esac + case "$rc2" in + 0) winner=$((winner + 1)) ;; + 3) losers=$((losers + 1)) ;; + *) fail "concurrent begin child 2 must exit 0 or 3, got $rc2" ;; + esac + [ "$winner" = 1 ] || fail "exactly one concurrent begin must claim the send right (winners=$winner rc1=$rc1 rc2=$rc2)" + [ "$losers" = 1 ] || fail "the other concurrent begin must be mid-delivery (losers=$losers rc1=$rc1 rc2=$rc2)" + assert_present "$inflight" "the winning begin must hold the exclusive inflight marker" + + home="$TMP_ROOT/c21send" + setup_home "$home" + _test_21_setup_resumable "$home" + slug=$(cat "$home/setup.slug") + go="$home/go" + rm -f "$go" "$home/ready1" "$home/ready2" + : > "$home/chunks.log" + home_env "$home" env FM_EXT_DISCORD_REPLY_MAX_CHARS=50 \ + PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$home/chunks.log" "$go" \ + "$home/ready1" "$home/out1" <<'PY' & +import fcntl, os, sys, time +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent, go, ready, out = sys.argv[1:6] +os.environ["FM_HOME"] = home +Path(ready).write_text("1") +while not Path(go).is_file(): + time.sleep(0.01) +def send(payload): + time.sleep(0.2) + with open(sent, "a", encoding="utf-8") as fh: + fcntl.flock(fh, fcntl.LOCK_EX) + fh.write(str(payload["chunk_index"]) + "\n") + fcntl.flock(fh, fcntl.LOCK_UN) + return {"ok": True, "discord_message_id": "21b-%s" % payload["chunk_index"]} +Path(out).write_text(",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + p1=$! + home_env "$home" env FM_EXT_DISCORD_REPLY_MAX_CHARS=50 \ + PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$home/chunks.log" "$go" \ + "$home/ready2" "$home/out2" <<'PY' & +import fcntl, os, sys, time +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, sent, go, ready, out = sys.argv[1:6] +os.environ["FM_HOME"] = home +Path(ready).write_text("1") +while not Path(go).is_file(): + time.sleep(0.01) +def send(payload): + time.sleep(0.2) + with open(sent, "a", encoding="utf-8") as fh: + fcntl.flock(fh, fcntl.LOCK_EX) + fh.write(str(payload["chunk_index"]) + "\n") + fcntl.flock(fh, fcntl.LOCK_UN) + return {"ok": True, "discord_message_id": "21b-%s" % payload["chunk_index"]} +Path(out).write_text(",".join(outbox_poster.drain_outbox(send=send, home=Path(home)))) +PY + p2=$! + for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do + [ -f "$home/ready1" ] && [ -f "$home/ready2" ] && break + sleep 0.05 + done + [ -f "$home/ready1" ] && [ -f "$home/ready2" ] \ + || fail "both concurrent poster processes must reach the start gate" + touch "$go" + wait "$p1" "$p2" || true + [ -f "$home/out1" ] && [ -f "$home/out2" ] \ + || fail "both concurrent poster processes must finish" + chunk1=$(grep -c '^1$' "$home/chunks.log" || true) + [ "$chunk1" = 1 ] || fail "exactly one poster must send the next chunk (chunk 1 count=$chunk1 log=$(tr '\n' ',' < "$home/chunks.log"))" + grep -q '^0$' "$home/chunks.log" && fail "resume must not repost chunk 0" + if grep -q . "$home/chunks.log"; then + sort "$home/chunks.log" | uniq -d | grep -q . \ + && fail "no chunk index may be posted twice (log=$(tr '\n' ',' < "$home/chunks.log"))" + fi + pass "21 concurrent resume claims: only one poster sends the next chunk" +} + +# --- 22. pre-send failures release inflight; abort drops it first ----------- + +test_22_presend_release_and_abort_inflight_first() { + local home slug posting progress inflight payload out rc order inflight_line posting_line + home="$TMP_ROOT/c22split" + setup_home "$home" + _test_21_setup_resumable "$home" + slug=$(cat "$home/setup.slug") + posting="$home/state/ext-outbox/${slug}.answer.1.posting" + progress="$home/state/ext-outbox/${slug}.answer.1.progress.json" + inflight="$home/state/ext-outbox/${slug}.answer.1.inflight" + payload="$home/state/ext-outbox/${slug}.answer.1.json" + out=$(home_env "$home" env FM_EXT_DISCORD_REPLY_MAX_CHARS=50 \ + PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$payload" <<'PY' +import os, sys +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, path = sys.argv[1], sys.argv[2] +os.environ["FM_HOME"] = home +def boom(*_args, **_kwargs): + raise RuntimeError("forced split failure") +outbox_poster.split_reply = boom +def send(_payload): + raise AssertionError("send must not run after a pre-send split failure") +print(outbox_poster.deliver_one(Path(path), send=send, home=Path(home))) +PY + ) + assert_contains "$out" "mid-delivery" "split failure after posted chunks must refuse without sending" + assert_absent "$inflight" "split failure after posted chunks must release the exclusive inflight claim" + assert_present "$posting" "split failure after posted chunks must keep posting for resume" + assert_present "$progress" "split failure after posted chunks must keep progress" + home_env "$home" "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 \ + >/dev/null; rc=$? + expect_code 0 "$rc" "begin after split-failure release" + assert_present "$inflight" "begin after split-failure release must be able to claim inflight" + + home="$TMP_ROOT/c22mismatch" + setup_home "$home" + _test_21_setup_resumable "$home" + slug=$(cat "$home/setup.slug") + posting="$home/state/ext-outbox/${slug}.answer.1.posting" + progress="$home/state/ext-outbox/${slug}.answer.1.progress.json" + inflight="$home/state/ext-outbox/${slug}.answer.1.inflight" + payload="$home/state/ext-outbox/${slug}.answer.1.json" + jq '.total = 99' "$progress" > "$home/bad-progress.json" + home_env "$home" "$OUTBOX" progress --slug "$slug" --kind answer --generation 1 \ + --progress-file "$home/bad-progress.json" >/dev/null + out=$(home_env "$home" env FM_EXT_DISCORD_REPLY_MAX_CHARS=50 \ + PYTHONPATH="$PLUGIN" "$PYTHON_BIN" - "$home" "$payload" <<'PY' +import os, sys +from pathlib import Path +sys.path.insert(0, os.environ["PYTHONPATH"]) +import outbox_poster +home, path = sys.argv[1], sys.argv[2] +os.environ["FM_HOME"] = home +def send(_payload): + raise AssertionError("send must not run after a chunk-count mismatch") +print(outbox_poster.deliver_one(Path(path), send=send, home=Path(home))) +PY + ) + assert_contains "$out" "mid-delivery" "chunk-count mismatch must refuse without sending" + assert_absent "$inflight" "chunk-count mismatch must release the exclusive inflight claim" + assert_present "$posting" "chunk-count mismatch must keep posting for resume" + home_env "$home" "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 \ + >/dev/null; rc=$? + expect_code 0 "$rc" "begin after mismatch release" + assert_present "$inflight" "begin after mismatch release must be able to claim inflight" + + home="$TMP_ROOT/c22abort" + setup_home "$home" + _test_21_setup_resumable "$home" + slug=$(cat "$home/setup.slug") + posting="$home/state/ext-outbox/${slug}.answer.1.posting" + progress="$home/state/ext-outbox/${slug}.answer.1.progress.json" + inflight="$home/state/ext-outbox/${slug}.answer.1.inflight" + home_env "$home" "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 >/dev/null + assert_present "$inflight" "abort-order fixture must hold inflight before abort" + rm -f -- "$posting" "$progress" + assert_absent "$posting" "fixture must drop posting to simulate a mid-abort crash" + assert_absent "$progress" "fixture must drop progress to simulate a mid-abort crash" + assert_present "$inflight" "stale inflight must remain until abort runs" + home_env "$home" "$OUTBOX" abort --slug "$slug" --kind answer --generation 1 >/dev/null + assert_absent "$inflight" "abort must release leftover inflight even when posting is already gone" + home_env "$home" "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 \ + >/dev/null; rc=$? + expect_code 0 "$rc" "begin after abort cleared leftover inflight" + + home="$TMP_ROOT/c22order" + setup_home "$home" + _test_21_setup_resumable "$home" + slug=$(cat "$home/setup.slug") + inflight="$home/state/ext-outbox/${slug}.answer.1.inflight" + home_env "$home" "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 >/dev/null + assert_present "$inflight" "abort-order probe must start with a claimed inflight" + order="$home/abort-order.log" + ( + # shellcheck source=bin/fm-ext-lib.sh + . "$ROOT/bin/fm-ext-lib.sh" + eval "$(declare -f fm_ext_private_artifact_remove | sed '1s/fm_ext_private_artifact_remove/_orig_remove/')" + fm_ext_private_artifact_remove() { + printf '%s\n' "$2" >> "$order" + _orig_remove "$@" + } + fm_ext_outbox_abort "$home/state/ext-outbox" "$slug" answer 1 + ) + [ -s "$order" ] || fail "abort must record artifact removals" + inflight_line=$(grep -n "\.inflight$" "$order" | head -1 | cut -d: -f1) + posting_line=$(grep -n "\.posting$" "$order" | head -1 | cut -d: -f1) + [ -n "$inflight_line" ] || fail "abort must remove inflight" + [ -n "$posting_line" ] || fail "abort must remove posting" + [ "$inflight_line" -lt "$posting_line" ] \ + || fail "abort must release inflight before removing posting (order=$(tr '\n' ',' < "$order"))" + pass "22 pre-send failures release inflight; abort drops inflight first" +} + +# --- 23. stale inflight steal: dead+TTL only; live concurrent still one ----- + +_dead_pid() { + local pid + true & + pid=$! + wait "$pid" || true + if kill -0 "$pid" 2>/dev/null; then + fail "could not obtain a dead pid for inflight steal tests" + fi + printf '%s\n' "$pid" +} + +_rewrite_inflight() { + local file=$1 pid=$2 recorded_at=$3 tmp + tmp="${file}.rewrite.$$" + jq -c --argjson pid "$pid" --argjson recorded_at "$recorded_at" \ + '.pid=$pid | .recorded_at=$recorded_at' "$file" > "$tmp" \ + || fail "could not rewrite inflight claim" + cat "$tmp" > "$file" || fail "could not replace inflight claim" + rm -f -- "$tmp" + chmod 600 "$file" || true +} + +test_23_stale_inflight_ttl_and_dead_pid_steal() { + local home slug inflight posting dead now ttl rc rc1 rc2 p1 p2 go winner losers owner lockbase + now=100000 + ttl=30 + home="$TMP_ROOT/c23steal" + setup_home "$home" + _test_21_setup_resumable "$home" + slug=$(cat "$home/setup.slug") + inflight="$home/state/ext-outbox/${slug}.answer.1.inflight" + home_env "$home" env FM_EXT_NOW_OVERRIDE="$now" FM_EXT_INFLIGHT_TTL_SECS="$ttl" \ + "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 >/dev/null + assert_present "$inflight" "begin must record an inflight claim" + assert_grep '"pid"' "$inflight" "inflight claim must record owner pid" + owner=$(jq -r '.pid' "$inflight") + [ "$owner" = "$$" ] || fail "inflight owner pid must be the claiming poster (got $owner want $$)" + + _rewrite_inflight "$inflight" "$owner" "$((now - ttl - 10))" + home_env "$home" env FM_EXT_NOW_OVERRIDE="$now" FM_EXT_INFLIGHT_TTL_SECS="$ttl" \ + "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 \ + >/dev/null 2>/dev/null; rc=$? + expect_code 3 "$rc" "live owner past TTL must still refuse steal" + assert_present "$inflight" "live-owner refuse must keep the inflight claim" + + dead=$(_dead_pid) + _rewrite_inflight "$inflight" "$dead" "$now" + home_env "$home" env FM_EXT_NOW_OVERRIDE="$now" FM_EXT_INFLIGHT_TTL_SECS="$ttl" \ + "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 \ + >/dev/null 2>/dev/null; rc=$? + expect_code 3 "$rc" "dead owner inside TTL must refuse steal" + + _rewrite_inflight "$inflight" "$dead" "$((now - ttl - 10))" + home_env "$home" env FM_EXT_NOW_OVERRIDE="$now" FM_EXT_INFLIGHT_TTL_SECS="$ttl" \ + "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 >/dev/null; rc=$? + expect_code 0 "$rc" "dead owner past TTL must steal once" + owner=$(jq -r '.pid' "$inflight") + [ "$owner" = "$$" ] || fail "stolen inflight must record the new owner pid (got $owner want $$)" + home_env "$home" env FM_EXT_NOW_OVERRIDE="$now" FM_EXT_INFLIGHT_TTL_SECS="$ttl" \ + "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 \ + >/dev/null 2>/dev/null; rc=$? + expect_code 3 "$rc" "after a successful steal, a second begin must refuse" + + home="$TMP_ROOT/c23live" + setup_home "$home" + _test_21_setup_resumable "$home" + slug=$(cat "$home/setup.slug") + inflight="$home/state/ext-outbox/${slug}.answer.1.inflight" + posting="$home/state/ext-outbox/${slug}.answer.1.posting" + go="$home/go" + rm -f "$go" "$home/rc1" "$home/rc2" + ( + while [ ! -f "$go" ]; do sleep 0.01; done + home_env "$home" env FM_EXT_INFLIGHT_TTL_SECS=0 \ + "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 \ + >/dev/null 2>"$home/begin1.err" + echo $? > "$home/rc1" + sleep 1 + ) & + p1=$! + ( + while [ ! -f "$go" ]; do sleep 0.01; done + home_env "$home" env FM_EXT_INFLIGHT_TTL_SECS=0 \ + "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 \ + >/dev/null 2>"$home/begin2.err" + echo $? > "$home/rc2" + sleep 1 + ) & + p2=$! + sleep 0.05 + touch "$go" + wait "$p1" "$p2" || true + rc1=$(cat "$home/rc1") + rc2=$(cat "$home/rc2") + winner=0 + losers=0 + case "$rc1" in + 0) winner=$((winner + 1)) ;; + 3) losers=$((losers + 1)) ;; + *) fail "TTL=0 concurrent begin child 1 must exit 0 or 3, got $rc1" ;; + esac + case "$rc2" in + 0) winner=$((winner + 1)) ;; + 3) losers=$((losers + 1)) ;; + *) fail "TTL=0 concurrent begin child 2 must exit 0 or 3, got $rc2" ;; + esac + [ "$winner" = 1 ] || fail "two live concurrent posters must still have one winner under TTL=0 (winners=$winner rc1=$rc1 rc2=$rc2)" + [ "$losers" = 1 ] || fail "two live concurrent posters must still have one mid-delivery loser under TTL=0 (losers=$losers rc1=$rc1 rc2=$rc2)" + assert_present "$inflight" "the live winner must still hold inflight" + assert_present "$posting" "concurrent live refuse must not drop posting" + + home="$TMP_ROOT/c23lock" + mkdir -p "$home/state/ext-outbox" + lockbase="probe.answer.1.inflight.lock" + mkdir "$home/state/ext-outbox/$lockbase" + if ( + # shellcheck source=bin/fm-ext-lib.sh + . "$ROOT/bin/fm-ext-lib.sh" + FM_EXT_INFLIGHT_TTL_SECS=0 + fm_ext_outbox_inflight_steallock_stale "$home/state/ext-outbox" "$lockbase" + ); then + fail "a fresh steal-lock must not be reclaimable when claim TTL is 0" + fi + + home="$TMP_ROOT/c23stealrace" + setup_home "$home" + _test_21_setup_resumable "$home" + slug=$(cat "$home/setup.slug") + inflight="$home/state/ext-outbox/${slug}.answer.1.inflight" + home_env "$home" "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 >/dev/null + dead=$(_dead_pid) + _rewrite_inflight "$inflight" "$dead" "$(($(date +%s) - 10))" + go="$home/go" + rm -f "$go" "$home/rc1" "$home/rc2" + ( + while [ ! -f "$go" ]; do sleep 0.01; done + home_env "$home" env FM_EXT_INFLIGHT_TTL_SECS=0 \ + "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 \ + >/dev/null 2>"$home/begin1.err" + echo $? > "$home/rc1" + sleep 1 + ) & + p1=$! + ( + while [ ! -f "$go" ]; do sleep 0.01; done + home_env "$home" env FM_EXT_INFLIGHT_TTL_SECS=0 \ + "$OUTBOX" begin --slug "$slug" --kind answer --generation 1 \ + >/dev/null 2>"$home/begin2.err" + echo $? > "$home/rc2" + sleep 1 + ) & + p2=$! + sleep 0.05 + touch "$go" + wait "$p1" "$p2" || true + rc1=$(cat "$home/rc1") + rc2=$(cat "$home/rc2") + winner=0 + losers=0 + case "$rc1" in + 0) winner=$((winner + 1)) ;; + 3) losers=$((losers + 1)) ;; + *) fail "TTL=0 steal-vs-steal child 1 must exit 0 or 3, got $rc1" ;; + esac + case "$rc2" in + 0) winner=$((winner + 1)) ;; + 3) losers=$((losers + 1)) ;; + *) fail "TTL=0 steal-vs-steal child 2 must exit 0 or 3, got $rc2" ;; + esac + [ "$winner" = 1 ] || fail "two concurrent TTL=0 stealers must not both win (winners=$winner rc1=$rc1 rc2=$rc2)" + [ "$losers" = 1 ] || fail "two concurrent TTL=0 stealers must have one mid-delivery loser (losers=$losers rc1=$rc1 rc2=$rc2)" + pass "23 stale inflight steal requires dead pid and TTL; live concurrent and TTL=0 steal-vs-steal still one winner" +} + +# --- bootstrap opt-in ------------------------------------------------------- + +test_bootstrap_arms_ext_watch_shim() { + local home out + home="$TMP_ROOT/boot" + setup_home "$home" + out=$(home_env "$home" "$BOOTSTRAP" 2>/dev/null || true) + assert_contains "$out" "EXT: local bridge on" "bootstrap must announce the local bridge" + assert_present "$home/state/ext-watch.check.sh" "bootstrap must drop the ext poll shim" + assert_grep "fm-ext-poll.sh" "$home/state/ext-watch.check.sh" "shim must exec fm-ext-poll.sh" + home_env "$home" "$BOOTSTRAP" >/dev/null 2>&1 || true + pass "bootstrap arms the ext-watch identity shim" +} + +test_poll_noop_when_inactive() { + local home out + home="$TMP_ROOT/poll-off" + mkdir -p "$home" + out=$(home_env "$home" "$POLL") + [ -z "$out" ] || fail "inactive poll must be silent, got: $out" + pass "poll is a hard no-op when the bridge is off" +} + +# Plugin must not dispatch the terminal tool. +test_plugin_has_no_terminal_dispatch() { + if grep -R -n --include='*.py' 'dispatch_tool(' "$PLUGIN" | grep -v 'never' >/dev/null 2>&1; then + fail "gateway plugin must not dispatch_tool(terminal)" + fi + pass "gateway plugin does not dispatch the terminal tool" +} + +export GUILD CHANNEL THREAD AUTHOR +test_1_allowlisted_intake_and_non_fm +test_2_correlation_persists +test_3_immediate_ack +test_4_followup_after_inbox_gone +test_5_multiple_followups_duplicate_generation +test_6_idempotent_intake_and_emit +test_7_unsent_outbox_receipt_once +test_8_restart_one_wake_per_offer +test_9_mid_send_refuse_and_cas_receipt +test_10_unauthorized_and_missing_allowlist +test_11_hermes_tui_launch_gating_unchanged +test_12_hermes_refused_as_secondmate +test_13_transient_http_clears_posting_and_retries +test_14_mid_delivery_refuses_plugin_repost +test_15_wake_failure_does_not_leave_silent_offered +test_16_ambiguous_urlerror_keeps_mid_delivery +test_17_permanent_4xx_is_terminal_failed +test_18_under_limit_is_one_post +test_19_over_limit_posts_chunks_in_order_without_fmx_token +test_20_later_chunk_transient_resumes_without_repost +test_21_concurrent_resume_exclusive_inflight_claim +test_22_presend_release_and_abort_inflight_first +test_23_stale_inflight_ttl_and_dead_pid_steal +test_bootstrap_arms_ext_watch_shim +test_poll_noop_when_inactive +test_plugin_has_no_terminal_dispatch + +echo "all fm-ext-bridge tests passed"