feat(otel): opt-in OpenTelemetry setup, per-agent switch and local OTLP relay

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ on:
   # it produced was the daemon cross-compile, and only because it touched
   # `crates/`. A PR that cannot go red is not a reviewed PR.
   pull_request:
-    branches: [main, feat/fp-cli]
+    branches: [main, feat/fp-cli, feat/tui-ux]
 
 concurrency:
   group: ci-${{ github.ref }}

.github/workflows/osv-scanner.yml

@@ -27,7 +27,7 @@ name: Supply Chain
 
 on:
   pull_request:
-    branches: [main]
+    branches: [main, feat/tui-ux]
   push:
     branches: [main]
   schedule:

CHANGELOG.md

@@ -4,6 +4,9 @@
 
 ### Features
 
+- Add opt-in coding-agent OpenTelemetry setup, reversible per-agent transcript switching, and a loopback OTLP relay (#880).
+- Add optional TypeScript and Python OpenTelemetry span exporters with durable OTLP/JSON spool delivery and no hard runtime dependencies (#880).
+- Port OpenTelemetry help and setup into the new TUI, retaining per-agent transcript overrides and reversible settings. OpenTelemetry stays out of `failproofai config`: it is opt-in, per agent, through `failproofai otel enable <agent>` (#880).
 - New terminal look for the CLI, step one: the palette gains the redesign's label grey, progress-track grey and a red for failures. Descriptions keep the terminal's own colour, and the greys fall back to the dim attribute below 24-bit, so every screen stays readable on light themes (#881)
 - Pickers in the new look: an UPPERCASE heading with a live "n of m selected", a `›` cursor and `■ □` boxes, twelve rows before scrolling, plain `a` to toggle all, and the key hints at the foot. The `policies -i` agent menu now uses the shared picker (#881)
 - Every `--help` page in the new look, cut to usage, options and examples, with every real flag and its real default. The index opens with three getting-started steps, then lists every command by what it is for. `policies add|remove|show --help` now prints the one policies page (#881)
@@ -29,6 +32,8 @@
 
 ### Fixes
 
+- Preserve Codex OTEL header comments, omit local Claude auth-header settings, and require an executable or Copilot extension before configuring VS Code (#880).
+- Keep Python OTEL imports lazy, label direct loopback destinations correctly, omit local Codex auth headers, and scope help options to each verb (#880).
 - `failproofai policies` showed nothing for a machine still on the pre-pack builtins, while those builtins were enforcing. It now says how many under BUILT IN, with the command that moves them into a pack (#881)
 - `failproofai config --disconnect` told you to restart the daemon to stop it sending. It stops on its own within a few seconds, because it polls the credential, and now says so (#881)
 - A config change that landed while the daemon was restarting its collector after a backfill was recorded as already running and never applied. `failproofai config` re-adding an agent hits exactly that, writing its backfill request and then the selection, so the agent stayed off until some unrelated edit (#881)
@@ -50,6 +55,7 @@
 
 ### Dependencies
 
+- Update vulnerable existing build and SDK test lockfile dependencies reported by the PR's OSV scan (#880).
 - Bump `next` to 16.3.8, `sharp` to 0.35.5 and `source-map-js` to 1.2.2, which OSV-Scanner flagged; the dashboard ships `next` and `sharp`. In the SDKs' dev lockfiles: `source-map-js` (TypeScript) and `banks`, `langgraph-sdk`, `multidict` and `pydantic-ai-slim` (Python) (#881)
 
 ## 1.0.10 — 2026-10-05

__tests__/ci/otel-stacked-ci.test.ts

@@ -0,0 +1,13 @@
+import { describe, expect, it } from "vitest";
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { parse } from "yaml";
+
+describe("OTEL stacked PR checks", () => {
+  it.each(["ci.yml", "osv-scanner.yml"])("%s checks the TUI base without adding feature pushes", file => {
+    const workflow = parse(readFileSync(resolve(__dirname, "../../.github/workflows", file), "utf8"));
+    expect(workflow.on.pull_request.branches).toContain("feat/tui-ux");
+    expect(workflow.on.pull_request.branches).toContain("main");
+    expect(workflow.on.push.branches).toEqual(["main"]);
+  });
+});

__tests__/dashboard/jev-activity-row.test.tsx

@@ -137,6 +137,7 @@ vi.mock("@/app/actions/update-policy-params", () => ({
 }));
 
 import HooksClient from "@/app/policies/hooks-client";
+import { getHookActivityAction } from "@/app/actions/get-hook-activity";
 
 /** The table row whose Tool cell reads `tool`. */
 async function rowFor(tool: string): Promise<HTMLElement> {
@@ -162,6 +163,11 @@ async function rowFor(tool: string): Promise<HTMLElement> {
  * it, so a blind retry would close the panel it had just opened.
  */
 async function openRow(user: ReturnType<typeof userEvent.setup>, tool: string, marker: string) {
+  // The initial filter effect refetches and clears expandedRow after 300ms.
+  // Waiting only for a visible row can click BEFORE that effect, so a panel
+  // successfully opened by this helper closes just after the assertion. Wait
+  // for the initial refetch before interacting; keep all UI assertions intact.
+  await waitFor(() => expect(vi.mocked(getHookActivityAction).mock.calls.length).toBeGreaterThanOrEqual(2));
   await waitFor(async () => {
     if (screen.queryByText(marker, { exact: false })) return;
     await user.click(await rowFor(tool));
@@ -172,6 +178,7 @@ async function openRow(user: ReturnType<typeof userEvent.setup>, tool: string, m
 describe("the activity tab with Jev rows", () => {
   beforeEach(() => {
     entries = [];
+    vi.mocked(getHookActivityAction).mockClear();
   });
 
   it("marks a Jev clear in the Decision cell and explains it in the detail panel", async () => {

__tests__/e2e/cli/otel.e2e.test.ts

@@ -0,0 +1,54 @@
+// @vitest-environment node
+import { afterAll, describe, expect, it } from "vitest";
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+
+const root = mkdtempSync(join(tmpdir(), "fpai-otel-bundle-"));
+const home = join(root, "home");
+const binary = join(root, "cli.mjs");
+const repo = resolve(__dirname, "../../..");
+// Like the installed tarball, external runtime deps sit next to the bundle.
+symlinkSync(join(repo, "node_modules"), join(root, "node_modules"), "dir");
+mkdirSync(join(home, ".failproofai"), { recursive: true });
+mkdirSync(join(home, ".codex"), { recursive: true });
+writeFileSync(join(home, ".failproofai/credentials.json"), JSON.stringify({
+  ingest: { url: "http://127.0.0.1:1/v1/events", key: "capture-test" },
+}));
+const original = '# original\nmodel = "test"\n[projects."/work"]\ntrust_level = "trusted"\n';
+writeFileSync(join(home, ".codex/config.toml"), original);
+const built = spawnSync("bun", ["build", "--target=node", "--format=esm", `--outfile=${binary}`,
+  "bin/failproofai.mjs", "--external", "posthog-node", "--external", "sql.js"], {
+  cwd: repo, encoding: "utf8", timeout: 30000,
+});
+afterAll(() => rmSync(root, { recursive: true, force: true }));
+
+function cli(...args: string[]) {
+  return spawnSync("node", [binary, "otel", ...args], {
+    env: { ...process.env, HOME: home, USERPROFILE: home, FAILPROOFAI_HOME: join(home, ".failproofai"),
+      FAILPROOFAI_TELEMETRY_DISABLED: "1", FAILPROOFAI_INGEST_KEY: "", FAILPROOFAI_INGEST_URL: "" },
+    encoding: "utf8", timeout: 15000,
+  });
+}
+
+describe("OTEL standalone bundled CLI", () => {
+  it("bundles JSONC and TOML without unresolved UMD relative imports", () => {
+    expect(built.status, built.stderr).toBe(0);
+    expect(existsSync(binary)).toBe(true);
+    expect(cli("enable", "--help").status).toBe(0);
+  });
+
+  it("enables and restores Codex from a Node-run standalone bundle against an older/unreachable server", () => {
+    const enabled = cli("enable", "codex", "--local", "--yes");
+    expect(enabled.status, enabled.stderr + enabled.stdout).toBe(0);
+    expect(readFileSync(join(home, ".codex/config.toml"), "utf8")).toContain("/v1/logs");
+    const status = cli("status");
+    expect(status.status, status.stderr).toBe(0);
+    expect(status.stdout).toContain("unknown");
+    expect(status.stdout).toContain("local relay");
+    const disabled = cli("disable", "all");
+    expect(disabled.status, disabled.stderr + disabled.stdout).toBe(0);
+    expect(readFileSync(join(home, ".codex/config.toml"), "utf8")).toBe(original);
+  });
+});

__tests__/hooks/configure-wizard.test.ts

@@ -93,6 +93,7 @@ vi.mock("../../src/hooks/integrations", async (importOriginal) => {
 
 import {
   multiSelect,
+  selectOne,
   promptCloudKey,
   OPEN_SOURCE,
   paint,
@@ -105,6 +106,7 @@ import {
   type TTYOut,
 } from "../../src/hooks/tui";
 import { connectToCloud } from "../../src/hooks/cloud-connection";
+import * as otelCli from "../../src/hooks/otel-cli";
 import { validateIngestKey, writeIngestCredential, writeCollectorSettings } from "../../src/hooks/collector-config";
 import { introspectKey } from "../../src/hooks/cloud-introspect";
 import { installHooks, notEnforcingReason, enforcingPolicyCount } from "../../src/hooks/manager";
@@ -220,6 +222,7 @@ beforeEach(() => {
     .mockResolvedValue({ policy: { ok: true, policyCount: 2, deployment: 7 }, ingest: { ok: true }, anyConfigured: true });
   vi.mocked(validateIngestKey).mockReset().mockResolvedValue({ ok: true });
   vi.mocked(introspectKey).mockReset().mockResolvedValue({ kind: "unsupported" });
+  vi.mocked(selectOne).mockReset().mockResolvedValue(false);
   // Healthy and already running is the default: the DAEMON step is one line
   // and needs no password. Daemon tests override these.
   vi.mocked(isDaemonSupportedPlatform).mockReset().mockReturnValue(true);
@@ -725,6 +728,28 @@ describe("connect step", () => {
     });
   });
 
+  it.each(["typed", "unattended", "token", "oss", "saved"])(
+    "never asks about OpenTelemetry or turns it on (%s setup): it is opt-in through `failproofai otel`",
+    async mode => {
+      const enable = vi.spyOn(otelCli, "runOtelCommand");
+      if (mode === "saved") {
+        writeIngestCredential({ url: "https://app.befailproof.ai/v1/events", key: "saved-key-3f2a" });
+        drive({ cloud: { kind: "saved", verified: true } });
+      } else if (mode === "typed") {
+        drive({ cloud: { kind: "typed", key: "a-real-looking-key" }, agents: ["claude"] });
+      }
+      try {
+        const result = await runConfigureWizard(mode === "unattended" ? headlessIO() : ttyIO(),
+          mode === "token" ? { token: "a-real-looking-key" } : mode === "oss" ? { oss: true } : {});
+        expect(result.applied).toBe(true);
+        expect(selectOne).not.toHaveBeenCalled();
+        expect(enable).not.toHaveBeenCalled();
+      } finally {
+        enable.mockRestore();
+      }
+    },
+  );
+
   it("checks a typed key without writing it, and reports a refusal", async () => {
     vi.mocked(validateIngestKey).mockResolvedValue({ ok: false, reason: "the server rejected that key (401)" });
     await runConfigureWizard(ttyIO());

__tests__/hooks/daemon-download.test.ts

@@ -406,7 +406,8 @@ describe("hooks/daemon-download", () => {
     });
 
     it("ensureFailproofaidBinary prefers the package and never touches the network", async () => {
-      installPlatformPackage("linux-x64");
+      // This entry point resolves the real host, unlike the explicit-key helpers.
+      installPlatformPackage(`${process.platform}-${process.arch}`);
       // Any fetch at all fails this test: a machine that already has the
       // binary from npm must not wait on github.com to install it.
       const server = await startServer();
@@ -430,7 +431,7 @@ describe("hooks/daemon-download", () => {
     it("works on an air-gapped machine, where the download channel is switched off", async () => {
       // FAILPROOFAI_NO_DOWNLOAD gates fetching, not copying — on exactly these
       // machines npm is the only channel that can supply a daemon at all.
-      installPlatformPackage("linux-x64");
+      installPlatformPackage(`${process.platform}-${process.arch}`);
       process.env.FAILPROOFAI_NO_DOWNLOAD = "1";
       process.env.FAILPROOFAI_DAEMON_BASE_URL = "http://127.0.0.1:1/never";
 

__tests__/hooks/fp-home.test.ts

@@ -228,6 +228,7 @@ describe("HOME_CLASSES", () => {
     fpcliDir: "fpcliDir",
     packsInstalledFile: "packsDir",
     packArtifactsDir: "packsDir",
+    otelAgentStateFile: "otelAgentStateDir",
   };
 
   /** Every exported function that returns a path inside the home. */

__tests__/hooks/otel-output.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from "vitest";
+import { OTEL_ORG_OFF } from "../../src/hooks/otel-cli";
+import { otelReportLines } from "../../src/hooks/otel-output";
+
+describe("OTEL screen language", () => {
+  it("uses state dots for status, including external and managed agents", () => {
+    expect(otelReportLines("status", { exitCode: 0, lines: [
+      "Org OTEL ingest: on",
+      "Claude Code: on by failproofai · destination: Cloud",
+      "Codex: on, not managed · destination: Cloud",
+      "Gemini CLI: off · destination: none · org: on",
+      "Copilot (VS Code): unreadable settings · destination: none",
+      "Relay: listening on 127.0.0.1 · last upload: none",
+    ] })).toEqual([
+      "● Org OTEL ingest: on",
+      "● Claude Code: on by failproofai · destination: Cloud",
+      "● Codex: on, not managed · destination: Cloud",
+      "○ Gemini CLI: off · destination: none · org: on",
+      "▲ Copilot (VS Code): unreadable settings · destination: none",
+      "● Relay: listening on 127.0.0.1 · last upload: none",
+    ]);
+  });
+
+  it("marks enable/disable side effects and keeps details underneath", () => {
+    expect(otelReportLines("enable", { exitCode: 0, lines: [
+      "Claude Code OTEL enabled → Cloud.",
+      "The daemon picks this up within seconds.",
+      OTEL_ORG_OFF,
+    ] })).toEqual([
+      "✓ Claude Code OTEL enabled → Cloud.",
+      "  The daemon picks this up within seconds.",
+      "▲ OTEL ingest is off for your org.",
+      "  An admin can turn it on in Settings → OpenTelemetry.",
+    ]);
+    expect(otelReportLines("disable", { exitCode: 0, lines: ["Codex OTEL disabled; previous settings restored."] })[0]).toMatch(/^✓ /);
+  });
+
+  it("gives a skipped Copilot a fix line", () => {
+    expect(otelReportLines("enable", { exitCode: 0, lines: ["VS Code not found — skipping copilot"] })).toEqual([
+      "▲ VS Code not found — skipping copilot",
+      "  Install VS Code or its Copilot extension, then run `failproofai otel enable copilot`.",
+    ]);
+  });
+
+  it("leaves shell output and failures to their existing renderers", () => {
+    const lines = ["export OTEL_SERVICE_NAME='my-agent'"];
+    expect(otelReportLines("env", { exitCode: 0, lines })).toBe(lines);
+    const failure = ["No Cloud API key configured. Run `failproofai config` first."];
+    expect(otelReportLines("enable", { exitCode: 1, lines: failure })).toBe(failure);
+  });
+});

__tests__/hooks/otel-setup-v2.test.ts

@@ -0,0 +1,16 @@
+import { describe, expect, it } from "vitest";
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+describe("OTEL collector restack contract", () => {
+  it("keeps raw OTLP tasks outside the selected transcript lane", () => {
+    const main = readFileSync(resolve(__dirname, "../../crates/failproofaid/src/main.rs"), "utf8");
+    const selected = main.indexOf("selection: &cfg.agents");
+    const transcriptFilter = main.indexOf("tasks.retain(|task|");
+    const otlp = main.indexOf("tasks.extend(otlp_tasks)");
+    expect(selected).toBeGreaterThan(0);
+    expect(transcriptFilter).toBeGreaterThan(selected);
+    expect(otlp).toBeGreaterThan(transcriptFilter);
+    expect(main.slice(transcriptFilter, otlp)).toContain("cfg.settings.sessions_for(source)");
+  });
+});