diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c66e8949..ccbf33f7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ on: # it produced was the daemon cross-compile, and only because it touched # `crates/`. A PR that cannot go red is not a reviewed PR. pull_request: - branches: [main, feat/fp-cli] + branches: [main, feat/fp-cli, feat/tui-ux] concurrency: group: ci-${{ github.ref }} diff --git a/.github/workflows/osv-scanner.yml b/.github/workflows/osv-scanner.yml index 0c9c44f3f..8ef6dd0b5 100644 --- a/.github/workflows/osv-scanner.yml +++ b/.github/workflows/osv-scanner.yml @@ -27,7 +27,7 @@ name: Supply Chain on: pull_request: - branches: [main] + branches: [main, feat/tui-ux] push: branches: [main] schedule: diff --git a/CHANGELOG.md b/CHANGELOG.md index 684947b7d..53dee9b10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ ### Features +- Add opt-in coding-agent OpenTelemetry setup, reversible per-agent transcript switching, and a loopback OTLP relay (#880). +- Add optional TypeScript and Python OpenTelemetry span exporters with durable OTLP/JSON spool delivery and no hard runtime dependencies (#880). +- Port OpenTelemetry help and setup into the new TUI, retaining per-agent transcript overrides and reversible settings. OpenTelemetry stays out of `failproofai config`: it is opt-in, per agent, through `failproofai otel enable ` (#880). - New terminal look for the CLI, step one: the palette gains the redesign's label grey, progress-track grey and a red for failures. Descriptions keep the terminal's own colour, and the greys fall back to the dim attribute below 24-bit, so every screen stays readable on light themes (#881) - Pickers in the new look: an UPPERCASE heading with a live "n of m selected", a `›` cursor and `■ □` boxes, twelve rows before scrolling, plain `a` to toggle all, and the key hints at the foot. The `policies -i` agent menu now uses the shared picker (#881) - Every `--help` page in the new look, cut to usage, options and examples, with every real flag and its real default. The index opens with three getting-started steps, then lists every command by what it is for. `policies add|remove|show --help` now prints the one policies page (#881) @@ -29,6 +32,8 @@ ### Fixes +- Preserve Codex OTEL header comments, omit local Claude auth-header settings, and require an executable or Copilot extension before configuring VS Code (#880). +- Keep Python OTEL imports lazy, label direct loopback destinations correctly, omit local Codex auth headers, and scope help options to each verb (#880). - `failproofai policies` showed nothing for a machine still on the pre-pack builtins, while those builtins were enforcing. It now says how many under BUILT IN, with the command that moves them into a pack (#881) - `failproofai config --disconnect` told you to restart the daemon to stop it sending. It stops on its own within a few seconds, because it polls the credential, and now says so (#881) - A config change that landed while the daemon was restarting its collector after a backfill was recorded as already running and never applied. `failproofai config` re-adding an agent hits exactly that, writing its backfill request and then the selection, so the agent stayed off until some unrelated edit (#881) @@ -50,6 +55,7 @@ ### Dependencies +- Update vulnerable existing build and SDK test lockfile dependencies reported by the PR's OSV scan (#880). - Bump `next` to 16.3.8, `sharp` to 0.35.5 and `source-map-js` to 1.2.2, which OSV-Scanner flagged; the dashboard ships `next` and `sharp`. In the SDKs' dev lockfiles: `source-map-js` (TypeScript) and `banks`, `langgraph-sdk`, `multidict` and `pydantic-ai-slim` (Python) (#881) ## 1.0.10 — 2026-10-05 diff --git a/Cargo.lock b/Cargo.lock index 943a3344c..fd3a0981b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,6 +33,52 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "base64" version = "0.22.1" @@ -283,7 +329,9 @@ dependencies = [ name = "fpai-collect" version = "1.0.11-beta.0" dependencies = [ + "axum", "getrandom 0.3.4", + "httpdate", "notify", "reqwest", "rusqlite", @@ -844,12 +892,24 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "mio" version = "1.2.2" diff --git a/__tests__/ci/otel-stacked-ci.test.ts b/__tests__/ci/otel-stacked-ci.test.ts new file mode 100644 index 000000000..c4979621e --- /dev/null +++ b/__tests__/ci/otel-stacked-ci.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { parse } from "yaml"; + +describe("OTEL stacked PR checks", () => { + it.each(["ci.yml", "osv-scanner.yml"])("%s checks the TUI base without adding feature pushes", file => { + const workflow = parse(readFileSync(resolve(__dirname, "../../.github/workflows", file), "utf8")); + expect(workflow.on.pull_request.branches).toContain("feat/tui-ux"); + expect(workflow.on.pull_request.branches).toContain("main"); + expect(workflow.on.push.branches).toEqual(["main"]); + }); +}); diff --git a/__tests__/dashboard/jev-activity-row.test.tsx b/__tests__/dashboard/jev-activity-row.test.tsx index 32c8f1892..dad9032f3 100644 --- a/__tests__/dashboard/jev-activity-row.test.tsx +++ b/__tests__/dashboard/jev-activity-row.test.tsx @@ -137,6 +137,7 @@ vi.mock("@/app/actions/update-policy-params", () => ({ })); import HooksClient from "@/app/policies/hooks-client"; +import { getHookActivityAction } from "@/app/actions/get-hook-activity"; /** The table row whose Tool cell reads `tool`. */ async function rowFor(tool: string): Promise { @@ -162,6 +163,11 @@ async function rowFor(tool: string): Promise { * it, so a blind retry would close the panel it had just opened. */ async function openRow(user: ReturnType, tool: string, marker: string) { + // The initial filter effect refetches and clears expandedRow after 300ms. + // Waiting only for a visible row can click BEFORE that effect, so a panel + // successfully opened by this helper closes just after the assertion. Wait + // for the initial refetch before interacting; keep all UI assertions intact. + await waitFor(() => expect(vi.mocked(getHookActivityAction).mock.calls.length).toBeGreaterThanOrEqual(2)); await waitFor(async () => { if (screen.queryByText(marker, { exact: false })) return; await user.click(await rowFor(tool)); @@ -172,6 +178,7 @@ async function openRow(user: ReturnType, tool: string, m describe("the activity tab with Jev rows", () => { beforeEach(() => { entries = []; + vi.mocked(getHookActivityAction).mockClear(); }); it("marks a Jev clear in the Decision cell and explains it in the detail panel", async () => { diff --git a/__tests__/e2e/cli/otel.e2e.test.ts b/__tests__/e2e/cli/otel.e2e.test.ts new file mode 100644 index 000000000..8cb4ca7f5 --- /dev/null +++ b/__tests__/e2e/cli/otel.e2e.test.ts @@ -0,0 +1,54 @@ +// @vitest-environment node +import { afterAll, describe, expect, it } from "vitest"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const root = mkdtempSync(join(tmpdir(), "fpai-otel-bundle-")); +const home = join(root, "home"); +const binary = join(root, "cli.mjs"); +const repo = resolve(__dirname, "../../.."); +// Like the installed tarball, external runtime deps sit next to the bundle. +symlinkSync(join(repo, "node_modules"), join(root, "node_modules"), "dir"); +mkdirSync(join(home, ".failproofai"), { recursive: true }); +mkdirSync(join(home, ".codex"), { recursive: true }); +writeFileSync(join(home, ".failproofai/credentials.json"), JSON.stringify({ + ingest: { url: "http://127.0.0.1:1/v1/events", key: "capture-test" }, +})); +const original = '# original\nmodel = "test"\n[projects."/work"]\ntrust_level = "trusted"\n'; +writeFileSync(join(home, ".codex/config.toml"), original); +const built = spawnSync("bun", ["build", "--target=node", "--format=esm", `--outfile=${binary}`, + "bin/failproofai.mjs", "--external", "posthog-node", "--external", "sql.js"], { + cwd: repo, encoding: "utf8", timeout: 30000, +}); +afterAll(() => rmSync(root, { recursive: true, force: true })); + +function cli(...args: string[]) { + return spawnSync("node", [binary, "otel", ...args], { + env: { ...process.env, HOME: home, USERPROFILE: home, FAILPROOFAI_HOME: join(home, ".failproofai"), + FAILPROOFAI_TELEMETRY_DISABLED: "1", FAILPROOFAI_INGEST_KEY: "", FAILPROOFAI_INGEST_URL: "" }, + encoding: "utf8", timeout: 15000, + }); +} + +describe("OTEL standalone bundled CLI", () => { + it("bundles JSONC and TOML without unresolved UMD relative imports", () => { + expect(built.status, built.stderr).toBe(0); + expect(existsSync(binary)).toBe(true); + expect(cli("enable", "--help").status).toBe(0); + }); + + it("enables and restores Codex from a Node-run standalone bundle against an older/unreachable server", () => { + const enabled = cli("enable", "codex", "--local", "--yes"); + expect(enabled.status, enabled.stderr + enabled.stdout).toBe(0); + expect(readFileSync(join(home, ".codex/config.toml"), "utf8")).toContain("/v1/logs"); + const status = cli("status"); + expect(status.status, status.stderr).toBe(0); + expect(status.stdout).toContain("unknown"); + expect(status.stdout).toContain("local relay"); + const disabled = cli("disable", "all"); + expect(disabled.status, disabled.stderr + disabled.stdout).toBe(0); + expect(readFileSync(join(home, ".codex/config.toml"), "utf8")).toBe(original); + }); +}); diff --git a/__tests__/hooks/configure-wizard.test.ts b/__tests__/hooks/configure-wizard.test.ts index dc86e19cb..ca78a4b71 100644 --- a/__tests__/hooks/configure-wizard.test.ts +++ b/__tests__/hooks/configure-wizard.test.ts @@ -93,6 +93,7 @@ vi.mock("../../src/hooks/integrations", async (importOriginal) => { import { multiSelect, + selectOne, promptCloudKey, OPEN_SOURCE, paint, @@ -105,6 +106,7 @@ import { type TTYOut, } from "../../src/hooks/tui"; import { connectToCloud } from "../../src/hooks/cloud-connection"; +import * as otelCli from "../../src/hooks/otel-cli"; import { validateIngestKey, writeIngestCredential, writeCollectorSettings } from "../../src/hooks/collector-config"; import { introspectKey } from "../../src/hooks/cloud-introspect"; import { installHooks, notEnforcingReason, enforcingPolicyCount } from "../../src/hooks/manager"; @@ -220,6 +222,7 @@ beforeEach(() => { .mockResolvedValue({ policy: { ok: true, policyCount: 2, deployment: 7 }, ingest: { ok: true }, anyConfigured: true }); vi.mocked(validateIngestKey).mockReset().mockResolvedValue({ ok: true }); vi.mocked(introspectKey).mockReset().mockResolvedValue({ kind: "unsupported" }); + vi.mocked(selectOne).mockReset().mockResolvedValue(false); // Healthy and already running is the default: the DAEMON step is one line // and needs no password. Daemon tests override these. vi.mocked(isDaemonSupportedPlatform).mockReset().mockReturnValue(true); @@ -725,6 +728,28 @@ describe("connect step", () => { }); }); + it.each(["typed", "unattended", "token", "oss", "saved"])( + "never asks about OpenTelemetry or turns it on (%s setup): it is opt-in through `failproofai otel`", + async mode => { + const enable = vi.spyOn(otelCli, "runOtelCommand"); + if (mode === "saved") { + writeIngestCredential({ url: "https://app.befailproof.ai/v1/events", key: "saved-key-3f2a" }); + drive({ cloud: { kind: "saved", verified: true } }); + } else if (mode === "typed") { + drive({ cloud: { kind: "typed", key: "a-real-looking-key" }, agents: ["claude"] }); + } + try { + const result = await runConfigureWizard(mode === "unattended" ? headlessIO() : ttyIO(), + mode === "token" ? { token: "a-real-looking-key" } : mode === "oss" ? { oss: true } : {}); + expect(result.applied).toBe(true); + expect(selectOne).not.toHaveBeenCalled(); + expect(enable).not.toHaveBeenCalled(); + } finally { + enable.mockRestore(); + } + }, + ); + it("checks a typed key without writing it, and reports a refusal", async () => { vi.mocked(validateIngestKey).mockResolvedValue({ ok: false, reason: "the server rejected that key (401)" }); await runConfigureWizard(ttyIO()); diff --git a/__tests__/hooks/daemon-download.test.ts b/__tests__/hooks/daemon-download.test.ts index 33f0a8c2f..c64e140cd 100644 --- a/__tests__/hooks/daemon-download.test.ts +++ b/__tests__/hooks/daemon-download.test.ts @@ -406,7 +406,8 @@ describe("hooks/daemon-download", () => { }); it("ensureFailproofaidBinary prefers the package and never touches the network", async () => { - installPlatformPackage("linux-x64"); + // This entry point resolves the real host, unlike the explicit-key helpers. + installPlatformPackage(`${process.platform}-${process.arch}`); // Any fetch at all fails this test: a machine that already has the // binary from npm must not wait on github.com to install it. const server = await startServer(); @@ -430,7 +431,7 @@ describe("hooks/daemon-download", () => { it("works on an air-gapped machine, where the download channel is switched off", async () => { // FAILPROOFAI_NO_DOWNLOAD gates fetching, not copying — on exactly these // machines npm is the only channel that can supply a daemon at all. - installPlatformPackage("linux-x64"); + installPlatformPackage(`${process.platform}-${process.arch}`); process.env.FAILPROOFAI_NO_DOWNLOAD = "1"; process.env.FAILPROOFAI_DAEMON_BASE_URL = "http://127.0.0.1:1/never"; diff --git a/__tests__/hooks/fp-home.test.ts b/__tests__/hooks/fp-home.test.ts index e20a2fde1..617749b93 100644 --- a/__tests__/hooks/fp-home.test.ts +++ b/__tests__/hooks/fp-home.test.ts @@ -228,6 +228,7 @@ describe("HOME_CLASSES", () => { fpcliDir: "fpcliDir", packsInstalledFile: "packsDir", packArtifactsDir: "packsDir", + otelAgentStateFile: "otelAgentStateDir", }; /** Every exported function that returns a path inside the home. */ diff --git a/__tests__/hooks/otel-output.test.ts b/__tests__/hooks/otel-output.test.ts new file mode 100644 index 000000000..97d75abc9 --- /dev/null +++ b/__tests__/hooks/otel-output.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; +import { OTEL_ORG_OFF } from "../../src/hooks/otel-cli"; +import { otelReportLines } from "../../src/hooks/otel-output"; + +describe("OTEL screen language", () => { + it("uses state dots for status, including external and managed agents", () => { + expect(otelReportLines("status", { exitCode: 0, lines: [ + "Org OTEL ingest: on", + "Claude Code: on by failproofai · destination: Cloud", + "Codex: on, not managed · destination: Cloud", + "Gemini CLI: off · destination: none · org: on", + "Copilot (VS Code): unreadable settings · destination: none", + "Relay: listening on 127.0.0.1 · last upload: none", + ] })).toEqual([ + "● Org OTEL ingest: on", + "● Claude Code: on by failproofai · destination: Cloud", + "● Codex: on, not managed · destination: Cloud", + "○ Gemini CLI: off · destination: none · org: on", + "▲ Copilot (VS Code): unreadable settings · destination: none", + "● Relay: listening on 127.0.0.1 · last upload: none", + ]); + }); + + it("marks enable/disable side effects and keeps details underneath", () => { + expect(otelReportLines("enable", { exitCode: 0, lines: [ + "Claude Code OTEL enabled → Cloud.", + "The daemon picks this up within seconds.", + OTEL_ORG_OFF, + ] })).toEqual([ + "✓ Claude Code OTEL enabled → Cloud.", + " The daemon picks this up within seconds.", + "▲ OTEL ingest is off for your org.", + " An admin can turn it on in Settings → OpenTelemetry.", + ]); + expect(otelReportLines("disable", { exitCode: 0, lines: ["Codex OTEL disabled; previous settings restored."] })[0]).toMatch(/^✓ /); + }); + + it("gives a skipped Copilot a fix line", () => { + expect(otelReportLines("enable", { exitCode: 0, lines: ["VS Code not found — skipping copilot"] })).toEqual([ + "▲ VS Code not found — skipping copilot", + " Install VS Code or its Copilot extension, then run `failproofai otel enable copilot`.", + ]); + }); + + it("leaves shell output and failures to their existing renderers", () => { + const lines = ["export OTEL_SERVICE_NAME='my-agent'"]; + expect(otelReportLines("env", { exitCode: 0, lines })).toBe(lines); + const failure = ["No Cloud API key configured. Run `failproofai config` first."]; + expect(otelReportLines("enable", { exitCode: 1, lines: failure })).toBe(failure); + }); +}); diff --git a/__tests__/hooks/otel-setup-v2.test.ts b/__tests__/hooks/otel-setup-v2.test.ts new file mode 100644 index 000000000..703c3be80 --- /dev/null +++ b/__tests__/hooks/otel-setup-v2.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +describe("OTEL collector restack contract", () => { + it("keeps raw OTLP tasks outside the selected transcript lane", () => { + const main = readFileSync(resolve(__dirname, "../../crates/failproofaid/src/main.rs"), "utf8"); + const selected = main.indexOf("selection: &cfg.agents"); + const transcriptFilter = main.indexOf("tasks.retain(|task|"); + const otlp = main.indexOf("tasks.extend(otlp_tasks)"); + expect(selected).toBeGreaterThan(0); + expect(transcriptFilter).toBeGreaterThan(selected); + expect(otlp).toBeGreaterThan(transcriptFilter); + expect(main.slice(transcriptFilter, otlp)).toContain("cfg.settings.sessions_for(source)"); + }); +}); diff --git a/__tests__/hooks/otel.test.ts b/__tests__/hooks/otel.test.ts new file mode 100644 index 000000000..0ce3a6101 --- /dev/null +++ b/__tests__/hooks/otel.test.ts @@ -0,0 +1,414 @@ +// @vitest-environment node +import { beforeEach, afterEach, describe, it, expect, vi } from "vitest"; +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, existsSync, rmSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import { spawnSync } from "node:child_process"; +import { runOtelCommand, OTEL_ORG_OFF } from "../../src/hooks/otel-cli"; +import { agentConfigPath, editAgentConfig, parseAgentConfig, vscodeSettingsPath, vscodeInstalled, vscodeProbe, agentOtelWrites } from "../../src/hooks/otel-writers"; +import { configFile, credentialsFile, otelAgentStateFile, otelAgentStateDir, otlpSpoolDir, resettablePaths } from "../../src/hooks/fp-home"; +import { readConfig, updateConfig } from "../../src/hooks/fp-config"; +import { agentSessionsEnabled } from "../../src/hooks/collector-config"; +import { introspectKey } from "../../src/hooks/cloud-introspect"; + +let home: string; +const fetchImpl = vi.fn(async () => new Response(JSON.stringify({ permissions: ["events:add"] }))); +const originalEnv = { ...process.env }; +const options = () => ({ home, fetchImpl }); +function seed(path: string, text: string): void { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, text); +} +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "fpai-otel-test-")); + process.env.FAILPROOFAI_HOME = join(home, ".failproofai"); + delete process.env.FAILPROOFAI_INGEST_KEY; + delete process.env.FAILPROOFAI_INGEST_URL; + delete process.env.XDG_CONFIG_HOME; + seed(credentialsFile(), JSON.stringify({ ingest: { url: "https://cloud.example/v1/events", key: "capture-test-secret" } })); + seed(configFile(), JSON.stringify({ mode: { kind: "cloud" }, collector: { sessions: false }, future: { keep: true } })); + fetchImpl.mockReset().mockImplementation(async () => new Response(JSON.stringify({ permissions: ["events:add"] }))); +}); +afterEach(() => { + for (const key of ["FAILPROOFAI_HOME", "FAILPROOFAI_INGEST_KEY", "FAILPROOFAI_INGEST_URL", "XDG_CONFIG_HOME"]) { + if (originalEnv[key] === undefined) delete process.env[key]; + else process.env[key] = originalEnv[key]; + } + rmSync(home, { recursive: true, force: true }); +}); + +describe("managed OTEL settings", () => { + it("migrations never discard restore credentials or undelivered OTLP batches", () => { + expect(resettablePaths()).not.toContain(otelAgentStateDir()); + expect(resettablePaths()).not.toContain(otlpSpoolDir()); + }); + it("round-trips Claude byte-equal, backs up once, preserves unknown keys and keeps keys out of config", async () => { + const path = agentConfigPath("claude", home); + const before = '{\n "env": {"EXISTING": "yes", "OTEL_EXPORTER_OTLP_HEADERS": "Authorization=old-secret"},\n "hooks": {"keep": true}\n}\n'; + seed(path, before); + expect((await runOtelCommand(["enable", "claude"], options())).exitCode).toBe(0); + const enabled = JSON.parse(readFileSync(path, "utf8")); + expect(enabled.env.CLAUDE_CODE_ENABLE_TELEMETRY).toBe("1"); + expect(enabled.env.OTEL_EXPORTER_OTLP_ENDPOINT).toBe("https://cloud.example"); + expect(enabled.env.OTEL_LOG_USER_PROMPTS).toBe("1"); + expect(enabled.env.EXISTING).toBe("yes"); + expect(enabled.hooks).toEqual({ keep: true }); + expect(readFileSync(`${path}.failproofai-bak`, "utf8")).toBe(before); + const raw = readFileSync(configFile(), "utf8"); + expect(raw).not.toContain("old-secret"); + expect(raw).not.toContain("capture-test-secret"); + expect(statSync(otelAgentStateFile("claude")).mode & 0o777).toBe(0o600); + expect((await runOtelCommand(["enable", "claude", "--no-content", "--local"], options())).exitCode).toBe(0); + expect(readFileSync(`${path}.failproofai-bak`, "utf8")).toBe(before); + expect((await runOtelCommand(["disable", "claude"], options())).exitCode).toBe(0); + expect(readFileSync(path, "utf8")).toBe(before); + expect(JSON.parse(readFileSync(configFile(), "utf8")).future).toEqual({ keep: true }); + expect(readConfig().collector.otlp?.enabled).toBe(false); + }); + + it("local Claude writes only documented content flags and never writes auth header variables", async () => { + const path = agentConfigPath("claude", home); + const before = '{"env":{"KEEP":"yes"}}\n'; + seed(path, before); + await runOtelCommand(["enable", "claude", "--local"], options()); + const env = JSON.parse(readFileSync(path, "utf8")).env; + expect(env.OTEL_LOG_ASSISTANT_RESPONSES).toBeUndefined(); + for (const key of ["OTEL_LOG_USER_PROMPTS", "OTEL_LOG_TOOL_DETAILS", "OTEL_LOG_TOOL_CONTENT"]) expect(env[key]).toBe("1"); + expect(Object.keys(env).some(key => key.endsWith("_HEADERS"))).toBe(false); + expect(agentOtelWrites("claude", "http://127.0.0.1:4318", "key", true, true) + .some(write => write.path.some(key => key.endsWith("_HEADERS")))).toBe(false); + await runOtelCommand(["disable", "claude"], options()); + expect(readFileSync(path, "utf8")).toBe(before); + }); + + it("Cloud-to-local Claude restores prior headers and stops owning them", async () => { + const path = agentConfigPath("claude", home); + const before = '{"env":{"OTEL_EXPORTER_OTLP_HEADERS":"Authorization=prior","OTEL_EXPORTER_OTLP_LOGS_HEADERS":"custom=prior"}}\n'; + seed(path, before); + await runOtelCommand(["enable", "claude"], options()); + await runOtelCommand(["enable", "claude", "--local"], options()); + expect(JSON.parse(readFileSync(path, "utf8")).env).toMatchObject({ + OTEL_EXPORTER_OTLP_HEADERS: "Authorization=prior", OTEL_EXPORTER_OTLP_LOGS_HEADERS: "custom=prior", + }); + const record = JSON.parse(readFileSync(configFile(), "utf8")).otel.agents.claude; + expect(record.keys.some((key: { path: string[] }) => key.path.some(p => p.endsWith("_HEADERS")))).toBe(false); + await runOtelCommand(["disable", "claude"], options()); + expect(readFileSync(path, "utf8")).toBe(before); + }); + + it("restores only owned keys after unrelated Claude changes", async () => { + const path = agentConfigPath("claude", home); + seed(path, '{"env":{"KEEP":"x","OTEL_LOG_USER_PROMPTS":"old"},"theme":"old"}'); + await runOtelCommand(["enable", "claude"], options()); + const changed = JSON.parse(readFileSync(path, "utf8")); + changed.theme = "new"; + changed.env.NEW_KEY = "keep"; + writeFileSync(path, JSON.stringify(changed)); + await runOtelCommand(["disable", "claude"], options()); + expect(JSON.parse(readFileSync(path, "utf8"))).toEqual({ + env: { KEEP: "x", OTEL_LOG_USER_PROMPTS: "old", NEW_KEY: "keep" }, theme: "new", + }); + }); + + it("re-enabling after unrelated edits preserves them on disable and status still recognizes owned keys", async () => { + const path = agentConfigPath("claude", home); + seed(path, '{"theme":"old"}\n'); + await runOtelCommand(["enable", "claude"], options()); + const changed = JSON.parse(readFileSync(path, "utf8")); + changed.theme = "new"; + writeFileSync(path, JSON.stringify(changed)); + expect((await runOtelCommand(["status"], options())).lines.join("\n")).toContain("Claude Code: on by failproofai"); + await runOtelCommand(["enable", "claude", "--no-content"], options()); + await runOtelCommand(["disable", "claude"], options()); + expect(JSON.parse(readFileSync(path, "utf8")).theme).toBe("new"); + }); + + it("Codex changes only OTEL table regions with comments, nested exporter tables and multiline strings", async () => { + const path = agentConfigPath("codex", home); + const prefix = '# user config\nmodel = "test"\nprompt = """\n[not_a_table]\n"""\n[projects."/work"] # trust\ntrust_level = "trusted"\n\n'; + const otel = '# OTEL\n[otel] # existing\nlog_user_prompt = false\nenvironment = "keep"\n[otel.exporter.otlp-http]\nendpoint = "http://old/v1/logs"\nprotocol = "json"\n'; + const suffix = '# introduces MCP table\n\n[mcp_servers.keep]\n# comment kept\ncommand = "keep"\n'; + const before = prefix + otel + suffix; + seed(path, before); + const result = await runOtelCommand(["enable", "codex"], options()); + expect(result.exitCode, result.lines.join("\n")).toBe(0); + const next = readFileSync(path, "utf8"); + expect(next.startsWith(prefix + "# OTEL\n")).toBe(true); + expect(next).toContain("[otel] # existing\n"); + expect(next.endsWith(suffix)).toBe(true); + const parsed = parseAgentConfig(next, "codex"); + expect(parsed.otel).toMatchObject({ + environment: "keep", + exporter: { "otlp-http": { endpoint: "https://cloud.example/v1/logs", headers: { Authorization: "Bearer capture-test-secret" } } }, + trace_exporter: { "otlp-http": { endpoint: "https://cloud.example/v1/traces" } }, + }); + await runOtelCommand(["disable", "codex"], options()); + expect(readFileSync(path, "utf8")).toBe(before); + }); + + it("Codex supports quoted OTEL headers, CRLF, no final newline; refuses root assignments and malformed files", () => { + const writes = agentOtelWrites("codex", "http://127.0.0.1:4318", "", true, false); + const before = 'model="x"\r\n["otel"]\r\nlog_user_prompt=true\r\n[other]\r\nvalue=1'; + const next = editAgentConfig(before, "codex", writes); + expect(next.startsWith('model="x"\r\n')).toBe(true); + expect(next.endsWith("[other]\r\nvalue=1")).toBe(true); + expect(() => editAgentConfig('otel = {log_user_prompt=true}\n', "codex", writes)).toThrow("root dotted"); + expect(() => editAgentConfig("[broken", "codex", writes)).toThrow(); + }); + + it("local Codex omits headers from all exporters and restores original config bytes", async () => { + const path = agentConfigPath("codex", home); + const before = '# kept\n[otel] # old config\nexporter = { otlp-http = { endpoint="https://old/v1/logs", protocol="json", headers={Authorization="Bearer old"} } }\n[other]\nkey="kept"\n'; + seed(path, before); + const enabled = await runOtelCommand(["enable", "codex", "--local"], options()); + expect(enabled.exitCode, enabled.lines.join("\n")).toBe(0); + const text = readFileSync(path, "utf8"); + const parsed = parseAgentConfig(text, "codex").otel as Record }>; + for (const name of ["exporter", "trace_exporter", "metrics_exporter"]) { + expect(parsed[name]["otlp-http"]).not.toHaveProperty("headers"); + expect(parsed[name]["otlp-http"].endpoint).toContain("http://127.0.0.1:4318/v1/"); + } + expect(text).not.toContain("headers"); + expect((await runOtelCommand(["disable", "codex"], options())).exitCode).toBe(0); + expect(readFileSync(path, "utf8")).toBe(before); + }); + + it.each(["gemini", "copilot"] as const)("writes %s with automatic auth-free relay and restores JSONC byte-equal", async agent => { + const path = agentConfigPath(agent, home); + const before = '{\n// user comment\n"editor.fontSize": 13,\n"telemetry": {"unknown": 42},\n}\n'; + seed(path, before); + if (agent === "copilot") mkdirSync(join(home, ".vscode/extensions/github.copilot-chat-0.55.0"), { recursive: true }); + const result = await runOtelCommand(["enable", agent, "--no-content"], options()); + expect(result.exitCode).toBe(0); + expect(result.lines.join("\n")).toContain("automatically"); + const text = readFileSync(path, "utf8"); + expect(text).toContain("// user comment"); + expect(text).not.toContain("capture-test-secret"); + expect(text).toContain("127.0.0.1:4318"); + const parsed = parseAgentConfig(text, agent); + expect(agent === "gemini" ? (parsed.telemetry as Record).logPrompts : parsed["github.copilot.chat.otel.captureContent"]).toBe(false); + expect(readConfig().collector.otlp?.enabled).toBe(true); + await runOtelCommand(["disable", agent], options()); + expect(readFileSync(path, "utf8")).toBe(before); + }); + + it("skips VS Code when absent and resolves macOS, Linux and Windows user paths", async () => { + const savedPath = process.env.PATH; + process.env.PATH = ""; + // System-wide installs (/usr/share/code/code…) would otherwise answer for a dev machine with VS Code. + const appPaths = vi.spyOn(vscodeProbe, "appPaths").mockReturnValue([]); + try { + mkdirSync(dirname(agentConfigPath("copilot", home)), { recursive: true }); + const before = readFileSync(configFile(), "utf8"); + expect((await runOtelCommand(["enable", "copilot"], options())).lines).toEqual(["VS Code not found — skipping copilot"]); + expect(readFileSync(configFile(), "utf8")).toBe(before); + expect(vscodeInstalled(home)).toBe(false); + } finally { process.env.PATH = savedPath; appPaths.mockRestore(); } + expect(existsSync(agentConfigPath("copilot", home))).toBe(false); + expect(vscodeSettingsPath(home, "darwin")).toContain("Library/Application Support/Code/User/settings.json"); + expect(vscodeSettingsPath(home, "linux")).toContain(".config/Code/User/settings.json"); + const before = process.env.APPDATA; + process.env.APPDATA = join(home, "Roaming"); + expect(vscodeSettingsPath(home, "win32")).toBe(join(home, "Roaming/Code/User/settings.json")); + if (before === undefined) delete process.env.APPDATA; else process.env.APPDATA = before; + }); + + it("detects an executable or Copilot extension, not an unrelated extension", () => { + const savedPath = process.env.PATH; + process.env.PATH = ""; + // System-wide installs (/usr/share/code/code…) would otherwise answer for a dev machine with VS Code. + const appPaths = vi.spyOn(vscodeProbe, "appPaths").mockReturnValue([]); + try { + mkdirSync(join(home, ".vscode/extensions/unrelated.extension-1.0"), { recursive: true }); + expect(vscodeInstalled(home)).toBe(false); + mkdirSync(join(home, ".vscode/extensions/github.copilot-1.0"), { recursive: true }); + expect(vscodeInstalled(home)).toBe(true); + rmSync(join(home, ".vscode"), { recursive: true }); + const bin = join(home, "bin"); + mkdirSync(bin); + writeFileSync(join(bin, "code-insiders"), "#!/bin/sh\n", { mode: 0o755 }); + process.env.PATH = bin; + expect(vscodeInstalled(home)).toBe(true); + } finally { process.env.PATH = savedPath; appPaths.mockRestore(); } + }); + + it("detects VS Code at a fixed install location with nothing on PATH", () => { + const savedPath = process.env.PATH; + process.env.PATH = ""; + const app = join(home, "opt/visual-studio-code/code"); + mkdirSync(dirname(app), { recursive: true }); + writeFileSync(app, "#!/bin/sh\n", { mode: 0o755 }); + const appPaths = vi.spyOn(vscodeProbe, "appPaths").mockReturnValue([join(home, "missing/code"), app]); + try { + expect(vscodeInstalled(home, "linux")).toBe(true); + expect(appPaths).toHaveBeenCalledWith(home, "linux"); + } finally { process.env.PATH = savedPath; appPaths.mockRestore(); } + expect(vscodeProbe.appPaths(home, "linux")).toEqual(["/usr/share/code/code", "/opt/visual-studio-code/code"]); + }); + + it("never overwrites an invalid config or creates a backup for it", async () => { + const path = agentConfigPath("claude", home); + seed(path, "{ invalid json"); + expect((await runOtelCommand(["enable", "claude"], options())).exitCode).toBe(1); + expect(readFileSync(path, "utf8")).toBe("{ invalid json"); + expect(existsSync(`${path}.failproofai-bak`)).toBe(false); + }); +}); + +describe("opt-in, one path and status", () => { + it("config v2 selection writes preserve legacy OTEL ownership, overrides and relay preferences", async () => { + const selection = { selected: ["claude", "codex"], seen: ["claude", "codex", "goose"] }; + updateConfig({ agents: selection, collector: { sessions: true, otlp: { enabled: false, port: 14318 } } }); + await runOtelCommand(["enable", "claude", "--local", "--yes"], options()); + const ownership = JSON.parse(readFileSync(configFile(), "utf8")).otel; + expect(readConfig().agents).toEqual(selection); + expect(agentSessionsEnabled("claude")).toBe(false); + expect(agentSessionsEnabled("codex")).toBe(true); + const rerun = { selected: ["claude"], seen: selection.seen }; + updateConfig({ agents: rerun, mode: "cloud" }); + expect(JSON.parse(readFileSync(configFile(), "utf8")).otel).toEqual(ownership); + expect(readConfig().collector.otlp).toEqual({ enabled: true, port: 14318 }); + expect(agentSessionsEnabled("claude")).toBe(false); + await runOtelCommand(["disable", "all"], options()); + expect(readConfig().agents).toEqual(rerun); + expect(readConfig().collector.agents?.claude).toBeUndefined(); + expect(readConfig().collector.otlp).toEqual({ enabled: false, port: 14318 }); + expect(agentSessionsEnabled("claude")).toBe(true); + }); + + it("one-path question defaults No; --yes sets only Claude override and disable restores absence", async () => { + updateConfig({ collector: { sessions: true } }); + const path = agentConfigPath("claude", home); + const question = vi.fn(async () => false); + await runOtelCommand(["enable", "claude"], { ...options(), confirm: question }); + expect(question).toHaveBeenCalledWith("failproofai already sends Claude Code sessions to Cloud. Send them with OTEL instead? Transcript upload for Claude Code stops."); + expect(existsSync(path)).toBe(false); + // Real non-TTY prompt has the conservative default, too. + await runOtelCommand(["enable", "claude"], options()); + expect(existsSync(path)).toBe(false); + await runOtelCommand(["enable", "claude", "--yes"], options()); + expect(agentSessionsEnabled("claude")).toBe(false); + expect(agentSessionsEnabled("codex")).toBe(true); + updateConfig({ telemetry: { enabled: false } }); + expect(agentSessionsEnabled("claude")).toBe(false); + await runOtelCommand(["disable", "claude"], options()); + expect(readConfig().collector.agents?.claude?.sessions).toBeUndefined(); + expect(agentSessionsEnabled("claude")).toBe(true); + expect(existsSync(path)).toBe(false); + }); + + it("Gemini has no transcript source and does not ask a misleading switch question", async () => { + updateConfig({ collector: { sessions: true } }); + const confirm = vi.fn(async () => false); + const result = await runOtelCommand(["enable", "gemini"], { ...options(), confirm }); + expect(result.exitCode).toBe(0); + expect(confirm).not.toHaveBeenCalled(); + expect(readConfig().collector.agents?.gemini).toBeUndefined(); + }); + + it("restores explicit sessions override and keeps relay on until last local agent disables", async () => { + updateConfig({ collector: { agents: { claude: { sessions: true } }, otlp: { enabled: false, port: 14318 } } }); + await runOtelCommand(["enable", "claude", "--yes", "--local"], options()); + await runOtelCommand(["enable", "gemini"], options()); + await runOtelCommand(["disable", "claude"], options()); + expect(readConfig().collector.agents?.claude?.sessions).toBe(true); + expect(readConfig().collector.otlp?.enabled).toBe(true); + await runOtelCommand(["disable", "gemini"], options()); + expect(readConfig().collector.otlp?.enabled).toBe(false); + }); + + it("status shows all agents, Cloud/local, relay, last upload and unknown for old introspect", async () => { + await runOtelCommand(["enable", "claude"], options()); + await runOtelCommand(["enable", "gemini"], options()); + const result = await runOtelCommand(["status"], options()); + const output = result.lines.join("\n"); + for (const agent of ["Claude Code", "Codex", "Gemini CLI", "Copilot"]) expect(output).toContain(agent); + expect(output).toContain("on by failproofai"); + expect(output).toContain("https://cloud.example"); + expect(output).toContain("http://127.0.0.1:4318"); + expect(output).toContain("org: unknown"); + expect(output).toContain("Relay:"); + expect(output).toContain("last upload: none"); + const identity = await introspectKey("https://cloud.example", "key", fetchImpl); + expect(identity.kind === "ok" && identity.identity.otelIngest).toBeUndefined(); + }); + + it("status uses recorded relay mode, not a loopback Cloud hostname", async () => { + process.env.FAILPROOFAI_INGEST_URL = "http://127.0.0.1:18765/v1/events"; + await runOtelCommand(["enable", "codex"], options()); + let line = (await runOtelCommand(["status"], options())).lines.find(line => line.startsWith("Codex:")); + expect(line).toContain("destination: Cloud (http://127.0.0.1:18765)"); + expect(line).not.toContain("local relay"); + await runOtelCommand(["enable", "codex", "--local"], options()); + line = (await runOtelCommand(["status"], options())).lines.find(line => line.startsWith("Codex:")); + expect(line).toContain("destination: local relay (http://127.0.0.1:4318)"); + }); + + it("reports unmanaged OTEL and warns but writes when org ingest is off", async () => { + seed(agentConfigPath("codex", home), '[otel]\nexporter = { "otlp-http" = { endpoint="http://external/v1/logs", protocol="json" } }\n'); + expect((await runOtelCommand(["status"], options())).lines.join("\n")).toContain("Codex: on, not managed"); + fetchImpl.mockImplementation(async () => new Response(JSON.stringify({ permissions: ["events:add"], otel_ingest: false }))); + const result = await runOtelCommand(["enable", "claude"], options()); + expect(result.exitCode).toBe(0); + expect(result.lines).toContain(OTEL_ORG_OFF); + expect((await runOtelCommand(["status"], options())).lines.join("\n")).toContain("org: off"); + }); + + it("missing key errors before writes; env is shell-safe and has the correct protocol/header", async () => { + const result = await runOtelCommand(["env", "--service", "service's name"], options()); + expect(result.lines.join("\n")).toContain("http/protobuf"); + expect(result.lines.join("\n")).toContain("Authorization=Bearer%20capture-test-secret"); + expect(result.lines.join("\n")).toContain("'service'\\''s name'"); + expect((await runOtelCommand(["env", "--local"], options())).lines.join("\n")).not.toContain("capture-test-secret"); + rmSync(credentialsFile()); + const error = await runOtelCommand(["enable", "claude"], options()); + expect(error.exitCode).toBe(1); + expect(error.lines.join("\n")).toContain("failproofai config"); + expect(existsSync(agentConfigPath("claude", home))).toBe(false); + }); + + it("is not part of `failproofai config`: setup never imports the OTEL command", () => { + const source = readFileSync(join(__dirname, "../../src/hooks/configure-wizard.ts"), "utf8"); + expect(source).not.toContain('from "./otel-cli"'); + expect(source).not.toContain('import("./otel-cli")'); + }); + + it("analytics properties contain only agent, local and content", async () => { + const onEvent = vi.fn(); + await runOtelCommand(["enable", "claude", "--local", "--no-content"], { ...options(), onEvent }); + await runOtelCommand(["disable", "claude"], { ...options(), onEvent }); + expect(onEvent.mock.calls).toEqual([ + ["otel_enabled", { agent: "claude", local: true, content: false }], + ["otel_disabled", { agent: "claude", local: true, content: false }], + ]); + }); + + it.each([{ args: [] }, { args: ["enable"] }, { args: ["disable"] }, { args: ["status"] }, { args: ["env"] }])("real CLI help $args follows config v2's screen style", ({ args }) => { + const result = spawnSync("bun", ["bin/failproofai.mjs", "otel", ...args, "--help"], { + cwd: join(__dirname, "../.."), env: { ...process.env, HOME: home, FAILPROOFAI_TELEMETRY_DISABLED: "1" }, + encoding: "utf8", timeout: 15000, + }); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("otel"); + expect(result.stdout).toContain("USAGE"); + expect(result.stdout).toContain("EXAMPLES"); + expect(result.stdout).not.toMatch(/NOTES|╭|╰/); + }); + + it.each([ + { verb: "status", flags: [] }, + { verb: "enable", flags: ["--local", "--no-content", "--yes"] }, + { verb: "disable", flags: [] }, + { verb: "env", flags: ["--service", "--local"] }, + ])("$verb help mentions only flags that verb accepts", ({ verb, flags }) => { + const result = spawnSync("bun", ["bin/failproofai.mjs", "otel", verb, "--help"], { + cwd: join(__dirname, "../.."), env: { ...process.env, HOME: home, FAILPROOFAI_TELEMETRY_DISABLED: "1" }, + encoding: "utf8", timeout: 15000, + }); + expect(result.status, result.stderr).toBe(0); + for (const flag of ["--local", "--no-content", "--yes", "--service"]) { + if (flags.includes(flag)) expect(result.stdout).toContain(flag); + else expect(result.stdout).not.toContain(flag); + } + }); +}); diff --git a/bin/failproofai.mjs b/bin/failproofai.mjs index 69a5f6f23..eee5e5c0d 100755 --- a/bin/failproofai.mjs +++ b/bin/failproofai.mjs @@ -453,7 +453,7 @@ async function runCli() { // at all, only the paragraph in the top-level dump that this rewrite moved. // `help` and `publish` are new. `policy` and `pack` are canonicalized to // `policies` above and never reach this list. - const SUBCOMMANDS = ["policies", "audit", "config", "uninstall", "backfill", "flush", "harness", "jev", "publish", "update", "migrate", "help"]; + const SUBCOMMANDS = ["policies", "audit", "config", "uninstall", "backfill", "flush", "harness", "otel", "jev", "publish", "update", "migrate", "help"]; // ── help ───────────────────────────────────────────────────────────────── // // The index and the reference manual used to be the same document: 152 lines, @@ -551,6 +551,7 @@ async function runCli() { ...section("Observe", [ ["(no args)", "Open the policy dashboard on localhost:8020"], ["audit", "Scan your agents' history for risky and wasteful patterns"], + ["otel", "Opt in to OpenTelemetry from agents and apps"], ]), ...section("Less often", [ ["harness", "Collect sessions from extra locations per agent"], @@ -874,6 +875,56 @@ async function runCli() { // could reach it to notice. The pack lane is entered from // `policies add|remove|show` below, which is the only door it has. + if (args[0] === "otel") { + const subArgs = args.slice(1); + if (!subArgs.length || subArgs.includes("--help") || subArgs.includes("-h")) { + const verb = ["status", "enable", "disable", "env"].includes(subArgs[0]) ? subArgs[0] : null; + const usages = { + status: "failproofai otel status", + enable: "failproofai otel enable [--local] [--no-content] [--yes]", + disable: "failproofai otel disable ", + env: "failproofai otel env [--service ] [--local]", + }; + const options = { + enable: [ + ["--local", "Use the loopback OTLP relay. Automatic for Gemini CLI and Copilot."], + ["--no-content", "Do not capture prompt text or tool details."], + ["--yes", "Accept switching this agent from transcript upload to OTEL."], + ], + env: [ + ["--service ", "Service name for env (default: my-agent)."], + ["--local", "Use the loopback OTLP relay."], + ], + status: [], + disable: [], + }; + const acceptedOptions = verb ? options[verb] : [...options.enable, options.env[0]]; + await printHelp({ + name: verb ? `otel ${verb}` : "otel", + usage: (verb ? [usages[verb]] : Object.values(usages)).map(usage => [ + usage, + verb === "enable" || verb === "disable" || !verb ? "Agents: claude, codex, gemini, copilot, all" : "", + ]), + options: acceptedOptions, + examples: verb === "status" ? ["failproofai otel status"] : + verb === "disable" ? ["failproofai otel disable all"] : + verb === "env" ? ["failproofai otel env --service my-agent --local"] : + ["failproofai otel enable claude", "failproofai otel enable codex --local --no-content", "failproofai otel disable all"], + }); + process.exit(0); + } + lastSubcommand = "otel"; + const { runOtelCommand } = await import("../src/hooks/otel-cli"); + const { otelReportLines } = await import("../src/hooks/otel-output"); + const result = await runOtelCommand(subArgs, { onEvent: track }); + // env is shell input and must not be wrapped in a styled report. + if (subArgs[0] === "env" && result.exitCode === 0) console.log(result.lines.join("\n")); + else await printReport(`otel ${subArgs[0]}`, otelReportLines(subArgs[0], result), { ok: result.exitCode === 0 }); + lastSubcommand = null; + await exitAfterFlush(result.exitCode); + return; + } + if (args[0] === "harness") { const subArgs = args.slice(1); if (subArgs.length === 0 || subArgs.includes("--help") || subArgs.includes("-h")) { diff --git a/bun.lock b/bun.lock index 9736f0395..ad51b469e 100644 --- a/bun.lock +++ b/bun.lock @@ -6,7 +6,9 @@ "name": "failproofai", "dependencies": { "html-to-image": "^1.11.13", + "jsonc-parser": "^3.3.1", "posthog-node": "^5.50.0", + "smol-toml": "^1.9.1", "sql.js": "^1.14.1", "yaml": "^2.9.0", }, @@ -833,6 +835,8 @@ "json5": ["json5@1.0.2", "", { "dependencies": { "minimist": "^1.2.0" }, "bin": { "json5": "lib/cli.js" } }, "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA=="], + "jsonc-parser": ["jsonc-parser@3.3.1", "", {}, "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ=="], + "jsx-ast-utils": ["jsx-ast-utils@3.3.5", "", { "dependencies": { "array-includes": "^3.1.6", "array.prototype.flat": "^1.3.1", "object.assign": "^4.1.4", "object.values": "^1.1.6" } }, "sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ=="], "keyv": ["keyv@5.6.0", "", { "dependencies": { "@keyv/serialize": "^1.1.1" } }, "sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw=="], @@ -1117,6 +1121,8 @@ "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], + "smol-toml": ["smol-toml@1.9.1", "", {}, "sha512-j7WyPdf0CJoutMzMoPLF/bVXHtKZIT9Rb4eGeEbWw8X2t5oirjPgpL+OpaENZrILYaXqYwnVH+qgUwVbl+pUJQ=="], + "source-map": ["source-map@0.7.6", "", {}, "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ=="], "source-map-js": ["source-map-js@1.2.2", "", {}, "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw=="], diff --git a/crates/failproofaid/src/main.rs b/crates/failproofaid/src/main.rs index a7b04fecc..c59aff6cd 100644 --- a/crates/failproofaid/src/main.rs +++ b/crates/failproofaid/src/main.rs @@ -874,6 +874,11 @@ fn collector_tasks() -> Vec { } }; + // Initialize the additive lane before announcing this deployment. TLS + // client construction can be slow; doing it after the startup milestone + // widens the config-snapshot race for a credential rotated at startup. + let otlp_tasks = fpai_collect::otlp::tasks(home.clone(), ingest.clone(), &cfg.settings.otlp); + eprintln!( "[failproofaid] collector enabled: sessions={} hooks={} ({:?}) -> {}", cfg.settings.sessions, cfg.settings.hooks, cfg.settings.hooks_verbosity, ingest.url, @@ -966,7 +971,13 @@ fn collector_tasks() -> Vec { })); } - if cfg.settings.sessions { + if cfg.settings.sessions + || cfg + .settings + .agents + .values() + .any(|a| a.sessions == Some(true)) + { // Session transcripts, gated on the `sessions` opt-in because — unlike // hook activity — these carry prompts, file contents and whatever was // pasted into a terminal. @@ -1413,6 +1424,14 @@ fn collector_tasks() -> Vec { } } + // Filter only transcript tasks; hook activity and delivery are untouched. + tasks.retain(|task| { + let source = task.name.split(':').next().unwrap_or(&task.name); + !HARNESS_KEYS.contains(&source) && source != "claude-subagent" + || cfg.settings.sessions_for(source) + }); + tasks.extend(otlp_tasks); + tasks.extend([ // Latency: delivers a batch within milliseconds of it being published. fpai_collect::TaskSpec::new("spool-watcher", move |sd| { @@ -2263,6 +2282,8 @@ mod tests { "hermes:team", "spool-watcher", "spool-sweeper", + // SDK OTEL batches drain even when the TCP relay is off. + "otlp-delivery", ] .iter() .map(|s| s.to_string()) @@ -2295,6 +2316,7 @@ mod tests { "codex:box", "health", "hook-activity", + "otlp-delivery", "spool-sweeper", "spool-watcher", ], @@ -2348,7 +2370,13 @@ mod tests { names.sort(); assert_eq!( names, - ["health", "hook-activity", "spool-sweeper", "spool-watcher"] + [ + "health", + "hook-activity", + "otlp-delivery", + "spool-sweeper", + "spool-watcher" + ] ); } @@ -2368,7 +2396,14 @@ mod tests { fn every_task_the_collector_builds_has_a_row_in_the_agent_table() { // A source missing from the table would be started on every machine // with no selection and on none with one — silently either way. - let infrastructure = ["health", "hook-activity", "spool-watcher", "spool-sweeper"]; + let infrastructure = [ + "health", + "hook-activity", + "spool-watcher", + "spool-sweeper", + "otlp-delivery", + "otlp-relay", + ]; for name in task_names(&config_with_agents(None), &["prod"]) { let source = name.split(':').next().unwrap(); assert!( diff --git a/crates/failproofaid/tests/agent_selection_e2e.rs b/crates/failproofaid/tests/agent_selection_e2e.rs index 327f89bde..2635fd743 100644 --- a/crates/failproofaid/tests/agent_selection_e2e.rs +++ b/crates/failproofaid/tests/agent_selection_e2e.rs @@ -300,7 +300,7 @@ fn with_no_agents_key_every_source_starts() { let s = Scratch::new("all"); s.config(&config(None, "")); let daemon = spawn_daemon(&s); - daemon.wait_for("collector started tasks=18", 1); + daemon.wait_for("collector started tasks=19", 1); for source in [ "claude", "claude-subagent", @@ -337,9 +337,10 @@ fn an_unselected_agent_and_its_extra_paths_never_start() { let daemon = spawn_daemon(&s); // health, hook-activity, claude and claude-subagent with one extra path - // each, two Hermes profile databases, and the two delivery tasks. Logged by - // the supervisor before any task runs, so it is the whole deployment. - daemon.wait_for("collector started tasks=10", 1); + // each, two Hermes profile databases, and the three delivery tasks (the + // additive SDK OTLP spool included). Logged by the supervisor before any + // task runs, so it is the whole deployment. + daemon.wait_for("collector started tasks=11", 1); daemon.wait_for(&file_source("claude", 0, 7), 2); daemon.wait_for(&file_source("claude-subagent", 0, 7), 2); daemon.wait_for("sqlite source started source=\"hermes\"", 2); @@ -370,7 +371,7 @@ fn an_unknown_selected_name_is_warned_about_on_every_build_even_without_sessions let warning = "[failproofaid] agents.selected names claud, which has no collector source; \ nothing is collected for it. Known: claude, codex, copilot, openclaw, pi, \ factory, antigravity, cursor, goose, opencode, devin, hermes"; - daemon.wait_for("collector started tasks=4", 1); + daemon.wait_for("collector started tasks=5", 1); daemon.wait_for(warning, 1); // Once per build: a cycle builds again and says it again. @@ -392,7 +393,7 @@ fn changing_the_selection_mid_run_cycles_the_right_sources() { "", )); let daemon = spawn_daemon(&s); - daemon.wait_for("collector started tasks=6", 1); + daemon.wait_for("collector started tasks=7", 1); daemon.wait_for(&file_source("claude", 0, 7), 1); daemon.wait_for(&file_source("claude-subagent", 0, 7), 1); daemon.assert_absent("source=\"codex\""); @@ -404,7 +405,7 @@ fn changing_the_selection_mid_run_cycles_the_right_sources() { "", )); daemon.wait_for("collector configuration changed; cycling the collector", 1); - daemon.wait_for("collector started tasks=5", 1); + daemon.wait_for("collector started tasks=6", 1); daemon.wait_for(&file_source("codex", 0, 7), 1); // The second deployment did not start claude again. assert_eq!( @@ -557,7 +558,7 @@ fn an_agent_re_added_on_a_running_daemon_never_resumes_its_stale_cursors() { s.cursors("codex", &[&stale]); let daemon = spawn_daemon(&s); - daemon.wait_for("collector started tasks=6", 1); + daemon.wait_for("collector started tasks=7", 1); let now = SystemTime::now(); s.request(&format!( diff --git a/crates/failproofaid/tests/otlp_relay_e2e.rs b/crates/failproofaid/tests/otlp_relay_e2e.rs new file mode 100644 index 000000000..076fbaf06 --- /dev/null +++ b/crates/failproofaid/tests/otlp_relay_e2e.rs @@ -0,0 +1,197 @@ +//! A relay bind failure and live enable/disable must never share fate with the +//! daemon's enforcement socket. Real binary, loopback HTTP and Unix Ping. +use std::io::{BufRead, Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use fpai_ipc::{ClientMessage, PROTOCOL_VERSION, ServerMessage, read_message, write_message}; + +struct Daemon { + child: Child, + home: PathBuf, + log: Arc>, +} +impl Drop for Daemon { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + let _ = std::fs::remove_dir_all(&self.home); + } +} +fn settings(home: &Path, port: u16, enabled: bool) { + let body = serde_json::json!({ + "mode": {"kind":"cloud"}, "collector": { + "sessions": true, "agents": {"claude":{"sessions":false},"codex":{"sessions":false}}, + "hooks": false, "otlp": {"enabled":enabled,"port":port} + }, "telemetry":{"enabled":false}, "audit":{"auto":false} + }); + std::fs::write(home.join("config.json"), body.to_string()).unwrap(); +} +fn start(port: u16, enabled: bool) -> Daemon { + let home = std::env::temp_dir().join(format!( + "fpai-otlp-daemon-{}", + fpai_collect::uploader::new_request_id() + )); + std::fs::create_dir_all(&home).unwrap(); + settings(&home, port, enabled); + std::fs::write( + home.join("credentials.json"), + r#"{"ingest":{"url":"http://127.0.0.1:59999/v1/events","key":"test"}}"#, + ) + .unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_failproofaid")) + .env("HOME", &home) + .env("FAILPROOFAI_HOME", &home) + .env( + "FAILPROOFAI_DAEMON_SOCKET", + home.join("run/failproofaid.sock"), + ) + .env("FAILPROOFAI_TELEMETRY_DISABLED", "1") + .env("FAILPROOFAI_COLLECTOR_CONFIG_POLL_MS", "100") + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let log = Arc::new(Mutex::new(String::new())); + let sink = log.clone(); + let pipe = child.stderr.take().unwrap(); + std::thread::spawn(move || { + for line in std::io::BufReader::new(pipe).lines().map_while(Result::ok) { + let mut log = sink.lock().unwrap(); + log.push_str(&line); + log.push('\n'); + } + }); + Daemon { child, home, log } +} +fn wait(predicate: impl Fn() -> bool) { + let start = Instant::now(); + while !predicate() { + assert!(start.elapsed() < Duration::from_secs(20), "timed out"); + std::thread::sleep(Duration::from_millis(50)); + } +} +fn ping(daemon: &Daemon) { + let mut stream = UnixStream::connect(daemon.home.join("run/failproofaid.sock")).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + write_message( + &mut stream, + &ClientMessage::Ping { + protocol_version: PROTOCOL_VERSION, + }, + ) + .unwrap(); + let response: ServerMessage = read_message(&mut stream).unwrap(); + assert!(matches!(response, ServerMessage::Pong { .. })); +} + +#[test] +fn busy_port_logs_once_and_daemon_keeps_answering() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let mut daemon = start(port, true); + wait(|| daemon.log.lock().unwrap().contains("OTLP relay off:")); + ping(&daemon); + std::thread::sleep(Duration::from_millis(500)); + assert!(daemon.child.try_wait().unwrap().is_none()); + assert_eq!( + daemon + .log + .lock() + .unwrap() + .matches("OTLP relay off:") + .count(), + 1 + ); + let health = std::fs::read_to_string(daemon.home.join("state/otlp-health.json")).unwrap(); + assert!(health.contains("port_busy")); + // Both default and subagent Claude tasks must be absent. + let log = daemon.log.lock().unwrap(); + assert!(!log.contains("source=\"claude\"")); + assert!(!log.contains("source=\"claude-subagent\"")); + assert!(!log.contains("source=\"codex\"")); +} + +#[test] +fn relay_starts_and_stops_on_config_reload_without_daemon_restart() { + let reserved = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = reserved.local_addr().unwrap().port(); + drop(reserved); + let daemon = start(port, false); + wait(|| daemon.home.join("run/failproofaid.sock").exists()); + // The socket is bound before the collector manager snapshots its config. + // Wait for the first deployment before testing a subsequent hand edit. + wait(|| daemon.log.lock().unwrap().contains("collector started")); + std::thread::sleep(Duration::from_millis(250)); + ping(&daemon); + assert!(TcpStream::connect(("127.0.0.1", port)).is_err()); + settings(&daemon.home, port, true); + wait(|| TcpStream::connect(("127.0.0.1", port)).is_ok()); + let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + stream.write_all(b"POST /v1/logs HTTP/1.1\r\nHost: localhost\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}").unwrap(); + let mut response = String::new(); + stream.read_to_string(&mut response).unwrap(); + assert!(response.starts_with("HTTP/1.1 200")); + ping(&daemon); + settings(&daemon.home, port, false); + wait(|| TcpStream::connect(("127.0.0.1", port)).is_err()); + ping(&daemon); +} + +#[test] +fn sdk_spool_delivers_without_opening_a_tcp_listener() { + let capture = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let origin_port = capture.local_addr().unwrap().port(); + let reserved = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let relay_port = reserved.local_addr().unwrap().port(); + drop(reserved); + let daemon = start(relay_port, false); + std::fs::write( + daemon.home.join("credentials.json"), + format!( + r#"{{"ingest":{{"url":"http://127.0.0.1:{origin_port}/v1/events","key":"test"}}}}"# + ), + ) + .unwrap(); + let spool = daemon.home.join("state/spool-otlp"); + std::fs::create_dir_all(&spool).unwrap(); + // Exact shared SDK format: metadata line followed by ExportTraceServiceRequest JSON. + let batch = spool.join("otlp-sdk-contract.jsonl"); + std::fs::write(&batch, b"{\"path\":\"/v1/traces\",\"content_type\":\"application/json\",\"encoding\":null}\n{\"resourceSpans\":[]}").unwrap(); + capture.set_nonblocking(true).unwrap(); + let start = Instant::now(); + let mut stream = loop { + if let Ok((stream, _)) = capture.accept() { + break stream; + } + assert!( + start.elapsed() < Duration::from_secs(20), + "SDK batch never reached capture server" + ); + std::thread::sleep(Duration::from_millis(50)); + }; + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut request = vec![0; 4096]; + let read = stream.read(&mut request).unwrap(); + let request = String::from_utf8_lossy(&request[..read]).to_lowercase(); + assert!(request.starts_with("post /v1/traces ")); + assert!(request.contains("authorization: bearer test")); + assert!(request.contains("content-type: application/json")); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}") + .unwrap(); + wait(|| !batch.exists()); + assert!(TcpStream::connect(("127.0.0.1", relay_port)).is_err()); + ping(&daemon); +} diff --git a/crates/fpai-collect/Cargo.toml b/crates/fpai-collect/Cargo.toml index 6edb91135..1131036c2 100644 --- a/crates/fpai-collect/Cargo.toml +++ b/crates/fpai-collect/Cargo.toml @@ -13,7 +13,9 @@ serde_json = "1" # The collector is the only async part of the daemon. `failproofaid`'s socket # server stays blocking std threads on purpose (see supervisor.rs), so this # runtime is created here and never leaks into the enforcement path. -tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "fs", "sync", "io-util"] } +tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "fs", "sync", "io-util", "net"] } +axum = { version = "0.8", default-features = false, features = ["http1", "tokio"] } +httpdate = "1" tracing = "0.1" # `rustls-tls` rather than the default native-tls: the daemon cross-compiles to # four targets and pulling OpenSSL onto each of them is the kind of build diff --git a/crates/fpai-collect/src/config.rs b/crates/fpai-collect/src/config.rs index a08971c64..811a04e50 100644 --- a/crates/fpai-collect/src/config.rs +++ b/crates/fpai-collect/src/config.rs @@ -130,6 +130,12 @@ pub struct Settings { /// sending them. #[serde(default)] pub sessions: bool, + /// Absent per-agent override inherits the global sessions opt-in. + #[serde(default)] + pub agents: BTreeMap, + /// Raw OTLP relay is additive and disabled by default. + #[serde(default)] + pub otlp: OtlpSettings, /// Ship hook activity. Defaults to TRUE once ingest is configured — it /// carries decisions and tool names, never file contents, and it is the /// capability that justifies collecting from inside failproofai at all. @@ -175,6 +181,33 @@ pub struct SourceSettings { pub extra_paths: Vec, } +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct AgentSettings { + #[serde(default)] + pub sessions: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +pub struct OtlpSettings { + #[serde(default)] + pub enabled: bool, + #[serde(default = "default_otlp_port")] + pub port: u16, +} + +fn default_otlp_port() -> u16 { + 4318 +} + +impl Default for OtlpSettings { + fn default() -> Self { + Self { + enabled: false, + port: default_otlp_port(), + } + } +} + fn default_true() -> bool { true } @@ -187,6 +220,8 @@ impl Default for Settings { fn default() -> Self { Settings { sessions: false, + agents: BTreeMap::new(), + otlp: OtlpSettings::default(), hooks: true, hooks_verbosity: HooksVerbosity::default(), redact: Redact::default(), @@ -198,6 +233,18 @@ impl Default for Settings { } impl Settings { + pub fn sessions_for(&self, agent: &str) -> bool { + let agent = if agent == "claude-subagent" { + "claude" + } else { + agent + }; + self.agents + .get(agent) + .and_then(|a| a.sessions) + .unwrap_or(self.sessions) + } + /// Raw `extra_paths` entries for one source, env override winning whole. /// /// `FAILPROOFAI__EXTRA_PATHS`, comma-separated — the same diff --git a/crates/fpai-collect/src/lib.rs b/crates/fpai-collect/src/lib.rs index a6149f937..88787f7f2 100644 --- a/crates/fpai-collect/src/lib.rs +++ b/crates/fpai-collect/src/lib.rs @@ -17,6 +17,7 @@ pub mod delivery; pub mod extra_paths; pub mod filetail; pub mod health; +pub mod otlp; pub mod redact; pub mod sources; pub mod spool; diff --git a/crates/fpai-collect/src/otlp.rs b/crates/fpai-collect/src/otlp.rs new file mode 100644 index 000000000..7309debe5 --- /dev/null +++ b/crates/fpai-collect/src/otlp.rs @@ -0,0 +1,712 @@ +//! Opaque OTLP/HTTP relay. No translation, decompression or event-uploader reuse. +//! Each batch is a JSON metadata line followed by the exact request bytes. +//! SDK exporters publish the same format; only atomic .jsonl files are read. +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Write}; +use std::net::{Ipv4Addr, SocketAddr}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime}; + +use axum::Router; +use axum::body::{Body, to_bytes}; +use axum::extract::{Request, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::Response; +use axum::routing::post; +use serde::{Deserialize, Serialize}; + +use crate::config::{Ingest, OtlpSettings}; +use crate::supervisor::{Shutdown, TaskError, TaskSpec}; +use crate::uploader::{ParkedName, new_request_id}; + +pub const MAX_BODY_BYTES: usize = 8 * 1024 * 1024; +const PATHS: [&str; 3] = ["/v1/traces", "/v1/logs", "/v1/metrics"]; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Metadata { + pub path: String, + pub content_type: String, + pub encoding: Option, +} + +#[derive(Default, Serialize, Deserialize)] +struct RelayHealth { + relay: String, + last_upload: Option, +} + +struct Relay { + spool: PathBuf, + failed: PathBuf, + health_file: PathBuf, + health: Mutex, + client: reqwest::Client, + origin: String, + key: String, + receiving: tokio::sync::Semaphore, +} + +impl Relay { + fn new(home: &Path, ingest: &Ingest) -> Result { + let url = reqwest::Url::parse(&ingest.url).map_err(|e| TaskError(e.to_string()))?; + let origin = url.origin().ascii_serialization(); + let health_file = home.join("state/otlp-health.json"); + let mut health = fs::read(&health_file) + .ok() + .and_then(|b| serde_json::from_slice::(&b).ok()) + .unwrap_or_default(); + health.relay = "off".to_string(); + Ok(Self { + spool: home.join("state/spool-otlp"), + // Nested under failed, isolated from the NDJSON events sweeper. + failed: home.join("state/failed/otlp"), + health_file, + health: Mutex::new(health), + client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(30)) + .build() + .map_err(|e| TaskError(e.to_string()))?, + origin, + key: ingest.key.clone(), + receiving: tokio::sync::Semaphore::new(16), + }) + } + + fn report(&self, relay: Option<&str>, upload: Option) { + let Ok(mut health) = self.health.lock() else { + return; + }; + if let Some(relay) = relay { + health.relay = relay.to_string(); + } + if let Some(upload) = upload { + health.last_upload = Some(upload); + } + if let Ok(bytes) = serde_json::to_vec(&*health) { + let _ = atomic_write(&self.health_file, &bytes); + } + } + + async fn deliver(&self, path: &Path, sd: &Shutdown) { + let bytes = match tokio::fs::read(path).await { + Ok(bytes) => bytes, + Err(_) => return, + }; + let Some(split) = bytes.iter().position(|b| *b == b'\n') else { + self.park(path, None).await; + return; + }; + let metadata: Metadata = match serde_json::from_slice(&bytes[..split]) { + Ok(metadata) => metadata, + Err(_) => { + self.park(path, None).await; + return; + } + }; + if !PATHS.contains(&metadata.path.as_str()) || bytes.len() - split - 1 > MAX_BODY_BYTES { + self.park(path, Some(400)).await; + return; + } + let body = &bytes[split + 1..]; + let mut status = None; + for attempt in 0..5 { + if sd.is_set() { + return; + } + let mut request = self + .client + .post(format!("{}{}", self.origin, metadata.path)) + .bearer_auth(&self.key) + .header("Content-Type", &metadata.content_type) + .body(body.to_vec()); + if let Some(encoding) = &metadata.encoding { + request = request.header("Content-Encoding", encoding); + } + let response = request.send().await; + let mut delay = Duration::from_secs(1 << attempt); + match response { + Ok(response) => { + let code = response.status().as_u16(); + status = Some(code); + if code == 200 { + let content_type = response + .headers() + .get("content-type") + .and_then(|h| h.to_str().ok()) + .unwrap_or("") + .to_string(); + // Bound response size too; an ingest proxy need not be trusted. + let mut response = response; + let mut result = Vec::new(); + while let Ok(Some(chunk)) = response.chunk().await { + if result.len() + chunk.len() > 64 * 1024 { + break; + } + result.extend_from_slice(&chunk); + } + let rejected = rejected_count(&result, &content_type); + if rejected > 0 { + tracing::warn!(rejected, signal = %metadata.path, "OTLP partial_success delivered"); + } + self.report(None, Some(format!("200 delivered; rejected={rejected}"))); + let _ = tokio::fs::remove_file(path).await; + return; + } + self.report(None, Some(format!("{code} parked"))); + if !matches!(code, 429 | 502 | 503 | 504) { + break; + } + delay = retry_after(response.headers()).unwrap_or(delay); + } + Err(_) => self.report(None, Some("network error; retained".to_string())), + } + if attempt < 4 && !sd.sleep(delay).await { + return; + } + } + self.park(path, status).await; + } + + async fn park(&self, path: &Path, status: Option) { + if tokio::fs::create_dir_all(&self.failed).await.is_err() { + return; + } + let Some(name) = path.file_name().and_then(|s| s.to_str()) else { + return; + }; + let mut parked = ParkedName::parse(name); + parked.attempt += 1; + parked.client_status = status.filter(|c| (400..500).contains(c) && *c != 429); + parked.poison = parked.attempt >= 3; + let target = self.failed.join(parked.render()); + if tokio::fs::rename(path, target).await.is_ok() { + let _ = File::open(&self.failed).and_then(|f| f.sync_all()); + } + } +} + +fn retry_after(headers: &HeaderMap) -> Option { + let value = headers.get("retry-after")?.to_str().ok()?; + if let Ok(seconds) = value.parse::() { + return Some(Duration::from_secs(seconds)); + } + Some( + httpdate::parse_http_date(value) + .ok()? + .duration_since(SystemTime::now()) + .unwrap_or_default(), + ) +} + +fn varint(bytes: &[u8], cursor: &mut usize) -> Option { + let mut value = 0; + for shift in (0..64).step_by(7) { + let byte = *bytes.get(*cursor)?; + *cursor += 1; + value |= u64::from(byte & 127) << shift; + if byte & 128 == 0 { + return Some(value); + } + } + None +} + +/// Export response field 1 is partial_success; its field 1 is rejected count. +/// Only response metadata is read — payload translation belongs to Cloud. +fn rejected_count(bytes: &[u8], content_type: &str) -> u64 { + if content_type.contains("json") || bytes.first() == Some(&b'{') { + let Ok(value) = serde_json::from_slice::(bytes) else { + return 0; + }; + let partial = value + .get("partialSuccess") + .or_else(|| value.get("partial_success")); + return partial + .and_then(|p| { + [ + "rejectedSpans", + "rejectedLogRecords", + "rejectedDataPoints", + "rejected_spans", + "rejected_log_records", + "rejected_data_points", + ] + .iter() + .find_map(|key| { + p.get(key) + .and_then(|n| n.as_u64().or_else(|| n.as_str()?.parse().ok())) + }) + }) + .unwrap_or(0); + } + let mut cursor = 0; + if varint(bytes, &mut cursor) != Some(10) { + return 0; + } + let Some(length) = varint(bytes, &mut cursor) else { + return 0; + }; + let Some(end) = cursor.checked_add(length as usize) else { + return 0; + }; + let Some(partial) = bytes.get(cursor..end) else { + return 0; + }; + cursor = 0; + if varint(partial, &mut cursor) != Some(8) { + return 0; + } + varint(partial, &mut cursor).unwrap_or(0) +} + +fn atomic_write(path: &Path, bytes: &[u8]) -> io::Result<()> { + let dir = path + .parent() + .ok_or_else(|| io::Error::other("missing directory"))?; + fs::create_dir_all(dir)?; + let temp = dir.join(format!(".{}.tmp", new_request_id())); + let result = (|| { + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&temp)?; + file.write_all(bytes)?; + file.sync_all()?; + fs::rename(&temp, path)?; + File::open(dir)?.sync_all() + })(); + if result.is_err() { + let _ = fs::remove_file(temp); + } + result +} + +pub fn spool_request(spool: &Path, metadata: &Metadata, body: &[u8]) -> io::Result { + if !PATHS.contains(&metadata.path.as_str()) || body.len() > MAX_BODY_BYTES { + return Err(io::Error::other("invalid OTLP path or body size")); + } + let mut bytes = serde_json::to_vec(metadata)?; + bytes.push(b'\n'); + bytes.extend_from_slice(body); + let path = spool.join(format!("otlp-{}.jsonl", new_request_id())); + atomic_write(&path, &bytes)?; + Ok(path) +} + +async fn receive(State(relay): State>, request: Request) -> Response { + let Ok(_permit) = relay.receiving.try_acquire() else { + return Response::builder().status(503).body(Body::empty()).unwrap(); + }; + let path = request.uri().path().to_string(); + let content_type = request + .headers() + .get("content-type") + .and_then(|h| h.to_str().ok()) + .unwrap_or("application/x-protobuf") + .to_string(); + if !matches!( + content_type.split(';').next(), + Some("application/json" | "application/x-protobuf" | "application/protobuf") + ) { + return Response::builder().status(415).body(Body::empty()).unwrap(); + } + let encoding = request + .headers() + .get("content-encoding") + .and_then(|h| h.to_str().ok()) + .map(str::to_string); + let is_json = content_type.contains("json"); + let body = match tokio::time::timeout( + Duration::from_secs(30), + to_bytes(request.into_body(), MAX_BODY_BYTES), + ) + .await + { + Ok(Ok(body)) => body, + Ok(Err(_)) => { + return Response::builder() + .status(StatusCode::PAYLOAD_TOO_LARGE) + .body(Body::empty()) + .unwrap(); + } + Err(_) => { + return Response::builder() + .status(StatusCode::REQUEST_TIMEOUT) + .body(Body::empty()) + .unwrap(); + } + }; + let spool = relay.spool.clone(); + let metadata = Metadata { + path, + content_type, + encoding, + }; + match tokio::task::spawn_blocking(move || spool_request(&spool, &metadata, &body)).await { + Ok(Ok(_)) => Response::builder() + .status(200) + .header( + "Content-Type", + if is_json { + "application/json" + } else { + "application/x-protobuf" + }, + ) + .body(Body::from(if is_json { + b"{}".as_slice() + } else { + b"".as_slice() + })) + .unwrap(), + _ => Response::builder().status(503).body(Body::empty()).unwrap(), + } +} + +async fn listen(relay: Arc, port: u16, sd: Shutdown) -> Result<(), TaskError> { + // Binding an IPv4 address, never a configurable host, makes loopback-only + // structural: no config or environment variable can expose the listener. + let address = SocketAddr::from((Ipv4Addr::LOCALHOST, port)); + let listener = match tokio::net::TcpListener::bind(address).await { + Ok(listener) => listener, + Err(err) => { + tracing::warn!(port, %err, "OTLP relay off: loopback port unavailable; daemon continues"); + relay.report(Some("port_busy"), None); + return Ok(()); // Clean exit: supervisor does not restart/log repeatedly. + } + }; + relay.report(Some("listening"), None); + let app = Router::new() + .route("/v1/traces", post(receive)) + .route("/v1/logs", post(receive)) + .route("/v1/metrics", post(receive)) + .with_state(relay.clone()); + let result = axum::serve(listener, app) + .with_graceful_shutdown(async move { while sd.sleep(Duration::from_millis(100)).await {} }) + .await; + relay.report(Some("off"), None); + result.map_err(|e| TaskError(e.to_string())) +} + +async fn deliver_loop(relay: Arc, sd: Shutdown) -> Result<(), TaskError> { + let mut last_failed = tokio::time::Instant::now(); + while !sd.is_set() { + for directory in [&relay.spool, &relay.failed] { + if directory == &relay.failed && last_failed.elapsed() < Duration::from_secs(3600) { + continue; + } + if directory == &relay.failed { + last_failed = tokio::time::Instant::now(); + } + let Ok(mut entries) = tokio::fs::read_dir(directory).await else { + continue; + }; + let mut delivered = 0; + while let Ok(Some(entry)) = entries.next_entry().await { + let name = entry.file_name().to_string_lossy().to_string(); + if !name.starts_with("otlp-") || !name.ends_with(".jsonl") { + continue; + } + if entry.file_type().await.is_ok_and(|t| t.is_file()) { + relay.deliver(&entry.path(), &sd).await; + delivered += 1; + if delivered >= 32 || sd.is_set() { + break; + } + } + } + } + if !sd.sleep(Duration::from_millis(250)).await { + break; + } + } + Ok(()) +} + +pub fn tasks(home: PathBuf, ingest: Ingest, settings: &OtlpSettings) -> Vec { + let relay = match Relay::new(&home, &ingest) { + Ok(relay) => Arc::new(relay), + Err(err) => { + tracing::warn!(%err, "OTLP relay disabled"); + return Vec::new(); + } + }; + let delivery = relay.clone(); + let port = settings.port; + // SDK exporters opt in by publishing raw OTLP batches. Delivery needs no + // TCP listener; an untouched machine has no batches and sends nothing. + let mut tasks = vec![TaskSpec::new("otlp-delivery", move |sd| { + deliver_loop(delivery.clone(), sd) + })]; + if settings.enabled { + tasks.push(TaskSpec::new("otlp-relay", move |sd| { + listen(relay.clone(), port, sd) + })); + } + tasks +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicBool; + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{header, method, path}, + }; + + fn temp() -> PathBuf { + let dir = std::env::temp_dir().join(format!("fpai-otlp-{}", new_request_id())); + fs::create_dir_all(&dir).unwrap(); + dir + } + fn shutdown() -> Shutdown { + Shutdown::for_test(Arc::new(AtomicBool::new(false))) + } + fn batch(relay: &Relay, body: &[u8]) -> PathBuf { + spool_request( + &relay.spool, + &Metadata { + path: "/v1/logs".into(), + content_type: "application/json".into(), + encoding: Some("gzip".into()), + }, + body, + ) + .unwrap() + } + + #[tokio::test] + async fn durable_opaque_forward_and_partial_success() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/logs")) + .and(header("authorization", "Bearer test")) + .and(header("content-type", "application/json")) + .and(header("content-encoding", "gzip")) + .respond_with( + ResponseTemplate::new(200).set_body_json( + serde_json::json!({"partialSuccess":{"rejectedLogRecords":"2"}}), + ), + ) + .expect(1) + .mount(&server) + .await; + let home = temp(); + let relay = Relay::new( + &home, + &Ingest { + url: format!("{}/v1/events", server.uri()), + key: "test".into(), + }, + ) + .unwrap(); + let body = b"\x1f\x8b\x00opaque-gzip"; + let file = batch(&relay, body); + let stored = fs::read(&file).unwrap(); + assert!(stored.ends_with(body)); + relay.deliver(&file, &shutdown()).await; + assert!(!file.exists()); + assert_eq!(server.received_requests().await.unwrap()[0].body, body); + assert!( + fs::read_to_string(&relay.health_file) + .unwrap() + .contains("rejected=2") + ); + fs::remove_dir_all(home).unwrap(); + } + + #[tokio::test] + async fn retries_503_retry_after_then_delivers() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(503).insert_header("Retry-After", "1")) + .up_to_n_times(1) + .expect(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + let home = temp(); + let relay = Relay::new( + &home, + &Ingest { + url: server.uri(), + key: "test".into(), + }, + ) + .unwrap(); + let file = batch(&relay, b"{}"); + let start = tokio::time::Instant::now(); + relay.deliver(&file, &shutdown()).await; + assert!(start.elapsed() >= Duration::from_secs(1)); + assert!(!file.exists()); + fs::remove_dir_all(home).unwrap(); + } + + #[tokio::test] + async fn parks_400_and_404_without_retry_or_loss() { + for code in [400, 404] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(code)) + .expect(1) + .mount(&server) + .await; + let home = temp(); + let relay = Relay::new( + &home, + &Ingest { + url: server.uri(), + key: "test".into(), + }, + ) + .unwrap(); + let file = batch(&relay, b"unchanged"); + let original = fs::read(&file).unwrap(); + relay.deliver(&file, &shutdown()).await; + assert!(!file.exists()); + let parked = fs::read_dir(&relay.failed) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + assert!( + parked + .to_str() + .unwrap() + .contains(&format!(".a1.c{code}.jsonl")) + ); + assert_eq!(fs::read(&parked).unwrap(), original); + fs::remove_dir_all(home).unwrap(); + } + } + + #[tokio::test] + async fn busy_port_exits_cleanly_without_supervisor_failure() { + let listener = tokio::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let port = listener.local_addr().unwrap().port(); + let home = temp(); + let relay = Arc::new( + Relay::new( + &home, + &Ingest { + url: "http://127.0.0.1:1".into(), + key: "test".into(), + }, + ) + .unwrap(), + ); + assert!(listen(relay.clone(), port, shutdown()).await.is_ok()); + assert!( + fs::read_to_string(&relay.health_file) + .unwrap() + .contains("port_busy") + ); + fs::remove_dir_all(home).unwrap(); + } + + #[tokio::test] + async fn loopback_receiver_limits_and_acknowledges_only_durable_batches() { + let home = temp(); + let relay = Arc::new( + Relay::new( + &home, + &Ingest { + url: "http://127.0.0.1:1".into(), + key: "test".into(), + }, + ) + .unwrap(), + ); + let reserved = tokio::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let port = reserved.local_addr().unwrap().port(); + drop(reserved); + let flag = Arc::new(AtomicBool::new(false)); + let task = tokio::spawn(listen( + relay.clone(), + port, + Shutdown::for_test(flag.clone()), + )); + let client = reqwest::Client::new(); + let url = format!("http://127.0.0.1:{port}/v1/traces"); + let mut response = None; + for _ in 0..30 { + match client + .post(&url) + .header("Content-Type", "application/x-protobuf") + .body(vec![0, 255, 1]) + .send() + .await + { + Ok(r) => { + response = Some(r); + break; + } + Err(_) => tokio::time::sleep(Duration::from_millis(10)).await, + } + } + assert_eq!(response.unwrap().status(), 200); + let stored = fs::read_dir(&relay.spool) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + assert!(fs::read(stored).unwrap().ends_with(&[0, 255, 1])); + assert_eq!( + client + .post(url) + .body(vec![0; MAX_BODY_BYTES + 1]) + .send() + .await + .unwrap() + .status(), + 413 + ); + flag.store(true, std::sync::atomic::Ordering::Relaxed); + task.await.unwrap().unwrap(); + fs::remove_dir_all(home).unwrap(); + } + + #[test] + fn protobuf_partial_success_and_retry_dates() { + assert_eq!(rejected_count(&[10, 2, 8, 3], "application/x-protobuf"), 3); + assert_eq!(rejected_count(&[], "application/x-protobuf"), 0); + let mut headers = HeaderMap::new(); + headers.insert( + "retry-after", + httpdate::fmt_http_date(SystemTime::now() + Duration::from_secs(10)) + .parse() + .unwrap(), + ); + assert!(retry_after(&headers).unwrap() >= Duration::from_secs(8)); + } + + #[test] + fn per_agent_overrides_and_relay_are_opt_in() { + let settings: crate::config::Settings = serde_json::from_value(serde_json::json!({ + "sessions": true, "agents": { "claude": { "sessions": false }, "codex": {} } + })) + .unwrap(); + assert!(!settings.sessions_for("claude")); + assert!(!settings.sessions_for("claude-subagent")); + assert!(settings.sessions_for("codex")); + assert!(!settings.otlp.enabled); + assert_eq!(settings.otlp.port, 4318); + } +} diff --git a/crates/fpai-collect/tests/config.rs b/crates/fpai-collect/tests/config.rs index 984d4826e..3d779f736 100644 --- a/crates/fpai-collect/tests/config.rs +++ b/crates/fpai-collect/tests/config.rs @@ -50,6 +50,34 @@ fn absent_config_means_collection_is_off() { fs::remove_dir_all(&home).ok(); } +#[test] +fn config_v2_selection_and_legacy_otel_transport_settings_are_independent() { + let home = tmp_home("otel-v2"); + fs::write( + home.join("config.json"), + r#"{ + "agents": {"selected": ["claude", "codex"], "seen": ["claude", "codex"]}, + "collector": { + "sessions": true, + "agents": {"claude": {"sessions": false}}, + "otlp": {"enabled": true, "port": 14318} + }, + "otel": {"agents": {"claude": {"local": true}}} + }"#, + ) + .unwrap(); + let cfg = without_env_overrides(|| config::load(&home).unwrap()); + assert!(cfg.agents.traces("claude")); + assert!(cfg.agents.traces("codex")); + assert!(!cfg.agents.traces("goose")); + assert!(!cfg.settings.sessions_for("claude")); + assert!(!cfg.settings.sessions_for("claude-subagent")); + assert!(cfg.settings.sessions_for("codex")); + assert!(cfg.settings.otlp.enabled); + assert_eq!(cfg.settings.otlp.port, 14318); + fs::remove_dir_all(home).ok(); +} + #[test] fn a_key_alone_does_not_enable_session_collection() { // The privacy default, and the one most likely to be eroded by accident. diff --git a/docs/agenteye/opentelemetry.mdx b/docs/agenteye/opentelemetry.mdx new file mode 100644 index 000000000..d355f6d00 --- /dev/null +++ b/docs/agenteye/opentelemetry.mdx @@ -0,0 +1,132 @@ +--- +title: "OpenTelemetry" +description: "Send coding-agent and application telemetry to FailproofAI Cloud with OTLP." +icon: "radio" +--- + +OpenTelemetry lets you send agent traces, logs, and metrics to FailproofAI Cloud using the standard OTLP/HTTP protocol. It is optional. Your existing FailproofAI integrations continue working without enabling it. + +## Turn on ingest for your organisation + +An organisation admin opens **Settings → OpenTelemetry** and turns on ingest. Use a Cloud API key with `events:add` permission, the same permission used for agent reporting. + +If ingest is off, Cloud refuses OTLP exports. Your local setup can still be prepared; run `failproofai otel status` to check the organisation switch before starting an agent. + +## Connect a coding agent + +Connect your machine with `failproofai config`, then choose an agent: + +```bash +failproofai otel enable claude +failproofai otel enable codex +failproofai otel enable gemini +failproofai otel enable copilot +``` + +The setup command chooses Cloud or a local authenticated relay as appropriate. Gemini CLI and Copilot use the local relay automatically. Start your configured FailproofAI daemon before using those integrations. + +If FailproofAI already sends an agent's session transcripts, setup asks whether to switch that agent to OTEL. The default is No. Accepting stops transcript uploads for that agent; it does not change policy enforcement or reporting from other agents. + +## Connect an application + +Use your application's OpenTelemetry instrumentation with these environment variables: + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT=https://app.befailproof.ai +export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +export OTEL_EXPORTER_OTLP_HEADERS='Authorization=Bearer%20' +export OTEL_SERVICE_NAME=my-agent +``` + +Replace `` with your API key. Keep the secret out of shell history, screen recordings, and committed configuration. For a configured machine, `failproofai otel env --service my-agent` prints the block with its existing key. Its output contains a credential. + +Cloud accepts OTLP/HTTP exports for traces, logs, and metrics, as protobuf or JSON, including gzip-compressed requests. It does not accept OTLP/gRPC. + +Metrics are acknowledged for exporter compatibility but are not stored or displayed. Supported agent traces and coding-agent logs become sessions; fields the agent does not export remain absent. + +## Capture prompt text + + +Text capture can include prompts, tool arguments, and other sensitive content. `failproofai otel enable` turns it on. Add `--no-content` to keep prompt and tool-detail capture off. + + +```bash +failproofai otel enable claude --no-content +failproofai otel enable codex --no-content +``` + +For an agent you configure manually, use its own content-capture controls: + + + + Add these variables to your agent's environment alongside the application block above: + + ```bash + export CLAUDE_CODE_ENABLE_TELEMETRY=1 + export OTEL_LOGS_EXPORTER=otlp + export OTEL_METRICS_EXPORTER=otlp + export OTEL_LOG_USER_PROMPTS=1 + export OTEL_LOG_TOOL_DETAILS=1 + ``` + + Set both content variables to `0` to avoid capturing prompts and tool details. + + + In Codex's user configuration, use its OTEL settings: + + ```toml + [otel] + log_user_prompt = true + exporter = { otlp-http = { endpoint = "https://app.befailproof.ai/v1/logs", protocol = "json", headers = { Authorization = "Bearer " } } } + trace_exporter = { otlp-http = { endpoint = "https://app.befailproof.ai/v1/traces", protocol = "json", headers = { Authorization = "Bearer " } } } + metrics_exporter = { otlp-http = { endpoint = "https://app.befailproof.ai/v1/metrics", protocol = "json", headers = { Authorization = "Bearer " } } } + ``` + + Codex uses the complete endpoint you provide. Set `log_user_prompt` to `false` to leave prompts out. + + + Use Gemini's telemetry settings with the local relay: + + ```json + { + "telemetry": { + "enabled": true, + "target": "local", + "otlpEndpoint": "http://127.0.0.1:4318", + "otlpProtocol": "http", + "useCollector": true, + "logPrompts": true + } + } + ``` + + Run `failproofai otel enable gemini` to enable the authenticated relay. Set `logPrompts` to `false` for no prompt capture. + + + Use these VS Code user settings: + + ```json + { + "github.copilot.chat.otel.enabled": true, + "github.copilot.chat.otel.exporterType": "otlp-http", + "github.copilot.chat.otel.otlpEndpoint": "http://127.0.0.1:4318", + "github.copilot.chat.otel.captureContent": true + } + ``` + + Run `failproofai otel enable copilot` to enable the authenticated relay. Set `captureContent` to `false` to omit content. + + + +## Check or undo setup + +```bash +failproofai otel status +failproofai otel disable all +``` + +Status distinguishes settings managed by FailproofAI from telemetry you configured elsewhere. Disable restores the previous managed settings and the agent's previous transcript-upload choice. + + + Command options, local relay setup, and troubleshooting. + diff --git a/docs/docs.json b/docs/docs.json index ec8079b32..50ddf63e7 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -122,6 +122,7 @@ "group": "Plug in your agent", "pages": [ "start/integrations", + "agenteye/opentelemetry", "start/integrations/custom-agents", "start/integrations/langchain", "start/integrations/crewai", @@ -241,6 +242,7 @@ "expanded": false, "pages": [ "reference/failproof-cli", + "reference/otel", "reference/local-dashboard", "policies/local-configuration", "reference/cloud-cli", diff --git a/docs/reference/custom-agents-typescript.mdx b/docs/reference/custom-agents-typescript.mdx index 16e66eb26..0b226e6d2 100644 --- a/docs/reference/custom-agents-typescript.mdx +++ b/docs/reference/custom-agents-typescript.mdx @@ -43,6 +43,38 @@ Identical to the Python SDK: create an `events:add` key under **Admin → Keys** ## Configuration +### OpenTelemetry spans (opt in) + +Install OpenTelemetry separately with `npm install @opentelemetry/api @opentelemetry/sdk-trace-base`. +Then add FailproofAI to your provider's processors: + +```ts +import * as failproofai from "@failproofai/sdk"; +import { BasicTracerProvider } from "@opentelemetry/sdk-trace-base"; +import type { SpanProcessor } from "@opentelemetry/sdk-trace-base"; + +const spanProcessors: SpanProcessor[] = []; +await failproofai.instrument("otel", { spanProcessors }); +const provider = new BasicTracerProvider({ spanProcessors }); +const span = provider.getTracer("my-agent").startSpan("agent", { + attributes: { "gen_ai.operation.name": "invoke_agent" }, +}); +span.end(); +await provider.forceFlush(); +``` + +For an existing provider, keep its other processors in the same list. Providers +that expose `addSpanProcessor` can instead be passed as `{ provider }`. Without +options, setup registers a new provider only if no provider is already registered. +`OtelSpanExporter` can also be used directly with your own processor. + +Exports go through the connected FailproofAI daemon; the SDK sends no network +requests and needs no key or local listener. Ask an admin to enable +[OpenTelemetry ingest](/agenteye/opentelemetry). A bare `instrument()` does not +enable this exporter. `uninstrument("otel")` disables only its own processor. +Your span attributes control content capture; flush your provider before exit. +OpenTelemetry stays an optional application dependency, not a hard SDK dependency. + ```ts failproofai.configure({ environment: "production", diff --git a/docs/reference/custom-agents.mdx b/docs/reference/custom-agents.mdx index cb891e148..3881e461c 100644 --- a/docs/reference/custom-agents.mdx +++ b/docs/reference/custom-agents.mdx @@ -58,6 +58,35 @@ The package is installed as `failproofai-sdk` and imported in Python as `failpro ## Configuration +### OpenTelemetry spans (opt in) + +Install `opentelemetry-sdk` separately, then attach the optional exporter: + +```python +import failproofai_sdk +from opentelemetry.sdk.trace import TracerProvider + +provider = TracerProvider() +failproofai_sdk.instrument("otel", provider=provider) +with provider.get_tracer("my-agent").start_as_current_span( + "agent", attributes={"gen_ai.operation.name": "invoke_agent"} +): + pass +provider.force_flush() +``` + +Setup uses your registered provider if you omit `provider`, creating one only +when none is registered. You can also pass `failproofai_sdk.OtelSpanExporter()` +directly to your own span processor. Other exporters remain unchanged. + +The connected FailproofAI daemon delivers exports; the SDK needs no network +setup, key, or local listener. An admin must enable +[OpenTelemetry ingest](/agenteye/opentelemetry). A bare `instrument()` never +enables OTEL. `uninstrument("otel")` disables only its own processor. +Your instrumentation controls content capture. Flush the provider before exit. +The SDK still declares zero hard runtime dependencies and imports OpenTelemetry +only when requested. + ```python import failproofai_sdk diff --git a/docs/reference/failproof-cli.mdx b/docs/reference/failproof-cli.mdx index e9eabfd50..fc30731e0 100644 --- a/docs/reference/failproof-cli.mdx +++ b/docs/reference/failproof-cli.mdx @@ -58,6 +58,7 @@ Run `failproofai` without arguments to open the local policy dashboard. | `failproofai audit --status` | Show the report address, interval, and next scheduled scan | | `failproofai audit --no-schedule` | Stop recurring scans without deleting audit history | | `failproofai harness list` | List extra capture paths | +| `failproofai otel status` | Inspect optional [OpenTelemetry reporting](/reference/otel) | | `failproofai jev --url --key-stdin` | Set Jev up in one step; the provider is taken from the URL's host | | `failproofai jev setup --provider --key-stdin` | Let [Jev](/reference/jev-providers) judge tool calls through your own endpoint and key | | `failproofai jev setup --provider failproofai` | Let Jev judge tool calls [through FailproofAI Cloud](/reference/jev-cloud), with this machine's Cloud key | diff --git a/docs/reference/otel.mdx b/docs/reference/otel.mdx new file mode 100644 index 000000000..9b761425e --- /dev/null +++ b/docs/reference/otel.mdx @@ -0,0 +1,71 @@ +--- +title: "OpenTelemetry CLI" +description: "Opt in to OTEL for coding agents, inspect delivery, and restore your previous settings." +icon: "radio" +--- + +`failproofai otel` configures optional OpenTelemetry reporting to FailproofAI Cloud. First connect your machine with `failproofai config` and ask an organisation admin to enable **Settings → OpenTelemetry**. + +Interactive setup asks about OpenTelemetry after the Cloud key field and the agent picker. The optional step defaults to No. Choose Yes to pick OTEL agents; the same enable code runs after the connection is written. Esc before setup applies changes nothing. Unattended setup, a supplied key, open-source mode, and keeping a saved connection do not opt you in. + +## Enable an agent + +```bash +failproofai otel enable claude +failproofai otel enable codex --local +failproofai otel enable gemini --no-content +failproofai otel enable copilot +failproofai otel enable all --yes +``` + +The supported agents are `claude`, `codex`, `gemini`, and `copilot` (VS Code). Copilot setup requires a VS Code executable or an installed Copilot extension. A leftover settings directory alone is not enough; setup skips Copilot when neither is found. + +`--local` sends to the local relay instead of directly to Cloud. Gemini CLI and Copilot use the relay automatically because their settings cannot store the authentication header. + +The relay listens only on your machine. Your configured FailproofAI daemon picks up the change within seconds; you do not need to restart it. If another application owns the relay port, status reports it as off and the daemon keeps running. + + +Text capture is on by default. Add `--no-content` to omit prompts and tool details. Agent versions can expose different content controls; review what your agent exports before using it with sensitive data. + + +### Choose one reporting path + +If transcript reporting is on for an agent, setup asks: + +> failproofai already sends Claude Code sessions to Cloud. Send them with OTEL instead? Transcript upload for Claude Code stops. + +The default is No. `--yes` accepts the switch without a prompt. Other agents and policy enforcement are unaffected. + +Config v2 keeps the remembered tracing list at `agents.selected` (and detection history at `agents.seen`). The per-agent transcript preference remains at `collector.agents..sessions`, OTEL ownership at `otel.agents`, and relay settings at `collector.otlp.enabled` / `collector.otlp.port`. These separate keys keep old OTEL configurations readable: setup can change the tracing list without losing the transport preference or the state needed to restore your agent settings. + +## Inspect setup + +```bash +failproofai otel status +``` + +Status shows every agent, whether its settings are managed by FailproofAI, its destination, the organisation's ingest switch, relay state, and the last relay upload result. + +`unknown` for the organisation switch means the server did not provide it or could not be reached. It is not an error and does not prevent configuration on an older Cloud version. + +If Cloud does not support OTEL yet, the relay retains rejected exports instead of discarding them. If organisation ingest is off, ask an admin to turn it on before starting your agents. + +## Print configuration for an application + +```bash +failproofai otel env --service my-agent +failproofai otel env --service my-agent --local +``` + +The output is a shell environment block for an OTLP/HTTP exporter. Cloud output includes the configured API key; treat it as a secret. Local output omits the key. To use local output, first enable a local agent so the relay is running. + +## Restore previous settings + +```bash +failproofai otel disable claude +failproofai otel disable all +``` + +Disable restores only the settings FailproofAI wrote, keeps unrelated changes, and restores each agent's prior transcript-reporting choice. When the last managed local agent is disabled, the relay stops accepting exports. + +Every command supports `--help`. See [OpenTelemetry in Cloud](/agenteye/opentelemetry) for application and coding-agent examples. diff --git a/package.json b/package.json index 2c1a9bfa3..18cb4e938 100644 --- a/package.json +++ b/package.json @@ -107,7 +107,9 @@ }, "dependencies": { "html-to-image": "^1.11.13", + "jsonc-parser": "^3.3.1", "posthog-node": "^5.50.0", + "smol-toml": "^1.9.1", "sql.js": "^1.14.1", "yaml": "^2.9.0" }, diff --git a/sdk/python/README.md b/sdk/python/README.md index 3cb8adafc..5931ed9ab 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -74,6 +74,47 @@ standard library. See `skill/references/frameworks.md` for the per-framework mapping, and `docs/` for a per-framework integration guide with runnable examples beside it. +## OpenTelemetry (opt in) + +Already emitting OpenTelemetry spans? Install `opentelemetry-sdk` separately, +then attach FailproofAI's exporter without changing your existing adapters: + +```bash +pip install opentelemetry-sdk +``` + +```python +import failproofai_sdk +from opentelemetry.sdk.trace import TracerProvider + +provider = TracerProvider() +failproofai_sdk.instrument("otel", provider=provider) +tracer = provider.get_tracer("my-agent") +with tracer.start_as_current_span( + "agent", attributes={"gen_ai.operation.name": "invoke_agent"} +): + pass +provider.force_flush() +``` + +Without a `provider`, `instrument("otel")` uses the registered provider or creates +one if none exists. You can also pass `failproofai_sdk.OtelSpanExporter()` to your +own `SimpleSpanProcessor` or `BatchSpanProcessor`. `uninstrument("otel")` disables +only our processor and leaves your provider and other exporters unchanged. + +The exporter writes durable OTLP/JSON requests to +`$FAILPROOFAI_HOME/state/spool-otlp`, otherwise `~/.failproofai/state/spool-otlp`. +It never sends HTTP or reads a Cloud key. Connect the daemon with +`failproofai config` and ask an admin to enable Settings → OpenTelemetry. +SDK delivery needs no local TCP listener. `spool_dir` explicitly overrides the +destination for tests or managed deployments; `configure(base_dir=...)` continues +to control only the original event adapters. + +A bare `instrument()` never enables OTEL. OpenTelemetry is imported only when +requested, and the SDK still declares zero hard runtime dependencies. Your +instrumentation controls prompt/content attributes; inspect them before exporting +sensitive data. Flush your provider before short-lived processes exit. + ## Scopes The same identity layer, for code the adapters do not cover. `session_id` and diff --git a/sdk/python/docs/README.md b/sdk/python/docs/README.md index 314dcd43e..aa2e7aa04 100644 --- a/sdk/python/docs/README.md +++ b/sdk/python/docs/README.md @@ -28,6 +28,7 @@ That is the whole integration. | CrewAI | [crewai/](crewai/) | [crewai/examples/](crewai/examples/) | | LlamaIndex | [llama_index/](llama_index/) | [llama_index/examples/](llama_index/examples/) | | Pydantic AI | [pydantic_ai/](pydantic_ai/) | [pydantic_ai/examples/](pydantic_ai/examples/) | +| OpenTelemetry (opt in) | [otel/](otel/) | [otel/examples/](otel/examples/) | | **no framework** (your own agent) | [manual/](manual/) | [manual/examples/](manual/examples/) | Using something else — AutoGen, Haystack, Semantic Kernel, your own loop? Read diff --git a/sdk/python/docs/otel/README.md b/sdk/python/docs/otel/README.md new file mode 100644 index 000000000..292908c09 --- /dev/null +++ b/sdk/python/docs/otel/README.md @@ -0,0 +1,36 @@ +# OpenTelemetry spans + +Use `failproofai_sdk.instrument("otel", provider=provider)` to attach the optional +OTLP/JSON exporter to your OpenTelemetry `TracerProvider`. Install +`opentelemetry-sdk` in the application, not as a FailproofAI runtime dependency. +`pip install 'failproofai-sdk[otel]'` is the optional convenience extra. + +```python +import failproofai_sdk +from opentelemetry.sdk.trace import TracerProvider + +provider = TracerProvider() +failproofai_sdk.instrument("otel", provider=provider) +with provider.get_tracer("my-agent").start_as_current_span( + "agent", attributes={"gen_ai.operation.name": "invoke_agent"} +): + pass +provider.force_flush() +provider.shutdown() +``` + +The exporter publishes requests into the relay's spool. Run a connected +`failproofaid` on the same machine; no auth header or exporter network setup is +needed. An org admin must enable OpenTelemetry ingest in Cloud. Export related +spans together and include the root agent span. + +You can also pass `failproofai_sdk.OtelSpanExporter()` to your existing span +processor. Existing processors continue unchanged. `uninstrument("otel")` stops +new exports from the processor installed by FailproofAI and leaves the provider +registered. Prompt and response attributes are sent as supplied by your app. + +See the [SDK README](../../README.md#opentelemetry-opt-in) for spool overrides and +the zero-dependency contract. + +Run [examples/quickstart.py](examples/quickstart.py) for one span, or +[examples/agent_trace.py](examples/agent_trace.py) for an agent/model/tool trace. diff --git a/sdk/python/docs/otel/examples/agent_trace.py b/sdk/python/docs/otel/examples/agent_trace.py new file mode 100644 index 000000000..22517c630 --- /dev/null +++ b/sdk/python/docs/otel/examples/agent_trace.py @@ -0,0 +1,30 @@ +"""Export one related agent/model/tool trace through the local spool. + +pip install 'failproofai-sdk[otel]' +python docs/otel/examples/agent_trace.py +""" +import failproofai_sdk +from opentelemetry.sdk.trace import TracerProvider + + +def main(): + provider = TracerProvider() + failproofai_sdk.instrument("otel", provider=provider) + tracer = provider.get_tracer("planner") + with tracer.start_as_current_span( + "planner", attributes={"gen_ai.operation.name": "invoke_agent"} + ): + with tracer.start_as_current_span( + "model", attributes={"gen_ai.operation.name": "chat"} + ): + pass + with tracer.start_as_current_span( + "search", attributes={"gen_ai.operation.name": "execute_tool"} + ): + pass + provider.force_flush() + provider.shutdown() + + +if __name__ == "__main__": + main() diff --git a/sdk/python/docs/otel/examples/quickstart.py b/sdk/python/docs/otel/examples/quickstart.py new file mode 100644 index 000000000..9866645a6 --- /dev/null +++ b/sdk/python/docs/otel/examples/quickstart.py @@ -0,0 +1,22 @@ +"""An optional OTEL span, durably exported with no HTTP setup. + +pip install 'failproofai-sdk[otel]' +python docs/otel/examples/quickstart.py +""" +import failproofai_sdk +from opentelemetry.sdk.trace import TracerProvider + + +def main(): + provider = TracerProvider() + failproofai_sdk.instrument("otel", provider=provider) + with provider.get_tracer("my-agent").start_as_current_span( + "agent", attributes={"gen_ai.operation.name": "invoke_agent"} + ): + pass + provider.force_flush() + provider.shutdown() + + +if __name__ == "__main__": + main() diff --git a/sdk/python/failproofai_sdk/__init__.py b/sdk/python/failproofai_sdk/__init__.py index f971aa828..5291fcbae 100644 --- a/sdk/python/failproofai_sdk/__init__.py +++ b/sdk/python/failproofai_sdk/__init__.py @@ -38,6 +38,7 @@ "propagate", "instrument", "uninstrument", + "OtelSpanExporter", "_writer", ] @@ -123,6 +124,13 @@ def uninstrument(framework: str | None = None): return _impl(framework) +def __getattr__(name): + if name == "OtelSpanExporter": + from failproofai_sdk.integrations.otel import OtelSpanExporter + return OtelSpanExporter + raise AttributeError(name) + + # MUST be last: any `import failproofai_sdk.` binds the *module* onto this # package as `failproofai_sdk._writer`. Rebinding it here to the instance is what # keeps the published `failproofai_sdk._writer.flush_now()` recipe working — and diff --git a/sdk/python/failproofai_sdk/integrations/__init__.py b/sdk/python/failproofai_sdk/integrations/__init__.py index 4e60d8262..1bf89528c 100644 --- a/sdk/python/failproofai_sdk/integrations/__init__.py +++ b/sdk/python/failproofai_sdk/integrations/__init__.py @@ -58,6 +58,7 @@ def uninstall() -> None "crewai": "failproofai_sdk.integrations.crewai", "llama_index": "failproofai_sdk.integrations.llama_index", "pydantic_ai": "failproofai_sdk.integrations.pydantic_ai", + "otel": "failproofai_sdk.integrations.otel", } # Spellings people actually type. LangGraph is served by the LangChain adapter @@ -79,6 +80,8 @@ def uninstall() -> None "crewai": ("crewai",), "llama_index": ("llama_index", "llama_index.core"), "pydantic_ai": ("pydantic_ai",), + # OTEL is explicit-only: installing it must not duplicate existing adapters. + "otel": (), } # Guards _ACTIVE and every install/uninstall. `instrument()` is called from diff --git a/sdk/python/failproofai_sdk/integrations/otel.py b/sdk/python/failproofai_sdk/integrations/otel.py new file mode 100644 index 000000000..1bc66f511 --- /dev/null +++ b/sdk/python/failproofai_sdk/integrations/otel.py @@ -0,0 +1,189 @@ +"""Explicit OTLP/JSON exporter; disk only. OpenTelemetry is an optional runtime import.""" +import base64 +import json +import logging +import math +import os +import threading +import uuid +from pathlib import Path + +_MAX_BYTES = 8 * 1024 * 1024 +_METADATA = b'{"path":"/v1/traces","content_type":"application/json","encoding":null}\n' + + +def _value(value): + if isinstance(value, bool): + return {"boolValue": value} + if isinstance(value, str): + return {"stringValue": value} + if isinstance(value, int): + return {"intValue": str(value)} + if isinstance(value, float): + return {"doubleValue": value if math.isfinite(value) else str(value)} + if isinstance(value, bytes): + return {"bytesValue": base64.b64encode(value).decode("ascii")} + if isinstance(value, (list, tuple)): + return {"arrayValue": {"values": [_value(v) for v in value]}} + if hasattr(value, "items"): + return {"kvlistValue": {"values": _attributes(value)}} + return {} + + +def _attributes(attributes=None): + return [{"key": str(k), "value": _value(v)} for k, v in (attributes or {}).items() if v is not None] + + +def _context(context): + result = {"traceId": f"{context.trace_id:032x}", "spanId": f"{context.span_id:016x}", + "flags": int(context.trace_flags)} + if context.trace_state: + result["traceState"] = context.trace_state.to_header() + return result + + +def _resource_span(span): + data = _context(span.context) + if span.parent: + data["parentSpanId"] = f"{span.parent.span_id:016x}" + data.update({ + "name": span.name, "kind": span.kind.value + 1, + "startTimeUnixNano": str(span.start_time), "endTimeUnixNano": str(span.end_time), + "attributes": _attributes(span.attributes), + "status": {"code": span.status.status_code.value, **({"message": span.status.description} if span.status.description else {})}, + "droppedAttributesCount": span.dropped_attributes, + "droppedEventsCount": span.dropped_events, "droppedLinksCount": span.dropped_links, + "events": [{"name": event.name, "timeUnixNano": str(event.timestamp), + "attributes": _attributes(event.attributes), + "droppedAttributesCount": event.dropped_attributes} for event in span.events], + "links": [{**_context(link.context), "attributes": _attributes(link.attributes), + "droppedAttributesCount": link.dropped_attributes} for link in span.links], + }) + scope = span.instrumentation_scope + return { + "resource": {"attributes": _attributes(span.resource.attributes)}, + **({"schemaUrl": span.resource.schema_url} if span.resource.schema_url else {}), + "scopeSpans": [{ + "scope": {"name": scope.name if scope else "", **({"version": scope.version} if scope and scope.version else {})}, + **({"schemaUrl": scope.schema_url} if scope and scope.schema_url else {}), + "spans": [data], + }], + } + + +class OtelSpanExporter: + """Exports standard spans durably to the daemon's raw OTLP spool, never HTTP.""" + + def __init__(self, *, spool_dir=None): + self._directory = Path(spool_dir) if spool_dir is not None else None + self._stopped = False + self._lock = threading.Lock() + + def export(self, spans): + from opentelemetry.sdk.trace.export import SpanExportResult + with self._lock: + if self._stopped: + return SpanExportResult.FAILURE + try: + directory = self._directory or Path(os.environ.get("FAILPROOFAI_HOME", Path.home() / ".failproofai")) / "state" / "spool-otlp" + items, chunks, size = [], [], 20 + for span in spans: + encoded = json.dumps(_resource_span(span), ensure_ascii=True, separators=(",", ":")).encode("utf8") + if len(encoded) + 20 > _MAX_BYTES: + raise ValueError("OTEL span exceeds the 8 MiB relay limit") + if size + len(encoded) + 1 > _MAX_BYTES: + chunks.append(b'{"resourceSpans":[' + b",".join(items) + b"]}") + items, size = [], 20 + items.append(encoded) + size += len(encoded) + 1 + if items: + chunks.append(b'{"resourceSpans":[' + b",".join(items) + b"]}") + for chunk in chunks: + self._publish(directory, chunk) + return SpanExportResult.SUCCESS + except Exception: + logging.getLogger(__name__).warning("OTEL export failed; request was not published", exc_info=True) + return SpanExportResult.FAILURE + + def _publish(self, directory, body): + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + name = f"otlp-sdk-{uuid.uuid4()}" + temporary = directory / f".{name}.tmp" + try: + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(_METADATA + body) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, directory / f"{name}.jsonl") + if os.name != "nt": + descriptor = os.open(directory, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + finally: + temporary.unlink(missing_ok=True) + + def shutdown(self): + with self._lock: + self._stopped = True + + def force_flush(self, timeout_millis=30000): + return True + + +class _Gate: + def __init__(self, processor): + self.processor = processor + self.enabled = True + + def on_start(self, span, parent_context=None): + if self.enabled: + self.processor.on_start(span, parent_context=parent_context) + + def on_end(self, span): + if self.enabled: + self.processor.on_end(span) + + def _on_ending(self, span): + if self.enabled: + callback = getattr(self.processor, "_on_ending", None) + if callback: + callback(span) + + def shutdown(self): + self.processor.shutdown() + + def force_flush(self, timeout_millis=30000): + return self.processor.force_flush(timeout_millis) if self.enabled else True + + +class _Adapter: + name = "otel" + module = "opentelemetry" + + def __init__(self): + self.gate = None + + def install(self, **options): + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + provider = options.get("provider") or trace.get_tracer_provider() + if not hasattr(provider, "add_span_processor"): + if options.get("provider") is not None: + raise ValueError("OTEL provider must support add_span_processor()") + provider = TracerProvider() + trace.set_tracer_provider(provider) + self.gate = _Gate(BatchSpanProcessor(OtelSpanExporter(spool_dir=options.get("spool_dir")))) + provider.add_span_processor(self.gate) + + def uninstall(self): + if self.gate: + self.gate.enabled = False + self.gate.shutdown() + self.gate = None + + +adapter = _Adapter() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 9ad8c32c4..e107419ac 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -50,7 +50,7 @@ Changelog = "https://github.com/FailproofAI/failproofai/blob/main/sdk/python/CHA # `tomli` is `tomllib`'s backport: 3.10 has no stdlib TOML parser, and that test # reads this file to prove the promise, so on 3.10 it needs one rather than # skipping the check on the oldest interpreter we advertise. -dev = ["pytest>=7", "tomli>=2; python_version<'3.11'", "pytest-asyncio>=1.3,<2"] +dev = ["pytest>=7", "tomli>=2; python_version<'3.11'", "pytest-asyncio>=1.3,<2", "opentelemetry-sdk>=1.39,<2"] # Convenience aliases that pull the FRAMEWORK in. The adapter code always ships # in the base wheel and lazy-imports, so these gate nothing on our side — most @@ -72,6 +72,7 @@ crewai = [ ] llamaindex = ["llama-index-core>=0.14.23,<0.15"] pydantic-ai = ["pydantic-ai-slim>=2.0,<3"] # 2.0 removed Agent(instrument=...) +otel = ["opentelemetry-sdk>=1.39,<2"] # optional SpanExporter / processor APIs [tool.setuptools.dynamic] version = { attr = "failproofai_sdk._version.__version__" } diff --git a/sdk/python/tests/test_docs.py b/sdk/python/tests/test_docs.py index 4fbd936a8..31fb45eef 100644 --- a/sdk/python/tests/test_docs.py +++ b/sdk/python/tests/test_docs.py @@ -40,6 +40,7 @@ "crewai": "crewai", "llama_index": "llama_index", "pydantic_ai": "pydantic_ai", + "otel": "otel", } #: directory -> the failproofai-sdk extra its guide must name. @@ -48,6 +49,7 @@ "crewai": "crewai", "llama_index": "llamaindex", "pydantic_ai": "pydantic-ai", + "otel": "otel", } #: directory -> third-party modules its examples may import. `manual/` maps to @@ -59,6 +61,7 @@ "llama_index": {"llama_index"}, "pydantic_ai": {"pydantic_ai", "pydantic"}, "manual": {"openai"}, + "otel": {"opentelemetry"}, } #: stdlib and local helpers every example may import. `_shared` is the trace @@ -191,7 +194,12 @@ def test_a_framework_guide_names_its_own_extra(directory, extra): def test_a_framework_guide_shows_instrument_and_a_session(directory): text = (DOCS / directory / "README.md").read_text(encoding="utf-8") assert "failproofai_sdk.instrument(" in text, f"{directory} never shows instrument()" - assert "failproofai_sdk.session()" in text, f"{directory} never shows a session" + if directory == "otel": + # OTEL supplies trace identity itself, not the SDK event scopes. + assert 'instrument("otel"' in text + assert "start_as_current_span(" in text + else: + assert "failproofai_sdk.session()" in text, f"{directory} never shows a session" @pytest.mark.parametrize("guide", _guides(), ids=lambda p: p.parent.name) @@ -285,7 +293,11 @@ def test_an_example_does_not_thread_identity_by_hand(path): @pytest.mark.parametrize("path", EXAMPLES, ids=EXAMPLE_IDS) def test_an_example_opens_a_session(path): source = path.read_text(encoding="utf-8") - assert "failproofai_sdk.session()" in source, f"{path.name} never opens a session" + if path.parents[1].name == "otel": + assert "start_as_current_span(" in source + assert '"gen_ai.operation.name": "invoke_agent"' in source + else: + assert "failproofai_sdk.session()" in source, f"{path.name} never opens a session" @pytest.mark.parametrize( @@ -295,7 +307,8 @@ def test_an_example_opens_a_session(path): ) def test_a_framework_example_instruments(path): source = path.read_text(encoding="utf-8") - assert "failproofai_sdk.instrument()" in source, f"{path.name} never instruments" + call = 'failproofai_sdk.instrument("otel",' if path.parents[1].name == "otel" else "failproofai_sdk.instrument()" + assert call in source, f"{path.name} never instruments" @pytest.mark.parametrize( diff --git a/sdk/python/tests/test_integrations.py b/sdk/python/tests/test_integrations.py index 4d2830e07..ffd09e658 100644 --- a/sdk/python/tests/test_integrations.py +++ b/sdk/python/tests/test_integrations.py @@ -106,7 +106,7 @@ def fake(monkeypatch): def test_the_four_adapters_are_pre_registered(): # Registered before their modules exist so that adding an adapter is one new # file, never an edit to the registry (which four agents would conflict on). - assert integrations.available() == ("crewai", "langchain", "llama_index", "pydantic_ai") + assert integrations.available() == ("crewai", "langchain", "llama_index", "otel", "pydantic_ai") @pytest.mark.parametrize( diff --git a/sdk/python/tests/test_otel.py b/sdk/python/tests/test_otel.py new file mode 100644 index 000000000..88164e454 --- /dev/null +++ b/sdk/python/tests/test_otel.py @@ -0,0 +1,147 @@ +import json +import subprocess +import sys +from pathlib import Path + +import pytest +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult +from opentelemetry.trace import Link, SpanKind, Status, StatusCode + +import failproofai_sdk +from failproofai_sdk.integrations import _core + + +def test_module_and_exporter_import_without_opentelemetry(): + package_root = Path(failproofai_sdk.__file__).resolve().parent.parent + result = subprocess.run( + [sys.executable, "-I", "-S", "-c", f""" +import importlib.abc +import sys +sys.path.insert(0, {str(package_root)!r}) +class AbsentOpenTelemetry(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "opentelemetry" or fullname.startswith("opentelemetry."): + raise ModuleNotFoundError("OpenTelemetry is not installed") +sys.meta_path.insert(0, AbsentOpenTelemetry()) +from failproofai_sdk.integrations.otel import OtelSpanExporter +import failproofai_sdk +assert failproofai_sdk.OtelSpanExporter is OtelSpanExporter +exporter = OtelSpanExporter() +assert exporter.force_flush() +exporter.shutdown() +assert not any(name.startswith("opentelemetry") for name in sys.modules) +print("Imported OTEL module and exporter without OpenTelemetry") +"""], + capture_output=True, text=True, timeout=15, + ) + assert result.returncode == 0, result.stderr + assert "without OpenTelemetry" in result.stdout + + +@pytest.fixture(autouse=True) +def enabled_otel(monkeypatch): + # The CrewAI test module disables its own vendor telemetry at collection. + # These tests explicitly exercise OTEL, without changing that other suite. + monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False) + + +def batches(home): + result = [] + for file in (home / "state/spool-otlp").glob("*.jsonl"): + header, body = file.read_bytes().split(b"\n", 1) + assert json.loads(header) == {"path": "/v1/traces", "content_type": "application/json", "encoding": None} + assert file.stat().st_mode & 0o777 == 0o600 + result.extend(json.loads(body)["resourceSpans"]) + return result + + +def test_real_spans_preserve_context_attributes_events_and_precise_time(tmp_path, monkeypatch): + monkeypatch.setenv("FAILPROOFAI_HOME", str(tmp_path)) + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(failproofai_sdk.OtelSpanExporter())) + tracer = provider.get_tracer("scope", "1.0") + parent = tracer.start_span("agent", start_time=1780000000000000123) + child = tracer.start_span("chat", kind=SpanKind.CLIENT, + context=trace.set_span_in_context(parent), + start_time=1780000000000000456, + attributes={"gen_ai.operation.name": "chat", "count": 42, "enabled": True, "values": ("a", "b")}, + links=[Link(parent.get_span_context(), attributes={"linked": True})]) + child.add_event("prompt", {"text": "hi"}, timestamp=1780000000000000789) + child.set_status(Status(StatusCode.ERROR, "failed")) + child.end(end_time=1780000001000000012) + parent.end(end_time=1780000001000000013) + assert provider.force_flush() + output = next(span for resource in batches(tmp_path) for scope in resource["scopeSpans"] + for span in scope["spans"] if span["name"] == "chat") + assert output["traceId"] == f"{child.get_span_context().trace_id:032x}" + assert output["parentSpanId"] == f"{parent.get_span_context().span_id:016x}" + assert output["kind"] == 3 + assert output["startTimeUnixNano"] == "1780000000000000456" + assert output["endTimeUnixNano"] == "1780000001000000012" + assert output["status"] == {"code": 2, "message": "failed"} + assert {"key": "count", "value": {"intValue": "42"}} in output["attributes"] + assert output["events"][0]["timeUnixNano"] == "1780000000000000789" + assert output["links"][0]["spanId"] == f"{parent.get_span_context().span_id:016x}" + assert not list((tmp_path / "state/spool-otlp").glob("*.tmp")) + provider.shutdown() + + +def test_exporter_reports_failures_and_shutdown(tmp_path): + file = tmp_path / "not-a-directory" + file.write_text("x") + exporter = failproofai_sdk.OtelSpanExporter(spool_dir=file) + provider = TracerProvider() + span = provider.get_tracer("app").start_span("test") + span.end() + assert exporter.export([span]) == SpanExportResult.FAILURE + exporter.shutdown() + assert exporter.export([]) == SpanExportResult.FAILURE + assert exporter.force_flush() + provider.shutdown() + + +def test_batches_split_at_relay_limit(tmp_path, monkeypatch): + from opentelemetry.sdk.trace import SpanLimits + monkeypatch.setenv("FAILPROOFAI_HOME", str(tmp_path)) + provider = TracerProvider(span_limits=SpanLimits(max_attribute_length=10_000_000)) + span = provider.get_tracer("app").start_span("large", attributes={"content": "x" * (4 * 1024 * 1024)}) + span.end() + assert failproofai_sdk.OtelSpanExporter().export([span, span]) == SpanExportResult.SUCCESS + assert len(list((tmp_path / "state/spool-otlp").glob("*.jsonl"))) == 2 + assert len(batches(tmp_path)) == 2 + provider.shutdown() + + +def test_instrument_is_explicit_gated_and_keeps_other_processors(tmp_path, monkeypatch): + monkeypatch.setenv("FAILPROOFAI_HOME", str(tmp_path)) + _core.set_strict(True) + provider = TracerProvider() + other = [] + + class Existing: + def on_start(self, *_args, **_kwargs): pass + def on_end(self, span): other.append(span.name) + def shutdown(self): pass + def force_flush(self, *_args): return True + def _on_ending(self, span): pass + + provider.add_span_processor(Existing()) + try: + assert "otel" not in failproofai_sdk.integrations._detected() + assert failproofai_sdk.instrument("otel", provider=provider) == ("otel",) + assert failproofai_sdk.instrument("otel", provider=provider) == () + tracer = provider.get_tracer("app") + tracer.start_span("before").end() + assert provider.force_flush() + assert len(batches(tmp_path)) == 1 + assert failproofai_sdk.uninstrument("otel") == ("otel",) + tracer.start_span("after").end() + assert provider.force_flush() + assert len(batches(tmp_path)) == 1 + assert other == ["before", "after"] + finally: + failproofai_sdk.uninstrument("otel") + _core.set_strict(None) + provider.shutdown() diff --git a/sdk/python/tests/test_zero_dependencies.py b/sdk/python/tests/test_zero_dependencies.py index 95be7b72e..9915684a7 100644 --- a/sdk/python/tests/test_zero_dependencies.py +++ b/sdk/python/tests/test_zero_dependencies.py @@ -61,13 +61,14 @@ "crewai.py": {"crewai"}, "llama_index.py": {"llama_index", "llama_index_instrumentation", "pydantic"}, "pydantic_ai.py": {"pydantic_ai"}, + "otel.py": {"opentelemetry"}, } #: Framework top-level names that must never appear in `sys.modules` after a bare #: `import failproofai_sdk`. FRAMEWORK_ROOTS = { "langchain", "langchain_core", "langgraph", "crewai", - "llama_index", "llama_index_instrumentation", "pydantic", "pydantic_ai", + "llama_index", "llama_index_instrumentation", "pydantic", "pydantic_ai", "opentelemetry", } @@ -142,6 +143,7 @@ def test_no_runtime_dependencies_are_declared(): #: convenience library drifting in — anything here is one `--extra dev` away from a #: user's environment, and none of it is covered by the zero-dependency promise. ALLOWED_DEV_DEPENDENCIES = { + "opentelemetry-sdk": "real optional OTEL SpanExporter and processor contract tests; never a base dependency", "pytest": "the test runner", "pytest-asyncio": "the adapters are half-async; their scopes are exercised " "under `async with`, which needs an async test runner", @@ -159,6 +161,7 @@ def test_no_runtime_dependencies_are_declared(): "crewai": {"crewai", "onnxruntime"}, "llamaindex": {"llama-index-core"}, "pydantic-ai": {"pydantic-ai-slim"}, + "otel": {"opentelemetry-sdk"}, } diff --git a/sdk/python/uv.lock b/sdk/python/uv.lock index 1e61a0991..9bab04bd7 100644 --- a/sdk/python/uv.lock +++ b/sdk/python/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -1012,6 +1012,7 @@ crewai = [ { name = "onnxruntime", marker = "python_full_version < '3.11'" }, ] dev = [ + { name = "opentelemetry-sdk" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "tomli", marker = "python_full_version < '3.11'" }, @@ -1025,6 +1026,9 @@ langgraph = [ llamaindex = [ { name = "llama-index-core" }, ] +otel = [ + { name = "opentelemetry-sdk" }, +] pydantic-ai = [ { name = "pydantic-ai-slim", version = "2.54.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "pydantic-ai-slim", version = "2.55.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -1037,12 +1041,14 @@ requires-dist = [ { name = "langgraph", marker = "extra == 'langgraph'", specifier = ">=1.2,<2" }, { name = "llama-index-core", marker = "extra == 'llamaindex'", specifier = ">=0.14.23,<0.15" }, { name = "onnxruntime", marker = "python_full_version < '3.11' and extra == 'crewai'", specifier = ">=1.14,<1.24" }, + { name = "opentelemetry-sdk", marker = "extra == 'dev'", specifier = ">=1.39,<2" }, + { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.39,<2" }, { name = "pydantic-ai-slim", marker = "extra == 'pydantic-ai'", specifier = ">=2.0,<3" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.3,<2" }, { name = "tomli", marker = "python_full_version < '3.11' and extra == 'dev'", specifier = ">=2" }, ] -provides-extras = ["dev", "langchain", "langgraph", "crewai", "llamaindex", "pydantic-ai"] +provides-extras = ["dev", "langchain", "langgraph", "crewai", "llamaindex", "pydantic-ai", "otel"] [[package]] name = "filelock" diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 8c00d453c..26d8cf653 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -82,6 +82,49 @@ is unreachable. ### 2. Adapters +#### OpenTelemetry (opt in) + +If your app already emits OpenTelemetry spans, send them through FailproofAI Cloud +without replacing its framework adapters. Install OpenTelemetry separately: + +```bash +npm install @opentelemetry/api @opentelemetry/sdk-trace-base +``` + +```ts +import * as failproofai from "@failproofai/sdk"; +import { BasicTracerProvider } from "@opentelemetry/sdk-trace-base"; +import type { SpanProcessor } from "@opentelemetry/sdk-trace-base"; + +const spanProcessors: SpanProcessor[] = []; // include your existing processors +await failproofai.instrument("otel", { spanProcessors }); +const provider = new BasicTracerProvider({ spanProcessors }); +const tracer = provider.getTracer("my-agent"); +const span = tracer.startSpan("agent", { + attributes: { "gen_ai.operation.name": "invoke_agent" }, +}); +span.end(); +await provider.forceFlush(); +``` + +With no options, `instrument("otel")` creates a provider only if none is registered. +For an existing OpenTelemetry v2 provider, add the processor before constructing +the provider, as above. For providers exposing `addSpanProcessor`, pass `{ provider }`. +You can also supply `new failproofai.OtelSpanExporter()` to your own span processor. +`uninstrument("otel")` disables only FailproofAI's processor; other exporters keep working. + +The exporter writes OTLP/JSON export requests durably into the local relay spool +(`$FAILPROOFAI_HOME/state/spool-otlp`, otherwise `~/.failproofai/state/spool-otlp`). +It never sends HTTP or reads the Cloud key. Connect the daemon with `failproofai config` +and ask an admin to enable Settings → OpenTelemetry. No local TCP listener is +needed for SDK exports. `spoolDir` is an explicit override for tests or managed +deployments. `configure({ baseDir })` still applies only to the existing event adapters. + +OTEL is never auto-enabled by a bare `instrument()`. The SDK still declares zero +hard runtime dependencies. Your spans control content capture; do not export +sensitive attributes unless intended. Export related spans together for complete +agent traces; call `provider.forceFlush()` before a short-lived process exits. + ```ts await failproofai.instrument(); // whatever it can find await failproofai.instrument("langchain"); // exactly one diff --git a/sdk/typescript/integration/ai.test.ts b/sdk/typescript/integration/ai.test.ts index 3ed9106a7..dd5921685 100644 --- a/sdk/typescript/integration/ai.test.ts +++ b/sdk/typescript/integration/ai.test.ts @@ -258,8 +258,15 @@ describe.each(FIXTURES)("%s", (fixture) => { }; expect(report.text).toBe("It is 20C in Paris."); if (major >= 7) { - // v7 has no OpenTelemetry dependency; the integration records the call. - expect(report.customer).toBe("absent"); + // v7 has no OpenTelemetry dependency of its own. The optional OTEL + // exporter's dev tests can make it resolvable from the parent SDK + // project; when present, their provider must still own the global slot + // and receive only their service span (ai 7 emits no OTEL spans). + if (report.customer !== "absent") { + expect(report.customer, describeTrace(result)).toEqual({ + registered: true, ended: ["http.request"], + }); + } expect(shape(result.events), describeTrace(result)).toEqual(GEN); return; } diff --git a/sdk/typescript/package-lock.json b/sdk/typescript/package-lock.json index 2ed2f574c..bd73b56bc 100644 --- a/sdk/typescript/package-lock.json +++ b/sdk/typescript/package-lock.json @@ -13,6 +13,8 @@ }, "devDependencies": { "@eslint/js": "^9.38.0", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/sdk-trace-base": "^2.12.0", "@types/node": "^22.10.2", "eslint": "^9.38.0", "typescript": "^5.9.2", @@ -281,6 +283,56 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "dev": true, + "engines": { "node": ">=8.0.0" } + }, + "node_modules/@opentelemetry/core": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.12.0.tgz", + "integrity": "sha512-1HsSAuvT4/my0QrXWsyXFdWjKaPeCHsismKZgEEIb8NK13LP6DrMRlsUbvD/0F9mDWK3xDom9AMv3IscxgOnOQ==", + "dev": true, + "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, + "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" }, + "engines": { "node": "^18.19.0 || >=20.6.0" } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.12.0.tgz", + "integrity": "sha512-+rSRoOdln6NhXTVcBdHSlZxJDNzwxadeweK2ltZO3tVtfp6aklABc06xclDk6ovRR+EiGgIcSbh/6NgjFhnb3A==", + "dev": true, + "dependencies": { "@opentelemetry/core": "2.12.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, + "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" }, + "engines": { "node": "^18.19.0 || >=20.6.0" } + }, + "node_modules/@opentelemetry/sdk-trace": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace/-/sdk-trace-2.12.0.tgz", + "integrity": "sha512-ImVuRQ6faOsTi1Wna88s2avgrrf/iDgXPigIeynkUhjstMuoQUDOJXgkez2j1mlU4btjXcXBGa+q7qKFYR3GsA==", + "dev": true, + "dependencies": { "@opentelemetry/core": "2.12.0", "@opentelemetry/resources": "2.12.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, + "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" }, + "engines": { "node": "^18.19.0 || >=20.6.0" } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.12.0.tgz", + "integrity": "sha512-JOERgZCxRM06FEJ5kEWxlSRChah6MmXDx3KwCruqEv/iEyLkTokCSemf2y1w8oIHtfrr8EVTpzebq/31vn2Cpg==", + "dev": true, + "dependencies": { "@opentelemetry/core": "2.12.0", "@opentelemetry/resources": "2.12.0", "@opentelemetry/sdk-trace": "2.12.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, + "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" }, + "engines": { "node": "^18.19.0 || >=20.6.0" } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz", + "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==", + "dev": true, + "engines": { "node": ">=14" } + }, "node_modules/@oxc-project/types": { "version": "0.150.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.150.0.tgz", diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 002fb18c1..e03b3bd42 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -389,6 +389,8 @@ }, "devDependencies": { "@eslint/js": "^9.38.0", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/sdk-trace-base": "^2.12.0", "@types/node": "^22.10.2", "eslint": "^9.38.0", "typescript": "^5.9.2", diff --git a/sdk/typescript/src/edge/index.ts b/sdk/typescript/src/edge/index.ts index e9ff394cd..ec508f60b 100644 --- a/sdk/typescript/src/edge/index.ts +++ b/sdk/typescript/src/edge/index.ts @@ -42,6 +42,15 @@ import { notice } from "./notice.js"; export { VERSION as version }; export const DEFAULT_AGENT_ID = "main"; +/** File export is unavailable at the Edge, like the other telemetry surfaces. */ +export class OtelSpanExporter { + export(_spans: unknown[], callback: (result: { code: number }) => void): void { + notice(); + callback({ code: 0 }); + } + forceFlush(): Promise { return Promise.resolve(); } + shutdown(): Promise { return Promise.resolve(); } +} /** The same registered symbol as the Node build's, so comparisons still hold. */ export const AUTO: unique symbol = Symbol.for("failproofai.AUTO") as never; diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index c32437283..db9ed6c86 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -88,6 +88,8 @@ export { uninstrument, } from "./integrations/index.js"; export type { FrameworkName, InstrumentOptions } from "./integrations/index.js"; +export { OtelSpanExporter } from "./integrations/otel.js"; +export type { OtelReadableSpan, OtelSpanExporterOptions } from "./integrations/otel.js"; export interface ConfigureOptions { /** diff --git a/sdk/typescript/src/integrations/index.ts b/sdk/typescript/src/integrations/index.ts index e2b9181b7..f0ab6543b 100644 --- a/sdk/typescript/src/integrations/index.ts +++ b/sdk/typescript/src/integrations/index.ts @@ -52,13 +52,14 @@ import * as compat from "./compat.js"; import type { Adapter } from "./core.js"; import * as core from "./core.js"; -export type FrameworkName = "langchain" | "ai" | "mastra" | "llamaindex"; +export type FrameworkName = "langchain" | "ai" | "mastra" | "llamaindex" | "otel"; const REGISTRY: Record Promise<{ adapter: Adapter }>> = { langchain: () => import("./langchain.js"), ai: () => import("./ai.js"), mastra: () => import("./mastra.js"), llamaindex: () => import("./llamaindex.js"), + otel: () => import("./otel.js"), }; /** @@ -90,6 +91,8 @@ const DETECT: Record = { ai: ["ai"], mastra: ["@mastra/core"], llamaindex: ["llamaindex", "@llamaindex/core"], + // Never auto-enable a second reporting path on an existing application. + otel: [], }; const active = new Map(); diff --git a/sdk/typescript/src/integrations/otel.ts b/sdk/typescript/src/integrations/otel.ts new file mode 100644 index 000000000..c52c9444d --- /dev/null +++ b/sdk/typescript/src/integrations/otel.ts @@ -0,0 +1,204 @@ +/** + * Opt-in OTLP/JSON SpanExporter. Only disk I/O: the daemon authenticates and + * delivers opaque batches. No OpenTelemetry import until instrument("otel"). + */ +import { randomUUID } from "node:crypto"; +import { closeSync, fsyncSync, mkdirSync, openSync, renameSync, rmSync, writeSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { importModule, resolveFrom, tryRequire } from "../node-require.js"; +import type { Adapter } from "./core.js"; + +type ObjectValue = Record; +export interface OtelReadableSpan { + name: string; + kind: number; + spanContext(): { traceId: string; spanId: string; traceFlags?: number; traceState?: { serialize(): string } }; + parentSpanId?: string; + parentSpanContext?: { spanId: string }; + startTime: readonly [number, number]; + endTime: readonly [number, number]; + attributes: ObjectValue; + status: { code: number; message?: string }; + resource?: { attributes: ObjectValue; schemaUrl?: string }; + instrumentationScope?: { name: string; version?: string; schemaUrl?: string }; + instrumentationLibrary?: { name: string; version?: string; schemaUrl?: string }; + events?: readonly { name: string; time: readonly [number, number]; attributes?: ObjectValue; droppedAttributesCount?: number }[]; + links?: readonly { context: { traceId: string; spanId: string; traceFlags?: number; traceState?: { serialize(): string } }; attributes?: ObjectValue; droppedAttributesCount?: number }[]; + droppedAttributesCount?: number; + droppedEventsCount?: number; + droppedLinksCount?: number; +} +export interface OtelExportResult { code: number; error?: Error } +export interface OtelSpanExporterOptions { spoolDir?: string } +const MAX_BYTES = 8 * 1024 * 1024; +const metadata = '{"path":"/v1/traces","content_type":"application/json","encoding":null}\n'; + +function anyValue(value: unknown): ObjectValue { + if (typeof value === "string") return { stringValue: value }; + if (typeof value === "boolean") return { boolValue: value }; + if (typeof value === "bigint") return { intValue: value.toString() }; + if (typeof value === "number") { + if (!Number.isFinite(value)) return { doubleValue: String(value) }; + return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value }; + } + if (value instanceof Uint8Array) return { bytesValue: Buffer.from(value).toString("base64") }; + if (Array.isArray(value)) return { arrayValue: { values: value.map(anyValue) } }; + if (value && typeof value === "object") return { kvlistValue: { values: attributes(value as ObjectValue) } }; + return {}; +} +function attributes(value: ObjectValue = {}): ObjectValue[] { + return Object.entries(value).filter(([, v]) => v !== undefined && v !== null) + .map(([key, v]) => ({ key, value: anyValue(v) })); +} +function nanos(time: readonly [number, number]): string { + return (BigInt(time[0]) * 1_000_000_000n + BigInt(time[1])).toString(); +} +function resourceSpan(span: OtelReadableSpan): ObjectValue { + const context = span.spanContext(); + const scope = span.instrumentationScope ?? span.instrumentationLibrary; + return { + resource: { attributes: attributes(span.resource?.attributes) }, + ...(span.resource?.schemaUrl ? { schemaUrl: span.resource.schemaUrl } : {}), + scopeSpans: [{ + scope: { name: scope?.name ?? "", ...(scope?.version ? { version: scope.version } : {}) }, + ...(scope?.schemaUrl ? { schemaUrl: scope.schemaUrl } : {}), + spans: [{ + traceId: context.traceId, spanId: context.spanId, + ...(span.parentSpanContext?.spanId || span.parentSpanId ? { parentSpanId: span.parentSpanContext?.spanId ?? span.parentSpanId } : {}), + ...(context.traceState ? { traceState: context.traceState.serialize() } : {}), + flags: context.traceFlags ?? 0, name: span.name, kind: span.kind + 1, + startTimeUnixNano: nanos(span.startTime), endTimeUnixNano: nanos(span.endTime), + attributes: attributes(span.attributes), status: span.status, + droppedAttributesCount: span.droppedAttributesCount ?? 0, + droppedEventsCount: span.droppedEventsCount ?? 0, droppedLinksCount: span.droppedLinksCount ?? 0, + events: (span.events ?? []).map(event => ({ + name: event.name, timeUnixNano: nanos(event.time), attributes: attributes(event.attributes), + droppedAttributesCount: event.droppedAttributesCount ?? 0, + })), + links: (span.links ?? []).map(link => ({ + traceId: link.context.traceId, spanId: link.context.spanId, flags: link.context.traceFlags ?? 0, + ...(link.context.traceState ? { traceState: link.context.traceState.serialize() } : {}), + attributes: attributes(link.attributes), droppedAttributesCount: link.droppedAttributesCount ?? 0, + })), + }], + }], + }; +} + +/** Structural implementation of OpenTelemetry's SpanExporter; zero hard deps. */ +export class OtelSpanExporter { + private stopped = false; + private readonly directory?: string; + constructor(options: OtelSpanExporterOptions = {}) { this.directory = options.spoolDir; } + + export(spans: readonly OtelReadableSpan[], callback: (result: OtelExportResult) => void): void { + let result: OtelExportResult = { code: 0 }; + try { + if (this.stopped) throw new Error("OTEL exporter has shut down."); + const dir = this.directory ?? join(process.env.FAILPROOFAI_HOME || join(homedir(), ".failproofai"), "state", "spool-otlp"); + // Encode each span once and split batches at the relay's request limit. + const chunks: string[] = []; + let items: string[] = []; + let size = Buffer.byteLength('{"resourceSpans":[]}'); + for (const span of spans) { + const encoded = JSON.stringify(resourceSpan(span)); + const bytes = Buffer.byteLength(encoded); + if (bytes + 20 > MAX_BYTES) throw new Error("OTEL span exceeds the 8 MiB relay limit."); + if (size + bytes + 1 > MAX_BYTES) { + chunks.push(`{"resourceSpans":[${items.join(",")}]}`); items = []; size = 20; + } + items.push(encoded); size += bytes + 1; + } + if (items.length) chunks.push(`{"resourceSpans":[${items.join(",")}]}`); + for (const body of chunks) this.publish(dir, body); + } catch (error) { + result = { code: 1, error: error instanceof Error ? error : new Error(String(error)) }; + } + callback(result); + } + private publish(dir: string, body: string): void { + mkdirSync(dir, { recursive: true, mode: 0o700 }); + const name = `otlp-sdk-${randomUUID()}`; + const temp = join(dir, `.${name}.tmp`); + let fd: number | undefined; + try { + fd = openSync(temp, "wx", 0o600); + const bytes = Buffer.from(metadata + body); + let offset = 0; + while (offset < bytes.length) offset += writeSync(fd, bytes, offset, bytes.length - offset); + fsyncSync(fd); closeSync(fd); fd = undefined; + renameSync(temp, join(dir, `${name}.jsonl`)); + if (process.platform !== "win32") { + const directory = openSync(dir, "r"); + try { fsyncSync(directory); } finally { closeSync(directory); } + } + } finally { + if (fd !== undefined) closeSync(fd); + rmSync(temp, { force: true }); + } + } + forceFlush(): Promise { return Promise.resolve(); } + shutdown(): Promise { this.stopped = true; return Promise.resolve(); } +} + +interface Processor { + onStart(...args: unknown[]): void; + onEnd(...args: unknown[]): void; + shutdown(): Promise; + forceFlush(): Promise; +} +interface Provider { addSpanProcessor(processor: Processor): void } +interface TraceSdk { + BatchSpanProcessor: new (exporter: OtelSpanExporter) => Processor; + BasicTracerProvider: new (options: { spanProcessors: Processor[] }) => unknown; +} +interface TraceApi { trace: { setGlobalTracerProvider(provider: unknown): boolean } } +let installed: { enabled: boolean; processor: Processor; list?: Processor[] } | null = null; +export const adapter: Adapter = { + name: "otel", + async install(options = {}) { + const provider = options.provider as Provider | undefined; + const list = options.spanProcessors as Processor[] | undefined; + if (provider && typeof provider.addSpanProcessor !== "function") { + throw new Error('instrument("otel") needs { spanProcessors } before constructing an OTEL v2 provider, or a provider with addSpanProcessor().'); + } + const path = resolveFrom("@opentelemetry/sdk-trace-base"); + if (!path) throw new Error('Install @opentelemetry/sdk-trace-base to use instrument("otel").'); + const module = tryRequire("@opentelemetry/sdk-trace-base") ?? + await importModule(path) as TraceSdk; + const exporter = new OtelSpanExporter({ spoolDir: options.spoolDir as string | undefined }); + const batch = new module.BatchSpanProcessor(exporter); + const state = { enabled: true, processor: batch, list }; + // Providers have no portable remove API. A gate makes uninstall effective + // without shutting down anybody else's processors or provider. + const gate: Processor = { + onStart: (...args) => { if (state.enabled) batch.onStart(...args); }, + onEnd: (...args) => { if (state.enabled) batch.onEnd(...args); }, + forceFlush: () => state.enabled ? batch.forceFlush() : Promise.resolve(), + shutdown: () => batch.shutdown(), + }; + installed = state; + if (Array.isArray(list)) { list.push(gate); state.processor = gate; } + else if (provider) { provider.addSpanProcessor(gate); state.processor = gate; } + else { + state.processor = gate; + const apiPath = resolveFrom("@opentelemetry/api"); + if (!apiPath) throw new Error("Install @opentelemetry/api to register the OTEL provider."); + const api = tryRequire("@opentelemetry/api") ?? await importModule(apiPath) as TraceApi; + if (!api.trace.setGlobalTracerProvider(new module.BasicTracerProvider({ spanProcessors: [gate] }))) { + throw new Error('An OTEL provider is already registered; use instrument("otel", { spanProcessors }) before constructing it.'); + } + } + }, + uninstall() { + if (!installed) return; + installed.enabled = false; + if (installed.list) { + const index = installed.list.indexOf(installed.processor); + if (index >= 0) installed.list.splice(index, 1); + } + void installed.processor.shutdown().catch(() => { /* SDK teardown never throws into the app. */ }); + installed = null; + }, +}; diff --git a/sdk/typescript/test/integrations.test.ts b/sdk/typescript/test/integrations.test.ts index 91c502932..a63a187ba 100644 --- a/sdk/typescript/test/integrations.test.ts +++ b/sdk/typescript/test/integrations.test.ts @@ -324,7 +324,7 @@ describe("RunTracker", () => { describe("the registry", () => { it("lists the frameworks it can instrument", () => { - expect(available()).toEqual(["ai", "langchain", "llamaindex", "mastra"]); + expect(available()).toEqual(["ai", "langchain", "llamaindex", "mastra", "otel"]); }); it("throws on an unknown name, listing the valid ones", async () => { diff --git a/sdk/typescript/test/otel.test.ts b/sdk/typescript/test/otel.test.ts new file mode 100644 index 000000000..fbccd6864 --- /dev/null +++ b/sdk/typescript/test/otel.test.ts @@ -0,0 +1,125 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BasicTracerProvider, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import type { SpanProcessor, ReadableSpan } from "@opentelemetry/sdk-trace-base"; +import { ROOT_CONTEXT, SpanKind, SpanStatusCode, trace } from "@opentelemetry/api"; +import { OtelSpanExporter } from "../src/index.js"; +import { instrument, uninstrument, activeFrameworks } from "../src/integrations/index.js"; +import { setStrict } from "../src/integrations/core.js"; + +let home: string; +let previousHome: string | undefined; +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "fpai-otel-sdk-")); + previousHome = process.env.FAILPROOFAI_HOME; + process.env.FAILPROOFAI_HOME = home; +}); +afterEach(() => { + uninstrument("otel"); + setStrict(null); + if (previousHome === undefined) delete process.env.FAILPROOFAI_HOME; + else process.env.FAILPROOFAI_HOME = previousHome; + rmSync(home, { recursive: true, force: true }); +}); +function files() { return readdirSync(join(home, "state/spool-otlp")); } +function body(name: string) { + const text = readFileSync(join(home, "state/spool-otlp", name), "utf8"); + const split = text.indexOf("\n"); + expect(JSON.parse(text.slice(0, split))).toEqual({ + path: "/v1/traces", content_type: "application/json", encoding: null, + }); + return JSON.parse(text.slice(split + 1)) as { resourceSpans: Array<{ + resource: { attributes: unknown[] }; + scopeSpans: Array<{ scope: { name: string }; spans: Array> }>; + }> }; +} + +describe("OTLP/JSON SpanExporter", () => { + it("exports real ended spans, links, events, precise nanos, status and typed attributes durably without network", async () => { + const exporter = new OtelSpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const tracer = provider.getTracer("test-scope", "1.0"); + const parent = tracer.startSpan("agent", { startTime: [1780000000, 123], attributes: { "gen_ai.operation.name": "invoke_agent" } }); + const child = tracer.startSpan("chat", { + startTime: [1780000000, 456], kind: SpanKind.CLIENT, + attributes: { "gen_ai.operation.name": "chat", count: 42, enabled: true, decimal: 1.5, values: ["a", "b"] }, + links: [{ context: parent.spanContext(), attributes: { linked: true } }], + }, trace.setSpanContext(ROOT_CONTEXT, parent.spanContext())); + child.addEvent("prompt", { text: "hello" }, [1780000000, 789]); + child.setStatus({ code: SpanStatusCode.ERROR, message: "failed" }); + child.end([1780000001, 12]); parent.end([1780000001, 13]); + await provider.forceFlush(); + expect(files()).toHaveLength(2); + const spans = files().flatMap(name => body(name).resourceSpans.flatMap(r => r.scopeSpans.flatMap(s => s.spans))); + const output = spans.find(span => span.name === "chat")!; + expect(output).toMatchObject({ + traceId: child.spanContext().traceId, spanId: child.spanContext().spanId, + parentSpanId: parent.spanContext().spanId, kind: 3, + startTimeUnixNano: "1780000000000000456", endTimeUnixNano: "1780000001000000012", + status: { code: 2, message: "failed" }, + events: [{ name: "prompt", timeUnixNano: "1780000000000000789" }], + links: [{ traceId: parent.spanContext().traceId, spanId: parent.spanContext().spanId }], + }); + expect(output.attributes).toContainEqual({ key: "count", value: { intValue: "42" } }); + expect(output.attributes).toContainEqual({ key: "values", value: { arrayValue: { values: [{ stringValue: "a" }, { stringValue: "b" }] } } }); + for (const name of files()) { + expect(name).toMatch(/^otlp-sdk-.*\.jsonl$/); + expect(statSync(join(home, "state/spool-otlp", name)).mode & 0o777).toBe(0o600); + } + await provider.shutdown(); + }); + + it("reports disk and post-shutdown failures through the callback, once", async () => { + const file = join(home, "not-a-directory"); + writeFileSync(file, "x"); + const exporter = new OtelSpanExporter({ spoolDir: file }); + const provider = new BasicTracerProvider(); + const span = provider.getTracer("test").startSpan("test"); + span.end(); + const results: number[] = []; + exporter.export([span as unknown as ReadableSpan], result => results.push(result.code)); + await exporter.shutdown(); + exporter.export([], result => results.push(result.code)); + expect(results).toEqual([1, 1]); + await provider.shutdown(); + }); + + it("splits an export above 8 MiB into complete bounded requests", async () => { + const provider = new BasicTracerProvider({ spanLimits: { attributeValueLengthLimit: 10_000_000 } }); + const span = provider.getTracer("test").startSpan("big", { attributes: { content: "x".repeat(4 * 1024 * 1024) } }); + span.end(); + const results: number[] = []; + new OtelSpanExporter().export([span as unknown as ReadableSpan, span as unknown as ReadableSpan], result => results.push(result.code)); + expect(results).toEqual([0]); + expect(files()).toHaveLength(2); + for (const name of files()) expect(body(name).resourceSpans).toHaveLength(1); + await provider.shutdown(); + }); + + it("instrument('otel') adds one gated processor and leaves other exporters unchanged", async () => { + setStrict(true); + const processors: SpanProcessor[] = []; + expect(await instrument("otel", { spanProcessors: processors })).toEqual(["otel"]); + expect(await instrument("otel", { spanProcessors: processors })).toEqual([]); + const observed: string[] = []; + const existing: SpanProcessor = { + onStart() {}, onEnd(span) { observed.push(span.name); }, + forceFlush: () => Promise.resolve(), shutdown: () => Promise.resolve(), + }; + const provider = new BasicTracerProvider({ spanProcessors: [existing, ...processors] }); + const tracer = provider.getTracer("app"); + tracer.startSpan("before").end(); + await provider.forceFlush(); + expect(files()).toHaveLength(1); + expect(activeFrameworks()).toContain("otel"); + expect(uninstrument("otel")).toEqual(["otel"]); + tracer.startSpan("after").end(); + await provider.forceFlush(); + expect(files()).toHaveLength(1); + expect(observed).toEqual(["before", "after"]); + expect(processors).toHaveLength(0); + await provider.shutdown(); + }); +}); diff --git a/src/hooks/cloud-introspect.ts b/src/hooks/cloud-introspect.ts index 84cfc996d..39a673824 100644 --- a/src/hooks/cloud-introspect.ts +++ b/src/hooks/cloud-introspect.ts @@ -44,6 +44,8 @@ export interface KeyIdentity { /** The EFFECTIVE permission set — what the server enforces. */ permissions: string[]; expiresAt?: string | null; + /** Missing on older Cloud versions; absence is unknown, never an error. */ + otelIngest?: boolean; } export type IntrospectResult = @@ -128,6 +130,7 @@ export async function introspectKey( orgName: typeof b.org_name === "string" ? b.org_name : undefined, permissions: b.permissions.filter((p): p is string => typeof p === "string"), expiresAt: typeof b.expires_at === "string" ? b.expires_at : null, + otelIngest: typeof b.otel_ingest === "boolean" ? b.otel_ingest : undefined, }, }; } diff --git a/src/hooks/collector-config.ts b/src/hooks/collector-config.ts index 5828ddf09..ee5091970 100644 --- a/src/hooks/collector-config.ts +++ b/src/hooks/collector-config.ts @@ -13,7 +13,7 @@ * the only thing that writes them. */ import { credentialsFile, failproofaiHome as layoutHome } from "./fp-home"; -import { readCredentials, writeCredentials, updateConfig } from "./fp-config"; +import { readConfig, readCredentials, writeCredentials, updateConfig } from "./fp-config"; /** * The hosted ingest endpoint — a COMPLETE endpoint, not a base to join onto. @@ -42,6 +42,8 @@ export interface IngestCredential { export interface CollectorSettings { /** Ship agent session transcripts. Separate opt-in from having a key. */ sessions: boolean; + agents?: Record; + otlp?: { enabled: boolean; port?: number }; /** Ship hook activity. */ hooks: boolean; hooksVerbosity?: "all" | "decisions" | "off"; @@ -60,6 +62,11 @@ export function failproofaiHome(): string { return layoutHome(); } +export function agentSessionsEnabled(agent: string): boolean { + const collector = readConfig().collector; + return collector.agents?.[agent]?.sessions ?? collector.sessions; +} + export function ingestPath(): string { return credentialsFile(); } @@ -122,6 +129,8 @@ export function writeCollectorSettings(settings: CollectorSettings): void { redact: settings.redact ?? "minimal", environment: settings.environment ?? "local", machineId: settings.machineId, + agents: settings.agents ?? readConfig().collector.agents, + otlp: settings.otlp ?? readConfig().collector.otlp, }, }); } diff --git a/src/hooks/configure-wizard.ts b/src/hooks/configure-wizard.ts index c2ce66077..cc784d5e6 100644 --- a/src/hooks/configure-wizard.ts +++ b/src/hooks/configure-wizard.ts @@ -7,8 +7,8 @@ * 2. Connect — one masked API-key field; Tab for open source. * 3. Agents — which agents failproofai traces: hooks, collection and * backfill all follow this one saved list. - * 4. Done — applied straight after the agents step; there is no separate - * review. Esc at any step before that changes nothing. + * 4. Done — applied after the last answer; there is no separate review. + * Esc at any step before that changes nothing. * * ## Two ordering rules that are not cosmetic * @@ -1069,7 +1069,9 @@ export async function runConfigureWizard( // ── Apply ───────────────────────────────────────────────────────────────── // - // Enter on the agents step was the last decision. ORDER MATTERS here: + // The agents step was the last decision. OpenTelemetry is not part of setup: + // it is opt-in, per agent, through `failproofai otel enable `. + // ORDER MATTERS here: // // 1. The daemon first, because setup requires it: if it cannot be installed // this run must leave the machine exactly as it found it. Writing hooks diff --git a/src/hooks/fp-config.ts b/src/hooks/fp-config.ts index 41ec050a4..103bd1b9b 100644 --- a/src/hooks/fp-config.ts +++ b/src/hooks/fp-config.ts @@ -281,6 +281,14 @@ export interface FpConfig { }; collector: { sessions: boolean; + /** + * Per-agent transcript transport overrides. Orthogonal to `agents.selected`, + * which gates collection: a traced agent using OTEL keeps its hooks, but + * must not also send transcripts. Retain the original OTEL key layout so + * upgrades and recovery snapshots can restore the previous preference. + */ + agents?: Record; + otlp?: { enabled: boolean; port?: number }; hooks: boolean; hooksVerbosity: "all" | "decisions" | "off"; redact: "minimal" | "off"; @@ -460,6 +468,25 @@ function readSources(raw: unknown): FpConfig["collector"]["sources"] { return Object.keys(out).length > 0 ? out : undefined; } +function readAgentSessions(raw: unknown): FpConfig["collector"]["agents"] { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) return undefined; + const agents = Object.fromEntries(Object.entries(raw).flatMap(([agent, settings]) => { + if (!settings || typeof settings !== "object" || Array.isArray(settings)) return []; + const sessions = (settings as Record).sessions; + return typeof sessions === "boolean" ? [[agent, { sessions }]] : []; + })); + return Object.keys(agents).length ? agents : undefined; +} + +function readOtlpSettings(raw: unknown): FpConfig["collector"]["otlp"] { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) return undefined; + const settings = raw as Record; + return { + enabled: settings.enabled === true, + ...(typeof settings.port === "number" && Number.isInteger(settings.port) ? { port: settings.port } : {}), + }; +} + /** * `agents` → the in-memory shape. Kept as written (strings only, de-duplicated * in order): the daemon owns the list of real agent ids and warns about an @@ -569,6 +596,8 @@ export function projectConfig(parsed: Record): FpConfig { typeof collector.environment === "string" ? collector.environment : "local", machineId: typeof collector.machine_id === "string" ? collector.machine_id : undefined, sources: readSources(collector.sources), + agents: readAgentSessions(collector.agents), + otlp: readOtlpSettings(collector.otlp), }, // Only an explicit `false` switches it off. Absent, or any other value, // reads as on — the shipped default, and what a config with no @@ -632,6 +661,9 @@ const OWNED_CONFIG_KEYS: readonly (readonly string[])[] = [ ["collector", "environment"], ["collector", "machine_id"], ["collector", "sources", "*", "extra_paths"], + ["collector", "agents", "*", "sessions"], + ["collector", "otlp", "enabled"], + ["collector", "otlp", "port"], ["telemetry", "enabled"], ["audit", "auto"], ["audit", "interval_days"], @@ -709,6 +741,8 @@ export function writeConfig(config: FpConfig, raw?: Record): vo // Omitted when unset rather than written null: the Rust side treats an // absent machine_id as "derive one", and an explicit null is not that. ...(c.machineId ? { machine_id: c.machineId } : {}), + ...(c.agents ? { agents: c.agents } : {}), + ...(c.otlp ? { otlp: c.otlp } : {}), // Extra capture paths per harness. Emitted ONLY when non-empty, so a // machine that never configured one gets a file byte-identical to what // this function produced before the field existed — the same reasoning as diff --git a/src/hooks/fp-home.ts b/src/hooks/fp-home.ts index 395bfe2ae..344ca93aa 100644 --- a/src/hooks/fp-home.ts +++ b/src/hooks/fp-home.ts @@ -139,6 +139,14 @@ export const versionFile = (home?: string) => atHome(home, "VERSION"); /** Non-secret configuration. World-readable by design; never holds a token. */ export const configFile = (home?: string) => atHome(home, "config.json"); +export const otelAgentStateDir = (home?: string) => atHome(home, "state", "otel-agents"); +export const otelAgentStateFile = (agent: string) => { + if (!["claude", "codex", "gemini", "copilot"].includes(agent)) throw new Error("Unknown OTEL agent."); + return resolve(otelAgentStateDir(), `${agent}.json`); +}; +export const otlpSpoolDir = (home?: string) => atHome(home, "state", "spool-otlp"); +export const otlpHealthFile = (home?: string) => atHome(home, "state", "otlp-health.json"); + /** * Every credential, owner-only. * @@ -576,6 +584,7 @@ export const HOME_CLASSES: readonly { path: (home?: string) => string; class: Da // the collector preferences, `[audit] auto`, the telemetry opt-out, and // `collector.sources.*.extra_paths` — the entire output of `harness add-path`. { path: configFile, class: "user-typed" }, + { path: otelAgentStateDir, class: "user-typed" }, // The builtin enable/disable set and per-policy params. `resetHome` used to // clear this and carry eight named keys back over; it is simply kept now. { path: globalPolicyConfigFile, class: "user-typed" }, @@ -602,6 +611,7 @@ export const HOME_CLASSES: readonly { path: (home?: string) => string; class: Da // these is PERMANENT and not merely slow is `cursorsDir` below: the watermark // has already advanced past them, so nothing will ever read that range again. { path: spoolDir, class: "undelivered" }, + { path: otlpSpoolDir, class: "undelivered" }, { path: failedDir, class: "undelivered" }, // The SDK spool (`events/` + `failed/`), same argument. { path: customAgentsDir, class: "undelivered" }, @@ -657,6 +667,7 @@ export const HOME_CLASSES: readonly { path: (home?: string) => string; class: Da { path: auditCacheDir, class: "derived" }, { path: auditScheduleFile, class: "derived" }, { path: collectorHealthFile, class: "derived" }, + { path: otlpHealthFile, class: "derived" }, { path: codexSessionPathsFile, class: "derived" }, { path: shimsDir, class: "derived" }, { path: sessionPauseDir, class: "derived" }, diff --git a/src/hooks/integrations.ts b/src/hooks/integrations.ts index bb1bf5e68..3d1c2ae76 100644 --- a/src/hooks/integrations.ts +++ b/src/hooks/integrations.ts @@ -85,7 +85,7 @@ export class UnreadableAgentConfigError extends Error { } } -function readJsonFile(path: string): Record { +export function readJsonFile(path: string): Record { if (!existsSync(path)) return {}; let raw: string; try { diff --git a/src/hooks/otel-cli.ts b/src/hooks/otel-cli.ts new file mode 100644 index 000000000..aa11a344b --- /dev/null +++ b/src/hooks/otel-cli.ts @@ -0,0 +1,344 @@ +/** Opt-in agent setup; no OTLP translation or changes to the events uploader. */ +import { + chmodSync, existsSync, mkdirSync, readFileSync, + statSync, unlinkSync, writeFileSync, +} from "node:fs"; +import { dirname } from "node:path"; +import { createHash } from "node:crypto"; +import { homedir } from "node:os"; +import { connect } from "node:net"; +import { readConfig, readCredentials } from "./fp-config"; +import { configFile, otelAgentStateFile, otlpHealthFile } from "./fp-home"; +import { writeConfigFileAtomic } from "./safe-config-write"; +import { agentSessionsEnabled } from "./collector-config"; +import { introspectKey } from "./cloud-introspect"; +import { readJsonFile } from "./integrations"; +import { selectOne } from "./tui"; +import { + AGENT_LABELS, OTEL_AGENTS, agentConfigPath, agentOtelWrites, editAgentConfig, + getSetting, parseAgentConfig, vscodeInstalled, + type JsonObject, type OtelAgent, type SettingWrite, +} from "./otel-writers"; + +export const OTEL_ORG_OFF = "OTEL ingest is off for your org. An admin can turn it on in Settings → OpenTelemetry."; +export interface OtelResult { + lines: string[]; + exitCode: number; + events?: { name: "otel_enabled" | "otel_disabled"; agent: OtelAgent; local: boolean; content: boolean }[]; +} +interface AgentRecord { + file: string; + local: boolean; + content: boolean; + destination: string; + keys: { path: string[]; hadPrevious: boolean; previous?: unknown; private?: boolean }[]; + previousSessions?: boolean; + switchedSessions: boolean; +} +interface PrivateState { + original: string | null; + originalMode?: number; + writtenHash: string; + previous: SettingWrite[]; +} +export interface OtelOptions { + home?: string; + fetchImpl?: typeof fetch; + confirm?: (question: string) => Promise; + onEvent?: (name: string, properties: { agent: OtelAgent; local: boolean; content: boolean }) => void | Promise; +} + +function rawConfig(): JsonObject { + return readJsonFile(configFile()); +} +function object(value: unknown): JsonObject { + if (value === undefined) return {}; + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid OTEL configuration object."); + return value as JsonObject; +} +function records(raw: JsonObject): Partial> { + return object(object(raw.otel).agents) as Partial>; +} +function saveConfig(raw: JsonObject): void { + writeConfigFileAtomic(configFile(), `${JSON.stringify(raw, null, 2)}\n`, { backup: false }); +} +function digest(text: string): string { + return createHash("sha256").update(text).digest("hex"); +} +function privateState(agent: OtelAgent): PrivateState { + return JSON.parse(readFileSync(otelAgentStateFile(agent), "utf8")) as PrivateState; +} +function credential(): { key: string; origin: string } { + const ingest = readCredentials().ingest; + const key = process.env.FAILPROOFAI_INGEST_KEY || ingest?.key; + if (!key) throw new Error("No Cloud API key configured. Run `failproofai config` first."); + const url = new URL(process.env.FAILPROOFAI_INGEST_URL || ingest?.url || "https://app.befailproof.ai"); + if (!["https:", "http:"].includes(url.protocol) || url.username || url.password) throw new Error("Invalid Cloud ingest URL."); + return { key, origin: url.origin }; +} +function backupOnce(path: string, original: string | null): void { + if (existsSync(`${path}.failproofai-bak`)) return; + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(`${path}.failproofai-bak`, original ?? "", { flag: "wx", mode: 0o600 }); + chmodSync(`${path}.failproofai-bak`, 0o600); +} +function savePrivate(agent: OtelAgent, state: PrivateState): void { + const path = otelAgentStateFile(agent); + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + writeConfigFileAtomic(path, JSON.stringify(state), { backup: false, mode: 0o600 }); + chmodSync(path, 0o600); +} +function configureRelay(raw: JsonObject): void { + const collector = object(raw.collector); + raw.collector = collector; + const local = Object.values(records(raw)).some(record => record?.local); + if (local) collector.otlp = { ...object(collector.otlp), enabled: true }; + else if (collector.otlp) collector.otlp = { ...object(collector.otlp), enabled: false }; +} +function hasExternalOtel(agent: OtelAgent, config: JsonObject): boolean { + if (agent === "claude") return getSetting(config, ["env", "CLAUDE_CODE_ENABLE_TELEMETRY"]) === "1"; + if (agent === "codex") { + return ["exporter", "trace_exporter", "metrics_exporter"].some(k => { + const value = getSetting(config, ["otel", k]); + return value !== undefined && value !== "none" && value !== "statsig"; + }); + } + return getSetting(config, agent === "gemini" ? ["telemetry", "enabled"] : ["github.copilot.chat.otel.enabled"]) === true; +} +async function defaultConfirm(question: string): Promise { + return await selectOne({ + message: question, + choices: [ + { label: "No", value: false, hint: "keep transcript upload" }, + { label: "Yes", value: true, hint: "switch this agent to OTEL" }, + ], + }) === true; +} +function restoreAgent(agent: OtelAgent, raw: JsonObject): void { + const all = records(raw); + const record = all[agent]; + if (!record) return; + const state = privateState(agent); + if (existsSync(record.file)) { + const current = readFileSync(record.file, "utf8"); + if (digest(current) === state.writtenHash) { + if (state.original === null) unlinkSync(record.file); + else writeConfigFileAtomic(record.file, state.original, { backup: false, mode: state.originalMode }); + } else { + const restored = editAgentConfig(current, agent, state.previous); + writeConfigFileAtomic(record.file, restored, { backup: false, mode: state.originalMode }); + } + } + if (record.switchedSessions) { + const collector = object(raw.collector); + const agents = object(collector.agents); + const settings = object(agents[agent]); + if (record.previousSessions === undefined) delete settings.sessions; + else settings.sessions = record.previousSessions; + if (Object.keys(settings).length) agents[agent] = settings; + else delete agents[agent]; + if (Object.keys(agents).length) collector.agents = agents; + else delete collector.agents; + } + delete all[agent]; + object(raw.otel).agents = all; +} +async function enableAgent(agent: OtelAgent, yes: boolean, localFlag: boolean, content: boolean, options: OtelOptions): Promise { + const { key, origin } = credential(); + const raw = rawConfig(); + const previousConfig = structuredClone(raw); + const file = agentConfigPath(agent, options.home ?? homedir()); + if (agent === "copilot" && !vscodeInstalled(options.home ?? homedir())) { + return { lines: ["VS Code not found — skipping copilot"], exitCode: 0 }; + } + const existing = records(raw)[agent]; + // Gemini has no transcript source in this collector. + const transcriptsEnabled = agent !== "gemini" && agentSessionsEnabled(agent); + if (!existing && transcriptsEnabled && !yes) { + const question = `failproofai already sends ${AGENT_LABELS[agent]} sessions to Cloud. Send them with OTEL instead? Transcript upload for ${AGENT_LABELS[agent]} stops.`; + if (!await (options.confirm ?? defaultConfirm)(question)) return { lines: [`${AGENT_LABELS[agent]} unchanged (default No).`], exitCode: 0 }; + } + const local = localFlag || agent === "gemini" || agent === "copilot"; + const port = readConfig().collector.otlp?.port ?? 4318; + if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("collector.otlp.port must be between 1 and 65535."); + const destination = local ? `http://127.0.0.1:${port}` : origin; + const original = existsSync(file) ? readFileSync(file, "utf8") : null; + const parsed = parseAgentConfig(original ?? "", agent); + const writes = agentOtelWrites(agent, destination, key, local, content); + // Switching from Cloud to local stops owning header keys. Restore any keys + // from an earlier enable before applying the new (header-free) write set. + const oldState = existing ? privateState(agent) : null; + const released = oldState?.previous.filter(previous => + !writes.some(write => JSON.stringify(write.path) === JSON.stringify(previous.path))) ?? []; + const next = editAgentConfig(original ?? "", agent, [...released, ...writes]); + // On repeated enable keep the ORIGINAL previous values, not our last write. + const state = oldState ?? { + original, + originalMode: original === null ? undefined : statSync(file).mode & 0o777, + writtenHash: "", + previous: writes.map(w => ({ path: w.path, value: getSetting(parsed, w.path) })), + }; + if (existing && original !== null && digest(original) !== state.writtenHash) { + // Re-enabling must not turn the fast byte-restore path into permission to + // discard changes made after the previous enable. + state.original = editAgentConfig(original, agent, state.previous); + } + if (existing) { + state.previous = writes.map(write => state.previous.find(previous => + JSON.stringify(previous.path) === JSON.stringify(write.path)) ?? + { path: write.path, value: getSetting(parsed, write.path) }); + } + const collector = object(raw.collector); + const agents = object(collector.agents); + const previousSessions = object(agents[agent]).sessions as boolean | undefined; + const record: AgentRecord = { + file, local, content, destination, + keys: state.previous.map(w => ({ + path: w.path, + hadPrevious: w.value !== undefined, + // Prior agent credentials belong in owner-only state, never config.json. + ...(w.path.some(k => /headers|exporter/i.test(k)) ? { private: true } : { previous: w.value }), + })), + previousSessions: existing ? existing.previousSessions : previousSessions, + switchedSessions: existing ? existing.switchedSessions : transcriptsEnabled, + }; + if (record.switchedSessions) { + agents[agent] = { ...object(agents[agent]), sessions: false }; + collector.agents = agents; + } + raw.collector = collector; + const otel = object(raw.otel); + otel.agents = { ...records(raw), [agent]: record }; + raw.otel = otel; + configureRelay(raw); + // Recovery state and one-time backup precede the first agent-file write. + backupOnce(file, original); + savePrivate(agent, { ...state, writtenHash: digest(next) }); + try { + // Publish ownership BEFORE the agent settings so a killed CLI can always + // disable/restore what it wrote, including the one-path transcript choice. + saveConfig(raw); + writeConfigFileAtomic(file, next, { backup: false, mode: 0o600 }); + // Agent files now contain a credential, even if previously world-readable. + chmodSync(file, 0o600); + } catch (err) { + if (original !== null) writeConfigFileAtomic(file, original, { backup: false, mode: state.originalMode }); + else if (existsSync(file)) unlinkSync(file); + saveConfig(previousConfig); + throw err; + } + const lines = [`${AGENT_LABELS[agent]} OTEL enabled → ${local ? "local relay" : "Cloud"} (${content ? "text capture on" : "no content"}).`]; + if (local && !localFlag) lines.push(`${AGENT_LABELS[agent]} settings cannot carry an auth header; using the local relay automatically.`); + if (local) lines.push("The daemon picks this up within seconds — no restart. Use `failproofai otel status` to check the relay."); + const result = await introspectKey(origin, key, options.fetchImpl, 3000); + if (result.kind === "ok" && result.identity.otelIngest === false) lines.push(OTEL_ORG_OFF); + await options.onEvent?.("otel_enabled", { agent, local, content }); + return { lines, exitCode: 0, events: [{ name: "otel_enabled", agent, local, content }] }; +} +function relayListening(port: number): Promise { + return new Promise(resolve => { + const socket = connect({ host: "127.0.0.1", port }); + const done = (ok: boolean) => { socket.destroy(); resolve(ok); }; + socket.setTimeout(200); + socket.once("connect", () => done(true)); + socket.once("error", () => done(false)); + socket.once("timeout", () => done(false)); + }); +} +async function status(options: OtelOptions): Promise { + let org = "unknown"; + try { + const { origin, key } = credential(); + const result = await introspectKey(origin, key, options.fetchImpl, 3000); + if (result.kind === "ok" && result.identity.otelIngest !== undefined) org = result.identity.otelIngest ? "on" : "off"; + } catch { /* Status works on an unconfigured machine too. */ } + const all = records(rawConfig()); + const lines = [`Org OTEL ingest: ${org}`]; + for (const agent of OTEL_AGENTS) { + const record = all[agent]; + let parsed: JsonObject = {}; + let unreadable = false; + try { + const file = record?.file ?? agentConfigPath(agent, options.home ?? homedir()); + if (existsSync(file)) parsed = parseAgentConfig(readFileSync(file, "utf8"), agent); + } catch { unreadable = true; } + const on = hasExternalOtel(agent, parsed); + let managed = false; + if (record && on) { + try { + const state = privateState(agent); + const expected = parseAgentConfig(editAgentConfig(state.original ?? "", agent, + agentOtelWrites(agent, record.destination, credential().key, record.local, record.content)), agent); + managed = record.keys.every(k => JSON.stringify(getSetting(parsed, k.path)) === JSON.stringify(getSetting(expected, k.path))); + } catch { /* Missing recovery state or key. */ } + } + const dest = record?.destination ?? getSetting(parsed, agent === "claude" ? ["env", "OTEL_EXPORTER_OTLP_ENDPOINT"] : + agent === "gemini" ? ["telemetry", "otlpEndpoint"] : + agent === "copilot" ? ["github.copilot.chat.otel.otlpEndpoint"] : ["otel", "exporter", "otlp-http", "endpoint"]); + const destination = dest ? `${record?.local ? "local relay" : "Cloud"} (${dest})` : "none"; + lines.push(`${AGENT_LABELS[agent]}: ${unreadable ? "unreadable settings" : on ? managed ? "on by failproofai" : "on, not managed" : "off"} · destination: ${destination} · org: ${org}`); + } + let health: JsonObject = {}; + try { health = JSON.parse(readFileSync(otlpHealthFile(), "utf8")) as JsonObject; } catch { /* No relay yet. */ } + const requested = readConfig().collector.otlp?.enabled === true; + const live = requested && health.relay === "listening" && await relayListening(readConfig().collector.otlp?.port ?? 4318); + lines.push(`Relay: ${live ? "listening on 127.0.0.1" : requested ? health.relay === "port_busy" ? "off (port busy)" : "off (daemon not listening)" : "off"} · last upload: ${health.last_upload ?? "none"}`); + return { lines, exitCode: 0 }; +} + +export async function runOtelCommand(args: string[], options: OtelOptions = {}): Promise { + try { + const [verb, ...rest] = args; + const flags = new Set(verb === "enable" ? ["--local", "--no-content", "--yes"] : verb === "env" ? ["--local", "--service"] : []); + const unexpected = rest.find((arg, i) => arg.startsWith("-") && !flags.has(arg) && rest[i - 1] !== "--service"); + if (unexpected) throw new Error(`Unexpected argument: ${unexpected}`); + if (verb === "status") { + if (rest.length) throw new Error("status takes no arguments."); + return await status(options); + } + if (verb === "env") { + const { origin, key } = credential(); + const index = rest.indexOf("--service"); + const service = index >= 0 ? rest[index + 1] : "my-agent"; + if (!service || service.startsWith("-")) throw new Error("Missing value after --service."); + const consumed = rest.filter((_, i) => i !== index && i !== index + 1 || index < 0).filter(x => x !== "--local"); + if (consumed.length) throw new Error("env takes only --service and --local."); + const local = rest.includes("--local"); + const endpoint = local ? `http://127.0.0.1:${readConfig().collector.otlp?.port ?? 4318}` : origin; + const quote = (s: string) => `'${s.replace(/'/g, "'\\''")}'`; + const values = { + OTEL_EXPORTER_OTLP_ENDPOINT: endpoint, OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf", + ...(!local ? { OTEL_EXPORTER_OTLP_HEADERS: `Authorization=Bearer%20${encodeURIComponent(key)}` } : {}), + OTEL_SERVICE_NAME: service, + }; + return { lines: Object.entries(values).map(([k, v]) => `export ${k}=${quote(v)}`), exitCode: 0 }; + } + if (verb !== "enable" && verb !== "disable") throw new Error("Expected status, enable, disable or env. Run `failproofai otel --help`."); + const target = rest[0]; + if (!target || (target !== "all" && !OTEL_AGENTS.includes(target as OtelAgent))) throw new Error(`Choose an agent: ${OTEL_AGENTS.join(", ")}, all.`); + if (rest.slice(1).some(x => !flags.has(x))) throw new Error("Unexpected positional argument."); + const agents = target === "all" ? [...OTEL_AGENTS] : [target as OtelAgent]; + const result: OtelResult = { lines: [], exitCode: 0, events: [] }; + for (const agent of agents) { + if (verb === "enable") { + const enabled = await enableAgent(agent, rest.includes("--yes"), rest.includes("--local"), !rest.includes("--no-content"), options); + result.lines.push(...enabled.lines); + result.events?.push(...enabled.events ?? []); + } else { + const raw = rawConfig(); + const record = records(raw)[agent]; + if (!record) { result.lines.push(`${AGENT_LABELS[agent]}: no managed OTEL settings.`); continue; } + restoreAgent(agent, raw); + configureRelay(raw); + saveConfig(raw); + unlinkSync(otelAgentStateFile(agent)); + result.lines.push(`${AGENT_LABELS[agent]} OTEL disabled; previous settings restored.`); + await options.onEvent?.("otel_disabled", { agent, local: record.local, content: record.content }); + result.events?.push({ name: "otel_disabled", agent, local: record.local, content: record.content }); + } + } + return result; + } catch (err) { + return { lines: [err instanceof Error ? err.message : String(err)], exitCode: 1 }; + } +} diff --git a/src/hooks/otel-output.ts b/src/hooks/otel-output.ts new file mode 100644 index 000000000..b8753328f --- /dev/null +++ b/src/hooks/otel-output.ts @@ -0,0 +1,24 @@ +import { OTEL_ORG_OFF, type OtelResult } from "./otel-cli"; + +/** Presentation only: writers and shell-ready `env` output keep their contract. */ +export function otelReportLines(verb: string, result: OtelResult): string[] { + if (result.exitCode !== 0 || verb === "env") return result.lines; + return result.lines.flatMap(line => { + if (verb === "status") { + const glyph = line.includes("unreadable settings") ? "▲" : + /^[^:]+: (?:on\b|listening\b)/.test(line) ? "●" : "○"; + return [`${glyph} ${line}`]; + } + if (line === OTEL_ORG_OFF) return [ + "▲ OTEL ingest is off for your org.", + " An admin can turn it on in Settings → OpenTelemetry.", + ]; + if (line.startsWith("VS Code not found")) return [ + `▲ ${line}`, + " Install VS Code or its Copilot extension, then run `failproofai otel enable copilot`.", + ]; + if (/OTEL enabled|OTEL disabled/.test(line)) return [`✓ ${line}`]; + if (/unchanged|no managed OTEL/.test(line)) return [`○ ${line}`]; + return [` ${line}`]; + }); +} diff --git a/src/hooks/otel-writers.ts b/src/hooks/otel-writers.ts new file mode 100644 index 000000000..0f3c61e48 --- /dev/null +++ b/src/hooks/otel-writers.ts @@ -0,0 +1,225 @@ +/** + * Agent-owned OTEL configuration. JSONC edits preserve comments; the TOML + * scanner identifies tables without mistaking a multiline string for one. + * smol-toml validates the complete document before any modification. + */ +import { parse as parseToml } from "smol-toml"; +import { applyEdits, modify, parse as parseJsonc, type ParseError } from "jsonc-parser/lib/esm/main.js"; +import { homedir } from "node:os"; +import { resolve } from "node:path"; +import { accessSync, constants, readdirSync, statSync } from "node:fs"; + +export const OTEL_AGENTS = ["claude", "codex", "gemini", "copilot"] as const; +export type OtelAgent = typeof OTEL_AGENTS[number]; +export const AGENT_LABELS: Record = { + claude: "Claude Code", codex: "Codex", gemini: "Gemini CLI", copilot: "Copilot", +}; +export type JsonObject = Record; +export interface SettingWrite { path: string[]; value?: unknown } + +export function vscodeSettingsPath(home = homedir(), platform = process.platform): string { + if (platform === "darwin") return resolve(home, "Library/Application Support/Code/User/settings.json"); + if (platform === "win32") return resolve(process.env.APPDATA ?? resolve(home, "AppData/Roaming"), "Code/User/settings.json"); + return resolve(process.env.XDG_CONFIG_HOME ?? resolve(home, ".config"), "Code/User/settings.json"); +} + +export function agentConfigPath(agent: OtelAgent, home = homedir()): string { + return agent === "copilot" ? vscodeSettingsPath(home) : + resolve(home, `.${agent}`, agent === "codex" ? "config.toml" : "settings.json"); +} + +/** + * Fixed install locations checked after PATH. A mutable seam so tests can run + * on a machine that really has VS Code installed: these paths are system-wide, + * so no temp HOME or PATH hides them. + */ +export const vscodeProbe = { + appPaths(home: string, platform: NodeJS.Platform): string[] { + return platform === "darwin" ? [ + "/Applications/Visual Studio Code.app/Contents/MacOS/Electron", + "/Applications/Visual Studio Code - Insiders.app/Contents/MacOS/Electron", + resolve(home, "Applications/Visual Studio Code.app/Contents/MacOS/Electron"), + ] : platform === "win32" ? [ + resolve(process.env.LOCALAPPDATA ?? resolve(home, "AppData/Local"), "Programs/Microsoft VS Code/Code.exe"), + resolve(process.env.ProgramFiles ?? "C:/Program Files", "Microsoft VS Code/Code.exe"), + ] : ["/usr/share/code/code", "/opt/visual-studio-code/code"]; + }, +}; + +export function vscodeInstalled(home = homedir(), platform = process.platform): boolean { + const executable = (path: string) => { + try { + accessSync(path, platform === "win32" ? constants.F_OK : constants.X_OK); + return statSync(path).isFile(); + } catch { return false; } + }; + const names = platform === "win32" ? ["code.cmd", "code-insiders.cmd", "Code.exe", "Code - Insiders.exe"] : ["code", "code-insiders"]; + if ((process.env.PATH ?? "").split(platform === "win32" ? ";" : ":") + .some(dir => names.some(name => executable(resolve(dir, name))))) return true; + if (vscodeProbe.appPaths(home, platform).some(executable)) return true; + return [".vscode", ".vscode-insiders"].some(folder => { + try { + return readdirSync(resolve(home, folder, "extensions"), { withFileTypes: true }) + .some(entry => entry.isDirectory() && /^github\.copilot(?:-chat)?-\d/i.test(entry.name)); + } catch { return false; } + }); +} + +export function parseAgentConfig(text: string, agent: OtelAgent): JsonObject { + if (agent === "codex") return parseToml(text) as JsonObject; + if (!text.trim()) return {}; + const errors: ParseError[] = []; + const value: unknown = parseJsonc(text, errors, { allowTrailingComma: true }); + if (errors.length || !value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Refusing to modify an invalid agent settings object."); + } + return value as JsonObject; +} + +export function getSetting(object: JsonObject, path: string[]): unknown { + let value: unknown = object; + for (const key of path) { + if (!value || typeof value !== "object") return undefined; + value = (value as JsonObject)[key]; + } + return value; +} + +function setSetting(object: JsonObject, path: string[], value: unknown): void { + const [key, ...rest] = path; + if (!rest.length) { + if (value === undefined) delete object[key]; + else object[key] = value; + return; + } + const child = object[key]; + if (child !== undefined && (!child || typeof child !== "object" || Array.isArray(child))) { + throw new Error(`Refusing to replace a non-object ${key} setting.`); + } + const nested = (child ?? {}) as JsonObject; + setSetting(nested, rest, value); + if (Object.keys(nested).length) object[key] = nested; + else delete object[key]; +} + +function tomlValue(value: unknown): string { + if (typeof value === "string") return JSON.stringify(value); + if (typeof value === "boolean" || typeof value === "number") return String(value); + if (value instanceof Date) return value.toISOString(); + if (Array.isArray(value)) return `[${value.map(tomlValue).join(", ")}]`; + if (value && typeof value === "object") return `{ ${Object.entries(value).map(([k, v]) => `${JSON.stringify(k)} = ${tomlValue(v)}`).join(", ")} }`; + throw new Error("Unsupported TOML value."); +} + +function tables(text: string): { start: number; otel: boolean; rootHeader?: string }[] { + const result: { start: number; otel: boolean; rootHeader?: string }[] = []; + let multiline: string | null = null; + let offset = 0; + for (const line of text.match(/[^\n]*\n|[^\n]+$/g) ?? []) { + if (!multiline) { + const header = /^\s*(\[\[?.*?\]\]?)\s*(?:#.*)?(?:\r?\n)?$/.exec(line); + if (header) { + const sample = parseToml(`${header[1]}\n__failproofai_probe = true\n`); + const root = Object.keys(sample).length === 1 && Object.hasOwn(sample, "otel") && + Object.hasOwn(sample.otel as object, "__failproofai_probe"); + result.push({ start: offset, otel: Object.hasOwn(sample, "otel"), rootHeader: root ? line.replace(/\r?\n$/, "") : undefined }); + } + } + let quote: string | null = null; + for (let i = 0; i < line.length; i++) { + if (multiline) { + if (line.startsWith(multiline, i) && (multiline === "'''" || line[i - 1] !== "\\")) { + i += 2; multiline = null; + } + } else if (quote) { + if (quote === '"' && line[i] === "\\") i++; + else if (line[i] === quote) quote = null; + } else if (line[i] === "#") break; + else if (line.startsWith('"""', i) || line.startsWith("'''", i)) { + multiline = line.slice(i, i + 3); i += 2; + } else if (line[i] === '"' || line[i] === "'") quote = line[i]; + } + offset += line.length; + } + return result; +} + +export function editAgentConfig(text: string, agent: OtelAgent, writes: SettingWrite[]): string { + const parsed = parseAgentConfig(text, agent); + if (agent !== "codex") { + let next = text.trim() ? text : "{}\n"; + for (const write of writes) { + setSetting(parsed, write.path, write.value); + next = applyEdits(next, modify(next, write.path, write.value, { + formattingOptions: { insertSpaces: true, tabSize: 2 }, + })); + } + for (const root of ["env", "telemetry"]) { + if (getSetting(parsed, [root]) === undefined) { + next = applyEdits(next, modify(next, [root], undefined, {})); + } + } + return next; + } + for (const write of writes) setSetting(parsed, write.path, write.value); + const headers = tables(text); + if (Object.hasOwn(parseToml(text.slice(0, headers[0]?.start ?? text.length)), "otel")) { + throw new Error("Codex OTEL must use an [otel] table, not a root dotted or inline assignment."); + } + const newline = text.includes("\r\n") ? "\r\n" : "\n"; + const otel = parsed.otel as JsonObject | undefined; + const rootHeader = headers.find(h => h.rootHeader)?.rootHeader ?? "[otel]"; + const replacement = otel ? `${rootHeader}${newline}${Object.entries(otel).map(([k, v]) => `${JSON.stringify(k)} = ${tomlValue(v)}${newline}`).join("")}` : ""; + let next = text; + const ranges = headers.map((h, i) => { + const end = headers[i + 1]?.start ?? text.length; + // Comments introducing the NEXT table are not part of the table we own. + const tail = /(?<=\n)(?:[ \t]*(?:#[^\r\n]*)?\r?\n)+$/.exec(text.slice(h.start, end)); + return { ...h, end: tail ? h.start + tail.index : end }; + }).filter(h => h.otel); + for (let i = ranges.length - 1; i >= 0; i--) { + const range = ranges[i]; + next = next.slice(0, range.start) + (i === 0 ? replacement : "") + next.slice(range.end); + } + if (!ranges.length && replacement) next += `${next && !next.endsWith("\n") ? newline : ""}${replacement}`; + parseToml(next); + return next; +} + +export function agentOtelWrites(agent: OtelAgent, endpoint: string, key: string, local: boolean, content: boolean): SettingWrite[] { + const auth = local ? {} : { Authorization: `Bearer ${key}` }; + const env: JsonObject = { + CLAUDE_CODE_ENABLE_TELEMETRY: "1", + OTEL_LOGS_EXPORTER: "otlp", OTEL_METRICS_EXPORTER: "otlp", + OTEL_EXPORTER_OTLP_ENDPOINT: endpoint, OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf", + ...(!local ? { OTEL_EXPORTER_OTLP_HEADERS: `Authorization=Bearer%20${encodeURIComponent(key)}` } : {}), + OTEL_LOG_USER_PROMPTS: content ? "1" : "0", + OTEL_LOG_TOOL_DETAILS: content ? "1" : "0", + // Tool output needs OTEL_LOG_TOOL_CONTENT; assistant responses follow OTEL_LOG_USER_PROMPTS, so omit OTEL_LOG_ASSISTANT_RESPONSES. + OTEL_LOG_TOOL_CONTENT: content ? "1" : "0", + }; + if (agent === "claude") { + for (const signal of ["LOGS", "METRICS"]) { + env[`OTEL_EXPORTER_OTLP_${signal}_ENDPOINT`] = `${endpoint}/v1/${signal.toLowerCase()}`; + env[`OTEL_EXPORTER_OTLP_${signal}_PROTOCOL`] = "http/protobuf"; + if (!local) env[`OTEL_EXPORTER_OTLP_${signal}_HEADERS`] = env.OTEL_EXPORTER_OTLP_HEADERS; + } + return Object.entries(env).map(([k, value]) => ({ path: ["env", k], value })); + } + if (agent === "codex") { + const exporter = (signal: string) => ({ "otlp-http": { + endpoint: `${endpoint}/v1/${signal}`, protocol: "json", ...(!local ? { headers: auth } : {}), + } }); + return Object.entries({ + exporter: exporter("logs"), trace_exporter: exporter("traces"), metrics_exporter: exporter("metrics"), + log_user_prompt: content, + log_agent_responses: content, + }).map(([k, value]) => ({ path: ["otel", k], value })); + } + if (agent === "gemini") return Object.entries({ + enabled: true, target: "local", otlpEndpoint: endpoint, otlpProtocol: "http", logPrompts: content, useCollector: true, outfile: undefined, + }).map(([k, value]) => ({ path: ["telemetry", k], value })); + return Object.entries({ + enabled: true, exporterType: "otlp-http", otlpEndpoint: endpoint, captureContent: content, + }).map(([k, value]) => ({ path: [`github.copilot.chat.otel.${k}`], value })); +} diff --git a/src/hooks/safe-config-write.ts b/src/hooks/safe-config-write.ts index ecf7cf603..74d1593ea 100644 --- a/src/hooks/safe-config-write.ts +++ b/src/hooks/safe-config-write.ts @@ -53,6 +53,8 @@ export function configBackupPath(path: string): string { /** Hooks for tests to simulate an interruption at the one step that matters. */ export interface AtomicWriteDeps { + /** Explicit permissions for a newly credential-bearing file. */ + mode?: number; rename?: (from: string, to: string) => void; /** Keep `.failproofai-backup` of the previous version (default true). */ backup?: boolean; @@ -144,9 +146,9 @@ export function writeConfigFileAtomic(path: string, content: string, deps: Atomi let mode: number | undefined; try { // A link being replaced lends nothing: its target's mode is someone else's. - mode = replacingLink ? undefined : statSync(target).mode & 0o777; + mode = deps.mode ?? (replacingLink ? undefined : statSync(target).mode & 0o777); } catch { - mode = undefined; // a new file: the process umask decides, as writeFileSync would + mode = deps.mode; // otherwise the process umask decides, as writeFileSync would } const temporary = join(dir, `.${basename(target)}.failproofai-${process.pid}-${randomBytes(4).toString("hex")}.tmp`);