From 9685b38e86b7a162691d084fe6dbc4f921839244 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:35:31 +0000 Subject: [PATCH 1/8] feat: Add `ollama` instrumentation --- AGENTS.md | 2 + e2e/README.md | 7 +- e2e/config/pr-comment-scenarios.json | 23 + e2e/helpers/scenario-harness.ts | 3 + .../ollama-v0.6-latest.cassette.json | 228 +++++++ .../__cassettes__/ollama-v0.6.cassette.json | 228 +++++++ .../ollama-v0.6-auto.span-tree.json | 329 ++++++++++ .../ollama-v0.6-auto.span-tree.txt | 257 ++++++++ .../ollama-v0.6-latest-auto.span-tree.json | 329 ++++++++++ .../ollama-v0.6-latest-auto.span-tree.txt | 257 ++++++++ .../ollama-v0.6-latest-wrapped.span-tree.json | 329 ++++++++++ .../ollama-v0.6-latest-wrapped.span-tree.txt | 257 ++++++++ .../ollama-v0.6-wrapped.span-tree.json | 329 ++++++++++ .../ollama-v0.6-wrapped.span-tree.txt | 257 ++++++++ .../ollama-instrumentation/assertions.ts | 227 +++++++ .../ollama-instrumentation/constants.mjs | 5 + .../ollama-instrumentation/package.json | 18 + .../ollama-instrumentation/pnpm-lock.yaml | 32 + .../ollama-instrumentation/scenario.impl.mjs | 254 ++++++++ .../ollama-instrumentation/scenario.mjs | 10 + .../ollama-instrumentation/scenario.test.ts | 76 +++ .../ollama-instrumentation/scenario.ts | 10 + js/src/auto-instrumentations/configs/all.ts | 2 + .../auto-instrumentations/configs/ollama.ts | 32 + js/src/auto-instrumentations/index.ts | 1 + js/src/exports.ts | 1 + .../instrumentation/braintrust-plugin.test.ts | 27 + js/src/instrumentation/braintrust-plugin.ts | 13 + js/src/instrumentation/config.ts | 3 + .../plugins/instrumentation-names.test.ts | 2 + .../plugins/ollama-channels.ts | 47 ++ .../plugins/ollama-plugin.test.ts | 321 ++++++++++ .../instrumentation/plugins/ollama-plugin.ts | 562 ++++++++++++++++++ js/src/span-origin.ts | 1 + js/src/vendor-sdk-types/ollama.ts | 117 ++++ js/src/wrappers/ollama.test.ts | 62 ++ js/src/wrappers/ollama.ts | 118 ++++ 37 files changed, 4773 insertions(+), 3 deletions(-) create mode 100644 e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6-latest.cassette.json create mode 100644 e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6.cassette.json create mode 100644 e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json create mode 100644 e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt create mode 100644 e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json create mode 100644 e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt create mode 100644 e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json create mode 100644 e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt create mode 100644 e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json create mode 100644 e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt create mode 100644 e2e/scenarios/ollama-instrumentation/assertions.ts create mode 100644 e2e/scenarios/ollama-instrumentation/constants.mjs create mode 100644 e2e/scenarios/ollama-instrumentation/package.json create mode 100644 e2e/scenarios/ollama-instrumentation/pnpm-lock.yaml create mode 100644 e2e/scenarios/ollama-instrumentation/scenario.impl.mjs create mode 100644 e2e/scenarios/ollama-instrumentation/scenario.mjs create mode 100644 e2e/scenarios/ollama-instrumentation/scenario.test.ts create mode 100644 e2e/scenarios/ollama-instrumentation/scenario.ts create mode 100644 js/src/auto-instrumentations/configs/ollama.ts create mode 100644 js/src/instrumentation/plugins/ollama-channels.ts create mode 100644 js/src/instrumentation/plugins/ollama-plugin.test.ts create mode 100644 js/src/instrumentation/plugins/ollama-plugin.ts create mode 100644 js/src/vendor-sdk-types/ollama.ts create mode 100644 js/src/wrappers/ollama.test.ts create mode 100644 js/src/wrappers/ollama.ts diff --git a/AGENTS.md b/AGENTS.md index c30331676..8bb1399b7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,6 +55,8 @@ pnpm run test:e2e:record # Re-record provider cassettes and update snap When adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output. +New instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary. + Span-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair. **From repo root:** diff --git a/e2e/README.md b/e2e/README.md index ebe595db9..5d2aa5912 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -105,6 +105,7 @@ Provider credentials are only required when recording or explicitly running live - `CURSOR_API_KEY` - `OPENROUTER_API_KEY` - `MISTRAL_API_KEY` +- `OLLAMA_API_KEY` - `HUGGINGFACE_API_KEY` - `COHERE_API_KEY` - `GROQ_API_KEY` @@ -165,7 +166,7 @@ pnpm --filter=@braintrust/js-e2e-tests run test:e2e:record -- ANTHROPIC_API_KEY=... AWS_BEARER_TOKEN_BEDROCK=... \ OPENAI_API_KEY=... GEMINI_API_KEY=... \ COHERE_API_KEY=... GROQ_API_KEY=... HUGGINGFACE_API_KEY=... \ -MISTRAL_API_KEY=... OPENROUTER_API_KEY=... \ +MISTRAL_API_KEY=... OLLAMA_API_KEY=... OPENROUTER_API_KEY=... \ CURSOR_API_KEY=... \ pnpm --filter=@braintrust/js-e2e-tests run test:e2e:record ``` @@ -173,7 +174,7 @@ CURSOR_API_KEY=... \ After recording, run again **without any provider keys** to confirm the cassette is sufficient: ```bash -unset ANTHROPIC_API_KEY AWS_BEARER_TOKEN_BEDROCK OPENAI_API_KEY GEMINI_API_KEY GOOGLE_API_KEY GOOGLE_GENAI_API_KEY COHERE_API_KEY GROQ_API_KEY HUGGINGFACE_API_KEY MISTRAL_API_KEY OPENROUTER_API_KEY CURSOR_API_KEY +unset ANTHROPIC_API_KEY AWS_BEARER_TOKEN_BEDROCK OPENAI_API_KEY GEMINI_API_KEY GOOGLE_API_KEY GOOGLE_GENAI_API_KEY COHERE_API_KEY GROQ_API_KEY HUGGINGFACE_API_KEY MISTRAL_API_KEY OLLAMA_API_KEY OPENROUTER_API_KEY CURSOR_API_KEY pnpm --filter=@braintrust/js-e2e-tests run test:e2e ``` @@ -185,7 +186,7 @@ After any successful record run, stale cassette variants are cleaned only inside These scenarios have cassette wiring in place and will use cassettes once they're recorded: -`anthropic-bedrock-instrumentation`, `anthropic-instrumentation`, `openai-instrumentation`, `openai-codex-instrumentation`, `ai-sdk-instrumentation`, `ai-sdk-otel-export`, `claude-agent-sdk-instrumentation`, `cohere-instrumentation`, `cursor-sdk-instrumentation`, `github-copilot-instrumentation`, `google-adk-instrumentation`, `google-genai-instrumentation`, `groq-instrumentation`, `huggingface-instrumentation`, `mistral-instrumentation`, `openrouter-agent-instrumentation`, `openrouter-instrumentation`, `wrap-langchain-js-traces`. +`anthropic-bedrock-instrumentation`, `anthropic-instrumentation`, `openai-instrumentation`, `openai-codex-instrumentation`, `ai-sdk-instrumentation`, `ai-sdk-otel-export`, `claude-agent-sdk-instrumentation`, `cohere-instrumentation`, `cursor-sdk-instrumentation`, `github-copilot-instrumentation`, `google-adk-instrumentation`, `google-genai-instrumentation`, `groq-instrumentation`, `huggingface-instrumentation`, `mistral-instrumentation`, `ollama-instrumentation`, `openrouter-agent-instrumentation`, `openrouter-instrumentation`, `wrap-langchain-js-traces`. ### Cassette format diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index 65a85ab6e..a5470a3ab 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -397,6 +397,29 @@ } ] }, + { + "scenarioDirName": "ollama-instrumentation", + "label": "Ollama Instrumentation", + "metadataScenario": "ollama-instrumentation", + "variants": [ + { + "variantKey": "ollama-v0.6-wrapped", + "label": "v0.6 pinned wrapped" + }, + { + "variantKey": "ollama-v0.6-auto", + "label": "v0.6 pinned auto-hook" + }, + { + "variantKey": "ollama-v0.6-latest-wrapped", + "label": "v0.6 latest wrapped" + }, + { + "variantKey": "ollama-v0.6-latest-auto", + "label": "v0.6 latest auto-hook" + } + ] + }, { "scenarioDirName": "cohere-instrumentation", "label": "Cohere Instrumentation", diff --git a/e2e/helpers/scenario-harness.ts b/e2e/helpers/scenario-harness.ts index b41e3a8be..8390f5a87 100644 --- a/e2e/helpers/scenario-harness.ts +++ b/e2e/helpers/scenario-harness.ts @@ -315,6 +315,7 @@ function getCassetteServerRoutes(): CassetteServerRoute[] { upstreamOrigin: "https://router.huggingface.co", }, { prefix: "/mistral", upstreamOrigin: "https://api.mistral.ai" }, + { prefix: "/ollama", upstreamOrigin: "https://ollama.com" }, { prefix: "/openai", upstreamOrigin: "https://api.openai.com" }, { prefix: "/openrouter", upstreamOrigin: "https://openrouter.ai" }, ]; @@ -349,6 +350,7 @@ function getCassetteEnv(wiring: ActiveCassetteWiring): Record { HUGGINGFACE_ROUTER_BASE_URL: `${serverUrl}/huggingface-router`, MISTRAL_API_URL: `${serverUrl}/mistral`, MISTRAL_BASE_URL: `${serverUrl}/mistral`, + OLLAMA_HOST: `${serverUrl}/ollama`, OPENAI_BASE_URL: `${serverUrl}/openai/v1`, OPENROUTER_BASE_URL: `${serverUrl}/openrouter/api/v1`, }; @@ -398,6 +400,7 @@ const CASSETTE_PROVIDER_KEYS: Array<{ { envVars: ["GROQ_API_KEY"], placeholder: "gsk_cassette-placeholder" }, { envVars: ["HUGGINGFACE_API_KEY"], placeholder: "hf_cassette-placeholder" }, { envVars: ["MISTRAL_API_KEY"], placeholder: "cassette-placeholder" }, + { envVars: ["OLLAMA_API_KEY"], placeholder: "cassette-placeholder" }, { envVars: ["OPENAI_API_KEY"], placeholder: "sk-cassette-placeholder" }, { envVars: ["OPENROUTER_API_KEY"], diff --git a/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6-latest.cassette.json b/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6-latest.cassette.json new file mode 100644 index 000000000..f6c999c5e --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6-latest.cassette.json @@ -0,0 +1,228 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "2a86bcda6598aa82", + "matchKey": "POST ollama.com/api/chat", + "recordedAt": "2026-07-30T14:17:27.786Z", + "request": { + "body": { + "kind": "json", + "value": { + "messages": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "stream": false, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/chat" + }, + "response": { + "body": { + "kind": "json", + "value": { + "created_at": "2026-07-30T14:17:27.836768072Z", + "done": true, + "done_reason": "stop", + "eval_count": 40, + "message": { + "content": "OK", + "role": "assistant", + "thinking": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK." + }, + "model": "gpt-oss:20b", + "prompt_eval_count": 67, + "total_duration": 820667686 + } + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-length": "349", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:27 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-c66f99321115f5f107758a3fad47a00f-bcbe1f3d1e4384ae-00", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "c66f99321115f5f107758a3fad47a00f/13600342272066684078", + "x-frame-options": "DENY", + "x-request-id": "127f9275-9630-4810-9a7c-6b72c370cb3c" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "923a3e6055148564", + "matchKey": "POST ollama.com/api/chat", + "recordedAt": "2026-07-30T14:17:29.278Z", + "request": { + "body": { + "kind": "json", + "value": { + "messages": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "stream": true, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/chat" + }, + "response": { + "body": { + "kind": "text", + "value": "{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.34967424Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"The\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.359378568Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" user\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.390259604Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" says\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.396460009Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\":\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.396601569Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.3975552Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"Reply\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.405247206Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" with\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.407624928Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.418610508Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.464847596Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.465359817Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" So\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.465435117Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" we\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.465480977Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" must\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.478837208Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" output\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.478898448Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.488134706Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" the\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.496465303Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" word\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.532802133Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.532881813Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.537893028Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"\\\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.547788406Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" No\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.555467652Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" extra\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.571374226Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" spaces\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.571476426Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\",\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.587682829Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" no\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.592180173Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" newline\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.593067434Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"?\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.606547985Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" They\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.615570543Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" said\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.645977628Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.646838859Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"Reply\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.655935567Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" with\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.665154714Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.673848022Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.689652135Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.689813995Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" Usually\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.702106365Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" we\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.710015642Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" output\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.717246658Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" the\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.726397266Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" word\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.735010853Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.789715719Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" Probably\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.790339829Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" just\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.790963009Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.79104392Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.79129428Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"\\\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.79150011Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" No\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.800693548Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" punctuation\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.807873884Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.808396504Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" So\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.819012393Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" output\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.840200251Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\":\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.840856441Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.843193773Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:29.248242423Z\",\"message\":{\"role\":\"assistant\",\"content\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:29.255514869Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:29.330188961Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\"},\"done\":true,\"done_reason\":\"stop\",\"total_duration\":1363219361,\"prompt_eval_count\":67,\"eval_count\":65}\n" + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:28 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-4455c46106092326278890284531f2ca-df664b0c9e5442e6-00", + "transfer-encoding": "chunked", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "4455c46106092326278890284531f2ca/16097636435650429670", + "x-frame-options": "DENY", + "x-request-id": "55fbd7c8-7512-4361-a3ae-8f2810927893" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 0, + "id": "99f1d8c4b0c43219", + "matchKey": "POST ollama.com/api/generate", + "recordedAt": "2026-07-30T14:17:33.894Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "prompt": "Reply with exactly OK.", + "stream": false, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/generate" + }, + "response": { + "body": { + "kind": "json", + "value": { + "created_at": "2026-07-30T14:17:33.941708086Z", + "done": true, + "done_reason": "stop", + "eval_count": 40, + "model": "gpt-oss:20b", + "prompt_eval_count": 67, + "response": "OK", + "thinking": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "total_duration": 4489076853 + } + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-length": "320", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:33 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-106b492de1a5a0ce278890284531f004-f53ba4690e6cf0bc-00", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "106b492de1a5a0ce278890284531f004/17670898334038814908", + "x-frame-options": "DENY", + "x-request-id": "203de0b5-0587-4e63-88c6-e083c1074a17" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "54471dd63ef97520", + "matchKey": "POST ollama.com/api/generate", + "recordedAt": "2026-07-30T14:17:34.958Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "prompt": "Reply with exactly STREAM.", + "stream": true, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/generate" + }, + "response": { + "body": { + "kind": "text", + "value": "{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.389627377Z\",\"response\":\"\",\"thinking\":\"The\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.395533242Z\",\"response\":\"\",\"thinking\":\" user\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.403797509Z\",\"response\":\"\",\"thinking\":\" says\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.412127386Z\",\"response\":\"\",\"thinking\":\":\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.420263983Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.431670083Z\",\"response\":\"\",\"thinking\":\"Reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.439038379Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.445798215Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.453676261Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.462879529Z\",\"response\":\"\",\"thinking\":\".\\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.48754039Z\",\"response\":\"\",\"thinking\":\" So\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.497012667Z\",\"response\":\"\",\"thinking\":\" we\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.503352273Z\",\"response\":\"\",\"thinking\":\" must\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.508533877Z\",\"response\":\"\",\"thinking\":\" output\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.518284895Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.524960831Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.532656867Z\",\"response\":\"\",\"thinking\":\" word\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.555859397Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.563573723Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.564276844Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.566337645Z\",\"response\":\"\",\"thinking\":\" No\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.573933902Z\",\"response\":\"\",\"thinking\":\" extra\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.581418518Z\",\"response\":\"\",\"thinking\":\" spaces\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.589888165Z\",\"response\":\"\",\"thinking\":\",\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.601848775Z\",\"response\":\"\",\"thinking\":\" no\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.610821713Z\",\"response\":\"\",\"thinking\":\" newline\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.615298726Z\",\"response\":\"\",\"thinking\":\"?\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.624505524Z\",\"response\":\"\",\"thinking\":\" They\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.632406001Z\",\"response\":\"\",\"thinking\":\" said\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.642595159Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.650027446Z\",\"response\":\"\",\"thinking\":\"Reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.661598875Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.669144222Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.675096107Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.686189736Z\",\"response\":\"\",\"thinking\":\".\\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.692248381Z\",\"response\":\"\",\"thinking\":\" Usually\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.705089872Z\",\"response\":\"\",\"thinking\":\" we\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.709504965Z\",\"response\":\"\",\"thinking\":\" output\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.720336364Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.72721519Z\",\"response\":\"\",\"thinking\":\" word\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.735638087Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.747191607Z\",\"response\":\"\",\"thinking\":\" Probably\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.760642398Z\",\"response\":\"\",\"thinking\":\" just\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.778758373Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.779113164Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.798139999Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.799441891Z\",\"response\":\"\",\"thinking\":\" No\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.799505571Z\",\"response\":\"\",\"thinking\":\" punctuation\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.808817408Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.827693204Z\",\"response\":\"\",\"thinking\":\" So\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.837738383Z\",\"response\":\"\",\"thinking\":\" final\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.849834023Z\",\"response\":\"\",\"thinking\":\" answer\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.856697219Z\",\"response\":\"\",\"thinking\":\":\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.863521814Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.865494336Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.927466028Z\",\"response\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.937183716Z\",\"response\":\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:35.009953287Z\",\"response\":\"\",\"done\":true,\"done_reason\":\"stop\",\"total_duration\":946827533,\"prompt_eval_count\":67,\"eval_count\":66}\n" + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:34 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-e8e48c6fbc8884675481ff7328ae2d0b-7a094f57d5536a16-00", + "transfer-encoding": "chunked", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "e8e48c6fbc8884675481ff7328ae2d0b/8793646986077366806", + "x-frame-options": "DENY", + "x-request-id": "97ff230e-37fe-4bb0-a698-396b552dcdcb" + }, + "status": 200, + "statusText": "OK" + } + } + ], + "meta": { + "createdAt": "2026-07-30T14:15:13.600Z" + } +} diff --git a/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6.cassette.json b/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6.cassette.json new file mode 100644 index 000000000..e092efa99 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6.cassette.json @@ -0,0 +1,228 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "2a86bcda6598aa82", + "matchKey": "POST ollama.com/api/chat", + "recordedAt": "2026-07-30T14:17:15.919Z", + "request": { + "body": { + "kind": "json", + "value": { + "messages": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "stream": false, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/chat" + }, + "response": { + "body": { + "kind": "json", + "value": { + "created_at": "2026-07-30T14:17:15.968759871Z", + "done": true, + "done_reason": "stop", + "eval_count": 40, + "message": { + "content": "OK", + "role": "assistant", + "thinking": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK." + }, + "model": "gpt-oss:20b", + "prompt_eval_count": 67, + "total_duration": 2759051201 + } + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-length": "350", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:15 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-499b9cbcd8010b751b766ba772b23e2e-45c927d9e2a30383-00", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "499b9cbcd8010b751b766ba772b23e2e/5028594275699590019", + "x-frame-options": "DENY", + "x-request-id": "69feef07-e84b-461c-b965-82ca63d5cd59" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "923a3e6055148564", + "matchKey": "POST ollama.com/api/chat", + "recordedAt": "2026-07-30T14:17:16.947Z", + "request": { + "body": { + "kind": "json", + "value": { + "messages": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "stream": true, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/chat" + }, + "response": { + "body": { + "kind": "text", + "value": "{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.420595067Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"The\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.428180554Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" user\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.434645149Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" says\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.443487256Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\":\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.454888486Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.464434994Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"Reply\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.468907788Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" with\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.475863774Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.486244792Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.494834779Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.501858345Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" So\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.513830845Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" the\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.522514833Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" assistant\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.528664688Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" should\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.536021444Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" reply\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.54387051Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" with\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.553723509Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.561350065Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" the\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.588566508Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" word\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.598017396Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.60322265Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.609424945Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"\\\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.619468064Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" No\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.643468554Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" other\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.644496975Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" characters\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.65059612Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\",\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.665049332Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" no\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.666145933Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" punctuation\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.675696741Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\",\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.685230109Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" no\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.68602731Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" whitespace\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.702263893Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.718099466Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" Just\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.764431305Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.794734681Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.843230121Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"\\\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.895195035Z\",\"message\":{\"role\":\"assistant\",\"content\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.897310446Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.996437479Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\"},\"done\":true,\"done_reason\":\"stop\",\"total_duration\":913140194,\"prompt_eval_count\":67,\"eval_count\":47}\n" + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:16 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-222faed9a3a0e3c91b766ba772b23075-d9f7f454c56193bb-00", + "transfer-encoding": "chunked", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "222faed9a3a0e3c91b766ba772b23075/15706290870403830715", + "x-frame-options": "DENY", + "x-request-id": "56b16227-d494-42bb-abf8-9db1b40d9f6d" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 0, + "id": "99f1d8c4b0c43219", + "matchKey": "POST ollama.com/api/generate", + "recordedAt": "2026-07-30T14:17:18.001Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "prompt": "Reply with exactly OK.", + "stream": false, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/generate" + }, + "response": { + "body": { + "kind": "json", + "value": { + "created_at": "2026-07-30T14:17:18.050054605Z", + "done": true, + "done_reason": "stop", + "eval_count": 40, + "model": "gpt-oss:20b", + "prompt_eval_count": 67, + "response": "OK", + "thinking": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "total_duration": 947970164 + } + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-length": "320", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:18 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-163ade5657ad33301b766ba772b2374e-25ddb75508a07d45-00", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "163ade5657ad33301b766ba772b2374e/2728538525101227333", + "x-frame-options": "DENY", + "x-request-id": "ccf7a2c3-d525-44ad-b0fa-cfa55a512239" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "54471dd63ef97520", + "matchKey": "POST ollama.com/api/generate", + "recordedAt": "2026-07-30T14:17:19.889Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "prompt": "Reply with exactly STREAM.", + "stream": true, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/generate" + }, + "response": { + "body": { + "kind": "text", + "value": "{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.599724432Z\",\"response\":\"\",\"thinking\":\"The\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.601082173Z\",\"response\":\"\",\"thinking\":\" user\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.601825504Z\",\"response\":\"\",\"thinking\":\" says\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.603175275Z\",\"response\":\"\",\"thinking\":\":\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.604566096Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.605298347Z\",\"response\":\"\",\"thinking\":\"Reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.605716007Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657090971Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657144751Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.65718348Z\",\"response\":\"\",\"thinking\":\".\\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657229391Z\",\"response\":\"\",\"thinking\":\" So\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657271011Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657398341Z\",\"response\":\"\",\"thinking\":\" assistant\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657473381Z\",\"response\":\"\",\"thinking\":\" should\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657507981Z\",\"response\":\"\",\"thinking\":\" reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657562171Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657608451Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657649941Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657975201Z\",\"response\":\"\",\"thinking\":\" word\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658022951Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658079621Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658124381Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658174071Z\",\"response\":\"\",\"thinking\":\" No\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658224241Z\",\"response\":\"\",\"thinking\":\" other\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658275511Z\",\"response\":\"\",\"thinking\":\" characters\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658624692Z\",\"response\":\"\",\"thinking\":\",\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658671932Z\",\"response\":\"\",\"thinking\":\" no\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658725682Z\",\"response\":\"\",\"thinking\":\" punctuation\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658773082Z\",\"response\":\"\",\"thinking\":\",\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.720832694Z\",\"response\":\"\",\"thinking\":\" no\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721380494Z\",\"response\":\"\",\"thinking\":\" whitespace\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721458554Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721841315Z\",\"response\":\"\",\"thinking\":\" Just\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721914635Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721965435Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722044485Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722130975Z\",\"response\":\"\",\"thinking\":\" Ensure\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722193605Z\",\"response\":\"\",\"thinking\":\" no\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722236845Z\",\"response\":\"\",\"thinking\":\" trailing\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722295275Z\",\"response\":\"\",\"thinking\":\" spaces\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722455815Z\",\"response\":\"\",\"thinking\":\" or\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722508865Z\",\"response\":\"\",\"thinking\":\" new\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722556475Z\",\"response\":\"\",\"thinking\":\"lines\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722604435Z\",\"response\":\"\",\"thinking\":\"?\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722650915Z\",\"response\":\"\",\"thinking\":\" The\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723595826Z\",\"response\":\"\",\"thinking\":\" instruction\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723652066Z\",\"response\":\"\",\"thinking\":\":\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723690436Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723731326Z\",\"response\":\"\",\"thinking\":\"Reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723800286Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723847616Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.724054236Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.840079094Z\",\"response\":\"\",\"thinking\":\".\\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.840384084Z\",\"response\":\"\",\"thinking\":\" Usually\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.840527404Z\",\"response\":\"\",\"thinking\":\" means\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.841446975Z\",\"response\":\"\",\"thinking\":\" output\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.842998126Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.843058326Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.843340866Z\",\"response\":\"\",\"thinking\":\" word\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.843399236Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.844343077Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.844413087Z\",\"response\":\"\",\"thinking\":\" So\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.846992499Z\",\"response\":\"\",\"thinking\":\" I\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.84708117Z\",\"response\":\"\",\"thinking\":\" will\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.848529051Z\",\"response\":\"\",\"thinking\":\" output\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.849822492Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.851341573Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.851916643Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.85971428Z\",\"response\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.85989297Z\",\"response\":\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.937042165Z\",\"response\":\"\",\"done\":true,\"done_reason\":\"stop\",\"total_duration\":1721668042,\"prompt_eval_count\":67,\"eval_count\":79}\n" + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:19 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-55d003c552c6028927ca4a55359c9d81-3b8fecdadd2af271-00", + "transfer-encoding": "chunked", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "55d003c552c6028927ca4a55359c9d81/4291909394664977009", + "x-frame-options": "DENY", + "x-request-id": "41903295-2fcb-4eb9-a612-8789e04d7cd7" + }, + "status": 200, + "statusText": "OK" + } + } + ], + "meta": { + "createdAt": "2026-07-30T13:54:11.198Z" + } +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json new file mode 100644 index 000000000..1fe6de648 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json @@ -0,0 +1,329 @@ +{ + "span_tree": [ + { + "name": "ollama-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ollama-chat-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "ollama-chat-stream-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\".", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 47, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 114 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "ollama-generate-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ollama-generate-stream-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\". Ensure no trailing spaces or newlines? The instruction: \"Reply with exactly STREAM.\" Usually means output exactly the word STREAM. So I will output \"STREAM\".", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 79, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 146 + } + } + ], + "metadata": { + "operation": "generate-stream", + "testRunId": "" + } + }, + { + "name": "ollama-tool-call-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use get_temperature for Paris.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_temperature" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama", + "tools": [ + { + "function": { + "description": "Get the temperature for a city", + "name": "get_temperature", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 12, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "tool-call", + "testRunId": "" + } + }, + { + "name": "ollama-embed-operation", + "children": [ + { + "name": "ollama.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing", + "ollama instrumentation" + ], + "output": { + "embedding_length": 4 + }, + "metadata": { + "model": "embeddinggemma", + "provider": "ollama" + }, + "metrics": { + "prompt_tokens": 4, + "tokens": 4 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ollama-embeddings-operation", + "children": [ + { + "name": "ollama.embeddings", + "type": "llm", + "children": [], + "input": "braintrust tracing", + "output": { + "embedding_length": 3 + }, + "metadata": { + "model": "all-minilm", + "provider": "ollama" + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "ollama-error-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Trigger an error.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama" + }, + "error": "synthetic Ollama failure" + } + ], + "metadata": { + "operation": "error", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt new file mode 100644 index 000000000..efdb2be28 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt @@ -0,0 +1,257 @@ +span_tree: +└── ollama-instrumentation-root [task] + metadata: { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + ├── ollama-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\".", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 47, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 114 + │ } + ├── ollama-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-generate-stream-operation + │ metadata: { + │ "operation": "generate-stream", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\". Ensure no trailing spaces or newlines? The instruction: \"Reply with exactly STREAM.\" Usually means output exactly the word STREAM. So I will output \"STREAM\".", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 79, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 146 + │ } + ├── ollama-tool-call-operation + │ metadata: { + │ "operation": "tool-call", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Use get_temperature for Paris.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "get_temperature" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the temperature for a city", + │ "name": "get_temperature", + │ "parameters": { + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 12, + │ "tokens": 17 + │ } + ├── ollama-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── ollama.embed [llm] + │ input: [ + │ "braintrust tracing", + │ "ollama instrumentation" + │ ] + │ output: { + │ "embedding_length": 4 + │ } + │ metadata: { + │ "model": "embeddinggemma", + │ "provider": "ollama" + │ } + │ metrics: { + │ "prompt_tokens": 4, + │ "tokens": 4 + │ } + ├── ollama-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── ollama.embeddings [llm] + │ input: "braintrust tracing" + │ output: { + │ "embedding_length": 3 + │ } + │ metadata: { + │ "model": "all-minilm", + │ "provider": "ollama" + │ } + └── ollama-error-operation + metadata: { + "operation": "error", + "testRunId": "" + } + └── ollama.chat [llm] + input: [ + { + "content": "Trigger an error.", + "role": "user" + } + ] + metadata: { + "model": "gpt-oss:20b", + "provider": "ollama" + } + error: "synthetic Ollama failure" diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json new file mode 100644 index 000000000..e6959f455 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json @@ -0,0 +1,329 @@ +{ + "span_tree": [ + { + "name": "ollama-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ollama-chat-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "ollama-chat-stream-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So output: STREAM.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 65, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 132 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "ollama-generate-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ollama-generate-stream-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So final answer: STREAM.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 66, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 133 + } + } + ], + "metadata": { + "operation": "generate-stream", + "testRunId": "" + } + }, + { + "name": "ollama-tool-call-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use get_temperature for Paris.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_temperature" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama", + "tools": [ + { + "function": { + "description": "Get the temperature for a city", + "name": "get_temperature", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 12, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "tool-call", + "testRunId": "" + } + }, + { + "name": "ollama-embed-operation", + "children": [ + { + "name": "ollama.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing", + "ollama instrumentation" + ], + "output": { + "embedding_length": 4 + }, + "metadata": { + "model": "embeddinggemma", + "provider": "ollama" + }, + "metrics": { + "prompt_tokens": 4, + "tokens": 4 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ollama-embeddings-operation", + "children": [ + { + "name": "ollama.embeddings", + "type": "llm", + "children": [], + "input": "braintrust tracing", + "output": { + "embedding_length": 3 + }, + "metadata": { + "model": "all-minilm", + "provider": "ollama" + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "ollama-error-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Trigger an error.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama" + }, + "error": "synthetic Ollama failure" + } + ], + "metadata": { + "operation": "error", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt new file mode 100644 index 000000000..b75077038 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt @@ -0,0 +1,257 @@ +span_tree: +└── ollama-instrumentation-root [task] + metadata: { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + ├── ollama-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So output: STREAM.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 65, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 132 + │ } + ├── ollama-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-generate-stream-operation + │ metadata: { + │ "operation": "generate-stream", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So final answer: STREAM.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 66, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 133 + │ } + ├── ollama-tool-call-operation + │ metadata: { + │ "operation": "tool-call", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Use get_temperature for Paris.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "get_temperature" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the temperature for a city", + │ "name": "get_temperature", + │ "parameters": { + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 12, + │ "tokens": 17 + │ } + ├── ollama-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── ollama.embed [llm] + │ input: [ + │ "braintrust tracing", + │ "ollama instrumentation" + │ ] + │ output: { + │ "embedding_length": 4 + │ } + │ metadata: { + │ "model": "embeddinggemma", + │ "provider": "ollama" + │ } + │ metrics: { + │ "prompt_tokens": 4, + │ "tokens": 4 + │ } + ├── ollama-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── ollama.embeddings [llm] + │ input: "braintrust tracing" + │ output: { + │ "embedding_length": 3 + │ } + │ metadata: { + │ "model": "all-minilm", + │ "provider": "ollama" + │ } + └── ollama-error-operation + metadata: { + "operation": "error", + "testRunId": "" + } + └── ollama.chat [llm] + input: [ + { + "content": "Trigger an error.", + "role": "user" + } + ] + metadata: { + "model": "gpt-oss:20b", + "provider": "ollama" + } + error: "synthetic Ollama failure" diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json new file mode 100644 index 000000000..e6959f455 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json @@ -0,0 +1,329 @@ +{ + "span_tree": [ + { + "name": "ollama-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ollama-chat-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "ollama-chat-stream-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So output: STREAM.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 65, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 132 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "ollama-generate-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ollama-generate-stream-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So final answer: STREAM.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 66, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 133 + } + } + ], + "metadata": { + "operation": "generate-stream", + "testRunId": "" + } + }, + { + "name": "ollama-tool-call-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use get_temperature for Paris.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_temperature" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama", + "tools": [ + { + "function": { + "description": "Get the temperature for a city", + "name": "get_temperature", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 12, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "tool-call", + "testRunId": "" + } + }, + { + "name": "ollama-embed-operation", + "children": [ + { + "name": "ollama.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing", + "ollama instrumentation" + ], + "output": { + "embedding_length": 4 + }, + "metadata": { + "model": "embeddinggemma", + "provider": "ollama" + }, + "metrics": { + "prompt_tokens": 4, + "tokens": 4 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ollama-embeddings-operation", + "children": [ + { + "name": "ollama.embeddings", + "type": "llm", + "children": [], + "input": "braintrust tracing", + "output": { + "embedding_length": 3 + }, + "metadata": { + "model": "all-minilm", + "provider": "ollama" + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "ollama-error-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Trigger an error.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama" + }, + "error": "synthetic Ollama failure" + } + ], + "metadata": { + "operation": "error", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt new file mode 100644 index 000000000..b75077038 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt @@ -0,0 +1,257 @@ +span_tree: +└── ollama-instrumentation-root [task] + metadata: { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + ├── ollama-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So output: STREAM.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 65, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 132 + │ } + ├── ollama-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-generate-stream-operation + │ metadata: { + │ "operation": "generate-stream", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So final answer: STREAM.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 66, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 133 + │ } + ├── ollama-tool-call-operation + │ metadata: { + │ "operation": "tool-call", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Use get_temperature for Paris.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "get_temperature" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the temperature for a city", + │ "name": "get_temperature", + │ "parameters": { + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 12, + │ "tokens": 17 + │ } + ├── ollama-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── ollama.embed [llm] + │ input: [ + │ "braintrust tracing", + │ "ollama instrumentation" + │ ] + │ output: { + │ "embedding_length": 4 + │ } + │ metadata: { + │ "model": "embeddinggemma", + │ "provider": "ollama" + │ } + │ metrics: { + │ "prompt_tokens": 4, + │ "tokens": 4 + │ } + ├── ollama-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── ollama.embeddings [llm] + │ input: "braintrust tracing" + │ output: { + │ "embedding_length": 3 + │ } + │ metadata: { + │ "model": "all-minilm", + │ "provider": "ollama" + │ } + └── ollama-error-operation + metadata: { + "operation": "error", + "testRunId": "" + } + └── ollama.chat [llm] + input: [ + { + "content": "Trigger an error.", + "role": "user" + } + ] + metadata: { + "model": "gpt-oss:20b", + "provider": "ollama" + } + error: "synthetic Ollama failure" diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json new file mode 100644 index 000000000..1fe6de648 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json @@ -0,0 +1,329 @@ +{ + "span_tree": [ + { + "name": "ollama-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ollama-chat-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "ollama-chat-stream-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\".", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 47, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 114 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "ollama-generate-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ollama-generate-stream-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\". Ensure no trailing spaces or newlines? The instruction: \"Reply with exactly STREAM.\" Usually means output exactly the word STREAM. So I will output \"STREAM\".", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 79, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 146 + } + } + ], + "metadata": { + "operation": "generate-stream", + "testRunId": "" + } + }, + { + "name": "ollama-tool-call-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use get_temperature for Paris.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_temperature" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama", + "tools": [ + { + "function": { + "description": "Get the temperature for a city", + "name": "get_temperature", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 12, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "tool-call", + "testRunId": "" + } + }, + { + "name": "ollama-embed-operation", + "children": [ + { + "name": "ollama.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing", + "ollama instrumentation" + ], + "output": { + "embedding_length": 4 + }, + "metadata": { + "model": "embeddinggemma", + "provider": "ollama" + }, + "metrics": { + "prompt_tokens": 4, + "tokens": 4 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ollama-embeddings-operation", + "children": [ + { + "name": "ollama.embeddings", + "type": "llm", + "children": [], + "input": "braintrust tracing", + "output": { + "embedding_length": 3 + }, + "metadata": { + "model": "all-minilm", + "provider": "ollama" + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "ollama-error-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Trigger an error.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama" + }, + "error": "synthetic Ollama failure" + } + ], + "metadata": { + "operation": "error", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt new file mode 100644 index 000000000..efdb2be28 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt @@ -0,0 +1,257 @@ +span_tree: +└── ollama-instrumentation-root [task] + metadata: { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + ├── ollama-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\".", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 47, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 114 + │ } + ├── ollama-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-generate-stream-operation + │ metadata: { + │ "operation": "generate-stream", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\". Ensure no trailing spaces or newlines? The instruction: \"Reply with exactly STREAM.\" Usually means output exactly the word STREAM. So I will output \"STREAM\".", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 79, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 146 + │ } + ├── ollama-tool-call-operation + │ metadata: { + │ "operation": "tool-call", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Use get_temperature for Paris.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "get_temperature" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the temperature for a city", + │ "name": "get_temperature", + │ "parameters": { + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 12, + │ "tokens": 17 + │ } + ├── ollama-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── ollama.embed [llm] + │ input: [ + │ "braintrust tracing", + │ "ollama instrumentation" + │ ] + │ output: { + │ "embedding_length": 4 + │ } + │ metadata: { + │ "model": "embeddinggemma", + │ "provider": "ollama" + │ } + │ metrics: { + │ "prompt_tokens": 4, + │ "tokens": 4 + │ } + ├── ollama-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── ollama.embeddings [llm] + │ input: "braintrust tracing" + │ output: { + │ "embedding_length": 3 + │ } + │ metadata: { + │ "model": "all-minilm", + │ "provider": "ollama" + │ } + └── ollama-error-operation + metadata: { + "operation": "error", + "testRunId": "" + } + └── ollama.chat [llm] + input: [ + { + "content": "Trigger an error.", + "role": "user" + } + ] + metadata: { + "model": "gpt-oss:20b", + "provider": "ollama" + } + error: "synthetic Ollama failure" diff --git a/e2e/scenarios/ollama-instrumentation/assertions.ts b/e2e/scenarios/ollama-instrumentation/assertions.ts new file mode 100644 index 000000000..334d03a00 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/assertions.ts @@ -0,0 +1,227 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; +import { + withScenarioHarness, + type ScenarioRunContext, +} from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; +import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; +import { + EMBEDDING_MODEL, + GENERATION_MODEL, + LEGACY_EMBEDDING_MODEL, + ROOT_NAME, + SCENARIO_NAME, +} from "./constants.mjs"; + +type RunOllamaScenario = (harness: { + runNodeScenarioDir: (options: { + entry: string; + nodeArgs: string[]; + runContext?: ScenarioRunContext; + scenarioDir: string; + timeoutMs: number; + }) => Promise; + runScenarioDir: (options: { + entry: string; + runContext?: ScenarioRunContext; + scenarioDir: string; + timeoutMs: number; + }) => Promise; +}) => Promise; + +function findProviderSpan( + events: CapturedLogEvent[], + operationName: string, + providerSpanName: string, +): CapturedLogEvent | undefined { + const operation = findLatestSpan(events, operationName); + const spans = findChildSpans(events, providerSpanName, operation?.span.id); + return ( + spans.find( + (candidate) => + candidate.output !== undefined || candidate.row.error !== undefined, + ) ?? spans[0] + ); +} + +function selectedEvents(events: CapturedLogEvent[]): CapturedLogEvent[] { + const operations = [ + ["ollama-chat-operation", "ollama.chat"], + ["ollama-chat-stream-operation", "ollama.chat"], + ["ollama-generate-operation", "ollama.generate"], + ["ollama-generate-stream-operation", "ollama.generate"], + ["ollama-tool-call-operation", "ollama.chat"], + ["ollama-embed-operation", "ollama.embed"], + ["ollama-embeddings-operation", "ollama.embeddings"], + ["ollama-error-operation", "ollama.chat"], + ] as const; + return [ + findLatestSpan(events, ROOT_NAME), + ...operations.flatMap(([operationName, providerSpanName]) => [ + findLatestSpan(events, operationName), + findProviderSpan(events, operationName, providerSpanName), + ]), + ].filter((event): event is CapturedLogEvent => event !== undefined); +} + +function expectTokenMetrics(event: CapturedLogEvent | undefined): void { + expect(event?.metrics).toMatchObject({ + prompt_tokens: expect.any(Number), + completion_tokens: expect.any(Number), + tokens: expect.any(Number), + }); +} + +export function defineOllamaInstrumentationAssertions(options: { + name: string; + runScenario: RunOllamaScenario; + snapshotName: string; + testFileUrl: string; + timeoutMs: number; +}): void { + const snapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.span-tree.json`, + ); + + describe(options.name, () => { + let events: CapturedLogEvent[] = []; + + beforeAll(async () => { + await withScenarioHarness(async (harness) => { + await options.runScenario(harness); + events = harness.events(); + }); + }, options.timeoutMs); + + test("captures chat and generate calls with streaming parity", () => { + for (const [operationName, spanName, streaming] of [ + ["ollama-chat-operation", "ollama.chat", false], + ["ollama-chat-stream-operation", "ollama.chat", true], + ["ollama-generate-operation", "ollama.generate", false], + ["ollama-generate-stream-operation", "ollama.generate", true], + ] as const) { + const span = findProviderSpan(events, operationName, spanName); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: GENERATION_MODEL, + provider: "ollama", + }); + expect(span?.output).toEqual([ + expect.objectContaining({ + index: 0, + finish_reason: expect.any(String), + message: expect.objectContaining({ + role: "assistant", + content: expect.any(String), + }), + }), + ]); + expectTokenMetrics(span); + if (streaming) { + expect(span?.metrics?.time_to_first_token).toEqual( + expect.any(Number), + ); + } + expect(span?.row.context).toMatchObject({ + span_origin: { + instrumentation: { name: "ollama" }, + }, + }); + } + }); + + test("captures canonical tool definitions and calls", () => { + const span = findProviderSpan( + events, + "ollama-tool-call-operation", + "ollama.chat", + ); + const metadata = span?.row.metadata as + | Record + | undefined; + const output = span?.output as + | Array<{ + finish_reason?: string; + message?: { + tool_calls?: Array<{ + id?: string; + type?: string; + function?: { name?: string; arguments?: string }; + }>; + }; + }> + | undefined; + + expect(metadata?.tools).toEqual([ + { + type: "function", + function: { + name: "get_temperature", + description: "Get the temperature for a city", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, + }, + ]); + expect(output?.[0]?.finish_reason).toBe("tool_calls"); + expect(output?.[0]?.message?.tool_calls?.[0]).toMatchObject({ + id: "ollama_call_get_temperature_0", + type: "function", + function: { + name: "get_temperature", + arguments: '{"city":"Paris"}', + }, + }); + }); + + test("captures current and legacy embedding calls compactly", () => { + const embed = findProviderSpan( + events, + "ollama-embed-operation", + "ollama.embed", + ); + const embeddings = findProviderSpan( + events, + "ollama-embeddings-operation", + "ollama.embeddings", + ); + + expect(embed?.row.metadata).toMatchObject({ + model: EMBEDDING_MODEL, + provider: "ollama", + }); + expect(embed?.output).toEqual({ embedding_length: 4 }); + expect(embed?.metrics).toMatchObject({ + prompt_tokens: 4, + tokens: 4, + }); + + expect(embeddings?.row.metadata).toMatchObject({ + model: LEGACY_EMBEDDING_MODEL, + provider: "ollama", + }); + expect(embeddings?.output).toEqual({ embedding_length: 3 }); + }); + + test("captures provider errors without changing caller behavior", () => { + const span = findProviderSpan( + events, + "ollama-error-operation", + "ollama.chat", + ); + expect(span?.row.error).toBe("synthetic Ollama failure"); + }); + + test("captures the scenario root and matches the span tree", async () => { + const root = findLatestSpan(events, ROOT_NAME); + expect(root?.row.metadata).toMatchObject({ scenario: SCENARIO_NAME }); + await matchSpanTreeSnapshot(selectedEvents(events), snapshotPath); + }); + }); +} diff --git a/e2e/scenarios/ollama-instrumentation/constants.mjs b/e2e/scenarios/ollama-instrumentation/constants.mjs new file mode 100644 index 000000000..e69d98a71 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/constants.mjs @@ -0,0 +1,5 @@ +export const ROOT_NAME = "ollama-instrumentation-root"; +export const SCENARIO_NAME = "ollama-instrumentation"; +export const GENERATION_MODEL = "gpt-oss:20b"; +export const EMBEDDING_MODEL = "embeddinggemma"; +export const LEGACY_EMBEDDING_MODEL = "all-minilm"; diff --git a/e2e/scenarios/ollama-instrumentation/package.json b/e2e/scenarios/ollama-instrumentation/package.json new file mode 100644 index 000000000..e9a3b6934 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/package.json @@ -0,0 +1,18 @@ +{ + "name": "@braintrust/e2e-ollama-instrumentation", + "private": true, + "braintrustScenario": { + "bump": { + "dependencies": { + "ollama-v0-6-latest": { + "package": "ollama", + "range": "0.6" + } + } + } + }, + "dependencies": { + "ollama-v0-6": "npm:ollama@0.6.3", + "ollama-v0-6-latest": "npm:ollama@0.6.3" + } +} diff --git a/e2e/scenarios/ollama-instrumentation/pnpm-lock.yaml b/e2e/scenarios/ollama-instrumentation/pnpm-lock.yaml new file mode 100644 index 000000000..d90313710 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/pnpm-lock.yaml @@ -0,0 +1,32 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + ollama-v0-6: + specifier: npm:ollama@0.6.3 + version: ollama@0.6.3 + ollama-v0-6-latest: + specifier: npm:ollama@0.6.3 + version: ollama@0.6.3 + +packages: + + ollama@0.6.3: + resolution: {integrity: sha512-KEWEhIqE5wtfzEIZbDCLH51VFZ6Z3ZSa6sIOg/E/tBV8S51flyqBOXi+bRxlOYKDf8i327zG9eSTb8IJxvm3Zg==} + + whatwg-fetch@3.6.20: + resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==} + +snapshots: + + ollama@0.6.3: + dependencies: + whatwg-fetch: 3.6.20 + + whatwg-fetch@3.6.20: {} diff --git a/e2e/scenarios/ollama-instrumentation/scenario.impl.mjs b/e2e/scenarios/ollama-instrumentation/scenario.impl.mjs new file mode 100644 index 000000000..4a0718764 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/scenario.impl.mjs @@ -0,0 +1,254 @@ +import { wrapOllama } from "braintrust"; +import { + collectAsync, + runOperation, + runTracedScenario, +} from "../../helpers/provider-runtime.mjs"; +import { + EMBEDDING_MODEL, + GENERATION_MODEL, + LEGACY_EMBEDDING_MODEL, + ROOT_NAME, + SCENARIO_NAME, +} from "./constants.mjs"; + +export const OLLAMA_SCENARIO_TIMEOUT_MS = 240_000; + +function requireOllamaConfig() { + const apiKey = process.env.OLLAMA_API_KEY; + const host = process.env.OLLAMA_HOST; + if (!apiKey) { + throw new Error("Expected OLLAMA_API_KEY to be set for e2e"); + } + if (!host) { + throw new Error("Expected OLLAMA_HOST to be set for e2e"); + } + return { apiKey, host }; +} + +function jsonResponse(body, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +async function syntheticFetch(input, init) { + const url = String(input); + const body = + typeof init?.body === "string" && init.body.length > 0 + ? JSON.parse(init.body) + : {}; + + if (url.endsWith("/api/embed")) { + return jsonResponse({ + model: body.model, + embeddings: [ + [0.1, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0.8], + ], + prompt_eval_count: 4, + total_duration: 100, + load_duration: 10, + }); + } + + if (url.endsWith("/api/embeddings")) { + return jsonResponse({ embedding: [0.1, 0.2, 0.3] }); + } + + if (url.endsWith("/api/chat") && Array.isArray(body.tools)) { + return jsonResponse({ + model: body.model, + message: { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "get_temperature", + arguments: { city: "Paris" }, + }, + }, + ], + }, + done: true, + done_reason: "stop", + prompt_eval_count: 12, + eval_count: 5, + }); + } + + return jsonResponse({ error: "synthetic Ollama failure" }, 500); +} + +function generationOptions() { + return { + temperature: 0, + num_predict: 128, + }; +} + +async function runOllamaInstrumentationScenario( + Ollama, + { decorateClient } = {}, +) { + const { apiKey, host } = requireOllamaConfig(); + const headers = { Authorization: `Bearer ${apiKey}` }; + const liveBaseClient = new Ollama({ host, headers }); + const syntheticBaseClient = new Ollama({ + host: "http://synthetic.ollama.test", + headers, + fetch: syntheticFetch, + }); + const liveClient = decorateClient + ? decorateClient(liveBaseClient) + : liveBaseClient; + const syntheticClient = decorateClient + ? decorateClient(syntheticBaseClient) + : syntheticBaseClient; + + await runTracedScenario({ + callback: async () => { + await runOperation("ollama-chat-operation", "chat", async () => { + const response = await liveClient.chat({ + model: GENERATION_MODEL, + messages: [{ role: "user", content: "Reply with exactly OK." }], + options: generationOptions(), + think: false, + }); + if (!response?.message?.content) { + throw new Error("Expected Ollama chat response content"); + } + }); + + await runOperation( + "ollama-chat-stream-operation", + "chat-stream", + async () => { + const stream = await liveClient.chat({ + model: GENERATION_MODEL, + messages: [{ role: "user", content: "Reply with exactly STREAM." }], + options: generationOptions(), + stream: true, + think: false, + }); + const chunks = await collectAsync(stream); + if (chunks.length === 0) { + throw new Error("Expected Ollama chat stream chunks"); + } + }, + ); + + await runOperation("ollama-generate-operation", "generate", async () => { + const response = await liveClient.generate({ + model: GENERATION_MODEL, + prompt: "Reply with exactly OK.", + options: generationOptions(), + think: false, + }); + if (!response?.response) { + throw new Error("Expected Ollama generate response content"); + } + }); + + await runOperation( + "ollama-generate-stream-operation", + "generate-stream", + async () => { + const stream = await liveClient.generate({ + model: GENERATION_MODEL, + prompt: "Reply with exactly STREAM.", + options: generationOptions(), + stream: true, + think: false, + }); + const chunks = await collectAsync(stream); + if (chunks.length === 0) { + throw new Error("Expected Ollama generate stream chunks"); + } + }, + ); + + await runOperation( + "ollama-tool-call-operation", + "tool-call", + async () => { + await syntheticClient.chat({ + model: GENERATION_MODEL, + messages: [ + { + role: "user", + content: "Use get_temperature for Paris.", + }, + ], + tools: [ + { + type: "function", + function: { + name: "get_temperature", + description: "Get the temperature for a city", + parameters: { + type: "object", + properties: { + city: { type: "string" }, + }, + required: ["city"], + }, + }, + }, + ], + }); + }, + ); + + await runOperation("ollama-embed-operation", "embed", async () => { + await syntheticClient.embed({ + model: EMBEDDING_MODEL, + input: ["braintrust tracing", "ollama instrumentation"], + }); + }); + + await runOperation( + "ollama-embeddings-operation", + "embeddings", + async () => { + await syntheticClient.embeddings({ + model: LEGACY_EMBEDDING_MODEL, + prompt: "braintrust tracing", + }); + }, + ); + + await runOperation("ollama-error-operation", "error", async () => { + try { + await syntheticClient.chat({ + model: GENERATION_MODEL, + messages: [{ role: "user", content: "Trigger an error." }], + }); + throw new Error("Expected synthetic Ollama request to fail"); + } catch (error) { + if ( + error instanceof Error && + error.message === "Expected synthetic Ollama request to fail" + ) { + throw error; + } + } + }); + }, + metadata: { scenario: SCENARIO_NAME }, + projectNameBase: "e2e-ollama-instrumentation", + rootName: ROOT_NAME, + }); +} + +export async function runWrappedOllamaInstrumentation(Ollama) { + await runOllamaInstrumentationScenario(Ollama, { + decorateClient: wrapOllama, + }); +} + +export async function runAutoOllamaInstrumentation(Ollama) { + await runOllamaInstrumentationScenario(Ollama); +} diff --git a/e2e/scenarios/ollama-instrumentation/scenario.mjs b/e2e/scenarios/ollama-instrumentation/scenario.mjs new file mode 100644 index 000000000..4d3ef4ae0 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/scenario.mjs @@ -0,0 +1,10 @@ +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoOllamaInstrumentation } from "./scenario.impl.mjs"; + +const ollamaPackageName = + process.env.OLLAMA_PACKAGE_NAME ?? "ollama-v0-6-latest"; + +runMain(async () => { + const { Ollama } = await import(ollamaPackageName); + await runAutoOllamaInstrumentation(Ollama); +}); diff --git a/e2e/scenarios/ollama-instrumentation/scenario.test.ts b/e2e/scenarios/ollama-instrumentation/scenario.test.ts new file mode 100644 index 000000000..525d0f504 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/scenario.test.ts @@ -0,0 +1,76 @@ +import { describe } from "vitest"; +import { + prepareScenarioDir, + readInstalledPackageVersion, + resolveScenarioDir, +} from "../../helpers/scenario-harness"; +import { defineOllamaInstrumentationAssertions } from "./assertions"; +import { OLLAMA_SCENARIO_TIMEOUT_MS } from "./scenario.impl.mjs"; + +const originalScenarioDir = resolveScenarioDir(import.meta.url); +const scenarioDir = await prepareScenarioDir({ + scenarioDir: originalScenarioDir, +}); +const ollamaScenarios = await Promise.all( + [ + { + dependencyName: "ollama-v0-6", + variantKey: "ollama-v0.6", + }, + { + dependencyName: "ollama-v0-6-latest", + variantKey: "ollama-v0.6-latest", + }, + ].map(async (scenario) => ({ + ...scenario, + version: await readInstalledPackageVersion( + scenarioDir, + scenario.dependencyName, + ), + })), +); + +describe.sequential("variants", () => { + for (const scenario of ollamaScenarios) { + describe.sequential(`ollama sdk ${scenario.version}`, () => { + defineOllamaInstrumentationAssertions({ + name: "wrapped instrumentation", + runScenario: async ({ runScenarioDir }) => { + await runScenarioDir({ + entry: "scenario.ts", + env: { OLLAMA_PACKAGE_NAME: scenario.dependencyName }, + runContext: { + variantKey: scenario.variantKey, + originalScenarioDir, + }, + scenarioDir, + timeoutMs: OLLAMA_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: `${scenario.variantKey}-wrapped`, + testFileUrl: import.meta.url, + timeoutMs: OLLAMA_SCENARIO_TIMEOUT_MS, + }); + + defineOllamaInstrumentationAssertions({ + name: "auto-hook instrumentation", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: "scenario.mjs", + env: { OLLAMA_PACKAGE_NAME: scenario.dependencyName }, + nodeArgs: ["--import", "braintrust/hook.mjs"], + runContext: { + variantKey: scenario.variantKey, + originalScenarioDir, + }, + scenarioDir, + timeoutMs: OLLAMA_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: `${scenario.variantKey}-auto`, + testFileUrl: import.meta.url, + timeoutMs: OLLAMA_SCENARIO_TIMEOUT_MS, + }); + }); + } +}); diff --git a/e2e/scenarios/ollama-instrumentation/scenario.ts b/e2e/scenarios/ollama-instrumentation/scenario.ts new file mode 100644 index 000000000..6fb1e33d4 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/scenario.ts @@ -0,0 +1,10 @@ +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedOllamaInstrumentation } from "./scenario.impl.mjs"; + +const ollamaPackageName = + process.env.OLLAMA_PACKAGE_NAME ?? "ollama-v0-6-latest"; + +runMain(async () => { + const { Ollama } = await import(ollamaPackageName); + await runWrappedOllamaInstrumentation(Ollama); +}); diff --git a/js/src/auto-instrumentations/configs/all.ts b/js/src/auto-instrumentations/configs/all.ts index 905715e43..e6886ceb1 100644 --- a/js/src/auto-instrumentations/configs/all.ts +++ b/js/src/auto-instrumentations/configs/all.ts @@ -23,6 +23,7 @@ import { huggingFaceTransformersConfigs } from "./huggingface-transformers"; import { langchainConfigs } from "./langchain"; import { langSmithConfigs } from "./langsmith"; import { mistralConfigs } from "./mistral"; +import { ollamaConfigs } from "./ollama"; import { openAIAgentsCoreConfigs } from "./openai-agents"; import { openaiConfigs } from "./openai"; import { openAICodexConfigs } from "./openai-codex"; @@ -88,6 +89,7 @@ const defaultInstrumentationConfigGroups: readonly InstrumentationConfigGroup[] configs: openRouterAgentConfigs, }, { integrations: ["mistral"], configs: mistralConfigs }, + { integrations: ["ollama"], configs: ollamaConfigs }, { integrations: ["googleADK"], configs: googleADKConfigs }, { integrations: ["cohere"], configs: cohereConfigs }, { integrations: ["groq"], configs: groqConfigs }, diff --git a/js/src/auto-instrumentations/configs/ollama.ts b/js/src/auto-instrumentations/configs/ollama.ts new file mode 100644 index 000000000..1db36b8f0 --- /dev/null +++ b/js/src/auto-instrumentations/configs/ollama.ts @@ -0,0 +1,32 @@ +import type { InstrumentationConfig } from "../orchestrion-js"; +import { ollamaChannels } from "../../instrumentation/plugins/ollama-channels"; + +const methods = [ + ["chat", ollamaChannels.chat.channelName], + ["generate", ollamaChannels.generate.channelName], + ["embed", ollamaChannels.embed.channelName], + ["embeddings", ollamaChannels.embeddings.channelName], +] as const; + +// Ollama's Node entry points in dist/index.{mjs,cjs} subclass the Ollama class +// exported here. These methods are defined on that shared base class, so +// instrumenting these files covers both the Node and browser package exports. +const implementationFiles = ["dist/browser.mjs", "dist/browser.cjs"] as const; + +export const ollamaConfigs: InstrumentationConfig[] = [ + ...implementationFiles.flatMap((filePath) => + methods.map(([methodName, channelName]) => ({ + channelName, + module: { + name: "ollama", + versionRange: ">=0.6.0 <0.7.0", + filePath, + }, + functionQuery: { + className: "Ollama", + methodName, + kind: "Async" as const, + }, + })), + ), +]; diff --git a/js/src/auto-instrumentations/index.ts b/js/src/auto-instrumentations/index.ts index 49f728f82..a92465f4e 100644 --- a/js/src/auto-instrumentations/index.ts +++ b/js/src/auto-instrumentations/index.ts @@ -42,6 +42,7 @@ export { huggingFaceConfigs } from "./configs/huggingface"; export { openRouterAgentConfigs } from "./configs/openrouter-agent"; export { openRouterConfigs } from "./configs/openrouter"; export { mistralConfigs } from "./configs/mistral"; +export { ollamaConfigs } from "./configs/ollama"; export { googleADKConfigs } from "./configs/google-adk"; export { cloudflareAIChatConfigs } from "./configs/cloudflare-ai-chat"; export { cohereConfigs } from "./configs/cohere"; diff --git a/js/src/exports.ts b/js/src/exports.ts index 02104f8c2..9da607b55 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -221,6 +221,7 @@ export { wrapHuggingFaceTransformers } from "./wrappers/huggingface-transformers export { wrapOpenRouterAgent } from "./wrappers/openrouter-agent"; export { wrapOpenRouter } from "./wrappers/openrouter"; export { wrapMistral } from "./wrappers/mistral"; +export { wrapOllama } from "./wrappers/ollama"; export { wrapCohere } from "./wrappers/cohere"; export { wrapGroq } from "./wrappers/groq"; export { wrapBedrockRuntime } from "./wrappers/bedrock-runtime"; diff --git a/js/src/instrumentation/braintrust-plugin.test.ts b/js/src/instrumentation/braintrust-plugin.test.ts index 05c8d4838..cdbe15d5c 100644 --- a/js/src/instrumentation/braintrust-plugin.test.ts +++ b/js/src/instrumentation/braintrust-plugin.test.ts @@ -12,6 +12,7 @@ import { HuggingFaceTransformersPlugin } from "./plugins/huggingface-transformer import { OpenRouterAgentPlugin } from "./plugins/openrouter-agent-plugin"; import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; import { MistralPlugin } from "./plugins/mistral-plugin"; +import { OllamaPlugin } from "./plugins/ollama-plugin"; import { CoherePlugin } from "./plugins/cohere-plugin"; import { GroqPlugin } from "./plugins/groq-plugin"; import { GitHubCopilotPlugin } from "./plugins/github-copilot-plugin"; @@ -90,6 +91,10 @@ vi.mock("./plugins/mistral-plugin", () => ({ MistralPlugin: createPluginClassMock(), })); +vi.mock("./plugins/ollama-plugin", () => ({ + OllamaPlugin: createPluginClassMock(), +})); + vi.mock("./plugins/cohere-plugin", () => ({ CoherePlugin: createPluginClassMock(), })); @@ -543,6 +548,18 @@ describe("BraintrustPlugin", () => { expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); }); + it("should not create Ollama plugin when ollama: false", () => { + const plugin = new BraintrustPlugin({ + integrations: { ollama: false }, + }); + plugin.enable(); + + expect(OllamaPlugin).not.toHaveBeenCalled(); + expect(OpenAIPlugin).toHaveBeenCalledTimes(1); + expect(AnthropicPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); + }); + it("should not create Cohere plugin when cohere: false", () => { const plugin = new BraintrustPlugin({ integrations: { cohere: false }, @@ -648,6 +665,7 @@ describe("BraintrustPlugin", () => { openrouter: false, openrouterAgent: false, mistral: false, + ollama: false, cohere: false, groq: false, gitHubCopilot: false, @@ -672,6 +690,7 @@ describe("BraintrustPlugin", () => { expect(OpenRouterPlugin).not.toHaveBeenCalled(); expect(OpenRouterAgentPlugin).not.toHaveBeenCalled(); expect(MistralPlugin).not.toHaveBeenCalled(); + expect(OllamaPlugin).not.toHaveBeenCalled(); expect(CoherePlugin).not.toHaveBeenCalled(); expect(GroqPlugin).not.toHaveBeenCalled(); expect(GitHubCopilotPlugin).not.toHaveBeenCalled(); @@ -881,6 +900,7 @@ describe("BraintrustPlugin", () => { const openRouterAgentMock = vi.mocked(OpenRouterAgentPlugin).mock .results[0].value; const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; + const ollamaMock = vi.mocked(OllamaPlugin).mock.results[0].value; const cohereMock = vi.mocked(CoherePlugin).mock.results[0].value; const groqMock = vi.mocked(GroqPlugin).mock.results[0].value; const piCodingAgentMock = @@ -903,6 +923,7 @@ describe("BraintrustPlugin", () => { expect(openRouterMock.enable).toHaveBeenCalledTimes(1); expect(openRouterAgentMock.enable).toHaveBeenCalledTimes(1); expect(mistralMock.enable).toHaveBeenCalledTimes(1); + expect(ollamaMock.enable).toHaveBeenCalledTimes(1); expect(cohereMock.enable).toHaveBeenCalledTimes(1); expect(groqMock.enable).toHaveBeenCalledTimes(1); expect(piCodingAgentMock.enable).toHaveBeenCalledTimes(1); @@ -935,6 +956,7 @@ describe("BraintrustPlugin", () => { const openRouterAgentMock = vi.mocked(OpenRouterAgentPlugin).mock .results[0].value; const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; + const ollamaMock = vi.mocked(OllamaPlugin).mock.results[0].value; const cohereMock = vi.mocked(CoherePlugin).mock.results[0].value; const groqMock = vi.mocked(GroqPlugin).mock.results[0].value; const piCodingAgentMock = @@ -959,6 +981,7 @@ describe("BraintrustPlugin", () => { expect(openRouterMock.disable).toHaveBeenCalledTimes(1); expect(openRouterAgentMock.disable).toHaveBeenCalledTimes(1); expect(mistralMock.disable).toHaveBeenCalledTimes(1); + expect(ollamaMock.disable).toHaveBeenCalledTimes(1); expect(cohereMock.disable).toHaveBeenCalledTimes(1); expect(groqMock.disable).toHaveBeenCalledTimes(1); expect(piCodingAgentMock.disable).toHaveBeenCalledTimes(1); @@ -1009,6 +1032,7 @@ describe("BraintrustPlugin", () => { expect(OpenRouterPlugin).not.toHaveBeenCalled(); expect(OpenRouterAgentPlugin).not.toHaveBeenCalled(); expect(MistralPlugin).not.toHaveBeenCalled(); + expect(OllamaPlugin).not.toHaveBeenCalled(); expect(CoherePlugin).not.toHaveBeenCalled(); expect(GroqPlugin).not.toHaveBeenCalled(); expect(PiCodingAgentPlugin).not.toHaveBeenCalled(); @@ -1035,6 +1059,7 @@ describe("BraintrustPlugin", () => { expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); expect(MistralPlugin).toHaveBeenCalledTimes(1); + expect(OllamaPlugin).toHaveBeenCalledTimes(1); expect(CoherePlugin).toHaveBeenCalledTimes(1); expect(GroqPlugin).toHaveBeenCalledTimes(1); expect(PiCodingAgentPlugin).toHaveBeenCalledTimes(1); @@ -1056,6 +1081,7 @@ describe("BraintrustPlugin", () => { openrouter: true, openrouterAgent: true, mistral: false, + ollama: false, cohere: false, groq: true, langchain: true, @@ -1089,6 +1115,7 @@ describe("BraintrustPlugin", () => { expect(groqMock.disable).toHaveBeenCalledTimes(1); expect(langChainMock.disable).toHaveBeenCalledTimes(1); expect(MistralPlugin).not.toHaveBeenCalled(); + expect(OllamaPlugin).not.toHaveBeenCalled(); expect(CoherePlugin).not.toHaveBeenCalled(); }); }); diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index ec88ddec2..201d39970 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -12,6 +12,7 @@ import { HuggingFaceTransformersPlugin } from "./plugins/huggingface-transformer import { OpenRouterAgentPlugin } from "./plugins/openrouter-agent-plugin"; import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; import { MistralPlugin } from "./plugins/mistral-plugin"; +import { OllamaPlugin } from "./plugins/ollama-plugin"; import { GoogleADKPlugin } from "./plugins/google-adk-plugin"; import { CoherePlugin } from "./plugins/cohere-plugin"; import { GroqPlugin } from "./plugins/groq-plugin"; @@ -43,6 +44,7 @@ export interface BraintrustPluginConfig { * - HuggingFace Inference SDK * - LangChain.js and LangGraph * - Mistral SDK + * - Ollama SDK * - Cohere SDK * * The plugin is automatically enabled when the Braintrust library is loaded. @@ -64,6 +66,7 @@ export class BraintrustPlugin extends BasePlugin { private openRouterPlugin: OpenRouterPlugin | null = null; private openRouterAgentPlugin: OpenRouterAgentPlugin | null = null; private mistralPlugin: MistralPlugin | null = null; + private ollamaPlugin: OllamaPlugin | null = null; private googleADKPlugin: GoogleADKPlugin | null = null; private coherePlugin: CoherePlugin | null = null; private groqPlugin: GroqPlugin | null = null; @@ -156,6 +159,11 @@ export class BraintrustPlugin extends BasePlugin { this.mistralPlugin.enable(); } + if (integrations.ollama !== false) { + this.ollamaPlugin = new OllamaPlugin(); + this.ollamaPlugin.enable(); + } + // Enable Google ADK integration (default: true) if (integrations.googleADK !== false) { this.googleADKPlugin = new GoogleADKPlugin(); @@ -301,6 +309,11 @@ export class BraintrustPlugin extends BasePlugin { this.mistralPlugin = null; } + if (this.ollamaPlugin) { + this.ollamaPlugin.disable(); + this.ollamaPlugin = null; + } + if (this.googleADKPlugin) { this.googleADKPlugin.disable(); this.googleADKPlugin = null; diff --git a/js/src/instrumentation/config.ts b/js/src/instrumentation/config.ts index c51a5b48d..03ecf52b7 100644 --- a/js/src/instrumentation/config.ts +++ b/js/src/instrumentation/config.ts @@ -17,6 +17,7 @@ export interface InstrumentationIntegrationsConfig { openrouter?: boolean; openrouterAgent?: boolean; mistral?: boolean; + ollama?: boolean; cohere?: boolean; groq?: boolean; bedrock?: boolean; @@ -93,6 +94,7 @@ const envIntegrationAliases: Record< openrouteragent: "openrouterAgent", "openrouter-agent": "openrouterAgent", mistral: "mistral", + ollama: "ollama", googleadk: "googleADK", "google-adk": "googleADK", cohere: "cohere", @@ -140,6 +142,7 @@ export function getDefaultInstrumentationIntegrations(): Record< openrouter: true, openrouterAgent: true, mistral: true, + ollama: true, cohere: true, groq: true, bedrock: true, diff --git a/js/src/instrumentation/plugins/instrumentation-names.test.ts b/js/src/instrumentation/plugins/instrumentation-names.test.ts index 2e342f9b2..fbb0b0561 100644 --- a/js/src/instrumentation/plugins/instrumentation-names.test.ts +++ b/js/src/instrumentation/plugins/instrumentation-names.test.ts @@ -23,6 +23,7 @@ import { huggingFaceChannels } from "./huggingface-channels"; import { langChainChannels } from "./langchain-channels"; import { langSmithChannels } from "./langsmith-channels"; import { mistralChannels } from "./mistral-channels"; +import { ollamaChannels } from "./ollama-channels"; import { openAIAgentsCoreChannels } from "./openai-agents-channels"; import { openAIChannels } from "./openai-channels"; import { openAICodexChannels } from "./openai-codex-channels"; @@ -61,6 +62,7 @@ describe("built-in instrumentation provenance names", () => { [langChainChannels.configure, INSTRUMENTATION_NAMES.LANGCHAIN], [langSmithChannels.createRun, INSTRUMENTATION_NAMES.LANGSMITH], [mistralChannels.chatComplete, INSTRUMENTATION_NAMES.MISTRAL], + [ollamaChannels.chat, INSTRUMENTATION_NAMES.OLLAMA], [ openAIAgentsCoreChannels.onTraceStart, INSTRUMENTATION_NAMES.OPENAI_AGENTS, diff --git a/js/src/instrumentation/plugins/ollama-channels.ts b/js/src/instrumentation/plugins/ollama-channels.ts new file mode 100644 index 000000000..70b7d8533 --- /dev/null +++ b/js/src/instrumentation/plugins/ollama-channels.ts @@ -0,0 +1,47 @@ +import { INSTRUMENTATION_NAMES } from "../../span-origin"; +import type { + OllamaChatRequest, + OllamaChatResponse, + OllamaChatResult, + OllamaEmbedRequest, + OllamaEmbedResponse, + OllamaEmbeddingsRequest, + OllamaEmbeddingsResponse, + OllamaGenerateRequest, + OllamaGenerateResponse, + OllamaGenerateResult, +} from "../../vendor-sdk-types/ollama"; +import { channel, defineChannels } from "../core/channel-definitions"; + +export const ollamaChannels = defineChannels( + "ollama", + { + chat: channel< + [OllamaChatRequest], + OllamaChatResult, + Record, + OllamaChatResponse + >({ + channelName: "chat", + kind: "async", + }), + generate: channel< + [OllamaGenerateRequest], + OllamaGenerateResult, + Record, + OllamaGenerateResponse + >({ + channelName: "generate", + kind: "async", + }), + embed: channel<[OllamaEmbedRequest], OllamaEmbedResponse>({ + channelName: "embed", + kind: "async", + }), + embeddings: channel<[OllamaEmbeddingsRequest], OllamaEmbeddingsResponse>({ + channelName: "embeddings", + kind: "async", + }), + }, + { instrumentationName: INSTRUMENTATION_NAMES.OLLAMA }, +); diff --git a/js/src/instrumentation/plugins/ollama-plugin.test.ts b/js/src/instrumentation/plugins/ollama-plugin.test.ts new file mode 100644 index 000000000..4c9690d63 --- /dev/null +++ b/js/src/instrumentation/plugins/ollama-plugin.test.ts @@ -0,0 +1,321 @@ +import { describe, expect, it } from "vitest"; +import { + aggregateOllamaChatChunks, + aggregateOllamaGenerateChunks, + extractOllamaChatInput, + extractOllamaChatOutput, + extractOllamaEmbedOutput, + extractOllamaEmbeddingsOutput, + extractOllamaGenerateInput, + extractOllamaMetrics, +} from "./ollama-plugin"; + +describe("Ollama instrumentation extraction", () => { + it("normalizes chat inputs, tools, and supported request metadata", () => { + const result = extractOllamaChatInput([ + { + model: "gpt-oss:20b", + messages: [ + { role: "system", content: "Be concise." }, + { role: "user", content: "What is the weather?" }, + { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "get_weather", + arguments: { city: "Paris" }, + }, + }, + ], + }, + { + role: "tool", + tool_name: "get_weather", + content: '{"temperature":18}', + }, + ], + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get weather", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + }, + }, + }, + ], + format: "json", + options: { + temperature: 0, + top_p: 0.8, + num_predict: 32, + frequency_penalty: 0.1, + presence_penalty: 0.2, + stop: ["DONE"], + top_k: 10, + }, + stream: true, + }, + ]); + + expect(result.input).toEqual([ + { role: "system", content: "Be concise." }, + { role: "user", content: "What is the weather?" }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "ollama_call_get_weather_0", + type: "function", + function: { + name: "get_weather", + arguments: '{"city":"Paris"}', + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "ollama_call_get_weather_0", + content: '{"temperature":18}', + }, + ]); + expect(result.metadata).toEqual({ + provider: "ollama", + model: "gpt-oss:20b", + temperature: 0, + top_p: 0.8, + max_tokens: 32, + frequency_penalty: 0.1, + presence_penalty: 0.2, + stop: ["DONE"], + response_format: "json", + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get weather", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + }, + }, + }, + ], + }); + }); + + it("converts multimodal chat images into attachments", () => { + const result = extractOllamaChatInput([ + { + model: "llava", + messages: [ + { + role: "user", + content: "Describe this image.", + images: ["data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB"], + }, + ], + }, + ]); + + const content = ( + result.input as Array<{ + content: Array<{ + type: string; + image_url?: { url?: { reference?: unknown } }; + }>; + }> + )[0].content; + expect(content[0]).toEqual({ + type: "text", + text: "Describe this image.", + }); + expect(content[1]).toMatchObject({ + type: "image_url", + image_url: { + url: { + reference: { + type: "braintrust_attachment", + content_type: "image/png", + }, + }, + }, + }); + }); + + it("normalizes chat output and tool calls as OpenAI choices", () => { + expect( + extractOllamaChatOutput({ + model: "gpt-oss:20b", + message: { + role: "assistant", + content: "", + thinking: "I should call the tool.", + tool_calls: [ + { + function: { + name: "get_weather", + arguments: { city: "Paris" }, + }, + }, + ], + }, + done: true, + done_reason: "stop", + }), + ).toEqual([ + { + index: 0, + finish_reason: "tool_calls", + message: { + role: "assistant", + content: null, + reasoning: "I should call the tool.", + tool_calls: [ + { + id: "ollama_call_get_weather_0", + type: "function", + function: { + name: "get_weather", + arguments: '{"city":"Paris"}', + }, + }, + ], + }, + }, + ]); + }); + + it("normalizes raw generation input as canonical messages", () => { + expect( + extractOllamaGenerateInput([ + { + model: "gpt-oss:20b", + system: "Be concise.", + prompt: "Say OK.", + options: { temperature: 0, num_predict: 8 }, + }, + ]), + ).toEqual({ + input: [ + { role: "system", content: "Be concise." }, + { role: "user", content: "Say OK." }, + ], + metadata: { + provider: "ollama", + model: "gpt-oss:20b", + temperature: 0, + max_tokens: 8, + }, + }); + }); + + it("extracts token metrics and ignores invalid counts", () => { + expect( + extractOllamaMetrics({ + prompt_eval_count: 7, + eval_count: 3, + }), + ).toEqual({ + prompt_tokens: 7, + completion_tokens: 3, + tokens: 10, + }); + expect( + extractOllamaMetrics({ + prompt_eval_count: -1, + eval_count: Number.NaN, + }), + ).toEqual({}); + }); + + it("aggregates chat and generation streams into final outputs", () => { + const chat = aggregateOllamaChatChunks( + [ + { + model: "gpt-oss:20b", + message: { role: "assistant", content: "Hello" }, + done: false, + }, + { + model: "gpt-oss:20b", + message: { role: "assistant", content: "!" }, + done: true, + done_reason: "stop", + prompt_eval_count: 5, + eval_count: 2, + }, + ], + undefined, + undefined, + 1, + ); + expect(chat.output).toEqual([ + { + index: 0, + finish_reason: "stop", + message: { role: "assistant", content: "Hello!" }, + }, + ]); + expect(chat.metrics).toMatchObject({ + prompt_tokens: 5, + completion_tokens: 2, + tokens: 7, + }); + expect(chat.metadata).toEqual({ model: "gpt-oss:20b" }); + + const generation = aggregateOllamaGenerateChunks( + [ + { model: "gpt-oss:20b", response: "O", done: false }, + { + model: "gpt-oss:20b", + response: "K", + done: true, + done_reason: "stop", + prompt_eval_count: 4, + eval_count: 2, + }, + ], + undefined, + undefined, + 1, + ); + expect(generation.output).toEqual([ + { + index: 0, + finish_reason: "stop", + message: { role: "assistant", content: "OK" }, + }, + ]); + expect(generation.metrics).toMatchObject({ + prompt_tokens: 4, + completion_tokens: 2, + tokens: 6, + }); + }); + + it("summarizes current and legacy embedding responses", () => { + expect( + extractOllamaEmbedOutput({ + model: "embeddinggemma", + embeddings: [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6], + ], + }), + ).toEqual({ embedding_length: 3 }); + expect( + extractOllamaEmbeddingsOutput({ + embedding: [0.1, 0.2], + }), + ).toEqual({ embedding_length: 2 }); + }); +}); diff --git a/js/src/instrumentation/plugins/ollama-plugin.ts b/js/src/instrumentation/plugins/ollama-plugin.ts new file mode 100644 index 000000000..831fb07e1 --- /dev/null +++ b/js/src/instrumentation/plugins/ollama-plugin.ts @@ -0,0 +1,562 @@ +import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { processInputAttachments } from "../../wrappers/attachment-utils"; +import type { + OllamaChatResponse, + OllamaEmbedResponse, + OllamaEmbeddingsResponse, + OllamaGenerateResponse, + OllamaMessage, + OllamaTool, + OllamaToolCall, + OllamaUsageResponse, +} from "../../vendor-sdk-types/ollama"; +import { BasePlugin } from "../core"; +import { + traceAsyncChannel, + traceStreamingChannel, + unsubscribeAll, +} from "../core/channel-tracing"; +import { ollamaChannels } from "./ollama-channels"; + +export class OllamaPlugin extends BasePlugin { + protected onEnable(): void { + this.unsubscribers.push( + traceStreamingChannel(ollamaChannels.chat, { + name: "ollama.chat", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaChatInput, + extractOutput: extractOllamaChatOutput, + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + aggregateChunks: aggregateOllamaChatChunks, + }), + traceStreamingChannel(ollamaChannels.generate, { + name: "ollama.generate", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaGenerateInput, + extractOutput: extractOllamaGenerateOutput, + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + aggregateChunks: aggregateOllamaGenerateChunks, + }), + traceAsyncChannel(ollamaChannels.embed, { + name: "ollama.embed", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaEmbedInput, + extractOutput: extractOllamaEmbedOutput, + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + }), + traceAsyncChannel(ollamaChannels.embeddings, { + name: "ollama.embeddings", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaEmbeddingsInput, + extractOutput: extractOllamaEmbeddingsOutput, + extractMetrics: () => ({}), + }), + ); + } + + protected onDisable(): void { + this.unsubscribers = unsubscribeAll(this.unsubscribers); + } +} + +function getRequestArg(args: unknown): Record | undefined { + const values = + Array.isArray(args) || isArrayLike(args) ? Array.from(args) : [args]; + return values.find((value) => isObject(value)) as + | Record + | undefined; +} + +function isArrayLike(value: unknown): value is ArrayLike { + return ( + isObject(value) && + typeof value.length === "number" && + Number.isInteger(value.length) && + value.length >= 0 + ); +} + +function isNonNegativeNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +function normalizeFinishReason( + response: { done_reason?: string } | undefined, + hasToolCalls = false, +): string { + if (hasToolCalls) { + return "tool_calls"; + } + return response?.done_reason || "stop"; +} + +function stringifyArguments(value: unknown): string { + if (typeof value === "string") { + return value; + } + try { + return JSON.stringify(value ?? {}); + } catch { + return String(value); + } +} + +function syntheticToolCallId(name: string, index: number): string { + const normalizedName = name.replace(/[^a-zA-Z0-9_-]/g, "_") || "tool"; + return `ollama_call_${normalizedName}_${index}`; +} + +function normalizeToolCall( + toolCall: OllamaToolCall, + index: number, +): Record | undefined { + const name = toolCall.function?.name; + if (typeof name !== "string" || name.length === 0) { + return undefined; + } + + return { + id: + typeof toolCall.id === "string" && toolCall.id.length > 0 + ? toolCall.id + : syntheticToolCallId(name, index), + type: "function", + function: { + name, + arguments: stringifyArguments(toolCall.function?.arguments), + }, + }; +} + +function normalizeToolCalls( + toolCalls: OllamaToolCall[] | undefined, +): Record[] { + if (!Array.isArray(toolCalls)) { + return []; + } + + return toolCalls.flatMap((toolCall, index) => { + const normalized = normalizeToolCall(toolCall, index); + return normalized ? [normalized] : []; + }); +} + +function imageBytes(value: unknown): Uint8Array | undefined { + if (value instanceof Uint8Array) { + return value; + } + if (value instanceof ArrayBuffer) { + return new Uint8Array(value); + } + if (typeof value !== "string" || value.startsWith("data:")) { + return undefined; + } + try { + const decoded = atob(value.slice(0, 24)); + return Uint8Array.from(decoded, (character) => character.charCodeAt(0)); + } catch { + return undefined; + } +} + +function inferImageMediaType(value: unknown): string | undefined { + if (typeof value === "string") { + const dataUrlType = value.match(/^data:(image\/[^;]+);base64,/i)?.[1]; + if (dataUrlType) { + return dataUrlType; + } + } + + const bytes = imageBytes(value); + if (!bytes) { + return undefined; + } + if ( + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47 + ) { + return "image/png"; + } + if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { + return "image/jpeg"; + } + const signature = String.fromCharCode(...bytes.slice(0, 12)); + if (signature.startsWith("GIF87a") || signature.startsWith("GIF89a")) { + return "image/gif"; + } + if (signature.startsWith("RIFF") && signature.slice(8, 12) === "WEBP") { + return "image/webp"; + } + return undefined; +} + +function normalizeTextAndImages( + content: string | null, + images: unknown[] | undefined, +): { + content: unknown; + unrecognizedImages?: unknown[]; +} { + if (!images || images.length === 0) { + return { content }; + } + + const imageParts: Record[] = []; + const unrecognizedImages: unknown[] = []; + for (const image of images) { + const mediaType = inferImageMediaType(image); + if (!mediaType) { + unrecognizedImages.push(image); + continue; + } + const processed = processInputAttachments({ + type: "image", + image, + mediaType, + }); + imageParts.push({ + type: "image_url", + image_url: { url: processed.image }, + }); + } + + return { + content: + imageParts.length > 0 + ? [...(content ? [{ type: "text", text: content }] : []), ...imageParts] + : content, + ...(unrecognizedImages.length > 0 ? { unrecognizedImages } : {}), + }; +} + +function normalizeMessage( + message: OllamaMessage, +): Record | undefined { + if (typeof message.role !== "string") { + return undefined; + } + + const toolCalls = normalizeToolCalls(message.tool_calls); + if (message.role === "tool") { + const toolName = + typeof message.tool_name === "string" && message.tool_name.length > 0 + ? message.tool_name + : "tool"; + return { + role: "tool", + tool_call_id: syntheticToolCallId(toolName, 0), + content: typeof message.content === "string" ? message.content : "", + }; + } + + const normalizedContent = normalizeTextAndImages( + typeof message.content === "string" + ? message.content || (toolCalls.length > 0 ? null : "") + : toolCalls.length > 0 + ? null + : "", + message.images, + ); + return { + role: message.role, + content: normalizedContent.content, + ...(typeof message.thinking === "string" && message.thinking.length > 0 + ? { reasoning: message.thinking } + : {}), + ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}), + ...(normalizedContent.unrecognizedImages + ? { images: normalizedContent.unrecognizedImages } + : {}), + }; +} + +function normalizeMessages(messages: unknown): Record[] { + if (!Array.isArray(messages)) { + return []; + } + return messages.flatMap((message) => { + if (!isObject(message)) { + return []; + } + const normalized = normalizeMessage(message as OllamaMessage); + return normalized ? [normalized] : []; + }); +} + +function normalizeToolDefinition( + tool: OllamaTool, +): Record | undefined { + const fn = tool.function; + if (!fn || typeof fn.name !== "string" || fn.name.length === 0) { + return undefined; + } + + return { + type: "function", + function: { + name: fn.name, + ...(typeof fn.description === "string" + ? { description: fn.description } + : {}), + ...(isObject(fn.parameters) ? { parameters: fn.parameters } : {}), + }, + }; +} + +function extractToolsMetadata(tools: unknown): Record { + if (!Array.isArray(tools)) { + return {}; + } + const normalized = tools.flatMap((tool) => { + if (!isObject(tool)) { + return []; + } + const definition = normalizeToolDefinition(tool as OllamaTool); + return definition ? [definition] : []; + }); + return normalized.length > 0 ? { tools: normalized } : {}; +} + +function extractOptionsMetadata(options: unknown): Record { + if (!isObject(options)) { + return {}; + } + + const metadata: Record = {}; + const mappings = [ + ["temperature", "temperature"], + ["top_p", "top_p"], + ["num_predict", "max_tokens"], + ["frequency_penalty", "frequency_penalty"], + ["presence_penalty", "presence_penalty"], + ["stop", "stop"], + ] as const; + for (const [source, target] of mappings) { + if (options[source] !== undefined) { + metadata[target] = options[source]; + } + } + return metadata; +} + +function extractRequestMetadata( + request: Record | undefined, +): Record { + return { + provider: "ollama", + ...(typeof request?.model === "string" ? { model: request.model } : {}), + ...extractOptionsMetadata(request?.options), + ...(request?.format !== undefined + ? { response_format: request.format } + : {}), + }; +} + +export function extractOllamaChatInput(args: unknown): { + input: unknown; + metadata: Record; +} { + const request = getRequestArg(args); + return { + input: normalizeMessages(request?.messages), + metadata: { + ...extractRequestMetadata(request), + ...extractToolsMetadata(request?.tools), + }, + }; +} + +export function extractOllamaGenerateInput(args: unknown): { + input: unknown; + metadata: Record; +} { + const request = getRequestArg(args); + const normalizedPrompt = normalizeTextAndImages( + typeof request?.prompt === "string" ? request.prompt : "", + Array.isArray(request?.images) ? request.images : undefined, + ); + const input = [ + ...(typeof request?.system === "string" + ? [{ role: "system", content: request.system }] + : []), + { + role: "user", + content: normalizedPrompt.content, + ...(normalizedPrompt.unrecognizedImages + ? { images: normalizedPrompt.unrecognizedImages } + : {}), + }, + ]; + + return { + input, + metadata: extractRequestMetadata(request), + }; +} + +export function extractOllamaEmbedInput(args: unknown): { + input: unknown; + metadata: Record; +} { + const request = getRequestArg(args); + return { + input: request?.input, + metadata: extractRequestMetadata(request), + }; +} + +export function extractOllamaEmbeddingsInput(args: unknown): { + input: unknown; + metadata: Record; +} { + const request = getRequestArg(args); + return { + input: request?.prompt, + metadata: extractRequestMetadata(request), + }; +} + +export function extractOllamaChatOutput(result: OllamaChatResponse): unknown { + if (!isObject(result) || !isObject(result.message)) { + return undefined; + } + + const message = normalizeMessage(result.message as OllamaMessage); + if (!message) { + return undefined; + } + const toolCalls = Array.isArray(message.tool_calls) ? message.tool_calls : []; + + return [ + { + index: 0, + finish_reason: normalizeFinishReason(result, toolCalls.length > 0), + message, + }, + ]; +} + +export function extractOllamaGenerateOutput( + result: OllamaGenerateResponse, +): unknown { + if (!isObject(result) || typeof result.response !== "string") { + return undefined; + } + + return [ + { + index: 0, + finish_reason: normalizeFinishReason(result), + message: { + role: "assistant", + content: result.response, + ...(typeof result.thinking === "string" && result.thinking.length > 0 + ? { reasoning: result.thinking } + : {}), + }, + }, + ]; +} + +export function extractOllamaEmbedOutput(result: OllamaEmbedResponse): unknown { + const embedding = Array.isArray(result?.embeddings) + ? result.embeddings[0] + : undefined; + return Array.isArray(embedding) + ? { embedding_length: embedding.length } + : undefined; +} + +export function extractOllamaEmbeddingsOutput( + result: OllamaEmbeddingsResponse, +): unknown { + return Array.isArray(result?.embedding) + ? { embedding_length: result.embedding.length } + : undefined; +} + +export function extractOllamaResponseMetadata( + result: OllamaUsageResponse, +): Record | undefined { + return typeof result?.model === "string" + ? { model: result.model } + : undefined; +} + +export function extractOllamaMetrics( + result: OllamaUsageResponse, +): Record { + const metrics: Record = {}; + const promptTokens = result?.prompt_eval_count; + const completionTokens = result?.eval_count; + + if (isNonNegativeNumber(promptTokens)) { + metrics.prompt_tokens = promptTokens; + } + if (isNonNegativeNumber(completionTokens)) { + metrics.completion_tokens = completionTokens; + } + if ( + isNonNegativeNumber(promptTokens) || + isNonNegativeNumber(completionTokens) + ) { + metrics.tokens = (promptTokens ?? 0) + (completionTokens ?? 0); + } + return metrics; +} + +export function aggregateOllamaChatChunks( + chunks: OllamaChatResponse[], + _result?: unknown, + _event?: unknown, + _startTime?: number, +): { + output: unknown; + metrics: Record; + metadata?: Record; +} { + const last = chunks.at(-1); + const message: OllamaMessage = { + role: "assistant", + content: chunks.map((chunk) => chunk.message?.content ?? "").join(""), + thinking: chunks.map((chunk) => chunk.message?.thinking ?? "").join(""), + tool_calls: chunks.flatMap((chunk) => chunk.message?.tool_calls ?? []), + }; + const response: OllamaChatResponse = { + ...last, + message, + }; + return { + output: extractOllamaChatOutput(response), + metrics: extractOllamaMetrics(last ?? {}), + metadata: extractOllamaResponseMetadata(last ?? {}), + }; +} + +export function aggregateOllamaGenerateChunks( + chunks: OllamaGenerateResponse[], + _result?: unknown, + _event?: unknown, + _startTime?: number, +): { + output: unknown; + metrics: Record; + metadata?: Record; +} { + const last = chunks.at(-1); + const response: OllamaGenerateResponse = { + ...last, + response: chunks.map((chunk) => chunk.response ?? "").join(""), + thinking: chunks.map((chunk) => chunk.thinking ?? "").join(""), + }; + return { + output: extractOllamaGenerateOutput(response), + metrics: extractOllamaMetrics(last ?? {}), + metadata: extractOllamaResponseMetadata(last ?? {}), + }; +} diff --git a/js/src/span-origin.ts b/js/src/span-origin.ts index 0fd2f5fc8..cf41cb14b 100644 --- a/js/src/span-origin.ts +++ b/js/src/span-origin.ts @@ -29,6 +29,7 @@ export const INSTRUMENTATION_NAMES = { LANGSMITH: "langsmith", MASTRA: "mastra", MISTRAL: "mistral", + OLLAMA: "ollama", OPENAI: "openai", OPENAI_AGENTS: "openai-agents", OPENAI_CODEX: "openai-codex", diff --git a/js/src/vendor-sdk-types/ollama.ts b/js/src/vendor-sdk-types/ollama.ts new file mode 100644 index 000000000..1438dda39 --- /dev/null +++ b/js/src/vendor-sdk-types/ollama.ts @@ -0,0 +1,117 @@ +// ollama types + +export interface OllamaOptions { + num_predict?: number; + temperature?: number; + top_p?: number; + frequency_penalty?: number; + presence_penalty?: number; + stop?: string[]; +} + +export interface OllamaToolCall { + id?: string; + function?: { + name?: string; + arguments?: unknown; + }; +} + +export interface OllamaTool { + type?: string; + function?: { + name?: string; + description?: string; + parameters?: Record; + }; +} + +export interface OllamaMessage { + role?: string; + content?: string; + thinking?: string; + images?: unknown[]; + tool_calls?: OllamaToolCall[]; + tool_name?: string; +} + +export interface OllamaChatRequest { + model: string; + messages?: OllamaMessage[]; + stream?: boolean; + format?: string | Record; + tools?: OllamaTool[]; + options?: OllamaOptions; +} + +export interface OllamaGenerateRequest { + model: string; + prompt: string; + suffix?: string; + system?: string; + stream?: boolean; + format?: string | Record; + images?: unknown[]; + options?: OllamaOptions; +} + +export interface OllamaEmbedRequest { + model: string; + input: string | string[]; + dimensions?: number; + options?: OllamaOptions; +} + +export interface OllamaEmbeddingsRequest { + model: string; + prompt: string; + options?: OllamaOptions; +} + +export interface OllamaUsageResponse { + model?: string; + prompt_eval_count?: number; + eval_count?: number; +} + +export interface OllamaChatResponse extends OllamaUsageResponse { + message?: OllamaMessage; + done?: boolean; + done_reason?: string; +} + +export interface OllamaGenerateResponse extends OllamaUsageResponse { + response?: string; + thinking?: string; + done?: boolean; + done_reason?: string; +} + +export interface OllamaEmbedResponse extends OllamaUsageResponse { + embeddings?: number[][]; +} + +export interface OllamaEmbeddingsResponse { + embedding?: number[]; +} + +export interface OllamaAsyncIterator extends AsyncIterable { + abort?: () => void; +} + +export type OllamaChatResult = + | OllamaChatResponse + | OllamaAsyncIterator; + +export type OllamaGenerateResult = + | OllamaGenerateResponse + | OllamaAsyncIterator; + +export interface OllamaClient { + chat?: (request: OllamaChatRequest) => Promise; + generate?: (request: OllamaGenerateRequest) => Promise; + embed?: (request: OllamaEmbedRequest) => Promise; + embeddings?: ( + request: OllamaEmbeddingsRequest, + ) => Promise; +} diff --git a/js/src/wrappers/ollama.test.ts b/js/src/wrappers/ollama.test.ts new file mode 100644 index 000000000..c9355592d --- /dev/null +++ b/js/src/wrappers/ollama.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it, vi } from "vitest"; +import { ollamaChannels } from "../instrumentation/plugins/ollama-channels"; +import type { OllamaClient } from "../vendor-sdk-types/ollama"; +import { wrapOllama } from "./ollama"; + +describe("wrapOllama", () => { + it("emits channel events for every supported generation surface", async () => { + const client: OllamaClient = { + chat: vi.fn(async () => ({ + message: { role: "assistant", content: "OK" }, + done: true, + })), + generate: vi.fn(async () => ({ response: "OK", done: true })), + embed: vi.fn(async () => ({ embeddings: [[0.1, 0.2]] })), + embeddings: vi.fn(async () => ({ embedding: [0.1, 0.2] })), + }; + const chatSpy = vi + .spyOn(ollamaChannels.chat, "tracePromise") + .mockImplementation((fn) => fn()); + const generateSpy = vi + .spyOn(ollamaChannels.generate, "tracePromise") + .mockImplementation((fn) => fn()); + const embedSpy = vi + .spyOn(ollamaChannels.embed, "tracePromise") + .mockImplementation((fn) => fn()); + const embeddingsSpy = vi + .spyOn(ollamaChannels.embeddings, "tracePromise") + .mockImplementation((fn) => fn()); + + const wrapped = wrapOllama(client); + await wrapped.chat?.({ + model: "gpt-oss:20b", + messages: [{ role: "user", content: "Say OK." }], + }); + await wrapped.generate?.({ + model: "gpt-oss:20b", + prompt: "Say OK.", + }); + await wrapped.embed?.({ model: "embeddinggemma", input: "hello" }); + await wrapped.embeddings?.({ + model: "all-minilm", + prompt: "hello", + }); + + expect(chatSpy).toHaveBeenCalledOnce(); + expect(generateSpy).toHaveBeenCalledOnce(); + expect(embedSpy).toHaveBeenCalledOnce(); + expect(embeddingsSpy).toHaveBeenCalledOnce(); + expect(wrapOllama(client)).toBe(wrapped); + expect(wrapOllama(wrapped)).toBe(wrapped); + }); + + it("returns unsupported objects unchanged", () => { + const client = {}; + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + expect(wrapOllama(client)).toBe(client); + expect(warn).toHaveBeenCalledWith( + "Unsupported Ollama library. Not wrapping.", + ); + }); +}); diff --git a/js/src/wrappers/ollama.ts b/js/src/wrappers/ollama.ts new file mode 100644 index 000000000..a573b0446 --- /dev/null +++ b/js/src/wrappers/ollama.ts @@ -0,0 +1,118 @@ +import { ollamaChannels } from "../instrumentation/plugins/ollama-channels"; +import type { + OllamaChatRequest, + OllamaChatResult, + OllamaClient, + OllamaEmbedRequest, + OllamaEmbedResponse, + OllamaEmbeddingsRequest, + OllamaEmbeddingsResponse, + OllamaGenerateRequest, + OllamaGenerateResult, +} from "../vendor-sdk-types/ollama"; + +/** + * Wrap an Ollama client so generation and embedding calls emit Braintrust + * diagnostics-channel events. + */ +export function wrapOllama(ollama: T): T { + if (isSupportedOllamaClient(ollama)) { + return ollamaProxy(ollama) as T; + } + + // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. + console.warn("Unsupported Ollama library. Not wrapping."); + return ollama; +} + +const ollamaProxyCache = new WeakMap(); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function hasFunction(value: Record, name: string): boolean { + return typeof value[name] === "function"; +} + +function isSupportedOllamaClient(value: unknown): value is OllamaClient { + return ( + isRecord(value) && + ["chat", "generate", "embed", "embeddings"].some((name) => + hasFunction(value, name), + ) + ); +} + +function ollamaProxy(ollama: OllamaClient): OllamaClient { + const cached = ollamaProxyCache.get(ollama); + if (cached) { + return cached; + } + + const proxy = new Proxy(ollama, { + get(target, prop, receiver) { + switch (prop) { + case "chat": + return typeof target.chat === "function" + ? wrapChat(target.chat.bind(target)) + : target.chat; + case "generate": + return typeof target.generate === "function" + ? wrapGenerate(target.generate.bind(target)) + : target.generate; + case "embed": + return typeof target.embed === "function" + ? wrapEmbed(target.embed.bind(target)) + : target.embed; + case "embeddings": + return typeof target.embeddings === "function" + ? wrapEmbeddings(target.embeddings.bind(target)) + : target.embeddings; + default: + return Reflect.get(target, prop, receiver); + } + }, + }); + ollamaProxyCache.set(ollama, proxy); + ollamaProxyCache.set(proxy, proxy); + return proxy; +} + +function wrapChat( + chat: (request: OllamaChatRequest) => Promise, +): NonNullable { + return (request) => + ollamaChannels.chat.tracePromise(() => chat(request), { + arguments: [request], + }); +} + +function wrapGenerate( + generate: (request: OllamaGenerateRequest) => Promise, +): NonNullable { + return (request) => + ollamaChannels.generate.tracePromise(() => generate(request), { + arguments: [request], + }); +} + +function wrapEmbed( + embed: (request: OllamaEmbedRequest) => Promise, +): NonNullable { + return (request) => + ollamaChannels.embed.tracePromise(() => embed(request), { + arguments: [request], + }); +} + +function wrapEmbeddings( + embeddings: ( + request: OllamaEmbeddingsRequest, + ) => Promise, +): NonNullable { + return (request) => + ollamaChannels.embeddings.tracePromise(() => embeddings(request), { + arguments: [request], + }); +} From 44bbfa2a937593ca9294045f971e7c22cfb381cc Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:51:33 +0000 Subject: [PATCH 2/8] Update PR #2315 --- .changeset/ollama-instrumentation.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/ollama-instrumentation.md diff --git a/.changeset/ollama-instrumentation.md b/.changeset/ollama-instrumentation.md new file mode 100644 index 000000000..52443f3be --- /dev/null +++ b/.changeset/ollama-instrumentation.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat: Add `ollama` instrumentation From 388e2f42a95ca8437739a8b71ee65facc99b6c31 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:04:26 +0000 Subject: [PATCH 3/8] Update PR #2315 --- .../pi-coding-agent-v0-auto-hook.span-tree.json | 2 +- .../pi-coding-agent-v0-auto-hook.span-tree.txt | 2 +- .../pi-coding-agent-v0-latest-auto-hook.span-tree.json | 2 +- .../pi-coding-agent-v0-latest-auto-hook.span-tree.txt | 2 +- .../pi-coding-agent-v0-latest-wrapped.span-tree.json | 2 +- .../pi-coding-agent-v0-latest-wrapped.span-tree.txt | 2 +- .../pi-coding-agent-v0-wrapped.span-tree.json | 2 +- .../pi-coding-agent-v0-wrapped.span-tree.txt | 2 +- js/src/instrumentation/plugins/ollama-plugin.ts | 10 ++++------ 9 files changed, 12 insertions(+), 14 deletions(-) diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json index 448e3137e..04093af29 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt index 1b306df81..29fdfabcc 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json index 2e256c94b..dea4dfab8 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt index ad01a3041..0c93a9b26 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json index 2e256c94b..dea4dfab8 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt index ad01a3041..0c93a9b26 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json index 448e3137e..04093af29 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt index 1b306df81..29fdfabcc 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/js/src/instrumentation/plugins/ollama-plugin.ts b/js/src/instrumentation/plugins/ollama-plugin.ts index 831fb07e1..be3eec9ee 100644 --- a/js/src/instrumentation/plugins/ollama-plugin.ts +++ b/js/src/instrumentation/plugins/ollama-plugin.ts @@ -399,7 +399,7 @@ export function extractOllamaGenerateInput(args: unknown): { }; } -export function extractOllamaEmbedInput(args: unknown): { +function extractOllamaEmbedInput(args: unknown): { input: unknown; metadata: Record; } { @@ -410,7 +410,7 @@ export function extractOllamaEmbedInput(args: unknown): { }; } -export function extractOllamaEmbeddingsInput(args: unknown): { +function extractOllamaEmbeddingsInput(args: unknown): { input: unknown; metadata: Record; } { @@ -441,9 +441,7 @@ export function extractOllamaChatOutput(result: OllamaChatResponse): unknown { ]; } -export function extractOllamaGenerateOutput( - result: OllamaGenerateResponse, -): unknown { +function extractOllamaGenerateOutput(result: OllamaGenerateResponse): unknown { if (!isObject(result) || typeof result.response !== "string") { return undefined; } @@ -480,7 +478,7 @@ export function extractOllamaEmbeddingsOutput( : undefined; } -export function extractOllamaResponseMetadata( +function extractOllamaResponseMetadata( result: OllamaUsageResponse, ): Record | undefined { return typeof result?.model === "string" From 93be74217962401c0fd9d87e4688f9572c13b559 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Mon, 3 Aug 2026 09:53:18 +0000 Subject: [PATCH 4/8] Update PR #2315 --- js/src/wrappers/ollama.test.ts | 3 ++- js/src/wrappers/ollama.ts | 19 ++++++------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/js/src/wrappers/ollama.test.ts b/js/src/wrappers/ollama.test.ts index c9355592d..1318d82aa 100644 --- a/js/src/wrappers/ollama.test.ts +++ b/js/src/wrappers/ollama.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import { debugLogger } from "../debug-logger"; import { ollamaChannels } from "../instrumentation/plugins/ollama-channels"; import type { OllamaClient } from "../vendor-sdk-types/ollama"; import { wrapOllama } from "./ollama"; @@ -52,7 +53,7 @@ describe("wrapOllama", () => { it("returns unsupported objects unchanged", () => { const client = {}; - const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const warn = vi.spyOn(debugLogger, "warn").mockImplementation(() => {}); expect(wrapOllama(client)).toBe(client); expect(warn).toHaveBeenCalledWith( diff --git a/js/src/wrappers/ollama.ts b/js/src/wrappers/ollama.ts index a573b0446..07c66b731 100644 --- a/js/src/wrappers/ollama.ts +++ b/js/src/wrappers/ollama.ts @@ -1,4 +1,6 @@ +import { debugLogger } from "../debug-logger"; import { ollamaChannels } from "../instrumentation/plugins/ollama-channels"; +import { isObject } from "../../util"; import type { OllamaChatRequest, OllamaChatResult, @@ -20,26 +22,17 @@ export function wrapOllama(ollama: T): T { return ollamaProxy(ollama) as T; } - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn("Unsupported Ollama library. Not wrapping."); + debugLogger.warn("Unsupported Ollama library. Not wrapping."); return ollama; } const ollamaProxyCache = new WeakMap(); -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null; -} - -function hasFunction(value: Record, name: string): boolean { - return typeof value[name] === "function"; -} - function isSupportedOllamaClient(value: unknown): value is OllamaClient { return ( - isRecord(value) && - ["chat", "generate", "embed", "embeddings"].some((name) => - hasFunction(value, name), + isObject(value) && + ["chat", "generate", "embed", "embeddings"].some( + (name) => typeof value[name] === "function", ) ); } From 223396eef540e417f644f9db7fa737bf911b9b7b Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:51:10 +0000 Subject: [PATCH 5/8] Update PR #2315 --- e2e/config/pr-comment-scenarios.json | 16 +- .../ollama-v0.6-auto.span-tree.json | 22 --- .../ollama-v0.6-auto.span-tree.txt | 14 -- .../ollama-v0.6-latest-auto.span-tree.json | 22 --- .../ollama-v0.6-latest-auto.span-tree.txt | 14 -- .../ollama-v0.6-latest-wrapped.span-tree.json | 22 --- .../ollama-v0.6-latest-wrapped.span-tree.txt | 14 -- .../ollama-v0.6-wrapped.span-tree.json | 22 --- .../ollama-v0.6-wrapped.span-tree.txt | 14 -- .../ollama-instrumentation/assertions.ts | 15 +- .../ollama-instrumentation/constants.mjs | 1 - .../ollama-instrumentation/scenario.impl.mjs | 16 -- .../auto-instrumentations/configs/ollama.ts | 1 - .../plugins/ollama-channels.ts | 6 - .../plugins/ollama-plugin.test.ts | 143 +++++++++++++++++- .../instrumentation/plugins/ollama-plugin.ts | 131 +++++++++++----- js/src/vendor-sdk-types/ollama.ts | 13 -- js/src/wrappers/ollama.test.ts | 44 ++++-- js/src/wrappers/ollama.ts | 71 +++++---- 19 files changed, 307 insertions(+), 294 deletions(-) diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index a5470a3ab..41bdccb92 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -403,20 +403,12 @@ "metadataScenario": "ollama-instrumentation", "variants": [ { - "variantKey": "ollama-v0.6-wrapped", - "label": "v0.6 pinned wrapped" + "variantKey": "ollama-v0.6", + "label": "v0.6 pinned" }, { - "variantKey": "ollama-v0.6-auto", - "label": "v0.6 pinned auto-hook" - }, - { - "variantKey": "ollama-v0.6-latest-wrapped", - "label": "v0.6 latest wrapped" - }, - { - "variantKey": "ollama-v0.6-latest-auto", - "label": "v0.6 latest auto-hook" + "variantKey": "ollama-v0.6-latest", + "label": "v0.6 latest" } ] }, diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json index 1fe6de648..c0779a0ce 100644 --- a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json @@ -272,28 +272,6 @@ "testRunId": "" } }, - { - "name": "ollama-embeddings-operation", - "children": [ - { - "name": "ollama.embeddings", - "type": "llm", - "children": [], - "input": "braintrust tracing", - "output": { - "embedding_length": 3 - }, - "metadata": { - "model": "all-minilm", - "provider": "ollama" - } - } - ], - "metadata": { - "operation": "embeddings", - "testRunId": "" - } - }, { "name": "ollama-error-operation", "children": [ diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt index efdb2be28..fe6c0d5f8 100644 --- a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt @@ -224,20 +224,6 @@ span_tree: │ "prompt_tokens": 4, │ "tokens": 4 │ } - ├── ollama-embeddings-operation - │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ └── ollama.embeddings [llm] - │ input: "braintrust tracing" - │ output: { - │ "embedding_length": 3 - │ } - │ metadata: { - │ "model": "all-minilm", - │ "provider": "ollama" - │ } └── ollama-error-operation metadata: { "operation": "error", diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json index e6959f455..cf9cd619c 100644 --- a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json @@ -272,28 +272,6 @@ "testRunId": "" } }, - { - "name": "ollama-embeddings-operation", - "children": [ - { - "name": "ollama.embeddings", - "type": "llm", - "children": [], - "input": "braintrust tracing", - "output": { - "embedding_length": 3 - }, - "metadata": { - "model": "all-minilm", - "provider": "ollama" - } - } - ], - "metadata": { - "operation": "embeddings", - "testRunId": "" - } - }, { "name": "ollama-error-operation", "children": [ diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt index b75077038..8560418e2 100644 --- a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt @@ -224,20 +224,6 @@ span_tree: │ "prompt_tokens": 4, │ "tokens": 4 │ } - ├── ollama-embeddings-operation - │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ └── ollama.embeddings [llm] - │ input: "braintrust tracing" - │ output: { - │ "embedding_length": 3 - │ } - │ metadata: { - │ "model": "all-minilm", - │ "provider": "ollama" - │ } └── ollama-error-operation metadata: { "operation": "error", diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json index e6959f455..cf9cd619c 100644 --- a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json @@ -272,28 +272,6 @@ "testRunId": "" } }, - { - "name": "ollama-embeddings-operation", - "children": [ - { - "name": "ollama.embeddings", - "type": "llm", - "children": [], - "input": "braintrust tracing", - "output": { - "embedding_length": 3 - }, - "metadata": { - "model": "all-minilm", - "provider": "ollama" - } - } - ], - "metadata": { - "operation": "embeddings", - "testRunId": "" - } - }, { "name": "ollama-error-operation", "children": [ diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt index b75077038..8560418e2 100644 --- a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt @@ -224,20 +224,6 @@ span_tree: │ "prompt_tokens": 4, │ "tokens": 4 │ } - ├── ollama-embeddings-operation - │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ └── ollama.embeddings [llm] - │ input: "braintrust tracing" - │ output: { - │ "embedding_length": 3 - │ } - │ metadata: { - │ "model": "all-minilm", - │ "provider": "ollama" - │ } └── ollama-error-operation metadata: { "operation": "error", diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json index 1fe6de648..c0779a0ce 100644 --- a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json @@ -272,28 +272,6 @@ "testRunId": "" } }, - { - "name": "ollama-embeddings-operation", - "children": [ - { - "name": "ollama.embeddings", - "type": "llm", - "children": [], - "input": "braintrust tracing", - "output": { - "embedding_length": 3 - }, - "metadata": { - "model": "all-minilm", - "provider": "ollama" - } - } - ], - "metadata": { - "operation": "embeddings", - "testRunId": "" - } - }, { "name": "ollama-error-operation", "children": [ diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt index efdb2be28..fe6c0d5f8 100644 --- a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt @@ -224,20 +224,6 @@ span_tree: │ "prompt_tokens": 4, │ "tokens": 4 │ } - ├── ollama-embeddings-operation - │ metadata: { - │ "operation": "embeddings", - │ "testRunId": "" - │ } - │ └── ollama.embeddings [llm] - │ input: "braintrust tracing" - │ output: { - │ "embedding_length": 3 - │ } - │ metadata: { - │ "model": "all-minilm", - │ "provider": "ollama" - │ } └── ollama-error-operation metadata: { "operation": "error", diff --git a/e2e/scenarios/ollama-instrumentation/assertions.ts b/e2e/scenarios/ollama-instrumentation/assertions.ts index 334d03a00..a8dcd1df7 100644 --- a/e2e/scenarios/ollama-instrumentation/assertions.ts +++ b/e2e/scenarios/ollama-instrumentation/assertions.ts @@ -10,7 +10,6 @@ import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; import { EMBEDDING_MODEL, GENERATION_MODEL, - LEGACY_EMBEDDING_MODEL, ROOT_NAME, SCENARIO_NAME, } from "./constants.mjs"; @@ -54,7 +53,6 @@ function selectedEvents(events: CapturedLogEvent[]): CapturedLogEvent[] { ["ollama-generate-stream-operation", "ollama.generate"], ["ollama-tool-call-operation", "ollama.chat"], ["ollama-embed-operation", "ollama.embed"], - ["ollama-embeddings-operation", "ollama.embeddings"], ["ollama-error-operation", "ollama.chat"], ] as const; return [ @@ -180,17 +178,12 @@ export function defineOllamaInstrumentationAssertions(options: { }); }); - test("captures current and legacy embedding calls compactly", () => { + test("captures embedding calls compactly", () => { const embed = findProviderSpan( events, "ollama-embed-operation", "ollama.embed", ); - const embeddings = findProviderSpan( - events, - "ollama-embeddings-operation", - "ollama.embeddings", - ); expect(embed?.row.metadata).toMatchObject({ model: EMBEDDING_MODEL, @@ -201,12 +194,6 @@ export function defineOllamaInstrumentationAssertions(options: { prompt_tokens: 4, tokens: 4, }); - - expect(embeddings?.row.metadata).toMatchObject({ - model: LEGACY_EMBEDDING_MODEL, - provider: "ollama", - }); - expect(embeddings?.output).toEqual({ embedding_length: 3 }); }); test("captures provider errors without changing caller behavior", () => { diff --git a/e2e/scenarios/ollama-instrumentation/constants.mjs b/e2e/scenarios/ollama-instrumentation/constants.mjs index e69d98a71..025d1d866 100644 --- a/e2e/scenarios/ollama-instrumentation/constants.mjs +++ b/e2e/scenarios/ollama-instrumentation/constants.mjs @@ -2,4 +2,3 @@ export const ROOT_NAME = "ollama-instrumentation-root"; export const SCENARIO_NAME = "ollama-instrumentation"; export const GENERATION_MODEL = "gpt-oss:20b"; export const EMBEDDING_MODEL = "embeddinggemma"; -export const LEGACY_EMBEDDING_MODEL = "all-minilm"; diff --git a/e2e/scenarios/ollama-instrumentation/scenario.impl.mjs b/e2e/scenarios/ollama-instrumentation/scenario.impl.mjs index 4a0718764..7595b4cdc 100644 --- a/e2e/scenarios/ollama-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/ollama-instrumentation/scenario.impl.mjs @@ -7,7 +7,6 @@ import { import { EMBEDDING_MODEL, GENERATION_MODEL, - LEGACY_EMBEDDING_MODEL, ROOT_NAME, SCENARIO_NAME, } from "./constants.mjs"; @@ -53,10 +52,6 @@ async function syntheticFetch(input, init) { }); } - if (url.endsWith("/api/embeddings")) { - return jsonResponse({ embedding: [0.1, 0.2, 0.3] }); - } - if (url.endsWith("/api/chat") && Array.isArray(body.tools)) { return jsonResponse({ model: body.model, @@ -209,17 +204,6 @@ async function runOllamaInstrumentationScenario( }); }); - await runOperation( - "ollama-embeddings-operation", - "embeddings", - async () => { - await syntheticClient.embeddings({ - model: LEGACY_EMBEDDING_MODEL, - prompt: "braintrust tracing", - }); - }, - ); - await runOperation("ollama-error-operation", "error", async () => { try { await syntheticClient.chat({ diff --git a/js/src/auto-instrumentations/configs/ollama.ts b/js/src/auto-instrumentations/configs/ollama.ts index 1db36b8f0..5ef28eb64 100644 --- a/js/src/auto-instrumentations/configs/ollama.ts +++ b/js/src/auto-instrumentations/configs/ollama.ts @@ -5,7 +5,6 @@ const methods = [ ["chat", ollamaChannels.chat.channelName], ["generate", ollamaChannels.generate.channelName], ["embed", ollamaChannels.embed.channelName], - ["embeddings", ollamaChannels.embeddings.channelName], ] as const; // Ollama's Node entry points in dist/index.{mjs,cjs} subclass the Ollama class diff --git a/js/src/instrumentation/plugins/ollama-channels.ts b/js/src/instrumentation/plugins/ollama-channels.ts index 70b7d8533..c6e4f2044 100644 --- a/js/src/instrumentation/plugins/ollama-channels.ts +++ b/js/src/instrumentation/plugins/ollama-channels.ts @@ -5,8 +5,6 @@ import type { OllamaChatResult, OllamaEmbedRequest, OllamaEmbedResponse, - OllamaEmbeddingsRequest, - OllamaEmbeddingsResponse, OllamaGenerateRequest, OllamaGenerateResponse, OllamaGenerateResult, @@ -38,10 +36,6 @@ export const ollamaChannels = defineChannels( channelName: "embed", kind: "async", }), - embeddings: channel<[OllamaEmbeddingsRequest], OllamaEmbeddingsResponse>({ - channelName: "embeddings", - kind: "async", - }), }, { instrumentationName: INSTRUMENTATION_NAMES.OLLAMA }, ); diff --git a/js/src/instrumentation/plugins/ollama-plugin.test.ts b/js/src/instrumentation/plugins/ollama-plugin.test.ts index 4c9690d63..11cb77745 100644 --- a/js/src/instrumentation/plugins/ollama-plugin.test.ts +++ b/js/src/instrumentation/plugins/ollama-plugin.test.ts @@ -1,15 +1,18 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import iso from "../../isomorph"; +import { configureNode } from "../../node/config"; import { aggregateOllamaChatChunks, aggregateOllamaGenerateChunks, extractOllamaChatInput, extractOllamaChatOutput, extractOllamaEmbedOutput, - extractOllamaEmbeddingsOutput, extractOllamaGenerateInput, extractOllamaMetrics, } from "./ollama-plugin"; +configureNode(); + describe("Ollama instrumentation extraction", () => { it("normalizes chat inputs, tools, and supported request metadata", () => { const result = extractOllamaChatInput([ @@ -28,13 +31,41 @@ describe("Ollama instrumentation extraction", () => { arguments: { city: "Paris" }, }, }, + { + function: { + name: "get_forecast", + arguments: { city: "Paris" }, + }, + }, ], }, + { + role: "tool", + tool_name: "get_forecast", + content: '{"forecast":"sunny"}', + }, { role: "tool", tool_name: "get_weather", content: '{"temperature":18}', }, + { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "get_weather", + arguments: { city: "Paris" }, + }, + }, + ], + }, + { + role: "tool", + tool_name: "get_weather", + content: '{"temperature":19}', + }, ], tools: [ { @@ -78,13 +109,45 @@ describe("Ollama instrumentation extraction", () => { arguments: '{"city":"Paris"}', }, }, + { + id: "ollama_call_get_forecast_1", + type: "function", + function: { + name: "get_forecast", + arguments: '{"city":"Paris"}', + }, + }, ], }, + { + role: "tool", + tool_call_id: "ollama_call_get_forecast_1", + content: '{"forecast":"sunny"}', + }, { role: "tool", tool_call_id: "ollama_call_get_weather_0", content: '{"temperature":18}', }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "ollama_call_get_weather_2", + type: "function", + function: { + name: "get_weather", + arguments: '{"city":"Paris"}', + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "ollama_call_get_weather_2", + content: '{"temperature":19}', + }, ]); expect(result.metadata).toEqual({ provider: "ollama", @@ -151,6 +214,63 @@ describe("Ollama instrumentation extraction", () => { }); }); + it("converts local image paths into attachments", () => { + const statSync = vi + .spyOn(iso, "statSync") + .mockReturnValue({ isFile: () => true }); + const result = extractOllamaGenerateInput([ + { + model: "llava", + prompt: "Describe this image.", + images: ["/tmp/example.webp"], + }, + ]); + statSync.mockRestore(); + + const content = ( + result.input as Array<{ + content: Array<{ + type: string; + image_url?: { url?: { reference?: unknown } }; + }>; + }> + )[0].content; + expect(content[1]).toMatchObject({ + type: "image_url", + image_url: { + url: { + reference: { + type: "braintrust_attachment", + filename: "example.webp", + content_type: "image/webp", + }, + }, + }, + }); + }); + + it("preserves image-looking strings that are not readable local paths", () => { + const statSync = vi.spyOn(iso, "statSync").mockImplementation(() => { + throw new Error("missing"); + }); + const result = extractOllamaGenerateInput([ + { + model: "llava", + prompt: "Describe this image.", + images: ["missing.webp"], + }, + ]); + statSync.mockRestore(); + + expect(result.input).toEqual([ + { + role: "user", + content: "Describe this image.", + images: ["missing.webp"], + }, + ]); + }); + it("normalizes chat output and tool calls as OpenAI choices", () => { expect( extractOllamaChatOutput({ @@ -235,6 +355,18 @@ describe("Ollama instrumentation extraction", () => { eval_count: Number.NaN, }), ).toEqual({}); + expect( + extractOllamaMetrics({ + prompt_eval_count: Number.NaN, + eval_count: 3, + }), + ).toEqual({ completion_tokens: 3, tokens: 3 }); + expect( + extractOllamaMetrics({ + prompt_eval_count: 7, + eval_count: -1, + }), + ).toEqual({ prompt_tokens: 7, tokens: 7 }); }); it("aggregates chat and generation streams into final outputs", () => { @@ -302,7 +434,7 @@ describe("Ollama instrumentation extraction", () => { }); }); - it("summarizes current and legacy embedding responses", () => { + it("summarizes embedding responses", () => { expect( extractOllamaEmbedOutput({ model: "embeddinggemma", @@ -312,10 +444,5 @@ describe("Ollama instrumentation extraction", () => { ], }), ).toEqual({ embedding_length: 3 }); - expect( - extractOllamaEmbeddingsOutput({ - embedding: [0.1, 0.2], - }), - ).toEqual({ embedding_length: 2 }); }); }); diff --git a/js/src/instrumentation/plugins/ollama-plugin.ts b/js/src/instrumentation/plugins/ollama-plugin.ts index be3eec9ee..4aa8bdc65 100644 --- a/js/src/instrumentation/plugins/ollama-plugin.ts +++ b/js/src/instrumentation/plugins/ollama-plugin.ts @@ -1,9 +1,10 @@ import { SpanTypeAttribute, isObject } from "../../../util/index"; +import iso from "../../isomorph"; +import { Attachment } from "../../logger"; import { processInputAttachments } from "../../wrappers/attachment-utils"; import type { OllamaChatResponse, OllamaEmbedResponse, - OllamaEmbeddingsResponse, OllamaGenerateResponse, OllamaMessage, OllamaTool, @@ -47,13 +48,6 @@ export class OllamaPlugin extends BasePlugin { extractMetadata: extractOllamaResponseMetadata, extractMetrics: extractOllamaMetrics, }), - traceAsyncChannel(ollamaChannels.embeddings, { - name: "ollama.embeddings", - type: SpanTypeAttribute.LLM, - extractInput: extractOllamaEmbeddingsInput, - extractOutput: extractOllamaEmbeddingsOutput, - extractMetrics: () => ({}), - }), ); } @@ -133,13 +127,14 @@ function normalizeToolCall( function normalizeToolCalls( toolCalls: OllamaToolCall[] | undefined, + syntheticIdOffset = 0, ): Record[] { if (!Array.isArray(toolCalls)) { return []; } return toolCalls.flatMap((toolCall, index) => { - const normalized = normalizeToolCall(toolCall, index); + const normalized = normalizeToolCall(toolCall, syntheticIdOffset + index); return normalized ? [normalized] : []; }); } @@ -209,19 +204,52 @@ function normalizeTextAndImages( const imageParts: Record[] = []; const unrecognizedImages: unknown[] = []; for (const image of images) { - const mediaType = inferImageMediaType(image); + let localImagePath: string | undefined; + let localPathMediaType: string | undefined; + if ( + typeof image === "string" && + !/^data:/i.test(image) && + !/^https?:\/\//i.test(image) + ) { + const extension = image.match(/\.([a-z0-9]+)$/i)?.[1]?.toLowerCase(); + const extensionMediaType = { + png: "image/png", + jpg: "image/jpeg", + jpeg: "image/jpeg", + gif: "image/gif", + webp: "image/webp", + }[extension ?? ""]; + if (extensionMediaType && iso.statSync) { + try { + if (iso.statSync(image).isFile()) { + localImagePath = image; + localPathMediaType = extensionMediaType; + } + } catch { + // Ollama treats unreadable strings as base64, so preserve them below. + } + } + } + + const mediaType = localPathMediaType ?? inferImageMediaType(image); if (!mediaType) { unrecognizedImages.push(image); continue; } - const processed = processInputAttachments({ - type: "image", - image, - mediaType, - }); + const processedImage = localImagePath + ? new Attachment({ + data: localImagePath, + filename: localImagePath.split(/[\\/]/).at(-1) ?? "image", + contentType: mediaType, + }) + : processInputAttachments({ + type: "image", + image, + mediaType, + }).image; imageParts.push({ type: "image_url", - image_url: { url: processed.image }, + image_url: { url: processedImage }, }); } @@ -236,12 +264,17 @@ function normalizeTextAndImages( function normalizeMessage( message: OllamaMessage, + toolCallId?: string, + syntheticToolCallIdOffset = 0, ): Record | undefined { if (typeof message.role !== "string") { return undefined; } - const toolCalls = normalizeToolCalls(message.tool_calls); + const toolCalls = normalizeToolCalls( + message.tool_calls, + syntheticToolCallIdOffset, + ); if (message.role === "tool") { const toolName = typeof message.tool_name === "string" && message.tool_name.length > 0 @@ -249,7 +282,7 @@ function normalizeMessage( : "tool"; return { role: "tool", - tool_call_id: syntheticToolCallId(toolName, 0), + tool_call_id: toolCallId ?? syntheticToolCallId(toolName, 0), content: typeof message.content === "string" ? message.content : "", }; } @@ -279,11 +312,48 @@ function normalizeMessages(messages: unknown): Record[] { if (!Array.isArray(messages)) { return []; } + + const pendingToolCallIds = new Map(); + let syntheticToolCallIdOffset = 0; return messages.flatMap((message) => { if (!isObject(message)) { return []; } - const normalized = normalizeMessage(message as OllamaMessage); + + const ollamaMessage = message as OllamaMessage; + let toolCallId: string | undefined; + if (ollamaMessage.role === "tool") { + const toolName = ollamaMessage.tool_name; + if (typeof toolName === "string") { + toolCallId = pendingToolCallIds.get(toolName)?.shift(); + } + } + + const normalized = normalizeMessage( + ollamaMessage, + toolCallId, + syntheticToolCallIdOffset, + ); + if (Array.isArray(ollamaMessage.tool_calls)) { + syntheticToolCallIdOffset += ollamaMessage.tool_calls.length; + } + if (ollamaMessage.role === "assistant" && normalized) { + const toolCalls = Array.isArray(normalized.tool_calls) + ? normalized.tool_calls + : []; + for (const toolCall of toolCalls) { + if (!isObject(toolCall) || !isObject(toolCall.function)) { + continue; + } + const name = toolCall.function.name; + const id = toolCall.id; + if (typeof name === "string" && typeof id === "string") { + const ids = pendingToolCallIds.get(name) ?? []; + ids.push(id); + pendingToolCallIds.set(name, ids); + } + } + } return normalized ? [normalized] : []; }); } @@ -410,17 +480,6 @@ function extractOllamaEmbedInput(args: unknown): { }; } -function extractOllamaEmbeddingsInput(args: unknown): { - input: unknown; - metadata: Record; -} { - const request = getRequestArg(args); - return { - input: request?.prompt, - metadata: extractRequestMetadata(request), - }; -} - export function extractOllamaChatOutput(result: OllamaChatResponse): unknown { if (!isObject(result) || !isObject(result.message)) { return undefined; @@ -470,14 +529,6 @@ export function extractOllamaEmbedOutput(result: OllamaEmbedResponse): unknown { : undefined; } -export function extractOllamaEmbeddingsOutput( - result: OllamaEmbeddingsResponse, -): unknown { - return Array.isArray(result?.embedding) - ? { embedding_length: result.embedding.length } - : undefined; -} - function extractOllamaResponseMetadata( result: OllamaUsageResponse, ): Record | undefined { @@ -503,7 +554,9 @@ export function extractOllamaMetrics( isNonNegativeNumber(promptTokens) || isNonNegativeNumber(completionTokens) ) { - metrics.tokens = (promptTokens ?? 0) + (completionTokens ?? 0); + metrics.tokens = + (isNonNegativeNumber(promptTokens) ? promptTokens : 0) + + (isNonNegativeNumber(completionTokens) ? completionTokens : 0); } return metrics; } diff --git a/js/src/vendor-sdk-types/ollama.ts b/js/src/vendor-sdk-types/ollama.ts index 1438dda39..73dfa98d6 100644 --- a/js/src/vendor-sdk-types/ollama.ts +++ b/js/src/vendor-sdk-types/ollama.ts @@ -62,12 +62,6 @@ export interface OllamaEmbedRequest { options?: OllamaOptions; } -export interface OllamaEmbeddingsRequest { - model: string; - prompt: string; - options?: OllamaOptions; -} - export interface OllamaUsageResponse { model?: string; prompt_eval_count?: number; @@ -91,10 +85,6 @@ export interface OllamaEmbedResponse extends OllamaUsageResponse { embeddings?: number[][]; } -export interface OllamaEmbeddingsResponse { - embedding?: number[]; -} - export interface OllamaAsyncIterator extends AsyncIterable { abort?: () => void; } @@ -111,7 +101,4 @@ export interface OllamaClient { chat?: (request: OllamaChatRequest) => Promise; generate?: (request: OllamaGenerateRequest) => Promise; embed?: (request: OllamaEmbedRequest) => Promise; - embeddings?: ( - request: OllamaEmbeddingsRequest, - ) => Promise; } diff --git a/js/src/wrappers/ollama.test.ts b/js/src/wrappers/ollama.test.ts index 1318d82aa..9a8ebff46 100644 --- a/js/src/wrappers/ollama.test.ts +++ b/js/src/wrappers/ollama.test.ts @@ -13,7 +13,6 @@ describe("wrapOllama", () => { })), generate: vi.fn(async () => ({ response: "OK", done: true })), embed: vi.fn(async () => ({ embeddings: [[0.1, 0.2]] })), - embeddings: vi.fn(async () => ({ embedding: [0.1, 0.2] })), }; const chatSpy = vi .spyOn(ollamaChannels.chat, "tracePromise") @@ -24,11 +23,10 @@ describe("wrapOllama", () => { const embedSpy = vi .spyOn(ollamaChannels.embed, "tracePromise") .mockImplementation((fn) => fn()); - const embeddingsSpy = vi - .spyOn(ollamaChannels.embeddings, "tracePromise") - .mockImplementation((fn) => fn()); - const wrapped = wrapOllama(client); + expect(wrapped.chat).toBe(wrapped.chat); + expect(wrapped.generate).toBe(wrapped.generate); + expect(wrapped.embed).toBe(wrapped.embed); await wrapped.chat?.({ model: "gpt-oss:20b", messages: [{ role: "user", content: "Say OK." }], @@ -38,15 +36,10 @@ describe("wrapOllama", () => { prompt: "Say OK.", }); await wrapped.embed?.({ model: "embeddinggemma", input: "hello" }); - await wrapped.embeddings?.({ - model: "all-minilm", - prompt: "hello", - }); expect(chatSpy).toHaveBeenCalledOnce(); expect(generateSpy).toHaveBeenCalledOnce(); expect(embedSpy).toHaveBeenCalledOnce(); - expect(embeddingsSpy).toHaveBeenCalledOnce(); expect(wrapOllama(client)).toBe(wrapped); expect(wrapOllama(wrapped)).toBe(wrapped); }); @@ -60,4 +53,35 @@ describe("wrapOllama", () => { "Unsupported Ollama library. Not wrapping.", ); }); + + it("refreshes stable wrappers when client methods change", async () => { + const originalChat = vi.fn(async () => ({ + message: { role: "assistant", content: "original" }, + done: true, + })); + const client: OllamaClient = { chat: originalChat }; + const tracePromise = vi + .spyOn(ollamaChannels.chat, "tracePromise") + .mockImplementation((fn) => fn()); + const wrapped = wrapOllama(client); + const firstWrappedChat = wrapped.chat; + + expect(firstWrappedChat).toBe(wrapped.chat); + + const replacementChat = vi.fn(async () => ({ + message: { role: "assistant", content: "replacement" }, + done: true, + })); + client.chat = replacementChat; + + expect(wrapped.chat).not.toBe(firstWrappedChat); + expect(wrapped.chat).toBe(wrapped.chat); + await wrapped.chat?.({ model: "gpt-oss:20b", messages: [] }); + expect(originalChat).not.toHaveBeenCalled(); + expect(replacementChat).toHaveBeenCalledOnce(); + + client.chat = undefined; + expect(wrapped.chat).toBeUndefined(); + tracePromise.mockRestore(); + }); }); diff --git a/js/src/wrappers/ollama.ts b/js/src/wrappers/ollama.ts index 07c66b731..3f90067a9 100644 --- a/js/src/wrappers/ollama.ts +++ b/js/src/wrappers/ollama.ts @@ -7,8 +7,6 @@ import type { OllamaClient, OllamaEmbedRequest, OllamaEmbedResponse, - OllamaEmbeddingsRequest, - OllamaEmbeddingsResponse, OllamaGenerateRequest, OllamaGenerateResult, } from "../vendor-sdk-types/ollama"; @@ -31,7 +29,7 @@ const ollamaProxyCache = new WeakMap(); function isSupportedOllamaClient(value: unknown): value is OllamaClient { return ( isObject(value) && - ["chat", "generate", "embed", "embeddings"].some( + ["chat", "generate", "embed"].some( (name) => typeof value[name] === "function", ) ); @@ -43,25 +41,49 @@ function ollamaProxy(ollama: OllamaClient): OllamaClient { return cached; } + let chatSource: OllamaClient["chat"]; + let wrappedChat: OllamaClient["chat"]; + let generateSource: OllamaClient["generate"]; + let wrappedGenerate: OllamaClient["generate"]; + let embedSource: OllamaClient["embed"]; + let wrappedEmbed: OllamaClient["embed"]; + const proxy = new Proxy(ollama, { get(target, prop, receiver) { switch (prop) { - case "chat": - return typeof target.chat === "function" - ? wrapChat(target.chat.bind(target)) - : target.chat; - case "generate": - return typeof target.generate === "function" - ? wrapGenerate(target.generate.bind(target)) - : target.generate; - case "embed": - return typeof target.embed === "function" - ? wrapEmbed(target.embed.bind(target)) - : target.embed; - case "embeddings": - return typeof target.embeddings === "function" - ? wrapEmbeddings(target.embeddings.bind(target)) - : target.embeddings; + case "chat": { + const source = target.chat; + if (source !== chatSource) { + chatSource = source; + wrappedChat = + typeof source === "function" + ? wrapChat(source.bind(target)) + : source; + } + return wrappedChat; + } + case "generate": { + const source = target.generate; + if (source !== generateSource) { + generateSource = source; + wrappedGenerate = + typeof source === "function" + ? wrapGenerate(source.bind(target)) + : source; + } + return wrappedGenerate; + } + case "embed": { + const source = target.embed; + if (source !== embedSource) { + embedSource = source; + wrappedEmbed = + typeof source === "function" + ? wrapEmbed(source.bind(target)) + : source; + } + return wrappedEmbed; + } default: return Reflect.get(target, prop, receiver); } @@ -98,14 +120,3 @@ function wrapEmbed( arguments: [request], }); } - -function wrapEmbeddings( - embeddings: ( - request: OllamaEmbeddingsRequest, - ) => Promise, -): NonNullable { - return (request) => - ollamaChannels.embeddings.tracePromise(() => embeddings(request), { - arguments: [request], - }); -} From e9415ed1e9246cf5070ded20d6b79f0ed3172ec2 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:05:50 +0000 Subject: [PATCH 6/8] Update PR #2315 --- .../core/stream-patcher.test.ts | 69 ++++++++++++++ js/src/instrumentation/core/stream-patcher.ts | 48 +++++++++- .../plugins/ollama-plugin.test.ts | 38 ++++++-- .../instrumentation/plugins/ollama-plugin.ts | 92 ++++++++++--------- 4 files changed, 193 insertions(+), 54 deletions(-) diff --git a/js/src/instrumentation/core/stream-patcher.test.ts b/js/src/instrumentation/core/stream-patcher.test.ts index 8a71b90e5..f2c86b13d 100644 --- a/js/src/instrumentation/core/stream-patcher.test.ts +++ b/js/src/instrumentation/core/stream-patcher.test.ts @@ -258,6 +258,75 @@ describe("patchStreamIfNeeded", () => { expect(onComplete).toHaveBeenCalledWith([1, 2]); }); + it("should complete an abortable stream when abort() is called", async () => { + const iterator = { + values: [1, 2, 3], + index: 0, + async next() { + const value = this.values[this.index++]; + return value === undefined + ? { done: true as const, value: undefined } + : { done: false as const, value }; + }, + }; + const abort = vi.fn(() => "aborted"); + const stream = { + abort, + [Symbol.asyncIterator]() { + return iterator; + }, + }; + const onComplete = vi.fn(); + const patched = patchStreamIfNeeded(stream, { + onComplete, + }) as typeof stream; + const patchedIterator = patched[Symbol.asyncIterator](); + + await patchedIterator.next(); + await patchedIterator.next(); + + expect(patched.abort()).toBe("aborted"); + expect(abort).toHaveBeenCalledOnce(); + expect(onComplete).toHaveBeenCalledOnce(); + expect(onComplete).toHaveBeenCalledWith([1, 2]); + + patched.abort(); + expect(onComplete).toHaveBeenCalledOnce(); + }); + + it("should complete an abortable self-iterator when abort() is called", async () => { + const abort = vi.fn(() => "aborted"); + const stream = { + values: [1, 2, 3], + index: 0, + abort, + async next() { + const value = this.values[this.index++]; + return value === undefined + ? { done: true as const, value: undefined } + : { done: false as const, value }; + }, + [Symbol.asyncIterator]() { + return this; + }, + }; + const onComplete = vi.fn(); + const patched = patchStreamIfNeeded(stream, { + onComplete, + }) as typeof stream; + + await patched.next(); + await patched.next(); + + expect(patched.abort()).toBe("aborted"); + expect(abort).toHaveBeenCalledOnce(); + expect(onComplete).toHaveBeenCalledOnce(); + expect(onComplete).toHaveBeenCalledWith([1, 2]); + + patched.abort(); + expect(onComplete).toHaveBeenCalledOnce(); + }); + it("should handle error injection via throw()", async () => { const stream = { async *[Symbol.asyncIterator]() { diff --git a/js/src/instrumentation/core/stream-patcher.ts b/js/src/instrumentation/core/stream-patcher.ts index 8bb25d80b..30f6d2d27 100644 --- a/js/src/instrumentation/core/stream-patcher.ts +++ b/js/src/instrumentation/core/stream-patcher.ts @@ -6,6 +6,8 @@ * even though they cannot replace return values. */ +import { debugLogger } from "../../debug-logger"; + /** * Check if a value is an async iterable (stream). */ @@ -142,6 +144,46 @@ export function patchStreamIfNeeded( return stream; } + const chunks: TChunk[] = []; + let completed = false; + const patchAbortIfPresent = () => { + try { + if ( + "abort" in stream && + typeof (stream as { abort?: unknown }).abort === "function" + ) { + const originalAbort = ( + stream as { abort: (...args: unknown[]) => unknown } + ).abort.bind(stream); + (stream as { abort: (...args: unknown[]) => unknown }).abort = ( + ...args + ) => { + try { + return originalAbort(...args); + } finally { + if (!completed) { + completed = true; + try { + void Promise.resolve(options.onComplete(chunks)).catch( + (error) => { + debugLogger.error( + "Error in stream onComplete handler:", + error, + ); + }, + ); + } catch (error) { + debugLogger.error("Error in stream onComplete handler:", error); + } + } + } + }; + } + } catch (error) { + debugLogger.warn("Failed to patch stream abort method:", error); + } + }; + // Only patch iterator methods directly when the stream is its own iterator. // Some SDKs expose a separate iterator from Symbol.asyncIterator(); patching // stream.next in those cases is a no-op because consumers never call it. @@ -156,8 +198,6 @@ export function patchStreamIfNeeded( typeof stream.return === "function" ? stream.return.bind(stream) : null; const originalThrow = typeof stream.throw === "function" ? stream.throw.bind(stream) : null; - const chunks: TChunk[] = []; - let completed = false; stream.next = async (...args: [] | [undefined]) => { try { @@ -256,6 +296,7 @@ export function patchStreamIfNeeded( value: true, }); + patchAbortIfPresent(); return stream; } catch (error) { // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. @@ -278,8 +319,6 @@ export function patchStreamIfNeeded( const patchedIteratorFn = function (this: any) { const iterator = originalIteratorFn.call(this); const originalNext = iterator.next.bind(iterator); - const chunks: TChunk[] = []; - let completed = false; // Patch the next() method iterator.next = async function (...args: [] | [undefined]) { @@ -397,6 +436,7 @@ export function patchStreamIfNeeded( // Replace the Symbol.asyncIterator method (stream as any)[Symbol.asyncIterator] = patchedIteratorFn; + patchAbortIfPresent(); return stream; } catch (error) { // If patching fails for any reason, log warning and return original diff --git a/js/src/instrumentation/plugins/ollama-plugin.test.ts b/js/src/instrumentation/plugins/ollama-plugin.test.ts index 11cb77745..bdaf3c3f5 100644 --- a/js/src/instrumentation/plugins/ollama-plugin.test.ts +++ b/js/src/instrumentation/plugins/ollama-plugin.test.ts @@ -321,14 +321,16 @@ describe("Ollama instrumentation extraction", () => { model: "gpt-oss:20b", system: "Be concise.", prompt: "Say OK.", + suffix: "Done.", options: { temperature: 0, num_predict: 8 }, }, ]), ).toEqual({ - input: [ - { role: "system", content: "Be concise." }, - { role: "user", content: "Say OK." }, - ], + input: { + system: "Be concise.", + prompt: "Say OK.", + suffix: "Done.", + }, metadata: { provider: "ollama", model: "gpt-oss:20b", @@ -374,12 +376,16 @@ describe("Ollama instrumentation extraction", () => { [ { model: "gpt-oss:20b", - message: { role: "assistant", content: "Hello" }, + message: { + role: "assistant", + content: "Hello", + thinking: "Check", + }, done: false, }, { model: "gpt-oss:20b", - message: { role: "assistant", content: "!" }, + message: { role: "assistant", content: "!", thinking: " done" }, done: true, done_reason: "stop", prompt_eval_count: 5, @@ -394,7 +400,11 @@ describe("Ollama instrumentation extraction", () => { { index: 0, finish_reason: "stop", - message: { role: "assistant", content: "Hello!" }, + message: { + role: "assistant", + content: "Hello!", + reasoning: "Check done", + }, }, ]); expect(chat.metrics).toMatchObject({ @@ -406,10 +416,16 @@ describe("Ollama instrumentation extraction", () => { const generation = aggregateOllamaGenerateChunks( [ - { model: "gpt-oss:20b", response: "O", done: false }, + { + model: "gpt-oss:20b", + response: "O", + thinking: "Check", + done: false, + }, { model: "gpt-oss:20b", response: "K", + thinking: " done", done: true, done_reason: "stop", prompt_eval_count: 4, @@ -424,7 +440,11 @@ describe("Ollama instrumentation extraction", () => { { index: 0, finish_reason: "stop", - message: { role: "assistant", content: "OK" }, + message: { + role: "assistant", + content: "OK", + reasoning: "Check done", + }, }, ]); expect(generation.metrics).toMatchObject({ diff --git a/js/src/instrumentation/plugins/ollama-plugin.ts b/js/src/instrumentation/plugins/ollama-plugin.ts index 4aa8bdc65..8a492387d 100644 --- a/js/src/instrumentation/plugins/ollama-plugin.ts +++ b/js/src/instrumentation/plugins/ollama-plugin.ts @@ -3,8 +3,11 @@ import iso from "../../isomorph"; import { Attachment } from "../../logger"; import { processInputAttachments } from "../../wrappers/attachment-utils"; import type { + OllamaChatRequest, OllamaChatResponse, + OllamaEmbedRequest, OllamaEmbedResponse, + OllamaGenerateRequest, OllamaGenerateResponse, OllamaMessage, OllamaTool, @@ -56,23 +59,6 @@ export class OllamaPlugin extends BasePlugin { } } -function getRequestArg(args: unknown): Record | undefined { - const values = - Array.isArray(args) || isArrayLike(args) ? Array.from(args) : [args]; - return values.find((value) => isObject(value)) as - | Record - | undefined; -} - -function isArrayLike(value: unknown): value is ArrayLike { - return ( - isObject(value) && - typeof value.length === "number" && - Number.isInteger(value.length) && - value.length >= 0 - ); -} - function isNonNegativeNumber(value: unknown): value is number { return typeof value === "number" && Number.isFinite(value) && value >= 0; } @@ -415,7 +401,13 @@ function extractOptionsMetadata(options: unknown): Record { } function extractRequestMetadata( - request: Record | undefined, + request: + | { + format?: unknown; + model?: unknown; + options?: unknown; + } + | undefined, ): Record { return { provider: "ollama", @@ -427,41 +419,57 @@ function extractRequestMetadata( }; } -export function extractOllamaChatInput(args: unknown): { +export function extractOllamaChatInput([request]: [ + OllamaChatRequest, + ...unknown[], +]): { input: unknown; metadata: Record; } { - const request = getRequestArg(args); return { - input: normalizeMessages(request?.messages), + input: normalizeMessages(request.messages), metadata: { ...extractRequestMetadata(request), - ...extractToolsMetadata(request?.tools), + ...extractToolsMetadata(request.tools), }, }; } -export function extractOllamaGenerateInput(args: unknown): { +export function extractOllamaGenerateInput([request]: [ + OllamaGenerateRequest, + ...unknown[], +]): { input: unknown; metadata: Record; } { - const request = getRequestArg(args); const normalizedPrompt = normalizeTextAndImages( - typeof request?.prompt === "string" ? request.prompt : "", - Array.isArray(request?.images) ? request.images : undefined, + typeof request.prompt === "string" ? request.prompt : "", + Array.isArray(request.images) ? request.images : undefined, ); - const input = [ - ...(typeof request?.system === "string" - ? [{ role: "system", content: request.system }] - : []), - { - role: "user", - content: normalizedPrompt.content, - ...(normalizedPrompt.unrecognizedImages - ? { images: normalizedPrompt.unrecognizedImages } - : {}), - }, - ]; + const input = + typeof request.suffix === "string" + ? { + ...(typeof request.system === "string" + ? { system: request.system } + : {}), + prompt: normalizedPrompt.content, + suffix: request.suffix, + ...(normalizedPrompt.unrecognizedImages + ? { images: normalizedPrompt.unrecognizedImages } + : {}), + } + : [ + ...(typeof request.system === "string" + ? [{ role: "system", content: request.system }] + : []), + { + role: "user", + content: normalizedPrompt.content, + ...(normalizedPrompt.unrecognizedImages + ? { images: normalizedPrompt.unrecognizedImages } + : {}), + }, + ]; return { input, @@ -469,13 +477,15 @@ export function extractOllamaGenerateInput(args: unknown): { }; } -function extractOllamaEmbedInput(args: unknown): { +function extractOllamaEmbedInput([request]: [ + OllamaEmbedRequest, + ...unknown[], +]): { input: unknown; metadata: Record; } { - const request = getRequestArg(args); return { - input: request?.input, + input: request.input, metadata: extractRequestMetadata(request), }; } From 7cc99e6254da60cedef410edb1bdf6023dc7ae8e Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:25:48 +0000 Subject: [PATCH 7/8] Update PR #2315 --- ...i-coding-agent-v0-auto-hook.span-tree.json | 2 +- ...pi-coding-agent-v0-auto-hook.span-tree.txt | 2 +- ...g-agent-v0-latest-auto-hook.span-tree.json | 2 +- ...ng-agent-v0-latest-auto-hook.span-tree.txt | 2 +- ...ing-agent-v0-latest-wrapped.span-tree.json | 2 +- ...ding-agent-v0-latest-wrapped.span-tree.txt | 2 +- .../pi-coding-agent-v0-wrapped.span-tree.json | 2 +- .../pi-coding-agent-v0-wrapped.span-tree.txt | 2 +- ...coding-agent-v079-auto-hook.span-tree.json | 146 ------------------ ...-coding-agent-v079-auto-hook.span-tree.txt | 121 --------------- ...i-coding-agent-v079-wrapped.span-tree.json | 146 ------------------ ...pi-coding-agent-v079-wrapped.span-tree.txt | 121 --------------- .../scenario.impl.mjs | 21 ++- .../core/channel-tracing.test.ts | 44 ++++++ .../instrumentation/core/channel-tracing.ts | 58 +++---- .../core/stream-patcher.test.ts | 11 +- js/src/instrumentation/core/stream-patcher.ts | 41 ++--- .../plugins/ollama-plugin.test.ts | 49 +++++- .../instrumentation/plugins/ollama-plugin.ts | 77 +++++---- 19 files changed, 216 insertions(+), 635 deletions(-) delete mode 100644 e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.json delete mode 100644 e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.txt delete mode 100644 e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.json delete mode 100644 e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.txt diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json index 04093af29..664950e5b 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt index 29fdfabcc..ed7efd569 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json index dea4dfab8..7569fe54b 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt index 0c93a9b26..4db394c56 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json index dea4dfab8..7569fe54b 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt index 0c93a9b26..4db394c56 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json index 04093af29..664950e5b 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt index 29fdfabcc..ed7efd569 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.json deleted file mode 100644 index ab2c68bb1..000000000 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "span_tree": [ - { - "name": "pi-coding-agent-root", - "type": "task", - "children": [ - { - "name": "pi-coding-agent-prompt-operation", - "children": [ - { - "name": "AgentSession.prompt", - "type": "task", - "children": [ - { - "name": "anthropic.messages.create", - "type": "llm", - "children": [], - "input": [ - { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", - "role": "system" - }, - { - "content": [ - { - "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - "type": "text" - } - ], - "role": "user" - }, - { - "content": null, - "role": "assistant", - "tool_calls": [ - { - "function": { - "arguments": "{\"command\":\"printf pi_tool_ok\"}", - "name": "bash" - }, - "id": "", - "type": "function" - } - ] - }, - { - "content": [ - { - "text": "pi_tool_ok", - "type": "text" - } - ], - "role": "tool", - "tool_call_id": "" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - "role": "assistant" - } - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "agent.streamFn", - "pi_coding_agent.stop_reason": "stop", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 23, - "duration": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 2299, - "time_to_first_token": 0, - "tokens": 2322 - } - }, - { - "name": "bash", - "type": "tool", - "children": [], - "input": { - "command": "printf pi_tool_ok" - }, - "output": { - "content": [ - { - "text": "pi_tool_ok", - "type": "text" - } - ] - }, - "metadata": { - "gen_ai.tool.call.id": "", - "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" - } - } - ], - "input": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - "role": "assistant" - } - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "AgentSession.prompt", - "pi_coding_agent.source": "rpc", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 80, - "duration": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 4524, - "tokens": 4604 - } - } - ], - "metadata": { - "operation": "prompt" - } - } - ], - "metadata": { - "scenario": "pi-coding-agent-instrumentation" - } - } - ] -} diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.txt deleted file mode 100644 index 1b3c2b6e4..000000000 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-auto-hook.span-tree.txt +++ /dev/null @@ -1,121 +0,0 @@ -span_tree: -└── pi-coding-agent-root [task] - metadata: { - "scenario": "pi-coding-agent-instrumentation" - } - └── pi-coding-agent-prompt-operation - metadata: { - "operation": "prompt" - } - └── AgentSession.prompt [task] - input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." - output: [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - "role": "assistant" - } - } - ] - metadata: { - "model": "claude-haiku-4-5", - "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "AgentSession.prompt", - "pi_coding_agent.source": "rpc", - "provider": "anthropic" - } - metrics: { - "completion_tokens": 80, - "duration": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 4524, - "tokens": 4604 - } - ├── anthropic.messages.create [llm] - │ input: [ - │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", - │ "role": "system" - │ }, - │ { - │ "content": [ - │ { - │ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ }, - │ { - │ "content": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"command\":\"printf pi_tool_ok\"}", - │ "name": "bash" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ }, - │ { - │ "content": [ - │ { - │ "text": "pi_tool_ok", - │ "type": "text" - │ } - │ ], - │ "role": "tool", - │ "tool_call_id": "" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "message": { - │ "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "claude-haiku-4-5", - │ "pi_coding_agent.api": "anthropic-messages", - │ "pi_coding_agent.model": "claude-haiku-4-5", - │ "pi_coding_agent.operation": "agent.streamFn", - │ "pi_coding_agent.stop_reason": "stop", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 23, - │ "duration": 0, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 2299, - │ "time_to_first_token": 0, - │ "tokens": 2322 - │ } - └── bash [tool] - input: { - "command": "printf pi_tool_ok" - } - output: { - "content": [ - { - "text": "pi_tool_ok", - "type": "text" - } - ] - } - metadata: { - "gen_ai.tool.call.id": "", - "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" - } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.json deleted file mode 100644 index ab2c68bb1..000000000 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "span_tree": [ - { - "name": "pi-coding-agent-root", - "type": "task", - "children": [ - { - "name": "pi-coding-agent-prompt-operation", - "children": [ - { - "name": "AgentSession.prompt", - "type": "task", - "children": [ - { - "name": "anthropic.messages.create", - "type": "llm", - "children": [], - "input": [ - { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", - "role": "system" - }, - { - "content": [ - { - "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - "type": "text" - } - ], - "role": "user" - }, - { - "content": null, - "role": "assistant", - "tool_calls": [ - { - "function": { - "arguments": "{\"command\":\"printf pi_tool_ok\"}", - "name": "bash" - }, - "id": "", - "type": "function" - } - ] - }, - { - "content": [ - { - "text": "pi_tool_ok", - "type": "text" - } - ], - "role": "tool", - "tool_call_id": "" - } - ], - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - "role": "assistant" - } - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "agent.streamFn", - "pi_coding_agent.stop_reason": "stop", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 23, - "duration": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 2299, - "time_to_first_token": 0, - "tokens": 2322 - } - }, - { - "name": "bash", - "type": "tool", - "children": [], - "input": { - "command": "printf pi_tool_ok" - }, - "output": { - "content": [ - { - "text": "pi_tool_ok", - "type": "text" - } - ] - }, - "metadata": { - "gen_ai.tool.call.id": "", - "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" - } - } - ], - "input": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - "output": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - "role": "assistant" - } - } - ], - "metadata": { - "model": "claude-haiku-4-5", - "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "AgentSession.prompt", - "pi_coding_agent.source": "rpc", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 80, - "duration": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 4524, - "tokens": 4604 - } - } - ], - "metadata": { - "operation": "prompt" - } - } - ], - "metadata": { - "scenario": "pi-coding-agent-instrumentation" - } - } - ] -} diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.txt deleted file mode 100644 index 1b3c2b6e4..000000000 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v079-wrapped.span-tree.txt +++ /dev/null @@ -1,121 +0,0 @@ -span_tree: -└── pi-coding-agent-root [task] - metadata: { - "scenario": "pi-coding-agent-instrumentation" - } - └── pi-coding-agent-prompt-operation - metadata: { - "operation": "prompt" - } - └── AgentSession.prompt [task] - input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." - output: [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - "role": "assistant" - } - } - ] - metadata: { - "model": "claude-haiku-4-5", - "pi_coding_agent.api": "anthropic-messages", - "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "AgentSession.prompt", - "pi_coding_agent.source": "rpc", - "provider": "anthropic" - } - metrics: { - "completion_tokens": 80, - "duration": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 4524, - "tokens": 4604 - } - ├── anthropic.messages.create [llm] - │ input: [ - │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", - │ "role": "system" - │ }, - │ { - │ "content": [ - │ { - │ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ }, - │ { - │ "content": null, - │ "role": "assistant", - │ "tool_calls": [ - │ { - │ "function": { - │ "arguments": "{\"command\":\"printf pi_tool_ok\"}", - │ "name": "bash" - │ }, - │ "id": "", - │ "type": "function" - │ } - │ ] - │ }, - │ { - │ "content": [ - │ { - │ "text": "pi_tool_ok", - │ "type": "text" - │ } - │ ], - │ "role": "tool", - │ "tool_call_id": "" - │ } - │ ] - │ output: [ - │ { - │ "finish_reason": "stop", - │ "index": 0, - │ "message": { - │ "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", - │ "role": "assistant" - │ } - │ } - │ ] - │ metadata: { - │ "model": "claude-haiku-4-5", - │ "pi_coding_agent.api": "anthropic-messages", - │ "pi_coding_agent.model": "claude-haiku-4-5", - │ "pi_coding_agent.operation": "agent.streamFn", - │ "pi_coding_agent.stop_reason": "stop", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 23, - │ "duration": 0, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 2299, - │ "time_to_first_token": 0, - │ "tokens": 2322 - │ } - └── bash [tool] - input: { - "command": "printf pi_tool_ok" - } - output: { - "content": [ - { - "text": "pi_tool_ok", - "type": "text" - } - ] - } - metadata: { - "gen_ai.tool.call.id": "", - "gen_ai.tool.name": "bash", - "pi_coding_agent.tool.name": "bash" - } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs b/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs index aa6f28014..80a35d9c0 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs @@ -9,8 +9,13 @@ export const SCENARIO_NAME = "pi-coding-agent-instrumentation"; async function runPiCodingAgentScenario({ decorateSDK, sdk }) { const instrumentedSDK = decorateSDK ? decorateSDK(sdk) : sdk; - const { AuthStorage, ModelRegistry, SessionManager, createAgentSession } = - instrumentedSDK; + const { + AuthStorage, + DefaultResourceLoader, + ModelRegistry, + SessionManager, + createAgentSession, + } = instrumentedSDK; const authStorage = AuthStorage.inMemory(); authStorage.setRuntimeApiKey("anthropic", process.env.ANTHROPIC_API_KEY); @@ -22,6 +27,13 @@ async function runPiCodingAgentScenario({ decorateSDK, sdk }) { if (!model) { throw new Error("Expected Pi Coding Agent Anthropic model"); } + const cwd = process.cwd(); + const resourceLoader = new DefaultResourceLoader({ + agentDir: cwd, + cwd, + noContextFiles: true, + }); + await resourceLoader.reload(); let session; await runTracedScenario({ @@ -32,10 +44,11 @@ async function runPiCodingAgentScenario({ decorateSDK, sdk }) { async () => { const result = await createAgentSession({ authStorage, - cwd: process.cwd(), + cwd, model, modelRegistry, - sessionManager: SessionManager.inMemory(process.cwd()), + resourceLoader, + sessionManager: SessionManager.inMemory(cwd), thinkingLevel: "off", tools: ["bash"], }); diff --git a/js/src/instrumentation/core/channel-tracing.test.ts b/js/src/instrumentation/core/channel-tracing.test.ts index 5af10b120..55c08f05b 100644 --- a/js/src/instrumentation/core/channel-tracing.test.ts +++ b/js/src/instrumentation/core/channel-tracing.test.ts @@ -314,4 +314,48 @@ describe("traceAsyncChannel current span binding", () => { expect(onError).toHaveBeenCalledTimes(1); expect(end).toHaveBeenCalledTimes(2); }); + + it("records stream cancellation as an error", async () => { + const onError = vi.fn(); + const child = { + end: vi.fn(), + log: vi.fn(), + } as unknown as Span; + const unsubscribe = traceStreamingChannel(testChannels.streamingCall, { + name: "streaming-channel-test", + startSpan: () => child, + type: "function", + extractInput: () => ({ input: "input", metadata: undefined }), + extractOutput: (result) => result, + extractMetrics: () => ({}), + onError, + }); + const stream = { + abort: vi.fn(), + async *[Symbol.asyncIterator]() { + yield { ok: true }; + }, + }; + + try { + const patched = await testChannels.streamingCall.tracePromise( + async () => stream as any, + { arguments: [{}] } as any, + ); + (patched as unknown as typeof stream).abort(); + await Promise.resolve(); + } finally { + unsubscribe(); + } + + const cancellationError = expect.objectContaining({ + message: "Stream cancelled before completion", + name: "AbortError", + }); + expect(child.log).toHaveBeenLastCalledWith({ error: cancellationError }); + expect(child.end).toHaveBeenCalledOnce(); + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ error: cancellationError }), + ); + }); }); diff --git a/js/src/instrumentation/core/channel-tracing.ts b/js/src/instrumentation/core/channel-tracing.ts index 8b200d651..a8ecc819c 100644 --- a/js/src/instrumentation/core/channel-tracing.ts +++ b/js/src/instrumentation/core/channel-tracing.ts @@ -563,6 +563,33 @@ export function traceStreamingChannel( if (isAsyncIterable(asyncEndEvent.result)) { let firstChunkTime: number | undefined; + const handleStreamError = (error: Error) => { + try { + span.log({ error }); + } catch (loggingError) { + debugLogger.error( + `Error logging failure for ${channelName}:`, + loggingError, + ); + } + try { + span.end(); + } catch (endingError) { + debugLogger.error( + `Error ending span for ${channelName}:`, + endingError, + ); + } + states.delete(event as object); + runStreamingErrorHook({ + channelName, + config, + error, + event: asyncEndEvent, + span, + startTime, + }); + }; patchStreamIfNeeded(asyncEndEvent.result, { onChunk: () => { @@ -661,33 +688,12 @@ export function traceStreamingChannel( }); } }, - onError: (error: Error) => { - try { - span.log({ error }); - } catch (loggingError) { - debugLogger.error( - `Error logging failure for ${channelName}:`, - loggingError, - ); - } - try { - span.end(); - } catch (endingError) { - debugLogger.error( - `Error ending span for ${channelName}:`, - endingError, - ); - } - states.delete(event as object); - runStreamingErrorHook({ - channelName, - config, - error, - event: asyncEndEvent, - span, - startTime, - }); + onCancel: () => { + const error = new Error("Stream cancelled before completion"); + error.name = "AbortError"; + handleStreamError(error); }, + onError: handleStreamError, }); return; } diff --git a/js/src/instrumentation/core/stream-patcher.test.ts b/js/src/instrumentation/core/stream-patcher.test.ts index f2c86b13d..f3954a41b 100644 --- a/js/src/instrumentation/core/stream-patcher.test.ts +++ b/js/src/instrumentation/core/stream-patcher.test.ts @@ -258,7 +258,7 @@ describe("patchStreamIfNeeded", () => { expect(onComplete).toHaveBeenCalledWith([1, 2]); }); - it("should complete an abortable stream when abort() is called", async () => { + it("should cancel an abortable stream when abort() is called", async () => { const iterator = { values: [1, 2, 3], index: 0, @@ -277,7 +277,9 @@ describe("patchStreamIfNeeded", () => { }, }; const onComplete = vi.fn(); + const onCancel = vi.fn(); const patched = patchStreamIfNeeded(stream, { + onCancel, onComplete, }) as typeof stream; const patchedIterator = patched[Symbol.asyncIterator](); @@ -287,11 +289,12 @@ describe("patchStreamIfNeeded", () => { expect(patched.abort()).toBe("aborted"); expect(abort).toHaveBeenCalledOnce(); - expect(onComplete).toHaveBeenCalledOnce(); - expect(onComplete).toHaveBeenCalledWith([1, 2]); + expect(onCancel).toHaveBeenCalledOnce(); + expect(onCancel).toHaveBeenCalledWith([1, 2]); + expect(onComplete).not.toHaveBeenCalled(); patched.abort(); - expect(onComplete).toHaveBeenCalledOnce(); + expect(onCancel).toHaveBeenCalledOnce(); }); it("should complete an abortable self-iterator when abort() is called", async () => { diff --git a/js/src/instrumentation/core/stream-patcher.ts b/js/src/instrumentation/core/stream-patcher.ts index 30f6d2d27..534a4d031 100644 --- a/js/src/instrumentation/core/stream-patcher.ts +++ b/js/src/instrumentation/core/stream-patcher.ts @@ -38,6 +38,12 @@ interface StreamPatchOptions { */ onComplete: (chunks: TChunk[]) => TFinal | void | Promise; + /** + * Called when the consumer cancels the stream before it completes. + * Falls back to onComplete when omitted. + */ + onCancel?: (chunks: TChunk[]) => void | Promise; + /** * Called if the stream errors. * If not provided, errors are re-thrown after collection stops. @@ -146,6 +152,13 @@ export function patchStreamIfNeeded( const chunks: TChunk[] = []; let completed = false; + const notifyCancellation = async () => { + try { + await (options.onCancel ?? options.onComplete)(chunks); + } catch (error) { + debugLogger.error("Error in stream cancellation handler:", error); + } + }; const patchAbortIfPresent = () => { try { if ( @@ -163,18 +176,7 @@ export function patchStreamIfNeeded( } finally { if (!completed) { completed = true; - try { - void Promise.resolve(options.onComplete(chunks)).catch( - (error) => { - debugLogger.error( - "Error in stream onComplete handler:", - error, - ); - }, - ); - } catch (error) { - debugLogger.error("Error in stream onComplete handler:", error); - } + void notifyCancellation(); } } }; @@ -259,12 +261,7 @@ export function patchStreamIfNeeded( stream.return = async (...args: [unknown?]) => { if (!completed) { completed = true; - try { - await options.onComplete(chunks); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error("Error in stream onComplete handler:", error); - } + await notifyCancellation(); } return originalReturn(...args); }; @@ -389,13 +386,7 @@ export function patchStreamIfNeeded( iterator.return = async function (...args: any[]) { if (!completed) { completed = true; - // Stream was cancelled/returned early - try { - await options.onComplete(chunks); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error("Error in stream onComplete handler:", error); - } + await notifyCancellation(); } return originalReturn(...args); }; diff --git a/js/src/instrumentation/plugins/ollama-plugin.test.ts b/js/src/instrumentation/plugins/ollama-plugin.test.ts index bdaf3c3f5..0006efea8 100644 --- a/js/src/instrumentation/plugins/ollama-plugin.test.ts +++ b/js/src/instrumentation/plugins/ollama-plugin.test.ts @@ -312,6 +312,46 @@ describe("Ollama instrumentation extraction", () => { }, }, ]); + + expect( + extractOllamaChatOutput( + { + message: { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "get_weather", + arguments: { city: "Paris" }, + }, + }, + ], + }, + done: true, + }, + 2, + ), + ).toEqual([ + { + index: 0, + finish_reason: "tool_calls", + message: { + role: "assistant", + content: null, + tool_calls: [ + { + id: "ollama_call_get_weather_2", + type: "function", + function: { + name: "get_weather", + arguments: '{"city":"Paris"}', + }, + }, + ], + }, + }, + ]); }); it("normalizes raw generation input as canonical messages", () => { @@ -326,11 +366,10 @@ describe("Ollama instrumentation extraction", () => { }, ]), ).toEqual({ - input: { - system: "Be concise.", - prompt: "Say OK.", - suffix: "Done.", - }, + input: [ + { role: "system", content: "Be concise." }, + { role: "user", content: "Say OK.", suffix: "Done." }, + ], metadata: { provider: "ollama", model: "gpt-oss:20b", diff --git a/js/src/instrumentation/plugins/ollama-plugin.ts b/js/src/instrumentation/plugins/ollama-plugin.ts index 8a492387d..dcf1bd70d 100644 --- a/js/src/instrumentation/plugins/ollama-plugin.ts +++ b/js/src/instrumentation/plugins/ollama-plugin.ts @@ -15,6 +15,7 @@ import type { OllamaUsageResponse, } from "../../vendor-sdk-types/ollama"; import { BasePlugin } from "../core"; +import type { AsyncEndOf } from "../core/channel-definitions"; import { traceAsyncChannel, traceStreamingChannel, @@ -29,7 +30,11 @@ export class OllamaPlugin extends BasePlugin { name: "ollama.chat", type: SpanTypeAttribute.LLM, extractInput: extractOllamaChatInput, - extractOutput: extractOllamaChatOutput, + extractOutput: (result, event) => + extractOllamaChatOutput( + result, + countOllamaToolCalls(event?.arguments[0]?.messages), + ), extractMetadata: extractOllamaResponseMetadata, extractMetrics: extractOllamaMetrics, aggregateChunks: aggregateOllamaChatChunks, @@ -125,6 +130,21 @@ function normalizeToolCalls( }); } +function countOllamaToolCalls(messages: unknown): number { + if (!Array.isArray(messages)) { + return 0; + } + + return messages.reduce( + (count, message) => + count + + (isObject(message) && Array.isArray(message.tool_calls) + ? message.tool_calls.length + : 0), + 0, + ); +} + function imageBytes(value: unknown): Uint8Array | undefined { if (value instanceof Uint8Array) { return value; @@ -446,30 +466,19 @@ export function extractOllamaGenerateInput([request]: [ typeof request.prompt === "string" ? request.prompt : "", Array.isArray(request.images) ? request.images : undefined, ); - const input = - typeof request.suffix === "string" - ? { - ...(typeof request.system === "string" - ? { system: request.system } - : {}), - prompt: normalizedPrompt.content, - suffix: request.suffix, - ...(normalizedPrompt.unrecognizedImages - ? { images: normalizedPrompt.unrecognizedImages } - : {}), - } - : [ - ...(typeof request.system === "string" - ? [{ role: "system", content: request.system }] - : []), - { - role: "user", - content: normalizedPrompt.content, - ...(normalizedPrompt.unrecognizedImages - ? { images: normalizedPrompt.unrecognizedImages } - : {}), - }, - ]; + const input = [ + ...(typeof request.system === "string" + ? [{ role: "system", content: request.system }] + : []), + { + role: "user", + content: normalizedPrompt.content, + ...(typeof request.suffix === "string" ? { suffix: request.suffix } : {}), + ...(normalizedPrompt.unrecognizedImages + ? { images: normalizedPrompt.unrecognizedImages } + : {}), + }, + ]; return { input, @@ -490,12 +499,19 @@ function extractOllamaEmbedInput([request]: [ }; } -export function extractOllamaChatOutput(result: OllamaChatResponse): unknown { +export function extractOllamaChatOutput( + result: OllamaChatResponse, + syntheticToolCallIdOffset = 0, +): unknown { if (!isObject(result) || !isObject(result.message)) { return undefined; } - const message = normalizeMessage(result.message as OllamaMessage); + const message = normalizeMessage( + result.message as OllamaMessage, + undefined, + syntheticToolCallIdOffset, + ); if (!message) { return undefined; } @@ -574,7 +590,7 @@ export function extractOllamaMetrics( export function aggregateOllamaChatChunks( chunks: OllamaChatResponse[], _result?: unknown, - _event?: unknown, + event?: AsyncEndOf, _startTime?: number, ): { output: unknown; @@ -593,7 +609,10 @@ export function aggregateOllamaChatChunks( message, }; return { - output: extractOllamaChatOutput(response), + output: extractOllamaChatOutput( + response, + countOllamaToolCalls(event?.arguments[0]?.messages), + ), metrics: extractOllamaMetrics(last ?? {}), metadata: extractOllamaResponseMetadata(last ?? {}), }; From 96bb8fe159222840776e87c17557db21ec40f817 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:31:14 +0000 Subject: [PATCH 8/8] Update PR #2315 --- js/src/instrumentation/plugins/ollama-plugin.ts | 4 ++-- js/src/vendor-sdk-types/ollama.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/js/src/instrumentation/plugins/ollama-plugin.ts b/js/src/instrumentation/plugins/ollama-plugin.ts index dcf1bd70d..f48040348 100644 --- a/js/src/instrumentation/plugins/ollama-plugin.ts +++ b/js/src/instrumentation/plugins/ollama-plugin.ts @@ -33,7 +33,7 @@ export class OllamaPlugin extends BasePlugin { extractOutput: (result, event) => extractOllamaChatOutput( result, - countOllamaToolCalls(event?.arguments[0]?.messages), + countOllamaToolCalls(event?.arguments?.[0]?.messages), ), extractMetadata: extractOllamaResponseMetadata, extractMetrics: extractOllamaMetrics, @@ -611,7 +611,7 @@ export function aggregateOllamaChatChunks( return { output: extractOllamaChatOutput( response, - countOllamaToolCalls(event?.arguments[0]?.messages), + countOllamaToolCalls(event?.arguments?.[0]?.messages), ), metrics: extractOllamaMetrics(last ?? {}), metadata: extractOllamaResponseMetadata(last ?? {}), diff --git a/js/src/vendor-sdk-types/ollama.ts b/js/src/vendor-sdk-types/ollama.ts index 73dfa98d6..d3626bf8f 100644 --- a/js/src/vendor-sdk-types/ollama.ts +++ b/js/src/vendor-sdk-types/ollama.ts @@ -3,6 +3,7 @@ export interface OllamaOptions { num_predict?: number; temperature?: number; + top_k?: number; top_p?: number; frequency_penalty?: number; presence_penalty?: number;